Build statically linked Linux binaries in CI (rescript-lang#5872)

* Build statically linked Linux executables in a container

* PR review feedback

Author: cknitt

.github/workflows/ci.yml

@@ -11,14 +11,53 @@ concurrency:
   # Cancel previous builds for pull requests only.
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
+env:
+  OCAMLRUNPARAM: b
+  DUNE_PROFILE: release
+
 jobs:
+  # Build statically linked Linux binaries in an Alpine-based Docker container
+  # See https://ocamlpro.com/blog/2021_09_02_generating_static_and_portable_executables_with_ocaml
+  # for more info.
+  # The container already comes with all required tools pre-installed
+  # (see https://github.com/rescript-lang/docker-rescript-ci-build/blob/main/Dockerfile).
+  static-binaries-linux:
+    runs-on: ubuntu-latest
+
+    container:
+      image: ghcr.io/rescript-lang/rescript-ci-build
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Apply static linking patch
+        run: git apply scripts/dune-static-linking.patch
+
+      - name: Build compiler binaries
+        run: opam exec -- dune build
+
+      - name: Build ninja binary
+        working-directory: ninja
+        env:
+          LDFLAGS: -static
+        run: python3 configure.py --bootstrap --verbose
+
+      - name: "Upload artifacts"
+        uses: actions/upload-artifact@v3
+        with:
+          name: static-binaries-linux
+          path: |
+            _build/install/default/bin
+            ninja/ninja
+
   build:
+    needs: static-binaries-linux
+
     strategy:
       fail-fast: false
       matrix:
-        # Stay on the oldest Ubuntu version that's still supported by Github Actions
-        # to avoid glibc incompatibilities as far as possible.
-        os: [macos-latest, ubuntu-20.04, windows-latest, macos-arm]
+        os: [macos-latest, ubuntu-latest, windows-latest, macos-arm]
         ocaml_compiler: [4.14.0]
 
     runs-on: ${{matrix.os}}
@@ -39,6 +78,18 @@ jobs:
         with:
           fetch-depth: 2 # to be able to check for changes in subfolder res_syntax later
 
+      - name: Download static linux binaries
+        if: runner.os == 'Linux'
+        uses: actions/download-artifact@v3
+        with:
+          name: static-binaries-linux
+
+      - name: Make static linux binaries executable
+        if: runner.os == 'Linux'
+        run: |
+          chmod +x ninja/ninja
+          chmod +x _build/install/default/bin/*
+
       - name: Use OCaml ${{matrix.ocaml_compiler}}
         uses: ocaml/setup-ocaml@v2
         with:
@@ -49,19 +100,31 @@ jobs:
       - name: "Install OPAM dependencies"
         run: opam install . --deps-only
 
+      - name: "Build compiler"
+        if: runner.os != 'Linux'
+        run: opam exec -- dune build
+
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
           node-version: 16
 
-      - name: "Build compiler"
-        run: |
-          opam exec -- dune build
-          node ./scripts/copyExes.js
-
       - name: Install npm packages
         run: npm ci --ignore-scripts
 
+      - name: "Windows: Use MSVC for ninja build"
+        if: runner.os == 'Windows'
+        uses: TheMrMilchmann/setup-msvc-dev@v2
+        with:
+          arch: x64
+
+      - name: Build ninja
+        if: runner.os != 'Linux'
+        run: node scripts/buildNinjaBinary.js
+
+      - name: Copy exes to platform bin dirs
+        run: node ./scripts/copyExes.js
+
       - name: "Check if syntax subfolder has changed"
         id: syntax-diff
         shell: bash
@@ -74,38 +137,28 @@ jobs:
 
       - name: "Syntax: Run roundtrip tests"
         if: ${{ env.syntax_status == 'changed' && runner.os != 'Windows' }}
-        run: opam exec -- make test-syntax-roundtrip
+        run: make test-syntax-roundtrip
 
       - name: "Syntax: Run tests (Windows)"
         if: ${{ env.syntax_status == 'changed' && runner.os == 'Windows' }}
-        run: opam exec -- make test-syntax
-
-      # Required for ninja build
-      - name: "Windows: Use MSVC"
-        if: runner.os == 'Windows'
-        uses: TheMrMilchmann/setup-msvc-dev@v2
-        with:
-          arch: x64
-
-      - name: Build ninja
-        run: node scripts/buildNinjaBinary.js
+        run: make test-syntax
 
       - name: Build runtime/stdlib
         if: runner.os != 'Windows'
         run: |
-          opam exec -- dune exec -- node ./scripts/ninja.js config
-          opam exec -- dune exec -- node ./scripts/ninja.js build
+          opam exec -- node ./scripts/ninja.js config
+          opam exec -- node ./scripts/ninja.js build
 
       - name: Check for changes in lib folder
         run: git diff --exit-code lib/js lib/es6
 
       - name: Run tests
         if: runner.os != 'Windows'
-        run: opam exec -- dune exec -- node scripts/ciTest.js -all
+        run: node scripts/ciTest.js -all
 
       - name: Run tests (Windows)
         if: runner.os == 'Windows'
-        run: opam exec -- dune exec -- node scripts/ciTest.js -mocha -theme -format
+        run: node scripts/ciTest.js -mocha -theme -format
 
       - name: Prepare artifact upload
         run: |
@@ -216,30 +269,3 @@ jobs:
         run: npx rescript -h && npx rescript build && cat src/Test.bs.js
         shell: bash
         working-directory: packages/test
-
-  # The following job tests installation from source on FreeBSD via cross-platform-actions.
-  # Disabled by default, as it takes ~11m.
-  # Can be enabled temporarily in a PR to verify that installation from source
-  # on an "exotic" platform still works.
-  #
-  # installationTestFreeBSD:
-  #   needs: package
-  #   runs-on: macos-latest
-  #
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v3
-  #
-  #     - name: Download artifacts
-  #       uses: actions/download-artifact@v3
-  #       with:
-  #         name: npm-packages
-  #         path: packages/test
-  #
-  #     - name: Test installation on FreeBSD
-  #       uses: cross-platform-actions/action@v0.6.2
-  #       with:
-  #         operating_system: freebsd
-  #         version: "13.0"
-  #         shell: bash
-  #         run: cd packages/test && ./test_freebsd.sh

.gitignore

@@ -36,7 +36,6 @@ _build
 *.rawlambda
 *.lambda
 *.zip
-*.patch
 *.mlast
 *.mliast
 # special file

Makefile

@@ -1,5 +1,7 @@
 SHELL = /bin/bash
 
+DUNE_BIN_DIR = ./_build/install/default/bin
+
 build:
 	dune build
 	./scripts/copyExes.js
@@ -8,42 +10,40 @@ watch:
 	dune build -w
 
 bench:
-	dune exec -- syntax_benchmarks
+	$(DUNE_BIN_DIR)/syntax_benchmarks
 
 dce:
-	opam exec reanalyze.exe -- -dce-cmt _build
+	reanalyze.exe -- -dce-cmt _build
 
 ninja/ninja:
 	./scripts/buildNinjaBinary.js
 
 ninja: ninja/ninja
 
 test: lib
-	dune exec -- node scripts/ciTest.js -all
+	node scripts/ciTest.js -all
 
-test-syntax: build
-	dune exec -- syntax_tests
-	dune exec -- bash ./scripts/test_syntax.sh
+test-syntax:
+	bash ./scripts/test_syntax.sh
 	make reanalyze
 	bash ./scripts/testok.sh
 
-test-syntax-roundtrip: build
-	dune exec -- syntax_tests
-	ROUNDTRIP_TEST=1 dune exec -- bash ./scripts/test.sh
+test-syntax-roundtrip:
+	ROUNDTRIP_TEST=1 bash ./scripts/test.sh
 	make reanalyze
 	bash ./scripts/testok.sh
 
-test-gentype: build
+test-gentype:
 	make -C jscomp/gentype_tests/typescript-react-example clean test
 
 test-all: test test-gentype
 
 reanalyze:
 	reanalyze.exe -set-exit-code -all-cmt _build/default/res_syntax -suppress res_syntax/testrunner
 
-lib: build ninja/ninja
-	dune exec -- node scripts/ninja.js config
-	dune exec -- node scripts/ninja.js build
+lib: ninja/ninja
+	node scripts/ninja.js config
+	node scripts/ninja.js build
 
 artifacts: lib
 	./scripts/prebuilt.js

scripts/buildNinjaBinary.js

@@ -1,11 +1,8 @@
 #!/usr/bin/env node
 
 const child_process = require("child_process");
-const fs = require("fs");
 const path = require("path");
 
-const { absolutePath: platformBinDir, ninja_exe } = require("./bin_path");
-
 const platform = process.platform;
 const ninjaDir = path.join(__dirname, "..", "ninja");
 const buildCommand = "python3 configure.py --bootstrap --verbose";
@@ -20,11 +17,3 @@ if (platform === "win32") {
   child_process.execSync(buildCommand, { stdio: [0, 1, 2], cwd: ninjaDir });
   child_process.execSync(`strip ninja`, { stdio: [0, 1, 2], cwd: ninjaDir });
 }
-
-// Copy exe to platform bin dir
-if (!fs.existsSync(platformBinDir)) {
-  fs.mkdirSync(platformBinDir);
-}
-
-const ext = process.platform === "win32" ? ".exe" : "";
-fs.copyFileSync(path.join(ninjaDir, "ninja" + ext), ninja_exe);

scripts/ciTest.js

@@ -3,6 +3,8 @@ var cp = require("child_process");
 var path = require("path");
 var fs = require("fs");
 
+var duneBinDir = require("./dune").duneBinDir;
+
 var ounitTest = false;
 var mochaTest = false;
 var themeTest = false;
@@ -43,7 +45,7 @@ if (all) {
 
 function runTests() {
   if (ounitTest) {
-    cp.execSync("ounit_tests", {
+    cp.execSync(path.join(duneBinDir, "ounit_tests"), {
       stdio: [0, 1, 2],
     });
   }

scripts/copyExes.js

@@ -4,24 +4,18 @@
 const path = require("path");
 const fs = require("fs");
 const child_process = require("child_process");
+const { duneBinDir } = require("./dune");
+const { absolutePath: platformBinDir } = require("./bin_path");
 
-const platformBinDir = require("./bin_path").absolutePath;
-const duneBinDir = path.join(
-  __dirname,
-  "..",
-  "_build",
-  "install",
-  "default",
-  "bin"
-);
+const ninjaDir = path.join(__dirname, "..", "ninja");
 
 if (!fs.existsSync(platformBinDir)) {
   fs.mkdirSync(platformBinDir);
 }
 
-function copyExe(exe) {
+function copyExe(dir, exe) {
   const ext = process.platform === "win32" ? ".exe" : "";
-  const src = path.join(duneBinDir, exe + ext);
+  const src = path.join(dir, exe + ext);
   const dest = path.join(platformBinDir, exe + ".exe");
 
   // For some reason, the copy operation fails in Windows CI if the file already exists.
@@ -36,6 +30,7 @@ function copyExe(exe) {
   }
 }
 
-copyExe("rescript");
-copyExe("bsc");
-copyExe("bsb_helper");
+copyExe(duneBinDir, "rescript");
+copyExe(duneBinDir, "bsc");
+copyExe(duneBinDir, "bsb_helper");
+copyExe(ninjaDir, "ninja");