Build the whole compiler with dune

Author: cknitt

.gitattributes

@@ -25,6 +25,7 @@ jobs:
 
     env:
       OCAMLRUNPARAM: b
+      DUNE_PROFILE: release
 
     steps:
       - name: "Windows: Set git to use LF"
@@ -45,6 +46,22 @@ jobs:
           opam-pin: false
           opam-depext: false
 
+      - name: "Install OPAM dependencies"
+        run: opam install . --deps-only
+
+      - name: "Build compiler"
+        run: |
+          opam exec -- dune build
+          ./scripts/copyExes.js
+
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install npm packages
+        run: npm ci --ignore-scripts
+
       - name: "Check if syntax subfolder has changed"
         id: syntax-diff
         shell: bash
@@ -55,25 +72,13 @@ jobs:
             echo "syntax_status=changed" >> $GITHUB_ENV
           fi
 
-      - name: "Syntax: install OPAM dependencies"
-        if: env.syntax_status == 'changed'
-        run: opam install . --deps-only
-        working-directory: res_syntax
-
-      - name: "Syntax: build CLI"
-        if: env.syntax_status == 'changed'
-        run: opam exec -- dune build
-        working-directory: res_syntax
-
       - name: "Syntax: Run roundtrip tests"
         if: ${{ env.syntax_status == 'changed' && runner.os != 'Windows' }}
-        run: opam exec -- make roundtrip-test
-        working-directory: res_syntax
+        run: opam exec -- make test-syntax-roundtrip
 
       - name: "Syntax: Run tests (Windows)"
         if: ${{ env.syntax_status == 'changed' && runner.os == 'Windows' }}
-        run: opam exec -- make test
-        working-directory: res_syntax
+        run: opam exec -- make test-syntax
 
       # Required for ninja build
       - name: "Windows: Use MSVC"
@@ -85,26 +90,27 @@ jobs:
       - name: Build ninja
         run: node scripts/buildNinjaBinary.js
 
-      - name: Use Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
+      - name: Build runtime/stdlib
+        if: runner.os != 'Windows'
+        run: |
+          opam exec -- dune exec -- node ./scripts/ninja.js config
+          opam exec -- dune exec -- node ./scripts/ninja.js build
 
-      - name: NPM install
-        run: opam exec -- npm ci
-        env:
-          RESCRIPT_FORCE_REBUILD: 1
+      - name: Check for changes in lib folder
+        run: git diff --exit-code lib/js lib/es6
 
       - name: Run tests
         if: runner.os != 'Windows'
-        run: opam exec -- node scripts/ciTest.js -all
+        run: opam exec -- dune exec -- node scripts/ciTest.js -all
 
       - name: Run tests (Windows)
         if: runner.os == 'Windows'
-        run: opam exec -- node scripts/ciTest.js -mocha -theme -format
+        run: opam exec -- dune exec -- node scripts/ciTest.js -mocha -theme -format
 
-      - name: Get artifact info
-        run: node .github/workflows/get_artifact_info.js
+      - name: Prepare artifact upload
+        run: |
+          ./scripts/prebuilt.js
+          node .github/workflows/get_artifact_info.js
 
       - name: "Upload artifacts: binaries"
         uses: actions/upload-artifact@v3
@@ -119,19 +125,14 @@ jobs:
           name: lib-ocaml
           path: lib/ocaml
 
-      # Important: the ninja build must run after the artifact upload because for some reason
-      # the bsc.exe created by the ninja build does not find the Pervasives module.
-      # So this is run here just to test the normal dev workflow, too.
-      - name: ninja config
-        if: runner.os != 'Windows'
-        run: opam exec -- node ./scripts/ninja.js config
-
-      - name: ninja build
-        if: runner.os != 'Windows'
-        run: opam exec -- node ./scripts/ninja.js build
-
-      - name: Check for changes in lib folder
-        run: git diff --exit-code lib/js lib/es6
+      - name: "Upload artifacts: cmi cache"
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v3
+        with:
+          name: cmij-cache
+          path: |
+            lib/cmi_cache.bin
+            lib/cmj_cache.bin
 
   package:
     needs: build
@@ -157,9 +158,6 @@ jobs:
       - name: Move artifacts
         run: ./scripts/moveArtifacts.sh
 
-      - name: Create ninja.tar.gz
-        run: git -C ninja archive --format=tar.gz HEAD -o ../vendor/ninja.tar.gz
-
       - name: Check artifact list
         run: node ./scripts/makeArtifactList.js -check
 

.github/workflows/ci.yml

@@ -1,13 +1,11 @@
 const fs = require("fs");
 const os = require("os");
 
-const artifactPath =
-  process.platform === "darwin" && process.arch === "arm64"
-    ? process.platform + process.arch
-    : process.platform;
-
+const artifactPath = require("../../scripts/bin_path").dirName;
 const artifactName = "binaries-" + artifactPath;
 
+console.log("Artifact path:", artifactPath);
+
 // Pass artifactPath and artifactName to subsequent GitHub actions
 fs.appendFileSync(
   process.env.GITHUB_ENV,

.github/workflows/get_artifact_info.js

@@ -45,39 +45,12 @@ _build
 .DS_Store
 
 node_modules
-oxml
-test.json
-jscomp/pre_load.js
-boot
 *.dump
 coverage
-*.g.js
-ppx_tools/
-vendor/ocaml/stdlib/target_camlheader
-vendor/ocaml/stdlib/target_camlheaderd
-man/
-lib/ocaml
-*.tgz
-bin/*
-jscomp/bin/*.exe
-
-# for npm publish
 *.exe
 
-!bin/bspack
-!/bin/bsb
-!/bin/bsc
-
-# This is intentional
-# so that `npm install && npm intall -g .`
-# the second install would use the cached ninja.exe (instead of being ignored)
-#
-#ninja
 .ninja_log
-.ninja_deps
 .bsdeps
-.bs_dir_cache
-
 .bsbuild
 lib/prebuilt.ninja
 lib/release.ninja
@@ -96,50 +69,20 @@ jscomp/cppoVendor.ninja
 jscomp/cppoEnv.ninja
 jscomp/bin/bsb
 jscomp/bin/bsc
+lib/ocaml
 lib/bs
 lib/js/jscomp/test
 jscomp/build_tests/*/lib/
 #ignore temporary directory
 *.goog.js
 *.jsoo.js
 *.so
-bin/ninja.exe
 tmp/
-jscomp/ext/hash_set.ml
-jscomp/ext/hash_set_poly.ml
-jscomp/ext/hash.ml
-jscomp/ext/hash_set_ident.ml
-jscomp/ext/hash_ident.ml
-jscomp/ext/map_ident.ml
-jscomp/ext/set_ident.ml
-jscomp/ext/hash_set_int.ml
-jscomp/ext/hash_int.ml
-jscomp/ext/map_int.ml
-jscomp/ext/vec_int.ml
-jscomp/ext/ordered_hash_map_local_ident.ml
-jscomp/ext/ordered_hash_set_string.ml
-jscomp/ext/vec.ml
-jscomp/ext/set_int.ml
-jscomp/ext/hash_set_string.ml
-jscomp/ext/hash_string.ml
-jscomp/ext/map_string.ml
-jscomp/ext/set_string.ml
-jscomp/ext/ext_json_parse.ml
-jscomp/syntax/reactjs_jsx_ppx_v2.ml
-jscomp/syntax/reactjs_jsx_ppx_v3.ml
-jscomp/bin/all_ounit_tests.ml
-native/
-vendor/ocaml
 jscomp/.lsp
-.vscode/launch.json
 tracing*.json
-darwin/bsc
 .nyc_output
-themes
+.vscode/launch.json
 .vscode/settings.json
-*.output # yacc output
-jscomp/ml/parser.output
-jscomp/ml/lexer.ml
 
 # opam local switch is stored in `_opam` folder
 _opam

.gitignore

@@ -3,6 +3,7 @@ lib/
 vendor/
 ninja/
 res_syntax/
+_build/
 _opam
 CHANGELOG.md
 README.md