Continuous benchmarking in CI (#7100)

* Syntax benchmark: cleanup

* Output benchmark results as JSON

* Output JSON lazily

* Increment opam cache key

* Build benchmarks only for OCaml >= 4.14.0

* Continuous benchmarking in CI

* Permissions

* Beautify

* Cleanup and fix benchmark

* More cleanup

* Do not attempt to run action for PRs created from other repos

* Fix caml_mach_absolute_time for Linux

* Add job summary

Author: cknitt

.github/workflows/ci.yml

@@ -11,6 +11,10 @@ on:
   pull_request:
     branches: [master, 11.0_release]
 
+permissions:
+  # allow posting comments to pull request
+  pull-requests: write
+  
 concurrency:
   group: ci-${{ github.ref }}-1
   # Cancel previous builds for pull requests only.
@@ -90,8 +94,9 @@ jobs:
             ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
             upload_binaries: true
             upload_libs: true
-            # Build the playground compiler on the fastest runner
+            # Build the playground compiler and run the benchmarks on the fastest runner
             build_playground: true
+            benchmarks: true
           - os: buildjet-2vcpu-ubuntu-2204-arm # ARM
             ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
             upload_binaries: true
@@ -150,7 +155,7 @@ jobs:
       # matrix.ocaml_compiler may contain commas
       - name: Get OPAM cache key
         shell: bash
-        run: echo "opam_cache_key=opam-env-v3-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV
+        run: echo "opam_cache_key=opam-env-v4-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV
 
       - name: Restore OPAM environment
         id: cache-opam-env
@@ -320,6 +325,32 @@ jobs:
         if: runner.os != 'Windows'
         run: make -C tests/gentype_tests/typescript-react-example clean test
 
+      - name: Run syntax benchmarks
+        if: matrix.benchmarks
+        run: ./_build/install/default/bin/syntax_benchmarks | tee tests/benchmark-output.json
+
+      - name: Download previous benchmark data
+        if: matrix.benchmarks
+        uses: actions/cache@v4
+        with:
+          path: ./tests/benchmark-cache
+          key: syntax-benchmark-v1
+ 
+      - name: Store benchmark result
+        # Do not run for PRs created from other repos as those won't be able to write to the pull request
+        if: ${{ matrix.benchmarks && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.event.repository.full_name) }}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Syntax Benchmarks
+          tool: customSmallerIsBetter
+          output-file-path: tests/benchmark-output.json
+          external-data-json-path: ./tests/benchmark-cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: "150%"
+          comment-always: true
+          comment-on-alert: true
+          summary-always: true
+
       - name: Build playground compiler
         if: matrix.build_playground
         run: |

dune-project

@@ -24,6 +24,10 @@
    (and
     :with-test
     (= 0.26.2)))
+  (yojson
+   (and
+    :with-test
+    (= 2.2.2)))
   (ocaml-lsp-server
    (and
     :with-dev-setup

rescript.opam

@@ -9,6 +9,7 @@ bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues"
 depends: [
   "ocaml" {>= "4.10"}
   "ocamlformat" {with-test & = "0.26.2"}
+  "yojson" {with-test & = "2.2.2"}
   "ocaml-lsp-server" {with-dev-setup & = "1.19.0"}
   "cppo" {= "1.6.9"}
   "js_of_ocaml" {= "5.8.1"}

tests/syntax_benchmarks/Benchmark.ml

@@ -75,59 +75,31 @@ end = struct
 end
 
 module Benchmark : sig
-  type t
+  type test_result = {ms_per_run: float; allocs_per_run: int}
 
-  val make : name:string -> f:(t -> unit) -> unit -> t
-  val launch : t -> unit
-  val report : t -> unit
+  val run : (unit -> unit) -> num_iterations:int -> test_result
 end = struct
   type t = {
-    name: string;
     mutable start: Time.t;
-    mutable n: int; (* current iterations count *)
-    mutable duration: Time.t;
-    bench_func: t -> unit;
+    mutable n: int; (* current iteration count *)
+    mutable total_duration: Time.t;
+    bench_func: unit -> unit;
     mutable timer_on: bool;
-    (* mutable result: benchmarkResult; *)
-    (* The initial states *)
     mutable start_allocs: float;
-    mutable start_bytes: float;
-    (* The net total of this test after being run. *)
-    mutable net_allocs: float;
-    mutable net_bytes: float;
+    mutable total_allocs: float;
   }
 
-  let report b =
-    print_endline (Format.sprintf "Benchmark: %s" b.name);
-    print_endline (Format.sprintf "Nbr of iterations: %d" b.n);
-    print_endline
-      (Format.sprintf "Benchmark ran during: %fms" (Time.print b.duration));
-    print_endline
-      (Format.sprintf "Avg time/op: %fms"
-         (Time.print b.duration /. float_of_int b.n));
-    print_endline
-      (Format.sprintf "Allocs/op: %d"
-         (int_of_float (b.net_allocs /. float_of_int b.n)));
-    print_endline
-      (Format.sprintf "B/op: %d"
-         (int_of_float (b.net_bytes /. float_of_int b.n)));
-
-    (* return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() *)
-    print_newline ();
-    ()
+  type test_result = {ms_per_run: float; allocs_per_run: int}
 
-  let make ~name ~f () =
+  let make f =
     {
-      name;
       start = Time.zero;
       n = 0;
       bench_func = f;
-      duration = Time.zero;
+      total_duration = Time.zero;
       timer_on = false;
       start_allocs = 0.;
-      start_bytes = 0.;
-      net_allocs = 0.;
-      net_bytes = 0.;
+      total_allocs = 0.;
     }
 
   (* total amount of memory allocated by the program since it started in words *)
@@ -139,79 +111,74 @@ end = struct
     if not b.timer_on then (
       let allocated_words = mallocs () in
       b.start_allocs <- allocated_words;
-      b.start_bytes <- allocated_words *. 8.;
       b.start <- Time.now ();
       b.timer_on <- true)
 
   let stop_timer b =
     if b.timer_on then (
       let allocated_words = mallocs () in
       let diff = Time.diff b.start (Time.now ()) in
-      b.duration <- Time.add b.duration diff;
-      b.net_allocs <- b.net_allocs +. (allocated_words -. b.start_allocs);
-      b.net_bytes <- b.net_bytes +. ((allocated_words *. 8.) -. b.start_bytes);
+      b.total_duration <- Time.add b.total_duration diff;
+      b.total_allocs <- b.total_allocs +. (allocated_words -. b.start_allocs);
       b.timer_on <- false)
 
   let reset_timer b =
     if b.timer_on then (
       let allocated_words = mallocs () in
       b.start_allocs <- allocated_words;
-      b.net_allocs <- allocated_words *. 8.;
-      b.start <- Time.now ());
-    b.net_allocs <- 0.;
-    b.net_bytes <- 0.
+      b.start <- Time.now ())
 
   let run_iteration b n =
     Gc.full_major ();
     b.n <- n;
     reset_timer b;
     start_timer b;
-    b.bench_func b;
+    b.bench_func ();
     stop_timer b
 
-  let launch b =
-    (* 150 runs * all the benchmarks means around 1m of benchmark time *)
-    for n = 1 to 150 do
+  let run f ~num_iterations =
+    let b = make f in
+    for n = 1 to num_iterations do
       run_iteration b n
-    done
+    done;
+    {
+      ms_per_run = Time.print b.total_duration /. float_of_int b.n;
+      allocs_per_run = int_of_float (b.total_allocs /. float_of_int b.n);
+    }
 end
 
 module Benchmarks : sig
   val run : unit -> unit
 end = struct
   type action = Parse | Print
+
   let string_of_action action =
     match action with
-    | Parse -> "parser"
-    | Print -> "printer"
-
-  (* TODO: we could at Reason here *)
-  type lang = Rescript
-  let string_of_lang lang =
-    match lang with
-    | Rescript -> "rescript"
+    | Parse -> "Parse"
+    | Print -> "Print"
 
   let parse_rescript src filename =
     let p = Parser.make src filename in
     let structure = ResParser.parse_implementation p in
     assert (p.diagnostics == []);
     structure
 
-  let benchmark filename lang action =
-    let src = IO.read_file filename in
-    let name =
-      filename ^ " " ^ string_of_lang lang ^ " " ^ string_of_action action
-    in
+  let data_dir = "tests/syntax_benchmarks/data"
+  let num_iterations = 150
+
+  let benchmark (filename, action) =
+    let path = Filename.concat data_dir filename in
+    let src = IO.read_file path in
     let benchmark_fn =
-      match (lang, action) with
-      | Rescript, Parse ->
-        fun _ ->
-          let _ = Sys.opaque_identity (parse_rescript src filename) in
+      match action with
+      | Parse ->
+        fun () ->
+          let _ = Sys.opaque_identity (parse_rescript src path) in
           ()
-      | Rescript, Print ->
-        let p = Parser.make src filename in
+      | Print ->
+        let p = Parser.make src path in
         let ast = ResParser.parse_implementation p in
-        fun _ ->
+        fun () ->
           let _ =
             Sys.opaque_identity
               (let cmt_tbl = CommentTable.make () in
@@ -221,21 +188,45 @@ end = struct
           in
           ()
     in
-    let b = Benchmark.make ~name ~f:benchmark_fn () in
-    Benchmark.launch b;
-    Benchmark.report b
+    Benchmark.run benchmark_fn ~num_iterations
+
+  let specs =
+    [
+      ("RedBlackTree.res", Parse);
+      ("RedBlackTree.res", Print);
+      ("RedBlackTreeNoComments.res", Print);
+      ("Napkinscript.res", Parse);
+      ("Napkinscript.res", Print);
+      ("HeroGraphic.res", Parse);
+      ("HeroGraphic.res", Print);
+    ]
 
   let run () =
-    let data_dir = "tests/syntax_benchmarks/data" in
-    benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Parse;
-    benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Print;
-    benchmark
-      (Filename.concat data_dir "RedBlackTreeNoComments.res")
-      Rescript Print;
-    benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Parse;
-    benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Print;
-    benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Parse;
-    benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Print
+    List.to_seq specs
+    |> Seq.flat_map (fun spec ->
+           let filename, action = spec in
+           let test_name = string_of_action action ^ " " ^ filename in
+           let {Benchmark.ms_per_run; allocs_per_run} = benchmark spec in
+           [
+             `Assoc
+               [
+                 ("name", `String (Format.sprintf "%s - time/run" test_name));
+                 ("unit", `String "ms");
+                 ("value", `Float ms_per_run);
+               ];
+             `Assoc
+               [
+                 ("name", `String (Format.sprintf "%s - allocs/run" test_name));
+                 ("unit", `String "words");
+                 ("value", `Int allocs_per_run);
+               ];
+           ]
+           |> List.to_seq)
+    |> Seq.iteri (fun i json ->
+           print_endline (if i == 0 then "[" else ",");
+           print_string (Yojson.to_string json));
+    print_newline ();
+    print_endline "]"
 end
 
 let () = Benchmarks.run ()

tests/syntax_benchmarks/dune

@@ -9,6 +9,7 @@
  (enabled_if
   (and
    (<> %{profile} browser)
+   (>= %{ocaml_version} "4.14.0")
    (or
     (= %{system} macosx)
     ; or one of Linuxes (see https://github.com/ocaml/ocaml/issues/10613)
@@ -22,6 +23,6 @@
  (foreign_stubs
   (language c)
   (names time))
- (libraries syntax))
+ (libraries syntax yojson))
 
 (data_only_dirs data)

tests/syntax_benchmarks/time.c

@@ -37,7 +37,7 @@ CAMLprim value caml_mach_absolute_time(value unit) {
 #elif defined(__linux__)
   struct timespec now;
   clock_gettime(CLOCK_MONOTONIC, &now);
-  result = now.tv_sec * 1000 + now.tv_nsec / 1000000;
+  result = now.tv_sec * 1000000000 + now.tv_nsec;
 #endif
 
   return caml_copy_int64(result);