Refine benchmark suite (#203)

* Add several benchmark suite to find our bottleneck

* Perform benchmark with release configuration

* Add primitive conversion benchmark cases

Author: kateinoigakukun

IntegrationTests/TestSuites/Package.swift

@@ -35,6 +35,6 @@ let package = Package(
                 .product(name: "JavaScriptEventLoop", package: "JavaScriptKit"),
             ]
         ),
-        .target(name: "BenchmarkTests", dependencies: ["JavaScriptKit"]),
+        .target(name: "BenchmarkTests", dependencies: ["JavaScriptKit", "CHelpers"]),
     ]
 )

IntegrationTests/TestSuites/Sources/BenchmarkTests/Benchmark.swift

@@ -8,10 +8,10 @@ class Benchmark {
     let title: String
     let runner = JSObject.global.benchmarkRunner.function!
 
-    func testSuite(_ name: String, _ body: @escaping () -> Void) {
+    func testSuite(_ name: String, _ body: @escaping (Int) -> Void) {
         let jsBody = JSClosure { arguments -> JSValue in
             let iteration = Int(arguments[0].number!)
-            for _ in 0 ..< iteration { body() }
+            body(iteration)
             return .undefined
         }
         runner("\(title)/\(name)", jsBody)

IntegrationTests/TestSuites/Sources/BenchmarkTests/main.swift

@@ -1,22 +1,75 @@
 import JavaScriptKit
+import CHelpers
 
 let serialization = Benchmark("Serialization")
 
+let noopFunction = JSObject.global.noopFunction.function!
+
+serialization.testSuite("JavaScript function call through Wasm import") { n in
+    for _ in 0 ..< n {
+        benchmark_helper_noop()
+    }
+}
+
+serialization.testSuite("JavaScript function call through Wasm import with int") { n in
+    for _ in 0 ..< n {
+        benchmark_helper_noop_with_int(42)
+    }
+}
+
+serialization.testSuite("JavaScript function call from Swift") { n in
+    for _ in 0 ..< n {
+        _ = noopFunction()
+    }
+}
+
 let swiftInt: Double = 42
-serialization.testSuite("Swift Int to JavaScript") {
+serialization.testSuite("Swift Int to JavaScript with assignment") { n in
+    let jsNumber = JSValue.number(swiftInt)
+    let object = JSObject.global
+    let key = JSString("numberValue")
+    for _ in 0 ..< n {
+        object[key] = jsNumber
+    }
+}
+
+serialization.testSuite("Swift Int to JavaScript with call") { n in
     let jsNumber = JSValue.number(swiftInt)
+    for _ in 0 ..< n {
+        _ = noopFunction(jsNumber)
+    }
+}
+
+serialization.testSuite("JavaScript Number to Swift Int") { n in
     let object = JSObject.global
-    for i in 0 ..< 100 {
-        object["numberValue\(i)"] = jsNumber
+    let key = JSString("jsNumber")
+    for _ in 0 ..< n {
+        _ = object[key].number
     }
 }
 
 let swiftString = "Hello, world"
-serialization.testSuite("Swift String to JavaScript") {
+serialization.testSuite("Swift String to JavaScript with assignment") { n in
     let jsString = JSValue.string(swiftString)
     let object = JSObject.global
-    for i in 0 ..< 100 {
-        object["stringValue\(i)"] = jsString
+    let key = JSString("stringValue")
+    for _ in 0 ..< n {
+        object[key] = jsString
+    }
+}
+
+serialization.testSuite("Swift String to JavaScript with call") { n in
+    let jsString = JSValue.string(swiftString)
+    for _ in 0 ..< n {
+        _ = noopFunction(jsString)
+    }
+}
+
+serialization.testSuite("JavaScript String to Swift String") { n in
+    let object = JSObject.global
+    let key = JSString("jsString")
+    for _ in 0 ..< n {
+        _ = object[key].string
     }
 }
 
@@ -25,8 +78,8 @@ let objectHeap = Benchmark("Object heap")
 let global = JSObject.global
 let Object = global.Object.function!
 global.objectHeapDummy = .object(Object.new())
-objectHeap.testSuite("Increment and decrement RC") {
-    for _ in 0 ..< 100 {
+objectHeap.testSuite("Increment and decrement RC") { n in
+    for _ in 0 ..< n {
         _ = global.objectHeapDummy
     }
 }

IntegrationTests/TestSuites/Sources/CHelpers/include/helpers.h

@@ -3,3 +3,8 @@
 /// @param pages Number of memory pages to increase memory by.
 int growMemory(int pages);
 
+__attribute__((__import_module__("benchmark_helper"), __import_name__("noop")))
+extern void benchmark_helper_noop(void);
+
+__attribute__((__import_module__("benchmark_helper"), __import_name__("noop_with_int")))
+extern void benchmark_helper_noop_with_int(int);

IntegrationTests/bin/benchmark-tests.js

@@ -1,41 +1,67 @@
 const { startWasiTask } = require("../lib");
 const { performance } = require("perf_hooks");
 
+const SAMPLE_ITERATION = 1000000
+
 global.benchmarkRunner = function (name, body) {
     console.log(`Running '${name}' ...`);
     const startTime = performance.now();
-    body(5000);
+    body(SAMPLE_ITERATION);
     const endTime = performance.now();
     console.log("done " + (endTime - startTime) + " ms");
 };
 
+global.noopFunction = function () {}
+global.jsNumber = 42
+global.jsString = "myString"
+
 class JSBenchmark {
     constructor(title) {
         this.title = title;
     }
     testSuite(name, body) {
         benchmarkRunner(`${this.title}/${name}`, (iteration) => {
-            for (let idx = 0; idx < iteration; idx++) {
-                body();
-            }
+            body(iteration);
         });
     }
 }
 
 const serialization = new JSBenchmark("Serialization");
-serialization.testSuite("Write JavaScript number directly", () => {
+serialization.testSuite("Call JavaScript function directly", (n) => {
+    for (let idx = 0; idx < n; idx++) {
+        global.noopFunction()
+    }
+});
+
+serialization.testSuite("Assign JavaScript number directly", (n) => {
     const jsNumber = 42;
     const object = global;
-    for (let idx = 0; idx < 100; idx++) {
-        object["numberValue" + idx] = jsNumber;
+    const key = "numberValue"
+    for (let idx = 0; idx < n; idx++) {
+        object[key] = jsNumber;
     }
 });
 
-serialization.testSuite("Write JavaScript string directly", () => {
+serialization.testSuite("Call with JavaScript number directly", (n) => {
+    const jsNumber = 42;
+    for (let idx = 0; idx < n; idx++) {
+        global.noopFunction(jsNumber)
+    }
+});
+
+serialization.testSuite("Write JavaScript string directly", (n) => {
     const jsString = "Hello, world";
     const object = global;
-    for (let idx = 0; idx < 100; idx++) {
-        object["stringValue" + idx] = jsString;
+    const key = "stringValue"
+    for (let idx = 0; idx < n; idx++) {
+        object[key] = jsString;
+    }
+});
+
+serialization.testSuite("Call with JavaScript string directly", (n) => {
+    const jsString = "Hello, world";
+    for (let idx = 0; idx < n; idx++) {
+        global.noopFunction(jsString)
     }
 });
 

IntegrationTests/lib.js

@@ -97,6 +97,10 @@ const startWasiTask = async (wasmPath, wasiConstructor = selectWASIBackend()) =>
     let { instance } = await WebAssembly.instantiate(wasmBinary, {
         wasi_snapshot_preview1: wasi.wasiImport,
         javascript_kit: swift.importObjects(),
+        benchmark_helper: {
+            noop: () => {},
+            noop_with_int: (_) => {},
+        }
     });
 
     swift.setInstance(instance);

Makefile

@@ -20,11 +20,11 @@ test:
 
 .PHONY: benchmark_setup
 benchmark_setup:
-	cd IntegrationTests && make benchmark_setup
+	cd IntegrationTests && CONFIGURATION=release make benchmark_setup
 
 .PHONY: run_benchmark
 run_benchmark:
-	cd IntegrationTests && make -s run_benchmark
+	cd IntegrationTests && CONFIGURATION=release make -s run_benchmark
 
 .PHONY: perf-tester
 perf-tester: