Add support for sharding tests across multiple workers (microsoft#32173)

* Add support for sharding tests across multiple workers

* Disable unittests when runners are expressly provided (unless they contain the unittest runner)

Author: weswigham

Gulpfile.js

@@ -415,6 +415,8 @@ task("runtests").flags = {
     "   --no-lint": "Disables lint",
     "   --timeout=<ms>": "Overrides the default test timeout.",
     "   --built": "Compile using the built version of the compiler.",
+    "   --shards": "Total number of shards running tests (default: 1)",
+    "   --shardId": "1-based ID of this shard (default: 1)",
 }
 
 const runTestsParallel = () => runConsoleTests("built/local/run.js", "min", /*runInParallel*/ true, /*watchMode*/ false);
@@ -430,6 +432,8 @@ task("runtests-parallel").flags = {
     "   --timeout=<ms>": "Overrides the default test timeout.",
     "   --built": "Compile using the built version of the compiler.",
     "   --skipPercent=<number>": "Skip expensive tests with <percent> chance to miss an edit. Default 5%.",
+    "   --shards": "Total number of shards running tests (default: 1)",
+    "   --shardId": "1-based ID of this shard (default: 1)",
 };
 
 task("diff", () => exec(getDiffTool(), [refBaseline, localBaseline], { ignoreExitCode: true }));

scripts/build/options.js

@@ -5,7 +5,7 @@ const os = require("os");
 /** @type {CommandLineOptions} */
 module.exports = minimist(process.argv.slice(2), {
     boolean: ["debug", "dirty", "inspect", "light", "colors", "lint", "lkg", "soft", "fix", "failed", "keepFailed", "force", "built"],
-    string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"],
+    string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout", "shards", "shardId"],
     alias: {
         "b": "browser",
         "d": "debug", "debug-brk": "debug",

scripts/build/tests.js

@@ -36,6 +36,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
     const testConfigFile = "test.config";
     const failed = cmdLineOptions.failed;
     const keepFailed = cmdLineOptions.keepFailed;
+    const shards = +cmdLineOptions.shards || undefined;
+    const shardId = +cmdLineOptions.shardId || undefined;
     if (!cmdLineOptions.dirty) {
         await cleanTestDirs();
         cancelToken.throwIfCancellationRequested();
@@ -63,8 +65,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
         testTimeout = 400000;
     }
 
-    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined) {
-        writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
+    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined || shards || shardId) {
+        writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed, shards, shardId);
     }
 
     const colors = cmdLineOptions.colors;
@@ -165,8 +167,10 @@ exports.cleanTestDirs = cleanTestDirs;
  * @param {string} [stackTraceLimit]
  * @param {string | number} [timeout]
  * @param {boolean} [keepFailed]
+ * @param {number | undefined} [shards]
+ * @param {number | undefined} [shardId]
  */
-function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
+function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed, shards, shardId) {
     const testConfigContents = JSON.stringify({
         test: tests ? [tests] : undefined,
         runners: runners ? runners.split(",") : undefined,
@@ -177,7 +181,9 @@ function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFold
         taskConfigsFolder,
         noColor: !cmdLineOptions.colors,
         timeout,
-        keepFailed
+        keepFailed,
+        shards,
+        shardId
     });
     log.info("Running tests with config: " + testConfigContents);
     fs.writeFileSync("test.config", testConfigContents);

src/harness/harness.ts

@@ -501,7 +501,7 @@ namespace Harness {
         }
 
         function enumerateTestFiles(runner: RunnerBase) {
-            return runner.enumerateTestFiles();
+            return runner.getTestFiles();
         }
 
         function listFiles(path: string, spec: RegExp, options: { recursive?: boolean } = {}) {

src/harness/runnerbase.ts

@@ -2,6 +2,9 @@ type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" |
 type CompilerTestKind = "conformance" | "compiler";
 type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 
+let shards = 1;
+let shardId = 1;
+
 abstract class RunnerBase {
     // contains the tests to run
     public tests: (string | Harness.FileBasedTest)[] = [];
@@ -19,6 +22,14 @@ abstract class RunnerBase {
 
     abstract enumerateTestFiles(): (string | Harness.FileBasedTest)[];
 
+    getTestFiles(): ReturnType<this["enumerateTestFiles"]> {
+        const all = this.enumerateTestFiles();
+        if (shards === 1) {
+            return all as ReturnType<this["enumerateTestFiles"]>;
+        }
+        return all.filter((_val, idx) => idx % shards === (shardId - 1)) as ReturnType<this["enumerateTestFiles"]>;
+    }
+
     /** The working directory where tests are found. Needed for batch testing where the input path will differ from the output path inside baselines */
     public workingDirectory = "";
 

src/testRunner/externalCompileRunner.ts

@@ -24,7 +24,7 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
      */
     initializeTests(): void {
         // Read in and evaluate the test list
-        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+        const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();
 
         // tslint:disable-next-line:no-this-assignment
         const cls = this;
@@ -113,7 +113,7 @@ class DockerfileRunner extends ExternalCompileRunnerBase {
     }
     initializeTests(): void {
         // Read in and evaluate the test list
-        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+        const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();
 
         // tslint:disable-next-line:no-this-assignment
         const cls = this;

src/testRunner/parallel/host.ts

@@ -222,7 +222,7 @@ namespace Harness.Parallel.Host {
             console.log("Discovering runner-based tests...");
             const discoverStart = +(new Date());
             for (const runner of runners) {
-                for (const test of runner.enumerateTestFiles()) {
+                for (const test of runner.getTestFiles()) {
                     const file = typeof test === "string" ? test : test.file;
                     let size: number;
                     if (!perfData) {

src/testRunner/projectsRunner.ts

@@ -32,7 +32,11 @@ namespace project {
 
     export class ProjectRunner extends RunnerBase {
         public enumerateTestFiles() {
-            return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
+            const all = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
+            if (shards === 1) {
+                return all;
+            }
+            return all.filter((_val, idx) => idx % shards === (shardId - 1));
         }
 
         public kind(): TestRunnerKind {

src/testRunner/runner.ts

@@ -80,6 +80,8 @@ interface TestConfig {
     timeout?: number;
     keepFailed?: boolean;
     skipPercent?: number;
+    shardId?: number;
+    shards?: number;
 }
 
 interface TaskSet {
@@ -114,6 +116,12 @@ function handleTestConfig() {
         if (testConfig.skipPercent !== undefined) {
             skipPercent = testConfig.skipPercent;
         }
+        if (testConfig.shardId) {
+            shardId = testConfig.shardId;
+        }
+        if (testConfig.shards) {
+            shards = testConfig.shards;
+        }
 
         if (testConfig.stackTraceLimit === "full") {
             (<any>Error).stackTraceLimit = Infinity;
@@ -129,6 +137,9 @@ function handleTestConfig() {
 
         const runnerConfig = testConfig.runners || testConfig.test;
         if (runnerConfig && runnerConfig.length > 0) {
+            if (testConfig.runners) {
+                runUnitTests = runnerConfig.indexOf("unittest") !== -1;
+            }
             for (const option of runnerConfig) {
                 if (!option) {
                     continue;

src/testRunner/rwcRunner.ts

@@ -225,7 +225,7 @@ class RWCRunner extends RunnerBase {
      */
     public initializeTests(): void {
         // Read in and evaluate the test list
-        for (const test of this.tests && this.tests.length ? this.tests : this.enumerateTestFiles()) {
+        for (const test of this.tests && this.tests.length ? this.tests : this.getTestFiles()) {
             this.runTest(typeof test === "string" ? test : test.file);
         }
     }

src/testRunner/test262Runner.ts

@@ -97,7 +97,7 @@ class Test262BaselineRunner extends RunnerBase {
     public initializeTests() {
         // this will set up a series of describe/it blocks to run between the setup and cleanup phases
         if (this.tests.length === 0) {
-            const testFiles = this.enumerateTestFiles();
+            const testFiles = this.getTestFiles();
             testFiles.forEach(fn => {
                 this.runTest(fn);
             });