Merge pull request microsoft#25004 from Microsoft/trackFailedTests

rbuckton · web-flow · commit 78bf10ad80e0 · 2018-06-25T16:55:43.000-07:00
Adds failed test tracking
diff --git a/.gitignore b/.gitignore
@@ -73,4 +73,5 @@ tests/cases/user/*/**/*.d.ts
 !tests/cases/user/zone.js/
 !tests/cases/user/bignumber.js/
 !tests/cases/user/discord.js/
-tests/baselines/reference/dt
+tests/baselines/reference/dt
+.failed-tests
diff --git a/Jakefile.js b/Jakefile.js
@@ -410,6 +410,8 @@ function runConsoleTests(defaultReporter, runInParallel) {
     const runners = process.env.runners || process.env.runner || process.env.ru;
     const tests = process.env.test || process.env.tests || process.env.t;
     const light = process.env.light === undefined || process.env.light !== "false";
+    const failed = process.env.failed;
+    const keepFailed = process.env.keepFailed || failed;
     const stackTraceLimit = process.env.stackTraceLimit;
     const colorsFlag = process.env.color || process.env.colors;
     const colors = colorsFlag !== "false" && colorsFlag !== "0";
@@ -440,16 +442,17 @@ function runConsoleTests(defaultReporter, runInParallel) {
         testTimeout = 800000;
     }
 
-    if (tests || runners || light || testTimeout || taskConfigsFolder) {
-        writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout);
+    if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
+        writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed);
     }
 
     // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
     // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
     if (!runInParallel) {
         var startTime = Travis.mark();
         var args = [];
-        args.push("-R", reporter);
+        args.push("-R", "scripts/failed-tests");
+        args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
         if (tests) args.push("-g", `"${tests}"`);
         args.push(colors ? "--colors" : "--no-colors");
         if (bail) args.push("--bail");
@@ -460,7 +463,14 @@ function runConsoleTests(defaultReporter, runInParallel) {
         }
         args.push(Paths.builtLocalRun);
 
-        var cmd = "mocha " + args.join(" ");
+        var cmd;
+        if (failed) {
+            args.unshift("scripts/run-failed-tests.js");
+            cmd = host + " " + args.join(" ");
+        }
+        else {
+            cmd = "mocha " + args.join(" ");
+        }
         var savedNodeEnv = process.env.NODE_ENV;
         process.env.NODE_ENV = "development";
         exec(cmd, function () {
@@ -521,7 +531,7 @@ function runConsoleTests(defaultReporter, runInParallel) {
 }
 
 // used to pass data from jake command line directly to run.js
-function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout) {
+function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed) {
     var testConfigContents = JSON.stringify({
         runners: runners ? runners.split(",") : undefined,
         test: tests ? [tests] : undefined,
@@ -530,7 +540,8 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
         taskConfigsFolder: taskConfigsFolder,
         stackTraceLimit: stackTraceLimit,
         noColor: !colors,
-        timeout: testTimeout
+        timeout: testTimeout,
+        keepFailed: keepFailed
     });
     fs.writeFileSync('test.config', testConfigContents, { encoding: "utf-8" });
 }
diff --git a/scripts/build/options.js b/scripts/build/options.js
@@ -4,7 +4,7 @@ const os = require("os");
 
 /** @type {CommandLineOptions} */
 module.exports = minimist(process.argv.slice(2), {
-    boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix"],
+    boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix", "failed", "keepFailed"],
     string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"],
     alias: {
         "b": "browser",
@@ -32,6 +32,8 @@ module.exports = minimist(process.argv.slice(2), {
         lint: process.env.lint || true,
         fix: process.env.fix || process.env.f,
         workers: process.env.workerCount || os.cpus().length,
+        failed: false,
+        keepFailed: false
     }
 });
 
@@ -52,6 +54,8 @@ module.exports = minimist(process.argv.slice(2), {
  * @property {string} reporter
  * @property {string} stackTraceLimit
  * @property {string|number} timeout
+ * @property {boolean} failed
+ * @property {boolean} keepFailed
  * 
  * @typedef {import("minimist").ParsedArgs & TypedOptions} CommandLineOptions
  */
diff --git a/scripts/build/tests.js b/scripts/build/tests.js
@@ -27,14 +27,16 @@ exports.localTest262Baseline = "internal/baselines/test262/local";
  */
 function runConsoleTests(runJs, defaultReporter, runInParallel) {
     let testTimeout = cmdLineOptions.timeout;
+    let tests = cmdLineOptions.tests;
     const lintFlag = cmdLineOptions.lint;
     const debug = cmdLineOptions.debug;
     const inspect = cmdLineOptions.inspect;
-    const tests = cmdLineOptions.tests;
     const runners = cmdLineOptions.runners;
     const light = cmdLineOptions.light;
     const stackTraceLimit = cmdLineOptions.stackTraceLimit;
     const testConfigFile = "test.config";
+    const failed = cmdLineOptions.failed;
+    const keepFailed = cmdLineOptions.keepFailed || failed;
     return cleanTestDirs()
         .then(() => {
             if (fs.existsSync(testConfigFile)) {
@@ -59,8 +61,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
                 testTimeout = 400000;
             }
 
-            if (tests || runners || light || testTimeout || taskConfigsFolder) {
-                writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout);
+            if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
+                writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
             }
 
             const colors = cmdLineOptions.colors;
@@ -75,7 +77,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
             // timeout normally isn"t necessary but Travis-CI has been timing out on compiler baselines occasionally
             // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
             if (!runInParallel) {
-                args.push("-R", reporter);
+                args.push("-R", "scripts/failed-tests");
+                args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
                 if (tests) {
                     args.push("-g", `"${tests}"`);
                 }
@@ -103,7 +106,12 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
                 args.push(runJs);
             }
             setNodeEnvToDevelopment();
-            return exec(host, [runJs]);
+            if (failed) {
+                return exec(host, ["scripts/run-failed-tests.js"].concat(args));
+            }
+            else {
+                return exec(host, args);
+            }
         })
         .then(({ exitCode }) => {
             if (exitCode !== 0) return finish(undefined, exitCode);
@@ -148,8 +156,9 @@ exports.cleanTestDirs = cleanTestDirs;
  * @param {string | number} [workerCount]
  * @param {string} [stackTraceLimit]
  * @param {string | number} [timeout]
+ * @param {boolean} [keepFailed]
  */
-function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout) {
+function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
     const testConfigContents = JSON.stringify({
         test: tests ? [tests] : undefined,
         runner: runners ? runners.split(",") : undefined,
@@ -159,6 +168,7 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
         taskConfigsFolder,
         noColor: !cmdLineOptions.colors,
         timeout,
+        keepFailed
     });
     log.info("Running tests with config: " + testConfigContents);
     fs.writeFileSync("test.config", testConfigContents);
diff --git a/scripts/failed-tests.d.ts b/scripts/failed-tests.d.ts
@@ -0,0 +1,22 @@
+import Mocha = require("mocha");
+
+export = FailedTestsReporter;
+
+declare class FailedTestsReporter extends Mocha.reporters.Base {
+    passes: Mocha.Test[];
+    failures: Mocha.Test[];
+    reporterOptions: FailedTestsReporter.ReporterOptions;
+    reporter?: Mocha.reporters.Base;
+    constructor(runner: Mocha.Runner, options?: { reporterOptions?: FailedTestsReporter.ReporterOptions });
+    static writeFailures(file: string, passes: ReadonlyArray<Mocha.Test>, failures: ReadonlyArray<Mocha.Test>, keepFailed: boolean, done: (err?: NodeJS.ErrnoException) => void): void;
+    done(failures: number, fn?: (failures: number) => void): void;
+}
+
+declare namespace FailedTestsReporter {
+    interface ReporterOptions {
+        file?: string;
+        keepFailed?: boolean;
+        reporter?: string | Mocha.ReporterConstructor;
+        reporterOptions?: any;
+    }
+}
diff --git a/scripts/failed-tests.js b/scripts/failed-tests.js
@@ -0,0 +1,117 @@
+// @ts-check
+const Mocha = require("mocha");
+const path = require("path");
+const fs = require("fs");
+const os = require("os");
+
+/**
+ * .failed-tests reporter
+ * 
+ * @typedef {Object} ReporterOptions
+ * @property {string} [file]
+ * @property {boolean} [keepFailed]
+ * @property {string|Mocha.ReporterConstructor} [reporter]
+ * @property {*} [reporterOptions]
+ */
+class FailedTestsReporter extends Mocha.reporters.Base {
+    /**
+     * @param {Mocha.Runner} runner 
+     * @param {{ reporterOptions?: ReporterOptions }} [options]
+     */
+    constructor(runner, options) {
+        super(runner, options);
+        if (!runner) return;
+
+        const reporterOptions = this.reporterOptions = options.reporterOptions || {};
+        if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests";
+        if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false;
+        if (reporterOptions.reporter) {
+            /** @type {Mocha.ReporterConstructor} */
+            let reporter;
+            if (typeof reporterOptions.reporter === "function") {
+                reporter = reporterOptions.reporter;
+            }
+            else if (Mocha.reporters[reporterOptions.reporter]) {
+                reporter = Mocha.reporters[reporterOptions.reporter];
+            }
+            else {
+                try {
+                    reporter = require(reporterOptions.reporter);
+                }
+                catch (_) {
+                    reporter = require(path.resolve(process.cwd(), reporterOptions.reporter));
+                }
+            }
+
+            const newOptions = Object.assign({}, options, { reporterOptions: reporterOptions.reporterOptions || {} });
+            this.reporter = new reporter(runner, newOptions);
+        }
+
+        /** @type {Mocha.Test[]} */
+        this.passes = [];
+        
+        /** @type {Mocha.Test[]} */
+        this.failures = [];
+        
+        runner.on("pass", test => this.passes.push(test));
+        runner.on("fail", test => this.failures.push(test));
+    }
+
+    /**
+     * @param {string} file 
+     * @param {ReadonlyArray<Mocha.Test>} passes 
+     * @param {ReadonlyArray<Mocha.Test>} failures 
+     * @param {boolean} keepFailed 
+     * @param {(err?: NodeJS.ErrnoException) => void} done 
+     */
+    static writeFailures(file, passes, failures, keepFailed, done) {
+        const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined);
+        if (failingTests.size > 0) {
+            for (const test of passes) {
+                const title = test.fullTitle().trim();
+                if (title) failingTests.delete(title);
+            }
+        }
+        for (const test of failures) {
+            const title = test.fullTitle().trim();
+            if (title) failingTests.add(title);
+        }
+        if (failingTests.size > 0) {
+            const failed = Array.from(failingTests).join(os.EOL);
+            fs.writeFile(file, failed, "utf8", done);
+        }
+        else if (!keepFailed) {
+            fs.unlink(file, done);
+        }
+        else {
+            done();
+        }
+
+        function readTests() {
+            return fs.readFileSync(file, "utf8")
+                .split(/\r?\n/g)
+                .map(line => line.trim())
+                .filter(line => line.length > 0);
+        }
+    }
+
+    /**
+     * @param {number} failures 
+     * @param {(failures: number) => void} [fn]
+     */
+    done(failures, fn) {
+        FailedTestsReporter.writeFailures(this.reporterOptions.file, this.passes, this.failures, this.reporterOptions.keepFailed || this.stats.tests === 0, (err) => {
+            const reporter = this.reporter;
+            if (reporter && reporter.done) {
+                reporter.done(failures, fn);
+            }
+            else if (fn) {
+                fn(failures);
+            }
+
+            if (err) console.error(err);
+        });
+    }
+}
+
+module.exports = FailedTestsReporter;
diff --git a/scripts/run-failed-tests.js b/scripts/run-failed-tests.js
diff --git a/src/testRunner/parallel/host.ts b/src/testRunner/parallel/host.ts
diff --git a/src/testRunner/runner.ts b/src/testRunner/runner.ts
