tester.js

// This package needs to be install:
//
// ```
// npm install browser-ui-test
// ```

const fs = require("fs");
const path = require("path");
const os = require("os");
const {Options, runTest} = require("browser-ui-test");

// If a test fails or errors, we will retry it two more times in case it was a flaky failure.
const NB_RETRY = 3;

function showHelp() {
    console.log("rustdoc-js options:");
    console.log("  --doc-folder [PATH]        : location of the generated doc folder");
    console.log("  --file [PATH]              : file to run (can be repeated)");
    console.log("  --debug                    : show extra information about script run");
    console.log("  --show-text                : render font in pages");
    console.log("  --no-headless              : disable headless mode");
    console.log("  --no-sandbox               : disable sandbox mode");
    console.log("  --help                     : show this message then quit");
    console.log("  --tests-folder [PATH]      : location of the .GOML tests folder");
    console.log("  --jobs [NUMBER]            : number of threads to run tests on");
    console.log("  --executable-path [PATH]   : path of the browser's executable to be used");
}

function isNumeric(s) {
    return /^\d+$/.test(s);
}

function parseOptions(args) {
    const opts = {
        "doc_folder": "",
        "tests_folder": "",
        "files": [],
        "debug": false,
        "show_text": false,
        "no_headless": false,
        "jobs": -1,
        "executable_path": null,
        "no_sandbox": false,
    };
    const correspondences = {
        "--doc-folder": "doc_folder",
        "--tests-folder": "tests_folder",
        "--debug": "debug",
        "--show-text": "show_text",
        "--no-headless": "no_headless",
        "--executable-path": "executable_path",
        "--no-sandbox": "no_sandbox",
    };

    for (let i = 0; i < args.length; ++i) {
        const arg = args[i];
        if (arg === "--doc-folder"
            || arg === "--tests-folder"
            || arg === "--file"
            || arg === "--jobs"
            || arg === "--executable-path") {
            i += 1;
            if (i >= args.length) {
                console.log("Missing argument after `" + arg + "` option.");
                return null;
            }
            const arg_value = args[i];
            if (arg === "--jobs") {
                if (!isNumeric(arg_value)) {
                    console.log(
                        "`--jobs` option expects a positive number, found `" + arg_value + "`");
                    return null;
                }
                opts["jobs"] = parseInt(arg_value);
            } else if (arg !== "--file") {
                opts[correspondences[arg]] = arg_value;
            } else {
                opts["files"].push(arg_value);
            }
        } else if (arg === "--help") {
            showHelp();
            process.exit(0);
        } else if (arg === "--no-sandbox") {
            console.log("`--no-sandbox` is being used. Be very careful!");
            opts[correspondences[arg]] = true;
        } else if (correspondences[arg]) {
            opts[correspondences[arg]] = true;
        } else {
            console.log("Unknown option `" + arg + "`.");
            console.log("Use `--help` to see the list of options");
            return null;
        }
    }
    if (opts["tests_folder"].length < 1) {
        console.log("Missing `--tests-folder` option.");
    } else if (opts["doc_folder"].length < 1) {
        console.log("Missing `--doc-folder` option.");
    } else {
        return opts;
    }
    return null;
}

/// Print single char status information without \n
function char_printer(n_tests) {
    const max_per_line = 10;
    let current = 0;
    return {
        successful: function() {
            current += 1;
            if (current % max_per_line === 0) {
                process.stdout.write(`. (${current}/${n_tests})${os.EOL}`);
            } else {
                process.stdout.write(".");
            }
        },
        erroneous: function() {
            current += 1;
            if (current % max_per_line === 0) {
                process.stderr.write(`F (${current}/${n_tests})${os.EOL}`);
            } else {
                process.stderr.write("F");
            }
        },
        finish: function() {
            if (current % max_per_line === 0) {
                // Don't output if we are already at a matching line end
                console.log("");
            } else {
                const spaces = " ".repeat(max_per_line - (current % max_per_line));
                process.stdout.write(`${spaces} (${current}/${n_tests})${os.EOL}${os.EOL}`);
            }
        },
    };
}

// Sort array by .file_name property
function by_filename(a, b) {
    return a.file_name - b.file_name;
}

async function runTests(opts, framework_options, files, results, status_bar, showTestFailures) {
    const tests_queue = [];

    for (const testPath of files) {
        const callback = runTest(testPath, {"options": framework_options})
            .then(out => {
                const [output, nb_failures] = out;
                results[nb_failures === 0 ? "successful" : "failed"].push({
                    file_name: testPath,
                    output: output,
                });
                if (nb_failures === 0) {
                    status_bar.successful();
                } else if (showTestFailures) {
                    status_bar.erroneous();
                }
            })
            .catch(err => {
                results.errored.push({
                    file_name: testPath,
                    output: err,
                });
                if (showTestFailures) {
                    status_bar.erroneous();
                }
            })
            .finally(() => {
                // We now remove the promise from the tests_queue.
                tests_queue.splice(tests_queue.indexOf(callback), 1);
            });
        tests_queue.push(callback);
        if (opts["jobs"] > 0 && tests_queue.length >= opts["jobs"]) {
            await Promise.race(tests_queue);
        }
    }
    if (tests_queue.length > 0) {
        await Promise.all(tests_queue);
    }
}

function createEmptyResults() {
    return {
        successful: [],
        failed: [],
        errored: [],
    };
}

async function main(argv) {
    const opts = parseOptions(argv.slice(2));
    if (opts === null) {
        process.exit(1);
    }

    // Print successful tests too
    let debug = false;
    // Run tests in sequentially
    let headless = true;
    const framework_options = new Options();
    try {
        // This is more convenient that setting fields one by one.
        const args = [
            "--variable", "DOC_PATH", opts["doc_folder"].split("\\").join("/"),
            "--enable-fail-on-js-error", "--allow-file-access-from-files",
        ];
        if (opts["debug"]) {
            debug = true;
            args.push("--debug");
        }
        if (opts["show_text"]) {
            args.push("--show-text");
        }
        if (opts["no_sandbox"]) {
            args.push("--no-sandbox");
        }
        if (opts["no_headless"]) {
            args.push("--no-headless");
            headless = false;
        }
        if (opts["executable_path"] !== null) {
            args.push("--executable-path");
            args.push(opts["executable_path"]);
        }
        framework_options.parseArguments(args);
    } catch (error) {
        console.error(`invalid argument: ${error}`);
        process.exit(1);
    }

    let files;
    if (opts["files"].length === 0) {
        files = fs.readdirSync(opts["tests_folder"]);
    } else {
        files = opts["files"];
    }
    files = files.filter(file => path.extname(file) === ".goml");
    if (files.length === 0) {
        console.error("rustdoc-gui: No test selected");
        process.exit(2);
    }
    files.forEach((file_name, index) => {
        files[index] = path.join(opts["tests_folder"], file_name);
    });
    files.sort();

    if (!headless) {
        opts["jobs"] = 1;
        console.log("`--no-headless` option is active, disabling concurrency for running tests.");
    }

    if (opts["jobs"] < 1) {
        const len = files.length;
        console.log(
            `Running ${len} rustdoc-gui (UNBOUNDED concurrency; use "-j#" for a limit) ...`,
        );
        process.setMaxListeners(files.length + 1);
    } else if (headless) {
        console.log(`Running ${files.length} rustdoc-gui (${opts["jobs"]} concurrently) ...`);
        process.setMaxListeners(opts["jobs"] + 1);
    } else {
        console.log(`Running ${files.length} rustdoc-gui ...`);
    }

    // We catch this "event" to display a nicer message in case of unexpected exit (because of a
    // missing `--no-sandbox`).
    const exitHandling = () => {
        if (!opts["no_sandbox"]) {
            console.log("");
            console.log(
                "`browser-ui-test` crashed unexpectedly. Please try again with adding `--test-args \
--no-sandbox` at the end. For example: `x.py test tests/rustdoc-gui --test-args --no-sandbox`");
            console.log("");
        }
    };
    process.on("exit", exitHandling);

    const originalFilesLen = files.length;
    const results = createEmptyResults();
    const status_bar = char_printer(files.length);

    let new_results;
    for (let it = 0; it < NB_RETRY && files.length > 0; ++it) {
        new_results = createEmptyResults();
        await runTests(opts, framework_options, files, new_results, status_bar, it + 1 >= NB_RETRY);
        Array.prototype.push.apply(results.successful, new_results.successful);
        // We generate the new list of files with the previously failing tests.
        files = Array.prototype.concat(new_results.failed, new_results.errored).map(
            f => f["file_name"]);
        if (files.length > originalFilesLen / 2) {
            // If we have too many failing tests, it's very likely not flaky failures anymore so
            // no need to retry.
            break;
        }
    }

    status_bar.finish();

    Array.prototype.push.apply(results.failed, new_results.failed);
    Array.prototype.push.apply(results.errored, new_results.errored);

    // We don't need this listener anymore.
    process.removeListener("exit", exitHandling);

    if (debug) {
        results.successful.sort(by_filename);
        results.successful.forEach(r => {
            console.log(r.output);
        });
    }

    if (results.failed.length > 0) {
        console.log("");
        results.failed.sort(by_filename);
        results.failed.forEach(r => {
            console.log(r.file_name, r.output);
        });
    }
    if (results.errored.length > 0) {
        console.log(os.EOL);
        // print run errors on the bottom so developers see them better
        results.errored.sort(by_filename);
        results.errored.forEach(r => {
            console.error(r.file_name, r.output);
        });
    }

    if (results.failed.length > 0 || results.errored.length > 0) {
        process.exit(1);
    }
    process.exit(0);
}

main(process.argv);