Always collect type and symbol baselines (microsoft#18621)

* Always generate type & symbol baselines

* Accept changed shadowed baselines

* Accept brand new type and symbol baselines

* Allow `getTypeAtLocation` to return undefined in the type writer

* Accept baselines which had missing type information

* Bind container for dynamically names enum members so they may be printed

* Accept type/symbol baselines for enums with computed members

* First pass at reducing typeWriter memory overhead

* Use generators to allow for type and symbol baselines with no cache

* Accept new baselines for tests whose output was fixed by better newline splitting

* Hard cap on number of declarations printed, cache declaration print text

* handle differing newlines better still to handle RWC newlines

* Lower abridging count, accept abridged baselines

* Limit max RWC error output size, limit RWC type and symbol baseline input size

* Move skip logic into type and symbol baseliner to streamline error handling

* Accept removal of empty baselines

* Canonicalize path earlier to handle odd paths in input files

* Do canonicalization earlier still, also ensure parallel perf profiles for different targets do not trample one another

* No need to pathify again

Author: weswigham

.gitignore

@@ -59,4 +59,4 @@ internal/
 .idea
 yarn.lock
 package-lock.json
-.parallelperf.json
+.parallelperf.*

src/compiler/binder.ts

@@ -1456,11 +1456,6 @@ namespace ts {
         }
 
         function declareSymbolAndAddToSymbolTable(node: Declaration, symbolFlags: SymbolFlags, symbolExcludes: SymbolFlags): Symbol {
-            // Just call this directly so that the return type of this function stays "void".
-            return declareSymbolAndAddToSymbolTableWorker(node, symbolFlags, symbolExcludes);
-        }
-
-        function declareSymbolAndAddToSymbolTableWorker(node: Declaration, symbolFlags: SymbolFlags, symbolExcludes: SymbolFlags): Symbol {
             switch (container.kind) {
                 // Modules, source files, and classes need specialized handling for how their
                 // members are declared (for example, a member of a class will go into a specific
@@ -1683,6 +1678,9 @@ namespace ts {
 
         function bindAnonymousDeclaration(node: Declaration, symbolFlags: SymbolFlags, name: __String) {
             const symbol = createSymbol(symbolFlags, name);
+            if (symbolFlags & SymbolFlags.EnumMember) {
+                symbol.parent = container.symbol;
+            }
             addDeclarationToSymbol(symbol, node, symbolFlags);
         }
 

src/harness/compilerRunner.ts

@@ -177,7 +177,7 @@ class CompilerBaselineRunner extends RunnerBase {
                     return;
                 }
 
-                Harness.Compiler.doTypeAndSymbolBaseline(justName, result, toBeCompiled.concat(otherFiles).filter(file => !!result.program.getSourceFile(file.unitName)));
+                Harness.Compiler.doTypeAndSymbolBaseline(justName, result.program, toBeCompiled.concat(otherFiles).filter(file => !!result.program.getSourceFile(file.unitName)));
             });
         });
     }

src/harness/harness.ts

@@ -148,6 +148,8 @@ namespace Utils {
         });
     }
 
+    export const canonicalizeForHarness = ts.createGetCanonicalFileName(/*caseSensitive*/ false); // This is done so tests work on windows _and_ linux
+
     export function assertInvariants(node: ts.Node, parent: ts.Node): void {
         if (node) {
             assert.isFalse(node.pos < 0, "node.pos < 0");
@@ -1446,10 +1448,7 @@ namespace Harness {
             });
         }
 
-        export function doTypeAndSymbolBaseline(baselinePath: string, result: CompilerResult, allFiles: {unitName: string, content: string}[], opts?: Harness.Baseline.BaselineOptions, multifile?: boolean) {
-            if (result.errors.length !== 0) {
-                return;
-            }
+        export function doTypeAndSymbolBaseline(baselinePath: string, program: ts.Program, allFiles: {unitName: string, content: string}[], opts?: Harness.Baseline.BaselineOptions, multifile?: boolean, skipTypeAndSymbolbaselines?: boolean) {
             // The full walker simulates the types that you would get from doing a full
             // compile.  The pull walker simulates the types you get when you just do
             // a type query for a random node (like how the LS would do it).  Most of the
@@ -1465,16 +1464,8 @@ namespace Harness {
             // These types are equivalent, but depend on what order the compiler observed
             // certain parts of the program.
 
-            const program = result.program;
-
             const fullWalker = new TypeWriterWalker(program, /*fullTypeCheck*/ true);
 
-            const fullResults = ts.createMap<TypeWriterResult[]>();
-
-            for (const sourceFile of allFiles) {
-                fullResults.set(sourceFile.unitName, fullWalker.getTypeAndSymbols(sourceFile.unitName));
-            }
-
             // Produce baselines.  The first gives the types for all expressions.
             // The second gives symbols for all identifiers.
             let typesError: Error, symbolsError: Error;
@@ -1515,76 +1506,77 @@ namespace Harness {
                     baselinePath.replace(/\.tsx?/, "") : baselinePath;
 
                 if (!multifile) {
-                    const fullBaseLine = generateBaseLine(fullResults, isSymbolBaseLine);
+                    const fullBaseLine = generateBaseLine(isSymbolBaseLine, skipTypeAndSymbolbaselines);
                     Harness.Baseline.runBaseline(outputFileName + fullExtension, () => fullBaseLine, opts);
                 }
                 else {
                     Harness.Baseline.runMultifileBaseline(outputFileName, fullExtension, () => {
-                        return iterateBaseLine(fullResults, isSymbolBaseLine);
+                        return iterateBaseLine(isSymbolBaseLine, skipTypeAndSymbolbaselines);
                     }, opts);
                 }
             }
 
-            function generateBaseLine(typeWriterResults: ts.Map<TypeWriterResult[]>, isSymbolBaseline: boolean): string {
+            function generateBaseLine(isSymbolBaseline: boolean, skipTypeAndSymbolbaselines?: boolean): string {
                 let result = "";
-                const gen = iterateBaseLine(typeWriterResults, isSymbolBaseline);
+                const gen = iterateBaseLine(isSymbolBaseline, skipTypeAndSymbolbaselines);
                 for (let {done, value} = gen.next(); !done; { done, value } = gen.next()) {
                     const [, content] = value;
                     result += content;
                 }
-                return result;
+                /* tslint:disable:no-null-keyword */
+                return result || null;
+                /* tslint:enable:no-null-keyword */
             }
 
-            function *iterateBaseLine(typeWriterResults: ts.Map<TypeWriterResult[]>, isSymbolBaseline: boolean): IterableIterator<[string, string]> {
-                let typeLines = "";
-                const typeMap: { [fileName: string]: { [lineNum: number]: string[]; } } = {};
+            function *iterateBaseLine(isSymbolBaseline: boolean, skipTypeAndSymbolbaselines?: boolean): IterableIterator<[string, string]> {
+                if (skipTypeAndSymbolbaselines) {
+                    return;
+                }
                 const dupeCase = ts.createMap<number>();
 
                 for (const file of allFiles) {
-                    const codeLines = file.content.split("\n");
-                    const key = file.unitName;
-                    typeWriterResults.get(file.unitName).forEach(result => {
+                    const { unitName } = file;
+                    let typeLines = "=== " + unitName + " ===\r\n";
+                    const codeLines = ts.flatMap(file.content.split(/\r?\n/g), e => e.split(/[\r\u2028\u2029]/g));
+                    const gen: IterableIterator<TypeWriterResult> = isSymbolBaseline ? fullWalker.getSymbols(unitName) : fullWalker.getTypes(unitName);
+                    let lastIndexWritten: number | undefined;
+                    for (let {done, value: result} = gen.next(); !done; { done, value: result } = gen.next()) {
                         if (isSymbolBaseline && !result.symbol) {
                             return;
                         }
-
+                        if (lastIndexWritten === undefined) {
+                            typeLines += codeLines.slice(0, result.line + 1).join("\r\n") + "\r\n";
+                        }
+                        else if (result.line !== lastIndexWritten) {
+                            if (!((lastIndexWritten + 1 < codeLines.length) && (codeLines[lastIndexWritten + 1].match(/^\s*[{|}]\s*$/) || codeLines[lastIndexWritten + 1].trim() === ""))) {
+                                typeLines += "\r\n";
+                            }
+                            typeLines += codeLines.slice(lastIndexWritten + 1, result.line + 1).join("\r\n") + "\r\n";
+                        }
+                        lastIndexWritten = result.line;
                         const typeOrSymbolString = isSymbolBaseline ? result.symbol : result.type;
                         const formattedLine = result.sourceText.replace(/\r?\n/g, "") + " : " + typeOrSymbolString;
-                        if (!typeMap[key]) {
-                            typeMap[key] = {};
-                        }
+                        typeLines += ">" + formattedLine + "\r\n";
+                    }
 
-                        let typeInfo = [formattedLine];
-                        const existingTypeInfo = typeMap[key][result.line];
-                        if (existingTypeInfo) {
-                            typeInfo = existingTypeInfo.concat(typeInfo);
+                    // Preserve legacy behavior
+                    if (lastIndexWritten === undefined) {
+                        for (let i = 0; i < codeLines.length; i++) {
+                            const currentCodeLine = codeLines[i];
+                            typeLines += currentCodeLine + "\r\n";
+                            typeLines += "No type information for this code.";
                         }
-                        typeMap[key][result.line] = typeInfo;
-                    });
-
-                    typeLines += "=== " + file.unitName + " ===\r\n";
-                    for (let i = 0; i < codeLines.length; i++) {
-                        const currentCodeLine = codeLines[i];
-                        typeLines += currentCodeLine + "\r\n";
-                        if (typeMap[key]) {
-                            const typeInfo = typeMap[key][i];
-                            if (typeInfo) {
-                                typeInfo.forEach(ty => {
-                                    typeLines += ">" + ty + "\r\n";
-                                });
-                                if (i + 1 < codeLines.length && (codeLines[i + 1].match(/^\s*[{|}]\s*$/) || codeLines[i + 1].trim() === "")) {
-                                }
-                                else {
-                                    typeLines += "\r\n";
-                                }
+                    }
+                    else {
+                        if (lastIndexWritten + 1 < codeLines.length) {
+                            if (!((lastIndexWritten + 1 < codeLines.length) && (codeLines[lastIndexWritten + 1].match(/^\s*[{|}]\s*$/) || codeLines[lastIndexWritten + 1].trim() === ""))) {
+                                typeLines += "\r\n";
                             }
+                            typeLines += codeLines.slice(lastIndexWritten + 1).join("\r\n");
                         }
-                        else {
-                            typeLines += "No type information for this code.";
-                        }
+                        typeLines += "\r\n";
                     }
-                    yield [checkDuplicatedFileName(file.unitName, dupeCase), typeLines];
-                    typeLines = "";
+                    yield [checkDuplicatedFileName(unitName, dupeCase), typeLines];
                 }
             }
         }
@@ -1726,7 +1718,7 @@ namespace Harness {
         }
 
         function sanitizeTestFilePath(name: string) {
-            return ts.normalizeSlashes(name.replace(/[\^<>:"|?*%]/g, "_")).replace(/\.\.\//g, "__dotdot/").toLowerCase();
+            return ts.toPath(ts.normalizeSlashes(name.replace(/[\^<>:"|?*%]/g, "_")).replace(/\.\.\//g, "__dotdot/"), "", Utils.canonicalizeForHarness);
         }
 
         // This does not need to exist strictly speaking, but many tests will need to be updated if it's removed
@@ -2070,7 +2062,6 @@ namespace Harness {
         export function runMultifileBaseline(relativeFileBase: string, extension: string, generateContent: () => IterableIterator<[string, string, number]> | IterableIterator<[string, string]>, opts?: BaselineOptions, referencedExtensions?: string[]): void {
             const gen = generateContent();
             const writtenFiles = ts.createMap<true>();
-            const canonicalize = ts.createGetCanonicalFileName(/*caseSensitive*/ false); // This is done so tests work on windows _and_ linux
             /* tslint:disable-next-line:no-null-keyword */
             const errors: Error[] = [];
             if (gen !== null) {
@@ -2086,8 +2077,7 @@ namespace Harness {
                     catch (e) {
                         errors.push(e);
                     }
-                    const path = ts.toPath(relativeFileName, "", canonicalize);
-                    writtenFiles.set(path, true);
+                    writtenFiles.set(relativeFileName, true);
                 }
             }
 
@@ -2100,8 +2090,7 @@ namespace Harness {
             const missing: string[] = [];
             for (const name of existing) {
                 const localCopy = name.substring(referenceDir.length - relativeFileBase.length);
-                const path = ts.toPath(localCopy, "", canonicalize);
-                if (!writtenFiles.has(path)) {
+                if (!writtenFiles.has(localCopy)) {
                     missing.push(localCopy);
                 }
             }
@@ -2114,13 +2103,14 @@ namespace Harness {
             if (errors.length || missing.length) {
                 let errorMsg = "";
                 if (errors.length) {
-                    errorMsg += `The baseline for ${relativeFileBase} has changed:${"\n    " + errors.map(e => e.message).join("\n    ")}`;
+                    errorMsg += `The baseline for ${relativeFileBase} in ${errors.length} files has changed:${"\n    " + errors.slice(0, 5).map(e => e.message).join("\n    ") + (errors.length > 5 ? "\n" + `    and ${errors.length - 5} more` : "")}`;
                 }
                 if (errors.length && missing.length) {
                     errorMsg += "\n";
                 }
                 if (missing.length) {
-                    errorMsg += `Baseline missing files:${"\n    " + missing.join("\n    ") + "\n"}Written:${"\n    " + ts.arrayFrom(writtenFiles.keys()).join("\n    ")}`;
+                    const writtenFilesArray = ts.arrayFrom(writtenFiles.keys());
+                    errorMsg += `Baseline missing ${missing.length} files:${"\n    " + missing.slice(0, 5).join("\n    ") + (missing.length > 5 ? "\n" + `    and ${missing.length - 5} more` : "") + "\n"}Written ${writtenFiles.size} files:${"\n    " + writtenFilesArray.slice(0, 5).join("\n    ") + (writtenFilesArray.length > 5 ? "\n" + `    and ${writtenFilesArray.length - 5} more` : "")}`;
                 }
                 throw new Error(errorMsg);
             }

src/harness/parallel/host.ts

@@ -26,9 +26,12 @@ namespace Harness.Parallel.Host {
         text?: string;
     }
 
-    const perfdataFileName = ".parallelperf.json";
-    function readSavedPerfData(): {[testHash: string]: number} {
-        const perfDataContents = Harness.IO.readFile(perfdataFileName);
+    const perfdataFileNameFragment = ".parallelperf";
+    function perfdataFileName(target?: string) {
+        return `${perfdataFileNameFragment}${target ? `.${target}` : ""}.json`;
+    }
+    function readSavedPerfData(target?: string): {[testHash: string]: number} {
+        const perfDataContents = Harness.IO.readFile(perfdataFileName(target));
         if (perfDataContents) {
             return JSON.parse(perfDataContents);
         }
@@ -46,7 +49,7 @@ namespace Harness.Parallel.Host {
         const { statSync }: { statSync(path: string): { size: number }; } = require("fs");
         let tasks: { runner: TestRunnerKind, file: string, size: number }[] = [];
         const newTasks: { runner: TestRunnerKind, file: string, size: number }[] = [];
-        const perfData = readSavedPerfData();
+        const perfData = readSavedPerfData(configOption);
         let totalCost = 0;
         let unknownValue: string | undefined;
         for (const runner of runners) {
@@ -290,7 +293,7 @@ namespace Harness.Parallel.Host {
                 reporter.epilogue();
             }
 
-            Harness.IO.writeFile(perfdataFileName, JSON.stringify(newPerfData, null, 4)); // tslint:disable-line:no-null-keyword
+            Harness.IO.writeFile(perfdataFileName(configOption), JSON.stringify(newPerfData, null, 4)); // tslint:disable-line:no-null-keyword
 
             process.exit(errorResults.length);
         }

src/harness/runner.ts

@@ -101,6 +101,7 @@ interface TaskSet {
     files: string[];
 }
 
+let configOption: string;
 function handleTestConfig() {
     if (testConfigContent !== "") {
         const testConfig = <TestConfig>JSON.parse(testConfigContent);
@@ -136,6 +137,13 @@ function handleTestConfig() {
                     continue;
                 }
 
+                if (!configOption) {
+                    configOption = option;
+                }
+                else {
+                    configOption += "+" + option;
+                }
+
                 switch (option) {
                     case "compiler":
                         runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));

src/harness/rwcRunner.ts

@@ -39,6 +39,8 @@ namespace RWC {
             const baseName = /(.*)\/(.*).json/.exec(ts.normalizeSlashes(jsonPath))[2];
             let currentDirectory: string;
             let useCustomLibraryFile: boolean;
+            let skipTypeAndSymbolbaselines = false;
+            const typeAndSymbolSizeLimit = 10000000;
             after(() => {
                 // Mocha holds onto the closure environment of the describe callback even after the test is done.
                 // Therefore we have to clean out large objects after the test is done.
@@ -52,6 +54,7 @@ namespace RWC {
                 // or to use lib.d.ts inside the json object. If the flag is true, use the lib.d.ts inside json file
                 // otherwise use the lib.d.ts from built/local
                 useCustomLibraryFile = undefined;
+                skipTypeAndSymbolbaselines = false;
             });
 
             it("can compile", function(this: Mocha.ITestCallbackContext) {
@@ -61,6 +64,7 @@ namespace RWC {
                 const ioLog: IOLog = JSON.parse(Harness.IO.readFile(jsonPath));
                 currentDirectory = ioLog.currentDirectory;
                 useCustomLibraryFile = ioLog.useCustomLibraryFile;
+                skipTypeAndSymbolbaselines = ioLog.filesRead.reduce((acc, elem) => (elem && elem.result && elem.result.contents) ? acc + elem.result.contents.length : acc, 0) > typeAndSymbolSizeLimit;
                 runWithIOLog(ioLog, () => {
                     opts = ts.parseCommandLine(ioLog.arguments, fileName => Harness.IO.readFile(fileName));
                     assert.equal(opts.errors.length, 0);
@@ -217,10 +221,10 @@ namespace RWC {
 
             it("has the expected types", () => {
                 // We don't need to pass the extension here because "doTypeAndSymbolBaseline" will append appropriate extension of ".types" or ".symbols"
-                Harness.Compiler.doTypeAndSymbolBaseline(baseName, compilerResult, inputFiles
+                Harness.Compiler.doTypeAndSymbolBaseline(baseName, compilerResult.program, inputFiles
                     .concat(otherFiles)
                     .filter(file => !!compilerResult.program.getSourceFile(file.unitName))
-                    .filter(e => !Harness.isDefaultLibraryFile(e.unitName)), baselineOpts, /*multifile*/ true);
+                    .filter(e => !Harness.isDefaultLibraryFile(e.unitName)), baselineOpts, /*multifile*/ true, skipTypeAndSymbolbaselines);
             });
 
             // Ideally, a generated declaration file will have no errors. But we allow generated