[Frontend][Index] Add frontend option to skip indexing the stdlib (for test performance)

Several tests related to indexing system modules were taking a considerable
amount of time (100+ seconds in the worst case) indexing the standard library.
This adds a frontend option to skip it and updates those tests to pass it.

Author: Nathan Hawes

Diff for: include/swift/Frontend/FrontendOptions.h

@@ -82,6 +82,9 @@ class FrontendOptions {
   /// Emit index data for imported serialized swift system modules.
   bool IndexSystemModules = false;
 
+  /// If indexing system modules, don't index the stdlib.
+  bool IndexIgnoreStdlib = false;
+
   /// The module for which we should verify all of the generic signatures.
   std::string VerifyGenericSignaturesInModule;
 

Diff for: include/swift/Index/IndexRecord.h

@@ -36,14 +36,18 @@ namespace index {
 /// \param indexSystemModules If true, emit index data for imported serialized
 /// swift system modules.
 ///
+/// \param skipStdlib If indexing system modules, don't index the standard
+/// library.
+///
 /// \param isDebugCompilation true for non-optimized compiler invocation.
 ///
 /// \param targetTriple The target for this compilation.
 ///
 /// \param dependencyTracker The set of dependencies seen while building.
 bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
                     StringRef indexStorePath, bool indexSystemModules,
-                    bool isDebugCompilation, StringRef targetTriple,
+                    bool skipStdlib, bool isDebugCompilation,
+                    StringRef targetTriple,
                     const DependencyTracker &dependencyTracker);
 
 /// Index the given module and store the results to \p indexStorePath.
@@ -64,15 +68,18 @@ bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
 /// \param indexSystemModules If true, emit index data for imported serialized
 /// swift system modules.
 ///
+/// \param skipStdlib If indexing system modules, don't index the standard
+/// library.
+///
 /// \param isDebugCompilation true for non-optimized compiler invocation.
 ///
 /// \param targetTriple The target for this compilation.
 ///
 /// \param dependencyTracker The set of dependencies seen while building.
 bool indexAndRecord(ModuleDecl *module, ArrayRef<std::string> indexUnitTokens,
                     StringRef moduleUnitToken, StringRef indexStorePath,
-                    bool indexSystemModules, bool isDebugCompilation,
-                    StringRef targetTriple,
+                    bool indexSystemModules, bool skipStdlib,
+                    bool isDebugCompilation, StringRef targetTriple,
                     const DependencyTracker &dependencyTracker);
 // FIXME: indexUnitTokens could be StringRef, but that creates an impedance
 // mismatch in the caller.

Diff for: include/swift/Option/FrontendOptions.td

@@ -604,6 +604,10 @@ def external_pass_pipeline_filename : Separate<["-"], "external-pass-pipeline-fi
 def index_system_modules : Flag<["-"], "index-system-modules">,
   HelpText<"Emit index data for imported serialized swift system modules">;
 
+def index_ignore_stdlib :
+  Flag<["-"], "index-ignore-stdlib">,
+  HelpText<"Avoid emitting index data for the standard library.">;
+
 def dump_interface_hash : Flag<["-"], "dump-interface-hash">,
    HelpText<"Parse input file(s) and dump interface token hash(es)">,
    ModeOpt;

Diff for: lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -67,6 +67,7 @@ bool ArgsToFrontendOptionsConverter::convert(
     Opts.BridgingHeaderDirForPrint = A->getValue();
   }
   Opts.IndexSystemModules |= Args.hasArg(OPT_index_system_modules);
+  Opts.IndexIgnoreStdlib |= Args.hasArg(OPT_index_ignore_stdlib);
 
   Opts.EmitVerboseSIL |= Args.hasArg(OPT_emit_verbose_sil);
   Opts.EmitSortedSIL |= Args.hasArg(OPT_emit_sorted_sil);

Diff for: lib/FrontendTool/FrontendTool.cpp

@@ -1729,7 +1729,8 @@ static bool emitIndexDataIfNeeded(SourceFile *PrimarySourceFile,
             PrimarySourceFile->getFilename());
     if (index::indexAndRecord(PrimarySourceFile, PSPs.OutputFilename,
                               opts.IndexStorePath, opts.IndexSystemModules,
-                              isDebugCompilation, Invocation.getTargetTriple(),
+                              opts.IndexIgnoreStdlib, isDebugCompilation,
+                              Invocation.getTargetTriple(),
                               *Instance.getDependencyTracker())) {
       return true;
     }
@@ -1741,7 +1742,7 @@ static bool emitIndexDataIfNeeded(SourceFile *PrimarySourceFile,
 
     if (index::indexAndRecord(Instance.getMainModule(), opts.InputsAndOutputs.copyOutputFilenames(),
                               moduleToken, opts.IndexStorePath,
-                              opts.IndexSystemModules,
+                              opts.IndexSystemModules, opts.IndexIgnoreStdlib,
                               isDebugCompilation, Invocation.getTargetTriple(),
                               *Instance.getDependencyTracker())) {
       return true;

Diff for: lib/Index/IndexRecord.cpp

@@ -389,6 +389,7 @@ static bool
 emitDataForSwiftSerializedModule(ModuleDecl *module,
                                  StringRef indexStorePath,
                                  bool indexSystemModules,
+                                 bool skipStdlib,
                                  StringRef targetTriple,
                                  const clang::CompilerInstance &clangCI,
                                  DiagnosticEngine &diags,
@@ -398,6 +399,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
 static void addModuleDependencies(ArrayRef<ModuleDecl::ImportedModule> imports,
                                   StringRef indexStorePath,
                                   bool indexSystemModules,
+                                  bool skipStdlib,
                                   StringRef targetTriple,
                                   const clang::CompilerInstance &clangCI,
                                   DiagnosticEngine &diags,
@@ -441,9 +443,10 @@ static void addModuleDependencies(ArrayRef<ModuleDecl::ImportedModule> imports,
             // We don't officially support binary swift modules, so normally
             // the index data for user modules would get generated while
             // building them.
-            if (mod->isSystemModule() && indexSystemModules) {
+            if (mod->isSystemModule() && indexSystemModules &&
+                (!skipStdlib || !mod->isStdlibModule())) {
               emitDataForSwiftSerializedModule(mod, indexStorePath,
-                                               indexSystemModules,
+                                               indexSystemModules, skipStdlib,
                                                targetTriple, clangCI, diags,
                                                unitWriter, initialFile);
               withoutUnitName = false;
@@ -470,6 +473,7 @@ static bool
 emitDataForSwiftSerializedModule(ModuleDecl *module,
                                  StringRef indexStorePath,
                                  bool indexSystemModules,
+                                 bool skipStdlib,
                                  StringRef targetTriple,
                                  const clang::CompilerInstance &clangCI,
                                  DiagnosticEngine &diags,
@@ -590,7 +594,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
   SmallVector<ModuleDecl::ImportedModule, 8> imports;
   module->getImportedModules(imports, importFilter);
   StringScratchSpace moduleNameScratch;
-  addModuleDependencies(imports, indexStorePath, indexSystemModules,
+  addModuleDependencies(imports, indexStorePath, indexSystemModules, skipStdlib,
                         targetTriple, clangCI, diags, unitWriter,
                         moduleNameScratch, initialFile);
 
@@ -605,7 +609,8 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
 static bool
 recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
                      StringRef indexStorePath, bool indexSystemModules,
-                     bool isDebugCompilation, StringRef targetTriple,
+                     bool skipStdlib, bool isDebugCompilation,
+                     StringRef targetTriple,
                      ArrayRef<const clang::FileEntry *> fileDependencies,
                      const clang::CompilerInstance &clangCI,
                      DiagnosticEngine &diags) {
@@ -628,11 +633,11 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
   importFilter |= ModuleDecl::ImportFilterKind::Public;
   importFilter |= ModuleDecl::ImportFilterKind::Private;
   importFilter |= ModuleDecl::ImportFilterKind::ImplementationOnly;
-  // FIXME: ImportFilterKind::ShadowedBySeparateOverlay?
+
   SmallVector<ModuleDecl::ImportedModule, 8> imports;
   primarySourceFile->getImportedModules(imports, importFilter);
   StringScratchSpace moduleNameScratch;
-  addModuleDependencies(imports, indexStorePath, indexSystemModules,
+  addModuleDependencies(imports, indexStorePath, indexSystemModules, skipStdlib,
                         targetTriple, clangCI, diags, unitWriter,
                         moduleNameScratch, primarySourceFile);
 
@@ -685,6 +690,7 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
                            StringRef indexUnitToken,
                            StringRef indexStorePath,
                            bool indexSystemModules,
+                           bool skipStdlib,
                            bool isDebugCompilation,
                            StringRef targetTriple,
                            const DependencyTracker &dependencyTracker) {
@@ -712,7 +718,7 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
 #endif
 
   return recordSourceFileUnit(primarySourceFile, indexUnitToken,
-                              indexStorePath, indexSystemModules,
+                              indexStorePath, indexSystemModules, skipStdlib,
                               isDebugCompilation, targetTriple,
                               fileDependencies.getArrayRef(),
                               clangCI, diags);
@@ -723,6 +729,7 @@ bool index::indexAndRecord(ModuleDecl *module,
                            StringRef moduleUnitToken,
                            StringRef indexStorePath,
                            bool indexSystemModules,
+                           bool skipStdlib,
                            bool isDebugCompilation,
                            StringRef targetTriple,
                            const DependencyTracker &dependencyTracker) {
@@ -758,7 +765,7 @@ bool index::indexAndRecord(ModuleDecl *module,
         return true;
       }
       if (recordSourceFileUnit(SF, indexUnitTokens[unitIndex],
-                               indexStorePath, indexSystemModules,
+                               indexStorePath, indexSystemModules, skipStdlib,
                                isDebugCompilation, targetTriple,
                                fileDependencies.getArrayRef(),
                                clangCI, diags))

Diff for: test/Index/Store/cross-import-overlay.swift

@@ -2,7 +2,7 @@
 // RUN: cp -r %S/../Inputs/CrossImport %t/CrossImport
 // RUN: %{python} %S/../../CrossImport/Inputs/rewrite-module-triples.py %t/CrossImport %module-target-triple
 
-// RUN: %target-swift-frontend -c -index-store-path %t/idx -index-system-modules -enable-cross-import-overlays %s -Fsystem %t/CrossImport -o %t/file1.o -module-name cross_import_overlay
+// RUN: %target-swift-frontend -c -index-store-path %t/idx -index-system-modules -index-ignore-stdlib -enable-cross-import-overlays %s -Fsystem %t/CrossImport -o %t/file1.o -module-name cross_import_overlay
 // RUN: c-index-test core -print-unit %t/idx > %t/units
 // RUN: %FileCheck %s --input-file %t/units --check-prefix=UNIT
 // RUN: %FileCheck %s --input-file %t/units --check-prefix=UNIT-NEGATIVE
@@ -16,6 +16,8 @@ fromB()
 from_ABAdditions()
 from__ABAdditionsCAdditions()
 
+// Check the overlay modules' names match the names of their underlying modules.
+//
 // UNIT: module-name: cross_import_overlay
 // UNIT: main-path: {{.*}}/cross-import-overlay.swift
 // UNIT: DEPEND START
@@ -59,6 +61,14 @@ from__ABAdditionsCAdditions()
 // UNIT: Record | system | A | {{.*}}/A.swiftmodule/{{.*}}
 // UNIT: DEPEND END
 
+// Make sure we aren't leaking the underscored overlay names anywhere
+//
 // UNIT-NEGATIVE-NOT: Unit | {{.*}} | _ABAdditions |
+// UNIT-NEGATIVE-NOT: Record | {{.*}} | _ABAdditions |
 // UNIT-NEGATIVE-NOT: Unit | {{.*}} | __ABAdditionsCAdditions |
+// UNIT-NEGATIVE-NOT: Record | {{.*}} | __ABAdditionsCAdditions |
+
+// Make sure we don't regress test performance by indexing the stdlib.
+//
+// UNIT-NEGATIVE-NOT: Record | system | Swift |
 

Diff for: test/Index/Store/driver-index-with-auxiliary-outputs.swift

@@ -4,7 +4,7 @@
 // make sure the frontend job doesn't try to emit the auxiliary outputs based
 // on the non-indexed files. (This is how Xcode currently constructs -index-file
 // invocations: take a normal build command and add extra arguments to it.)
-// RUN: %target-build-swift -index-file -index-file-path %S/Inputs/SwiftModuleA.swift %S/Inputs/SwiftModuleA.swift %s -index-store-path %t/idx -module-name driver_index -emit-objc-header-path %t/out.h -emit-module-interface-path %t/out.swiftinterface
+// RUN: %target-build-swift -index-file -index-file-path %S/Inputs/SwiftModuleA.swift %S/Inputs/SwiftModuleA.swift %s -index-store-path %t/idx -Xfrontend -index-ignore-stdlib -module-name driver_index -emit-objc-header-path %t/out.h -emit-module-interface-path %t/out.swiftinterface
 
 // RUN: test ! -f %t/out.h
 // RUN: test ! -f %t/out.swiftinterface

Diff for: test/Index/index_swift_only_systemmodule.swift

@@ -34,6 +34,7 @@ print(someFunc())
 // RUN: %target-swift-frontend \
 // RUN:     -typecheck \
 // RUN:     -index-system-modules \
+// RUN:     -index-ignore-stdlib \
 // RUN:     -index-store-path %t/idx \
 // RUN:     -sdk %t/SDK \
 // RUN:     -Fsystem %t/SDK/Frameworks \
@@ -66,6 +67,7 @@ print(someFunc())
 // RUN: %target-swift-frontend \
 // RUN:     -typecheck \
 // RUN:     -index-system-modules \
+// RUN:     -index-ignore-stdlib \
 // RUN:     -index-store-path %t/idx \
 // RUN:     -sdk %t/SDK \
 // RUN:     -Fsystem %t/SDK/Frameworks \
@@ -95,6 +97,7 @@ print(someFunc())
 // RUN: %target-swift-frontend \
 // RUN:     -typecheck \
 // RUN:     -index-system-modules \
+// RUN:     -index-ignore-stdlib \
 // RUN:     -index-store-path %t/idx \
 // RUN:     -sdk %t/SDK \
 // RUN:     -Fsystem %t/SDK/Frameworks \
@@ -123,6 +126,7 @@ print(someFunc())
 // RUN: %target-swift-frontend \
 // RUN:     -typecheck \
 // RUN:     -index-system-modules \
+// RUN:     -index-ignore-stdlib \
 // RUN:     -index-store-path %t/idx \
 // RUN:     -sdk %t/SDK \
 // RUN:     -Fsystem %t/SDK/Frameworks \