Merge remote-tracking branch 'origin/master' into master-next

Author: swift-ci

include/swift/ClangImporter/ClangImporter.h

@@ -43,6 +43,12 @@ namespace clang {
   class Type;
   class VisibleDeclConsumer;
   class DeclarationName;
+  class CompilerInvocation;
+namespace tooling {
+namespace dependencies {
+  struct FullDependenciesResult;
+}
+}
 }
 
 namespace swift {
@@ -148,6 +154,14 @@ class ClangImporter final : public ClangModuleLoader {
          std::string swiftPCHHash = "", DependencyTracker *tracker = nullptr,
          DWARFImporterDelegate *dwarfImporterDelegate = nullptr);
 
+  static std::vector<std::string>
+  getClangArguments(ASTContext &ctx, const ClangImporterOptions &importerOpts);
+
+  static std::unique_ptr<clang::CompilerInvocation>
+  createClangInvocation(ClangImporter *importer,
+                        const ClangImporterOptions &importerOpts,
+                        ArrayRef<std::string> invocationArgStrs,
+                        std::vector<std::string> *CC1Args = nullptr);
   ClangImporter(const ClangImporter &) = delete;
   ClangImporter(ClangImporter &&) = delete;
   ClangImporter &operator=(const ClangImporter &) = delete;
@@ -369,6 +383,10 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
+  void recordModuleDependencies(
+      ModuleDependenciesCache &cache,
+      const clang::tooling::dependencies::FullDependenciesResult &clangDependencies);
+
   Optional<ModuleDependencies> getModuleDependencies(
       StringRef moduleName, ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate) override;
@@ -460,6 +478,12 @@ class ClangImporter final : public ClangModuleLoader {
 ImportDecl *createImportDecl(ASTContext &Ctx, DeclContext *DC, ClangNode ClangN,
                              ArrayRef<clang::Module *> Exported);
 
+/// Extract the specified-or-defaulted -module-cache-path that winds up in
+/// the clang importer, for reuse as the .swiftmodule cache path when
+/// building a ModuleInterfaceLoader.
+std::string
+getModuleCachePathFromClang(const clang::CompilerInstance &Instance);
+
 } // end namespace swift
 
 #endif

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -198,12 +198,6 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
     bool RemarkOnRebuildFromInterface, bool DisableInterfaceFileLock);
 };
 
-/// Extract the specified-or-defaulted -module-cache-path that winds up in
-/// the clang importer, for reuse as the .swiftmodule cache path when
-/// building a ModuleInterfaceLoader.
-std::string
-getModuleCachePathFromClang(const clang::CompilerInstance &Instance);
-
 struct InterfaceSubContextDelegateImpl: InterfaceSubContextDelegate {
 private:
   SourceManager &SM;

lib/ClangImporter/ClangImporter.cpp

@@ -905,15 +905,10 @@ ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions,
   return PCHFilename.getValue();
 }
 
-std::unique_ptr<ClangImporter>
-ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts,
-                      std::string swiftPCHHash, DependencyTracker *tracker,
-                      DWARFImporterDelegate *dwarfImporterDelegate) {
-  std::unique_ptr<ClangImporter> importer{
-      new ClangImporter(ctx, importerOpts, tracker, dwarfImporterDelegate)};
-
+std::vector<std::string>
+ClangImporter::getClangArguments(ASTContext &ctx,
+                                 const ClangImporterOptions &importerOpts) {
   std::vector<std::string> invocationArgStrs;
-
   // Clang expects this to be like an actual command line. So we need to pass in
   // "clang" for argv[0]
   invocationArgStrs.push_back("clang");
@@ -927,6 +922,49 @@ ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts,
     break;
   }
   addCommonInvocationArguments(invocationArgStrs, ctx, importerOpts);
+  return invocationArgStrs;
+}
+
+std::unique_ptr<clang::CompilerInvocation>
+ClangImporter::createClangInvocation(ClangImporter *importer,
+                                     const ClangImporterOptions &importerOpts,
+                                     ArrayRef<std::string> invocationArgStrs,
+                                     std::vector<std::string> *CC1Args) {
+  std::vector<const char *> invocationArgs;
+  invocationArgs.reserve(invocationArgStrs.size());
+  for (auto &argStr : invocationArgStrs)
+    invocationArgs.push_back(argStr.c_str());
+  // Set up a temporary diagnostic client to report errors from parsing the
+  // command line, which may be important for Swift clients if, for example,
+  // they're using -Xcc options. Unfortunately this diagnostic engine has to
+  // use the default options because the /actual/ options haven't been parsed
+  // yet.
+  //
+  // The long-term client for Clang diagnostics is set up below, after the
+  // clang::CompilerInstance is created.
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> tempDiagOpts{
+    new clang::DiagnosticOptions
+  };
+
+  ClangDiagnosticConsumer tempDiagClient{importer->Impl, *tempDiagOpts,
+                                         importerOpts.DumpClangDiagnostics};
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> tempClangDiags =
+      clang::CompilerInstance::createDiagnostics(tempDiagOpts.get(),
+                                                 &tempDiagClient,
+                                                 /*owned*/false);
+
+  return clang::createInvocationFromCommandLine(invocationArgs, tempClangDiags,
+                                                nullptr, false, CC1Args);
+}
+
+std::unique_ptr<ClangImporter>
+ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts,
+                      std::string swiftPCHHash, DependencyTracker *tracker,
+                      DWARFImporterDelegate *dwarfImporterDelegate) {
+  std::unique_ptr<ClangImporter> importer{
+      new ClangImporter(ctx, importerOpts, tracker, dwarfImporterDelegate)};
+  importer->Impl.ClangArgs = getClangArguments(ctx, importerOpts);
+  ArrayRef<std::string> invocationArgStrs = importer->Impl.ClangArgs;
 
   if (importerOpts.DumpClangDiagnostics) {
     llvm::errs() << "'";
@@ -936,10 +974,7 @@ ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts,
     llvm::errs() << "'\n";
   }
 
-  std::vector<const char *> invocationArgs;
-  invocationArgs.reserve(invocationArgStrs.size());
-  for (auto &argStr : invocationArgStrs)
-    invocationArgs.push_back(argStr.c_str());
+
 
   if (llvm::sys::path::extension(importerOpts.BridgingHeader)
         .endswith(file_types::getExtension(file_types::TY_PCH))) {
@@ -957,27 +992,9 @@ ClangImporter::create(ASTContext &ctx, const ClangImporterOptions &importerOpts,
 
   // Create a new Clang compiler invocation.
   {
-    // Set up a temporary diagnostic client to report errors from parsing the
-    // command line, which may be important for Swift clients if, for example,
-    // they're using -Xcc options. Unfortunately this diagnostic engine has to
-    // use the default options because the /actual/ options haven't been parsed
-    // yet.
-    //
-    // The long-term client for Clang diagnostics is set up below, after the
-    // clang::CompilerInstance is created.
-    llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> tempDiagOpts{
-      new clang::DiagnosticOptions
-    };
-
-    ClangDiagnosticConsumer tempDiagClient{importer->Impl, *tempDiagOpts,
-                                           importerOpts.DumpClangDiagnostics};
-    llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> tempClangDiags =
-        clang::CompilerInstance::createDiagnostics(tempDiagOpts.get(),
-                                                   &tempDiagClient,
-                                                   /*owned*/false);
-
-    importer->Impl.Invocation =
-        clang::createInvocationFromCommandLine(invocationArgs, tempClangDiags);
+    importer->Impl.Invocation = createClangInvocation(importer.get(),
+                                                      importerOpts,
+                                                      invocationArgStrs);
     if (!importer->Impl.Invocation)
       return nullptr;
   }
@@ -3994,3 +4011,20 @@ bool ClangImporter::isInOverlayModuleForImportedModule(
   return !clangModule->ExportAsModule.empty() &&
     clangModule->ExportAsModule == overlayModule->getName().str();
 }
+
+/// Extract the specified-or-defaulted -module-cache-path that winds up in
+/// the clang importer, for reuse as the .swiftmodule cache path when
+/// building a ModuleInterfaceLoader.
+std::string
+swift::getModuleCachePathFromClang(const clang::CompilerInstance &Clang) {
+  if (!Clang.hasPreprocessor())
+    return "";
+  std::string SpecificModuleCachePath =
+      Clang.getPreprocessor().getHeaderSearchInfo().getModuleCachePath().str();
+
+  // The returned-from-clang module cache path includes a suffix directory
+  // that is specific to the clang version and invocation; we want the
+  // directory above that.
+  return llvm::sys::path::parent_path(SpecificModuleCachePath).str();
+}
+

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -188,12 +188,48 @@ static ClangModuleDependenciesCacheImpl *getOrCreateClangImpl(
   return clangImpl;
 }
 
+static std::string getModuleFilePath(StringRef moduleCacheDir,
+                                     StringRef moduleName,
+                                     StringRef contextHash) {
+  SmallString<128> outputPath(moduleCacheDir);
+  llvm::sys::path::append(outputPath, (llvm::Twine(moduleName)
+    + "-" + contextHash + ".pcm").str());
+  return outputPath.str().str();
+}
+
+static std::string getModuleFilePath(StringRef moduleCacheDir,
+                                     const ModuleDeps &dep) {
+  return getModuleFilePath(moduleCacheDir, dep.ModuleName, dep.ContextHash);
+}
+
 /// Record the module dependencies we found by scanning Clang modules into
 /// the module dependencies cache.
-static void recordModuleDependencies(
+void ClangImporter::recordModuleDependencies(
     ModuleDependenciesCache &cache,
     const FullDependenciesResult &clangDependencies) {
+  struct ModuleInfo {
+    std::string PCMPath;
+    std::string ModuleMapPath;
+  };
+  auto ModuleCacheDir = swift::getModuleCachePathFromClang(getClangInstance());
+
+  // A map keyed by module name and context hash.
+  llvm::StringMap<llvm::StringMap<ModuleInfo>> moduleInfoMap;
+
+  // Traverse all Clang modules to populate moduleInfoMap for cross
+  // referencing later.
+  for (const auto &clangModuleDep : clangDependencies.DiscoveredModules) {
+    moduleInfoMap[clangModuleDep.ModuleName][clangModuleDep.ContextHash] =
+      {
+        // Keep track of pcm path for output.
+        getModuleFilePath(ModuleCacheDir, clangModuleDep),
+        // Keep track of modulemap file for input.
+        clangModuleDep.ClangModuleMapFile
+      };
+  }
   for (const auto &clangModuleDep : clangDependencies.DiscoveredModules) {
+    assert(moduleInfoMap[clangModuleDep.ModuleName]
+      .count(clangModuleDep.ContextHash));
     // If we've already cached this information, we're done.
     if (cache.hasDependencies(clangModuleDep.ModuleName,
                               ModuleDependenciesKind::Clang))
@@ -204,14 +240,58 @@ static void recordModuleDependencies(
     for (const auto &fileDep : clangModuleDep.FileDeps) {
       fileDeps.push_back(fileDep.getKey().str());
     }
+    // Inherit all Clang driver args when creating the clang importer.
+    std::vector<std::string> allArgs = Impl.ClangArgs;
+    ClangImporterOptions Opts;
+    std::vector<std::string> cc1Args;
+
+    // Calling this to convert driver args to CC1 args.
+    createClangInvocation(this, Opts, allArgs, &cc1Args);
+    std::vector<std::string> swiftArgs;
+    // We are using Swift frontend mode.
+    swiftArgs.push_back("-frontend");
+    auto addClangArg = [&](StringRef arg) {
+      swiftArgs.push_back("-Xcc");
+      swiftArgs.push_back("-Xclang");
+      swiftArgs.push_back("-Xcc");
+      swiftArgs.push_back(arg);
+    };
+    // Add all args inheritted from creating the importer.
+    for (auto arg: cc1Args) {
+      addClangArg(arg);
+    }
+    // Add all args reported from the Clang dependencies scanner.
+    for(auto arg: clangModuleDep.NonPathCommandLine) {
+      addClangArg(arg);
+    }
 
+    // Add -fmodule-map-file and -fmodule-file for direct dependencies.
+    for (auto &dep: clangModuleDep.ClangModuleDeps) {
+      assert(moduleInfoMap[dep.ModuleName].count(dep.ContextHash));
+      addClangArg((llvm::Twine("-fmodule-map-file=")
+        + moduleInfoMap[dep.ModuleName][dep.ContextHash].ModuleMapPath).str());
+      addClangArg((llvm::Twine("-fmodule-file=")
+        + moduleInfoMap[dep.ModuleName][dep.ContextHash].PCMPath).str());
+    }
+    // Swift frontend action: -emit-pcm
+    swiftArgs.push_back("-emit-pcm");
+    swiftArgs.push_back("-module-name");
+    swiftArgs.push_back(clangModuleDep.ModuleName);
+
+    // Swift frontend option for output file path (Foo.pcm).
+    swiftArgs.push_back("-o");
+    swiftArgs.push_back(moduleInfoMap[clangModuleDep.ModuleName]
+      [clangModuleDep.ContextHash].PCMPath);
+
+    // Swift frontend option for input file path (Foo.modulemap).
+    swiftArgs.push_back(clangModuleDep.ClangModuleMapFile);
     // Module-level dependencies.
     llvm::StringSet<> alreadyAddedModules;
     auto dependencies = ModuleDependencies::forClangModule(
         clangModuleDep.ImplicitModulePCMPath,
         clangModuleDep.ClangModuleMapFile,
         clangModuleDep.ContextHash,
-        clangModuleDep.NonPathCommandLine,
+        swiftArgs,
         fileDeps);
     for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
       dependencies.addModuleDependency(moduleName.ModuleName, alreadyAddedModules);

lib/ClangImporter/ImporterImpl.h

@@ -390,6 +390,8 @@ class LLVM_LIBRARY_VISIBILITY ClangImporter::Implementation
   /// Clang parser, which is used to load textual headers.
   std::unique_ptr<clang::MangleContext> Mangler;
 
+  /// Clang arguments used to create the Clang invocation.
+  std::vector<std::string> ClangArgs;
 public:
   /// Mapping of already-imported declarations.
   llvm::DenseMap<std::pair<const clang::Decl *, Version>, Decl *> ImportedDecls;

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -41,22 +41,6 @@
 using namespace swift;
 using FileDependency = SerializationOptions::FileDependency;
 
-/// Extract the specified-or-defaulted -module-cache-path that winds up in
-/// the clang importer, for reuse as the .swiftmodule cache path when
-/// building a ModuleInterfaceLoader.
-std::string
-swift::getModuleCachePathFromClang(const clang::CompilerInstance &Clang) {
-  if (!Clang.hasPreprocessor())
-    return "";
-  std::string SpecificModuleCachePath =
-      Clang.getPreprocessor().getHeaderSearchInfo().getModuleCachePath().str();
-
-  // The returned-from-clang module cache path includes a suffix directory
-  // that is specific to the clang version and invocation; we want the
-  // directory above that.
-  return llvm::sys::path::parent_path(SpecificModuleCachePath).str();
-}
-
 #pragma mark - Forwarding Modules
 
 namespace {
@@ -1048,6 +1032,7 @@ void ModuleInterfaceLoader::collectVisibleTopLevelModuleNames(
 void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface(
     const SearchPathOptions &SearchPathOpts,
     const LangOptions &LangOpts) {
+  GenericArgs.push_back("-frontend");
   // Start with a SubInvocation that copies various state from our
   // invoking ASTContext.
   GenericArgs.push_back("-compile-module-from-interface");

test/ScanDependencies/Inputs/BuildModulesFromGraph.swift

@@ -0,0 +1,46 @@
+//===--------------- BuildModulesFromGraph.swift --------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+import Foundation
+
+let fileName = CommandLine.arguments[1]
+let swiftPath = CommandLine.arguments[2]
+let moduleName = CommandLine.arguments[3]
+let data = try! Data(contentsOf: URL(fileURLWithPath: fileName))
+
+let decoder = JSONDecoder()
+let moduleDependencyGraph = try! decoder.decode(
+  ModuleDependencyGraph.self, from: data)
+
+func findModuleBuildingCommand(_ moduleName: String) -> [String]? {
+  for (_, dep) in moduleDependencyGraph.modules {
+    if dep.modulePath.hasSuffix(moduleName) {
+      switch dep.details {
+      case .swift(let details):
+        return details.commandLine
+      case .clang(let details):
+        return details.commandLine
+      }
+    } else {
+      continue
+    }
+  }
+  return nil
+}
+
+if let command = findModuleBuildingCommand(moduleName) {
+  var result = swiftPath
+  command.forEach { result += " \($0)"}
+  print(result)
+  exit(0)
+} else {
+  fatalError("cannot find module building commands for \(moduleName)")
+}