Skip to content

Commit 6e3f896

Browse files
artemcmartemcm
committed
[Dependency Scanning] Refactor primary scan operations into 'ModuleDependencyScanner' class
From being a scattered collection of 'static' methods in ScanDependencies.cpp and member methods of ASTContext. This makes 'ScanDependencies.cpp' much easier to read, and abstracts the actual scanning logic away to a place with common state which will make it easier to reason about in the future.
1 parent ea0e5c3 commit 6e3f896

25 files changed

+1473
-1365
lines changed

‎include/swift/AST/ASTContext.h

-22
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@ namespace swift {
9292
class LazyIterableDeclContextData;
9393
class LazyMemberLoader;
9494
struct MacroDiscriminatorContext;
95-
class ModuleDependencyInfo;
9695
class PatternBindingDecl;
9796
class PatternBindingInitializer;
9897
class PluginLoader;
@@ -1045,27 +1044,6 @@ class ASTContext final {
10451044
/// Retrieve the module interface checker associated with this AST context.
10461045
ModuleInterfaceChecker *getModuleInterfaceChecker() const;
10471046

1048-
/// Retrieve the module dependencies for the module with the given name.
1049-
///
1050-
llvm::Optional<const ModuleDependencyInfo *> getModuleDependencies(
1051-
StringRef moduleName, ModuleDependenciesCache &cache,
1052-
InterfaceSubContextDelegate &delegate,
1053-
bool optionalDependencyLookup = false, bool isTestableImport = false,
1054-
llvm::Optional<std::pair<std::string, swift::ModuleDependencyKind>>
1055-
dependencyOf = llvm::None);
1056-
1057-
/// Retrieve the module dependencies for the Clang module with the given name.
1058-
llvm::Optional<const ModuleDependencyInfo *>
1059-
getClangModuleDependencies(StringRef moduleName,
1060-
ModuleDependenciesCache &cache,
1061-
InterfaceSubContextDelegate &delegate);
1062-
1063-
/// Retrieve the module dependencies for the Swift module with the given name.
1064-
llvm::Optional<const ModuleDependencyInfo *>
1065-
getSwiftModuleDependencies(StringRef moduleName,
1066-
ModuleDependenciesCache &cache,
1067-
InterfaceSubContextDelegate &delegate);
1068-
10691047
/// Compute the extra implicit framework search paths on Apple platforms:
10701048
/// $SDKROOT/System/Library/Frameworks/ and $SDKROOT/Library/Frameworks/.
10711049
std::vector<std::string> getDarwinImplicitFrameworkSearchPaths() const;

‎include/swift/AST/ModuleDependencies.h

+46-18
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
#include "llvm/Support/VirtualFileSystem.h"
4040
#include <string>
4141
#include <unordered_map>
42+
#include <unordered_set>
4243
#include <vector>
4344

4445
namespace swift {
@@ -86,17 +87,38 @@ enum class ModuleDependencyKind : int8_t {
8687
LastKind = SwiftPlaceholder + 1
8788
};
8889

89-
using ModuleDependencyID = std::pair<std::string, ModuleDependencyKind>;
90-
using ModuleDependencyIDSetVector =
91-
llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
92-
std::set<ModuleDependencyID>>;
90+
/// This is used to identify a specific module.
91+
struct ModuleDependencyID {
92+
std::string ModuleName;
93+
ModuleDependencyKind Kind;
94+
bool operator==(const ModuleDependencyID &Other) const {
95+
return std::tie(ModuleName, Kind) ==
96+
std::tie(Other.ModuleName, Other.Kind);
97+
}
98+
bool operator<(const ModuleDependencyID& Other) const {
99+
return std::tie(ModuleName, Kind) <
100+
std::tie(Other.ModuleName, Other.Kind);
101+
}
102+
};
93103

94104
struct ModuleDependencyKindHash {
95105
std::size_t operator()(ModuleDependencyKind k) const {
96106
using UnderlyingType = std::underlying_type<ModuleDependencyKind>::type;
97107
return std::hash<UnderlyingType>{}(static_cast<UnderlyingType>(k));
98108
}
99109
};
110+
struct ModuleDependencyIDHash {
111+
std::size_t operator()(ModuleDependencyID id) const {
112+
return llvm::hash_combine(id.ModuleName, id.Kind);
113+
}
114+
};
115+
116+
using ModuleDependencyIDSet =
117+
std::unordered_set<ModuleDependencyID,
118+
ModuleDependencyIDHash>;
119+
using ModuleDependencyIDSetVector =
120+
llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
121+
std::set<ModuleDependencyID>>;
100122

101123
namespace dependencies {
102124
std::string createEncodedModuleKindAndName(ModuleDependencyID id);
@@ -750,6 +772,7 @@ class ModuleDependencyInfo {
750772
collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName) const;
751773
};
752774

775+
using ModuleDependencyVector = llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>;
753776
using ModuleNameToDependencyMap = llvm::StringMap<ModuleDependencyInfo>;
754777
using ModuleDependenciesKindMap =
755778
std::unordered_map<ModuleDependencyKind,
@@ -861,6 +884,7 @@ class SwiftDependencyScanningService {
861884
}
862885

863886
bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
887+
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> getCachingFS() const { return CacheFS; }
864888

865889
llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
866890
assert(CacheFS && "Expect CachingOnDiskFileSystem");
@@ -895,9 +919,10 @@ class SwiftDependencyScanningService {
895919
/// Enforce clients not being allowed to query this cache directly, it must be
896920
/// wrapped in an instance of `ModuleDependenciesCache`.
897921
friend class ModuleDependenciesCache;
922+
friend class ModuleDependencyScanner;
923+
friend class ModuleDependencyScanningWorker;
898924
friend class ModuleDependenciesCacheDeserializer;
899925
friend class ModuleDependenciesCacheSerializer;
900-
friend class DependencyScanningTool;
901926

902927
/// Configure the current state of the cache to respond to queries
903928
/// for the specified scanning context hash.
@@ -964,8 +989,6 @@ class ModuleDependenciesCache {
964989
std::string scannerContextHash;
965990
/// The location of where the built modules will be output to
966991
std::string moduleOutputPath;
967-
/// The Clang dependency scanner tool
968-
clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
969992

970993
/// Retrieve the dependencies map that corresponds to the given dependency
971994
/// kind.
@@ -987,23 +1010,29 @@ class ModuleDependenciesCache {
9871010
bool hasDependency(StringRef moduleName,
9881011
llvm::Optional<ModuleDependencyKind> kind) const;
9891012

990-
/// Produce a reference to the Clang scanner tool associated with this cache
991-
clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() {
992-
return clangScanningTool;
993-
}
9941013
SwiftDependencyScanningService &getScanService() {
9951014
return globalScanningService;
9961015
}
997-
llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() {
1016+
const llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() const {
9981017
return alreadySeenClangModules;
9991018
}
10001019
void addSeenClangModule(clang::tooling::dependencies::ModuleID newModule) {
10011020
alreadySeenClangModules.insert(newModule);
10021021
}
1003-
std::string getModuleOutputPath() {
1022+
std::string getModuleOutputPath() const {
10041023
return moduleOutputPath;
10051024
}
10061025

1026+
/// Query all dependencies, direct and Swift overlay.
1027+
std::vector<ModuleDependencyID>
1028+
getAllDependencies(const ModuleDependencyID &moduleID) const;
1029+
1030+
/// Look for module dependencies for a module with the given ID
1031+
///
1032+
/// \returns the cached result, or \c None if there is no cached entry.
1033+
llvm::Optional<const ModuleDependencyInfo *>
1034+
findDependency(const ModuleDependencyID moduleID) const;
1035+
10071036
/// Look for module dependencies for a module with the given name
10081037
///
10091038
/// \returns the cached result, or \c None if there is no cached entry.
@@ -1015,6 +1044,9 @@ class ModuleDependenciesCache {
10151044
void recordDependency(StringRef moduleName,
10161045
ModuleDependencyInfo dependencies);
10171046

1047+
/// Record dependencies for the given module collection.
1048+
void recordDependencies(ModuleDependencyVector moduleDependencies);
1049+
10181050
/// Update stored dependencies for the given module.
10191051
void updateDependency(ModuleDependencyID moduleID,
10201052
ModuleDependencyInfo dependencies);
@@ -1039,12 +1071,8 @@ class ModuleDependenciesCache {
10391071
namespace std {
10401072
template <>
10411073
struct hash<swift::ModuleDependencyID> {
1042-
using UnderlyingKindType = std::underlying_type<swift::ModuleDependencyKind>::type;
10431074
std::size_t operator()(const swift::ModuleDependencyID &id) const {
1044-
auto underlyingKindValue = static_cast<UnderlyingKindType>(id.second);
1045-
1046-
return (hash<string>()(id.first) ^
1047-
(hash<UnderlyingKindType>()(underlyingKindValue)));
1075+
return llvm::hash_combine(id.ModuleName, id.Kind);
10481076
}
10491077
};
10501078
} // namespace std

‎include/swift/AST/ModuleLoader.h

+17-2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include "llvm/ADT/SetVector.h"
2929
#include "llvm/ADT/StringSet.h"
3030
#include "llvm/ADT/TinyPtrVector.h"
31+
#include "llvm/ADT/IntrusiveRefCntPtr.h"
3132
#include "llvm/Support/VersionTuple.h"
3233
#include <system_error>
3334

@@ -36,10 +37,19 @@ class FileCollectorBase;
3637
namespace vfs {
3738
class OutputBackend;
3839
}
40+
namespace cas {
41+
class CachingOnDiskFileSystem;
42+
}
3943
}
4044

4145
namespace clang {
4246
class DependencyCollector;
47+
namespace tooling {
48+
namespace dependencies {
49+
struct ModuleID;
50+
class DependencyScanningTool;
51+
}
52+
}
4353
}
4454

4555
namespace swift {
@@ -51,6 +61,7 @@ class ClassDecl;
5161
class FileUnit;
5262
class ModuleDecl;
5363
class ModuleDependencyInfo;
64+
struct ModuleDependencyID;
5465
class ModuleDependenciesCache;
5566
class NominalTypeDecl;
5667
class SourceFile;
@@ -327,8 +338,12 @@ class ModuleLoader {
327338

328339
/// Retrieve the dependencies for the given, named module, or \c None
329340
/// if no such module exists.
330-
virtual llvm::Optional<const ModuleDependencyInfo *>
331-
getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
341+
virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
342+
getModuleDependencies(StringRef moduleName,
343+
StringRef moduleOutputPath,
344+
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
345+
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
346+
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
332347
InterfaceSubContextDelegate &delegate,
333348
bool isTestableImport = false) = 0;
334349
};

‎include/swift/ClangImporter/ClangImporter.h

+17-11
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ class FuncDecl;
7575
class ImportDecl;
7676
class IRGenOptions;
7777
class ModuleDecl;
78+
struct ModuleDependencyID;
7879
class NominalTypeDecl;
7980
class StructDecl;
8081
class SwiftLookupTable;
@@ -433,32 +434,37 @@ class ClangImporter final : public ClangModuleLoader {
433434

434435
void verifyAllModules() override;
435436

436-
void recordModuleDependencies(
437-
ModuleDependenciesCache &cache,
438-
const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies);
437+
llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1> bridgeClangModuleDependencies(
438+
const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
439+
StringRef moduleOutputPath);
440+
441+
llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
442+
getModuleDependencies(StringRef moduleName, StringRef moduleOutputPath,
443+
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
444+
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
445+
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
446+
InterfaceSubContextDelegate &delegate,
447+
bool isTestableImport = false) override;
439448

440449
void recordBridgingHeaderOptions(
441450
ModuleDependencyInfo &MDI,
442451
const clang::tooling::dependencies::TranslationUnitDeps &deps);
443452

444-
llvm::Optional<const ModuleDependencyInfo *>
445-
getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
446-
InterfaceSubContextDelegate &delegate,
447-
bool isTestableImport = false) override;
448-
449453
/// Add dependency information for the bridging header.
450454
///
451-
/// \param moduleName the name of the Swift module whose dependency
455+
/// \param moduleID the name of the Swift module whose dependency
452456
/// information will be augmented with information about the given
453457
/// bridging header.
454458
///
459+
/// \param clangScanningTool The clang dependency scanner.
460+
///
455461
/// \param cache The module dependencies cache to update, with information
456462
/// about new Clang modules discovered along the way.
457463
///
458464
/// \returns \c true if an error occurred, \c false otherwise
459465
bool addBridgingHeaderDependencies(
460-
StringRef moduleName,
461-
ModuleDependencyKind moduleKind,
466+
ModuleDependencyID moduleID,
467+
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
462468
ModuleDependenciesCache &cache);
463469
clang::TargetInfo &getModuleAvailabilityTarget() const override;
464470
clang::ASTContext &getClangASTContext() const override;

‎include/swift/DependencyScan/ScanDependencies.h

+6-13
Original file line numberDiff line numberDiff line change
@@ -55,19 +55,20 @@ bool prescanDependencies(CompilerInstance &instance);
5555
bool batchScanDependencies(CompilerInstance &instance,
5656
llvm::StringRef batchInputFile);
5757

58-
/// Batch prescan the imports of modules specified in \c batchInputFile.
59-
bool batchPrescanDependencies(CompilerInstance &instance,
60-
llvm::StringRef batchInputFile);
61-
6258
// MARK: Dependency scanning execution
6359
/// Scans the dependencies of the main module of \c instance.
6460
llvm::ErrorOr<swiftscan_dependency_graph_t>
6561
performModuleScan(CompilerInstance &instance,
6662
ModuleDependenciesCache &cache);
6763

64+
llvm::ErrorOr<swiftscan_dependency_graph_t>
65+
performParallelModuleScan(CompilerInstance &instance,
66+
ModuleDependenciesCache &cache);
67+
6868
/// Scans the main module of \c instance for all direct module imports
6969
llvm::ErrorOr<swiftscan_import_set_t>
70-
performModulePrescan(CompilerInstance &instance);
70+
performModulePrescan(CompilerInstance &instance,
71+
ModuleDependenciesCache &cache);
7172

7273
/// Batch scan the dependencies for modules specified in \c batchInputFile.
7374
std::vector<llvm::ErrorOr<swiftscan_dependency_graph_t>>
@@ -76,14 +77,6 @@ performBatchModuleScan(CompilerInstance &invocationInstance,
7677
CompilerArgInstanceCacheMap *versionedPCMInstanceCache,
7778
llvm::StringSaver &saver,
7879
const std::vector<BatchScanInput> &BatchInput);
79-
80-
/// Batch prescan the imports of modules specified in \c batchInputFile.
81-
std::vector<llvm::ErrorOr<swiftscan_import_set_t>>
82-
performBatchModulePrescan(CompilerInstance &invocationInstance,
83-
ModuleDependenciesCache &cache,
84-
llvm::StringSaver &saver,
85-
const std::vector<BatchScanInput> &BatchInput);
86-
8780
} // end namespace dependencies
8881
} // end namespace swift
8982

‎include/swift/Frontend/FrontendOptions.h

+4-1
Original file line numberDiff line numberDiff line change
@@ -363,9 +363,12 @@ class FrontendOptions {
363363
/// The path at which to either serialize or deserialize the dependency scanner cache.
364364
std::string SerializedDependencyScannerCachePath;
365365

366-
/// Emit remarks indicating use of the serialized module dependency scanning cache
366+
/// Emit remarks indicating use of the serialized module dependency scanning cache.
367367
bool EmitDependencyScannerCacheRemarks = false;
368368

369+
/// Whether the dependency scanner invocation should use multiple threads.
370+
bool ParallelDependencyScan = false;
371+
369372
/// When performing an incremental build, ensure that cross-module incremental
370373
/// build metadata is available in any swift modules emitted by this frontend
371374
/// job.

‎include/swift/Option/FrontendOptions.td

+3
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,9 @@ def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">,
255255
def dependency_scan_cache_remarks : Flag<["-"], "Rdependency-scan-cache">,
256256
HelpText<"Emit remarks indicating use of the serialized module dependency scanning cache.">;
257257

258+
def parallel_scan : Flag<["-"], "parallel-scan">,
259+
HelpText<"Perform dependency scanning in-parallel.">;
260+
258261
def enable_copy_propagation : Flag<["-"], "enable-copy-propagation">,
259262
HelpText<"Run SIL copy propagation with lexical lifetimes to shorten object "
260263
"lifetimes while preserving variable lifetimes.">;

‎include/swift/Sema/SourceLoader.h

+5-2
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,11 @@ class SourceLoader : public ModuleLoader {
9797
// Parsing populates the Objective-C method tables.
9898
}
9999

100-
llvm::Optional<const ModuleDependencyInfo *>
101-
getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
100+
llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
101+
getModuleDependencies(StringRef moduleName, StringRef moduleOutputPath,
102+
llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
103+
const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
104+
clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
102105
InterfaceSubContextDelegate &delegate,
103106
bool isTestableImport) override;
104107
};

0 commit comments

Comments
 (0)