Skip to content

Commit b4dfb6b

Browse files
artemcmartemcm
committed
[Dependency Scanning] Implement parallel imported module resolution
'ModuleDependencyScanner' maintains a Thread Pool along with a pool of workers which are capable of executing a filesystem lookup of a named module dependency. When resolving imports of a given Swift module, each import's resolution operation can be issued asunchronously.
1 parent 6e3f896 commit b4dfb6b

15 files changed

+1393
-1125
lines changed

‎include/swift/AST/ModuleDependencies.h

+16
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,9 @@ class ModuleDependencyInfo {
667667
/// Whether the dependencies are for a Swift module: either Textual, Source, Binary, or Placeholder.
668668
bool isSwiftModule() const;
669669

670+
/// Whether the dependencies are for a textual interface Swift module or a Source Swift module.
671+
bool isTextualSwiftModule() const;
672+
670673
/// Whether the dependencies are for a textual Swift module.
671674
bool isSwiftInterfaceModule() const;
672675

@@ -1006,13 +1009,20 @@ class ModuleDependenciesCache {
10061009
ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
10071010

10081011
public:
1012+
/// Whether we have cached dependency information for the given module.
1013+
bool hasDependency(const ModuleDependencyID &moduleID) const;
10091014
/// Whether we have cached dependency information for the given module.
10101015
bool hasDependency(StringRef moduleName,
10111016
llvm::Optional<ModuleDependencyKind> kind) const;
1017+
/// Whether we have cached dependency information for the given module Name.
1018+
bool hasDependency(StringRef moduleName) const;
10121019

10131020
SwiftDependencyScanningService &getScanService() {
10141021
return globalScanningService;
10151022
}
1023+
const SwiftDependencyScanningService &getScanService() const {
1024+
return globalScanningService;
1025+
}
10161026
const llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() const {
10171027
return alreadySeenClangModules;
10181028
}
@@ -1040,6 +1050,12 @@ class ModuleDependenciesCache {
10401050
findDependency(StringRef moduleName,
10411051
llvm::Optional<ModuleDependencyKind> kind) const;
10421052

1053+
/// Look for module dependencies for a module with the given name
1054+
///
1055+
/// \returns the cached result, or \c None if there is no cached entry.
1056+
llvm::Optional<const ModuleDependencyInfo *>
1057+
findDependency(StringRef moduleName) const;
1058+
10431059
/// Record dependencies for the given module.
10441060
void recordDependency(StringRef moduleName,
10451061
ModuleDependencyInfo dependencies);

‎include/swift/DependencyScan/ModuleDependencyScanner.h

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
//===--- ModuleDependencyScanner.h - Import Swift modules --------*- C++
2+
//-*-===//
3+
//
4+
// This source file is part of the Swift.org open source project
5+
//
6+
// Copyright (c) 2020 Apple Inc. and the Swift project authors
7+
// Licensed under Apache License v2.0 with Runtime Library Exception
8+
//
9+
// See https://swift.org/LICENSE.txt for license information
10+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
//
12+
//===----------------------------------------------------------------------===//
13+
14+
#include "swift/AST/ASTContext.h"
15+
#include "swift/AST/ModuleDependencies.h"
16+
#include "swift/Frontend/ModuleInterfaceLoader.h"
17+
#include "swift/Serialization/SerializedModuleLoader.h"
18+
#include "llvm/Support/ThreadPool.h"
19+
20+
namespace swift {
21+
class DependencyTracker;
22+
}
23+
24+
namespace swift {
25+
26+
/// A dependency scanning worker which performs filesystem lookup
27+
/// of a named module dependency.
28+
class ModuleDependencyScanningWorker {
29+
public:
30+
ModuleDependencyScanningWorker(
31+
SwiftDependencyScanningService &globalScanningService,
32+
const CompilerInvocation &ScanCompilerInvocation,
33+
const SILOptions &SILOptions, ASTContext &ScanASTContext,
34+
DependencyTracker &DependencyTracker, DiagnosticEngine &diags);
35+
36+
private:
37+
/// Retrieve the module dependencies for the module with the given name.
38+
ModuleDependencyVector
39+
scanFilesystemForModuleDependency(StringRef moduleName,
40+
const ModuleDependenciesCache &cache,
41+
bool isTestableImport = false);
42+
43+
/// Retrieve the module dependencies for the Clang module with the given name.
44+
ModuleDependencyVector
45+
scanFilesystemForClangModuleDependency(StringRef moduleName,
46+
const ModuleDependenciesCache &cache);
47+
48+
/// Retrieve the module dependencies for the Swift module with the given name.
49+
ModuleDependencyVector
50+
scanFilesystemForSwiftModuleDependency(StringRef moduleName,
51+
const ModuleDependenciesCache &cache);
52+
53+
// An AST delegate for interface scanning.
54+
std::unique_ptr<InterfaceSubContextDelegateImpl> ScanningASTDelegate;
55+
// The Clang scanner tool used by this worker.
56+
clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
57+
// Swift and Clang module loaders acting as scanners.
58+
std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
59+
std::unique_ptr<ClangImporter> clangScannerModuleLoader;
60+
// Restrict access to the parent scanner class.
61+
friend class ModuleDependencyScanner;
62+
};
63+
64+
class ModuleDependencyScanner {
65+
public:
66+
ModuleDependencyScanner(SwiftDependencyScanningService &ScanningService,
67+
const CompilerInvocation &ScanCompilerInvocation,
68+
const SILOptions &SILOptions,
69+
ASTContext &ScanASTContext,
70+
DependencyTracker &DependencyTracker,
71+
DiagnosticEngine &diags, bool ParallelScan);
72+
73+
/// Identify the scanner invocation's main module's dependencies
74+
llvm::ErrorOr<ModuleDependencyInfo> getMainModuleDependencyInfo(
75+
ModuleDecl *mainModule,
76+
llvm::Optional<SwiftDependencyTracker> tracker = llvm::None);
77+
78+
/// Resolve module dependencies of the given module, computing a full
79+
/// transitive closure dependency graph.
80+
std::vector<ModuleDependencyID>
81+
getModuleDependencies(ModuleDependencyID moduleID,
82+
ModuleDependenciesCache &cache);
83+
84+
/// Query the module dependency info for the Clang module with the given name.
85+
/// Explicit by-name lookups are useful for batch mode scanning.
86+
llvm::Optional<const ModuleDependencyInfo *>
87+
getNamedClangModuleDependencyInfo(StringRef moduleName,
88+
ModuleDependenciesCache &cache);
89+
90+
/// Query the module dependency info for the Swift module with the given name.
91+
/// Explicit by-name lookups are useful for batch mode scanning.
92+
llvm::Optional<const ModuleDependencyInfo *>
93+
getNamedSwiftModuleDependencyInfo(StringRef moduleName,
94+
ModuleDependenciesCache &cache);
95+
96+
private:
97+
/// Resolve the direct dependencies of the given module.
98+
std::vector<ModuleDependencyID>
99+
resolveDirectModuleDependencies(ModuleDependencyID moduleID,
100+
ModuleDependenciesCache &cache);
101+
102+
/// Resolve imported module names of a given module to concrete
103+
/// modules. If `ParallelScan` is enabled, this operation is multithreaded.
104+
void
105+
resolveImportDependencies(const ModuleDependencyID &moduleID,
106+
ModuleDependenciesCache &cache,
107+
ModuleDependencyIDSetVector &directDependencies);
108+
109+
/// If a module has a bridging header, execute a dependency scan
110+
/// on it and record the dependencies.
111+
void resolveBridgingHeaderDependencies(
112+
const ModuleDependencyID &moduleID, ModuleDependenciesCache &cache,
113+
std::vector<std::string> &allClangModules,
114+
llvm::StringSet<> &alreadyKnownModules,
115+
ModuleDependencyIDSetVector &directDependencies);
116+
117+
/// Resolve all module dependencies comprised of Swift overlays
118+
/// of this module's Clang module dependencies.
119+
void resolveSwiftOverlayDependencies(
120+
const ModuleDependencyID &moduleID,
121+
const std::vector<std::string> &clangDependencies,
122+
ModuleDependenciesCache &cache,
123+
ModuleDependencyIDSetVector &swiftOverlayDependencies);
124+
125+
/// Identify all cross-import overlay modules of the specified
126+
/// dependency set and apply an action for each.
127+
void discoverCrossImportOverlayDependencies(
128+
StringRef mainModuleName, ArrayRef<ModuleDependencyID> allDependencies,
129+
ModuleDependenciesCache &cache,
130+
llvm::function_ref<void(ModuleDependencyID)> action);
131+
132+
/// Perform an operation utilizing one of the Scanning workers
133+
/// available to this scanner.
134+
template <typename Function, typename... Args>
135+
auto withDependencyScanningWorker(Function &&F, Args &&...ArgList);
136+
137+
private:
138+
const CompilerInvocation &ScanCompilerInvocation;
139+
ASTContext &ScanASTContext;
140+
DiagnosticEngine &Diagnostics;
141+
142+
/// The available pool of workers for filesystem module search
143+
unsigned NumThreads;
144+
std::list<std::unique_ptr<ModuleDependencyScanningWorker>> Workers;
145+
llvm::ThreadPool ScanningThreadPool;
146+
/// Protect worker access.
147+
std::mutex WorkersLock;
148+
};
149+
150+
} // namespace swift

‎include/swift/DependencyScan/ScanDependencies.h

-4
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,6 @@ llvm::ErrorOr<swiftscan_dependency_graph_t>
6161
performModuleScan(CompilerInstance &instance,
6262
ModuleDependenciesCache &cache);
6363

64-
llvm::ErrorOr<swiftscan_dependency_graph_t>
65-
performParallelModuleScan(CompilerInstance &instance,
66-
ModuleDependenciesCache &cache);
67-
6864
/// Scans the main module of \c instance for all direct module imports
6965
llvm::ErrorOr<swiftscan_import_set_t>
7066
performModulePrescan(CompilerInstance &instance,

‎include/swift/Frontend/FrontendOptions.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,8 @@ class FrontendOptions {
366366
/// Emit remarks indicating use of the serialized module dependency scanning cache.
367367
bool EmitDependencyScannerCacheRemarks = false;
368368

369-
/// Whether the dependency scanner invocation should use multiple threads.
369+
/// Whether the dependency scanner invocation should resolve imports
370+
/// to filesystem modules in parallel.
370371
bool ParallelDependencyScan = false;
371372

372373
/// When performing an incremental build, ensure that cross-module incremental

‎include/swift/Serialization/ModuleDependencyScanner.h ‎include/swift/Serialization/ScanningLoaders.h

+7-103
Original file line numberDiff line numberDiff line change
@@ -1,121 +1,23 @@
1-
//===--- ModuleDependencyScanner.h - Import Swift modules --------*- C++
2-
//-*-===//
3-
//
1+
//===--- ScanningLoaders.h - Swift module scanning --------------*- C++ -*-===//
42
// This source file is part of the Swift.org open source project
53
//
6-
// Copyright (c) 2020 Apple Inc. and the Swift project authors
4+
// Copyright (c) 2023 Apple Inc. and the Swift project authors
75
// Licensed under Apache License v2.0 with Runtime Library Exception
86
//
97
// See https://swift.org/LICENSE.txt for license information
108
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
119
//
1210
//===----------------------------------------------------------------------===//
1311

12+
#ifndef SWIFT_SCANNINGLOADERS_H
13+
#define SWIFT_SCANNINGLOADERS_H
14+
1415
#include "swift/AST/ASTContext.h"
1516
#include "swift/AST/ModuleDependencies.h"
1617
#include "swift/Frontend/ModuleInterfaceLoader.h"
1718
#include "swift/Serialization/SerializedModuleLoader.h"
1819

1920
namespace swift {
20-
class DependencyTracker;
21-
}
22-
23-
namespace swift {
24-
25-
class ModuleDependencyScanningWorker {
26-
private:
27-
ModuleDependencyScanningWorker(SwiftDependencyScanningService &globalScanningService,
28-
const CompilerInvocation &ScanCompilerInvocation,
29-
const SILOptions &SILOptions,
30-
ASTContext &ScanASTContext,
31-
DependencyTracker &DependencyTracker,
32-
DiagnosticEngine &diags);
33-
34-
/// Retrieve the module dependencies for the module with the given name.
35-
llvm::Optional<const ModuleDependencyInfo *> scanForModuleDependency(
36-
StringRef moduleName, ModuleDependenciesCache &cache,
37-
bool optionalDependencyLookup = false, bool isTestableImport = false,
38-
llvm::Optional<ModuleDependencyID> dependencyOf = llvm::None);
39-
40-
/// Retrieve the module dependencies for the Clang module with the given name.
41-
llvm::Optional<const ModuleDependencyInfo *>
42-
scanForClangModuleDependency(StringRef moduleName,
43-
ModuleDependenciesCache &cache);
44-
45-
/// Retrieve the module dependencies for the Swift module with the given name.
46-
llvm::Optional<const ModuleDependencyInfo *>
47-
scanForSwiftModuleDependency(StringRef moduleName,
48-
ModuleDependenciesCache &cache);
49-
50-
51-
private:
52-
DiagnosticEngine &Diagnostics;
53-
54-
// An AST delegate for interface scanning
55-
std::unique_ptr<InterfaceSubContextDelegateImpl> ScanningASTDelegate;
56-
// Clang scanner tool
57-
clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
58-
59-
// Swift and Clang module loaders acting as the corresponding
60-
// scanners.
61-
std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
62-
std::unique_ptr<ClangImporter> clangScannerModuleLoader;
63-
64-
// Only the parent scanner can use this
65-
friend class ModuleDependencyScanner;
66-
};
67-
68-
class ModuleDependencyScanner {
69-
public:
70-
ModuleDependencyScanner(SwiftDependencyScanningService &ScanningService,
71-
const CompilerInvocation &ScanCompilerInvocation,
72-
const SILOptions &SILOptions,
73-
ASTContext &ScanASTContext,
74-
DependencyTracker &DependencyTracker,
75-
DiagnosticEngine &diags);
76-
77-
/// Identify the scanner invocation's main module's dependencies
78-
llvm::ErrorOr<ModuleDependencyInfo> getMainModuleDependencyInfo(
79-
ModuleDecl *mainModule,
80-
llvm::Optional<SwiftDependencyTracker> tracker = llvm::None);
81-
82-
/// Resolve module dependencies of the given module, direct and transitive.
83-
std::vector<ModuleDependencyID>
84-
resolveDependencies(ModuleDependencyID moduleID,
85-
ModuleDependenciesCache &cache);
86-
87-
/// Retrieve the module dependencies for the Clang module with the given name.
88-
llvm::Optional<const ModuleDependencyInfo *>
89-
scanForClangModuleDependency(StringRef moduleName,
90-
ModuleDependenciesCache &cache);
91-
92-
/// Retrieve the module dependencies for the Swift module with the given name.
93-
llvm::Optional<const ModuleDependencyInfo *>
94-
scanForSwiftModuleDependency(StringRef moduleName,
95-
ModuleDependenciesCache &cache);
96-
97-
private:
98-
/// Resolve the direct dependencies of the given module.
99-
std::vector<ModuleDependencyID>
100-
resolveModuleImports(ModuleDependencyID moduleID,
101-
ModuleDependenciesCache &cache);
102-
103-
/// Identify all cross-import overlay modules of the specified
104-
/// dependency set and apply an action for each.
105-
void discoverCrossImportOverlayDependencies(
106-
StringRef mainModuleName, ArrayRef<ModuleDependencyID> allDependencies,
107-
ModuleDependenciesCache &cache,
108-
llvm::function_ref<void(ModuleDependencyID)> action);
109-
110-
private:
111-
const CompilerInvocation &ScanCompilerInvocation;
112-
ASTContext &ScanASTContext;
113-
DiagnosticEngine &Diagnostics;
114-
115-
// For now, the sole worker doing the scanning
116-
ModuleDependencyScanningWorker ScanningWorker;
117-
};
118-
11921
/// A module "loader" that looks for .swiftinterface and .swiftmodule files
12022
/// for the purpose of determining dependencies, but does not attempt to
12123
/// load the module files.
@@ -220,3 +122,5 @@ class PlaceholderSwiftModuleScanner : public SwiftModuleScanner {
220122
}
221123
};
222124
} // namespace swift
125+
126+
#endif // SWIFT_SCANNINGLOADERS_H

0 commit comments

Comments
 (0)