diff --git a/clang/include/clang/Basic/DiagnosticCASKinds.td b/clang/include/clang/Basic/DiagnosticCASKinds.td index 33a23da0d9b4a8..10bcc92e92da8c 100644 --- a/clang/include/clang/Basic/DiagnosticCASKinds.td +++ b/clang/include/clang/Basic/DiagnosticCASKinds.td @@ -11,6 +11,9 @@ let Component = "CAS" in { def err_builtin_cas_cannot_be_initialized : Error< "CAS cannot be initialized from '%0' on disk (check -fcas-path)">, DefaultFatal; +def err_builtin_actioncache_cannot_be_initialized : Error< + "ActionCache cannot be initialized from '%0' on disk (check -faction-cache-path)">, + DefaultFatal; def err_cas_cannot_parse_root_id : Error< "CAS cannot parse root-id '%0' specified by -fcas-fs">, DefaultFatal; def err_cas_filesystem_cannot_be_initialized : Error< diff --git a/clang/include/clang/CAS/CASOptions.h b/clang/include/clang/CAS/CASOptions.h index e660c13e263cc2..3ec55588703a8e 100644 --- a/clang/include/clang/CAS/CASOptions.h +++ b/clang/include/clang/CAS/CASOptions.h @@ -19,7 +19,9 @@ #include namespace llvm { +class Error; namespace cas { +class ActionCache; class CASDB; } // end namespace cas } // end namespace llvm @@ -52,10 +54,11 @@ class CASConfiguration { /// - "auto" is an alias for an automatically chosen location in the user's /// system cache. std::string CASPath; + std::string CachePath; friend bool operator==(const CASConfiguration &LHS, const CASConfiguration &RHS) { - return LHS.CASPath == RHS.CASPath; + return LHS.CASPath == RHS.CASPath && LHS.CachePath == RHS.CachePath; } friend bool operator!=(const CASConfiguration &LHS, const CASConfiguration &RHS) { @@ -73,7 +76,7 @@ class CASConfiguration { /// defined in CASConfiguration to enable caching a CAS instance. /// /// CASOptions includes \a getOrCreateCAS() and \a -/// getOrCreateCASAndHideConfig() for creating a CAS and caching it. +/// getOrCreateActionCache() for creating CAS and ActionCache. /// /// FIXME: The the caching is done here, instead of as a field in \a /// CompilerInstance, in order to ensure that \a @@ -92,24 +95,38 @@ class CASOptions : public CASConfiguration { getOrCreateCAS(DiagnosticsEngine &Diags, bool CreateEmptyCASOnFailure = false) const; - /// Get a CAS defined by the options above. Future calls will return the same + /// Get a ActionCache defined by the options above. Future calls will return + /// the same ActionCache instance... unless the configuration has changed, in + /// which case a new one will be created. + /// + /// If \p CreateEmptyCacheOnFailure, returns an empty in-memory ActionCache on + /// failure. Else, returns \c nullptr on failure. + std::shared_ptr + getOrCreateActionCache(DiagnosticsEngine &Diags, + bool CreateEmptyCacheOnFailures = false) const; + + /// Freeze CAS Configuration. Future calls will return the same /// CAS instance, even if the configuration changes again later. /// /// The configuration will be wiped out to prevent it being observable or /// affecting the output of something that takes \a CASOptions as an input. /// This also "locks in" the return value of \a getOrCreateCAS(): future /// calls will not check if the configuration has changed. - std::shared_ptr - getOrCreateCASAndHideConfig(DiagnosticsEngine &Diags); + void freezeConfig(DiagnosticsEngine &Diags); /// If the configuration is not for a persistent store, it modifies it to the /// default on-disk CAS, otherwise this is a noop. void ensurePersistentCAS(); private: + /// Initialize Cached CAS and ActionCache. + void initCache(DiagnosticsEngine &Diags) const; + struct CachedCAS { /// A cached CAS instance. std::shared_ptr CAS; + /// An ActionCache instnace. + std::shared_ptr AC; /// Remember how the CAS was created. CASConfiguration Config; diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 27f8745436e2fc..b94f580db70a1b 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -6446,6 +6446,12 @@ def fcas_path : Separate<["-"], "fcas-path">, " '-fcas-path=auto' chooses a path in the user's system cache.">, MarshallingInfoString>; +def faction_cache_path : Separate<["-"], "faction-cache-path">, + Group, MetaVarName<"|auto">, + HelpText<"Path to a persistent on-disk backing store for the builtin Cache." + " '-faction-cache-path=auto' chooses a path in the user's system cache.">, + MarshallingInfoString>; + // FIXME: Add to driver once it's supported by -fdepscan. def fcas_fs : Separate<["-"], "fcas-fs">, Group, MetaVarName<"">, diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index ced83d0bab40d3..476e4ac9912474 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -87,6 +87,9 @@ class CompilerInstance : public ModuleLoader { /// The CAS, if any. std::shared_ptr CAS; + /// The ActionCache, if any. + std::shared_ptr ActionCache; + /// The file manager. IntrusiveRefCntPtr FileMgr; @@ -427,6 +430,7 @@ class CompilerInstance : public ModuleLoader { /// Get the CAS, or create it using the configuration in CompilerInvocation. llvm::cas::CASDB &getOrCreateCAS(); + llvm::cas::ActionCache &getOrCreateActionCache(); /// } /// @name Source Manager diff --git a/clang/include/clang/Lex/PTHManager.h b/clang/include/clang/Lex/PTHManager.h index b12a00f2aa331d..548e73f8e51354 100644 --- a/clang/include/clang/Lex/PTHManager.h +++ b/clang/include/clang/Lex/PTHManager.h @@ -20,6 +20,8 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/CASDB.h" #include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CASID.h" #include "llvm/Support/Allocator.h" @@ -98,6 +100,7 @@ class PTHManager { llvm::SpecificBumpPtrAllocator IdentifierInfoCacheAlloc; llvm::cas::CASDB &CAS; + llvm::cas::ActionCache &Cache; IntrusiveRefCntPtr FS; LangOptions CanonicalLangOpts; Optional SerializedLangOpts; @@ -120,7 +123,7 @@ class PTHManager { ~PTHManager(); PTHManager() = delete; - PTHManager(llvm::cas::CASDB &CAS, + PTHManager(llvm::cas::CASDB &CAS, llvm::cas::ActionCache &Cache, IntrusiveRefCntPtr FS, Preprocessor &PP); void setPreprocessor(Preprocessor *pp) { PP = pp; } diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h index 0878be5453a90a..e224a806398f01 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h @@ -13,6 +13,7 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASID.h" #include "llvm/CAS/CASReference.h" #include "llvm/CAS/ThreadSafeFileSystem.h" @@ -34,7 +35,8 @@ namespace dependencies { class DependencyScanningCASFilesystem : public llvm::cas::ThreadSafeFileSystem { public: DependencyScanningCASFilesystem( - IntrusiveRefCntPtr WorkerFS); + IntrusiveRefCntPtr WorkerFS, + llvm::cas::ActionCache &Cache); ~DependencyScanningCASFilesystem(); @@ -104,6 +106,7 @@ class DependencyScanningCASFilesystem : public llvm::cas::ThreadSafeFileSystem { llvm::cas::CachingOnDiskFileSystem &getCachingFS(); llvm::cas::CASDB &CAS; + llvm::cas::ActionCache &Cache; Optional ClangFullVersionID; Optional DepDirectivesID; Optional EmptyBlobID; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h index 82f8c093d98bde..5c227dba38b938 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -12,6 +12,7 @@ #include "clang/CAS/CASOptions.h" #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h" #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "llvm/CAS/ActionCache.h" namespace clang { namespace tooling { @@ -58,6 +59,7 @@ class DependencyScanningService { public: DependencyScanningService( ScanningMode Mode, ScanningOutputFormat Format, CASOptions CASOpts, + std::shared_ptr Cache, IntrusiveRefCntPtr SharedFS, bool ReuseFileManager = true, bool OptimizeArgs = false, bool EagerLoadModules = false, bool OverrideCASTokenCache = false); @@ -79,6 +81,10 @@ class DependencyScanningService { } const CASOptions &getCASOpts() const { return CASOpts; } + llvm::cas::ActionCache &getCache() const { + assert(Cache && "Cache is not initialized"); + return *Cache; + } bool overrideCASTokenCache() const { return OverrideCASTokenCache; } @@ -90,6 +96,7 @@ class DependencyScanningService { const ScanningMode Mode; const ScanningOutputFormat Format; CASOptions CASOpts; + std::shared_ptr Cache; const bool ReuseFileManager; /// Whether to optimize the modules' command-line arguments. const bool OptimizeArgs; diff --git a/clang/lib/CAS/CASOptions.cpp b/clang/lib/CAS/CASOptions.cpp index b8480689034612..e5d88adfc1a69a 100644 --- a/clang/lib/CAS/CASOptions.cpp +++ b/clang/lib/CAS/CASOptions.cpp @@ -9,14 +9,15 @@ #include "clang/CAS/CASOptions.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticCAS.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" +#include "llvm/Support/Error.h" using namespace clang; using namespace llvm::cas; static std::shared_ptr -createCAS(const CASConfiguration &Config, DiagnosticsEngine &Diags, - bool CreateEmptyCASOnFailure) { +createCAS(const CASConfiguration &Config, DiagnosticsEngine &Diags) { if (Config.CASPath.empty()) return llvm::cas::createInMemoryCAS(); @@ -33,7 +34,7 @@ createCAS(const CASConfiguration &Config, DiagnosticsEngine &Diags, llvm::expectedToOptional(llvm::cas::createOnDiskCAS(Path))) return std::move(*MaybeCAS); Diags.Report(diag::err_builtin_cas_cannot_be_initialized) << Path; - return CreateEmptyCASOnFailure ? llvm::cas::createInMemoryCAS() : nullptr; + return nullptr; } std::shared_ptr @@ -42,22 +43,21 @@ CASOptions::getOrCreateCAS(DiagnosticsEngine &Diags, if (Cache.Config.IsFrozen) return Cache.CAS; - auto &CurrentConfig = static_cast(*this); - if (!Cache.CAS || CurrentConfig != Cache.Config) { - Cache.Config = CurrentConfig; - Cache.CAS = createCAS(Cache.Config, Diags, CreateEmptyCASOnFailure); - } - + initCache(Diags); + if (Cache.CAS) + return Cache.CAS; + if (!CreateEmptyCASOnFailure) + return nullptr; + Cache.CAS = llvm::cas::createInMemoryCAS(); return Cache.CAS; } -std::shared_ptr -CASOptions::getOrCreateCASAndHideConfig(DiagnosticsEngine &Diags) { +void CASOptions::freezeConfig(DiagnosticsEngine &Diags) { if (Cache.Config.IsFrozen) - return Cache.CAS; + return; - std::shared_ptr CAS = getOrCreateCAS(Diags); - assert(CAS == Cache.CAS && "Expected CAS to be cached"); + // Make sure the cache is initialized. + initCache(Diags); // Freeze the CAS and wipe out the visible config to hide it from future // accesses. For example, future diagnostics cannot see this. Something that @@ -67,13 +67,48 @@ CASOptions::getOrCreateCASAndHideConfig(DiagnosticsEngine &Diags) { CurrentConfig = CASConfiguration(); CurrentConfig.IsFrozen = Cache.Config.IsFrozen = true; - if (CAS) { + if (Cache.CAS) { // Set the CASPath to the hash schema, since that leaks through CASContext's // API and is observable. - CurrentConfig.CASPath = CAS->getHashSchemaIdentifier().str(); + CurrentConfig.CASPath = Cache.CAS->getHashSchemaIdentifier().str(); } + if (Cache.AC) + CurrentConfig.CachePath = ""; +} + +static std::shared_ptr +createCache(CASDB &CAS, const CASConfiguration &Config, + DiagnosticsEngine &Diags) { + if (Config.CachePath.empty()) + return llvm::cas::createInMemoryActionCache(CAS); + + // Compute the path. + std::string Path = Config.CASPath; + if (Path == "auto") + Path = getDefaultOnDiskActionCachePath(); - return CAS; + // FIXME: Pass on the actual error from the CAS. + if (auto MaybeCache = llvm::expectedToOptional( + llvm::cas::createOnDiskActionCache(CAS, Path))) + return std::move(*MaybeCache); + Diags.Report(diag::err_builtin_actioncache_cannot_be_initialized) << Path; + return nullptr; +} + +std::shared_ptr +CASOptions::getOrCreateActionCache(DiagnosticsEngine &Diags, + bool CreateEmptyOnFailure) const { + if (Cache.Config.IsFrozen) + return Cache.AC; + + initCache(Diags); + if (Cache.AC) + return Cache.AC; + if (!CreateEmptyOnFailure) + return nullptr; + + Cache.CAS = Cache.CAS ? Cache.CAS : llvm::cas::createInMemoryCAS(); + return llvm::cas::createInMemoryActionCache(*Cache.CAS); } void CASOptions::ensurePersistentCAS() { @@ -83,8 +118,19 @@ void CASOptions::ensurePersistentCAS() { llvm_unreachable("Cannot ensure persistent CAS if it's unknown / frozen"); case InMemoryCAS: CASPath = "auto"; + CachePath = "auto"; break; case OnDiskCAS: break; } } + +void CASOptions::initCache(DiagnosticsEngine &Diags) const { + auto &CurrentConfig = static_cast(*this); + if (CurrentConfig == Cache.Config && Cache.CAS && Cache.AC) + return; + + Cache.Config = CurrentConfig; + Cache.CAS = createCAS(Cache.Config, Diags); + Cache.AC = createCache(*Cache.CAS, Cache.Config, Diags); +} diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 50a9790dca9e27..872cc5dcb8ffce 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -470,8 +470,8 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) { // token-caching can fall back to ingesting into an in-memory CAS itself. llvm::vfs::FileSystem &FS = getFileManager().getVirtualFileSystem(); if (FS.isCASFS()) - PP->setPTHManager( - std::make_unique(getOrCreateCAS(), &FS, *PP)); + PP->setPTHManager(std::make_unique( + getOrCreateCAS(), getOrCreateActionCache(), &FS, *PP)); } if (PPOpts.DetailedRecord) @@ -864,6 +864,16 @@ llvm::cas::CASDB &CompilerInstance::getOrCreateCAS() { return *CAS; } +llvm::cas::ActionCache &CompilerInstance::getOrCreateActionCache() { + if (ActionCache) + return *ActionCache; + + ActionCache = getInvocation().getCASOpts().getOrCreateActionCache( + getDiagnostics(), + /*CreateEmptyActionCacheOnFailure=*/true); + return *ActionCache; +} + std::unique_ptr CompilerInstance::createOutputFile(StringRef OutputPath, bool Binary, bool RemoveFileOnSignal, bool UseTemporary, diff --git a/clang/lib/Lex/PTHLexer.cpp b/clang/lib/Lex/PTHLexer.cpp index b09cf3294e007b..b62fb30c35f465 100644 --- a/clang/lib/Lex/PTHLexer.cpp +++ b/clang/lib/Lex/PTHLexer.cpp @@ -26,6 +26,7 @@ #include "clang/Lex/Token.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/HierarchicalTreeBuilder.h" #include "llvm/Support/DJB.h" @@ -343,10 +344,10 @@ SourceLocation PTHLexer::getSourceLocation() { // PTHManager methods. //===----------------------------------------------------------------------===// -PTHManager::PTHManager(llvm::cas::CASDB &CAS, +PTHManager::PTHManager(llvm::cas::CASDB &CAS, llvm::cas::ActionCache &Cache, IntrusiveRefCntPtr FS, Preprocessor &PP) - : CAS(CAS), FS(std::move(FS)), PP(&PP) { + : CAS(CAS), Cache(Cache), FS(std::move(FS)), PP(&PP) { // TODO: Allow the filesystem NOT to have a CAS instance, or (maybe?) to have // a different CAS instance. This requires ingesting files from FS into CAS. @@ -613,9 +614,11 @@ Expected PTHManager::computePTH(llvm::cas::CASID InputFile) { llvm::cas::TreeEntry::Regular, "lang-opts"); llvm::cas::CASID CacheKey = CAS.getID(llvm::cantFail(Builder.create(CAS))); - if (Optional CachedPTH = - expectedToOptional(CAS.getCachedResult(CacheKey))) - return *CachedPTH; + auto Cached = Cache.get(CacheKey); + if (!Cached) + return Cached.takeError(); + if (Optional CachedPTH = *Cached) + return CAS.getID(*CachedPTH); SmallString<256> PTHString; { @@ -628,7 +631,7 @@ Expected PTHManager::computePTH(llvm::cas::CASID InputFile) { CanonicalLangOpts); } auto PTH = llvm::cantFail(CAS.createProxy(None, PTHString)); - llvm::cantFail(CAS.putCachedResult(CacheKey, PTH)); + llvm::cantFail(Cache.put(CacheKey, PTH.getRef())); return PTH; } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp index c21d505258679c..13a8dccf7ee518 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp @@ -9,6 +9,7 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h" #include "clang/Basic/Version.h" #include "clang/Lex/DependencyDirectivesScanner.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/HierarchicalTreeBuilder.h" @@ -34,8 +35,10 @@ static void reportAsFatalIfError(llvm::Error E) { using llvm::Error; DependencyScanningCASFilesystem::DependencyScanningCASFilesystem( - IntrusiveRefCntPtr WorkerFS) - : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()) {} + IntrusiveRefCntPtr WorkerFS, + llvm::cas::ActionCache &Cache) + : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), Cache(Cache) { +} DependencyScanningCASFilesystem::~DependencyScanningCASFilesystem() = default; @@ -84,12 +87,12 @@ template static void readle(StringRef &Slice, T &Out) { } static Error loadDepDirectives( - cas::CASDB &CAS, cas::CASID ID, + cas::CASDB &CAS, cas::ObjectRef Ref, llvm::SmallVectorImpl &DepTokens, llvm::SmallVectorImpl &DepDirectives) { using namespace dependency_directives_scan; - auto Blob = CAS.getProxy(ID); + auto Blob = CAS.getProxy(Ref); if (!Blob) return Blob.takeError(); @@ -164,9 +167,10 @@ void DependencyScanningCASFilesystem::scanForDirectives( } // Check the result cache. - if (Optional OutputID = - expectedToOptional(CAS.getCachedResult(*InputID))) { - reportAsFatalIfError(loadDepDirectives(CAS, *OutputID, Tokens, Directives)); + if (Optional OutputRef = + reportAsFatalIfError(Cache.get(*InputID))) { + reportAsFatalIfError( + loadDepDirectives(CAS, *OutputRef, Tokens, Directives)); return; } @@ -178,16 +182,15 @@ void DependencyScanningCASFilesystem::scanForDirectives( // Failure. Cache empty directives. Tokens.clear(); Directives.clear(); - reportAsFatalIfError( - CAS.putCachedResult(*InputID, CAS.getID(*EmptyBlobID))); + reportAsFatalIfError(Cache.put(*InputID, *EmptyBlobID)); return; } // Success. Add to the CAS and get back persistent output data. - cas::CASID DirectivesID = - CAS.getID(reportAsFatalIfError(storeDepDirectives(CAS, Directives))); + cas::ObjectRef DirectivesID = + reportAsFatalIfError(storeDepDirectives(CAS, Directives)); // Cache the computation. - reportAsFatalIfError(CAS.putCachedResult(*InputID, DirectivesID)); + reportAsFatalIfError(Cache.put(*InputID, DirectivesID)); } Expected diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp index b1488d3c13cb64..22d92919edee49 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/TargetSelect.h" @@ -17,10 +18,11 @@ using namespace dependencies; DependencyScanningService::DependencyScanningService( ScanningMode Mode, ScanningOutputFormat Format, CASOptions CASOpts, + std::shared_ptr Cache, IntrusiveRefCntPtr SharedFS, bool ReuseFileManager, bool OptimizeArgs, bool EagerLoadModules, bool OverrideCASTokenCache) - : Mode(Mode), Format(Format), CASOpts(std::move(CASOpts)), + : Mode(Mode), Format(Format), CASOpts(std::move(CASOpts)), Cache(Cache), ReuseFileManager(ReuseFileManager), OptimizeArgs(OptimizeArgs), EagerLoadModules(EagerLoadModules), OverrideCASTokenCache(OverrideCASTokenCache), diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 8156279acc693f..d3bc637e479534 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -416,7 +416,8 @@ DependencyScanningWorker::DependencyScanningWorker( if (Service.getMode() == ScanningMode::DependencyDirectivesScan) { if (Service.useCASScanning()) - DepCASFS = new DependencyScanningCASFilesystem(CacheFS); + DepCASFS = + new DependencyScanningCASFilesystem(CacheFS, Service.getCache()); else DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(), RealFS); diff --git a/clang/test/CAS/cas-backend.c b/clang/test/CAS/cas-backend.c index 639e970e5bd362..dcbbe5aef68c6d 100644 --- a/clang/test/CAS/cas-backend.c +++ b/clang/test/CAS/cas-backend.c @@ -2,7 +2,7 @@ // RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/casid // // RUN: %clang -cc1 -triple x86_64-apple-macos11 -fcas-backend \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Rcompile-job-cache -emit-obj -o %t/output.o \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb \ // RUN: -dependency-file %t/deps.d -MT %t/output.o 2>&1 \ @@ -13,7 +13,7 @@ // // RUN: CLANG_CAS_BACKEND_SAVE_CASID_FILE=1 %clang -cc1 \ // RUN: -triple x86_64-apple-macos11 -fcas-backend \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Rcompile-job-cache -emit-obj -o %t/output.o \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb \ // RUN: -dependency-file %t/deps.d -MT %t/output.o 2>&1 \ diff --git a/clang/test/CAS/daemon-cwd.c b/clang/test/CAS/daemon-cwd.c index 5195fd1ede0c6b..705bf1797be6ea 100644 --- a/clang/test/CAS/daemon-cwd.c +++ b/clang/test/CAS/daemon-cwd.c @@ -4,7 +4,7 @@ // RUN: rm -rf %t && mkdir -p %t/include // RUN: cp %S/Inputs/test.h %t/include -// RUN: %clang -cc1depscand -start %{clang-daemon-dir}/daemon-cwd +// RUN: %clang -cc1depscand -start %{clang-daemon-dir}/daemon-cwd -fcas-path %t/cas -faction-cache-path %t/cache // RUN: (cd %t && %clang -target x86_64-apple-macos11 -fdepscan=daemon \ // RUN: -fdepscan-prefix-map=%S=/^source \ // RUN: -fdepscan-prefix-map=%t=/^build \ diff --git a/clang/test/CAS/driver-cache-launcher.c b/clang/test/CAS/driver-cache-launcher.c index 913a6e58a60f3e..876cc4bb5e4c9f 100644 --- a/clang/test/CAS/driver-cache-launcher.c +++ b/clang/test/CAS/driver-cache-launcher.c @@ -13,12 +13,12 @@ // RUN: env LLVM_CACHE_CAS_PATH=%t/cas PATH="%t:$PATH" %clang-cache clang-symlink-outside-bindir -c %s -o %t.o -### 2>&1 | FileCheck %s -check-prefix=CLANG -DPREFIX=%t // CLANG: "-cc1depscan" "-fdepscan=auto" -// CLANG: "-fcas-path" "[[PREFIX]]/cas" "-fcas-token-cache" +// CLANG: "-fcas-path" "[[PREFIX]]/cas/cas" "-faction-cache-path" "[[PREFIX]]/cas/actioncache" "-fcas-token-cache" // CLANG: "-greproducible" // CLANG: "-x" "c" // CLANGPP: "-cc1depscan" "-fdepscan=auto" -// CLANGPP: "-fcas-path" "[[PREFIX]]/cas" "-fcas-token-cache" +// CLANGPP: "-fcas-path" "[[PREFIX]]/cas/cas" "-faction-cache-path" "[[PREFIX]]/cas/actioncache" "-fcas-token-cache" // CLANGPP: "-greproducible" // CLANGPP: "-x" "c++" @@ -28,7 +28,7 @@ // RUN: env LLVM_CACHE_CAS_PATH=%t/cas cache-build-session %clang-cache %clang -c %s -o %t.o -### 2>&1 | FileCheck %s -check-prefix=SESSION -DPREFIX=%t // SESSION: "-cc1depscan" "-fdepscan=daemon" "-fdepscan-share-identifier" -// SESSION: "-fcas-path" "[[PREFIX]]/cas" "-fcas-token-cache" +// SESSION: "-fcas-path" "[[PREFIX]]/cas/cas" "-faction-cache-path" "[[PREFIX]]/cas/actioncache" "-fcas-token-cache" // SESSION: "-greproducible" // RUN: cp -R %S/Inputs/cmake-build %t/cmake-build diff --git a/clang/test/CAS/fcache-compile-job-dependency-file.c b/clang/test/CAS/fcache-compile-job-dependency-file.c index f1b914af15cea8..002fa336eb2774 100644 --- a/clang/test/CAS/fcache-compile-job-dependency-file.c +++ b/clang/test/CAS/fcache-compile-job-dependency-file.c @@ -3,7 +3,7 @@ // RUN: llvm-cas --cas %t/cas --ingest --data %t > %t/casid // // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Rcompile-job-cache %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \ // RUN: -dependency-file %t/deps1.d -MT depends 2>&1 \ // RUN: | FileCheck %s --allow-empty --check-prefix=CACHE-MISS @@ -17,7 +17,7 @@ // RUN: ls %t/output.o && rm %t/output.o // // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Rcompile-job-cache %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \ // RUN: -dependency-file %t/deps2.d -MT depends 2>&1 \ // RUN: | FileCheck %s --check-prefix=CACHE-HIT @@ -29,7 +29,7 @@ // CACHE-MISS-NOT: remark: compile job cache hit // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Rcompile-job-cache %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \ // RUN: -dependency-file %t/deps3.d -MT other1 -MT other2 -MP 2>&1 \ // RUN: | FileCheck %s --check-prefix=CACHE-HIT @@ -42,7 +42,7 @@ // DEPS_OTHER: my_header.h: // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Rcompile-job-cache %t/main.c -emit-obj -o %t/output.o -isystem %t/sys \ // RUN: -sys-header-deps -dependency-file %t/deps4.d -MT depends 2>&1 \ // RUN: | FileCheck %s --check-prefix=CACHE-MISS diff --git a/clang/test/CAS/fcache-compile-job-serialized-diagnostics.c b/clang/test/CAS/fcache-compile-job-serialized-diagnostics.c index 91d62ee1ecb201..8c7a495edc8211 100644 --- a/clang/test/CAS/fcache-compile-job-serialized-diagnostics.c +++ b/clang/test/CAS/fcache-compile-job-serialized-diagnostics.c @@ -1,7 +1,7 @@ // RUN: rm -rf %t && mkdir -p %t // RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/casid -// RUN: %clang -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas \ +// RUN: %clang -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -faction-cache-path %t/cache \ // RUN: -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Wimplicit-function-declaration \ // RUN: -Wno-error=implicit-function-declaration \ @@ -13,7 +13,7 @@ // RUN: ls %t/output.o && rm %t/output.o -// RUN: %clang -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas \ +// RUN: %clang -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -faction-cache-path %t/cache \ // RUN: -fcas-fs @%t/casid -fcache-compile-job \ // RUN: -Wimplicit-function-declaration \ // RUN: -Wno-error=implicit-function-declaration \ diff --git a/clang/test/CAS/fcache-compile-job.c b/clang/test/CAS/fcache-compile-job.c index 1ba7302d71f8fe..5b4507ef3d7e0a 100644 --- a/clang/test/CAS/fcache-compile-job.c +++ b/clang/test/CAS/fcache-compile-job.c @@ -2,27 +2,27 @@ // RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/casid // // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ -// RUN: -Rcompile-job-cache-hit -emit-obj -o %t/output.o 2>&1 \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -Rcompile-job-cache-hit -emit-obj %s -o %t/output.o 2>&1 \ // RUN: | FileCheck %s --allow-empty --check-prefix=CACHE-MISS // RUN: ls %t/output.o && rm %t/output.o // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ -// RUN: -Rcompile-job-cache-hit -emit-obj -o %t/output.o 2>&1 \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -Rcompile-job-cache-hit -emit-obj %s -o %t/output.o 2>&1 \ // RUN: | FileCheck %s --check-prefix=CACHE-HIT // RUN: ls %t/output.o && rm %t/output.o // RUN: cd %t // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ -// RUN: -Rcompile-job-cache-hit -emit-obj -o output.o 2>&1 \ +// RUN: -fcas-path %t/cas -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -Rcompile-job-cache-hit -emit-obj %s -o output.o 2>&1 \ // RUN: | FileCheck %s --allow-empty --check-prefix=CACHE-HIT // RUN: ls %t/output.o // // Check for a cache hit if the CAS moves: // RUN: mv %t/cas %t/cas.moved // RUN: %clang -cc1 -triple x86_64-apple-macos11 \ -// RUN: -fcas-path %t/cas.moved -fcas-fs @%t/casid -fcache-compile-job \ -// RUN: -Rcompile-job-cache-hit -emit-obj -o output.o 2>&1 \ +// RUN: -fcas-path %t/cas.moved -faction-cache-path %t/cache -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -Rcompile-job-cache-hit -emit-obj %s -o output.o 2>&1 \ // RUN: | FileCheck %s --check-prefix=CACHE-HIT // RUN: ls %t/output.o // diff --git a/clang/test/CAS/fdepscan-daemon.c b/clang/test/CAS/fdepscan-daemon.c index 1572bccf59d3cd..128987f212a89e 100644 --- a/clang/test/CAS/fdepscan-daemon.c +++ b/clang/test/CAS/fdepscan-daemon.c @@ -2,8 +2,9 @@ // // REQUIRES: system-darwin, clang-cc1daemon -// RUN: %clang -cc1depscand -start %{clang-daemon-dir}/fdepscan-daemon -// RUN: %clang -target x86_64-apple-macos11 -I %S/Inputs -fdepscan=daemon -fdepscan-daemon=%{clang-daemon-dir}/fdepscan-daemon -fsyntax-only -x c %s +// RUN: %clang -cc1depscand -start %{clang-daemon-dir}/fdepscan-daemon -fcas-path %t/cas -faction-cache-path %t/cache +// RUN: %clang -target x86_64-apple-macos11 -I %S/Inputs \ +// RUN: -fdepscan=daemon -fdepscan-daemon=%{clang-daemon-dir}/fdepscan-daemon -fsyntax-only -x c %s // RUN: %clang -cc1depscand -shutdown %{clang-daemon-dir}/fdepscan-daemon #include "test.h" diff --git a/clang/test/CAS/include-tree-with-sanitizer.c b/clang/test/CAS/include-tree-with-sanitizer.c index 520b42e1707842..ec171fa4dee409 100644 --- a/clang/test/CAS/include-tree-with-sanitizer.c +++ b/clang/test/CAS/include-tree-with-sanitizer.c @@ -10,7 +10,7 @@ // RUN: -e "s/^.*miss for '//" \ // RUN: -e "s/' .*$//" > %t/cache-key-tree -// RUN: clang-cas-test -print-compile-job-cache-key -cas %t/cas @%t/cache-key-tree > %t/key.txt +// RUN: clang-cas-test -print-compile-job-cache-key -cas %t/cas/cas @%t/cache-key-tree > %t/key.txt // RUN: FileCheck %s -DSRC_FILE=%s -DOUT_DIR=%t -input-file %t/key.txt // // CHECK: -fsanitize=address \ diff --git a/clang/test/CAS/print-compile-job-cache-key.c b/clang/test/CAS/print-compile-job-cache-key.c index 266ccbd1274365..05caf7dbb96b60 100644 --- a/clang/test/CAS/print-compile-job-cache-key.c +++ b/clang/test/CAS/print-compile-job-cache-key.c @@ -43,7 +43,7 @@ // RUN: -e "s/^.*miss for '//" \ // RUN: -e "s/' .*$//" > %t/cache-key-tree -// RUN: clang-cas-test -print-compile-job-cache-key -cas %t/cas @%t/cache-key-tree | FileCheck %s -check-prefix=INCLUDE_TREE -DSRC_FILE=%s +// RUN: clang-cas-test -print-compile-job-cache-key -cas %t/cas/cas @%t/cache-key-tree | FileCheck %s -check-prefix=INCLUDE_TREE -DSRC_FILE=%s // // INCLUDE_TREE: command-line: llvmcas:// // INCLUDE_TREE: computation: llvmcas:// diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index fe2225b8402874..837c3a8e92cdb5 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -19,6 +19,7 @@ #include "clang/Tooling/Tooling.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Twine.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" @@ -776,6 +777,7 @@ int main(int argc, const char **argv) { CASOptions CASOpts; std::shared_ptr CAS; + std::shared_ptr Cache; IntrusiveRefCntPtr FS; if (outputFormatRequiresCAS()) { if (!InMemoryCAS) { @@ -785,12 +787,14 @@ int main(int argc, const char **argv) { CASOpts.ensurePersistentCAS(); } CAS = CASOpts.getOrCreateCAS(Diags); + // FIXME: Cache is not used here so just create a dummy in-memory cache. + Cache = CASOpts.getOrCreateActionCache(Diags); if (!CAS) return 1; if (Format != ScanningOutputFormat::IncludeTree) FS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); } - DependencyScanningService Service(ScanMode, Format, CASOpts, FS, + DependencyScanningService Service(ScanMode, Format, CASOpts, Cache, FS, ReuseFileManager, OptimizeArgs, EagerLoadModules, OverrideCASTokenCache); llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads)); diff --git a/clang/tools/driver/CacheLauncherMode.cpp b/clang/tools/driver/CacheLauncherMode.cpp index cab7a55ed60b95..8e9641732bb823 100644 --- a/clang/tools/driver/CacheLauncherMode.cpp +++ b/clang/tools/driver/CacheLauncherMode.cpp @@ -143,7 +143,14 @@ clang::handleClangCacheInvocation(SmallVectorImpl &Args, } } if (const char *CASPath = ::getenv("LLVM_CACHE_CAS_PATH")) { - Args.append({"-Xclang", "-fcas-path", "-Xclang", CASPath}); + llvm::SmallString<256> CASArg(CASPath); + llvm::sys::path::append(CASArg, "cas"); + Args.append({"-Xclang", "-fcas-path", "-Xclang", + Saver.save(CASArg.str()).data()}); + llvm::SmallString<256> CacheArg(CASPath); + llvm::sys::path::append(CacheArg, "actioncache"); + Args.append({"-Xclang", "-faction-cache-path", "-Xclang", + Saver.save(CacheArg.str()).data()}); } Args.append({"-greproducible", "-Xclang", "-fcas-token-cache"}); return None; diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp index d68e2b3b65b2ed..9a86ed69cd4cbc 100644 --- a/clang/tools/driver/cc1_main.cpp +++ b/clang/tools/driver/cc1_main.cpp @@ -33,6 +33,7 @@ #include "clang/FrontendTool/Utils.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CASOutputBackend.h" @@ -288,6 +289,7 @@ class CompileJobCache { bool ComputedJobNeedsReplay = false; std::shared_ptr CAS; + std::shared_ptr Cache; SmallString<256> ResultDiags; bool WriteOutputAsCASID = false; bool UseCASBackend = false; @@ -360,9 +362,13 @@ Optional CompileJobCache::initialize(CompilerInstance &Clang) { // // TODO: Extract CASOptions.Path first if we need it later since it'll // disappear here. - CAS = Invocation.getCASOpts().getOrCreateCASAndHideConfig(Diags); + Invocation.getCASOpts().freezeConfig(Diags); + CAS = Invocation.getCASOpts().getOrCreateCAS(Diags); if (!CAS) return 1; // Exit with error! + Cache = Invocation.getCASOpts().getOrCreateActionCache(Diags); + if (!Cache) + return 1; // Exit with error! // Canonicalize Invocation and save things in a side channel. // @@ -439,23 +445,20 @@ Optional CompileJobCache::tryReplayCachedResult(CompilerInstance &Clang) { if (!ResultCacheKey) return 1; - Expected Result = CAS->getCachedResult(*ResultCacheKey); + Optional Result; + if (auto E = Cache->get(*ResultCacheKey).moveInto(Result)) + consumeError(std::move(E)); // ignore error and treat it as a cache miss. + if (Result) { - if (Optional ResultRef = CAS->getReference(*Result)) { - Diags.Report(diag::remark_compile_job_cache_hit) - << ResultCacheKey->toString() << Result->toString(); - Optional Status = - replayCachedResult(Clang, *ResultRef, /*JustComputedResult=*/false); - assert(Status && "Expected a status for a cache hit"); - return *Status; - } - Diags.Report(diag::remark_compile_job_cache_miss_result_not_found) - << ResultCacheKey->toString() << Result->toString(); - } else { - llvm::consumeError(Result.takeError()); - Diags.Report(diag::remark_compile_job_cache_miss) - << ResultCacheKey->toString(); + Diags.Report(diag::remark_compile_job_cache_hit) + << ResultCacheKey->toString() << CAS->getID(*Result).toString(); + Optional Status = + replayCachedResult(Clang, *Result, /*JustComputedResult=*/false); + assert(Status && "Expected a status for a cache hit"); + return *Status; } + Diags.Report(diag::remark_compile_job_cache_miss) + << ResultCacheKey->toString(); // Create an on-disk backend for streaming the results live if we run the // computation. If we're writing the output as a CASID, skip it here, since @@ -635,8 +638,7 @@ void CompileJobCache::finishComputedResult(CompilerInstance &Clang, Expected Result = Builder.create(*CAS); if (!Result) llvm::report_fatal_error(Result.takeError()); - if (llvm::Error E = - CAS->putCachedResult(*ResultCacheKey, CAS->getID(*Result))) + if (llvm::Error E = Cache->put(*ResultCacheKey, CAS->getReference(*Result))) llvm::report_fatal_error(std::move(E)); // Replay / decanonicalize as necessary. diff --git a/clang/tools/driver/cc1depscan_main.cpp b/clang/tools/driver/cc1depscan_main.cpp index 5135d104f22d89..c910db253df6ad 100644 --- a/clang/tools/driver/cc1depscan_main.cpp +++ b/clang/tools/driver/cc1depscan_main.cpp @@ -28,6 +28,7 @@ #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" #include "llvm/Bitstream/BitstreamReader.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" @@ -366,7 +367,8 @@ static Expected scanAndUpdateCC1Inline( bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, const cc1depscand::DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, - const CASOptions &CASOpts, std::shared_ptr DB); + const CASOptions &CASOpts, std::shared_ptr DB, + std::shared_ptr Cache); static Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, @@ -507,6 +509,7 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, const llvm::opt::ArgList &Args, const CASOptions &CASOpts, std::shared_ptr DB, + std::shared_ptr Cache, llvm::Optional &RootID) { using namespace clang::driver; @@ -578,7 +581,7 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, PrefixMapping, *DaemonPath, Sharing, SaveArg, *DB); return scanAndUpdateCC1Inline(Exec, OldArgs, WorkingDirectory, NewArgs, ProduceIncludeTree, DiagnosticErrorOccurred, - PrefixMapping, SaveArg, CASOpts, DB); + PrefixMapping, SaveArg, CASOpts, DB, Cache); }; if (llvm::Error E = ScanAndUpdate().moveInto(RootID)) { Diag.Report(diag::err_cas_depscan_failed) << toString(std::move(E)); @@ -631,8 +634,13 @@ int cc1depscan_main(ArrayRef Argv, const char *Argv0, if (!CAS) return 1; + std::shared_ptr Cache = + CASOpts.getOrCreateActionCache(Diags); + if (!Cache) + return 1; + if (int Ret = scanAndUpdateCC1(Argv0, CC1Args->getValues(), NewArgs, Diags, - Args, CASOpts, CAS, RootID)) + Args, CASOpts, CAS, Cache, RootID)) return Ret; // FIXME: Use OutputBackend to OnDisk only now. @@ -869,6 +877,11 @@ int cc1depscand_main(ArrayRef Argv, const char *Argv0, if (!CAS) llvm::report_fatal_error("clang -cc1depscand: cannot create CAS"); + std::shared_ptr Cache = + CASOpts.getOrCreateActionCache(Diags); + if (!Cache) + llvm::report_fatal_error("clang -cc1depscand: cannot create ActionCache"); + IntrusiveRefCntPtr FS; if (!ProduceIncludeTree) FS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); @@ -877,7 +890,7 @@ int cc1depscand_main(ArrayRef Argv, const char *Argv0, ProduceIncludeTree ? tooling::dependencies::ScanningOutputFormat::IncludeTree : tooling::dependencies::ScanningOutputFormat::Tree, - CASOpts, FS, + CASOpts, Cache, FS, /*ReuseFileManager=*/false, /*SkipExcludedPPRanges=*/true); @@ -1119,7 +1132,8 @@ static Expected scanAndUpdateCC1Inline( bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, const cc1depscand::DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, - const CASOptions &CASOpts, std::shared_ptr DB) { + const CASOptions &CASOpts, std::shared_ptr DB, + std::shared_ptr Cache) { IntrusiveRefCntPtr FS; if (!ProduceIncludeTree) FS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*DB)); @@ -1129,7 +1143,7 @@ static Expected scanAndUpdateCC1Inline( ProduceIncludeTree ? tooling::dependencies::ScanningOutputFormat::IncludeTree : tooling::dependencies::ScanningOutputFormat::Tree, - CASOpts, FS, + CASOpts, Cache, FS, /*ReuseFileManager=*/false, /*SkipExcludedPPRanges=*/true); llvm::IntrusiveRefCntPtr UnderlyingFS = diff --git a/clang/unittests/CAS/CASOptionsTest.cpp b/clang/unittests/CAS/CASOptionsTest.cpp index aa8e60c4fc38c7..1a76152f28d45e 100644 --- a/clang/unittests/CAS/CASOptionsTest.cpp +++ b/clang/unittests/CAS/CASOptionsTest.cpp @@ -104,7 +104,7 @@ TEST(CASOptionsTest, getOrCreateCASInvalid) { ASSERT_EQ(Contents, MemBuffer->getBuffer()); } -TEST(CASOptionsTest, getOrCreateCASAndHideConfig) { +TEST(CASOptionsTest, freezeConfig) { DiagnosticsEngine Diags(new DiagnosticIDs(), new DiagnosticOptions, new IgnoringDiagConsumer()); @@ -112,7 +112,8 @@ TEST(CASOptionsTest, getOrCreateCASAndHideConfig) { unittest::TempDir Dir("cas-options", /*Unique=*/true); CASOptions Opts; Opts.CASPath = Dir.path("cas").str().str(); - std::shared_ptr CAS = Opts.getOrCreateCASAndHideConfig(Diags); + Opts.freezeConfig(Diags); + std::shared_ptr CAS = Opts.getOrCreateCAS(Diags); ASSERT_TRUE(CAS); EXPECT_EQ(CASOptions::UnknownCAS, Opts.getKind()); @@ -124,14 +125,10 @@ TEST(CASOptionsTest, getOrCreateCASAndHideConfig) { Opts.CASPath = ""; EXPECT_EQ(CAS, Opts.getOrCreateCAS(Diags)); EXPECT_EQ(CASOptions::UnknownCAS, Opts.getKind()); - EXPECT_EQ(CAS, Opts.getOrCreateCASAndHideConfig(Diags)); - EXPECT_EQ(CASOptions::UnknownCAS, Opts.getKind()); Opts.CASPath = Dir.path("ignored-cas").str().str(); EXPECT_EQ(CAS, Opts.getOrCreateCAS(Diags)); EXPECT_EQ(CASOptions::UnknownCAS, Opts.getKind()); - EXPECT_EQ(CAS, Opts.getOrCreateCASAndHideConfig(Diags)); - EXPECT_EQ(CASOptions::UnknownCAS, Opts.getKind()); } #endif diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index a8c6e9424fba5a..44e67aa5b0af68 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -39,7 +39,7 @@ TEST(IncludeTree, IncludeTreeScan) { DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::IncludeTree, - CASOptions(), nullptr); + CASOptions(), nullptr, nullptr); DependencyScanningTool ScanTool(Service, std::move(VFS)); std::vector CommandLine = {"clang", diff --git a/clang/unittests/Tooling/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScannerTest.cpp index a946854797c836..2cf80cc3da7e1f 100644 --- a/clang/unittests/Tooling/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScannerTest.cpp @@ -233,7 +233,7 @@ TEST(DependencyScanner, ScanDepsWithFS) { DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, CASOptions(), - nullptr); + nullptr, nullptr); DependencyScanningTool ScanTool(Service, VFS); std::string DepFile; @@ -256,7 +256,7 @@ TEST(DependencyScanner, DepScanFSWithCASProvider) { DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, CASOptions(), - nullptr); + nullptr, nullptr); { DependencyScanningWorkerFilesystem DepFS(Service.getSharedCache(), std::move(CASFS)); diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h index 56c16a3a1d3240..a0053bb9dbe932 100644 --- a/lld/MachO/Config.h +++ b/lld/MachO/Config.h @@ -18,6 +18,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/Support/CachePruning.h" #include "llvm/Support/GlobPattern.h" #include "llvm/Support/VersionTuple.h" @@ -200,6 +201,7 @@ struct Configuration { llvm::IntrusiveRefCntPtr fs; std::unique_ptr CAS; + std::unique_ptr actionCache; std::unique_ptr CASSchemas; bool depScanning = false; diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 00207ab63ccd47..29e9fa05f0c42d 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -37,6 +37,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/BinaryFormat/Magic.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" @@ -1507,11 +1508,16 @@ static bool linkWithResultCaching(InputArgList &args, bool canExitEarly, } CASID cacheKey = *optCacheKey; - Expected result = config->CAS->getCachedResult(cacheKey); - if (result) { - log("Caching: cache hit, result: " + result->toString() + + auto result = config->actionCache->get(cacheKey); + if (!result) { + error("ActionCache lookup failed: " + toString(result.takeError())); + return false; + } + if (*result) { + CASID resultID = config->CAS->getID(**result); + log("Caching: cache hit, result: " + resultID.toString() + ", key: " + cacheKey.toString()); - if (Error E = replayResult(CAS, std::move(*result))) { + if (Error E = replayResult(CAS, resultID)) { error("error replaying cached result: " + toString(std::move(E))); return false; } @@ -1522,7 +1528,6 @@ static bool linkWithResultCaching(InputArgList &args, bool canExitEarly, TimeTraceScope timeScope("Caching: link after cache miss"); log("Caching: cache miss, key: " + cacheKey.toString()); - consumeError(result.takeError()); SmallString<128> workingDirectory; if (std::error_code ec = sys::fs::current_path(workingDirectory)) { @@ -1579,12 +1584,13 @@ static bool linkWithResultCaching(InputArgList &args, bool canExitEarly, return false; } - cas::CASID resultID = CAS.getID(*resultTree); - if (Error E = CAS.putCachedResult(cacheKey, resultID)) { + if (Error E = + config->actionCache->put(cacheKey, CAS.getReference(*resultTree))) { error("error storing cached result: " + toString(std::move(E))); return false; } + cas::CASID resultID = CAS.getID(*resultTree); log("Caching: cached result: " + resultID.toString()); } @@ -1766,6 +1772,15 @@ bool macho::link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, error("error loading CAS at path '" + path + "': " + toString(CAS.takeError())); } + std::string defaultCachePath = llvm::cas::getDefaultOnDiskActionCachePath(); + StringRef cachePath = + args.getLastArgValue(OPT_cache_path, defaultCachePath); + auto cache = llvm::cas::createOnDiskActionCache(*config->CAS, cachePath); + if (cache) + config->actionCache = std::move(*cache); + else + error("error loading ActionCache at path '" + cachePath + + "': " + toString(cache.takeError())); } Optional CASFileSystemRootID; if (args.hasArg(OPT_fcas_fs)) diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td index b03f0753597ff7..8a12d86bad1410 100644 --- a/lld/MachO/Options.td +++ b/lld/MachO/Options.td @@ -100,6 +100,10 @@ def cas_path : Separate<["--"], "fcas-builtin-path">, HelpText<"Path to a persistent on-disk backing store for the builtin CAS.">, MetaVarName<"">, Group; +def cache_path : Separate<["--"], "faction-cache-path">, + HelpText<"Path to a persistent on-disk backing store for the action cache.">, + MetaVarName<"">, + Group; def fcas_fs : Separate<["--"], "fcas-fs">, HelpText<"Configure the filesystem to read from the provided CAS tree.">, MetaVarName<"">, diff --git a/lld/test/MachO/CAS/cache-results.s b/lld/test/MachO/CAS/cache-results.s index dc3666b643a44e..b37088e7db53f2 100644 --- a/lld/test/MachO/CAS/cache-results.s +++ b/lld/test/MachO/CAS/cache-results.s @@ -15,60 +15,60 @@ // Check result caching. -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-MISS // RUN: diff %t/exe %t/exe-normal // RUN: rm %t/exe -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-HIT // RUN: diff %t/exe %t/exe-normal // RUN: touch %t/main.o %t/alib/libt1.a %t/dlib/libt2.dylib -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-HIT // Check that changing output executable filename doesn't affect cache key for executables. -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe2 -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe2 -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-HIT // RUN: diff %t/exe2 %t/exe-normal // But changing output dylib filename affects cache key. -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose -dylib %t/main.o -o %t/exe1.dylib -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose -dylib %t/main.o -o %t/exe1.dylib -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-MISS -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose -dylib %t/main.o -o %t/exe2.dylib -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose -dylib %t/main.o -o %t/exe2.dylib -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-MISS // RUN: not diff %t/exe1.dylib %t/exe2.dylib // Check that re-exports in a dylib are handled for dependency scanning (libc++ re-exports libc++abi) -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose -lc++ %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose -lc++ %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-MISS // Change order of search paths. -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt2 -lt1 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt2 -lt1 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-MISS -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt2 -lt1 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt2 -lt1 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-HIT // Change .o contents // RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %t/main2.s -o %t/main.o -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-MISS -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-HIT // Change .a and .dylib contents // RUN: llvm-ar rcs %t/alib/libt1.a %t/t2.o // RUN: %lld -dylib -o %t/dlib/libt2.dylib %t/t1.o -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-MISS -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose %t/main.o -o %t/exe -lt1 -lt2 -L%t/alib -L%t/dlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-HIT // Check using relative paths. // RUN: mv %t/exe %t/exe.bak; cd %t -// RUN: %lld --fcas-builtin-path %t/cas --fcas-cache-results --verbose main.o -o exe -lt1 -lt2 -Lalib -Ldlib 2>&1 \ +// RUN: %lld --fcas-builtin-path %t/cas --faction-cache-path %t/cache --fcas-cache-results --verbose main.o -o exe -lt1 -lt2 -Lalib -Ldlib 2>&1 \ // RUN: | FileCheck %s -check-prefix=CACHE-HIT // RUN: diff %t/exe %t/exe.bak diff --git a/llvm/include/llvm/CAS/ActionCache.h b/llvm/include/llvm/CAS/ActionCache.h new file mode 100644 index 00000000000000..a7f606ad199fbc --- /dev/null +++ b/llvm/include/llvm/CAS/ActionCache.h @@ -0,0 +1,98 @@ +//===- llvm/CAS/ActionCache.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CAS_CASACTIONCACHE_H +#define LLVM_CAS_CASACTIONCACHE_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CAS/CASReference.h" +#include "llvm/Support/Error.h" + +namespace llvm::cas { + +class CASDB; +class CASID; +class ObjectProxy; + +/// A key for caching an operation. +/// It is implemented as a bag of bytes and provides a convenient constructor +/// for CAS types. +class CacheKey { +public: + StringRef getKey() const { return Key; } + + // TODO: Support CacheKey other than a CASID but rather any array of bytes. + // To do that, ActionCache need to be able to rehash the key into the index, + // which then `getOrCompute` method can be used to avoid multiple calls to + // has function. + CacheKey(const CASID &ID); + CacheKey(const ObjectProxy &Proxy); + CacheKey(const CASDB &CAS, const ObjectRef &Ref); + +private: + std::string Key; +}; + +/// A cache from a key describing an action to the result of doing it. +/// +/// Actions are expected to be pure (collision is an error). +class ActionCache { + virtual void anchor(); + +public: + /// Get a previously computed result for \p ActionKey. + Expected> get(const CacheKey &ActionKey) const { + return getImpl(arrayRefFromStringRef(ActionKey.getKey())); + } + + /// Cache \p Result for the \p ActionKey computation. + Error put(const CacheKey &ActionKey, const ObjectRef &Result) { + return putImpl(arrayRefFromStringRef(ActionKey.getKey()), Result); + } + + /// Cache \p Result using \p ObjectRef as a key. + Error put(const ObjectRef &RefKey, const ObjectRef &Result) { + CacheKey ActionKey(CAS, RefKey); + return putImpl(arrayRefFromStringRef(ActionKey.getKey()), Result); + } + + /// Get or compute a result for \p ActionKey. Equivalent to calling \a get() + /// (followed by \a compute() and \a put() on failure). + Expected getOrCompute(const CacheKey &ActionKey, + function_ref()> Compute); + + virtual ~ActionCache() = default; + +protected: + virtual Expected> + getImpl(ArrayRef ResolvedKey) const = 0; + virtual Error putImpl(ArrayRef ResolvedKey, + const ObjectRef &Result) = 0; + + ActionCache(CASDB &CAS) : CAS(CAS) {} + + CASDB &getCAS() const { return CAS; } + +private: + CASDB &CAS; +}; + +/// Create an action cache in memory. +std::unique_ptr createInMemoryActionCache(CASDB &CAS); + +/// Get a reasonable default on-disk path for a persistent ActionCache for the +/// current user. +std::string getDefaultOnDiskActionCachePath(); + +/// Create an action cache on disk. +Expected> createOnDiskActionCache(CASDB &CAS, + StringRef Path); +} // end namespace llvm::cas + +#endif // LLVM_CAS_CASACTIONCACHE_H diff --git a/llvm/include/llvm/CAS/CASDB.h b/llvm/include/llvm/CAS/CASDB.h index 2c3315e7e41a32..514c06f29d4cb0 100644 --- a/llvm/include/llvm/CAS/CASDB.h +++ b/llvm/include/llvm/CAS/CASDB.h @@ -145,6 +145,9 @@ class CASDB : public CASIDContext { /// Returns \c None if not stored in this CAS. virtual Optional getReference(const CASID &ID) const = 0; + /// Get a reference to the object has the hash value \p Hash. + virtual Optional getReference(ArrayRef Hash) const = 0; + /// Get a Ref from Handle. virtual ObjectRef getReference(ObjectHandle Handle) const = 0; @@ -167,13 +170,6 @@ class CASDB : public CASIDContext { virtual ArrayRef getData(ObjectHandle Node, bool RequiresNullTerminator = false) const = 0; -public: - /// ActionCache APIs. - // FIXME: Split out in the future. This should also be generic key-value - // storage interface, rather than CASID -> CASID. - virtual Expected getCachedResult(CASID InputID) = 0; - virtual Error putCachedResult(CASID InputID, CASID OutputID) = 0; - protected: virtual Expected storeFromOpenFileImpl(sys::fs::file_t FD, diff --git a/llvm/include/llvm/TableGen/ScanDependencies.h b/llvm/include/llvm/TableGen/ScanDependencies.h index cb72b55f885152..8267cc9e2a9b97 100644 --- a/llvm/include/llvm/TableGen/ScanDependencies.h +++ b/llvm/include/llvm/TableGen/ScanDependencies.h @@ -19,6 +19,7 @@ namespace llvm { namespace cas { +class ActionCache; class CachingOnDiskFileSystem; } // end namespace cas @@ -42,8 +43,8 @@ void scanTextForIncludes(StringRef Input, SmallVectorImpl &Includes); /// /// \p ExecID is the Blob for the running executable. It can be used as part of /// the key for caching to avoid (fixed) bugs poisoning results. -Error scanTextForIncludes(cas::CASDB &CAS, cas::ObjectRef ExecID, - const cas::ObjectProxy &Blob, +Error scanTextForIncludes(cas::CASDB &CAS, cas::ActionCache &Cache, + cas::ObjectRef ExecID, const cas::ObjectProxy &Blob, SmallVectorImpl &Includes); /// Match logic from SourceMgr::AddIncludeFile. @@ -57,7 +58,8 @@ Optional lookupIncludeID(cas::CachingOnDiskFileSystem &FS, /// /// \p ExecID is the Blob for the running executable. It can be used as part of /// the key for caching to avoid (fixed) bugs poisoning results. -Error accessAllIncludes(cas::CachingOnDiskFileSystem &FS, cas::ObjectRef ExecID, +Error accessAllIncludes(cas::CachingOnDiskFileSystem &FS, + cas::ActionCache &Cache, cas::ObjectRef ExecID, ArrayRef IncludeDirs, const cas::ObjectProxy &MainFileBlob); @@ -79,8 +81,8 @@ struct ScanIncludesResult { /// it'd be useful for build systems that want to collect dependencies ahead of /// time. Maybe can separate a tool/etc. for that. Expected -scanIncludes(cas::CASDB &CAS, cas::ObjectRef ExecID, StringRef MainFilename, - ArrayRef IncludeDirs, +scanIncludes(cas::CASDB &CAS, cas::ActionCache &Cache, cas::ObjectRef ExecID, + StringRef MainFilename, ArrayRef IncludeDirs, ArrayRef PrefixMappings = None, Optional *CapturedPM = nullptr); @@ -91,8 +93,8 @@ scanIncludes(cas::CASDB &CAS, cas::ObjectRef ExecID, StringRef MainFilename, /// \p ExecID is the Blob for the running executable. It can be used as part of /// the key for caching to avoid (fixed) bugs poisoning results. Expected -scanIncludesAndRemap(cas::CASDB &CAS, cas::ObjectRef ExecID, - std::string &MainFilename, +scanIncludesAndRemap(cas::CASDB &CAS, cas::ActionCache &Cache, + cas::ObjectRef ExecID, std::string &MainFilename, std::vector &IncludeDirs, ArrayRef PrefixMappings); diff --git a/llvm/lib/CAS/ActionCache.cpp b/llvm/lib/CAS/ActionCache.cpp new file mode 100644 index 00000000000000..510a987584bca1 --- /dev/null +++ b/llvm/lib/CAS/ActionCache.cpp @@ -0,0 +1,37 @@ +//===- ActionCache.cpp ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "BuiltinCAS.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/CASDB.h" +#include "llvm/CAS/CASID.h" + +using namespace llvm; +using namespace llvm::cas; + +void ActionCache::anchor() {} + +CacheKey::CacheKey(const CASID &ID) : Key(toStringRef(ID.getHash()).str()) {} +CacheKey::CacheKey(const ObjectProxy &Proxy) + : CacheKey(Proxy.getCAS(), Proxy.getRef()) {} +CacheKey::CacheKey(const CASDB &CAS, const ObjectRef &Ref) + : Key(toStringRef(CAS.getID(Ref).getHash())) {} + +Expected +ActionCache::getOrCompute(const CacheKey &ActionKey, + function_ref()> Computation) { + ArrayRef Key = arrayRefFromStringRef(ActionKey.getKey()); + if (Optional> Result = builtin::transpose(getImpl(Key))) + return std::move(*Result); + Optional Result; + if (Error E = Computation().moveInto(Result)) + return std::move(E); + if (Error E = putImpl(Key, *Result)) + return std::move(E); + return *Result; +} diff --git a/llvm/lib/CAS/ActionCaches.cpp b/llvm/lib/CAS/ActionCaches.cpp new file mode 100644 index 00000000000000..d86e8b9d365179 --- /dev/null +++ b/llvm/lib/CAS/ActionCaches.cpp @@ -0,0 +1,228 @@ +//===- ActionCaches.cpp -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "BuiltinCAS.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/CASDB.h" +#include "llvm/CAS/HashMappedTrie.h" +#include "llvm/CAS/OnDiskHashMappedTrie.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/Path.h" + +#define DEBUG_TYPE "action-caches" + +using namespace llvm; +using namespace llvm::cas; + +namespace { + +using HasherT = BLAKE3; +using HashType = decltype(HasherT::hash(std::declval &>())); + +template class CacheEntry { +public: + CacheEntry() = default; + CacheEntry(ArrayRef Hash) { llvm::copy(Hash, Value.data()); } + CacheEntry(const CacheEntry &Entry) { llvm::copy(Entry.Value, Value.data()); } + ArrayRef getValue() const { return Value; } + +private: + std::array Value; +}; + +class InMemoryActionCache final : public ActionCache { +public: + InMemoryActionCache(CASDB &CAS); + + Error putImpl(ArrayRef ActionKey, const ObjectRef &Result) final; + Expected> + getImpl(ArrayRef ActionKey) const final; + +private: + using DataT = CacheEntry; + using InMemoryCacheT = ThreadSafeHashMappedTrie; + + InMemoryCacheT Cache; +}; + +#if LLVM_ENABLE_ONDISK_CAS +class OnDiskActionCache final : public ActionCache { +public: + Error putImpl(ArrayRef ActionKey, const ObjectRef &Result) final; + Expected> + getImpl(ArrayRef ActionKey) const final; + + static Expected> create(CASDB &CAS, + StringRef Path); + +private: + static StringRef getHashName() { return "BLAKE3"; } + static StringRef getActionCacheTableName() { + static const std::string Name = + ("llvm.actioncache[" + getHashName() + "->" + getHashName() + "]") + .str(); + return Name; + } + static constexpr StringLiteral ActionCacheFile = "actions"; + static constexpr StringLiteral FilePrefix = "v1."; + + OnDiskActionCache(CASDB &CAS, StringRef RootPath, + OnDiskHashMappedTrie ActionCache); + + std::string Path; + OnDiskHashMappedTrie Cache; + using DataT = CacheEntry; +}; +#endif /* LLVM_ENABLE_ONDISK_CAS */ +} // end namespace + +static std::string hashToString(ArrayRef Hash) { + SmallString<64> Str; + toHex(Hash, /*LowerCase=*/true, Str); + return Str.str().str(); +} + +static Error createResultCachePoisonedError(StringRef Key, CASDB &CAS, + ObjectRef Output, + ArrayRef ExistingOutput) { + std::string OutID = CAS.getID(Output).toString(); + Optional ExistingRef = CAS.getReference(ExistingOutput); + std::string Existing = ExistingRef ? CAS.getID(*ExistingRef).toString() + : hashToString(ExistingOutput); + return createStringError(std::make_error_code(std::errc::invalid_argument), + "cache poisoned for '" + Key + "' (new='" + OutID + + "' vs. existing '" + Existing + "')"); +} + +static Error createResultCacheUnknownObjectError(StringRef Key, + StringRef Hash) { + return createStringError( + std::make_error_code(std::errc::no_such_device_or_address), + "the result object for key '" + Key + "' does not exist in CAS: '" + + Hash + "'"); +} + +// TODO: Check the hash schema is the same between action cache and CAS. If we +// can derive that from static type information, that would be even better. +InMemoryActionCache::InMemoryActionCache(CASDB &CAS) : ActionCache(CAS) {} + +Expected> +InMemoryActionCache::getImpl(ArrayRef Key) const { + auto Result = Cache.find(Key); + if (!Result) + return None; + return getCAS().getReference(Result->Data.getValue()); + Optional Out = getCAS().getReference(Result->Data.getValue()); + if (!Out) + return createResultCacheUnknownObjectError( + hashToString(Key), hashToString(Result->Data.getValue())); + return *Out; +} + +Error InMemoryActionCache::putImpl(ArrayRef Key, + const ObjectRef &Result) { + DataT Expected(getCAS().getID(Result).getHash()); + const InMemoryCacheT::value_type &Cached = *Cache.insertLazy( + Key, [&](auto ValueConstructor) { ValueConstructor.emplace(Expected); }); + + const DataT &Observed = Cached.Data; + if (Expected.getValue() == Observed.getValue()) + return Error::success(); + + return createResultCachePoisonedError(hashToString(Key), getCAS(), Result, + Observed.getValue()); +} + +std::unique_ptr cas::createInMemoryActionCache(CASDB &CAS) { + return std::make_unique(CAS); +} + +#if LLVM_ENABLE_ONDISK_CAS +OnDiskActionCache::OnDiskActionCache(CASDB &CAS, StringRef Path, + OnDiskHashMappedTrie Cache) + : ActionCache(CAS), Path(Path.str()), Cache(std::move(Cache)) {} + +Expected> +OnDiskActionCache::create(CASDB &CAS, StringRef AbsPath) { + if (std::error_code EC = sys::fs::create_directories(AbsPath)) + return createFileError(AbsPath, EC); + + SmallString<256> CachePath(AbsPath); + sys::path::append(CachePath, FilePrefix + ActionCacheFile); + constexpr uint64_t MB = 1024ull * 1024ull; + constexpr uint64_t GB = 1024ull * 1024ull * 1024ull; + + Optional ActionCache; + if (Error E = OnDiskHashMappedTrie::create( + CachePath, getActionCacheTableName(), sizeof(HashType) * 8, + /*DataSize=*/sizeof(DataT), /*MaxFileSize=*/GB, + /*MinFileSize=*/MB) + .moveInto(ActionCache)) + return std::move(E); + + return std::unique_ptr( + new OnDiskActionCache(CAS, AbsPath, std::move(*ActionCache))); +} + +Expected> +OnDiskActionCache::getImpl(ArrayRef Key) const { + // Check the result cache. + OnDiskHashMappedTrie::const_pointer ActionP = Cache.find(Key); + if (!ActionP) + return None; + + const DataT *Output = reinterpret_cast(ActionP->Data.data()); + Optional Out = getCAS().getReference(Output->getValue()); + if (!Out) + return createResultCacheUnknownObjectError( + hashToString(Key), hashToString(Output->getValue())); + + return *Out; +} + +Error OnDiskActionCache::putImpl(ArrayRef Key, + const ObjectRef &Result) { + DataT Expected(getCAS().getID(Result).getHash()); + OnDiskHashMappedTrie::pointer ActionP = Cache.insertLazy( + Key, [&](FileOffset TentativeOffset, + OnDiskHashMappedTrie::ValueProxy TentativeValue) { + assert(TentativeValue.Data.size() == sizeof(DataT)); + assert(isAddrAligned(Align::Of(), TentativeValue.Data.data())); + new (TentativeValue.Data.data()) DataT{Expected}; + }); + const DataT *Observed = reinterpret_cast(ActionP->Data.data()); + + if (Expected.getValue() == Observed->getValue()) + return Error::success(); + + return createResultCachePoisonedError(hashToString(Key), getCAS(), Result, + Observed->getValue()); +} + +static constexpr StringLiteral DefaultName = "actioncache"; + +std::string cas::getDefaultOnDiskActionCachePath() { + SmallString<128> Path; + if (!llvm::sys::path::cache_directory(Path)) + report_fatal_error("cannot get default cache directory"); + llvm::sys::path::append(Path, builtin::DefaultDir, DefaultName); + return Path.str().str(); +} + +Expected> +cas::createOnDiskActionCache(CASDB &CAS, StringRef Path) { + return OnDiskActionCache::create(CAS, Path); +} +# else +Expected> +cas::createOnDiskActionCache(CASDB &CAS, StringRef Path) { + return createStringError(inconvertibleErrorCode(), "OnDiskCache is disabled"); +} +#endif /* LLVM_ENABLE_ONDISK_CAS */ diff --git a/llvm/lib/CAS/BuiltinCAS.h b/llvm/lib/CAS/BuiltinCAS.h index e4406a4a55e415..b3442168acdb7a 100644 --- a/llvm/lib/CAS/BuiltinCAS.h +++ b/llvm/lib/CAS/BuiltinCAS.h @@ -207,30 +207,13 @@ class BuiltinCAS : public CASDB { "corrupt storage"); } - /// FIXME: This should not use Error. - Error createResultCacheMissError(CASID Input) const { - return createStringError(std::make_error_code(std::errc::invalid_argument), - "no result for '" + Input.toString() + "'"); - } - - Error createResultCachePoisonedError(CASID Input, CASID Output, - CASID ExistingOutput) const { - return createStringError(std::make_error_code(std::errc::invalid_argument), - "cache poisoned for '" + Input.toString() + - "' (new='" + Output.toString() + - "' vs. existing '" + - ExistingOutput.toString() + "')"); - } - - Error createResultCacheCorruptError(CASID Input) const { - return createStringError(std::make_error_code(std::errc::invalid_argument), - "result cache corrupt for '" + Input.toString() + - "'"); - } - Error validate(const CASID &ID) final; }; +// FIXME: Proxy not portable. Maybe also error-prone? +constexpr StringLiteral DefaultDirProxy = "/^llvm::cas::builtin::default"; +constexpr StringLiteral DefaultDir = "llvm.cas.builtin.default"; + } // end namespace builtin } // end namespace cas } // end namespace llvm diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt index ec42bbf375044f..6ac1053bcb3d67 100644 --- a/llvm/lib/CAS/CMakeLists.txt +++ b/llvm/lib/CAS/CMakeLists.txt @@ -1,4 +1,6 @@ add_llvm_component_library(LLVMCAS + ActionCache.cpp + ActionCaches.cpp BuiltinCAS.cpp CASDB.cpp CASFileSystem.cpp diff --git a/llvm/lib/CAS/InMemoryCAS.cpp b/llvm/lib/CAS/InMemoryCAS.cpp index 3568ddf9bbd7fd..dd2339db26220d 100644 --- a/llvm/lib/CAS/InMemoryCAS.cpp +++ b/llvm/lib/CAS/InMemoryCAS.cpp @@ -272,6 +272,11 @@ class InMemoryCAS : public BuiltinCAS { return toReference(*Object); return None; } + Optional getReference(ArrayRef Hash) const final { + if (InMemoryIndexT::const_pointer P = Index.find(Hash)) + return toReference(*P->Data); + return None; + } ObjectRef getReference(ObjectHandle Handle) const final { return toReference(asInMemoryObject(Handle)); } @@ -282,9 +287,6 @@ class InMemoryCAS : public BuiltinCAS { void print(raw_ostream &OS) const final; - Expected getCachedResult(CASID InputID) final; - Error putCachedResult(CASID InputID, CASID OutputID) final; - InMemoryCAS() = default; private: @@ -398,34 +400,6 @@ const InMemoryString &InMemoryCAS::getOrCreateString(StringRef String) { return S.Data.loadOrGenerate(Generator); } -Expected InMemoryCAS::getCachedResult(CASID InputID) { - InMemoryCacheT::pointer P = ActionCache.find(InputID.getHash()); - if (!P) - return createResultCacheMissError(InputID); - - /// TODO: Although, consider inserting null on cache misses and returning a - /// handle to avoid a second lookup! - assert(P->Data && "Unexpected null in result cache"); - return getID(*P->Data); -} - -Error InMemoryCAS::putCachedResult(CASID InputID, CASID OutputID) { - const InMemoryIndexT::value_type &Expected = indexHash(OutputID.getHash()); - const InMemoryCacheT::value_type &Cached = - *ActionCache.insertLazy(InputID.getHash(), [&](auto ValueConstructor) { - ValueConstructor.emplace(&Expected); - }); - - /// TODO: Although, consider changing \a getCachedResult() to insert nullptr - /// and returning a handle on cache misses! - assert(Cached.Data && "Unexpected null in result cache"); - const InMemoryIndexT::value_type &Observed = *Cached.Data; - if (&Expected == &Observed) - return Error::success(); - - return createResultCachePoisonedError(InputID, OutputID, getID(Observed)); -} - Error InMemoryCAS::forEachRef(ObjectHandle Handle, function_ref Callback) const { auto &Node = getInMemoryObject(Handle); diff --git a/llvm/lib/CAS/OnDiskCAS.cpp b/llvm/lib/CAS/OnDiskCAS.cpp index adbda20612c278..fbe04cf2d4e3a5 100644 --- a/llvm/lib/CAS/OnDiskCAS.cpp +++ b/llvm/lib/CAS/OnDiskCAS.cpp @@ -835,6 +835,7 @@ class OnDiskCAS : public BuiltinCAS { } Optional getReference(const CASID &ID) const final; + Optional getReference(ArrayRef Hash) const final; ObjectRef getReference(ObjectHandle Handle) const final { return getExternalReference(getInternalHandle(Handle).getRef()); } @@ -867,9 +868,6 @@ class OnDiskCAS : public BuiltinCAS { void print(raw_ostream &OS) const final; - Expected getCachedResult(CASID InputID) final; - Error putCachedResult(CASID InputID, CASID OutputID) final; - static Expected> open(StringRef Path); private: @@ -1455,6 +1453,16 @@ Optional OnDiskCAS::getReference(const CASID &ID) const { return None; } +Optional OnDiskCAS::getReference(ArrayRef Hash) const { + OnDiskHashMappedTrie::const_pointer P = Index.find(Hash); + if (!P) + return None; + IndexProxy I = getIndexProxyFromPointer(P); + if (Optional Ref = makeInternalRef(I.Offset, I.Ref.load())) + return getExternalReference(*Ref); + return None; +} + OnDiskHashMappedTrie::const_pointer OnDiskCAS::getInternalIndexPointer(const CASID &ID) const { // Recover the pointer from the FileOffset if ID comes from this CAS. @@ -1924,65 +1932,6 @@ Error OnDiskCAS::forEachRef(ObjectHandle Node, return Error::success(); } -Expected OnDiskCAS::getCachedResult(CASID InputID) { - // Check that InputID is valid. - // - // FIXME: InputID check is silly; we should have a separate ActionKey. - if (!getReference(InputID)) - return createUnknownObjectError(InputID); - - // Check the result cache. - // - // FIXME: Failure here should not be an error. - OnDiskHashMappedTrie::pointer ActionP = ActionCache.find(InputID.getHash()); - if (!ActionP) - return createResultCacheMissError(InputID); - const uint64_t Output = - reinterpret_cast(ActionP->Data.data()) - ->load(); - - // Return the result. - return getID(getExternalReference(InternalRef::getFromRawData(Output))); -} - -Error OnDiskCAS::putCachedResult(CASID InputID, CASID OutputID) { - // Check that both IDs are valid. - // - // FIXME: InputID check is silly; we should have a separate ActionKey. - if (!getReference(InputID)) - return createUnknownObjectError(InputID); - - Optional OutputRef; - if (Optional ExternalRef = getReference(OutputID)) - OutputRef = getInternalRef(*ExternalRef); - else - return createUnknownObjectError(OutputID); - - // Insert Input the result cache. - // - // FIXME: Consider templating OnDiskHashMappedTrie (really, renaming it to - // OnDiskHashMappedTrieBase and adding a type-safe layer on top). - const uint64_t Expected = OutputRef->getRawData(); - OnDiskHashMappedTrie::pointer ActionP = ActionCache.insertLazy( - InputID.getHash(), [&](FileOffset TentativeOffset, - OnDiskHashMappedTrie::ValueProxy TentativeValue) { - assert(TentativeValue.Data.size() == sizeof(ActionCacheResultT)); - assert(isAddrAligned(Align::Of(), - TentativeValue.Data.data())); - new (TentativeValue.Data.data()) ActionCacheResultT(Expected); - }); - const uint64_t Observed = - reinterpret_cast(ActionP->Data.data()) - ->load(); - - if (Expected == Observed) - return Error::success(); - - CASID ObservedID = - getID(getExternalReference(InternalRef::getFromRawData(Observed))); - return createResultCachePoisonedError(InputID, OutputID, ObservedID); -} - Expected> OnDiskCAS::open(StringRef AbsPath) { if (std::error_code EC = sys::fs::create_directories(AbsPath)) return createFileError(AbsPath, EC); @@ -2053,13 +2002,11 @@ Expected> cas::createOnDiskCAS(const Twine &Path) { #endif /* LLVM_ENABLE_ONDISK_CAS */ -// FIXME: Proxy not portable. Maybe also error-prone? -constexpr StringLiteral DefaultDirProxy = "/^llvm::cas::builtin::default"; -constexpr StringLiteral DefaultName = "llvm.cas.builtin.default"; +static constexpr StringLiteral DefaultName = "cas"; void cas::getDefaultOnDiskCASStableID(SmallVectorImpl &Path) { Path.assign(DefaultDirProxy.begin(), DefaultDirProxy.end()); - llvm::sys::path::append(Path, DefaultName); + llvm::sys::path::append(Path, DefaultDir, DefaultName); } std::string cas::getDefaultOnDiskCASStableID() { @@ -2072,7 +2019,7 @@ void cas::getDefaultOnDiskCASPath(SmallVectorImpl &Path) { // FIXME: Should this return 'Error' instead of hard-failing? if (!llvm::sys::path::cache_directory(Path)) report_fatal_error("cannot get default cache directory"); - llvm::sys::path::append(Path, DefaultName); + llvm::sys::path::append(Path, DefaultDir, DefaultName); } std::string cas::getDefaultOnDiskCASPath() { diff --git a/llvm/lib/TableGen/Main.cpp b/llvm/lib/TableGen/Main.cpp index 577db102fb09e4..1b59e1c426208a 100644 --- a/llvm/lib/TableGen/Main.cpp +++ b/llvm/lib/TableGen/Main.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/HierarchicalTreeBuilder.h" @@ -252,6 +253,7 @@ int llvm::TableGenMain(const char *argv0, TableGenMainFn *MainFn) { namespace { struct TableGenCache { std::unique_ptr CAS; + std::unique_ptr Cache; Optional ExecutableID; Optional IncludesTreeID; Optional ActionID; @@ -374,12 +376,18 @@ Error TableGenCache::lookupCachedResult(ArrayRef Args) { cas::createOnDiskCAS(cas::getDefaultOnDiskCASPath()).moveInto(CAS)) return E; + if (Error E = cas::createOnDiskActionCache( + *CAS, cas::getDefaultOnDiskActionCachePath()) + .moveInto(Cache)) + return E; + if (Error E = createExecutableBlob(Args[0]).moveInto(ExecutableID)) return E; OriginalInputFilename = InputFilename.getValue(); - Expected Scan = scanIncludesAndRemap( - *CAS, *ExecutableID, InputFilename, *&IncludeDirs, PrefixMappings); + Expected Scan = + scanIncludesAndRemap(*CAS, *Cache, *ExecutableID, InputFilename, + *&IncludeDirs, PrefixMappings); if (!Scan) return Scan.takeError(); @@ -393,9 +401,11 @@ Error TableGenCache::lookupCachedResult(ArrayRef Args) { return E; // Not an error for the result to be missing. - if (Optional ID = - expectedToOptional(CAS->getCachedResult(CAS->getID(*ActionID)))) - ResultID = CAS->getReference(*ID); + auto Result = Cache->get(CAS->getID(*ActionID)); + if (!Result) + return Result.takeError(); + if (Optional ID = *Result) + ResultID = *ID; return Error::success(); } @@ -479,7 +489,7 @@ Error TableGenCache::computeResult(TableGenMainFn *MainFn) { Expected Tree = Builder.create(*CAS); if (!Tree) return Tree.takeError(); - return CAS->putCachedResult(CAS->getID(*ActionID), CAS->getID(*Tree)); + return Cache->put(CAS->getID(*ActionID), CAS->getReference(*Tree)); } Error TableGenCache::replayResult() { diff --git a/llvm/lib/TableGen/ScanDependencies.cpp b/llvm/lib/TableGen/ScanDependencies.cpp index 11f66c99005151..10594a93e03e35 100644 --- a/llvm/lib/TableGen/ScanDependencies.cpp +++ b/llvm/lib/TableGen/ScanDependencies.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/TableGen/ScanDependencies.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/PrefixMapper.h" @@ -74,7 +75,7 @@ void tablegen::scanTextForIncludes(StringRef Input, } static Error -fetchCachedIncludedFiles(cas::CASDB &CAS, cas::CASID ID, +fetchCachedIncludedFiles(cas::CASDB &CAS, cas::ObjectRef ID, SmallVectorImpl &IncludedFiles) { Expected ExpectedBlob = CAS.getProxy(ID); if (!ExpectedBlob) @@ -83,8 +84,8 @@ fetchCachedIncludedFiles(cas::CASDB &CAS, cas::CASID ID, return Error::success(); } -static Error computeIncludedFiles(cas::CASDB &CAS, cas::CASID Key, - StringRef Input, +static Error computeIncludedFiles(cas::CASDB &CAS, cas::ActionCache &Cache, + cas::CASID Key, StringRef Input, SmallVectorImpl &IncludedFiles) { SmallString<256> ResultToCache; scanTextForIncludes(Input, IncludedFiles); @@ -94,12 +95,13 @@ static Error computeIncludedFiles(cas::CASDB &CAS, cas::CASID Key, CAS.createProxy(None, ResultToCache); if (!ExpectedResult) return ExpectedResult.takeError(); - if (Error E = CAS.putCachedResult(Key, *ExpectedResult)) + if (Error E = Cache.put(Key, ExpectedResult->getRef())) return E; return Error::success(); } -Error tablegen::scanTextForIncludes(cas::CASDB &CAS, cas::ObjectRef ExecID, +Error tablegen::scanTextForIncludes(cas::CASDB &CAS, cas::ActionCache &Cache, + cas::ObjectRef ExecID, const cas::ObjectProxy &Blob, SmallVectorImpl &Includes) { constexpr StringLiteral CacheKeyData = "llvm::tablegen::scanTextForIncludes"; @@ -110,10 +112,12 @@ Error tablegen::scanTextForIncludes(cas::CASDB &CAS, cas::ObjectRef ExecID, return Key.takeError(); cas::CASID KeyID = CAS.getID(*Key); - if (Optional ResultID = - expectedToOptional(CAS.getCachedResult(KeyID))) + auto Result = Cache.get(KeyID); + if (!Result) + return Result.takeError(); + if (Optional ResultID = *Result) return fetchCachedIncludedFiles(CAS, *ResultID, Includes); - return computeIncludedFiles(CAS, KeyID, Blob.getData(), Includes); + return computeIncludedFiles(CAS, Cache, KeyID, Blob.getData(), Includes); } Optional @@ -136,6 +140,7 @@ tablegen::lookupIncludeID(cas::CachingOnDiskFileSystem &FS, } Error tablegen::accessAllIncludes(cas::CachingOnDiskFileSystem &FS, + cas::ActionCache &Cache, cas::ObjectRef ExecID, ArrayRef IncludeDirs, const cas::ObjectProxy &MainFileBlob) { @@ -160,8 +165,8 @@ Error tablegen::accessAllIncludes(cas::CachingOnDiskFileSystem &FS, SmallVector IncludedFiles; while (!Worklist.empty()) { IncludedFiles.clear(); - if (Error E = scanTextForIncludes(CAS, ExecID, Worklist.pop_back_val(), - IncludedFiles)) + if (Error E = scanTextForIncludes(CAS, Cache, ExecID, + Worklist.pop_back_val(), IncludedFiles)) return E; // Add included files to the worklist. Ignore files not found, since the @@ -202,10 +207,12 @@ Error tablegen::createMainFileError(StringRef MainFilename, "': " + EC.message()); } -Expected tablegen::scanIncludes( - cas::CASDB &CAS, cas::ObjectRef ExecID, StringRef MainFilename, - ArrayRef IncludeDirs, ArrayRef PrefixMappings, - Optional *CapturedPM) { +Expected +tablegen::scanIncludes(cas::CASDB &CAS, cas::ActionCache &Cache, + cas::ObjectRef ExecID, StringRef MainFilename, + ArrayRef IncludeDirs, + ArrayRef PrefixMappings, + Optional *CapturedPM) { IntrusiveRefCntPtr FS; if (Error E = cas::createCachingOnDiskFileSystem(CAS).moveInto(FS)) return std::move(E); @@ -216,7 +223,7 @@ Expected tablegen::scanIncludes( return MainBlob.takeError(); // Helper for adding to the worklist. - if (Error E = accessAllIncludes(*FS, ExecID, IncludeDirs, *MainBlob)) + if (Error E = accessAllIncludes(*FS, Cache, ExecID, IncludeDirs, *MainBlob)) return std::move(E); Optional LocalPM; @@ -239,13 +246,13 @@ Expected tablegen::scanIncludes( } Expected -tablegen::scanIncludesAndRemap(cas::CASDB &CAS, cas::ObjectRef ExecID, - std::string &MainFilename, +tablegen::scanIncludesAndRemap(cas::CASDB &CAS, cas::ActionCache &Cache, + cas::ObjectRef ExecID, std::string &MainFilename, std::vector &IncludeDirs, ArrayRef PrefixMappings) { Optional PM; - auto Result = - scanIncludes(CAS, ExecID, MainFilename, IncludeDirs, PrefixMappings, &PM); + auto Result = scanIncludes(CAS, Cache, ExecID, MainFilename, IncludeDirs, + PrefixMappings, &PM); if (!Result) return Result.takeError(); diff --git a/llvm/tools/llvm-cas/llvm-cas.cpp b/llvm/tools/llvm-cas/llvm-cas.cpp index 55834347bfa261..4b2848249f7f26 100644 --- a/llvm/tools/llvm-cas/llvm-cas.cpp +++ b/llvm/tools/llvm-cas/llvm-cas.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/Optional.h" +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/CAS/CASFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" @@ -399,7 +400,8 @@ static Error recursiveAccess(CachingOnDiskFileSystem &FS, StringRef Path) { return Error::success(); } -static Expected ingestFileSystemImpl(CASDB &CAS, StringRef Path) { +static Expected +ingestFileSystemImpl(CASDB &CAS, StringRef Path) { auto FS = createCachingOnDiskFileSystem(CAS); if (!FS) return FS.takeError(); diff --git a/llvm/unittests/CAS/ActionCacheTest.cpp b/llvm/unittests/CAS/ActionCacheTest.cpp new file mode 100644 index 00000000000000..a407579781d288 --- /dev/null +++ b/llvm/unittests/CAS/ActionCacheTest.cpp @@ -0,0 +1,95 @@ +//===- ActionCacheTest.cpp ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CASTestConfig.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/CASDB.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Testing/Support/Error.h" +#include "llvm/Testing/Support/SupportHelpers.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::cas; + +TEST_P(CASTest, ActionCacheHit) { + std::unique_ptr CAS = createCAS(); + std::unique_ptr Cache = createActionCache(*CAS); + + Optional ID; + ASSERT_THAT_ERROR(CAS->createProxy(None, "1").moveInto(ID), Succeeded()); + ASSERT_THAT_ERROR(Cache->put(*ID, ID->getRef()), Succeeded()); + Optional Result; + ASSERT_THAT_ERROR(Cache->get(*ID).moveInto(Result), Succeeded()); + ASSERT_TRUE(Result); + ASSERT_EQ(*ID, *Result); +} + +TEST_P(CASTest, ActionCacheMiss) { + std::unique_ptr CAS = createCAS(); + std::unique_ptr Cache = createActionCache(*CAS); + + Optional ID1, ID2; + ASSERT_THAT_ERROR(CAS->createProxy(None, "1").moveInto(ID1), Succeeded()); + ASSERT_THAT_ERROR(CAS->createProxy(None, "2").moveInto(ID2), Succeeded()); + ASSERT_THAT_ERROR(Cache->put(*ID1, ID2->getRef()), Succeeded()); + // This is a cache miss for looking up a key doesn't exist. + Optional Result1; + ASSERT_THAT_ERROR(Cache->get(*ID2).moveInto(Result1), Succeeded()); + ASSERT_FALSE(Result1); + + ASSERT_THAT_ERROR(Cache->put(*ID2, ID1->getRef()), Succeeded()); + // Cache hit after adding the value. + Optional Result2; + ASSERT_THAT_ERROR(Cache->get(*ID2).moveInto(Result2), Succeeded()); + ASSERT_TRUE(Result2); + ASSERT_EQ(*ID1, *Result2); +} + +TEST_P(CASTest, ActionCacheRewrite) { + std::unique_ptr CAS = createCAS(); + std::unique_ptr Cache = createActionCache(*CAS); + + Optional ID1, ID2; + ASSERT_THAT_ERROR(CAS->createProxy(None, "1").moveInto(ID1), Succeeded()); + ASSERT_THAT_ERROR(CAS->createProxy(None, "2").moveInto(ID2), Succeeded()); + ASSERT_THAT_ERROR(Cache->put(*ID1, ID1->getRef()), Succeeded()); + // Writing to the same key with different value is error. + ASSERT_THAT_ERROR(Cache->put(*ID1, ID2->getRef()), Failed()); + // Writing the same value multiple times to the same key is fine. + ASSERT_THAT_ERROR(Cache->put(*ID1, ID1->getRef()), Succeeded()); +} + +TEST(OnDiskActionCache, ActionCacheResultInvalid) { + unittest::TempDir Temp("on-disk-cache", /*Unique=*/true); + std::unique_ptr CAS1 = createInMemoryCAS(); + std::unique_ptr CAS2 = createInMemoryCAS(); + + Optional ID1, ID2, ID3; + ASSERT_THAT_ERROR(CAS1->createProxy(None, "1").moveInto(ID1), Succeeded()); + ASSERT_THAT_ERROR(CAS1->createProxy(None, "2").moveInto(ID2), Succeeded()); + ASSERT_THAT_ERROR(CAS2->createProxy(None, "1").moveInto(ID3), Succeeded()); + + std::unique_ptr Cache1 = + cantFail(createOnDiskActionCache(*CAS1, Temp.path())); + // Test put and get. + ASSERT_THAT_ERROR(Cache1->put(*ID1, ID2->getRef()), Succeeded()); + Optional Result; + ASSERT_THAT_ERROR(Cache1->get(*ID1).moveInto(Result), Succeeded()); + ASSERT_TRUE(Result); + + // Create OnDiskCAS from the same location but a different underlying CAS. + std::unique_ptr Cache2 = + cantFail(createOnDiskActionCache(*CAS2, Temp.path())); + // Loading an key that points to an invalid object. + Optional Result2; + ASSERT_THAT_ERROR(Cache2->get(*ID3).moveInto(Result2), Failed()); + // Write a different value will cause error. + ASSERT_THAT_ERROR(Cache2->put(*ID3, ID3->getRef()), Failed()); +} diff --git a/llvm/unittests/CAS/CASDBTest.cpp b/llvm/unittests/CAS/CASDBTest.cpp index eeb62da446c584..9b7f7db9d667b7 100644 --- a/llvm/unittests/CAS/CASDBTest.cpp +++ b/llvm/unittests/CAS/CASDBTest.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "CASTestConfig.h" #include "llvm/CAS/CASDB.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/FileSystem.h" @@ -16,32 +17,7 @@ using namespace llvm; using namespace llvm::cas; -struct TestingAndDir { - std::unique_ptr DB; - Optional Temp; -}; - -class CASDBTest - : public testing::TestWithParam> { -protected: - Optional NextCASIndex; - - SmallVector Dirs; - - std::unique_ptr createCAS() { - auto TD = GetParam()((*NextCASIndex)++); - if (TD.Temp) - Dirs.push_back(std::move(*TD.Temp)); - return std::move(TD.DB); - } - void SetUp() { NextCASIndex = 0; } - void TearDown() { - NextCASIndex = None; - Dirs.clear(); - } -}; - -TEST_P(CASDBTest, PrintIDs) { +TEST_P(CASTest, PrintIDs) { std::unique_ptr CAS = createCAS(); Optional ID1, ID2; @@ -59,7 +35,7 @@ TEST_P(CASDBTest, PrintIDs) { EXPECT_EQ(ID2, ParsedID2); } -TEST_P(CASDBTest, Blobs) { +TEST_P(CASTest, Blobs) { std::unique_ptr CAS1 = createCAS(); StringRef ContentStrings[] = { "word", @@ -133,7 +109,7 @@ multiline text multiline text multiline text multiline text multiline text)", } } -TEST_P(CASDBTest, BlobsBig) { +TEST_P(CASTest, BlobsBig) { // A little bit of validation that bigger blobs are okay. Climb up to 1MB. std::unique_ptr CAS = createCAS(); SmallString<256> String1 = StringRef("a few words"); @@ -176,7 +152,7 @@ TEST_P(CASDBTest, BlobsBig) { } } -TEST_P(CASDBTest, LeafNodes) { +TEST_P(CASTest, LeafNodes) { std::unique_ptr CAS1 = createCAS(); StringRef ContentStrings[] = { "word", @@ -260,7 +236,7 @@ multiline text multiline text multiline text multiline text multiline text)", } } -TEST_P(CASDBTest, NodesBig) { +TEST_P(CASTest, NodesBig) { std::unique_ptr CAS = createCAS(); // Specifically check near 1MB for objects large enough they're likely to be @@ -297,17 +273,3 @@ TEST_P(CASDBTest, NodesBig) { for (auto ID: CreatedNodes) ASSERT_THAT_ERROR(CAS->validate(CAS->getID(ID)), Succeeded()); } - -INSTANTIATE_TEST_SUITE_P(InMemoryCAS, CASDBTest, ::testing::Values([](int) { - return TestingAndDir{createInMemoryCAS(), None}; - })); - -#if LLVM_ENABLE_ONDISK_CAS -static TestingAndDir createOnDisk(int I) { - unittest::TempDir Temp("on-disk-cas", /*Unique=*/true); - std::unique_ptr CAS; - EXPECT_THAT_ERROR(createOnDiskCAS(Temp.path()).moveInto(CAS), Succeeded()); - return TestingAndDir{std::move(CAS), std::move(Temp)}; -} -INSTANTIATE_TEST_SUITE_P(OnDiskCAS, CASDBTest, ::testing::Values(createOnDisk)); -#endif /* LLVM_ENABLE_ONDISK_CAS */ diff --git a/llvm/unittests/CAS/CASTestConfig.cpp b/llvm/unittests/CAS/CASTestConfig.cpp new file mode 100644 index 00000000000000..450dd77ba10648 --- /dev/null +++ b/llvm/unittests/CAS/CASTestConfig.cpp @@ -0,0 +1,40 @@ +//===- CASTestConfig.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CASTestConfig.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/CASDB.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::cas; + +TestingAndDir createInMemory(int I) { + std::unique_ptr CAS = createInMemoryCAS(); + return TestingAndDir{std::move(CAS), createInMemoryActionCache, None}; +} + +INSTANTIATE_TEST_SUITE_P(InMemoryCAS, CASTest, + ::testing::Values(createInMemory)); + +#if LLVM_ENABLE_ONDISK_CAS +TestingAndDir createOnDisk(int I) { + unittest::TempDir Temp("on-disk-cas", /*Unique=*/true); + std::unique_ptr CAS; + EXPECT_THAT_ERROR(createOnDiskCAS(Temp.path()).moveInto(CAS), Succeeded()); + std::string TempPath = Temp.path().str(); + auto CreateFn = [&, TempPath](CASDB &CAS) -> std::unique_ptr { + std::unique_ptr Cache; + EXPECT_THAT_ERROR(createOnDiskActionCache(CAS, TempPath).moveInto(Cache), + Succeeded()); + return Cache; + }; + return TestingAndDir{std::move(CAS), CreateFn, std::move(Temp)}; +} +INSTANTIATE_TEST_SUITE_P(OnDiskCAS, CASTest, ::testing::Values(createOnDisk)); +#endif /* LLVM_ENABLE_ONDISK_CAS */ diff --git a/llvm/unittests/CAS/CASTestConfig.h b/llvm/unittests/CAS/CASTestConfig.h index fc4d1f27233d2f..e1be81fc9422b0 100644 --- a/llvm/unittests/CAS/CASTestConfig.h +++ b/llvm/unittests/CAS/CASTestConfig.h @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASDB.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/FileSystem.h" @@ -13,12 +14,17 @@ #include "llvm/Testing/Support/SupportHelpers.h" #include "gtest/gtest.h" +#ifndef LLVM_UNITTESTS_CASTESTCONFIG_H +#define LLVM_UNITTESTS_CASTESTCONFIG_H + struct TestingAndDir { - std::unique_ptr DB; + std::unique_ptr CAS; + std::function(llvm::cas::CASDB &)> + CreateCacheFn; llvm::Optional Temp; }; -class CASDBTest +class CASTest : public testing::TestWithParam> { protected: llvm::Optional NextCASIndex; @@ -26,10 +32,15 @@ class CASDBTest llvm::SmallVector Dirs; std::unique_ptr createCAS() { - auto TD = GetParam()((*NextCASIndex)++); + auto TD = GetParam()(++(*NextCASIndex)); if (TD.Temp) Dirs.push_back(std::move(*TD.Temp)); - return std::move(TD.DB); + return std::move(TD.CAS); + } + std::unique_ptr + createActionCache(llvm::cas::CASDB &CAS) { + auto TD = GetParam()(*NextCASIndex); + return TD.CreateCacheFn(CAS); } void SetUp() { NextCASIndex = 0; } void TearDown() { @@ -38,12 +49,4 @@ class CASDBTest } }; -#if LLVM_ENABLE_ONDISK_CAS -static TestingAndDir createOnDisk(int I) { - llvm::unittest::TempDir Temp("on-disk-cas", /*Unique=*/true); - std::unique_ptr CAS; - EXPECT_THAT_ERROR(llvm::cas::createOnDiskCAS(Temp.path()).moveInto(CAS), - llvm::Succeeded()); - return TestingAndDir{std::move(CAS), std::move(Temp)}; -} -#endif /* LLVM_ENABLE_ONDISK_CAS */ +#endif diff --git a/llvm/unittests/CAS/CMakeLists.txt b/llvm/unittests/CAS/CMakeLists.txt index 18dc42ba4778db..bdcb47db6bedb9 100644 --- a/llvm/unittests/CAS/CMakeLists.txt +++ b/llvm/unittests/CAS/CMakeLists.txt @@ -5,8 +5,10 @@ set(LLVM_LINK_COMPONENTS ) add_llvm_unittest(CASTests + ActionCacheTest.cpp CASDBTest.cpp CASFileSystemTest.cpp + CASTestConfig.cpp CASOutputBackendTest.cpp CASProvidingFileSystemTest.cpp CachingOnDiskFileSystemTest.cpp