[SourceKit] Pass the main swift executable path when constructing a compiler invocation

#58786 (rdar://93030932) was failing because the `swift-frontend` invocations passed a `swiftExecutablePath` to `Invocation.parseArgs`. This caused the `ClangImporter` instance to point to a `clang` binary next to the `swift-frontend` executable while SourceKit used PATH to find `clang`. The clang executable next to `swift-frontend` doesn’t actually exist because `clang` lives in `llvm-linux-aarch64/bin` and `swift-frontend` lives in `swift-linux-aarch64/bin`.
So some checks for a minimum clang verison failed for the normal build (because the executable doesn’t actually exists) while they pass during the SourceKit build (which used `clang` from `PATH`). This in turn caused the `outline-atomics` to be enabled to the SourceKit clang compiler arguments but not the clang compiler arguments for a normal build and thus resulted in two separate module cache directories (which includes the enabled features in the module directory hash).

To fix this issue, also set the swift executable path for compiler invocations created from SourceKit.

Fixes #58786 (rdar://93030932)

Author: ahoppen

include/swift/IDE/Utils.h

@@ -89,6 +89,7 @@ bool initCompilerInvocation(
     FrontendOptions::ActionType Action, DiagnosticEngine &Diags,
     StringRef UnresolvedPrimaryFile,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
+    const std::string &swiftExecutablePath,
     const std::string &runtimeResourcePath,
     const std::string &diagnosticDocumentationPath, time_t sessionTimestamp,
     std::string &Error);

lib/IDE/Utils.cpp

@@ -276,6 +276,7 @@ bool ide::initCompilerInvocation(
     FrontendOptions::ActionType Action, DiagnosticEngine &Diags,
     StringRef UnresolvedPrimaryFile,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
+    const std::string &swiftExecutablePath,
     const std::string &runtimeResourcePath,
     const std::string &diagnosticDocumentationPath, time_t sessionTimestamp,
     std::string &Error) {
@@ -299,7 +300,9 @@ bool ide::initCompilerInvocation(
       driver::getSingleFrontendInvocationFromDriverArguments(
           Args, Diags,
           [&](ArrayRef<const char *> FrontendArgs) {
-            return Invocation.parseArgs(FrontendArgs, Diags);
+            return Invocation.parseArgs(
+                FrontendArgs, Diags, /*ConfigurationFileBuffers=*/nullptr,
+                /*workingDirectory=*/"", swiftExecutablePath);
           },
           /*ForceNoOutputs=*/true);
 

test/SourceKit/Misc/match-module-cache-with-compiler.swift

@@ -3,9 +3,6 @@
 
 // REQUIRES: shell
 
-// https://github.com/apple/swift/issues/58786
-// UNSUPPORTED: OS=linux-gnu
-
 // RUN: %empty-directory(%t)
 
 // RUN: COMPILER_ARGS=( \

tools/SourceKit/include/SourceKit/Core/Context.h

@@ -169,6 +169,9 @@ class SlowRequestSimulator {
 };
 
 class Context {
+  /// The path of the swift-frontend executable.
+  /// Used to find clang relative to it.
+  std::string SwiftExecutablePath;
   std::string RuntimeLibPath;
   std::string DiagnosticDocumentationPath;
   std::unique_ptr<LangSupport> SwiftLang;
@@ -178,12 +181,15 @@ class Context {
   std::shared_ptr<SlowRequestSimulator> SlowRequestSim;
 
 public:
-  Context(StringRef RuntimeLibPath, StringRef DiagnosticDocumentationPath,
+  Context(StringRef SwiftExecutablePath, StringRef RuntimeLibPath,
+          StringRef DiagnosticDocumentationPath,
           llvm::function_ref<std::unique_ptr<LangSupport>(Context &)>
               LangSupportFactoryFn,
           bool shouldDispatchNotificationsOnMain = true);
   ~Context();
 
+  StringRef getSwiftExecutablePath() const { return SwiftExecutablePath; }
+
   StringRef getRuntimeLibPath() const { return RuntimeLibPath; }
   StringRef getDiagnosticDocumentationPath() const {
     return DiagnosticDocumentationPath;

tools/SourceKit/lib/Core/Context.cpp

@@ -36,11 +36,12 @@ GlobalConfig::getCompletionOpts() const {
 }
 
 SourceKit::Context::Context(
-    StringRef RuntimeLibPath, StringRef DiagnosticDocumentationPath,
+    StringRef SwiftExecutablePath, StringRef RuntimeLibPath,
+    StringRef DiagnosticDocumentationPath,
     llvm::function_ref<std::unique_ptr<LangSupport>(Context &)>
         LangSupportFactoryFn,
     bool shouldDispatchNotificationsOnMain)
-    : RuntimeLibPath(RuntimeLibPath),
+    : SwiftExecutablePath(SwiftExecutablePath), RuntimeLibPath(RuntimeLibPath),
       DiagnosticDocumentationPath(DiagnosticDocumentationPath),
       NotificationCtr(
           new NotificationCenter(shouldDispatchNotificationsOnMain)),

tools/SourceKit/lib/SwiftLang/SwiftASTManager.cpp

@@ -544,10 +544,11 @@ struct SwiftASTManager::Implementation {
       std::shared_ptr<SwiftEditorDocumentFileMap> EditorDocs,
       std::shared_ptr<GlobalConfig> Config,
       std::shared_ptr<SwiftStatistics> Stats,
-      std::shared_ptr<RequestTracker> ReqTracker, StringRef RuntimeResourcePath,
-      StringRef DiagnosticDocumentationPath)
+      std::shared_ptr<RequestTracker> ReqTracker, StringRef SwiftExecutablePath,
+      StringRef RuntimeResourcePath, StringRef DiagnosticDocumentationPath)
       : EditorDocs(EditorDocs), Config(Config), Stats(Stats),
-        ReqTracker(ReqTracker), RuntimeResourcePath(RuntimeResourcePath),
+        ReqTracker(ReqTracker), SwiftExecutablePath(SwiftExecutablePath),
+        RuntimeResourcePath(RuntimeResourcePath),
         DiagnosticDocumentationPath(DiagnosticDocumentationPath),
         SessionTimestamp(llvm::sys::toTimeT(std::chrono::system_clock::now())) {
   }
@@ -556,6 +557,9 @@ struct SwiftASTManager::Implementation {
   std::shared_ptr<GlobalConfig> Config;
   std::shared_ptr<SwiftStatistics> Stats;
   std::shared_ptr<RequestTracker> ReqTracker;
+  /// The path of the swift-frontend executable.
+  /// Used to find clang relative to it.
+  std::string SwiftExecutablePath;
   std::string RuntimeResourcePath;
   std::string DiagnosticDocumentationPath;
   SourceManager SourceMgr;
@@ -625,10 +629,10 @@ SwiftASTManager::SwiftASTManager(
     std::shared_ptr<SwiftEditorDocumentFileMap> EditorDocs,
     std::shared_ptr<GlobalConfig> Config,
     std::shared_ptr<SwiftStatistics> Stats,
-    std::shared_ptr<RequestTracker> ReqTracker, StringRef RuntimeResourcePath,
-    StringRef DiagnosticDocumentationPath)
+    std::shared_ptr<RequestTracker> ReqTracker, StringRef SwiftExecutablePath,
+    StringRef RuntimeResourcePath, StringRef DiagnosticDocumentationPath)
     : Impl(*new Implementation(EditorDocs, Config, Stats, ReqTracker,
-                               RuntimeResourcePath,
+                               SwiftExecutablePath, RuntimeResourcePath,
                                DiagnosticDocumentationPath)) {}
 
 SwiftASTManager::~SwiftASTManager() {
@@ -665,8 +669,8 @@ bool SwiftASTManager::initCompilerInvocation(
     std::string &Error) {
   return ide::initCompilerInvocation(
       Invocation, OrigArgs, Action, Diags, UnresolvedPrimaryFile, FileSystem,
-      Impl.RuntimeResourcePath, Impl.DiagnosticDocumentationPath,
-      Impl.SessionTimestamp, Error);
+      Impl.SwiftExecutablePath, Impl.RuntimeResourcePath,
+      Impl.DiagnosticDocumentationPath, Impl.SessionTimestamp, Error);
 }
 
 bool SwiftASTManager::initCompilerInvocation(

tools/SourceKit/lib/SwiftLang/SwiftASTManager.h

@@ -235,6 +235,7 @@ class SwiftASTManager : public std::enable_shared_from_this<SwiftASTManager> {
                            std::shared_ptr<GlobalConfig> Config,
                            std::shared_ptr<SwiftStatistics> Stats,
                            std::shared_ptr<RequestTracker> ReqTracker,
+                           StringRef SwiftExecutablePath,
                            StringRef RuntimeResourcePath,
                            StringRef DiagnosticDocumentationPath);
   ~SwiftASTManager();

tools/SourceKit/lib/SwiftLang/SwiftLangSupport.cpp

@@ -272,6 +272,7 @@ configureCompletionInstance(std::shared_ptr<CompletionInstance> CompletionInst,
 
 SwiftLangSupport::SwiftLangSupport(SourceKit::Context &SKCtx)
     : NotificationCtr(SKCtx.getNotificationCenter()),
+      SwiftExecutablePath(SKCtx.getSwiftExecutablePath()),
       ReqTracker(SKCtx.getRequestTracker()), CCCache(new SwiftCompletionCache),
       CompileManager(RuntimeResourcePath, DiagnosticDocumentationPath) {
   llvm::SmallString<128> LibPath(SKCtx.getRuntimeLibPath());
@@ -283,7 +284,7 @@ SwiftLangSupport::SwiftLangSupport(SourceKit::Context &SKCtx)
   EditorDocuments = std::make_shared<SwiftEditorDocumentFileMap>();
   ASTMgr = std::make_shared<SwiftASTManager>(
       EditorDocuments, SKCtx.getGlobalConfiguration(), Stats, ReqTracker,
-      RuntimeResourcePath, DiagnosticDocumentationPath);
+      SwiftExecutablePath, RuntimeResourcePath, DiagnosticDocumentationPath);
 
   CompletionInst = std::make_shared<CompletionInstance>();
 

tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

@@ -350,6 +350,9 @@ struct SwiftStatistics {
 
 class SwiftLangSupport : public LangSupport {
   std::shared_ptr<NotificationCenter> NotificationCtr;
+  /// The path of the swift-frontend executable.
+  /// Used to find clang relative to it.
+  std::string SwiftExecutablePath;
   std::string RuntimeResourcePath;
   std::string DiagnosticDocumentationPath;
   std::shared_ptr<SwiftASTManager> ASTMgr;

tools/SourceKit/tools/sourcekitd/bin/InProc/sourcekitdInProc-darwin.exports

@@ -70,7 +70,7 @@ sourcekitd_variant_uid_get_value
 _ZN10sourcekitd13cancelRequestEPKv
 _ZN10sourcekitd13enableLoggingEN4llvm9StringRefE
 _ZN10sourcekitd13handleRequestEPvPKvNSt3__18functionIFvS0_EEE
-_ZN10sourcekitd17initializeServiceEN4llvm9StringRefES1_NSt3__18functionIFvPvEEE
+_ZN10sourcekitd17initializeServiceEN4llvm9StringRefES1_S1_NSt3__18functionIFvPvEEE
 _ZN10sourcekitd24createErrorRequestFailedEN4llvm9StringRefE
 _ZN10sourcekitd24disposeCancellationTokenEPKv
 _ZN9SourceKit6Logger12LoggingLevelE

tools/SourceKit/tools/sourcekitd/bin/InProc/sourcekitdInProc.cpp

@@ -80,6 +80,13 @@ static std::string getRuntimeLibPath() {
   return libPath.str().str();
 }
 
+static std::string getSwiftExecutablePath() {
+  llvm::SmallString<128> path;
+  getToolchainPrefixPath(path);
+  llvm::sys::path::append(path, "bin", "swift-frontend");
+  return path.str().str();
+}
+
 static std::string getDiagnosticDocumentationPath() {
   llvm::SmallString<128> docPath;
   getToolchainPrefixPath(docPath);
@@ -90,7 +97,7 @@ static std::string getDiagnosticDocumentationPath() {
 void sourcekitd_initialize(void) {
   if (sourcekitd::initializeClient()) {
     LOG_INFO_FUNC(High, "initializing");
-    sourcekitd::initializeService(getRuntimeLibPath(),
+    sourcekitd::initializeService(getSwiftExecutablePath(), getRuntimeLibPath(),
                                   getDiagnosticDocumentationPath(),
                                   postNotification);
   }

tools/SourceKit/tools/sourcekitd/bin/XPC/Service/XPCService.cpp

@@ -215,6 +215,13 @@ static std::string getRuntimeLibPath() {
   return path.str().str();
 }
 
+static std::string getSwiftExecutablePath() {
+  llvm::SmallString<128> path;
+  getToolchainPrefixPath(path);
+  llvm::sys::path::append(path, "bin", "swift-frontend");
+  return path.str().str();
+}
+
 static std::string getDiagnosticDocumentationPath() {
   llvm::SmallString<128> path;
   getToolchainPrefixPath(path);
@@ -362,8 +369,9 @@ int main(int argc, const char *argv[]) {
       ^const char *(sourcekitd_uid_t uid) {
         return xpcUIdentFromSKDUID(uid).c_str();
       });
-  sourcekitd::initializeService(
-      getRuntimeLibPath(), getDiagnosticDocumentationPath(), postNotification);
+  sourcekitd::initializeService(getSwiftExecutablePath(), getRuntimeLibPath(),
+                                getDiagnosticDocumentationPath(),
+                                postNotification);
 
   // Increase the file descriptor limit.
   // FIXME: Portability ?

tools/SourceKit/tools/sourcekitd/include/sourcekitd/Internal.h

@@ -154,11 +154,14 @@ class RequestDict {
 };
 
 /// Initialize the service. Must be called before attempting to handle requests.
+/// \param swiftExecutablePath The path of the swift-frontend executable.
+///                            Used to find clang relative to it.
 /// \param runtimeLibPath The path to the toolchain's library directory.
 /// \param diagnosticDocumentationPath The path to diagnostics documentation.
 /// \param postNotification Callback to post a notification.
 void initializeService(
-    llvm::StringRef runtimeLibPath, llvm::StringRef diagnosticDocumentationPath,
+    llvm::StringRef swiftExecutablePath, llvm::StringRef runtimeLibPath,
+    llvm::StringRef diagnosticDocumentationPath,
     std::function<void(sourcekitd_response_t)> postNotification);
 /// Shutdown the service.
 void shutdownService();

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -107,14 +107,15 @@ static void fillDictionaryForDiagnosticInfo(ResponseBuilder::Dictionary Elem,
 static SourceKit::Context *GlobalCtx = nullptr;
 
 void sourcekitd::initializeService(
-    StringRef runtimeLibPath, StringRef diagnosticDocumentationPath,
+    llvm::StringRef swiftExecutablePath, StringRef runtimeLibPath,
+    StringRef diagnosticDocumentationPath,
     std::function<void(sourcekitd_response_t)> postNotification) {
   INITIALIZE_LLVM();
   initializeSwiftModules();
   llvm::EnablePrettyStackTrace();
-  GlobalCtx =
-      new SourceKit::Context(runtimeLibPath, diagnosticDocumentationPath,
-                             SourceKit::createSwiftLangSupport);
+  GlobalCtx = new SourceKit::Context(swiftExecutablePath, runtimeLibPath,
+                                     diagnosticDocumentationPath,
+                                     SourceKit::createSwiftLangSupport);
   auto noteCenter = GlobalCtx->getNotificationCenter();
 
   noteCenter->addDocumentUpdateNotificationReceiver([postNotification](StringRef DocumentName) {

unittests/SourceKit/SwiftLang/CursorInfoTest.cpp

@@ -27,6 +27,12 @@ static StringRef getRuntimeLibPath() {
   return sys::path::parent_path(SWIFTLIB_DIR);
 }
 
+static SmallString<128> getSwiftExecutablePath() {
+  SmallString<128> path = sys::path::parent_path(getRuntimeLibPath());
+  sys::path::append(path, "bin", "swift-frontend");
+  return path;
+}
+
 static void *createCancallationToken() {
   static std::atomic<size_t> handle(1000);
   return reinterpret_cast<void *>(
@@ -125,7 +131,8 @@ class CursorInfoTest : public ::testing::Test {
   }
 
   CursorInfoTest()
-      : Ctx(*new SourceKit::Context(getRuntimeLibPath(),
+      : Ctx(*new SourceKit::Context(getSwiftExecutablePath(),
+                                    getRuntimeLibPath(),
                                     /*diagnosticDocumentationPath*/ "",
                                     SourceKit::createSwiftLangSupport,
                                     /*dispatchOnMain=*/false)) {

unittests/SourceKit/SwiftLang/EditingTest.cpp

@@ -32,6 +32,12 @@ static StringRef getRuntimeLibPath() {
   return sys::path::parent_path(SWIFTLIB_DIR);
 }
 
+static SmallString<128> getSwiftExecutablePath() {
+  SmallString<128> path = sys::path::parent_path(getRuntimeLibPath());
+  sys::path::append(path, "bin", "swift-frontend");
+  return path;
+}
+
 namespace {
 
 struct Token {
@@ -124,7 +130,8 @@ class EditTest : public ::testing::Test {
     // This is avoiding destroying \p SourceKit::Context because another
     // thread may be active trying to use it to post notifications.
     // FIXME: Use shared_ptr ownership to avoid such issues.
-    Ctx = new SourceKit::Context(getRuntimeLibPath(),
+    Ctx = new SourceKit::Context(getSwiftExecutablePath(),
+                                 getRuntimeLibPath(),
                                  /*diagnosticDocumentationPath*/ "",
                                  SourceKit::createSwiftLangSupport,
                                  /*dispatchOnMain=*/false);