[Macros] In-process plugin server

Separate swift-syntax libs for the compiler and for the library plugins.
Compiler communicates with library plugins using serialized messages
just like executable plugins.

* `lib/swift/host/compiler/lib_Compiler*.dylib`(`lib/CompilerSwiftSyntax`):
  swift-syntax libraries for compiler. Library evolution is disabled.
* Compiler (`ASTGen` and `swiftIDEUtilsBridging`) only depends on
  `lib/swift/host/compiler` libraries.
* `SwiftInProcPluginServer`: In-process plugin server shared library.
  This has one `swift_inproc_plugins_handle_message` entry point that
  receives a message and return the response.
* In the compiler
  * Add `-in-process-plugin-server-path` front-end option, which specifies
    the `SwiftInProcPluginServer` shared library path.
  * Remove `LoadedLibraryPlugin`, because all library plugins are managed
    by `SwiftInProcPluginServer`
  * Introduce abstract `CompilerPlugin` class that has 2 subclasses:
    * `LoadedExecutablePlugin` existing class that represents an
      executable plugin
    * `InProcessPlugins` wraps `dlopen`ed `SwiftInProcPluginServer`
  * Unified the code path in `TypeCheckMacros.cpp` and `ASTGen`, the
    difference between executable plugins and library plugins are now
    abstracted by `CompilerPlugin`

Author: rintaro

cmake/modules/AddPureSwift.cmake

@@ -4,13 +4,12 @@ include(macCatalystUtils)
 function(force_add_dependencies TARGET)
   foreach(DEPENDENCY ${ARGN})
     string(REGEX REPLACE [<>:\"/\\|?*] _ sanitized ${DEPENDENCY})
-    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/forced-${sanitized}-dep.swift
-      COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/forced-${sanitized}-dep.swift
+    set(depfile "${CMAKE_CURRENT_BINARY_DIR}/forced-${sanitized}-dep.swift")
+    add_custom_command(OUTPUT ${depfile}
+      COMMAND ${CMAKE_COMMAND} -E touch ${depfile}
       DEPENDS ${DEPENDENCY}
     )
-    target_sources(${TARGET} PRIVATE
-      ${CMAKE_CURRENT_BINARY_DIR}/forced-${sanitized}-dep.swift
-    )
+    target_sources(${TARGET} PRIVATE ${depfile})
   endforeach()
 endfunction()
 
@@ -218,15 +217,6 @@ function(add_pure_swift_host_library name)
   # Depends on all '*.h' files in 'include/module.modulemap'.
   force_add_dependencies(${name} importedHeaderDependencies)
 
-  # Workaround to touch the library and its objects so that we don't
-  # continually rebuild (again, see corresponding change in swift-syntax).
-  add_custom_command(
-      TARGET ${name}
-      POST_BUILD
-      COMMAND "${CMAKE_COMMAND}" -E touch_nocreate $<TARGET_FILE:${name}> $<TARGET_OBJECTS:${name}> "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${name}.swiftmodule" "${CMAKE_CURRENT_BINARY_DIR}/${name}.swiftmodule"
-      COMMAND_EXPAND_LISTS
-      COMMENT "Update mtime of library outputs workaround")
-
   # Link against dependencies.
   target_link_libraries(${name} PUBLIC
     ${APSHL_DEPENDENCIES}
@@ -236,15 +226,7 @@ function(add_pure_swift_host_library name)
     ${APSHL_SWIFT_DEPENDENCIES}
   )
 
-  # Make sure we can use the host libraries.
-  target_include_directories(${name} PUBLIC
-    "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
-  target_link_directories(${name} PUBLIC
-    "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
-
   if(APSHL_EMIT_MODULE)
-    # Determine where Swift modules will be built and installed.
-
     set(module_triple "${SWIFT_HOST_MODULE_TRIPLE}")
     set(module_dir "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
     set(module_base "${module_dir}/${name}.swiftmodule")
@@ -253,14 +235,6 @@ function(add_pure_swift_host_library name)
     set(module_private_interface_file "${module_base}/${module_triple}.private.swiftinterface")
     set(module_sourceinfo_file "${module_base}/${module_triple}.swiftsourceinfo")
 
-    set_target_properties(${name} PROPERTIES
-        # Set the default module name to the target name.
-        Swift_MODULE_NAME ${name}
-        # Install the Swift module into the appropriate location.
-        Swift_MODULE_DIRECTORY ${module_dir}
-        # NOTE: workaround for CMake not setting up include flags.
-        INTERFACE_INCLUDE_DIRECTORIES ${module_dir})
-
     # Create the module directory.
     add_custom_command(
         TARGET ${name}
@@ -280,12 +254,27 @@ function(add_pure_swift_host_library name)
         >)
   else()
     # Emit a swiftmodule in the current directory.
-    set_target_properties(${name} PROPERTIES
-        Swift_MODULE_NAME ${name}
-        Swift_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-    set(module_file "${CMAKE_CURRENT_BINARY_DIR}/${name}.swiftmodule")
+    set(module_dir "${CMAKE_CURRENT_BINARY_DIR}/modules")
+    set(module_file "${module_dir}/${name}.swiftmodule")
   endif()
 
+  set_target_properties(${name} PROPERTIES
+    # Set the default module name to the target name.
+    Swift_MODULE_NAME ${name}
+    # Install the Swift module into the appropriate location.
+    Swift_MODULE_DIRECTORY ${module_dir}
+    # NOTE: workaround for CMake not setting up include flags.
+    INTERFACE_INCLUDE_DIRECTORIES ${module_dir})
+
+  # Workaround to touch the library and its objects so that we don't
+  # continually rebuild (again, see corresponding change in swift-syntax).
+  add_custom_command(
+      TARGET ${name}
+      POST_BUILD
+      COMMAND "${CMAKE_COMMAND}" -E touch_nocreate $<TARGET_FILE:${name}> $<TARGET_OBJECTS:${name}> "${module_file}"
+      COMMAND_EXPAND_LISTS
+      COMMENT "Update mtime of library outputs workaround")
+
   # Downstream linking should include the swiftmodule in debug builds to allow lldb to
   # work correctly. Only do this on Darwin since neither gold (currently used by default
   # on Linux), nor the default Windows linker 'link' support '-add_ast_path'.

cmake/modules/AddSwift.cmake

@@ -1016,12 +1016,12 @@ function(add_swift_host_tool executable)
       set_property(
         TARGET ${executable}
         APPEND PROPERTY INSTALL_RPATH
-          "@executable_path/../${extra_relative_rpath}lib/swift/host")
+          "@executable_path/../${extra_relative_rpath}lib/swift/host/compiler")
     else()
       set_property(
         TARGET ${executable}
         APPEND PROPERTY INSTALL_RPATH
-          "$ORIGIN/../${extra_relative_rpath}lib/swift/host")
+          "$ORIGIN/../${extra_relative_rpath}lib/swift/host/compiler")
     endif()
   endif()
 

cmake/modules/AddSwiftUnittests.cmake

@@ -133,11 +133,11 @@ function(add_swift_unittest test_dirname)
     if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS)
       set_property(
         TARGET ${test_dirname}
-        APPEND PROPERTY INSTALL_RPATH "@executable_path/${relative_lib_path}/swift/host")
+        APPEND PROPERTY INSTALL_RPATH "@executable_path/${relative_lib_path}/swift/host/compiler")
     elseif(SWIFT_HOST_VARIANT_SDK MATCHES "LINUX|ANDROID|OPENBSD|FREEBSD")
       set_property(
         TARGET ${test_dirname}
-        APPEND PROPERTY INSTALL_RPATH "$ORIGIN/${relative_lib_path}/swift/host")
+        APPEND PROPERTY INSTALL_RPATH "$ORIGIN/${relative_lib_path}/swift/host/compiler")
     endif()
   endif()
 endfunction()

include/swift/AST/ASTBridging.h

@@ -1862,9 +1862,6 @@ void Plugin_setCapability(PluginHandle handle,
 /// Get a capability data set by \c Plugin_setCapability .
 PluginCapabilityPtr _Nullable Plugin_getCapability(PluginHandle handle);
 
-/// Get the executable file path of the plugin.
-const char *Plugin_getExecutableFilePath(PluginHandle handle);
-
 /// Lock the plugin. Clients should lock it during sending and recving the
 /// response.
 void Plugin_lock(PluginHandle handle);

include/swift/AST/ASTContext.h

@@ -84,8 +84,6 @@ namespace swift {
   class DifferentiableAttr;
   class ExtensionDecl;
   struct ExternalSourceLocs;
-  class LoadedExecutablePlugin;
-  class LoadedLibraryPlugin;
   class ForeignRepresentationInfo;
   class FuncDecl;
   class GenericContext;

include/swift/AST/MacroDefinition.h

@@ -26,24 +26,38 @@ namespace swift {
 class ASTContext;
 
 /// A reference to an external macro definition that is understood by ASTGen.
-struct ExternalMacroDefinition {
-  enum class PluginKind : int8_t {
-    InProcess = 0,
-    Executable = 1,
-    Error = -1,
+class ExternalMacroDefinition {
+  enum class Status : int8_t {
+    Success = 0,
+    Error,
   };
-  PluginKind kind;
+  Status status;
   /// ASTGen's notion of an macro definition, which is opaque to the C++ part
   /// of the compiler. If 'kind' is 'PluginKind::Error', this is a C-string to
   /// the error message
-  const void *opaqueHandle = nullptr;
+  const void *opaqueHandle;
+
+  ExternalMacroDefinition(Status status, const void *opaqueHandle)
+      : status(status), opaqueHandle(opaqueHandle) {}
+
+public:
+  static ExternalMacroDefinition success(const void *opaqueHandle) {
+    return ExternalMacroDefinition{Status::Success, opaqueHandle};
+  }
 
   static ExternalMacroDefinition error(NullTerminatedStringRef message) {
-    return ExternalMacroDefinition{PluginKind::Error,
+    return ExternalMacroDefinition{Status::Error,
                                    static_cast<const void *>(message.data())};
   }
-  bool isError() const { return kind == PluginKind::Error; }
+
+  const void *get() {
+    if (status != Status::Success)
+      return nullptr;
+    return opaqueHandle;
+  }
+  bool isError() const { return status == Status::Error; }
   NullTerminatedStringRef getErrorMessage() const {
+    assert(isError());
     return static_cast<const char *>(opaqueHandle);
   }
 };

include/swift/AST/PluginLoader.h

@@ -78,15 +78,14 @@ class PluginLoader {
   /// returns a nullptr.
   /// NOTE: This method is idempotent. If the plugin is already loaded, the same
   /// instance is simply returned.
-  llvm::Expected<LoadedLibraryPlugin *> loadLibraryPlugin(llvm::StringRef path);
+  llvm::Expected<CompilerPlugin *> getInProcessPlugins();
 
   /// Launch the specified executable plugin path resolving the path with the
   /// current VFS. If it fails to load the plugin, a diagnostic is emitted, and
   /// returns a nullptr.
   /// NOTE: This method is idempotent. If the plugin is already loaded, the same
   /// instance is simply returned.
-  llvm::Expected<LoadedExecutablePlugin *>
-  loadExecutablePlugin(llvm::StringRef path);
+  llvm::Expected<CompilerPlugin *> loadExecutablePlugin(llvm::StringRef path);
 
   /// Add the specified plugin associated with the module name to the dependency
   /// tracker if needed.