Add GeneratorEngine module with tests and macros (#31)

We'd like to avoid redundant rebuilding and regeneration for inputs that don't change. The new `GeneratorEngine` module achieves that by introducing a `Query` abstraction, which can be hashed and matched to its cached outputs if any are available.

The new `Engine` actor can run such `Query` and immediately return a cached result file if one is present and its hash matches the recorded one.

This required introducing new `@Query` and `@CacheKey` macros for making hashing easy and consistent on `QueryProtocol` types and their inputs.  We can't use the standard `Hashable` protocol, as that produces new hashes on every process launch due to randomized hashing seeds. With Swift Crypto's `HashFunction` we can generate consistent hashes and record them in `SQLiteBackedCache`.

To make the caching engine testable, I've introduced a new `FileSystem` actor protocol with `VirtualFileSystem` and `LocalFileSystem` actor implementations.

Author: MaxDesiatov

Docker/Dockerfile

@@ -1,4 +1,4 @@
-ARG swift_version=5.7
+ARG swift_version=5.9
 ARG ubuntu_version=jammy
 ARG base_image=swift:$swift_version-$ubuntu_version
 FROM $base_image
@@ -7,7 +7,7 @@ ARG swift_version
 ARG ubuntu_version
 
 # set as UTF-8
-RUN apt-get update && apt-get install -y locales locales-all
+RUN apt-get update && apt-get install -y locales locales-all libsqlite3-dev
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8

Package.resolved

@@ -45,6 +45,15 @@
         "version" : "1.0.4"
       }
     },
+    {
+      "identity" : "swift-crypto",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-crypto.git",
+      "state" : {
+        "revision" : "b51f1d6845b353a2121de1c6a670738ec33561a6",
+        "version" : "3.1.0"
+      }
+    },
     {
       "identity" : "swift-log",
       "kind" : "remoteSourceControl",
@@ -99,6 +108,15 @@
         "version" : "1.19.0"
       }
     },
+    {
+      "identity" : "swift-syntax",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-syntax.git",
+      "state" : {
+        "revision" : "ffa3cd6fc2aa62adbedd31d3efaf7c0d86a9f029",
+        "version" : "509.0.1"
+      }
+    },
     {
       "identity" : "swift-system",
       "kind" : "remoteSourceControl",

Package.swift

@@ -1,6 +1,7 @@
 // swift-tools-version: 5.9
 // The swift-tools-version declares the minimum version of Swift required to build this package.
 
+import CompilerPluginSupport
 import PackageDescription
 
 let package = Package(
@@ -20,10 +21,12 @@ let package = Package(
     .package(url: "https://github.com/apple/swift-argument-parser", from: "1.2.2"),
     .package(url: "https://github.com/apple/swift-async-algorithms.git", exact: "1.0.0-alpha"),
     .package(url: "https://github.com/apple/swift-atomics.git", from: "1.1.0"),
+    .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.4"),
+    .package(url: "https://github.com/apple/swift-crypto.git", from: "3.1.0"),
     .package(url: "https://github.com/apple/swift-nio.git", from: "2.58.0"),
     .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.19.0"),
     .package(url: "https://github.com/apple/swift-log.git", from: "1.5.3"),
-    .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.4"),
+    .package(url: "https://github.com/apple/swift-syntax.git", from: "509.0.1"),
   ],
   targets: [
     // Targets are the basic building blocks of a package. A target can define a module or a test suite.
@@ -33,7 +36,7 @@ let package = Package(
       dependencies: [
         "SwiftSDKGenerator",
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
-      ],      
+      ],
       swiftSettings: [
         .enableExperimentalFeature("StrictConcurrency=complete"),
       ]
@@ -44,7 +47,9 @@ let package = Package(
         .target(name: "AsyncProcess"),
         .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
         .product(name: "AsyncHTTPClient", package: "async-http-client"),
+        .product(name: "Logging", package: "swift-log"),
         .product(name: "SystemPackage", package: "swift-system"),
+        "GeneratorEngine",
       ],
       exclude: ["Dockerfiles"],
       swiftSettings: [
@@ -54,12 +59,47 @@ let package = Package(
     .testTarget(
       name: "SwiftSDKGeneratorTests",
       dependencies: [
-        .target(name: "SwiftSDKGenerator"),
+        "SwiftSDKGenerator",
       ],
       swiftSettings: [
         .enableExperimentalFeature("StrictConcurrency=complete"),
       ]
     ),
+    .target(
+      name: "GeneratorEngine",
+      dependencies: [
+        .product(name: "AsyncHTTPClient", package: "async-http-client"),
+        .product(name: "Crypto", package: "swift-crypto"),
+        .product(name: "Logging", package: "swift-log"),
+        .product(name: "SystemPackage", package: "swift-system"),
+        "Macros",
+        "SystemSQLite",
+      ]
+    ),
+    .testTarget(
+      name: "GeneratorEngineTests",
+      dependencies: [
+        "GeneratorEngine",
+      ]
+    ),
+    .macro(
+      name: "Macros",
+      dependencies: [
+        .product(name: "SwiftCompilerPlugin", package: "swift-syntax"),
+        .product(name: "SwiftSyntax", package: "swift-syntax"),
+        .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"),
+        .product(name: "SwiftSyntaxMacros", package: "swift-syntax"),
+        .product(name: "SwiftDiagnostics", package: "swift-syntax"),
+      ]
+    ),
+    .testTarget(
+      name: "MacrosTests",
+      dependencies: [
+        "Macros",
+        .product(name: "SwiftSyntaxMacros", package: "swift-syntax"),
+        .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"),
+      ]
+    ),
     .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"),
     .target(
       name: "AsyncProcess",

Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift

@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Macros
+
+@_exported import protocol Crypto.HashFunction
+import struct Foundation.URL
+import struct SystemPackage.FilePath
+
+/// Indicates that values of a conforming type can be hashed with an arbitrary hashing function. Unlike `Hashable`,
+/// this protocol doesn't utilize random seed values and produces consistent hash values across process launches.
+public protocol CacheKeyProtocol {
+  func hash(with hashFunction: inout some HashFunction)
+}
+
+extension Bool: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    "Swift.Bool".hash(with: &hashFunction)
+    hashFunction.update(data: self ? [1] : [0])
+  }
+}
+
+extension Int: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    "Swift.Int".hash(with: &hashFunction)
+    withUnsafeBytes(of: self) {
+      hashFunction.update(bufferPointer: $0)
+    }
+  }
+}
+
+extension String: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    var t = "Swift.String"
+    t.withUTF8 {
+      hashFunction.update(bufferPointer: .init($0))
+    }
+    var x = self
+    x.withUTF8 {
+      hashFunction.update(bufferPointer: .init($0))
+    }
+  }
+}
+
+extension FilePath: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    "SystemPackage.FilePath".hash(with: &hashFunction)
+    self.string.hash(with: &hashFunction)
+  }
+}
+
+extension URL: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    "Foundation.URL".hash(with: &hashFunction)
+    self.description.hash(with: &hashFunction)
+  }
+}
+
+extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    "Swift.Optional".hash(with: &hashFunction)
+    if let self {
+      true.hash(with: &hashFunction)
+      self.hash(with: &hashFunction)
+    } else {
+      false.hash(with: &hashFunction)
+    }
+  }
+}
+
+@attached(extension, conformances: CacheKeyProtocol, names: named(hash(with:)))
+public macro CacheKey() = #externalMacro(module: "Macros", type: "CacheKeyMacro")

Sources/GeneratorEngine/Cache/FileCacheRecord.swift

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import struct SystemPackage.FilePath
+
+struct FileCacheRecord {
+  let path: FilePath
+  let hash: String
+}
+
+extension FileCacheRecord: Codable {
+  enum CodingKeys: CodingKey {
+    case path
+    case hash
+  }
+
+  // FIXME: `Codable` on `FilePath` is broken
+  init(from decoder: any Decoder) throws {
+    let container = try decoder.container(keyedBy: CodingKeys.self)
+    self.path = try FilePath(container.decode(String.self, forKey: .path))
+    self.hash = try container.decode(String.self, forKey: .hash)
+  }
+
+  func encode(to encoder: any Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    try container.encode(self.path.string, forKey: .path)
+    try container.encode(self.hash, forKey: .hash)
+  }
+}