Fully generalize "whole match" in the engine and enable transforming custom types

* Track the whole match as an element of the "capture list" in the matching engine. Do so by emitting code as an implicit `capture` around the root node.
* No longer handle `matcher` as a special case within `capture` lowering, because the matcher can be arbitrarily nested within "output-forwarding" nodes, such as a `changeMatchingOptions` non-capturing group. Instead, make the bytecode emitter carry a result value so that a custom output can be propagated through any forwarding nodes.
  ```swift
  Regex {
    Capture(
      SemanticVersionParser()
        .ignoringCase()
        .matchingSemantics(.unicodeScalar)
    ) // This would not work previously.
  }
  ```
* Collapse DSLTree node `transform` into `capture`, because a transform can never be standalone (without a `capture` parent). This greatly simplifies `capture` lowering.
* Make the bytecode's capture transform use type `(Input, _StoredCapture) -> Any` so that it can transform any whole match, not just `Substring`. This means you can now transform any captured value, including a custom-consuming regex component's result!
  ```swift
   Regex {
    "version:"
    OneOrMore(.whitespace)
    Capture {
      SemanticVersionParser() // Regex<SemanticVersion>
    } transform: {
      // (SemanticVersion) -> SomethingElse
    }
  }
  ```
  The transforms of `Capture` and `TryCapture` are now generalized from taking `Substring` to taking generic parameter `W` (the whole match).
* Fix an issue where initial options were applied based solely on whether the bytecode had any instructions, failing examples such as `((?i:.))`. It now checks whether the first matchable atom has been emitted.

Author: rxwei

Sources/RegexBuilder/Variadics.swift

@@ -646,28 +646,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any
-            },
-            component.regex.root)))
+          self.init(node: .capture(
+            component.regex.root,
+            CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
           as reference: Reference<NewCapture>,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any
-              },
-              component.regex.root)))
+            component.regex.root,
+            CaptureTransform(transform)))
         }
       }
 
@@ -676,28 +671,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any?
-            },
-            component.regex.root)))
+          self.init(node: .capture(
+          component.regex.root,
+          CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           _ component: R,
           as reference: Reference<NewCapture>,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any?
-              },
-              component.regex.root)))
+            component.regex.root,
+            CaptureTransform(transform)))
         }
       }
 
@@ -725,28 +715,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any
-            },
-            component().regex.root)))
+          self.init(node: .capture(
+            component().regex.root,
+            CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           as reference: Reference<NewCapture>,
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture
+          transform: @escaping (W) throws -> NewCapture
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any
-              },
-              component().regex.root)))
+            component().regex.root,
+            CaptureTransform(transform)))
         }
       }
 
@@ -755,28 +740,23 @@ struct VariadicsGenerator: ParsableCommand {
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
-          self.init(node: .capture(.transform(
-            CaptureTransform(resultType: NewCapture.self) {
-              try transform($0) as Any?
-            },
-            component().regex.root)))
+          self.init(node: .capture(
+            component().regex.root,
+            CaptureTransform(transform)))
         }
 
       \(disfavored)\
         public init<\(genericParams), NewCapture>(
           as reference: Reference<NewCapture>,
           @\(concatBuilderName) _ component: () -> R,
-          transform: @escaping (Substring) throws -> NewCapture?
+          transform: @escaping (W) throws -> NewCapture?
         ) \(whereClauseTransformed) {
           self.init(node: .capture(
             reference: reference.id,
-            .transform(
-              CaptureTransform(resultType: NewCapture.self) {
-                try transform($0) as Any?
-              },
-              component().regex.root)))
+            component().regex.root,
+            CaptureTransform(transform)))
         }
       }
 

Sources/VariadicsGenerator/VariadicsGenerator.swift

@@ -24,13 +24,13 @@ public struct CaptureList {
 extension CaptureList {
   public struct Capture {
     public var name: String?
-    public var type: Any.Type?
+    public var type: Any.Type
     public var optionalDepth: Int
     public var location: SourceLocation
 
     public init(
       name: String? = nil,
-      type: Any.Type? = nil,
+      type: Any.Type = Substring.self,
       optionalDepth: Int,
       _ location: SourceLocation
     ) {
@@ -122,18 +122,15 @@ extension AST.Node {
       break
     }
   }
-
-  public var _captureList: CaptureList {
-    var caps = CaptureList()
-    self._addCaptures(to: &caps, optionalNesting: 0)
-    return caps
-  }
 }
 
 extension AST {
-  /// Get the capture list for this AST
+  /// The capture list (including the whole match) of this AST.
   public var captureList: CaptureList {
-    root._captureList
+    var caps = CaptureList()
+    caps.append(.init(optionalDepth: 0, .fake))
+    root._addCaptures(to: &caps, optionalNesting: 0)
+    return caps
   }
 }
 
@@ -151,12 +148,7 @@ extension CaptureList: Equatable {}
 
 extension CaptureList.Capture: CustomStringConvertible {
   public var description: String {
-    let typeStr: String
-    if let ty = type {
-      typeStr = "\(ty)"
-    } else {
-      typeStr = "Substring"
-    }
+    let typeStr = String(describing: type)
     let suffix = String(repeating: "?", count: optionalDepth)
     return typeStr + suffix
   }

Sources/_RegexParser/Regex/Parse/CaptureList.swift

@@ -225,7 +225,7 @@ extension CaptureStructure: CustomStringConvertible {
 extension AST {
   /// The capture structure of this AST for compiler communication.
   var captureStructure: CaptureStructure {
-    root._captureList._captureStructure(nestOptionals: true)
+    captureList._captureStructure(nestOptionals: true)
   }
 }
 
@@ -246,10 +246,7 @@ extension CaptureList {
 extension CaptureList.Capture {
   func _captureStructure(nestOptionals: Bool) -> CaptureStructure {
     if optionalDepth == 0 {
-      if let ty = type {
-        return .atom(name: name, type: .init(ty))
-      }
-      return .atom(name: name)
+      return .atom(name: name, type: type == Substring.self ? nil : .init(type))
     }
     var copy = self
     copy.optionalDepth = 0

Sources/_RegexParser/Regex/Parse/CaptureStructure.swift

@@ -77,7 +77,7 @@ extension RegexValidator {
     }
     switch ref.kind {
     case .absolute(let i):
-      guard i <= captures.captures.count else {
+      guard i < captures.captures.count else {
         throw error(.invalidReference(i), at: ref.innerLoc)
       }
     case .named(let name):