Merge pull request swiftlang#98 from hamishknight/time-and-place

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/AST.swift

@@ -92,11 +92,20 @@ extension AST {
 
   public struct Alternation: Hashable, _ASTNode {
     public let children: [AST]
-    public let location: SourceLocation
+    public let pipes: [SourceLocation]
+
+    public init(_ mems: [AST], pipes: [SourceLocation]) {
+      // An alternation must have at least two branches (though the branches
+      // may be empty AST nodes), and n - 1 pipes.
+      precondition(mems.count >= 2)
+      precondition(pipes.count == mems.count - 1)
 
-    public init(_ mems: [AST], _ location: SourceLocation) {
       self.children = mems
-      self.location = location
+      self.pipes = pipes
+    }
+
+    public var location: SourceLocation {
+      .init(children.first!.location.start ..< children.last!.location.end)
     }
   }
 

Sources/_MatchingEngine/Regex/AST/Atom.swift

@@ -314,7 +314,7 @@ extension AST.Atom {
 
     public var _dumpBase: String {
       // FIXME: better printing...
-      "\(kind)\(isInverted)"
+      "\(kind)\(isInverted)\(isPOSIX)"
     }
   }
 }

Sources/_MatchingEngine/Regex/AST/CustomCharClass.swift

@@ -21,14 +21,25 @@ extension AST {
       case custom(CustomCharacterClass)
 
       /// A character range `a-z`
-      case range(Atom, Atom)
+      case range(Range)
 
       /// A single character or escape
       case atom(Atom)
 
       /// A binary operator applied to sets of members `abc&&def`
       case setOperation([Member], Located<SetOp>, [Member])
     }
+    public struct Range: Hashable {
+      public var lhs: Atom
+      public var dashLoc: SourceLocation
+      public var rhs: Atom
+
+      public init(_ lhs: Atom, _ dashLoc: SourceLocation, _ rhs: Atom) {
+        self.lhs = lhs
+        self.dashLoc = dashLoc
+        self.rhs = rhs
+      }
+    }
     public enum SetOp: String, Hashable {
       case subtraction = "--"
       case intersection = "&&"
@@ -45,3 +56,18 @@ extension AST {
 extension AST.CustomCharacterClass {
   public var isInverted: Bool { start.value == .inverted }
 }
+
+extension CustomCC.Member {
+  private var _associatedValue: Any {
+    switch self {
+    case .custom(let c): return c
+    case .range(let r): return r
+    case .atom(let a): return a
+    case .setOperation(let lhs, let op, let rhs): return (lhs, op, rhs)
+    }
+  }
+
+  func `as`<T>(_ t: T.Type = T.self) -> T? {
+    _associatedValue as? T
+  }
+}

Sources/_MatchingEngine/Regex/AST/Group.swift

@@ -54,10 +54,9 @@ extension AST {
       case atomicScriptRun
 
       // (?iJmnsUxxxDPSWy{..}-iJmnsUxxxDPSW:)
-      // If hasImplicitScope is true, it was written as e.g (?i), and implicitly
-      // forms a group containing all the following elements of the current
-      // group.
-      case changeMatchingOptions(MatchingOptionSequence, hasImplicitScope: Bool)
+      // Isolated options are written as e.g (?i), and implicitly form a group
+      // containing all the following elements of the current group.
+      case changeMatchingOptions(MatchingOptionSequence, isIsolated: Bool)
 
       // NOTE: Comments appear to be groups, but are not parsed
       // the same. They parse more like quotes, so are not
@@ -74,16 +73,16 @@ extension AST.Group.Kind {
     }
   }
 
-  /// Whether this is a group with an implicit scope, e.g matching options
-  /// written as (?i) implicitly become parent groups for the rest of the
-  /// elements in the current group:
+  /// Whether this is a group with an implicit scope, e.g isolated matching
+  /// options implicitly become parent groups for the rest of the elements in
+  /// the current group:
   ///
   ///      (a(?i)bc)de -> (a(?i:bc))de
   ///
   public var hasImplicitScope: Bool {
     switch self {
-    case .changeMatchingOptions(_, let hasImplicitScope):
-      return hasImplicitScope
+    case .changeMatchingOptions(_, let isIsolated):
+      return isIsolated
     default:
       return false
     }

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -602,7 +602,7 @@ extension Source {
         // Matching option changing group (?iJmnsUxxxDPSWy{..}-iJmnsUxxxDPSW:).
         if let seq = try src.lexMatchingOptionSequence() {
           if src.tryEat(":") {
-            return .changeMatchingOptions(seq, hasImplicitScope: false)
+            return .changeMatchingOptions(seq, isIsolated: false)
           }
           // If this isn't start of an explicit group, we should have an
           // implicit group that covers the remaining elements of the current
@@ -611,7 +611,7 @@ extension Source {
           // also does it across alternations, which will require additional
           // handling.
           try src.expect(")")
-          return .changeMatchingOptions(seq, hasImplicitScope: true)
+          return .changeMatchingOptions(seq, isIsolated: true)
         }
 
         guard let next = src.peek() else {
@@ -1026,14 +1026,19 @@ extension Source {
   /// of a '-' character followed by an atom.
   mutating func lexCustomCharClassRangeEnd(
     priorGroupCount: Int
-  ) throws -> AST.Atom? {
+  ) throws -> (dashLoc: SourceLocation, AST.Atom)? {
     // Make sure we don't have a binary operator e.g '--', and the '-' is not
     // ending the custom character class (in which case it is literal).
+    let start = currentPosition
     guard peekCCBinOp() == nil && !starts(with: "-]") && tryEat("-") else {
       return nil
     }
-    return try lexAtom(isInCustomCharacterClass: true,
-                       priorGroupCount: priorGroupCount)
+    let dashLoc = Location(start ..< currentPosition)
+    guard let end = try lexAtom(isInCustomCharacterClass: true,
+                                priorGroupCount: priorGroupCount) else {
+      return nil
+    }
+    return (dashLoc, end)
   }
 }
 

Sources/_MatchingEngine/Regex/Parse/Parse.swift

@@ -89,17 +89,20 @@ extension Parser {
 
     if source.isEmpty { return .empty(.init(loc(_start))) }
 
-    var result = Array<AST>(singleElement: try parseConcatenation())
-    while source.tryEat("|") {
-      // TODO: track pipe locations too...
+    var result = [try parseConcatenation()]
+    var pipes: [SourceLocation] = []
+    while true {
+      let pipeStart = source.currentPosition
+      guard source.tryEat("|") else { break }
+      pipes.append(loc(pipeStart))
       result.append(try parseConcatenation())
     }
 
     if result.count == 1 {
       return result[0]
     }
 
-    return .alternation(.init(result, loc(_start)))
+    return .alternation(.init(result, pipes: pipes))
   }
 
   /// Parse a term, potentially separated from others by `|`
@@ -266,14 +269,14 @@ extension Parser {
       else { break }
 
       // Range between atoms.
-      if let rhs = try source.lexCustomCharClassRangeEnd(
+      if let (dashLoc, rhs) = try source.lexCustomCharClassRangeEnd(
         priorGroupCount: priorGroupCount
       ) {
         guard atom.literalCharacterValue != nil &&
               rhs.literalCharacterValue != nil else {
           throw ParseError.invalidCharacterClassRangeOperand
         }
-        members.append(.range(atom, rhs))
+        members.append(.range(.init(atom, dashLoc, rhs)))
         continue
       }
 

Sources/_MatchingEngine/Regex/Printing/DumpAST.swift

@@ -120,8 +120,8 @@ extension AST.Group.Kind: _ASTPrintable {
     case .nonAtomicLookbehind:   return "nonAtomicLookbehind"
     case .scriptRun:             return "scriptRun"
     case .atomicScriptRun:       return "atomicScriptRun"
-    case .changeMatchingOptions(let seq, let hasImplicitScope):
-      return "changeMatchingOptions<\(seq), \(hasImplicitScope)>"
+    case .changeMatchingOptions(let seq, let isIsolated):
+      return "changeMatchingOptions<\(seq), \(isIsolated)>"
     }
   }
 }
@@ -185,10 +185,15 @@ extension AST.CustomCharacterClass.Member: _ASTPrintable {
     switch self {
     case .custom(let cc): return "\(cc)"
     case .atom(let a): return "\(a)"
-    case .range(let lhs, let rhs):
-      return "range \(lhs) to \(rhs)"
+    case .range(let r): return "\(r)"
     case .setOperation(let lhs, let op, let rhs):
       return "op \(lhs) \(op.value) \(rhs)"
     }
   }
 }
+
+extension AST.CustomCharacterClass.Range: _ASTPrintable {
+  public var _dumpBase: String {
+    "\(lhs)-\(rhs)"
+  }
+}

Sources/_MatchingEngine/Regex/Printing/PrintAsCanonical.swift

@@ -85,10 +85,10 @@ extension PrettyPrinter {
     switch member {
     case .custom(let ccc):
       outputAsCanonical(ccc)
-    case .range(let a, let b):
-      output(a._canonicalBase)
+    case .range(let r):
+      output(r.lhs._canonicalBase)
       output("-")
-      output(b._canonicalBase)
+      output(r.rhs._canonicalBase)
     case .atom(let a):
       output(a._canonicalBase)
     case .setOperation:

Sources/_MatchingEngine/Regex/Printing/PrintAsPattern.swift

@@ -139,16 +139,16 @@ extension PrettyPrinter {
     switch member {
     case .custom(let ccc):
       printAsPattern(ccc)
-    case .range(let a, let b):
-      if let lhs = a.literalStringValue,
-         let rhs = b.literalStringValue {
+    case .range(let r):
+      if let lhs = r.lhs.literalStringValue,
+         let rhs = r.rhs.literalStringValue {
         indent()
         output(lhs._quoted)
         output("...")
         output(rhs._quoted)
         terminateLine()
       } else {
-        print("// TODO: Range \(a) to \(b)")
+        print("// TODO: Range \(r.lhs) to \(r.rhs)")
       }
     case .atom(let a):
       if let s = a.literalStringValue {

Sources/_MatchingEngine/Utility/Misc_2.swift

@@ -93,6 +93,10 @@ extension Collection {
     distance(from: startIndex, to: i)
   }
 
+  public func offsets(of r: Range<Index>) -> Range<Int> {
+    offset(of: r.lowerBound) ..< offset(of: r.upperBound)
+  }
+
   public func convertByOffset<
     C: Collection
   >(_ range: Range<Index>, in c: C) -> Range<C.Index> {

Sources/_StringProcessing/ASTBuilder.swift

@@ -17,7 +17,9 @@ AST.
 import _MatchingEngine
 
 func alt(_ asts: [AST]) -> AST {
-  .alternation(.init(asts, .fake))
+  return .alternation(
+    .init(asts, pipes: Array(repeating: .fake, count: asts.count - 1))
+  )
 }
 func alt(_ asts: AST...) -> AST {
   alt(asts)
@@ -90,9 +92,9 @@ public func atomicScriptRun(_ child: AST) -> AST {
   group(.atomicScriptRun, child)
 }
 func changeMatchingOptions(
-  _ seq: AST.MatchingOptionSequence, hasImplicitScope: Bool, _ child: AST
+  _ seq: AST.MatchingOptionSequence, isIsolated: Bool, _ child: AST
 ) -> AST {
-  group(.changeMatchingOptions(seq, hasImplicitScope: hasImplicitScope), child)
+  group(.changeMatchingOptions(seq, isIsolated: isIsolated), child)
 }
 
 func matchingOptions(
@@ -261,9 +263,13 @@ func prop_m(
 ) -> AST.CustomCharacterClass.Member {
   atom_m(.property(.init(kind, isInverted: inverted, isPOSIX: false)))
 }
+func range_m(
+  _ lower: AST.Atom, _ upper: AST.Atom
+) -> AST.CustomCharacterClass.Member {
+  .range(.init(lower, .fake, upper))
+}
 func range_m(
   _ lower: AST.Atom.Kind, _ upper: AST.Atom.Kind
 ) -> AST.CustomCharacterClass.Member {
-  .range(atom_a(lower), atom_a(upper))
+  range_m(atom_a(lower), atom_a(upper))
 }
-

Sources/_StringProcessing/CharacterClass.swift

@@ -384,10 +384,10 @@ extension AST.CustomCharacterClass {
             return nil
           }
           result.append(.characterClass(cc))
-        case .range(let lhs, let rhs):
+        case .range(let r):
           result.append(.range(
-            lhs.literalCharacterValue! ...
-            rhs.literalCharacterValue!))
+            r.lhs.literalCharacterValue! ...
+            r.rhs.literalCharacterValue!))
 
         case .atom(let a):
           if let cc = a.characterClass {

Sources/_StringProcessing/ConsumerInterface.swift

@@ -117,12 +117,12 @@ extension AST.CustomCharacterClass.Member {
     case .custom(let ccc):
       return try ccc.generateConsumer(opts)
 
-    case .range(let lower, let upper):
-      guard let lhs = lower.literalCharacterValue else {
-        throw unsupported("\(lower) in range")
+    case .range(let r):
+      guard let lhs = r.lhs.literalCharacterValue else {
+        throw unsupported("\(r.lhs) in range")
       }
-      guard let rhs = upper.literalCharacterValue else {
-        throw unsupported("\(upper) in range")
+      guard let rhs = r.rhs.literalCharacterValue else {
+        throw unsupported("\(r.rhs) in range")
       }
 
       return { input, bounds in