rdar://107955097 (FoundationPreview: Batch move string API (continued)) (#34)

* rdar://107955097 (FoundationPreview: Batch move string API (continued))

- Move localized uppercase and lowercase to FoundationLocalization
- if-def out of CharacterSet from FoundationPreview. It's not implemented at all there, and having a no-op stub is misleading

* rdar://107955097 (FoundationPreview: Batch move string API (continued))

- Move components separated by string and range of string functions

* rdar://107955097 (FoundationPreview: Batch move string API (continued))

Enable snake case options for JSON encoder and decoder. We haven't been able to enable this option because it needed `CharacterSet`, which hasn't been properly implemented for FoundationPreview. Now that we have `BuiltInUnicodeScalarSet`, which mirrors `CharacterSet`, we can switch to that and enable the options.

* rdar://107955097 (FoundationPreview: Batch move string API (continued))

- Move `StringProtocol.lineRange(for:)` and `paragraphRange(for:)` to FoundationEssentials
- Rename String+Regex.swift to RegexPatternCache.swift
- Consolidate extensions on various String family members and remove one redundant swift file

---------

Co-authored-by: I-Ting Tina Liu <iting_liu@apple.com>

Author: iCharlesHu

Sources/FoundationEssentials/BuiltInUnicodeScalarSet.swift

@@ -83,10 +83,10 @@ internal struct BuiltInUnicodeScalarSet {
         return new
     }
 
-    static let uppercaseLetter = Self.init(type: .uppercaseLetter)
-    static let lowercaseLetter = Self.init(type: .lowercaseLetter)
-    static let caseIgnorable = Self.init(type: .caseIgnorable)
-    static let graphemeExtend = Self.init(type: .graphemeExtend)
-    static let canonicalDecomposable = Self.init(type: .canonicalDecomposable)
+    static let uppercaseLetters = Self.init(type: .uppercaseLetter)
+    static let lowercaseLetters = Self.init(type: .lowercaseLetter)
+    static let caseIgnorables = Self.init(type: .caseIgnorable)
+    static let graphemeExtends = Self.init(type: .graphemeExtend)
+    static let canonicalDecomposables = Self.init(type: .canonicalDecomposable)
 }
 

Sources/FoundationEssentials/CharacterSet+Stub.swift

@@ -117,12 +117,9 @@ open class JSONEncoder {
         /// Use the keys specified by each type. This is the default strategy.
         case useDefaultKeys
 
-#if FOUNDATION_FRAMEWORK
-        // TODO: Reenable this option once String.rangeOfCharacter(from:) is moved
-
         /// Convert from "camelCaseKeys" to "snake_case_keys" before writing a key to JSON payload.
         ///
-        /// Capital characters are determined by testing membership in `CharacterSet.uppercaseLetters` and `CharacterSet.lowercaseLetters` (Unicode General Categories Lu and Lt).
+        /// Capital characters are determined by testing membership in Unicode General Categories Lu and Lt.
         /// The conversion to lower case uses `Locale.system`, also known as the ICU "root" locale. This means the result is consistent regardless of the current user's locale and language preferences.
         ///
         /// Converting from camel case to snake case:
@@ -135,7 +132,6 @@ open class JSONEncoder {
         ///
         /// - Note: Using a key encoding strategy has a nominal performance cost, as each string key has to be converted.
         case convertToSnakeCase
-#endif
 
         /// Provide a custom conversion to the key in the encoded JSON from the keys specified by the encoded types.
         /// The full path to the current encoding position is provided for context (in case you need to locate this key within the payload). The returned key is used in place of the last component in the coding path before encoding.
@@ -157,13 +153,13 @@ open class JSONEncoder {
             var searchRange = stringKey.index(after: wordStart)..<stringKey.endIndex
 
             // Find next uppercase character
-            while let upperCaseRange = stringKey.rangeOfCharacter(from: CharacterSet.uppercaseLetters, options: [], range: searchRange) {
+            while let upperCaseRange = stringKey[searchRange]._rangeOfCharacter(from: BuiltInUnicodeScalarSet.uppercaseLetters, options: []) {
                 let untilUpperCase = wordStart..<upperCaseRange.lowerBound
                 words.append(untilUpperCase)
 
                 // Find next lowercase character
                 searchRange = upperCaseRange.lowerBound..<searchRange.upperBound
-                guard let lowerCaseRange = stringKey.rangeOfCharacter(from: CharacterSet.lowercaseLetters, options: [], range: searchRange) else {
+                guard let lowerCaseRange = stringKey[searchRange]._rangeOfCharacter(from: BuiltInUnicodeScalarSet.lowercaseLetters, options: []) else {
                     // There are no more lower case letters. Just end here.
                     wordStart = searchRange.lowerBound
                     break
@@ -741,11 +737,9 @@ private struct _JSONKeyedEncodingContainer<K : CodingKey> : KeyedEncodingContain
         switch encoder.options.keyEncodingStrategy {
         case .useDefaultKeys:
             return key.stringValue
-#if FOUNDATION_FRAMEWORK
         case .convertToSnakeCase:
             let newKeyString = JSONEncoder.KeyEncodingStrategy._convertToSnakeCase(key.stringValue)
             return newKeyString
-#endif // FOUNDATION_FRAMEWORK
         case .custom(let converter):
             return converter(codingPathNode.path(with: key)).stringValue
         }

Sources/FoundationEssentials/JSON/JSONEncoder.swift

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+extension BidirectionalCollection where Index == String.Index {
+    internal func _alignIndex(roundingDown i: Index) -> Index {
+        return i < endIndex ? index(before: index(after: i)) : i
+    }
+
+    internal func _alignIndex(roundingUp i: Index) -> Index {
+        let truncated = _alignIndex(roundingDown: i)
+        if i > truncated && i < endIndex {
+            return index(after: i)
+        } else {
+            return i
+        }
+    }
+
+    internal func _boundaryAlignedRange<R: RangeExpression>(_ r: R) -> Range<Index> where R.Bound == String.Index {
+        let range = r.relative(to: self)
+        return _alignIndex(roundingDown: range.lowerBound)..<_alignIndex(roundingUp: range.upperBound)
+    }
+
+    internal func _checkRange(_ r: Range<Index>) -> Range<Index>? {
+        guard r.lowerBound >= startIndex, r.upperBound <= endIndex else {
+            return nil
+        }
+        return r
+    }
+}

Sources/FoundationEssentials/String/BidirectionalCollection+StringIndex.swift

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+struct RegexPatternCache: @unchecked Sendable {
+    private struct Key : Sendable, Hashable {
+        var pattern: String
+        var caseInsensitive: Bool
+    }
+
+    private let _lock: LockedState<[Key: Regex<AnyRegexOutput>]>
+
+    static let cache = RegexPatternCache()
+
+    fileprivate init() {
+        _lock = LockedState(initialState: .init())
+    }
+
+    func regex(for pattern: String, caseInsensitive: Bool) throws -> Regex<AnyRegexOutput>? {
+
+        let key = Key(pattern: pattern, caseInsensitive: caseInsensitive)
+
+        return try _lock.withLock { cache in
+
+            if let cached = cache[key] {
+                return cached
+            }
+
+            var r = try Regex(pattern)
+            if caseInsensitive {
+                r = r.ignoresCase()
+            }
+            cache[key] = r
+            return r
+        }
+    }
+}

Sources/FoundationEssentials/String/RegexPatternCache.swift

@@ -566,7 +566,11 @@ extension Character : _StringCompareOptionsConvertible {
     }
 
     var isExtendCharacter: Bool {
-        return _isExtendCharacter
+        guard !self.isASCII else {
+            return false
+        }
+
+        return unicodeScalars.allSatisfy { $0._isGraphemeExtend }
     }
 
 }
@@ -674,10 +678,103 @@ extension Substring {
         return result
     }
 
+#if FOUNDATION_FRAMEWORK
     func _rangeOfCharacter(from set: CharacterSet, options: String.CompareOptions) -> Range<Index>? {
         guard !isEmpty else { return nil }
 
-        return unicodeScalars._rangeOfCharacter(from: set, anchored: options.contains(.anchored), backwards: options.contains(.backwards))
+        return unicodeScalars._rangeOfCharacter(anchored: options.contains(.anchored), backwards: options.contains(.backwards), matchingPredicate: set.contains)
+    }
+#endif
+
+    func _rangeOfCharacter(from set: BuiltInUnicodeScalarSet, options: String.CompareOptions) -> Range<Index>? {
+        guard !isEmpty else { return nil }
+
+        return unicodeScalars._rangeOfCharacter(anchored: options.contains(.anchored), backwards: options.contains(.backwards), matchingPredicate: set.contains)
+    }
+
+    func _range(of strToFind: Substring, options: String.CompareOptions) throws -> Range<Index>? {
+        if options.contains(.regularExpression) {
+            guard let regex = try RegexPatternCache.cache.regex(for: String(strToFind), caseInsensitive: options.contains(.caseInsensitive)) else {
+                return nil
+            }
+
+            if options.contains(.anchored) {
+                guard let match = prefixMatch(of: regex) else { return nil }
+                return match.range
+            } else {
+                guard let match = firstMatch(of: regex) else { return nil }
+                return match.range
+            }
+        }
+
+        guard !isEmpty, !strToFind.isEmpty else {
+            return nil
+        }
+
+        let toHalfWidth = options.contains(.widthInsensitive)
+        let diacriticsInsensitive = options.contains(.diacriticInsensitive)
+        let caseFold = options.contains(.caseInsensitive)
+        let anchored = options.contains(.anchored)
+        let backwards = options.contains(.backwards)
+
+        let result: Range<Index>?
+        if options.contains(.literal) {
+            result = unicodeScalars._range(of: strToFind.unicodeScalars, toHalfWidth: toHalfWidth, diacriticsInsensitive: diacriticsInsensitive, caseFold: caseFold, anchored: anchored, backwards: backwards)
+        } else if !toHalfWidth && !diacriticsInsensitive && !caseFold {
+            // Fast path: iterate through UTF8 view when we don't need to transform string content
+            guard let utf8Result = utf8._range(of: strToFind.utf8, anchored: anchored, backwards: backwards) else {
+                 return nil
+            }
+
+            // Adjust the index to that of the original slice since we called `makeContiguousUTF8` before
+            guard let lower = String.Index(utf8Result.lowerBound, within: self), let upper = String.Index(utf8Result.upperBound, within: self) else {
+                return nil
+            }
+            result = lower..<upper
+
+        } else if _isASCII && strToFind._isASCII {
+            // Fast path: Iterate utf8 without having to decode as unicode scalars. In this case only case folding matters.
+
+            guard let utf8Result = utf8._range(of: strToFind.utf8, toHalfWidth: false, diacriticsInsensitive: false, caseFold: caseFold, anchored: anchored, backwards: backwards) else {
+                return nil
+            }
+
+            // Adjust the index to that of the original slice since we called `makeContiguousUTF8` before
+            guard let lower = String.Index(utf8Result.lowerBound, within: self), let upper = String.Index(utf8Result.upperBound, within: self) else {
+                return nil
+            }
+            result = lower..<upper
+
+        } else {
+            result = _range(of: strToFind, toHalfWidth: toHalfWidth, diacriticsInsensitive: diacriticsInsensitive, caseFold: caseFold, anchored: anchored, backwards: backwards)
+        }
+
+        return result
+    }
+
+    var _isASCII: Bool {
+        var mutated = self
+        return mutated.withUTF8 {
+            _allASCII($0)
+        }
+    }
+
+    func _components(separatedBy separator: Substring, options: String.CompareOptions = []) throws -> [String] {
+        var result = [String]()
+        var searchStart = startIndex
+        while searchStart < endIndex {
+            let r = try self[searchStart...]._range(of: separator, options: options)
+            guard let r, !r.isEmpty else {
+                break
+            }
+
+            result.append(String(self[searchStart ..< r.lowerBound]))
+            searchStart = r.upperBound
+        }
+
+        result.append(String(self[searchStart..<endIndex]))
+
+        return result
     }
 }
 
@@ -703,6 +800,42 @@ extension Substring.UnicodeScalarView {
 
         return ComparisonResult(stringIndex: idx1, idx2: idx2, endIndex1: endIndex, endIndex2: other.endIndex)
     }
+
+    func _rangeOfCharacter(anchored: Bool, backwards: Bool, matchingPredicate predicate: (Unicode.Scalar) -> Bool) -> Range<Index>? {
+        guard !isEmpty else { return nil }
+
+        let fromLoc: String.Index
+        let toLoc: String.Index
+        let step: Int
+        if backwards {
+            fromLoc = index(before: endIndex)
+            toLoc = anchored ? fromLoc : startIndex
+            step = -1
+        } else {
+            fromLoc = startIndex
+            toLoc = anchored ? fromLoc : index(before: endIndex)
+            step = 1
+        }
+
+        var done = false
+        var found = false
+
+        var idx = fromLoc
+        while !done {
+            let ch = self[idx]
+            if predicate(ch) {
+                done = true
+                found = true
+            } else if idx == toLoc {
+                done = true
+            } else {
+                formIndex(&idx, offsetBy: step)
+            }
+        }
+
+        guard found else { return nil }
+        return idx..<index(after: idx)
+    }
 }
 
 // MARK: - ComparisonResult Extension
@@ -744,3 +877,34 @@ extension BidirectionalCollection {
         return idx
     }
 }
+
+// Borrowed from stdlib
+internal func _allASCII(_ input: UnsafeBufferPointer<UInt8>) -> Bool {
+    if input.isEmpty { return true }
+    let ptr = input.baseAddress.unsafelyUnwrapped
+    var i = 0
+
+    let count = input.count
+    let stride = MemoryLayout<UInt>.stride
+    let address = Int(bitPattern: ptr)
+
+    let wordASCIIMask = UInt(truncatingIfNeeded: 0x8080_8080_8080_8080 as UInt64)
+    let byteASCIIMask = UInt8(truncatingIfNeeded: wordASCIIMask)
+
+    while (address &+ i) % stride != 0 && i < count {
+        guard ptr[i] & byteASCIIMask == 0 else { return false }
+        i &+= 1
+    }
+
+    while (i &+ stride) <= count {
+        let word: UInt = UnsafePointer(bitPattern: address &+ i).unsafelyUnwrapped.pointee
+        guard word & wordASCIIMask == 0 else { return false }
+        i &+= stride
+    }
+
+    while i < count {
+        guard ptr[i] & byteASCIIMask == 0 else { return false }
+        i &+= 1
+    }
+    return true
+}

Sources/FoundationEssentials/String/String+Comparison.swift

@@ -16,9 +16,9 @@ extension String {
         var new = ""
         new.reserveCapacity(utf8.count)
 
-        let uppercaseSet = BuiltInUnicodeScalarSet.uppercaseLetter
-        let lowercaseSet = BuiltInUnicodeScalarSet.lowercaseLetter
-        let cfcaseIgnorableSet = BuiltInUnicodeScalarSet.caseIgnorable
+        let uppercaseSet = BuiltInUnicodeScalarSet.uppercaseLetters
+        let lowercaseSet = BuiltInUnicodeScalarSet.lowercaseLetters
+        let cfcaseIgnorableSet = BuiltInUnicodeScalarSet.caseIgnorables
 
         var isLastCased = false
         for scalar in unicodeScalars {