Un-revert "[stdlib] String index interchange, etc." (#10812)

I failed to merge the upstream changes to swift-corelibs-foundation at the same
time as I merged that #9806, and it broke on linux. Going to get it right this
time.

Author: Dave Abrahams

stdlib/public/SDK/Foundation/ExtraStringAPIs.swift

@@ -10,25 +10,24 @@
 //
 //===----------------------------------------------------------------------===//
 
-// Random access for String.UTF16View, only when Foundation is
-// imported.  Making this API dependent on Foundation decouples the
-// Swift core from a UTF16 representation.
-extension String.UTF16View.Index : Strideable {
+extension String.UTF16View.Index {
   /// Construct from an integer offset.
+  @available(swift, deprecated: 3.2)
+  @available(swift, obsoleted: 4.0)
   public init(_ offset: Int) {
     _precondition(offset >= 0, "Negative UTF16 index offset not allowed")
     self.init(_offset: offset)
   }
 
+  @available(swift, deprecated: 3.2)
+  @available(swift, obsoleted: 4.0)
   public func distance(to other: String.UTF16View.Index) -> Int {
     return _offset.distance(to: other._offset)
   }
 
+  @available(swift, deprecated: 3.2)
+  @available(swift, obsoleted: 4.0)
   public func advanced(by n: Int) -> String.UTF16View.Index {
     return String.UTF16View.Index(_offset.advanced(by: n))
   }
 }
-
-extension String.UTF16View : RandomAccessCollection {}
-extension String.UTF16View.Indices : RandomAccessCollection {}
-

stdlib/public/SDK/Foundation/NSRange.swift

@@ -151,8 +151,8 @@ extension NSRange {
   where R.Bound == S.Index, S.Index == String.Index {
     let r = region.relative(to: target)
     self = NSRange(
-      location: r.lowerBound._utf16Index - target.startIndex._utf16Index,
-      length: r.upperBound._utf16Index - r.lowerBound._utf16Index
+      location: r.lowerBound.encodedOffset - target.startIndex.encodedOffset,
+      length: r.upperBound.encodedOffset - r.lowerBound.encodedOffset
     )
   }
 

stdlib/public/SDK/Foundation/NSStringAPI.swift

@@ -32,8 +32,8 @@ func _toNSArray<T, U : AnyObject>(_ a: [T], f: (T) -> U) -> NSArray {
 
 func _toNSRange(_ r: Range<String.Index>) -> NSRange {
   return NSRange(
-    location: r.lowerBound._utf16Index,
-    length: r.upperBound._utf16Index - r.lowerBound._utf16Index)
+    location: r.lowerBound.encodedOffset,
+    length: r.upperBound.encodedOffset - r.lowerBound.encodedOffset)
 }
 
 // We only need this for UnsafeMutablePointer, but there's not currently a way
@@ -72,10 +72,7 @@ extension String {
   /// Return an `Index` corresponding to the given offset in our UTF-16
   /// representation.
   func _index(_ utf16Index: Int) -> Index {
-    return Index(
-      _base: String.UnicodeScalarView.Index(_position: utf16Index),
-      in: characters
-    )
+    return Index(encodedOffset: utf16Index)
   }
 
   /// Return a `Range<Index>` corresponding to the given `NSRange` of
@@ -1260,7 +1257,7 @@ extension String {
   public
   func rangeOfComposedCharacterSequence(at anIndex: Index) -> Range<Index> {
     return _range(
-      _ns.rangeOfComposedCharacterSequence(at: anIndex._utf16Index))
+      _ns.rangeOfComposedCharacterSequence(at: anIndex.encodedOffset))
   }
 
   // - (NSRange)rangeOfComposedCharacterSequencesForRange:(NSRange)range
@@ -1610,15 +1607,15 @@ extension String {
   /// Returns a new string containing the characters of the
   /// `String` from the one at a given index to the end.
   public func substring(from index: Index) -> String {
-    return _ns.substring(from: index._utf16Index)
+    return _ns.substring(from: index.encodedOffset)
   }
 
   // - (NSString *)substringToIndex:(NSUInteger)anIndex
 
   /// Returns a new string containing the characters of the
   /// `String` up to, but not including, the one at a given index.
   public func substring(to index: Index) -> String {
-    return _ns.substring(to: index._utf16Index)
+    return _ns.substring(to: index.encodedOffset)
   }
 
   // - (NSString *)substringWithRange:(NSRange)aRange

stdlib/public/SDK/Foundation/URLComponents.swift

@@ -194,8 +194,8 @@ public struct URLComponents : ReferenceConvertible, Hashable, Equatable, _Mutabl
     private func _toStringRange(_ r : NSRange) -> Range<String.Index>? {
         guard r.location != NSNotFound else { return nil }
 
-        let utf16Start = String.UTF16View.Index(_offset: r.location)
-        let utf16End = String.UTF16View.Index(_offset: r.location + r.length)
+        let utf16Start = String.UTF16View.Index(encodedOffset: r.location)
+        let utf16End = String.UTF16View.Index(encodedOffset: r.location + r.length)
 
         guard let s = self.string else { return nil }
         guard let start = String.Index(utf16Start, within: s) else { return nil }

stdlib/public/core/CMakeLists.txt

@@ -126,6 +126,7 @@ set(SWIFTLIB_ESSENTIAL
   StringBuffer.swift
   StringComparable.swift
   StringCore.swift
+  StringIndex.swift
   StringInterpolation.swift
   StringLegacy.swift
   StringRangeReplaceableCollection.swift.gyb

stdlib/public/core/GroupInfo.json

@@ -17,6 +17,7 @@
     "StringComparable.swift",
     "StringCore.swift",
     "StringHashable.swift",
+    "StringIndex.swift",
     "StringIndexConversions.swift",
     "StringInterpolation.swift",
     "StringLegacy.swift",

stdlib/public/core/StringCharacterView.swift

@@ -63,7 +63,7 @@ extension String {
 
     /// The offset of this view's `_core` from an original core. This works
     /// around the fact that `_StringCore` is always zero-indexed.
-    /// `_coreOffset` should be subtracted from `UnicodeScalarIndex._position`
+    /// `_coreOffset` should be subtracted from `UnicodeScalarIndex.encodedOffset`
     /// before that value is used as a `_core` index.
     @_versioned
     internal var _coreOffset: Int
@@ -178,97 +178,66 @@ extension String.CharacterView : BidirectionalCollection {
     return UnicodeScalarView(_core, coreOffset: _coreOffset)
   }
 
-  /// A position in a string's `CharacterView` instance.
-  ///
-  /// You can convert between indices of the different string views by using
-  /// conversion initializers and the `samePosition(in:)` method overloads.
-  /// The following example finds the index of the first space in the string's
-  /// character view and then converts that to the same position in the UTF-8
-  /// view:
-  ///
-  ///     let hearts = "Hearts <3 ♥︎ 💘"
-  ///     if let i = hearts.characters.index(of: " ") {
-  ///         let j = i.samePosition(in: hearts.utf8)
-  ///         print(Array(hearts.utf8[..<j]))
-  ///     }
-  ///     // Prints "[72, 101, 97, 114, 116, 115]"
-  public struct Index : Comparable, CustomPlaygroundQuickLookable {
-    public // SPI(Foundation)    
-    init(_base: String.UnicodeScalarView.Index, in c: String.CharacterView) {
-      self._base = _base
-      self._countUTF16 = c._measureExtendedGraphemeClusterForward(from: _base)
-    }
-
-    internal init(_base: UnicodeScalarView.Index, _countUTF16: Int) {
-      self._base = _base
-      self._countUTF16 = _countUTF16
-    }
-
-    internal let _base: UnicodeScalarView.Index
-
-    /// The count of this extended grapheme cluster in UTF-16 code units.
-    internal let _countUTF16: Int
-
-    /// The integer offset of this index in UTF-16 code units.
-    public // SPI(Foundation)
-    var _utf16Index: Int {
-      return _base._position
-    }
-
-    /// The one past end index for this extended grapheme cluster in Unicode
-    /// scalars.
-    internal var _endBase: UnicodeScalarView.Index {
-      return UnicodeScalarView.Index(_position: _utf16Index + _countUTF16)
-    }
-
-    public var customPlaygroundQuickLook: PlaygroundQuickLook {
-      return .int(Int64(_utf16Index))
-    }
-  }
-
+  public typealias Index = String.Index
   public typealias IndexDistance = Int
 
   /// The position of the first character in a nonempty character view.
   /// 
   /// In an empty character view, `startIndex` is equal to `endIndex`.
   public var startIndex: Index {
-    return Index(_base: unicodeScalars.startIndex, in: self)
+    return unicodeScalars.startIndex
   }
 
   /// A character view's "past the end" position---that is, the position one
   /// greater than the last valid subscript argument.
   ///
   /// In an empty character view, `endIndex` is equal to `startIndex`.
   public var endIndex: Index {
-    return Index(_base: unicodeScalars.endIndex, in: self)
+    return unicodeScalars.endIndex
   }
 
+  internal func _index(atEncodedOffset n: Int) -> Index {
+    let stride = _measureExtendedGraphemeClusterForward(
+      from: Index(encodedOffset: n))
+    return Index(encodedOffset: n, .character(stride: UInt16(stride)))
+  }
+  
   /// Returns the next consecutive position after `i`.
   ///
   /// - Precondition: The next position is valid.
   public func index(after i: Index) -> Index {
-    _precondition(i._base < unicodeScalars.endIndex,
+    _precondition(
+      i < unicodeScalars.endIndex,
       "cannot increment beyond endIndex")
-    _precondition(i._base >= unicodeScalars.startIndex,
+    
+    _precondition(
+      i >= unicodeScalars.startIndex,
       "cannot increment invalid index")
-    return Index(_base: i._endBase, in: self)
+
+    var j = i
+    while true {
+      if case .character(let oldStride) = j._cache {
+        return _index(atEncodedOffset: j.encodedOffset + Int(oldStride))
+      }
+      j = _index(atEncodedOffset: j.encodedOffset)
+    }
   }
 
   /// Returns the previous consecutive position before `i`.
   ///
   /// - Precondition: The previous position is valid.
   public func index(before i: Index) -> Index {
-    _precondition(i._base > unicodeScalars.startIndex,
+    _precondition(i > unicodeScalars.startIndex,
       "cannot decrement before startIndex")
-    _precondition(i._base <= unicodeScalars.endIndex,
+    _precondition(i <= unicodeScalars.endIndex,
       "cannot decrement invalid index")
-    let predecessorLengthUTF16 =
-      _measureExtendedGraphemeClusterBackward(from: i._base)
+    
+    let stride = _measureExtendedGraphemeClusterBackward(
+      from: Index(encodedOffset: i.encodedOffset))
+    
     return Index(
-      _base: UnicodeScalarView.Index(
-        _position: i._utf16Index - predecessorLengthUTF16
-      ),
-      in: self
+      encodedOffset: i.encodedOffset &- stride,
+      .character(stride: numericCast(stride))
     )
   }
 
@@ -365,8 +334,8 @@ extension String.CharacterView : BidirectionalCollection {
   internal func _measureExtendedGraphemeClusterForward(
     from start: UnicodeScalarView.Index
   ) -> Int {
-    let startPosition = start._position
-    let endPosition = unicodeScalars.endIndex._position
+    let startPosition = start.encodedOffset
+    let endPosition = unicodeScalars.endIndex.encodedOffset
 
     // No more graphemes
     if startPosition == endPosition {
@@ -379,7 +348,7 @@ extension String.CharacterView : BidirectionalCollection {
     }
 
     // Our relative offset from the _StringCore's baseAddress pointer. If our
-    // _core is not a substring, this is the same as start._position. Otherwise,
+    // _core is not a substring, this is the same as start.encodedOffset. Otherwise,
     // it is the code unit relative offset into the substring and not the
     // absolute offset into the outer string.
     let startOffset = startPosition - _coreOffset
@@ -419,7 +388,7 @@ extension String.CharacterView : BidirectionalCollection {
   func _measureExtendedGraphemeClusterForwardSlow(
     startOffset: Int
   ) -> Int {
-    let endOffset = unicodeScalars.endIndex._position - _coreOffset
+    let endOffset = unicodeScalars.endIndex.encodedOffset - _coreOffset
     let numCodeUnits = endOffset - startOffset
     _sanityCheck(numCodeUnits >= 2, "should have at least two code units")
 
@@ -501,8 +470,8 @@ extension String.CharacterView : BidirectionalCollection {
   internal func _measureExtendedGraphemeClusterBackward(
     from end: UnicodeScalarView.Index
   ) -> Int {
-    let startPosition = unicodeScalars.startIndex._position
-    let endPosition = end._position
+    let startPosition = unicodeScalars.startIndex.encodedOffset
+    let endPosition = end.encodedOffset
 
     // No more graphemes
     if startPosition == endPosition {
@@ -559,7 +528,7 @@ extension String.CharacterView : BidirectionalCollection {
   ) -> Int {
     let startOffset = 0
     let numCodeUnits = endOffset - startOffset
-    _sanityCheck(unicodeScalars.startIndex._position - _coreOffset == 0,
+    _sanityCheck(unicodeScalars.startIndex.encodedOffset - _coreOffset == 0,
       "position/offset mismatch in _StringCore as a substring")
     _sanityCheck(numCodeUnits >= 2,
       "should have at least two code units")
@@ -643,31 +612,38 @@ extension String.CharacterView : BidirectionalCollection {
   ///
   /// - Parameter position: A valid index of the character view. `position`
   ///   must be less than the view's end index.
-  public subscript(i: Index) -> Character {
-    if i._countUTF16 == 1 {
-      // For single-code-unit graphemes, we can construct a Character directly
-      // from a single unicode scalar (if sub-surrogate).
-      let relativeOffset = i._base._position - _coreOffset
-      if _core.isASCII {
-        let asciiBuffer = _core.asciiBuffer._unsafelyUnwrappedUnchecked
-        // Bounds checks in an UnsafeBufferPointer (asciiBuffer) are only
-        // performed in Debug mode, so they need to be duplicated here.
-        // Falling back to the non-optimal behavior in the case they don't
-        // pass.
-        if relativeOffset >= asciiBuffer.startIndex &&
-          relativeOffset < asciiBuffer.endIndex {
-          return Character(Unicode.Scalar(asciiBuffer[relativeOffset]))
-        }
-      } else if _core._baseAddress != nil {
-        let cu = _core._nthContiguous(relativeOffset)
-        // Only constructible if sub-surrogate
-        if (cu < 0xd800) {
-          return Character(Unicode.Scalar(cu)._unsafelyUnwrappedUnchecked)
+  public subscript(i_: Index) -> Character {
+    var i = i_
+    while true {
+      if case .character(let stride) = i._cache {
+        if _fastPath(stride == 1) {
+          // For single-code-unit graphemes, we can construct a Character directly
+          // from a single unicode scalar (if sub-surrogate).
+          let relativeOffset = i.encodedOffset - _coreOffset
+          if _core.isASCII {
+            let asciiBuffer = _core.asciiBuffer._unsafelyUnwrappedUnchecked
+            // Bounds checks in an UnsafeBufferPointer (asciiBuffer) are only
+            // performed in Debug mode, so they need to be duplicated here.
+            // Falling back to the non-optimal behavior in the case they don't
+            // pass.
+            if relativeOffset >= asciiBuffer.startIndex &&
+            relativeOffset < asciiBuffer.endIndex {
+              return Character(Unicode.Scalar(asciiBuffer[relativeOffset]))
+            }
+          } else if _core._baseAddress != nil {
+            let cu = _core._nthContiguous(relativeOffset)
+            // Only constructible if sub-surrogate
+            if (cu < 0xd800) {
+              return Character(Unicode.Scalar(cu)._unsafelyUnwrappedUnchecked)
+            }
+          }
         }
+        
+        let s = self[i..<Index(encodedOffset: i.encodedOffset + Int(stride))]
+        return Character(s._ephemeralContent)
       }
+      i = _index(atEncodedOffset: i.encodedOffset)
     }
-
-    return Character(String(unicodeScalars[i._base..<i._endBase]))
   }
 }
 
@@ -696,8 +672,8 @@ extension String.CharacterView : RangeReplaceableCollection {
     with newElements: C
   ) where C : Collection, C.Element == Character {
     let rawSubRange: Range<Int> =
-      bounds.lowerBound._base._position - _coreOffset
-      ..< bounds.upperBound._base._position - _coreOffset
+      bounds.lowerBound.encodedOffset - _coreOffset
+      ..< bounds.upperBound.encodedOffset - _coreOffset
     let lazyUTF16 = newElements.lazy.flatMap { $0.utf16 }
     _core.replaceSubrange(rawSubRange, with: lazyUTF16)
   }
@@ -764,9 +740,9 @@ extension String.CharacterView {
   /// - Complexity: O(*n*) if the underlying string is bridged from
   ///   Objective-C, where *n* is the length of the string; otherwise, O(1).
   public subscript(bounds: Range<Index>) -> String.CharacterView {
-    let unicodeScalarRange = bounds.lowerBound._base..<bounds.upperBound._base
-    return String.CharacterView(unicodeScalars[unicodeScalarRange]._core,
-      coreOffset: unicodeScalarRange.lowerBound._position)
+    return String.CharacterView(
+      unicodeScalars[bounds]._core,
+      coreOffset: bounds.lowerBound.encodedOffset)
   }
 }
 

stdlib/public/core/StringCore.swift

@@ -428,7 +428,6 @@ public struct _StringCore {
       // In order to grow the substring in place, this _StringCore should point
       // at the substring at the end of a _StringBuffer.  Otherwise, some other
       // String is using parts of the buffer beyond our last byte.
-      let usedStart = _pointer(toElementAt:0)
       let usedEnd = _pointer(toElementAt:count)
 
       // Attempt to claim unused capacity in the buffer