SR-3052. Incorrect string result for String methods with non latin symbols (swiftlang#706)

* added SR-3052 tests

* expanded non latin string test

* [SR-3052] utf8 encoding fix

Author: naithar

Foundation/NSCFString.swift

@@ -140,19 +140,18 @@ internal func _CFSwiftStringGetCharacters(_ str: AnyObject, range: CFRange, buff
 }
 
 internal func _CFSwiftStringGetBytes(_ str: AnyObject, encoding: CFStringEncoding, range: CFRange, buffer: UnsafeMutablePointer<UInt8>?, maxBufLen: CFIndex, usedBufLen: UnsafeMutablePointer<CFIndex>?) -> CFIndex {
+    let convertedLength: CFIndex
     switch encoding {
         // TODO: Don't treat many encodings like they are UTF8
     case CFStringEncoding(kCFStringEncodingUTF8), CFStringEncoding(kCFStringEncodingISOLatin1), CFStringEncoding(kCFStringEncodingMacRoman), CFStringEncoding(kCFStringEncodingASCII), CFStringEncoding(kCFStringEncodingNonLossyASCII):
-        let encodingView = (str as! NSString)._swiftObject.utf8
-        let start = encodingView.startIndex
+        let encodingView = (str as! NSString).substring(with: NSRange(range)).utf8
         if let buffer = buffer {
-            for idx in 0..<range.length {
-                let characterIndex = encodingView.index(start, offsetBy: idx + range.location)
-                let character = encodingView[characterIndex]
+            for (idx, character) in encodingView.enumerated() {
                 buffer.advanced(by: idx).initialize(to: character)
             }
         }
-        usedBufLen?.pointee = range.length
+        usedBufLen?.pointee = encodingView.count
+        convertedLength = encodingView.count
 
     case CFStringEncoding(kCFStringEncodingUTF16):
         let encodingView = (str as! NSString)._swiftObject.utf16
@@ -169,13 +168,13 @@ internal func _CFSwiftStringGetBytes(_ str: AnyObject, encoding: CFStringEncodin
         }
         // Every character was 2 bytes
         usedBufLen?.pointee = range.length * 2
-
+        convertedLength = range.length
 
     default:
         fatalError("Attempted to get bytes of a Swift string using an unsupported encoding")
     }
 
-    return range.length
+    return convertedLength
 }
 
 internal func _CFSwiftStringCreateWithSubstring(_ str: AnyObject, range: CFRange) -> Unmanaged<AnyObject> {

TestFoundation/TestNSString.swift

@@ -81,7 +81,11 @@ class TestNSString : XCTestCase {
             ("test_resolvingSymlinksInPath", test_resolvingSymlinksInPath),
             ("test_expandingTildeInPath", test_expandingTildeInPath),
             ("test_standardizingPath", test_standardizingPath),
-            ("test_removingPercentEncoding", test_removingPercentEncoding),
+            ("test_addingPercentEncoding", test_addingPercentEncoding),
+            ("test_removingPercentEncodingInLatin", test_removingPercentEncodingInLatin),
+            ("test_removingPercentEncodingInNonLatin", test_removingPercentEncodingInNonLatin),
+            ("test_removingPersentEncodingWithoutEncoding", test_removingPersentEncodingWithoutEncoding),
+            ("test_addingPercentEncodingAndBack", test_addingPercentEncodingAndBack),
             ("test_stringByAppendingPathExtension", test_stringByAppendingPathExtension),
             ("test_deletingPathExtension", test_deletingPathExtension),
             ("test_ExternalRepresentation", test_ExternalRepresentation),
@@ -867,13 +871,66 @@ class TestNSString : XCTestCase {
         }
     }
 
-    func test_removingPercentEncoding() {
+    func test_addingPercentEncoding() {
+        let s1 = "a b".addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+        XCTAssertEqual(s1, "a%20b")
+        
+        let s2 = "\u{0434}\u{043E}\u{043C}".addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+        XCTAssertEqual(s2, "%D0%B4%D0%BE%D0%BC")
+    }
+    
+    func test_removingPercentEncodingInLatin() {
         let s1 = "a%20b".removingPercentEncoding
         XCTAssertEqual(s1, "a b")
         let s2 = "a%1 b".removingPercentEncoding
         XCTAssertNil(s2, "returns nil for a string with an invalid percent encoding")
     }
 
+    func test_removingPercentEncodingInNonLatin() {
+        let s1 = "\u{043C}\u{043E}\u{0439}%20\u{0434}\u{043E}\u{043C}".removingPercentEncoding
+        XCTAssertEqual(s1, "\u{043C}\u{043E}\u{0439} \u{0434}\u{043E}\u{043C}")
+        
+        let s2 = "%D0%B4%D0%BE%D0%BC".removingPercentEncoding
+        XCTAssertEqual(s2, "\u{0434}\u{043E}\u{043C}")
+        
+        let s3 = "\u{00E0}a%1 b".removingPercentEncoding
+        XCTAssertNil(s3, "returns nil for a string with an invalid percent encoding")
+    }
+    
+    func test_removingPersentEncodingWithoutEncoding() {
+        let cyrillicString = "\u{0434}\u{043E}\u{043C}"
+        let cyrillicEscapedString = cyrillicString.removingPercentEncoding
+        XCTAssertEqual(cyrillicString, cyrillicEscapedString)
+        
+        let chineseString = "\u{623F}\u{5B50}"
+        let chineseEscapedString = chineseString.removingPercentEncoding
+        XCTAssertEqual(chineseString, chineseEscapedString)
+        
+        let arabicString = "\u{0645}\u{0646}\u{0632}\u{0644}"
+        let arabicEscapedString = arabicString.removingPercentEncoding
+        XCTAssertEqual(arabicString, arabicEscapedString)
+        
+        let randomString = "\u{00E0}\u{00E6}"
+        let randomEscapedString = randomString.removingPercentEncoding
+        XCTAssertEqual(randomString, randomEscapedString)
+        
+        let latinString = "home"
+        let latinEscapedString = latinString.removingPercentEncoding
+        XCTAssertEqual(latinString, latinEscapedString)
+    }
+    
+    func test_addingPercentEncodingAndBack() {
+        let latingString = "a b"
+        let escapedLatingString = latingString.addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+        let returnedLatingString = escapedLatingString?.removingPercentEncoding
+        XCTAssertEqual(returnedLatingString, latingString)
+        
+        let cyrillicString = "\u{0434}\u{043E}\u{043C}"
+        let escapedCyrillicString = cyrillicString.addingPercentEncoding(withAllowedCharacters: .alphanumerics)
+        let returnedCyrillicString = escapedCyrillicString?.removingPercentEncoding
+        XCTAssertEqual(returnedCyrillicString, cyrillicString)
+    }
+    
     func test_stringByAppendingPathExtension() {
         let values = [
             NSString(string: "/tmp/scratch.old") : "/tmp/scratch.old.tiff",