Skip to content

Commit bf3ffd2

Browse files
swift-ciswift-ci
authored
Merge pull request swiftlang#1657 from spevans/pr_json_numberparsing
2 parents 042b6cb + c69ec85 commit bf3ffd2

File tree

3 files changed

+257
-41
lines changed

3 files changed

+257
-41
lines changed

‎Foundation/JSONSerialization.swift

+136-39
Original file line numberDiff line numberDiff line change
@@ -769,52 +769,149 @@ private struct JSONReader {
769769
}
770770

771771
//MARK: - Number parsing
772-
static let numberCodePoints: [UInt8] = [
773-
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, // 0...9
774-
0x2E, 0x2D, 0x2B, 0x45, 0x65, // . - + E e
775-
]
772+
private static let ZERO = UInt8(ascii: "0")
773+
private static let ONE = UInt8(ascii: "1")
774+
private static let NINE = UInt8(ascii: "9")
775+
private static let MINUS = UInt8(ascii: "-")
776+
private static let PLUS = UInt8(ascii: "+")
777+
private static let LOWER_EXPONENT = UInt8(ascii: "e")
778+
private static let UPPER_EXPONENT = UInt8(ascii: "E")
779+
private static let DECIMAL_SEPARATOR = UInt8(ascii: ".")
780+
private static let allDigits = (ZERO...NINE)
781+
private static let oneToNine = (ONE...NINE)
782+
783+
private static let numberCodePoints: [UInt8] = {
784+
var numberCodePoints = Array(ZERO...NINE)
785+
numberCodePoints.append(contentsOf: [DECIMAL_SEPARATOR, MINUS, PLUS, LOWER_EXPONENT, UPPER_EXPONENT])
786+
return numberCodePoints
787+
}()
788+
776789

777790
func parseNumber(_ input: Index, options opt: JSONSerialization.ReadingOptions) throws -> (Any, Index)? {
778-
func parseTypedNumber(_ address: UnsafePointer<UInt8>, count: Int) -> (Any, IndexDistance)? {
779-
let temp_buffer_size = 64
780-
var temp_buffer = [Int8](repeating: 0, count: temp_buffer_size)
781-
return temp_buffer.withUnsafeMutableBufferPointer { (buffer: inout UnsafeMutableBufferPointer<Int8>) -> (Any, IndexDistance)? in
782-
memcpy(buffer.baseAddress!, address, min(count, temp_buffer_size - 1)) // ensure null termination
783-
784-
let startPointer = buffer.baseAddress!
785-
let intEndPointer = UnsafeMutablePointer<UnsafeMutablePointer<Int8>?>.allocate(capacity: 1)
786-
defer { intEndPointer.deallocate() }
787-
let doubleEndPointer = UnsafeMutablePointer<UnsafeMutablePointer<Int8>?>.allocate(capacity: 1)
788-
defer { doubleEndPointer.deallocate() }
789-
let intResult = strtol(startPointer, intEndPointer, 10)
790-
let intDistance = startPointer.distance(to: intEndPointer[0]!)
791-
let doubleResult = strtod(startPointer, doubleEndPointer)
792-
let doubleDistance = startPointer.distance(to: doubleEndPointer[0]!)
793-
794-
guard doubleDistance > 0 else { return nil }
795-
if intDistance == doubleDistance {
796-
return (NSNumber(value: intResult), intDistance)
791+
792+
var isNegative = false
793+
var string = ""
794+
var isInteger = true
795+
var exponent = 0
796+
var positiveExponent = true
797+
var index = input
798+
var digitCount: Int?
799+
var ascii: UInt8 = 0 // set by nextASCII()
800+
801+
// Validate the input is a valid JSON number, also gather the following
802+
// about the input: isNegative, isInteger, the exponent and if it is +/-,
803+
// and finally the count of digits including excluding an '.'
804+
func checkJSONNumber() throws -> Bool {
805+
// Return true if the next character is any one of the valid JSON number characters
806+
func nextASCII() -> Bool {
807+
guard let (ch, nextIndex) = source.takeASCII(index),
808+
JSONReader.numberCodePoints.contains(ch) else { return false }
809+
810+
index = nextIndex
811+
ascii = ch
812+
string.append(Character(UnicodeScalar(ascii)))
813+
return true
814+
}
815+
816+
// Consume as many digits as possible and return with the next non-digit
817+
// or nil if end of string.
818+
func readDigits() -> UInt8? {
819+
while let (ch, nextIndex) = source.takeASCII(index) {
820+
if !JSONReader.allDigits.contains(ch) {
821+
return ch
822+
}
823+
string.append(Character(UnicodeScalar(ch)))
824+
index = nextIndex
825+
}
826+
return nil
827+
}
828+
829+
guard nextASCII() else { return false }
830+
831+
if ascii == JSONReader.MINUS {
832+
isNegative = true
833+
guard nextASCII() else { return false }
834+
}
835+
836+
if JSONReader.oneToNine.contains(ascii) {
837+
guard let ch = readDigits() else { return true }
838+
ascii = ch
839+
if [ JSONReader.DECIMAL_SEPARATOR, JSONReader.LOWER_EXPONENT, JSONReader.UPPER_EXPONENT ].contains(ascii) {
840+
guard nextASCII() else { return false } // There should be at least one char as readDigits didnt remove the '.eE'
841+
}
842+
} else if ascii == JSONReader.ZERO {
843+
guard nextASCII() else { return true }
844+
} else {
845+
throw NSError(domain: NSCocoaErrorDomain, code: CocoaError.propertyListReadCorrupt.rawValue,
846+
userInfo: ["NSDebugDescription" : "Numbers must start with a 1-9 at character \(input)." ])
847+
}
848+
849+
if ascii == JSONReader.DECIMAL_SEPARATOR {
850+
isInteger = false
851+
guard readDigits() != nil else { return true }
852+
guard nextASCII() else { return true }
853+
} else if JSONReader.allDigits.contains(ascii) {
854+
throw NSError(domain: NSCocoaErrorDomain, code: CocoaError.propertyListReadCorrupt.rawValue,
855+
userInfo: ["NSDebugDescription" : "Leading zeros not allowed at character \(input)." ])
856+
}
857+
858+
digitCount = string.count - (isInteger ? 0 : 1) - (isNegative ? 1 : 0)
859+
guard ascii == JSONReader.LOWER_EXPONENT || ascii == JSONReader.UPPER_EXPONENT else {
860+
// End of valid number characters
861+
return true
862+
}
863+
digitCount = digitCount! - 1
864+
865+
// Process the exponent
866+
isInteger = false
867+
guard nextASCII() else { return false }
868+
if ascii == JSONReader.MINUS {
869+
positiveExponent = false
870+
guard nextASCII() else { return false }
871+
} else if ascii == JSONReader.PLUS {
872+
positiveExponent = true
873+
guard nextASCII() else { return false }
874+
}
875+
guard JSONReader.allDigits.contains(ascii) else { return false }
876+
exponent = Int(ascii - JSONReader.ZERO)
877+
while nextASCII() {
878+
guard JSONReader.allDigits.contains(ascii) else { return false } // Invalid exponent character
879+
exponent = (exponent * 10) + Int(ascii - JSONReader.ZERO)
880+
if exponent > 324 {
881+
// Exponent is too large to store in a Double
882+
return false
797883
}
798-
return (NSNumber(value: doubleResult), doubleDistance)
799884
}
885+
return true
800886
}
801-
802-
if source.encoding == .utf8 {
803-
return parseTypedNumber(source.buffer.baseAddress!.advanced(by: input), count: source.buffer.count - input).map { return ($0.0, input + $0.1) }
804-
}
805-
else {
806-
var numberCharacters = [UInt8]()
807-
var index = input
808-
while let (ascii, nextIndex) = source.takeASCII(index), JSONReader.numberCodePoints.contains(ascii) {
809-
numberCharacters.append(ascii)
810-
index = nextIndex
887+
888+
guard try checkJSONNumber() == true else { return nil }
889+
digitCount = digitCount ?? string.count - (isInteger ? 0 : 1) - (isNegative ? 1 : 0)
890+
891+
// Try Int64() or UInt64() first
892+
if isInteger {
893+
if isNegative {
894+
if digitCount! <= 19, let intValue = Int64(string) {
895+
return (NSNumber(value: intValue), index)
896+
}
897+
} else {
898+
if digitCount! <= 20, let uintValue = UInt64(string) {
899+
return (NSNumber(value: uintValue), index)
900+
}
811901
}
812-
numberCharacters.append(0)
813-
814-
return numberCharacters.withUnsafeBufferPointer {
815-
parseTypedNumber($0.baseAddress!, count: $0.count)
816-
}.map { return ($0.0, index) }
817902
}
903+
904+
// Decimal holds more digits of precision but a smaller exponent than Double
905+
// so try that if the exponent fits and there are more digits than Double can hold
906+
if digitCount! > 17 && exponent >= -128 && exponent <= 127,
907+
let decimal = Decimal(string: string), decimal.isFinite {
908+
return (NSDecimalNumber(decimal: decimal), index)
909+
}
910+
// Fall back to Double() for everything else
911+
if let doubleValue = Double(string) {
912+
return (NSNumber(value: doubleValue), index)
913+
}
914+
return nil
818915
}
819916

820917
//MARK: - Value parsing

‎TestFoundation/TestJSONEncoder.swift

+108
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,113 @@ class TestJSONEncoder : XCTestCase {
499499
}
500500
}
501501

502+
func test_numericLimits() {
503+
struct DataStruct: Codable {
504+
let int8Value: Int8?
505+
let uint8Value: UInt8?
506+
let int16Value: Int16?
507+
let uint16Value: UInt16?
508+
let int32Value: Int32?
509+
let uint32Value: UInt32?
510+
let int64Value: Int64?
511+
let intValue: Int?
512+
let uintValue: UInt?
513+
let uint64Value: UInt64?
514+
let floatValue: Float?
515+
let doubleValue: Double?
516+
let decimalValue: Decimal?
517+
}
518+
519+
func decode(_ type: String, _ value: String) throws {
520+
var key = type.lowercased()
521+
key.append("Value")
522+
_ = try JSONDecoder().decode(DataStruct.self, from: "{ \"\(key)\": \(value) }".data(using: .utf8)!)
523+
}
524+
525+
func testGoodValue(_ type: String, _ value: String) {
526+
do {
527+
try decode(type, value)
528+
} catch {
529+
XCTFail("Unexpected error: \(error) for parsing \(value) to \(type)")
530+
}
531+
}
532+
533+
func testErrorThrown(_ type: String, _ value: String, errorMessage: String) {
534+
do {
535+
try decode(type, value)
536+
XCTFail("Decode of \(value) to \(type) should not succeed")
537+
} catch DecodingError.dataCorrupted(let context) {
538+
XCTAssertEqual(context.debugDescription, errorMessage)
539+
} catch {
540+
XCTAssertEqual(String(describing: error), errorMessage)
541+
}
542+
}
543+
544+
545+
var goodValues = [
546+
("Int8", "0"), ("Int8", "1"), ("Int8", "-1"), ("Int8", "-128"), ("Int8", "127"),
547+
("UInt8", "0"), ("UInt8", "1"), ("UInt8", "255"), ("UInt8", "-0"),
548+
549+
("Int16", "0"), ("Int16", "1"), ("Int16", "-1"), ("Int16", "-32768"), ("Int16", "32767"),
550+
("UInt16", "0"), ("UInt16", "1"), ("UInt16", "65535"), ("UInt16", "34.0"),
551+
552+
("Int32", "0"), ("Int32", "1"), ("Int32", "-1"), ("Int32", "-2147483648"), ("Int32", "2147483647"),
553+
("UInt32", "0"), ("UInt32", "1"), ("UInt32", "4294967295"),
554+
555+
("Int64", "0"), ("Int64", "1"), ("Int64", "-1"), ("Int64", "-9223372036854775808"), ("Int64", "9223372036854775807"),
556+
("UInt64", "0"), ("UInt64", "1"), ("UInt64", "18446744073709551615"),
557+
558+
("Double", "0"), ("Double", "1"), ("Double", "-1"), ("Double", "2.2250738585072014e-308"), ("Double", "1.7976931348623157e+308"),
559+
("Double", "5e-324"), ("Double", "3.141592653589793"),
560+
561+
("Decimal", "1.2"), ("Decimal", "3.14159265358979323846264338327950288419"),
562+
("Decimal", "3402823669209384634633746074317682114550000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
563+
("Decimal", "-3402823669209384634633746074317682114550000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
564+
]
565+
566+
if Int.max == Int64.max {
567+
goodValues += [
568+
("Int", "0"), ("Int", "1"), ("Int", "-1"), ("Int", "-9223372036854775808"), ("Int", "9223372036854775807"),
569+
("UInt", "0"), ("UInt", "1"), ("UInt", "18446744073709551615"),
570+
]
571+
} else {
572+
goodValues += [
573+
("Int", "0"), ("Int", "1"), ("Int", "-1"), ("Int", "-2147483648"), ("Int", "2147483647"),
574+
("UInt", "0"), ("UInt", "1"), ("UInt", "4294967295"),
575+
]
576+
}
577+
578+
let badValues = [
579+
("Int8", "-129"), ("Int8", "128"), ("Int8", "1.2"),
580+
("UInt8", "-1"), ("UInt8", "256"),
581+
582+
("Int16", "-32769"), ("Int16", "32768"),
583+
("UInt16", "-1"), ("UInt16", "65536"),
584+
585+
("Int32", "-2147483649"), ("Int32", "2147483648"),
586+
("UInt32", "-1"), ("UInt32", "4294967296"),
587+
588+
("Int64", "9223372036854775808"), ("Int64", "9223372036854775808"), ("Int64", "-100000000000000000000"),
589+
("UInt64", "-1"), ("UInt64", "18446744073709600000"), ("Int64", "10000000000000000000000000000000000000"),
590+
]
591+
592+
for value in goodValues {
593+
testGoodValue(value.0, value.1)
594+
}
595+
596+
for (type, value) in badValues {
597+
testErrorThrown(type, value, errorMessage: "Parsed JSON number <\(value)> does not fit in \(type).")
598+
}
599+
600+
// Invalid JSON number formats
601+
testErrorThrown("Int8", "0000000000000000000000000000001", errorMessage: "The given data was not valid JSON.")
602+
testErrorThrown("Double", "-.1", errorMessage: "The given data was not valid JSON.")
603+
testErrorThrown("Int32", "+1", errorMessage: "The given data was not valid JSON.")
604+
testErrorThrown("Int", ".012", errorMessage: "The given data was not valid JSON.")
605+
testErrorThrown("Double", "2.7976931348623158e+308", errorMessage: "The given data was not valid JSON.")
606+
}
607+
608+
502609
// MARK: - Helper Functions
503610
private var _jsonEmptyDictionary: Data {
504611
return "{}".data(using: .utf8)!
@@ -1089,6 +1196,7 @@ extension TestJSONEncoder {
10891196
("test_codingOfDouble", test_codingOfDouble),
10901197
("test_codingOfString", test_codingOfString),
10911198
("test_codingOfURL", test_codingOfURL),
1199+
("test_numericLimits", test_numericLimits),
10921200
]
10931201
}
10941202
}

‎TestFoundation/TestJSONSerialization.swift

+13-2
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,7 @@ extension TestJSONSerialization {
487487

488488
//MARK: - Number parsing
489489
func deserialize_numbers(objectType: ObjectType) {
490-
let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10]"
490+
let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10, -12.34e56, 12.34e-56, 12.34e+6, 0.002, 0.0043e+4]"
491491

492492
do {
493493
for encoding in supportedEncodings {
@@ -504,14 +504,19 @@ extension TestJSONSerialization {
504504
XCTAssertEqual(result?[5] as? Double, 0.001)
505505
XCTAssertEqual(result?[6] as? Int, 10)
506506
XCTAssertEqual(result?[6] as? Double, 10.0)
507+
XCTAssertEqual(result?[7] as? Double, -12.34e56)
508+
XCTAssertEqual(result?[8] as? Double, 12.34e-56)
509+
XCTAssertEqual(result?[9] as? Double, 12.34e6)
510+
XCTAssertEqual(result?[10] as? Double, 2e-3)
511+
XCTAssertEqual(result?[11] as? Double, 43)
507512
}
508513
} catch {
509514
XCTFail("Unexpected error: \(error)")
510515
}
511516
}
512517

513518
func deserialize_numbers_as_reference_types(objectType: ObjectType) {
514-
let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10]"
519+
let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10, -12.34e56, 12.34e-56, 12.34e+6, 0.002, 0.0043e+4]"
515520

516521
do {
517522
for encoding in supportedEncodings {
@@ -528,6 +533,12 @@ extension TestJSONSerialization {
528533
XCTAssertEqual(result?[5] as? NSNumber, 0.001)
529534
XCTAssertEqual(result?[6] as? NSNumber, 10)
530535
XCTAssertEqual(result?[6] as? NSNumber, 10.0)
536+
XCTAssertEqual(result?[7] as? NSNumber, -12.34e56)
537+
XCTAssertEqual(result?[8] as? NSNumber, 12.34e-56)
538+
XCTAssertEqual(result?[9] as? NSNumber, 12.34e6)
539+
XCTAssertEqual(result?[10] as? NSNumber, 2e-3)
540+
XCTAssertEqual(result?[11] as? NSNumber, 43)
541+
531542
}
532543
} catch {
533544
XCTFail("Unexpected error: \(error)")

0 commit comments

Comments
 (0)