JSONSerialization: Improve parsing of numbers

spevans · spevans · commit 1e32417299c5 · 2018-08-13T13:51:41.000+01:00
- Fully validate that the number conforms to the JSON number
  specification.

- Determine if the number should be parsed as a UInt64, Int64 or
  Decimal before falling back to Decimal.
diff --git a/Foundation/JSONSerialization.swift b/Foundation/JSONSerialization.swift
@@ -816,66 +816,145 @@ private struct JSONReader {
     }
     
     //MARK: - Number parsing
-    static let numberCodePoints: [UInt8] = [
-        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, // 0...9
-        0x2E, 0x2D, 0x2B, 0x45, 0x65, // . - + E e
-    ]
+    private static let ZERO = UInt8(ascii: "0")
+    private static let ONE = UInt8(ascii: "1")
+    private static let NINE = UInt8(ascii: "9")
+    private static let MINUS = UInt8(ascii: "-")
+    private static let PLUS = UInt8(ascii: "+")
+    private static let LOWER_EXPONENT = UInt8(ascii: "e")
+    private static let UPPER_EXPONENT = UInt8(ascii: "E")
+    private static let DECIMAL_SEPARATOR = UInt8(ascii: ".")
+    private static let allDigits = (ZERO...NINE)
+    private static let oneToNine = (ONE...NINE)
+
+    private static let numberCodePoints: [UInt8] = {
+        var numberCodePoints = Array(ZERO...NINE)
+        numberCodePoints.append(contentsOf: [DECIMAL_SEPARATOR, MINUS, PLUS, LOWER_EXPONENT, UPPER_EXPONENT])
+        return numberCodePoints
+    }()
+
 
     func parseNumber(_ input: Index, options opt: JSONSerialization.ReadingOptions) throws -> (Any, Index)? {
-        let ZERO = UInt8(ascii: "0")
-        let ONE = UInt8(ascii: "1")
-        let NINE = UInt8(ascii: "9")
-        let MINUS = UInt8(ascii: "-")
 
         var isNegative = false
         var string = ""
-
-        // Validate the first few characters look like a JSON encoded number:
-        // Optional '-' sign at start only 1 leading zero if followed by a decimal point.
+        var isInteger = true
+        var exponent = 0
+        var positiveExponent = true
         var index = input
-        func nextASCII() -> UInt8? {
-            guard let (ascii, nextIndex) = source.takeASCII(index),
-                JSONReader.numberCodePoints.contains(ascii) else { return nil }
-            index = nextIndex
-            return ascii
-        }
+        var digitCount: Int?
+        var ascii: UInt8 = 0    // set by nextASCII()
+
+        // Validate the input is a valid JSON number, also gather the following
+        // about the input: isNegative, isInteger, the exponent and if it is +/-,
+        // and finally the count of digits including excluding an '.'
+        func checkJSONNumber() throws -> Bool {
+            // Return true if the next character is any one of the valid JSON number characters
+            func nextASCII() -> Bool {
+                guard let (ch, nextIndex) = source.takeASCII(index),
+                    JSONReader.numberCodePoints.contains(ch) else { return false }
+
+                index = nextIndex
+                ascii = ch
+                string.append(Character(UnicodeScalar(ascii)))
+                return true
+            }
 
-        guard var ascii = nextASCII() else { return nil }
-        guard ascii == MINUS || (ascii >= ZERO && ascii <= NINE) else { return nil }
-        if ascii == MINUS {
-            string = "-"
-            isNegative = true
-            guard let d = nextASCII() else { return nil }
-            ascii = d
-        }
+            // Consume as many digits as possible and return with the next non-digit
+            // or nil if end of string.
+            func readDigits() -> UInt8? {
+                while let (ch, nextIndex) = source.takeASCII(index) {
+                    if !JSONReader.allDigits.contains(ch) {
+                        return ch
+                    }
+                    string.append(Character(UnicodeScalar(ch)))
+                    index = nextIndex
+                }
+                return nil
+            }
+
+            guard nextASCII() else { return false }
+
+            if ascii == JSONReader.MINUS {
+                isNegative = true
+                guard nextASCII() else { return false }
+            }
+
+            if JSONReader.oneToNine.contains(ascii) {
+                guard let ch = readDigits() else { return true }
+                ascii = ch
+                if [ JSONReader.DECIMAL_SEPARATOR, JSONReader.LOWER_EXPONENT, JSONReader.UPPER_EXPONENT ].contains(ascii) {
+                    guard nextASCII() else { return false } // There should be at least one char as readDigits didnt remove the '.eE'
+                }
+            } else if ascii == JSONReader.ZERO {
+                guard nextASCII() else { return true }
+            } else {
+                throw NSError(domain: NSCocoaErrorDomain, code: CocoaError.propertyListReadCorrupt.rawValue,
+                              userInfo: ["NSDebugDescription" : "Numbers must start with a 1-9 at character \(input)." ])
+            }
+
+            if ascii == JSONReader.DECIMAL_SEPARATOR {
+                isInteger = false
+                guard readDigits() != nil else { return true }
+                guard nextASCII() else { return true }
+            } else if JSONReader.allDigits.contains(ascii) {
+                throw NSError(domain: NSCocoaErrorDomain, code: CocoaError.propertyListReadCorrupt.rawValue,
+                              userInfo: ["NSDebugDescription" : "Leading zeros not allowed at character \(input)." ])
+            }
 
-        if ascii == ZERO {
-            if let ascii2 = nextASCII() {
-                if ascii2 >= ZERO && ascii2 <= NINE {
-                    throw NSError(domain: NSCocoaErrorDomain, code: CocoaError.propertyListReadCorrupt.rawValue,
-                                  userInfo: ["NSDebugDescription" : "Leading zeros not allowed at character \(input)." ])
+            digitCount = string.count - (isInteger ? 0 : 1) - (isNegative ? 1 : 0)
+            guard ascii == JSONReader.LOWER_EXPONENT || ascii == JSONReader.UPPER_EXPONENT else {
+                // End of valid number characters
+                return true
+            }
+            digitCount = digitCount! - 1
+
+            // Process the exponent
+            isInteger = false
+            guard nextASCII() else { return false }
+            if ascii == JSONReader.MINUS {
+                positiveExponent = false
+                guard nextASCII() else { return false }
+            } else if ascii == JSONReader.PLUS {
+                positiveExponent = true
+                guard nextASCII() else { return false }
+            }
+            guard JSONReader.allDigits.contains(ascii) else { return false }
+            exponent = Int(ascii - JSONReader.ZERO)
+            while nextASCII() {
+                guard JSONReader.allDigits.contains(ascii) else { return false } // Invalid exponent character
+                exponent = (exponent * 10) + Int(ascii - JSONReader.ZERO)
+                if exponent > 324 {
+                    // Exponent is too large to store in a Double
+                    return false
                 }
-                string.append("0")
-                ascii = ascii2
             }
-        } else if ascii < ONE || ascii > NINE {
-            throw NSError(domain: NSCocoaErrorDomain, code: CocoaError.propertyListReadCorrupt.rawValue,
-                          userInfo: ["NSDebugDescription" : "Numbers must start with a 1-9 at character \(input)." ])
-        }
-        string.append(Character(UnicodeScalar(ascii)))
-        while let ascii = nextASCII() {
-            string.append(Character(UnicodeScalar(ascii)))
+            return true
         }
 
-        if isNegative {
-            if let intValue = Int64(string) {
-                return (NSNumber(value: intValue), index)
-            }
-        } else {
-            if let uintValue = UInt64(string) {
-                return (NSNumber(value: uintValue), index)
+        guard try checkJSONNumber() == true else { return nil }
+        digitCount = digitCount ?? string.count - (isInteger ? 0 : 1) - (isNegative ? 1 : 0)
+
+        // Try Int64() or UInt64() first
+        if isInteger {
+            if isNegative {
+                if digitCount! <= 19, let intValue = Int64(string) {
+                    return (NSNumber(value: intValue), index)
+                }
+            } else {
+                if digitCount! <= 20, let uintValue = UInt64(string) {
+                    return (NSNumber(value: uintValue), index)
+                }
             }
         }
+
+        // Decimal holds more digits of precision but a smaller exponent than Double
+        // so try that if the exponent fits and there are more digits than Double can hold
+        if digitCount! > 17 && exponent >= -128 && exponent <= 127,
+            let decimal = Decimal(string: string), decimal.isFinite {
+            return (NSDecimalNumber(decimal: decimal), index)
+        }
+        // Fall back to Double() for everything else
         if let doubleValue = Double(string) {
             return (NSNumber(value: doubleValue), index)
         }
diff --git a/TestFoundation/TestJSONEncoder.swift b/TestFoundation/TestJSONEncoder.swift
@@ -513,6 +513,7 @@ class TestJSONEncoder : XCTestCase {
             let uint64Value: UInt64?
             let floatValue: Float?
             let doubleValue: Double?
+            let decimalValue: Decimal?
         }
 
         func decode(_ type: String, _ value: String) throws {
@@ -553,6 +554,13 @@ class TestJSONEncoder : XCTestCase {
 
             ("Int64", "0"), ("Int64", "1"), ("Int64", "-1"), ("Int64", "-9223372036854775808"), ("Int64", "9223372036854775807"),
             ("UInt64", "0"), ("UInt64", "1"), ("UInt64", "18446744073709551615"),
+
+            ("Double", "0"), ("Double", "1"), ("Double", "-1"), ("Double", "2.2250738585072014e-308"), ("Double", "1.7976931348623157e+308"),
+            ("Double", "5e-324"), ("Double", "3.141592653589793"),
+
+            ("Decimal", "1.2"), ("Decimal", "3.14159265358979323846264338327950288419"),
+            ("Decimal", "3402823669209384634633746074317682114550000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
+            ("Decimal", "-3402823669209384634633746074317682114550000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
         ]
 
         if Int.max == Int64.max {
@@ -589,11 +597,12 @@ class TestJSONEncoder : XCTestCase {
             testErrorThrown(type, value, errorMessage: "Parsed JSON number <\(value)> does not fit in \(type).")
         }
 
-        // Leading zeros are invalid
+        // Invalid JSON number formats
         testErrorThrown("Int8", "0000000000000000000000000000001", errorMessage: "The operation could not be completed")
         testErrorThrown("Double", "-.1", errorMessage: "The operation could not be completed")
         testErrorThrown("Int32", "+1", errorMessage: "The operation could not be completed")
         testErrorThrown("Int", ".012", errorMessage: "The operation could not be completed")
+        testErrorThrown("Double", "2.7976931348623158e+308", errorMessage: "The operation could not be completed")
     }
 
 
diff --git a/TestFoundation/TestJSONSerialization.swift b/TestFoundation/TestJSONSerialization.swift
@@ -487,7 +487,7 @@ extension TestJSONSerialization {
 
     //MARK: - Number parsing
     func deserialize_numbers(objectType: ObjectType) {
-        let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10]"
+        let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10, -12.34e56, 12.34e-56, 12.34e+6, 0.002, 0.0043e+4]"
 
         do {
             for encoding in supportedEncodings {
@@ -504,14 +504,19 @@ extension TestJSONSerialization {
                 XCTAssertEqual(result?[5] as? Double, 0.001)
                 XCTAssertEqual(result?[6] as? Int,    10)
                 XCTAssertEqual(result?[6] as? Double, 10.0)
+                XCTAssertEqual(result?[7] as? Double, -12.34e56)
+                XCTAssertEqual(result?[8] as? Double, 12.34e-56)
+                XCTAssertEqual(result?[9] as? Double, 12.34e6)
+                XCTAssertEqual(result?[10] as? Double, 2e-3)
+                XCTAssertEqual(result?[11] as? Double, 43)
             }
         } catch {
             XCTFail("Unexpected error: \(error)")
         }
     }
 
     func deserialize_numbers_as_reference_types(objectType: ObjectType) {
-        let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10]"
+        let subject = "[1, -1, 1.3, -1.3, 1e3, 1E-3, 10, -12.34e56, 12.34e-56, 12.34e+6, 0.002, 0.0043e+4]"
 
         do {
             for encoding in supportedEncodings {
@@ -528,6 +533,12 @@ extension TestJSONSerialization {
                 XCTAssertEqual(result?[5] as? NSNumber, 0.001)
                 XCTAssertEqual(result?[6] as? NSNumber, 10)
                 XCTAssertEqual(result?[6] as? NSNumber, 10.0)
+                XCTAssertEqual(result?[7] as? NSNumber, -12.34e56)
+                XCTAssertEqual(result?[8] as? NSNumber, 12.34e-56)
+                XCTAssertEqual(result?[9] as? NSNumber, 12.34e6)
+                XCTAssertEqual(result?[10] as? NSNumber, 2e-3)
+                XCTAssertEqual(result?[11] as? NSNumber, 43)
+
             }
         } catch {
             XCTFail("Unexpected error: \(error)")
