From a1c665d7945f8bc81a8674eb6fc6f42cf0ae6d2e Mon Sep 17 00:00:00 2001 From: Tina Liu Date: Wed, 11 Jan 2023 14:05:34 -0800 Subject: [PATCH 01/21] rdar://101694801 (Essentials String API: Case mapping) --- .../ICU/ICU+Foundation.swift | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Sources/FoundationInternationalization/ICU/ICU+Foundation.swift b/Sources/FoundationInternationalization/ICU/ICU+Foundation.swift index e965c78d2..6fe302952 100644 --- a/Sources/FoundationInternationalization/ICU/ICU+Foundation.swift +++ b/Sources/FoundationInternationalization/ICU/ICU+Foundation.swift @@ -82,6 +82,35 @@ internal func _withFixedUCharBuffer(size: Int32 = ULOC_FULLNAME_CAPACITY + ULOC_ } } +/// Allocate a buffer with `size` `CChar`s and execute the given block. +/// The closure should return the actual length of the string, or nil if there is an error in the ICU call or the result is zero length. +internal func _withResizingCharBuffer(initialSize: Int32 = 32, _ body: (UnsafeMutablePointer, Int32, inout UErrorCode) -> Int32?) -> String? { + withUnsafeTemporaryAllocation(of: CChar.self, capacity: Int(initialSize)) { + buffer in + var status = U_ZERO_ERROR + if let len = body(buffer.baseAddress!, initialSize, &status) { + if status == U_BUFFER_OVERFLOW_ERROR { + // Retry, once + return withUnsafeTemporaryAllocation(of: CChar.self, capacity: Int(len + 1)) { innerBuffer in + var innerStatus = U_ZERO_ERROR + if let innerLen = body(innerBuffer.baseAddress!, len + 1, &innerStatus) { + if innerStatus.isSuccess && innerLen > 0 { + return String(validatingUTF8: innerBuffer.baseAddress!) + } + } + + // At this point the retry has also failed + return nil + } + } else if status.isSuccess && len > 0 { + return String(validatingUTF8: buffer.baseAddress!) + } + } + + return nil + } +} + /// Allocate a buffer with `size` `CChar`s and execute the given block. The result is always null-terminated. /// The closure should return the actual length of the string, or nil if there is an error in the ICU call or the result is zero length. internal func _withFixedCharBuffer(size: Int32 = ULOC_FULLNAME_CAPACITY + ULOC_KEYWORD_AND_VALUES_CAPACITY, _ body: (UnsafeMutablePointer, Int32, inout UErrorCode) -> Int32?) -> String? { From 4803651cf783179b73630385e6e138c08f3bc93f Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Mon, 20 Feb 2023 12:29:08 -0800 Subject: [PATCH 02/21] rdar://102214045 (Re-core NSUUID on Swift UUID struct) --- Sources/FoundationEssentials/UUID.swift | 4 +- .../FoundationEssentials/UUID_Wrappers.swift | 170 ++++++++++++++++++ 2 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 Sources/FoundationEssentials/UUID_Wrappers.swift diff --git a/Sources/FoundationEssentials/UUID.swift b/Sources/FoundationEssentials/UUID.swift index c9d14336e..7a4874f2e 100644 --- a/Sources/FoundationEssentials/UUID.swift +++ b/Sources/FoundationEssentials/UUID.swift @@ -1,6 +1,6 @@ //===----------------------------------------------------------------------===// // -// This source file is part of the Swift Collections open source project +// This source file is part of the Swift open source project // // Copyright (c) 2022 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception @@ -9,7 +9,7 @@ // //===----------------------------------------------------------------------===// -@_implementationOnly import _CShims +@_implementationOnly import _CShims // uuid.h public typealias uuid_t = (UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8) public typealias uuid_string_t = (Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8, Int8) diff --git a/Sources/FoundationEssentials/UUID_Wrappers.swift b/Sources/FoundationEssentials/UUID_Wrappers.swift new file mode 100644 index 000000000..3fac71e8f --- /dev/null +++ b/Sources/FoundationEssentials/UUID_Wrappers.swift @@ -0,0 +1,170 @@ +//===----------------------------------------------------------------------===// + // + // This source file is part of the Swift Collections open source project + // + // Copyright (c) 2022 Apple Inc. and the Swift project authors + // Licensed under Apache License v2.0 with Runtime Library Exception + // + // See https://swift.org/LICENSE.txt for license information + // + //===----------------------------------------------------------------------===// + +#if FOUNDATION_FRAMEWORK + +@_implementationOnly import _ForSwiftFoundation + +// Needed this for backward compatibility even though we don't use it. +import Darwin.uuid + +@available(macOS 10.10, iOS 8.0, tvOS 9.0, watchOS 2.0, *) +extension UUID : ReferenceConvertible { + public typealias ReferenceType = NSUUID + + @_semantics("convertToObjectiveC") + public func _bridgeToObjectiveC() -> NSUUID { + return _NSSwiftUUID(value: self) + } + + public static func _forceBridgeFromObjectiveC(_ x: NSUUID, result: inout UUID?) { + if !_conditionallyBridgeFromObjectiveC(x, result: &result) { + fatalError("Unable to bridge \(_ObjectiveCType.self) to \(self)") + } + } + + public static func _conditionallyBridgeFromObjectiveC(_ input: NSUUID, result: inout UUID?) -> Bool { + // Is this NSUUID already backed by a UUID? + guard let swiftInput = input as? _NSSwiftUUID else { + // Fallback to using bytes + var bytes = uuid_t(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0) + input.getBytes(&bytes) + result = UUID(uuid: bytes) + return true + } + + result = swiftInput._storage + return true + } + + @_effects(readonly) + public static func _unconditionallyBridgeFromObjectiveC(_ source: NSUUID?) -> UUID { + var result: UUID? + _forceBridgeFromObjectiveC(source!, result: &result) + return result! + } + } + +@available(macOS 10.10, iOS 8.0, tvOS 9.0, watchOS 2.0, *) +extension NSUUID : _HasCustomAnyHashableRepresentation { + // Must be @nonobjc to avoid infinite recursion during bridging. + @nonobjc + public func _toCustomAnyHashable() -> AnyHashable? { + return AnyHashable(self as UUID) + } +} + +@objc(_NSSwiftUUID) +internal class _NSSwiftUUID : _NSUUIDBridge { + final var _storage: UUID + + fileprivate init(value: Foundation.UUID) { + _storage = value + super.init() + } + + override public init() { + _storage = Foundation.UUID() + super.init() + } + + required init?(coder: NSCoder) { + guard coder.allowsKeyedCoding else { + coder.failWithError(CocoaError(CocoaError.coderReadCorrupt, userInfo: [NSDebugDescriptionErrorKey : "Cannot be decoded without keyed coding"])) + return nil + } + + var decodedByteLength = 0 + let bytes = coder.decodeBytes(forKey: "NS.uuidbytes", returnedLength: &decodedByteLength) + + guard let bytes else { + if NSUUID._compatibilityBehavior { + let empty = uuid_t(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0) + _storage = Foundation.UUID(uuid: empty) + super.init() + return + } else { + coder.failWithError(CocoaError(CocoaError.coderValueNotFound, userInfo: [NSDebugDescriptionErrorKey : "UUID bytes not found in archive"])) + return nil + } + } + + guard decodedByteLength == MemoryLayout.size else { + if NSUUID._compatibilityBehavior { + let empty = uuid_t(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0) + _storage = Foundation.UUID(uuid: empty) + super.init() + return + } else { + coder.failWithError(CocoaError(CocoaError.coderReadCorrupt, userInfo: [NSDebugDescriptionErrorKey : "UUID bytes were not the expected length"])) + return nil + } + } + + let cUUID = bytes.withMemoryRebound(to: uuid_t.self, capacity: 1, { $0.pointee }) + _storage = Foundation.UUID(uuid: cUUID) + super.init() + } + + override func encode(with coder: NSCoder) { + var uuid = _storage.uuid + withUnsafeBytes(of: &uuid) { buffer in + buffer.withMemoryRebound(to: UInt8.self) { buffer in + coder.encodeBytes(buffer.baseAddress, length: MemoryLayout.size, forKey: "NS.uuidbytes") + } + } + } + + override public init?(uuidString: NSObject) { + guard let string = uuidString as? NSString, let swiftUUID = Foundation.UUID(uuidString: string as String) else { + if NSUUID._compatibilityBehavior { + let empty = uuid_t(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0) + _storage = Foundation.UUID(uuid: empty) + super.init() + return + } else { + return nil + } + } + _storage = swiftUUID + super.init() + } + + override public init(uuidBytes: UnsafePointer?) { + let cUUID = uuidBytes?.withMemoryRebound(to: uuid_t.self, capacity: 1, { + $0.pointee + }) ?? (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0) + _storage = Foundation.UUID(uuid: cUUID) + super.init() + } + + override open func getBytes(_ bytes: UnsafeMutablePointer) { + let size = MemoryLayout.size + withUnsafePointer(to: _storage.uuid) { source in + source.withMemoryRebound(to: UInt8.self, capacity: size) { buffer in + bytes.initialize(from: buffer, count: size) + } + } + } + + override open var uuidString: String { + @objc(UUIDString) get { + _storage.uuidString + } + } + + override var classForCoder: AnyClass { + return NSUUID.self + } +} + +#endif + From 55db333e2be6d1d9800cf3df95ff8a4767917986 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Wed, 1 Mar 2023 10:56:17 -0800 Subject: [PATCH 03/21] rdar://102214045 -- Use NSString instead of NSObject --- Sources/FoundationEssentials/UUID_Wrappers.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/FoundationEssentials/UUID_Wrappers.swift b/Sources/FoundationEssentials/UUID_Wrappers.swift index 3fac71e8f..b768bd9e7 100644 --- a/Sources/FoundationEssentials/UUID_Wrappers.swift +++ b/Sources/FoundationEssentials/UUID_Wrappers.swift @@ -123,8 +123,8 @@ internal class _NSSwiftUUID : _NSUUIDBridge { } } - override public init?(uuidString: NSObject) { - guard let string = uuidString as? NSString, let swiftUUID = Foundation.UUID(uuidString: string as String) else { + override public init?(uuidString: String) { + guard let swiftUUID = Foundation.UUID(uuidString: uuidString) else { if NSUUID._compatibilityBehavior { let empty = uuid_t(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0) _storage = Foundation.UUID(uuid: empty) From 6d137951b0ceac3a7707464727de4a668dee9620 Mon Sep 17 00:00:00 2001 From: Tina Liu Date: Sun, 5 Mar 2023 12:13:39 -0800 Subject: [PATCH 04/21] rdar://105100825 (NSLocale: Implement localeIdentifier and regionCode, soft deprecate countryCode) --- .../Locale/Locale+Language.swift | 5 +++-- .../Locale/Locale_Wrappers.swift | 10 ++++++++++ .../LocaleLanguageTests.swift | 4 +++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale+Language.swift b/Sources/FoundationInternationalization/Locale/Locale+Language.swift index d95e8e5e9..c6d0dee24 100644 --- a/Sources/FoundationInternationalization/Locale/Locale+Language.swift +++ b/Sources/FoundationInternationalization/Locale/Locale+Language.swift @@ -194,7 +194,7 @@ extension Locale { result = lang } else { result = _withFixedCharBuffer { buffer, size, status in - uloc_getLanguage(maximalIdentifier, buffer, size, &status) + uloc_getLanguage(components.identifier, buffer, size, &status) }.map { LanguageCode($0) } } return result @@ -207,6 +207,7 @@ extension Locale { result = script } else { result = _withFixedCharBuffer { buffer, size, status in + // Use `maximalIdentifier` to ensure that script code is present in the identifier. uloc_getScript(maximalIdentifier, buffer, size, &status) }.map { Script($0) } } @@ -220,7 +221,7 @@ extension Locale { result = script } else { result = _withFixedCharBuffer { buffer, size, status in - uloc_getCountry(maximalIdentifier, buffer, size, &status) + uloc_getCountry(components.identifier, buffer, size, &status) }.map { Region($0) } } return result diff --git a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift index e16175afd..d9e9c0bf4 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift @@ -525,11 +525,21 @@ internal class _NSSwiftLocale: _NSLocaleBridge { locale.languageCode ?? "" } + override var languageIdentifier: String { + let langIdentifier = locale.language.components.identifier + let localeWithOnlyLanguage = Locale(identifier: langIdentifier) + return localeWithOnlyLanguage.identifier(.bcp47) + } + @available(macOS, deprecated: 13) @available(iOS, deprecated: 16) @available(tvOS, deprecated: 16) @available(watchOS, deprecated: 9) override var countryCode: String? { locale.regionCode } + override var regionCode: String? { + locale.region?.identifier + } + @available(macOS, deprecated: 13) @available(iOS, deprecated: 16) @available(tvOS, deprecated: 16) @available(watchOS, deprecated: 9) override var scriptCode: String? { locale.scriptCode diff --git a/Tests/FoundationInternationalizationTests/LocaleLanguageTests.swift b/Tests/FoundationInternationalizationTests/LocaleLanguageTests.swift index 0af47e712..1f786899e 100644 --- a/Tests/FoundationInternationalizationTests/LocaleLanguageTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleLanguageTests.swift @@ -77,7 +77,9 @@ class LocaleLanguageTests: XCTestCase { verify("en-Kore-US", expectedParent: .init(identifier: "en-Kore"), minBCP47: "en-Kore", maxBCP47: "en-Kore-US", langCode: "en", script: "Kore", region: "US", lineDirection: .topToBottom, characterDirection: .leftToRight) verify("zh-TW", expectedParent: .init(identifier: "root"), minBCP47: "zh-TW", maxBCP47: "zh-Hant-TW", langCode: "zh", script: "Hant", region: "TW", lineDirection: .topToBottom, characterDirection: .leftToRight) verify("en-Latn-US", expectedParent: .init(identifier: "en-Latn"), minBCP47: "en", maxBCP47: "en-Latn-US", langCode: "en", script: "Latn", region: "US", lineDirection: .topToBottom, characterDirection: .leftToRight) - verify("ar-Arab", expectedParent: .init(identifier: "ar"), minBCP47: "ar", maxBCP47: "ar-Arab-EG", langCode: "ar", script: "Arab", region: "EG", lineDirection: .topToBottom, characterDirection: .rightToLeft) + verify("ar-Arab", expectedParent: .init(identifier: "ar"), minBCP47: "ar", maxBCP47: "ar-Arab-EG", langCode: "ar", script: "Arab", region: nil, lineDirection: .topToBottom, characterDirection: .rightToLeft) + + verify("en", expectedParent: .init(identifier: "root"), minBCP47: "en", maxBCP47: "en-Latn-US", langCode: "en", script: "Latn", region: nil, lineDirection: .topToBottom, characterDirection: .leftToRight) verify("root", expectedParent: .init(identifier: "root"), minBCP47: "root", maxBCP47: "root", langCode: "root", script: nil, region: nil, lineDirection: .topToBottom, characterDirection: .leftToRight) } From c6982991d3aa0def1abb70b0594258ffb9b18287 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Mon, 20 Feb 2023 08:42:27 -0800 Subject: [PATCH 05/21] rdar://105186248 (Fix more warnings in Swift) --- .../Data/Collections+DataProtocol.swift | 14 ++++++------ .../Data/ContiguousBytes.swift | 22 +++++++++---------- Sources/FoundationEssentials/Data/Data.swift | 10 ++++----- .../Data/DataProtocol.swift | 12 +++++----- .../Data/Pointers+DataProtocol.swift | 4 ++-- Sources/FoundationEssentials/Platform.swift | 4 ++-- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/Sources/FoundationEssentials/Data/Collections+DataProtocol.swift b/Sources/FoundationEssentials/Data/Collections+DataProtocol.swift index 5d49183b1..c01562dc5 100644 --- a/Sources/FoundationEssentials/Data/Collections+DataProtocol.swift +++ b/Sources/FoundationEssentials/Data/Collections+DataProtocol.swift @@ -12,21 +12,21 @@ //===--- DataProtocol -----------------------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Array: DataProtocol where Element == UInt8 { public var regions: CollectionOfOne> { return CollectionOfOne(self) } } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension ArraySlice: DataProtocol where Element == UInt8 { public var regions: CollectionOfOne> { return CollectionOfOne(self) } } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension ContiguousArray: DataProtocol where Element == UInt8 { public var regions: CollectionOfOne> { return CollectionOfOne(self) @@ -42,14 +42,14 @@ extension ContiguousArray: DataProtocol where Element == UInt8 { // } // } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension EmptyCollection : DataProtocol where Element == UInt8 { public var regions: EmptyCollection { return EmptyCollection() } } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Repeated: DataProtocol where Element == UInt8 { public typealias Regions = Repeated @@ -61,8 +61,8 @@ extension Repeated: DataProtocol where Element == UInt8 { //===--- MutableDataProtocol ----------------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Array: MutableDataProtocol where Element == UInt8 { } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension ContiguousArray: MutableDataProtocol where Element == UInt8 { } diff --git a/Sources/FoundationEssentials/Data/ContiguousBytes.swift b/Sources/FoundationEssentials/Data/ContiguousBytes.swift index 3fc984f06..c7569c6d7 100644 --- a/Sources/FoundationEssentials/Data/ContiguousBytes.swift +++ b/Sources/FoundationEssentials/Data/ContiguousBytes.swift @@ -14,7 +14,7 @@ /// Indicates that the conforming type is a contiguous collection of raw bytes /// whose underlying storage is directly accessible by withUnsafeBytes. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) public protocol ContiguousBytes { /// Calls the given closure with the contents of underlying storage. /// @@ -28,20 +28,20 @@ public protocol ContiguousBytes { //===--- Collection Conformances ------------------------------------------===// // FIXME: When possible, expand conformance to `where Element : Trivial`. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Array : ContiguousBytes where Element == UInt8 { } // FIXME: When possible, expand conformance to `where Element : Trivial`. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension ArraySlice : ContiguousBytes where Element == UInt8 { } // FIXME: When possible, expand conformance to `where Element : Trivial`. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension ContiguousArray : ContiguousBytes where Element == UInt8 { } //===--- Pointer Conformances ---------------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension UnsafeRawBufferPointer : ContiguousBytes { @inlinable public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { @@ -49,7 +49,7 @@ extension UnsafeRawBufferPointer : ContiguousBytes { } } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension UnsafeMutableRawBufferPointer : ContiguousBytes { @inlinable public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { @@ -58,7 +58,7 @@ extension UnsafeMutableRawBufferPointer : ContiguousBytes { } // FIXME: When possible, expand conformance to `where Element : Trivial`. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension UnsafeBufferPointer : ContiguousBytes where Element == UInt8 { @inlinable public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { @@ -67,7 +67,7 @@ extension UnsafeBufferPointer : ContiguousBytes where Element == UInt8 { } // FIXME: When possible, expand conformance to `where Element : Trivial`. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension UnsafeMutableBufferPointer : ContiguousBytes where Element == UInt8 { @inlinable public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { @@ -76,7 +76,7 @@ extension UnsafeMutableBufferPointer : ContiguousBytes where Element == UInt8 { } // FIXME: When possible, expand conformance to `where Element : Trivial`. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension EmptyCollection : ContiguousBytes where Element == UInt8 { @inlinable public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { @@ -85,7 +85,7 @@ extension EmptyCollection : ContiguousBytes where Element == UInt8 { } // FIXME: When possible, expand conformance to `where Element : Trivial`. -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension CollectionOfOne : ContiguousBytes where Element == UInt8 { @inlinable public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { @@ -98,7 +98,7 @@ extension CollectionOfOne : ContiguousBytes where Element == UInt8 { //===--- Conditional Conformances -----------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Slice : ContiguousBytes where Base : ContiguousBytes { public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> ResultType) rethrows -> ResultType { let offset = base.distance(from: base.startIndex, to: self.startIndex) diff --git a/Sources/FoundationEssentials/Data/Data.swift b/Sources/FoundationEssentials/Data/Data.swift index 19a1b1829..8a8df74ca 100644 --- a/Sources/FoundationEssentials/Data/Data.swift +++ b/Sources/FoundationEssentials/Data/Data.swift @@ -17,7 +17,7 @@ import Darwin internal func __DataInvokeDeallocatorVirtualMemory(_ mem: UnsafeMutableRawPointer, _ length: Int) { guard vm_deallocate( mach_task_self_, - unsafeBitCast(mem, to: vm_address_t.self), + vm_address_t(UInt(bitPattern: mem)), vm_size_t(length)) == ERR_SUCCESS else { fatalError("*** __DataInvokeDeallocatorVirtualMemory(\(mem), \(length)) failed") } @@ -80,7 +80,7 @@ internal func _withStackOrHeapBuffer(capacity: Int, _ body: (UnsafeMutableBuffer // coexist without a conflicting ObjC class name, so it was renamed. // The old name must not be used in the new runtime. @usableFromInline -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) internal final class __DataStorage : @unchecked Sendable { @usableFromInline static let maxSize = Int.max >> 1 @usableFromInline static let vmOpsThreshold = Platform.pageSize * 4 @@ -506,7 +506,7 @@ internal final class __DataStorage : @unchecked Sendable { } @frozen -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) public struct Data : Equatable, Hashable, RandomAccessCollection, MutableCollection, RangeReplaceableCollection, MutableDataProtocol, ContiguousBytes, Sendable { public typealias Index = Int @@ -2541,7 +2541,7 @@ extension Data { #endif //!FOUNDATION_FRAMEWORK -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Data : CustomStringConvertible, CustomDebugStringConvertible, CustomReflectable { /// A human-readable description for the data. public var description: String { @@ -2572,7 +2572,7 @@ extension Data : CustomStringConvertible, CustomDebugStringConvertible, CustomRe } } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Data : Codable { public init(from decoder: Decoder) throws { var container = try decoder.unkeyedContainer() diff --git a/Sources/FoundationEssentials/Data/DataProtocol.swift b/Sources/FoundationEssentials/Data/DataProtocol.swift index 573b6563d..462baf086 100644 --- a/Sources/FoundationEssentials/Data/DataProtocol.swift +++ b/Sources/FoundationEssentials/Data/DataProtocol.swift @@ -18,7 +18,7 @@ import Glibc //===--- DataProtocol -----------------------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) public protocol DataProtocol : RandomAccessCollection where Element == UInt8, SubSequence : DataProtocol { // FIXME: Remove in favor of opaque type on `regions`. associatedtype Regions: BidirectionalCollection where Regions.Element : DataProtocol & ContiguousBytes, Regions.Element.SubSequence : ContiguousBytes @@ -71,7 +71,7 @@ public protocol DataProtocol : RandomAccessCollection where Element == UInt8, Su //===--- MutableDataProtocol ----------------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) public protocol MutableDataProtocol : DataProtocol, MutableCollection, RangeReplaceableCollection { /// Replaces the contents of the buffer at the given range with zeroes. /// @@ -82,7 +82,7 @@ public protocol MutableDataProtocol : DataProtocol, MutableCollection, RangeRepl //===--- DataProtocol Extensions ------------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension DataProtocol { public func firstRange(of data: D) -> Range? { return self.firstRange(of: data, in: self.startIndex ..< self.endIndex) @@ -200,7 +200,7 @@ extension DataProtocol { } } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension DataProtocol where Self : ContiguousBytes { public func copyBytes(to ptr: UnsafeMutableBufferPointer, from range: R) where R.Bound == Index { precondition(ptr.baseAddress != nil) @@ -216,7 +216,7 @@ extension DataProtocol where Self : ContiguousBytes { //===--- MutableDataProtocol Extensions -----------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension MutableDataProtocol { public mutating func resetBytes(in range: R) where R.Bound == Index { let r = range.relative(to: self) @@ -227,7 +227,7 @@ extension MutableDataProtocol { //===--- DataProtocol Conditional Conformances ----------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension Slice : DataProtocol where Base : DataProtocol { public typealias Regions = [Base.Regions.Element.SubSequence] diff --git a/Sources/FoundationEssentials/Data/Pointers+DataProtocol.swift b/Sources/FoundationEssentials/Data/Pointers+DataProtocol.swift index d08bcf803..dd8839c42 100644 --- a/Sources/FoundationEssentials/Data/Pointers+DataProtocol.swift +++ b/Sources/FoundationEssentials/Data/Pointers+DataProtocol.swift @@ -11,14 +11,14 @@ //===----------------------------------------------------------------------===// -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension UnsafeRawBufferPointer : DataProtocol { public var regions: CollectionOfOne { return CollectionOfOne(self) } } -@available(macOS 10.10, iOS 8.0, *) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) extension UnsafeBufferPointer : DataProtocol where Element == UInt8 { public var regions: CollectionOfOne> { return CollectionOfOne(self) diff --git a/Sources/FoundationEssentials/Platform.swift b/Sources/FoundationEssentials/Platform.swift index a4093c90f..418cf9ede 100644 --- a/Sources/FoundationEssentials/Platform.swift +++ b/Sources/FoundationEssentials/Platform.swift @@ -43,9 +43,9 @@ internal struct Platform { #if canImport(Darwin) if vm_copy( mach_task_self_, - unsafeBitCast(source, to: vm_address_t.self), + vm_address_t(UInt(bitPattern: source)), vm_size_t(length), - unsafeBitCast(dest, to: vm_address_t.self)) != KERN_SUCCESS { + vm_address_t(UInt(bitPattern: dest))) != KERN_SUCCESS { memmove(dest, source, length) } #else From 0fa573b2c65d4ebcd5468c9d7f628eb20acda7c0 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Mon, 6 Mar 2023 17:42:23 -0800 Subject: [PATCH 06/21] rdar://106155597 (Fix regression) --- .../Locale/Locale.swift | 2 +- .../Locale/Locale_Cache.swift | 2 +- .../Locale/Locale_ICU.swift | 100 +++++++++++------- .../Locale/Locale_Wrappers.swift | 2 +- 4 files changed, 64 insertions(+), 42 deletions(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index 7f53a73cf..063b9688e 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -116,7 +116,7 @@ public struct Locale : Hashable, Equatable, Sendable { } /// This returns an instance of `Locale` that's set up exactly like it would be if the user changed the current locale to that identifier, set the preferences keys in the overrides dictionary, then called `current`. - internal static func localeAsIfCurrent(name: String?, overrides: [String: Any]? = nil, disableBundleMatching: Bool = false) -> Locale { + internal static func localeAsIfCurrent(name: String?, overrides: CFDictionary? = nil, disableBundleMatching: Bool = false) -> Locale { let (inner, _) = _Locale._currentLocale(name: name, overrides: overrides, disableBundleMatching: disableBundleMatching) return Locale(.fixed(inner)) } diff --git a/Sources/FoundationInternationalization/Locale/Locale_Cache.swift b/Sources/FoundationInternationalization/Locale/Locale_Cache.swift index aab3c0fb4..2158a8018 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Cache.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Cache.swift @@ -187,7 +187,7 @@ struct LocaleCache : Sendable { } var preferred: _Locale { - let (locale, _) = _Locale._currentLocale(name: nil, overrides: [:], disableBundleMatching: true) + let (locale, _) = _Locale._currentLocale(name: nil, overrides: nil, disableBundleMatching: true) return locale } diff --git a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift index 5d46eb858..aa2fe6e07 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift @@ -77,11 +77,13 @@ internal final class _Locale: Sendable, Hashable { var collationOrder: String? var firstWeekday: [Calendar.Identifier : Int]? var minDaysInFirstWeek: [Calendar.Identifier : Int]? - var ICUDateTimeSymbols: [String : [String]]? - var ICUDateFormatStrings: [String : String]? - var ICUTimeFormatStrings: [String : String]? - var ICUNumberFormatStrings: [String : String]? - var ICUNumberSymbols: [String : String]? +#if FOUNDATION_FRAMEWORK + var ICUDateTimeSymbols: CFDictionary? + var ICUDateFormatStrings: CFDictionary? + var ICUTimeFormatStrings: CFDictionary? + var ICUNumberFormatStrings: CFDictionary? + var ICUNumberSymbols: CFDictionary? +#endif var country: String? var measurementUnits: MeasurementUnit? var temperatureUnit: TemperatureUnit? @@ -92,55 +94,74 @@ internal final class _Locale: Sendable, Hashable { /// Interpret a dictionary (from user defaults) according to a predefined set of strings and convert it into the more strongly-typed `Prefs` values. /// Several dictionaries may need to be applied to the same instance, which is why this is structured as a mutating setter rather than an initializer. - mutating func apply(_ prefs: [String: Any]) { - guard !prefs.isEmpty else { return } - if let langs = prefs["AppleLanguages"] as? [Any] { - let filtered = langs.filter { $0 is String } - if !filtered.isEmpty { - self.languages = filtered as? [String] - } - } - if let locale = prefs["AppleLocale"] as? String { + /// Why use a `CFDictionary` instead of a Swift dictionary here? The input prefs may be a complete copy of the user's prefs, and we don't want to bridge a ton of unrelated data into Swift just to extract a few keys. Keeping it as a `CFDictionary` avoids that overhead, and we call into small CF helper functions to get the data we need, if it is there. + mutating func apply(_ prefs: CFDictionary) { + var exists: DarwinBoolean = false + + guard CFDictionaryGetCount(prefs) > 0 else { return } + + if let langs = __CFLocalePrefsCopyAppleLanguages(prefs)?.takeRetainedValue() as? [String] { + self.languages = langs + } + if let locale = __CFLocalePrefsCopyAppleLocale(prefs)?.takeRetainedValue() as? String { self.locale = locale } - if let metricUnits = prefs["AppleMetricUnits"] as? Bool { - self.metricUnits = metricUnits + + let isMetric = __CFLocalePrefsAppleMetricUnitsIsMetric(prefs, &exists) + if exists.boolValue { + self.metricUnits = isMetric } - if let measurementUnits = MeasurementUnit(prefs["AppleMeasurementUnits"] as? String) { - self.measurementUnits = measurementUnits + + let isCentimeters = __CFLocalePrefsAppleMeasurementUnitsIsCm(prefs, &exists) + if exists.boolValue { + self.measurementUnits = isCentimeters ? .centimeters : .inches } - if let temperatureUnit = TemperatureUnit(prefs["AppleTemperatureUnit"] as? String) { - self.temperatureUnit = temperatureUnit + + let isCelsius = __CFLocalePrefsAppleTemperatureUnitIsC(prefs, &exists) + if exists.boolValue { + self.temperatureUnit = isCelsius ? .celsius : .fahrenheit } - if let collationOrder = prefs["AppleCollationOrder"] as? String { - self.collationOrder = collationOrder + + let is24Hour = __CFLocalePrefsAppleForce24HourTime(prefs, &exists) + if exists.boolValue { + self.ICUForce24HourTime = is24Hour } - if let ICUDateTimeSymbols = prefs["AppleICUDateTimeSymbols"] as? [String : [String]] { - self.ICUDateTimeSymbols = ICUDateTimeSymbols + + let is12Hour = __CFLocalePrefsAppleForce12HourTime(prefs, &exists) + if exists.boolValue { + self.ICUForce12HourTime = is12Hour } - if let ICUForce24HourTime = prefs["AppleICUForce24HourTime"] as? Bool { - self.ICUForce24HourTime = ICUForce24HourTime + + if let collationOrder = __CFLocalePrefsCopyAppleCollationOrder(prefs)?.takeRetainedValue() as? String { + self.collationOrder = collationOrder + } + + if let country = __CFLocalePrefsCopyCountry(prefs)?.takeRetainedValue() as? String { + self.country = country } - if let ICUForce12HourTime = prefs["AppleICUForce12HourTime"] as? Bool { - self.ICUForce12HourTime = ICUForce12HourTime + + if let ICUDateTimeSymbols = __CFLocalePrefsCopyAppleICUDateTimeSymbols(prefs)?.takeRetainedValue() { + self.ICUDateTimeSymbols = ICUDateTimeSymbols } - if let ICUDateFormatStrings = prefs["AppleICUDateFormatStrings"] as? [String : String] { + + if let ICUDateFormatStrings = __CFLocalePrefsCopyAppleICUDateFormatStrings(prefs)?.takeRetainedValue() { self.ICUDateFormatStrings = ICUDateFormatStrings } - if let ICUTimeFormatStrings = prefs["AppleICUTimeFormatStrings"] as? [String : String] { + + if let ICUTimeFormatStrings = __CFLocalePrefsCopyAppleICUTimeFormatStrings(prefs)?.takeRetainedValue() { self.ICUTimeFormatStrings = ICUTimeFormatStrings } - if let ICUNumberFormatStrings = prefs["AppleICUNumberFormatStrings"] as? [String : String] { + + if let ICUNumberFormatStrings = __CFLocalePrefsCopyAppleICUNumberFormatStrings(prefs)?.takeRetainedValue() { self.ICUNumberFormatStrings = ICUNumberFormatStrings } - if let ICUNumberSymbols = prefs["AppleICUNumberSymbols"] as? [String : String] { + + if let ICUNumberSymbols = __CFLocalePrefsCopyAppleICUNumberSymbols(prefs)?.takeRetainedValue() { self.ICUNumberSymbols = ICUNumberSymbols } - if let country = prefs["Country"] as? String { - self.country = country - } + - if let firstWeekdaysPrefs = prefs["AppleFirstWeekday"] as? [String: Int] { + if let firstWeekdaysPrefs = __CFLocalePrefsCopyAppleFirstWeekday(prefs)?.takeRetainedValue() as? [String: Int] { var mapped: [Calendar.Identifier : Int] = [:] for (key, value) in firstWeekdaysPrefs { if let id = Calendar.Identifier(identifierString: key) { @@ -153,7 +174,7 @@ internal final class _Locale: Sendable, Hashable { } } - if let minDaysPrefs = prefs["AppleMinDaysInFirstWeek"] as? [String: Int] { + if let minDaysPrefs = __CFLocalePrefsCopyAppleMinDaysInFirstWeek(prefs)?.takeRetainedValue() as? [String: Int] { var mapped: [Calendar.Identifier : Int] = [:] for (key, value) in minDaysPrefs { if let id = Calendar.Identifier(identifierString: key) { @@ -1548,7 +1569,7 @@ internal final class _Locale: Sendable, Hashable { }() #endif // FOUNDATION_FRAMEWORK - internal static func _currentLocale(name: String?, overrides: [String: Any]?, disableBundleMatching: Bool) -> (_Locale, Bool) { + internal static func _currentLocale(name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> (_Locale, Bool) { /* NOTE: calling any CFPreferences function, or any function which calls into a CFPreferences function, *except* for __CFXPreferencesCopyCurrentApplicationStateWithDeadlockAvoidance (and accepting backstop values if its outparam is false), will deadlock. This is because CFPreferences calls os_log_*, which calls -descriptionWithLocale:, which calls CFLocaleCopyCurrent. */ @@ -1596,7 +1617,8 @@ internal final class _Locale: Sendable, Hashable { // Don't cache a locale built with incomplete prefs suitableForCache = false } - if let cfPrefs = cfPrefs as? [String : Any] { prefs.apply(cfPrefs) } + + prefs.apply(cfPrefs) #endif // FOUNDATION_FRAMEWORK #endif // os(...) diff --git a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift index d9e9c0bf4..934460b8c 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift @@ -42,7 +42,7 @@ extension NSLocale { } @objc - private class func _newLocaleAsIfCurrent(_ name: String?, overrides: [String: Any]?, disableBundleMatching: Bool) -> NSLocale? { + private class func _newLocaleAsIfCurrent(_ name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> NSLocale? { let inner = Locale.localeAsIfCurrent(name: name, overrides: overrides, disableBundleMatching: disableBundleMatching) return _NSSwiftLocale(inner) } From 204cdece9548d13d0b13113046ea1312522b77c5 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Mon, 27 Feb 2023 08:42:30 -0800 Subject: [PATCH 07/21] rdar://102214045: Override supportsSecureCoding in Swift subclass --- Sources/FoundationEssentials/UUID_Wrappers.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/FoundationEssentials/UUID_Wrappers.swift b/Sources/FoundationEssentials/UUID_Wrappers.swift index b768bd9e7..1d2a31bad 100644 --- a/Sources/FoundationEssentials/UUID_Wrappers.swift +++ b/Sources/FoundationEssentials/UUID_Wrappers.swift @@ -76,6 +76,8 @@ internal class _NSSwiftUUID : _NSUUIDBridge { super.init() } + override static var supportsSecureCoding: Bool { true } + required init?(coder: NSCoder) { guard coder.allowsKeyedCoding else { coder.failWithError(CocoaError(CocoaError.coderReadCorrupt, userInfo: [NSDebugDescriptionErrorKey : "Cannot be decoded without keyed coding"])) From f65a4dfa713cbbb39cc7562823de40b6265666c5 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Tue, 7 Mar 2023 16:40:11 -0800 Subject: [PATCH 08/21] rdar://106155597 -- Clean up --- .../LocaleTests.swift | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Tests/FoundationInternationalizationTests/LocaleTests.swift b/Tests/FoundationInternationalizationTests/LocaleTests.swift index 06c5ff315..cf7506097 100644 --- a/Tests/FoundationInternationalizationTests/LocaleTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleTests.swift @@ -58,6 +58,10 @@ final class LocaleTests : XCTestCase { // XCTAssertEqual("something", locale.localizedString(forCollatorIdentifier: "en")) } + @available(macOS, deprecated: 13) + @available(iOS, deprecated: 16) + @available(tvOS, deprecated: 16) + @available(watchOS, deprecated: 9) func test_properties_complexIdentifiers() { struct S { var identifier: String @@ -388,6 +392,10 @@ final class LocalePropertiesTests : XCTestCase { verify("en-GB-u-rg-uszzzz", .us, shouldRespectUserPref: true) } + @available(macOS, deprecated: 13) + @available(iOS, deprecated: 16) + @available(tvOS, deprecated: 16) + @available(watchOS, deprecated: 9) func test_properties() { let locale = Locale(identifier: "zh-Hant-HK") @@ -446,6 +454,11 @@ extension NSLocale { } final class LocalBridgingTests : XCTestCase { + + @available(macOS, deprecated: 13) + @available(iOS, deprecated: 16) + @available(tvOS, deprecated: 16) + @available(watchOS, deprecated: 9) func test_getACustomLocale() { let loc = getACustomLocale() let objCLoc = loc as! CustomNSLocaleSubclass @@ -479,25 +492,13 @@ final class LocalBridgingTests : XCTestCase { extension LocaleTests { func test_userPreferenceOverride_firstWeekday() { func verify(_ localeID: String, _ expected: Locale.Weekday, shouldRespectUserPrefForGregorian: Bool, shouldRespectUserPrefForIslamic: Bool, file: StaticString = #file, line: UInt = #line) { - let firstWeekdayKey = "AppleFirstWeekday" - // sunday is 1 - let forceWed = [ firstWeekdayKey: [ - Calendar.Identifier.gregorian.cldrIdentifier : 4 - ]] as CFDictionary - - let forceFriIslamic = [ firstWeekdayKey: [ - Calendar.Identifier.islamic.cldrIdentifier : 6 - ]] as CFDictionary - - let empty = [ firstWeekdayKey: [] ] as CFDictionary - - let localeNoPref = _CFLocaleCopyAsIfCurrentWithOverrides(localeID as CFString, empty) as Locale + let localeNoPref = Locale.localeAsIfCurrent(name: localeID, overrides: .init(firstWeekday: [:])) XCTAssertEqual(localeNoPref.firstDayOfWeek, expected, file: file, line: line) - let wed = _CFLocaleCopyAsIfCurrentWithOverrides(localeID as CFString, forceWed) as Locale + let wed = Locale.localeAsIfCurrent(name: localeID, overrides: .init(firstWeekday: [.gregorian : 4])) XCTAssertEqual(wed.firstDayOfWeek, shouldRespectUserPrefForGregorian ? .wednesday : expected, file: file, line: line) - let fri_islamic = _CFLocaleCopyAsIfCurrentWithOverrides(localeID as CFString, forceFriIslamic) as Locale + let fri_islamic = Locale.localeAsIfCurrent(name: localeID, overrides: .init(firstWeekday: [.islamic : 6])) XCTAssertEqual(fri_islamic.firstDayOfWeek, shouldRespectUserPrefForIslamic ? .friday : expected, file: file, line: line) } From 344ecfe5efce44a20eb1507b63caa7d380a405b4 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Tue, 7 Mar 2023 14:12:24 -0800 Subject: [PATCH 09/21] rdar://106155597 -- Enable tests to run by refactoring out LocalePreferences. Also improves situation for non-Darwin builds --- .../Locale/Locale.swift | 17 +- .../Locale/Locale_Cache.swift | 6 +- .../Locale/Locale_ICU.swift | 511 ++++++++++-------- .../Locale/Locale_Wrappers.swift | 2 +- .../LocaleComponentsTests.swift | 8 +- .../LocaleTestUtilities.swift | 89 +-- .../LocaleTests.swift | 14 +- 7 files changed, 320 insertions(+), 327 deletions(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index 063b9688e..432e68008 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -115,12 +115,21 @@ public struct Locale : Hashable, Equatable, Sendable { return Locale(.fixed(LocaleCache.cache.system)) } +#if FOUNDATION_FRAMEWORK + /// This returns an instance of `Locale` that's set up exactly like it would be if the user changed the current locale to that identifier, set the preferences keys in the overrides dictionary, then called `current`. + internal static func localeAsIfCurrent(name: String?, cfOverrides: CFDictionary? = nil, disableBundleMatching: Bool = false) -> Locale { + let (inner, _) = _Locale._currentLocaleWithCFOverrides(name: name, overrides: cfOverrides, disableBundleMatching: disableBundleMatching) + return Locale(.fixed(inner)) + } +#endif /// This returns an instance of `Locale` that's set up exactly like it would be if the user changed the current locale to that identifier, set the preferences keys in the overrides dictionary, then called `current`. - internal static func localeAsIfCurrent(name: String?, overrides: CFDictionary? = nil, disableBundleMatching: Bool = false) -> Locale { - let (inner, _) = _Locale._currentLocale(name: name, overrides: overrides, disableBundleMatching: disableBundleMatching) + internal static func localeAsIfCurrent(name: String?, overrides: LocalePreferences? = nil, disableBundleMatching: Bool = false) -> Locale { + // On Darwin, this overrides are applied on top of CFPreferences. + let (inner, _) = _Locale._currentLocaleWithOverrides(name: name, overrides: overrides, disableBundleMatching: disableBundleMatching) return Locale(.fixed(inner)) } + internal static func localeAsIfCurrentWithBundleLocalizations(_ availableLocalizations: [String], allowsMixedLocalizations: Bool) -> Locale? { guard let inner = _Locale._currentLocaleWithBundleLocalizations(availableLocalizations, allowsMixedLocalizations: allowsMixedLocalizations) else { return nil @@ -162,7 +171,7 @@ public struct Locale : Hashable, Equatable, Sendable { @available(macOS 13, iOS 16, tvOS 16, watchOS 9, *) internal init(identifier: String, calendarIdentifier: Calendar.Identifier, firstWeekday: Locale.Weekday?, minimumDaysInFirstWeek: Int?) { - var prefs = _Locale.Prefs() + var prefs = LocalePreferences() if let firstWeekday { prefs.firstWeekday = [calendarIdentifier : firstWeekday.icuIndex] } @@ -188,7 +197,7 @@ public struct Locale : Hashable, Equatable, Sendable { } #endif - /// Produce a copy of the `Locale` (including `Prefs`, if present), but with a different `Calendar.Identifier`. Date formatting uses this. + /// Produce a copy of the `Locale` (including `LocalePreferences`, if present), but with a different `Calendar.Identifier`. Date formatting uses this. internal func copy(newCalendarIdentifier identifier: Calendar.Identifier) -> Locale { switch kind { case .fixed(let l): diff --git a/Sources/FoundationInternationalization/Locale/Locale_Cache.swift b/Sources/FoundationInternationalization/Locale/Locale_Cache.swift index 2158a8018..93680dc17 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Cache.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Cache.swift @@ -66,7 +66,7 @@ struct LocaleCache : Sendable { if let cachedCurrentLocale { return cachedCurrentLocale } else { - let (locale, doCache) = _Locale._currentLocale(name: nil, overrides: nil, disableBundleMatching: false) + let (locale, doCache) = _Locale._currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: false) if doCache { self.cachedCurrentLocale = locale } @@ -110,7 +110,7 @@ struct LocaleCache : Sendable { return nsLocale } else { // We have neither a Swift Locale nor an NSLocale. Recalculate and set both. - let (locale, doCache) = _Locale._currentLocale(name: nil, overrides: nil, disableBundleMatching: false) + let (locale, doCache) = _Locale._currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: false) let nsLocale = _NSSwiftLocale(Locale(inner: locale)) if doCache { // It's possible this was an 'incomplete locale', in which case we will want to calculate it again later. @@ -187,7 +187,7 @@ struct LocaleCache : Sendable { } var preferred: _Locale { - let (locale, _) = _Locale._currentLocale(name: nil, overrides: nil, disableBundleMatching: true) + let (locale, _) = _Locale._currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: false) return locale } diff --git a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift index aa2fe6e07..6c37be06a 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift @@ -26,169 +26,6 @@ import FoundationEssentials let MAX_ICU_NAME_SIZE: Int32 = 1024 internal final class _Locale: Sendable, Hashable { - - /// Holds user preferences about `Locale`, retrieved from user defaults. It is only used when creating the `current` Locale. Fixed-identiifer locales never have preferences. - struct Prefs: Hashable { - enum MeasurementUnit { - case centimeters - case inches - - /// Init with the value of a user defaults string - init?(_ string: String?) { - guard let string else { return nil } - if string == "Centimeters" { self = .centimeters } - else if string == "Inches" { self = .inches } - else { return nil } - } - - /// Get the value as a user defaults string - var userDefaultString: String { - switch self { - case .centimeters: return "Centimeters" - case .inches: return "Inches" - } - } - } - - enum TemperatureUnit { - case fahrenheit - case celsius - - /// Init with the value of a user defaults string - init?(_ string: String?) { - guard let string else { return nil } - if string == "Celsius" { self = .celsius } - else if string == "Fahrenheit" { self = .fahrenheit } - else { return nil } - } - - /// Get the value as a user defaults string - var userDefaultString: String { - switch self { - case .celsius: return "Celsius" - case .fahrenheit: return "Fahrenheit" - } - } - } - - var metricUnits: Bool? - var languages: [String]? - var locale: String? - var collationOrder: String? - var firstWeekday: [Calendar.Identifier : Int]? - var minDaysInFirstWeek: [Calendar.Identifier : Int]? -#if FOUNDATION_FRAMEWORK - var ICUDateTimeSymbols: CFDictionary? - var ICUDateFormatStrings: CFDictionary? - var ICUTimeFormatStrings: CFDictionary? - var ICUNumberFormatStrings: CFDictionary? - var ICUNumberSymbols: CFDictionary? -#endif - var country: String? - var measurementUnits: MeasurementUnit? - var temperatureUnit: TemperatureUnit? - var ICUForce24HourTime: Bool? - var ICUForce12HourTime: Bool? - - init() { } - - /// Interpret a dictionary (from user defaults) according to a predefined set of strings and convert it into the more strongly-typed `Prefs` values. - /// Several dictionaries may need to be applied to the same instance, which is why this is structured as a mutating setter rather than an initializer. - /// Why use a `CFDictionary` instead of a Swift dictionary here? The input prefs may be a complete copy of the user's prefs, and we don't want to bridge a ton of unrelated data into Swift just to extract a few keys. Keeping it as a `CFDictionary` avoids that overhead, and we call into small CF helper functions to get the data we need, if it is there. - mutating func apply(_ prefs: CFDictionary) { - var exists: DarwinBoolean = false - - guard CFDictionaryGetCount(prefs) > 0 else { return } - - if let langs = __CFLocalePrefsCopyAppleLanguages(prefs)?.takeRetainedValue() as? [String] { - self.languages = langs - } - if let locale = __CFLocalePrefsCopyAppleLocale(prefs)?.takeRetainedValue() as? String { - self.locale = locale - } - - let isMetric = __CFLocalePrefsAppleMetricUnitsIsMetric(prefs, &exists) - if exists.boolValue { - self.metricUnits = isMetric - } - - let isCentimeters = __CFLocalePrefsAppleMeasurementUnitsIsCm(prefs, &exists) - if exists.boolValue { - self.measurementUnits = isCentimeters ? .centimeters : .inches - } - - let isCelsius = __CFLocalePrefsAppleTemperatureUnitIsC(prefs, &exists) - if exists.boolValue { - self.temperatureUnit = isCelsius ? .celsius : .fahrenheit - } - - let is24Hour = __CFLocalePrefsAppleForce24HourTime(prefs, &exists) - if exists.boolValue { - self.ICUForce24HourTime = is24Hour - } - - let is12Hour = __CFLocalePrefsAppleForce12HourTime(prefs, &exists) - if exists.boolValue { - self.ICUForce12HourTime = is12Hour - } - - if let collationOrder = __CFLocalePrefsCopyAppleCollationOrder(prefs)?.takeRetainedValue() as? String { - self.collationOrder = collationOrder - } - - if let country = __CFLocalePrefsCopyCountry(prefs)?.takeRetainedValue() as? String { - self.country = country - } - - if let ICUDateTimeSymbols = __CFLocalePrefsCopyAppleICUDateTimeSymbols(prefs)?.takeRetainedValue() { - self.ICUDateTimeSymbols = ICUDateTimeSymbols - } - - if let ICUDateFormatStrings = __CFLocalePrefsCopyAppleICUDateFormatStrings(prefs)?.takeRetainedValue() { - self.ICUDateFormatStrings = ICUDateFormatStrings - } - - if let ICUTimeFormatStrings = __CFLocalePrefsCopyAppleICUTimeFormatStrings(prefs)?.takeRetainedValue() { - self.ICUTimeFormatStrings = ICUTimeFormatStrings - } - - if let ICUNumberFormatStrings = __CFLocalePrefsCopyAppleICUNumberFormatStrings(prefs)?.takeRetainedValue() { - self.ICUNumberFormatStrings = ICUNumberFormatStrings - } - - if let ICUNumberSymbols = __CFLocalePrefsCopyAppleICUNumberSymbols(prefs)?.takeRetainedValue() { - self.ICUNumberSymbols = ICUNumberSymbols - } - - - if let firstWeekdaysPrefs = __CFLocalePrefsCopyAppleFirstWeekday(prefs)?.takeRetainedValue() as? [String: Int] { - var mapped: [Calendar.Identifier : Int] = [:] - for (key, value) in firstWeekdaysPrefs { - if let id = Calendar.Identifier(identifierString: key) { - mapped[id] = value - } - } - - if !mapped.isEmpty { - self.firstWeekday = mapped - } - } - - if let minDaysPrefs = __CFLocalePrefsCopyAppleMinDaysInFirstWeek(prefs)?.takeRetainedValue() as? [String: Int] { - var mapped: [Calendar.Identifier : Int] = [:] - for (key, value) in minDaysPrefs { - if let id = Calendar.Identifier(identifierString: key) { - mapped[id] = value - } - } - - if !mapped.isEmpty { - self.minDaysInFirstWeek = mapped - } - } - } - } - // Double-nil values are caches where the result may be nil. If the outer value is nil, the result has not yet been calculated. // Single-nil values are caches where the result may not be nil. If the value is nil, the result has not yet been calculated. struct State: Hashable, Sendable { @@ -257,12 +94,12 @@ internal final class _Locale: Sendable, Hashable { internal let identifier: String internal let doesNotRequireSpecialCaseHandling: Bool - private let prefs: Prefs? + private let prefs: LocalePreferences? private let lock: LockedState // MARK: - init - init(identifier: String, prefs: Prefs? = nil) { + init(identifier: String, prefs: LocalePreferences? = nil) { self.identifier = Locale._canonicalLocaleIdentifier(from: identifier) doesNotRequireSpecialCaseHandling = Self.identifierDoesNotRequireSpecialCaseHandling(self.identifier) self.prefs = prefs @@ -350,9 +187,9 @@ internal final class _Locale: Sendable, Hashable { case "AppleICUDateTimeSymbols": return prefs.ICUDateTimeSymbols case "AppleICUForce24HourTime": - return prefs.ICUForce24HourTime + return prefs.force24Hour case "AppleICUForce12HourTime": - return prefs.ICUForce12HourTime + return prefs.force12Hour case "AppleICUDateFormatStrings": return prefs.ICUDateFormatStrings case "AppleICUTimeFormatStrings": @@ -1180,7 +1017,7 @@ internal final class _Locale: Sendable, Hashable { internal var force24Hour: Bool { if let prefs { - return prefs.ICUForce24HourTime ?? false + return prefs.force24Hour ?? false } return false @@ -1188,7 +1025,7 @@ internal final class _Locale: Sendable, Hashable { internal var force12Hour: Bool { if let prefs { - return prefs.ICUForce12HourTime ?? false + return prefs.force12Hour ?? false } return false @@ -1569,11 +1406,64 @@ internal final class _Locale: Sendable, Hashable { }() #endif // FOUNDATION_FRAMEWORK - internal static func _currentLocale(name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> (_Locale, Bool) { - /* - NOTE: calling any CFPreferences function, or any function which calls into a CFPreferences function, *except* for __CFXPreferencesCopyCurrentApplicationStateWithDeadlockAvoidance (and accepting backstop values if its outparam is false), will deadlock. This is because CFPreferences calls os_log_*, which calls -descriptionWithLocale:, which calls CFLocaleCopyCurrent. - */ +#if FOUNDATION_FRAMEWORK + static func _prefsFromCFPrefs() -> (CFDictionary, Bool) { + // On Darwin, we check the current user preferences for Locale values + var wouldDeadlock: DarwinBoolean = false + let cfPrefs = __CFXPreferencesCopyCurrentApplicationStateWithDeadlockAvoidance(&wouldDeadlock).takeRetainedValue() + + if wouldDeadlock.boolValue { + // Don't cache a locale built with incomplete prefs + return (cfPrefs, false) + } else { + return (cfPrefs, true) + } + } + + /// Create a `Locale` that acts like a `currentLocale`, using `CFPreferences` values with `CFDictionary` overrides. + internal static func _currentLocaleWithCFOverrides(name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> (_Locale, Bool) { + let (cfPrefs, wouldDeadlock) = _prefsFromCFPrefs() + let suitableForCache = disableBundleMatching ? false : (wouldDeadlock ? false : true) + var prefs = LocalePreferences() + prefs.apply(cfPrefs) + if let overrides { + prefs.apply(overrides) + } + let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) + return (result, suitableForCache) + } + + /// Create a `Locale` that acts like a `currentLocale`, using `CFPreferences` values and `LocalePreferences` overrides. + internal static func _currentLocaleWithOverrides(name: String?, overrides: LocalePreferences?, disableBundleMatching: Bool) -> (_Locale, Bool) { + let (cfPrefs, wouldDeadlock) = _prefsFromCFPrefs() + let suitableForCache = disableBundleMatching ? false : (wouldDeadlock ? false : true) + var prefs = LocalePreferences() + prefs.apply(cfPrefs) + if let overrides { + prefs.apply(overrides) + } + let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) + return (result, suitableForCache) + } +#else + /// Create a `Locale` that acts like a `currentLocale`, using default values and `LocalePreferences` overrides. + internal static func _currentLocaleWithOverrides(name: String?, overrides: LocalePreferences?, disableBundleMatching: Bool) -> (_Locale, Bool) { + let suitableForCache = disableBundleMatching ? false : true + + // On this platform, preferences start with default values + var prefs = LocalePreferences() + prefs.locale = "en_US" + prefs.languages = ["en-US"] + + // Apply overrides + if let overrides { prefs.apply(overrides) } + let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) + return (result, suitableForCache) + } +#endif + + private static func _localeWithPreferences(name: String?, prefs: LocalePreferences, disableBundleMatching: Bool) -> _Locale { var ident: String? if let name { ident = Locale._canonicalLocaleIdentifier(from: name) @@ -1589,65 +1479,24 @@ internal final class _Locale: Sendable, Hashable { #endif // FOUNDATION_FRAMEWORK } - // If `disableBundleMatching` is true, caching needs to be turned off, only a single value is cached for the most common case of calling `localeCurrent`. - var suitableForCache = disableBundleMatching ? false : true - - var prefs = Prefs() - -#if os(Windows) - fatalError() - // TODO: Needs port from swift-corelibs-foundation's CFPreferences.c, CFBundle_Locale.c - /* - // From CFPreferences.c - let cfPrefs = __CFXPreferencesCopyCurrentApplicationState() - - // From CFBundle_Locale.c - // CFStringRef copyLocaleLanguageName(void); - // CFStringRef copyLocaleCountryName(void); - - ident = countryName ? "\(langName ? langName : "en")_\(countryName)" : "\(langName ? langName : "en")" - */ -#else - #if FOUNDATION_FRAMEWORK - // On Darwin, we check the current user preferences for Locale values - var wouldDeadlock: DarwinBoolean = false - let cfPrefs = __CFXPreferencesCopyCurrentApplicationStateWithDeadlockAvoidance(&wouldDeadlock).takeRetainedValue() - - if wouldDeadlock.boolValue { - // Don't cache a locale built with incomplete prefs - suitableForCache = false - } - - prefs.apply(cfPrefs) - - #endif // FOUNDATION_FRAMEWORK -#endif // os(...) - - if let overrides { prefs.apply(overrides) } - if let identSet = ident { ident = Locale._canonicalLocaleIdentifier(from: identSet) } else { let preferredLocale = prefs.locale // If CFBundleAllowMixedLocalizations is set, don't do any checking of the user's preferences for locale-matching purposes (32264371) - #if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) +#if FOUNDATION_FRAMEWORK let allowMixed = Bundle.main.infoDictionary?["CFBundleAllowMixedLocalizations"] as? Bool ?? false - #else +#else let allowMixed = false - #endif +#endif let performBundleMatching = !disableBundleMatching && !allowMixed let preferredLanguages = prefs.languages #if FOUNDATION_FRAMEWORK if preferredLanguages == nil && (preferredLocale == nil || performBundleMatching) { - // We were going to use preferredLanguages - if wouldDeadlock.boolValue { - Logger(log).debug("Lookup of 'AppleLanguages' from current preferences failed (lookup would deadlock due to re-entrancy); likely falling back to default locale identifier as current") - } else { - Logger(log).debug("Lookup of 'AppleLanguages' from current preferences failed lookup (app preferences do not contain the key); likely falling back to default locale identifier as current") - } + Logger(log).debug("Lookup of 'AppleLanguages' from current preferences failed lookup (app preferences do not contain the key); likely falling back to default locale identifier as current") } #endif @@ -1709,13 +1558,13 @@ internal final class _Locale: Sendable, Hashable { #endif } let locale = _Locale(identifier: ident!, prefs: prefs) - return (locale, suitableForCache) + return locale } internal static func _currentLocaleWithBundleLocalizations(_ availableLocalizations: [String], allowsMixedLocalizations: Bool) -> _Locale? { #if FOUNDATION_FRAMEWORK guard !allowsMixedLocalizations else { - let (result, _) = _currentLocale(name: nil, overrides: nil, disableBundleMatching: true) + let (result, _) = _currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: true) return result } @@ -1739,3 +1588,225 @@ internal final class _Locale: Sendable, Hashable { } +// MARK: - + +/// Holds user preferences about `Locale`, retrieved from user defaults. It is only used when creating the `current` Locale. Fixed-identiifer locales never have preferences. +struct LocalePreferences: Hashable { + enum MeasurementUnit { + case centimeters + case inches + + /// Init with the value of a user defaults string + init?(_ string: String?) { + guard let string else { return nil } + if string == "Centimeters" { self = .centimeters } + else if string == "Inches" { self = .inches } + else { return nil } + } + + /// Get the value as a user defaults string + var userDefaultString: String { + switch self { + case .centimeters: return "Centimeters" + case .inches: return "Inches" + } + } + } + + enum TemperatureUnit { + case fahrenheit + case celsius + + /// Init with the value of a user defaults string + init?(_ string: String?) { + guard let string else { return nil } + if string == "Celsius" { self = .celsius } + else if string == "Fahrenheit" { self = .fahrenheit } + else { return nil } + } + + /// Get the value as a user defaults string + var userDefaultString: String { + switch self { + case .celsius: return "Celsius" + case .fahrenheit: return "Fahrenheit" + } + } + } + + var metricUnits: Bool? + var languages: [String]? + var locale: String? + var collationOrder: String? + var firstWeekday: [Calendar.Identifier : Int]? + var minDaysInFirstWeek: [Calendar.Identifier : Int]? +#if FOUNDATION_FRAMEWORK + var ICUDateTimeSymbols: CFDictionary? + var ICUDateFormatStrings: CFDictionary? + var ICUTimeFormatStrings: CFDictionary? + var ICUNumberFormatStrings: CFDictionary? + var ICUNumberSymbols: CFDictionary? +#endif + var country: String? + var measurementUnits: MeasurementUnit? + var temperatureUnit: TemperatureUnit? + var force24Hour: Bool? + var force12Hour: Bool? + + init() { } + + init(metricUnits: Bool? = nil, + languages: [String]? = nil, + locale: String? = nil, + collationOrder: String? = nil, + firstWeekday: [Calendar.Identifier : Int]? = nil, + minDaysInFirstWeek: [Calendar.Identifier : Int]? = nil, + country: String? = nil, + measurementUnits: MeasurementUnit? = nil, + temperatureUnit: TemperatureUnit? = nil, + force24Hour: Bool? = nil, + force12Hour: Bool? = nil) { + + self.metricUnits = metricUnits + self.languages = languages + self.locale = locale + self.collationOrder = collationOrder + self.firstWeekday = firstWeekday + self.minDaysInFirstWeek = minDaysInFirstWeek + self.country = country + self.measurementUnits = measurementUnits + self.temperatureUnit = temperatureUnit + self.force24Hour = force24Hour + self.force12Hour = force12Hour + +#if FOUNDATION_FRAMEWORK + ICUDateTimeSymbols = nil + ICUDateFormatStrings = nil + ICUTimeFormatStrings = nil + ICUNumberFormatStrings = nil + ICUNumberSymbols = nil +#endif + } + +#if FOUNDATION_FRAMEWORK + /// Interpret a dictionary (from user defaults) according to a predefined set of strings and convert it into the more strongly-typed `LocalePreferences` values. + /// Several dictionaries may need to be applied to the same instance, which is why this is structured as a mutating setter rather than an initializer. + /// Why use a `CFDictionary` instead of a Swift dictionary here? The input prefs may be a complete copy of the user's prefs, and we don't want to bridge a ton of unrelated data into Swift just to extract a few keys. Keeping it as a `CFDictionary` avoids that overhead, and we call into small CF helper functions to get the data we need, if it is there. + mutating func apply(_ prefs: CFDictionary) { + var exists: DarwinBoolean = false + + guard CFDictionaryGetCount(prefs) > 0 else { return } + + if let langs = __CFLocalePrefsCopyAppleLanguages(prefs)?.takeRetainedValue() as? [String] { + self.languages = langs + } + if let locale = __CFLocalePrefsCopyAppleLocale(prefs)?.takeRetainedValue() as? String { + self.locale = locale + } + + let isMetric = __CFLocalePrefsAppleMetricUnitsIsMetric(prefs, &exists) + if exists.boolValue { + self.metricUnits = isMetric + } + + let isCentimeters = __CFLocalePrefsAppleMeasurementUnitsIsCm(prefs, &exists) + if exists.boolValue { + self.measurementUnits = isCentimeters ? .centimeters : .inches + } + + let isCelsius = __CFLocalePrefsAppleTemperatureUnitIsC(prefs, &exists) + if exists.boolValue { + self.temperatureUnit = isCelsius ? .celsius : .fahrenheit + } + + let is24Hour = __CFLocalePrefsAppleForce24HourTime(prefs, &exists) + if exists.boolValue { + self.force24Hour = is24Hour + } + + let is12Hour = __CFLocalePrefsAppleForce12HourTime(prefs, &exists) + if exists.boolValue { + self.force12Hour = is12Hour + } + + if let collationOrder = __CFLocalePrefsCopyAppleCollationOrder(prefs)?.takeRetainedValue() as? String { + self.collationOrder = collationOrder + } + + if let country = __CFLocalePrefsCopyCountry(prefs)?.takeRetainedValue() as? String { + self.country = country + } + + if let ICUDateTimeSymbols = __CFLocalePrefsCopyAppleICUDateTimeSymbols(prefs)?.takeRetainedValue() { + self.ICUDateTimeSymbols = ICUDateTimeSymbols + } + + if let ICUDateFormatStrings = __CFLocalePrefsCopyAppleICUDateFormatStrings(prefs)?.takeRetainedValue() { + self.ICUDateFormatStrings = ICUDateFormatStrings + } + + if let ICUTimeFormatStrings = __CFLocalePrefsCopyAppleICUTimeFormatStrings(prefs)?.takeRetainedValue() { + self.ICUTimeFormatStrings = ICUTimeFormatStrings + } + + if let ICUNumberFormatStrings = __CFLocalePrefsCopyAppleICUNumberFormatStrings(prefs)?.takeRetainedValue() { + self.ICUNumberFormatStrings = ICUNumberFormatStrings + } + + if let ICUNumberSymbols = __CFLocalePrefsCopyAppleICUNumberSymbols(prefs)?.takeRetainedValue() { + self.ICUNumberSymbols = ICUNumberSymbols + } + + + if let firstWeekdaysPrefs = __CFLocalePrefsCopyAppleFirstWeekday(prefs)?.takeRetainedValue() as? [String: Int] { + var mapped: [Calendar.Identifier : Int] = [:] + for (key, value) in firstWeekdaysPrefs { + if let id = Calendar.Identifier(identifierString: key) { + mapped[id] = value + } + } + + if !mapped.isEmpty { + self.firstWeekday = mapped + } + } + + if let minDaysPrefs = __CFLocalePrefsCopyAppleMinDaysInFirstWeek(prefs)?.takeRetainedValue() as? [String: Int] { + var mapped: [Calendar.Identifier : Int] = [:] + for (key, value) in minDaysPrefs { + if let id = Calendar.Identifier(identifierString: key) { + mapped[id] = value + } + } + + if !mapped.isEmpty { + self.minDaysInFirstWeek = mapped + } + } + } +#endif // FOUNDATION_FRAMEWORK + + /// For testing purposes, merge a set of override prefs into this one. + mutating func apply(_ prefs: LocalePreferences) { + if let other = prefs.metricUnits { self.metricUnits = other } + if let other = prefs.languages { self.languages = other } + if let other = prefs.locale { self.locale = other } + if let other = prefs.collationOrder { self.collationOrder = other } + if let other = prefs.firstWeekday { self.firstWeekday = other } + if let other = prefs.minDaysInFirstWeek { self.minDaysInFirstWeek = other } +#if FOUNDATION_FRAMEWORK + if let other = prefs.ICUDateTimeSymbols { self.ICUDateTimeSymbols = other } + if let other = prefs.ICUDateFormatStrings { self.ICUDateFormatStrings = other } + if let other = prefs.ICUTimeFormatStrings { self.ICUTimeFormatStrings = other } + if let other = prefs.ICUNumberFormatStrings { self.ICUNumberFormatStrings = other } + if let other = prefs.ICUNumberSymbols { self.ICUNumberSymbols = other } +#endif + if let other = prefs.country { self.country = other } + if let other = prefs.measurementUnits { self.measurementUnits = other } + if let other = prefs.temperatureUnit { self.temperatureUnit = other } + if let other = prefs.force24Hour { self.force24Hour = other } + if let other = prefs.force12Hour { self.force12Hour = other } + } +} + + diff --git a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift index 934460b8c..2b449e3bd 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift @@ -43,7 +43,7 @@ extension NSLocale { @objc private class func _newLocaleAsIfCurrent(_ name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> NSLocale? { - let inner = Locale.localeAsIfCurrent(name: name, overrides: overrides, disableBundleMatching: disableBundleMatching) + let inner = Locale.localeAsIfCurrent(name: name, cfOverrides: overrides, disableBundleMatching: disableBundleMatching) return _NSSwiftLocale(inner) } diff --git a/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift b/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift index 52138537c..e2f34dab3 100644 --- a/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift @@ -287,15 +287,15 @@ final class LocaleComponentsTests: XCTestCase { let nonCurrentDefault = Locale.Components(locale: loc) XCTAssertEqual(nonCurrentDefault.hourCycle, expectDefault, "default did not match", file: file, line: line) - let defaultLoc = Locale.likeCurrent(identifier: localeID, preferences: .init()) + let defaultLoc = Locale.localeAsIfCurrent(name: localeID, overrides: .init()) let defaultComp = Locale.Components(locale: defaultLoc) XCTAssertEqual(defaultComp.hourCycle, expectDefault, "explicit no override did not match", file: file, line: line) - let force24 = Locale.likeCurrent(identifier: localeID, preferences: .init(force24Hour: true)) + let force24 = Locale.localeAsIfCurrent(name: localeID, overrides: .init(force24Hour: true)) let force24Comp = Locale.Components(locale: force24) XCTAssertEqual(force24Comp.hourCycle, shouldRespectUserPref ? .zeroToTwentyThree : expectDefault, "force 24-hr did not match", file: file, line: line) - let force12 = Locale.likeCurrent(identifier: localeID, preferences: .init(force12Hour: true)) + let force12 = Locale.localeAsIfCurrent(name: localeID, overrides: .init(force12Hour: true)) let force12Comp = Locale.Components(locale: force12) XCTAssertEqual(force12Comp.hourCycle, shouldRespectUserPref ? .oneToTwelve : expectDefault, "force 12-hr did not match", file: file, line: line) } @@ -313,7 +313,7 @@ final class LocaleComponentsTests: XCTestCase { } func test_userPreferenceOverrideRoundtrip() { - let customLocale = Locale.likeCurrent(identifier: "en_US", preferences: .init(measurementSystem: .metric, force24Hour: true, firstWeekday: [.gregorian: .wednesday])) + let customLocale = Locale.localeAsIfCurrent(name: "en_US", overrides: .init(metricUnits: true, firstWeekday: [.gregorian: Locale.Weekday.wednesday.icuIndex], measurementUnits: .centimeters, force24Hour: true)) XCTAssertEqual(customLocale.identifier, "en_US") XCTAssertEqual(customLocale.hourCycle, .zeroToTwentyThree) XCTAssertEqual(customLocale.firstDayOfWeek, .wednesday) diff --git a/Tests/FoundationInternationalizationTests/LocaleTestUtilities.swift b/Tests/FoundationInternationalizationTests/LocaleTestUtilities.swift index 378220007..d74806c5f 100644 --- a/Tests/FoundationInternationalizationTests/LocaleTestUtilities.swift +++ b/Tests/FoundationInternationalizationTests/LocaleTestUtilities.swift @@ -20,95 +20,8 @@ @testable import FoundationInternationalization #endif // FOUNDATION_FRAMEWORK -let metricUnitsKey = "AppleMetricUnits" -let measurementUnitsKey = "AppleMeasurementUnits" -let force24HourKey = "AppleICUForce24HourTime" -let force12HourKey = "AppleICUForce12HourTime" -let temperatureUnitKey = "AppleTemperatureUnit" -let firstWeekdayKey = "AppleFirstWeekday" - -let cm = "Centimeters" -let inch = "Inches" - -struct LocalePreferences { - var measurementSystem: Locale.MeasurementSystem? - var force24Hour: Bool? - var force12Hour: Bool? - var temperatureUnit: UnitTemperature? - var firstWeekday: [Calendar.Identifier : Locale.Weekday]? - init(measurementSystem: Locale.MeasurementSystem? = nil, force24Hour: Bool? = nil, force12Hour: Bool? = nil, temperatureUnit: UnitTemperature? = nil, firstWeekday: [Calendar.Identifier : Locale.Weekday]? = nil) { - self.measurementSystem = measurementSystem - self.force24Hour = force24Hour - self.force12Hour = force12Hour - self.temperatureUnit = temperatureUnit - self.firstWeekday = firstWeekday - } -} - -extension Locale { - static func likeCurrent(identifier: String, preferences: LocalePreferences) -> Locale { - var override = [String : Any]() - if let measurementSystem = preferences.measurementSystem { - switch measurementSystem { - case .metric: - override[metricUnitsKey] = true - override[measurementUnitsKey] = cm - case .us: - override[metricUnitsKey] = false - override[measurementUnitsKey] = inch - case .uk: - override[metricUnitsKey] = true - override[measurementUnitsKey] = inch - default: - override[metricUnitsKey] = Null() - override[measurementUnitsKey] = Null() - } - } else { - override[metricUnitsKey] = Null() - override[measurementUnitsKey] = Null() - } - - if let force12Hour = preferences.force12Hour { - override[force12HourKey] = force12Hour - } else { - override[force12HourKey] = Null() - } - - if let force24Hour = preferences.force24Hour { - override[force24HourKey] = force24Hour - } else { - override[force24HourKey] = Null() - } - - if let temperatureUnit = preferences.temperatureUnit { - switch temperatureUnit { - case .celsius: - override[temperatureUnitKey] = "Celsius" - case .fahrenheit: - override[temperatureUnitKey] = "Fahrenheit" - default: - override[temperatureUnitKey] = Null() - } - } else { - override[temperatureUnitKey] = Null() - } - - if let firstWeekday = preferences.firstWeekday { - let mapped = Dictionary(uniqueKeysWithValues: firstWeekday.map({ key, value in - return (key.cldrIdentifier, value.icuIndex) - })) - - override[firstWeekdayKey] = mapped - } else { - override[firstWeekdayKey] = Null() - } - return Locale.localeAsIfCurrent(name: identifier, overrides: override) - } - -} - // MARK: - Stubs -fileprivate struct Null {} + #if !FOUNDATION_FRAMEWORK internal enum UnitTemperature { case celsius diff --git a/Tests/FoundationInternationalizationTests/LocaleTests.swift b/Tests/FoundationInternationalizationTests/LocaleTests.swift index cf7506097..128ff6bac 100644 --- a/Tests/FoundationInternationalizationTests/LocaleTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleTests.swift @@ -334,13 +334,13 @@ final class LocalePropertiesTests : XCTestCase { let loc = Locale(identifier: localeID) XCTAssertEqual(loc.hourCycle, expectDefault, "default did not match", file: file, line: line) - let defaultLoc = Locale.likeCurrent(identifier: localeID, preferences: .init()) + let defaultLoc = Locale.localeAsIfCurrent(name: localeID, overrides: .init()) XCTAssertEqual(defaultLoc.hourCycle, expectDefault, "explicit no override did not match", file: file, line: line) - let force24 = Locale.likeCurrent(identifier: localeID, preferences: .init(force24Hour: true)) + let force24 = Locale.localeAsIfCurrent(name: localeID, overrides: .init(force24Hour: true)) XCTAssertEqual(force24.hourCycle, shouldRespectUserPref ? .zeroToTwentyThree : expectDefault, "force 24-hr did not match", file: file, line: line) - let force12 = Locale.likeCurrent(identifier: localeID, preferences: .init(force12Hour: true)) + let force12 = Locale.localeAsIfCurrent(name: localeID, overrides: .init(force12Hour: true)) XCTAssertEqual(force12.hourCycle, shouldRespectUserPref ? .oneToTwelve : expectDefault, "force 12-hr did not match", file: file, line: line) } @@ -364,16 +364,16 @@ final class LocalePropertiesTests : XCTestCase { func test_userPreferenceOverride_measurementSystem() { func verify(_ localeID: String, _ expected: Locale.MeasurementSystem, shouldRespectUserPref: Bool, file: StaticString = #file, line: UInt = #line) { - let localeNoPref = Locale.likeCurrent(identifier: localeID, preferences: .init()) + let localeNoPref = Locale.localeAsIfCurrent(name: localeID, overrides: .init()) XCTAssertEqual(localeNoPref.measurementSystem, expected, file: file, line: line) - let fakeCurrentMetric = Locale.likeCurrent(identifier: localeID, preferences: .init(measurementSystem: .metric)) + let fakeCurrentMetric = Locale.localeAsIfCurrent(name: localeID, overrides: .init(metricUnits: true, measurementUnits: .centimeters)) XCTAssertEqual(fakeCurrentMetric.measurementSystem, shouldRespectUserPref ? .metric : expected, file: file, line: line) - let fakeCurrentUS = Locale.likeCurrent(identifier: localeID, preferences: .init(measurementSystem: .us)) + let fakeCurrentUS = Locale.localeAsIfCurrent(name: localeID, overrides: .init(metricUnits: false, measurementUnits: .inches)) XCTAssertEqual(fakeCurrentUS.measurementSystem, shouldRespectUserPref ? .us : expected, file: file, line: line) - let fakeCurrentUK = Locale.likeCurrent(identifier: localeID, preferences: .init(measurementSystem: .uk)) + let fakeCurrentUK = Locale.localeAsIfCurrent(name: localeID, overrides: .init(metricUnits: true, measurementUnits: .inches)) XCTAssertEqual(fakeCurrentUK.measurementSystem, shouldRespectUserPref ? .uk : expected, file: file, line: line) } From 0b409c1daeb952f3e764eee291b1119d3f697184 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Tue, 7 Mar 2023 16:32:24 -0800 Subject: [PATCH 10/21] rdar://106155597 - Clarify purpose of keeping CFDictionary around in Swift; fix ugly ternary --- .../Locale/Locale_ICU.swift | 85 ++++++++++--------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift index 6c37be06a..45e9adb6c 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift @@ -185,19 +185,19 @@ internal final class _Locale: Sendable, Hashable { } return result case "AppleICUDateTimeSymbols": - return prefs.ICUDateTimeSymbols + return prefs.icuDateTimeSymbols case "AppleICUForce24HourTime": return prefs.force24Hour case "AppleICUForce12HourTime": return prefs.force12Hour case "AppleICUDateFormatStrings": - return prefs.ICUDateFormatStrings + return prefs.icuDateFormatStrings case "AppleICUTimeFormatStrings": - return prefs.ICUTimeFormatStrings + return prefs.icuTimeFormatStrings case "AppleICUNumberFormatStrings": - return prefs.ICUNumberFormatStrings + return prefs.icuNumberFormatStrings case "AppleICUNumberSymbols": - return prefs.ICUNumberSymbols + return prefs.icuNumberSymbols default: return nil } @@ -1407,42 +1407,43 @@ internal final class _Locale: Sendable, Hashable { #endif // FOUNDATION_FRAMEWORK #if FOUNDATION_FRAMEWORK - static func _prefsFromCFPrefs() -> (CFDictionary, Bool) { + static func _prefsFromCFPrefs() -> (LocalePreferences, Bool) { // On Darwin, we check the current user preferences for Locale values var wouldDeadlock: DarwinBoolean = false let cfPrefs = __CFXPreferencesCopyCurrentApplicationStateWithDeadlockAvoidance(&wouldDeadlock).takeRetainedValue() + var prefs = LocalePreferences() + prefs.apply(cfPrefs) + if wouldDeadlock.boolValue { // Don't cache a locale built with incomplete prefs - return (cfPrefs, false) + return (prefs, false) } else { - return (cfPrefs, true) + return (prefs, true) } } /// Create a `Locale` that acts like a `currentLocale`, using `CFPreferences` values with `CFDictionary` overrides. internal static func _currentLocaleWithCFOverrides(name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> (_Locale, Bool) { - let (cfPrefs, wouldDeadlock) = _prefsFromCFPrefs() - let suitableForCache = disableBundleMatching ? false : (wouldDeadlock ? false : true) - var prefs = LocalePreferences() - prefs.apply(cfPrefs) + var (prefs, wouldDeadlock) = _prefsFromCFPrefs() if let overrides { prefs.apply(overrides) } let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) + + let suitableForCache = !disableBundleMatching && !wouldDeadlock return (result, suitableForCache) } /// Create a `Locale` that acts like a `currentLocale`, using `CFPreferences` values and `LocalePreferences` overrides. internal static func _currentLocaleWithOverrides(name: String?, overrides: LocalePreferences?, disableBundleMatching: Bool) -> (_Locale, Bool) { - let (cfPrefs, wouldDeadlock) = _prefsFromCFPrefs() - let suitableForCache = disableBundleMatching ? false : (wouldDeadlock ? false : true) - var prefs = LocalePreferences() - prefs.apply(cfPrefs) + var (prefs, wouldDeadlock) = _prefsFromCFPrefs() if let overrides { prefs.apply(overrides) } let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) + + let suitableForCache = !disableBundleMatching && !wouldDeadlock return (result, suitableForCache) } #else @@ -1641,11 +1642,13 @@ struct LocalePreferences: Hashable { var firstWeekday: [Calendar.Identifier : Int]? var minDaysInFirstWeek: [Calendar.Identifier : Int]? #if FOUNDATION_FRAMEWORK - var ICUDateTimeSymbols: CFDictionary? - var ICUDateFormatStrings: CFDictionary? - var ICUTimeFormatStrings: CFDictionary? - var ICUNumberFormatStrings: CFDictionary? - var ICUNumberSymbols: CFDictionary? + // The following `CFDictionary` ivars are used directly by `CFDateFormatter`. Keep them as `CFDictionary` to avoid bridging them into and out of Swift. We don't need to access them from Swift at all. + + var icuDateTimeSymbols: CFDictionary? + var icuDateFormatStrings: CFDictionary? + var icuTimeFormatStrings: CFDictionary? + var icuNumberFormatStrings: CFDictionary? + var icuNumberSymbols: CFDictionary? #endif var country: String? var measurementUnits: MeasurementUnit? @@ -1680,11 +1683,11 @@ struct LocalePreferences: Hashable { self.force12Hour = force12Hour #if FOUNDATION_FRAMEWORK - ICUDateTimeSymbols = nil - ICUDateFormatStrings = nil - ICUTimeFormatStrings = nil - ICUNumberFormatStrings = nil - ICUNumberSymbols = nil + icuDateTimeSymbols = nil + icuDateFormatStrings = nil + icuTimeFormatStrings = nil + icuNumberFormatStrings = nil + icuNumberSymbols = nil #endif } @@ -1737,24 +1740,24 @@ struct LocalePreferences: Hashable { self.country = country } - if let ICUDateTimeSymbols = __CFLocalePrefsCopyAppleICUDateTimeSymbols(prefs)?.takeRetainedValue() { - self.ICUDateTimeSymbols = ICUDateTimeSymbols + if let icuDateTimeSymbols = __CFLocalePrefsCopyAppleICUDateTimeSymbols(prefs)?.takeRetainedValue() { + self.icuDateTimeSymbols = icuDateTimeSymbols } - if let ICUDateFormatStrings = __CFLocalePrefsCopyAppleICUDateFormatStrings(prefs)?.takeRetainedValue() { - self.ICUDateFormatStrings = ICUDateFormatStrings + if let icuDateFormatStrings = __CFLocalePrefsCopyAppleICUDateFormatStrings(prefs)?.takeRetainedValue() { + self.icuDateFormatStrings = icuDateFormatStrings } - if let ICUTimeFormatStrings = __CFLocalePrefsCopyAppleICUTimeFormatStrings(prefs)?.takeRetainedValue() { - self.ICUTimeFormatStrings = ICUTimeFormatStrings + if let icuTimeFormatStrings = __CFLocalePrefsCopyAppleICUTimeFormatStrings(prefs)?.takeRetainedValue() { + self.icuTimeFormatStrings = icuTimeFormatStrings } - if let ICUNumberFormatStrings = __CFLocalePrefsCopyAppleICUNumberFormatStrings(prefs)?.takeRetainedValue() { - self.ICUNumberFormatStrings = ICUNumberFormatStrings + if let icuNumberFormatStrings = __CFLocalePrefsCopyAppleICUNumberFormatStrings(prefs)?.takeRetainedValue() { + self.icuNumberFormatStrings = icuNumberFormatStrings } - if let ICUNumberSymbols = __CFLocalePrefsCopyAppleICUNumberSymbols(prefs)?.takeRetainedValue() { - self.ICUNumberSymbols = ICUNumberSymbols + if let icuNumberSymbols = __CFLocalePrefsCopyAppleICUNumberSymbols(prefs)?.takeRetainedValue() { + self.icuNumberSymbols = icuNumberSymbols } @@ -1795,11 +1798,11 @@ struct LocalePreferences: Hashable { if let other = prefs.firstWeekday { self.firstWeekday = other } if let other = prefs.minDaysInFirstWeek { self.minDaysInFirstWeek = other } #if FOUNDATION_FRAMEWORK - if let other = prefs.ICUDateTimeSymbols { self.ICUDateTimeSymbols = other } - if let other = prefs.ICUDateFormatStrings { self.ICUDateFormatStrings = other } - if let other = prefs.ICUTimeFormatStrings { self.ICUTimeFormatStrings = other } - if let other = prefs.ICUNumberFormatStrings { self.ICUNumberFormatStrings = other } - if let other = prefs.ICUNumberSymbols { self.ICUNumberSymbols = other } + if let other = prefs.icuDateTimeSymbols { self.icuDateTimeSymbols = other } + if let other = prefs.icuDateFormatStrings { self.icuDateFormatStrings = other } + if let other = prefs.icuTimeFormatStrings { self.icuTimeFormatStrings = other } + if let other = prefs.icuNumberFormatStrings { self.icuNumberFormatStrings = other } + if let other = prefs.icuNumberSymbols { self.icuNumberSymbols = other } #endif if let other = prefs.country { self.country = other } if let other = prefs.measurementUnits { self.measurementUnits = other } From e6986a0934f132bdbd91dc9a77c4c1319ca1a01f Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Fri, 3 Mar 2023 16:26:09 -0800 Subject: [PATCH 11/21] rdar://106200399 (Fix Calendar Tests) --- .../Calendar/Calendar_ICU.swift | 46 +++++++++---------- .../Locale/Locale.swift | 24 +++++----- .../Locale/Locale_ICU.swift | 4 +- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/Sources/FoundationInternationalization/Calendar/Calendar_ICU.swift b/Sources/FoundationInternationalization/Calendar/Calendar_ICU.swift index dd21bb0e9..9316779f3 100644 --- a/Sources/FoundationInternationalization/Calendar/Calendar_ICU.swift +++ b/Sources/FoundationInternationalization/Calendar/Calendar_ICU.swift @@ -31,13 +31,12 @@ internal final class _Calendar: Equatable, @unchecked Sendable { // These custom values take precedence over the locale values private var customFirstWeekday: Int? private var customMinimumFirstDaysInWeek: Int? - private var localeIdentifier: String - - @available(macOS 13, iOS 16, tvOS 16, watchOS 9, *) - private var localeFirstWeekday: Locale.Weekday? - - private var localeMinimumFirstDaysInWeek: Int? + // Identifier of any locale used + private var localeIdentifier: String + // Custom user preferences of any locale used (current locale or current locale imitation only). We need to store this to correctly rebuild a Locale that has been stored inside Calendar as an identifier. + private var localePrefs: LocalePreferences? + let customGregorianStartDate: Date? internal init(identifier: Calendar.Identifier, @@ -54,27 +53,24 @@ internal final class _Calendar: Equatable, @unchecked Sendable { // We do not store the Locale here, as Locale stores a Calendar. We only keep the values we need that affect Calendar's operation. if let locale { localeIdentifier = locale.identifier - localeFirstWeekday = locale.forceFirstWeekday(identifier) - localeMinimumFirstDaysInWeek = locale.forceMinDaysInFirstWeek(identifier) + localePrefs = locale.prefs } else { localeIdentifier = "" - localeFirstWeekday = nil - localeMinimumFirstDaysInWeek = nil + localePrefs = nil } _timeZone = timeZone ?? TimeZone.default customFirstWeekday = firstWeekday customMinimumFirstDaysInWeek = minimumDaysInFirstWeek customGregorianStartDate = gregorianStartDate - - ucalendar = Self.icuCalendar(identifier: identifier, timeZone: _timeZone, localeIdentifier: localeIdentifier, localeFirstWeekday: localeFirstWeekday, localeMinimumDaysInFirstWeek: localeMinimumFirstDaysInWeek, firstWeekday: firstWeekday, minimumDaysInFirstWeek: minimumDaysInFirstWeek, gregorianStartDate: customGregorianStartDate) + + ucalendar = Self.icuCalendar(identifier: identifier, timeZone: _timeZone, localeIdentifier: localeIdentifier, localePrefs: localePrefs, firstWeekday: firstWeekday, minimumDaysInFirstWeek: minimumDaysInFirstWeek, gregorianStartDate: customGregorianStartDate) } static func icuCalendar(identifier: Calendar.Identifier, timeZone: TimeZone, localeIdentifier: String, - localeFirstWeekday: Locale.Weekday?, - localeMinimumDaysInFirstWeek: Int?, + localePrefs: LocalePreferences?, firstWeekday: Int?, minimumDaysInFirstWeek: Int?, gregorianStartDate: Date?) -> UnsafeMutablePointer { @@ -107,13 +103,14 @@ internal final class _Calendar: Equatable, @unchecked Sendable { if let firstWeekday { ucal_setAttribute(calendar, UCAL_FIRST_DAY_OF_WEEK, Int32(firstWeekday)) - } else if let forced = localeFirstWeekday { + } else if let forcedNumber = localePrefs?.firstWeekday?[identifier], let forced = Locale.Weekday(Int32(forcedNumber)) { + // Make sure we don't have an off-by-one error here by using the ICU function. This could probably be simplified. ucal_setAttribute(calendar, UCAL_FIRST_DAY_OF_WEEK, Int32(forced.icuIndex)) } if let minimumDaysInFirstWeek { ucal_setAttribute(calendar, UCAL_MINIMAL_DAYS_IN_FIRST_WEEK, Int32(truncatingIfNeeded: minimumDaysInFirstWeek)) - } else if let forced = localeMinimumDaysInFirstWeek { + } else if let forced = localePrefs?.minDaysInFirstWeek?[identifier] { ucal_setAttribute(calendar, UCAL_MINIMAL_DAYS_IN_FIRST_WEEK, Int32(truncatingIfNeeded: forced)) } @@ -132,8 +129,7 @@ internal final class _Calendar: Equatable, @unchecked Sendable { identifier: identifier, timeZone: _timeZone, localeIdentifier: localeIdentifier, - localeFirstWeekday: localeFirstWeekday, - localeMinimumDaysInFirstWeek: localeMinimumFirstDaysInWeek, + localePrefs: localePrefs, firstWeekday: customFirstWeekday, minimumDaysInFirstWeek: customMinimumFirstDaysInWeek, gregorianStartDate: customGregorianStartDate) @@ -141,13 +137,12 @@ internal final class _Calendar: Equatable, @unchecked Sendable { var locale: Locale { get { - return Locale(identifier: localeIdentifier, calendarIdentifier: identifier, firstWeekday: localeFirstWeekday, minimumDaysInFirstWeek: localeMinimumFirstDaysInWeek) + return Locale(identifier: localeIdentifier, calendarIdentifier: identifier, prefs: localePrefs) } set { lock.withLock { localeIdentifier = newValue.identifier - localeFirstWeekday = newValue.forceFirstWeekday(locale._calendarIdentifier) - localeMinimumFirstDaysInWeek = newValue.forceMinDaysInFirstWeek(locale._calendarIdentifier) + localePrefs = newValue.prefs _locked_regenerate() } } @@ -250,8 +245,8 @@ internal final class _Calendar: Equatable, @unchecked Sendable { lhs.firstWeekday == rhs.firstWeekday && lhs.minimumDaysInFirstWeek == rhs.minimumDaysInFirstWeek && lhs.localeIdentifier == rhs.localeIdentifier && - lhs.localeFirstWeekday == rhs.localeFirstWeekday && - lhs.localeMinimumFirstDaysInWeek == rhs.localeMinimumFirstDaysInWeek + lhs.localePrefs?.firstWeekday?[lhs.identifier] == rhs.localePrefs?.firstWeekday?[rhs.identifier] && + lhs.localePrefs?.minDaysInFirstWeek?[lhs.identifier] == rhs.localePrefs?.minDaysInFirstWeek?[rhs.identifier] } func hash(into hasher: inout Hasher) { @@ -261,8 +256,9 @@ internal final class _Calendar: Equatable, @unchecked Sendable { hasher.combine(_locked_firstWeekday) hasher.combine(_locked_minimumDaysInFirstWeek) hasher.combine(localeIdentifier) - hasher.combine(localeFirstWeekday) - hasher.combine(localeMinimumFirstDaysInWeek) + // It's important to include only properties that affect the Calendar itself. That allows e.g. currentLocale (with an irrelevant pref about something like preferred metric unit) to compare equal to a different locale. + hasher.combine(localePrefs?.firstWeekday?[identifier]) + hasher.combine(localePrefs?.minDaysInFirstWeek?[identifier]) lock.unlock() } diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index 432e68008..4cd21c7fe 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -169,15 +169,7 @@ public struct Locale : Hashable, Equatable, Sendable { } - @available(macOS 13, iOS 16, tvOS 16, watchOS 9, *) - internal init(identifier: String, calendarIdentifier: Calendar.Identifier, firstWeekday: Locale.Weekday?, minimumDaysInFirstWeek: Int?) { - var prefs = LocalePreferences() - if let firstWeekday { - prefs.firstWeekday = [calendarIdentifier : firstWeekday.icuIndex] - } - if let minimumDaysInFirstWeek { - prefs.minDaysInFirstWeek = [calendarIdentifier : minimumDaysInFirstWeek] - } + internal init(identifier: String, calendarIdentifier: Calendar.Identifier, prefs: LocalePreferences?) { self.kind = .fixed(_Locale(identifier: identifier, prefs: prefs)) } @@ -879,8 +871,18 @@ public struct Locale : Hashable, Equatable, Sendable { } } #endif // FOUNDATION_FRAMEWORK - -#if FOUNDATION_FRAMEWORK + + /// The whole bucket of preferences. + /// For use by `Calendar`, which wants to keep these values without a circular retain cycle with `Locale`. Only `current` locales and current-alikes have prefs. + internal var prefs: LocalePreferences? { + switch kind { + case .autoupdating: return LocaleCache.cache.current.prefs + case .fixed(let l): return l.prefs + case .bridged(_): return nil + } + } + + #if FOUNDATION_FRAMEWORK internal func pref(for key: String) -> Any? { switch kind { case .autoupdating: return LocaleCache.cache.current.pref(for: key) diff --git a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift index 45e9adb6c..d98159c68 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift @@ -94,7 +94,7 @@ internal final class _Locale: Sendable, Hashable { internal let identifier: String internal let doesNotRequireSpecialCaseHandling: Bool - private let prefs: LocalePreferences? + internal let prefs: LocalePreferences? private let lock: LockedState // MARK: - init @@ -1592,7 +1592,7 @@ internal final class _Locale: Sendable, Hashable { // MARK: - /// Holds user preferences about `Locale`, retrieved from user defaults. It is only used when creating the `current` Locale. Fixed-identiifer locales never have preferences. -struct LocalePreferences: Hashable { +internal struct LocalePreferences: Hashable { enum MeasurementUnit { case centimeters case inches From 8f9af73d82c56d45d25bcafaccbdec6c97605841 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Tue, 14 Mar 2023 11:12:35 -0700 Subject: [PATCH 12/21] rdar://106217659 (Fix Locale.currentLocale.decimalSeparator) --- .../Locale/Locale_ICU.swift | 37 +++++++++++++++---- .../LocaleTests.swift | 7 ++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift index d98159c68..a0fdd33e1 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift @@ -59,7 +59,7 @@ internal final class _Locale: Sendable, Hashable { var numberFormatters: [UInt32 /* UNumberFormatStyle */ : UnsafeMutablePointer] = [:] - mutating func formatter(for style: UNumberFormatStyle, identifier: String) -> UnsafeMutablePointer? { + mutating func formatter(for style: UNumberFormatStyle, identifier: String, numberSymbols: [UInt32 : String]?) -> UnsafeMutablePointer? { if let nf = numberFormatters[style.rawValue] { return nf } @@ -77,11 +77,22 @@ internal final class _Locale: Sendable, Hashable { unum_setAttribute(nf, UNUM_LENIENT_PARSE, 0) unum_setContext(nf, UDISPCTX_CAPITALIZATION_NONE, &status) + if let numberSymbols { + for (sym, str) in numberSymbols { + let icuSymbol = UNumberFormatSymbol(UInt32(sym)) + let utf16 = Array(str.utf16) + utf16.withUnsafeBufferPointer { + var status = U_ZERO_ERROR + unum_setSymbol(nf, icuSymbol, $0.baseAddress, Int32($0.count), &status) + } + } + } + numberFormatters[style.rawValue] = nf return nf } - + mutating func cleanup() { for nf in numberFormatters.values { unum_close(nf) @@ -815,7 +826,7 @@ internal final class _Locale: Sendable, Hashable { internal var decimalSeparator: String? { lock.withLock { state in - guard let nf = state.formatter(for: UNUM_DECIMAL, identifier: identifier) else { + guard let nf = state.formatter(for: UNUM_DECIMAL, identifier: identifier, numberSymbols: prefs?.numberSymbols) else { return nil } @@ -829,7 +840,7 @@ internal final class _Locale: Sendable, Hashable { internal var groupingSeparator: String? { lock.withLock { state in - guard let nf = state.formatter(for: UNUM_DECIMAL, identifier: identifier) else { + guard let nf = state.formatter(for: UNUM_DECIMAL, identifier: identifier, numberSymbols: prefs?.numberSymbols) else { return nil } @@ -878,7 +889,7 @@ internal final class _Locale: Sendable, Hashable { internal var currencySymbol: String? { lock.withLock { state in - guard let nf = state.formatter(for: UNUM_DECIMAL, identifier: identifier) else { + guard let nf = state.formatter(for: UNUM_DECIMAL, identifier: identifier, numberSymbols: prefs?.numberSymbols) else { return nil } @@ -907,7 +918,7 @@ internal final class _Locale: Sendable, Hashable { internal var currencyCode: String? { lock.withLock { state in - guard let nf = state.formatter(for: UNUM_CURRENCY, identifier: identifier) else { + guard let nf = state.formatter(for: UNUM_CURRENCY, identifier: identifier, numberSymbols: prefs?.numberSymbols) else { return nil } @@ -1647,9 +1658,12 @@ internal struct LocalePreferences: Hashable { var icuDateTimeSymbols: CFDictionary? var icuDateFormatStrings: CFDictionary? var icuTimeFormatStrings: CFDictionary? + + // The OS no longer writes out this preference, but we keep it here for compatibility with CFDateFormatter behavior. var icuNumberFormatStrings: CFDictionary? var icuNumberSymbols: CFDictionary? #endif + var numberSymbols: [UInt32 : String]? // Bridged version of `icuNumberSymbols` var country: String? var measurementUnits: MeasurementUnit? var temperatureUnit: TemperatureUnit? @@ -1668,7 +1682,8 @@ internal struct LocalePreferences: Hashable { measurementUnits: MeasurementUnit? = nil, temperatureUnit: TemperatureUnit? = nil, force24Hour: Bool? = nil, - force12Hour: Bool? = nil) { + force12Hour: Bool? = nil, + numberSymbols: [UInt32 : String]? = nil) { self.metricUnits = metricUnits self.languages = languages @@ -1681,6 +1696,7 @@ internal struct LocalePreferences: Hashable { self.temperatureUnit = temperatureUnit self.force24Hour = force24Hour self.force12Hour = force12Hour + self.numberSymbols = numberSymbols #if FOUNDATION_FRAMEWORK icuDateTimeSymbols = nil @@ -1757,7 +1773,13 @@ internal struct LocalePreferences: Hashable { } if let icuNumberSymbols = __CFLocalePrefsCopyAppleICUNumberSymbols(prefs)?.takeRetainedValue() { + // Store the CFDictionary for passing back to CFDateFormatter self.icuNumberSymbols = icuNumberSymbols + + // And bridge the mapping for our own usage in Locale + if let numberSymbolsPrefs = icuNumberSymbols as? [UInt32 : String] { + self.numberSymbols = numberSymbolsPrefs + } } @@ -1809,6 +1831,7 @@ internal struct LocalePreferences: Hashable { if let other = prefs.temperatureUnit { self.temperatureUnit = other } if let other = prefs.force24Hour { self.force24Hour = other } if let other = prefs.force12Hour { self.force12Hour = other } + if let other = prefs.numberSymbols { self.numberSymbols = other } } } diff --git a/Tests/FoundationInternationalizationTests/LocaleTests.swift b/Tests/FoundationInternationalizationTests/LocaleTests.swift index 128ff6bac..1a681f718 100644 --- a/Tests/FoundationInternationalizationTests/LocaleTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleTests.swift @@ -441,6 +441,13 @@ final class LocalePropertiesTests : XCTestCase { // Need to find a good test case for collator identifier // XCTAssertEqual("something", locale.collatorIdentifier) } + + func test_customizedProperties() { + let localePrefs = LocalePreferences(numberSymbols: [0 : "*", 1: "-"]) + let customizedLocale = Locale.localeAsIfCurrent(name: "en_US", overrides: localePrefs) + XCTAssertEqual(customizedLocale.decimalSeparator, "*") + XCTAssertEqual(customizedLocale.groupingSeparator, "-") + } } // MARK: - Bridging Tests From 3a2adee92509307f086049d9dde5d624ca752439 Mon Sep 17 00:00:00 2001 From: Jeremy Schonfeld Date: Mon, 6 Mar 2023 17:42:37 -0800 Subject: [PATCH 13/21] rdar://106320664 (Added debug description to UUID) --- Sources/FoundationEssentials/UUID_Wrappers.swift | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Sources/FoundationEssentials/UUID_Wrappers.swift b/Sources/FoundationEssentials/UUID_Wrappers.swift index 1d2a31bad..7bcde74dc 100644 --- a/Sources/FoundationEssentials/UUID_Wrappers.swift +++ b/Sources/FoundationEssentials/UUID_Wrappers.swift @@ -163,6 +163,16 @@ internal class _NSSwiftUUID : _NSUUIDBridge { } } + override var description: String { + self.uuidString + } + + override var debugDescription: String { + withUnsafePointer(to: self) { ptr in + "<\(Self.self) \(ptr.debugDescription)> \(self.uuidString)" + } + } + override var classForCoder: AnyClass { return NSUUID.self } From 1ba7cb8ec0fd8d8dc109925078abb8d23bbbfa98 Mon Sep 17 00:00:00 2001 From: Tina Liu Date: Tue, 21 Mar 2023 14:54:22 -0700 Subject: [PATCH 14/21] rdar://106555323 (NSLocale countryCode should behave the same as regionCode) Call through `region.identifier` for `countryCode` whenever possible. --- .../Locale/Locale.swift | 4 ++-- .../Locale/Locale_ICU.swift | 13 ----------- .../Locale/Locale_Wrappers.swift | 2 +- .../LocaleTests.swift | 23 ++++++++++++++++++- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index 4cd21c7fe..b729fa338 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -393,8 +393,8 @@ public struct Locale : Hashable, Equatable, Sendable { // n.b. this is called countryCode in ObjC let result: String? switch kind { - case .autoupdating: result = LocaleCache.cache.current.countryCode - case .fixed(let l): result = l.countryCode + case .autoupdating: result = LocaleCache.cache.current.region?.identifier + case .fixed(let l): result = l.region?.identifier #if FOUNDATION_FRAMEWORK case .bridged(let l): result = l.countryCode #endif diff --git a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift index a0fdd33e1..a3a58444d 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift @@ -343,19 +343,6 @@ internal final class _Locale: Sendable, Hashable { // MARK: - Country, Region, Subdivision, Variant - internal var countryCode: String? { - // n.b. the modern name for this is Region, not Country - lock.withLock { state in - if let comps = state.languageComponents { - return comps.region?.identifier - } else { - let comps = Locale.Language.Components(identifier: identifier) - state.languageComponents = comps - return comps.region?.identifier - } - } - } - internal func countryCodeDisplayName(for value: String) -> String? { lock.withLock { state in if let result = state.countryCodeDisplayNames[value] { diff --git a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift index 2b449e3bd..95d8466aa 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift @@ -533,7 +533,7 @@ internal class _NSSwiftLocale: _NSLocaleBridge { @available(macOS, deprecated: 13) @available(iOS, deprecated: 16) @available(tvOS, deprecated: 16) @available(watchOS, deprecated: 9) override var countryCode: String? { - locale.regionCode + locale.region?.identifier } override var regionCode: String? { diff --git a/Tests/FoundationInternationalizationTests/LocaleTests.swift b/Tests/FoundationInternationalizationTests/LocaleTests.swift index 1a681f718..143dbee01 100644 --- a/Tests/FoundationInternationalizationTests/LocaleTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleTests.swift @@ -467,7 +467,7 @@ final class LocalBridgingTests : XCTestCase { @available(tvOS, deprecated: 16) @available(watchOS, deprecated: 9) func test_getACustomLocale() { - let loc = getACustomLocale() + let loc = getACustomLocale("en_US") let objCLoc = loc as! CustomNSLocaleSubclass // Verify that accessing the properties of `l` calls back into ObjC @@ -476,6 +476,27 @@ final class LocalBridgingTests : XCTestCase { XCTAssertEqual(loc.currencyCode, "USD") XCTAssertEqual(objCLoc.last, "objectForKey:") // Everything funnels through the primitives + + XCTAssertEqual(loc.regionCode, "US") + XCTAssertEqual(objCLoc.countryCode, "US") + } + + @available(macOS, deprecated: 13) + @available(iOS, deprecated: 16) + @available(tvOS, deprecated: 16) + @available(watchOS, deprecated: 9) + func test_customLocaleCountryCode() { + let loc = getACustomLocale("en_US@rg=gbzzzz") + let objCLoc = loc as! CustomNSLocaleSubclass + + XCTAssertEqual(loc.identifier, "en_US@rg=gbzzzz") + XCTAssertEqual(objCLoc.last, "localeIdentifier") + + XCTAssertEqual(loc.currencyCode, "GBP") + XCTAssertEqual(objCLoc.last, "objectForKey:") // Everything funnels through the primitives + + XCTAssertEqual(loc.regionCode, "GB") + XCTAssertEqual(objCLoc.countryCode, "GB") } func test_AnyHashableCreatedFromNSLocale() { From d4a070ca995115c78c89402b4dda604b60fab9b5 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Tue, 21 Mar 2023 16:08:28 -0700 Subject: [PATCH 15/21] rdar://106792309 (Fix regression) --- Sources/FoundationEssentials/UUID_Wrappers.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/FoundationEssentials/UUID_Wrappers.swift b/Sources/FoundationEssentials/UUID_Wrappers.swift index 7bcde74dc..9305e126b 100644 --- a/Sources/FoundationEssentials/UUID_Wrappers.swift +++ b/Sources/FoundationEssentials/UUID_Wrappers.swift @@ -116,6 +116,7 @@ internal class _NSSwiftUUID : _NSUUIDBridge { super.init() } + #if false // rdar://106792309 override func encode(with coder: NSCoder) { var uuid = _storage.uuid withUnsafeBytes(of: &uuid) { buffer in @@ -124,6 +125,7 @@ internal class _NSSwiftUUID : _NSUUIDBridge { } } } + #endif override public init?(uuidString: String) { guard let swiftUUID = Foundation.UUID(uuidString: uuidString) else { From fb29eff8cb060e2900da44e0b53db082c968b1e3 Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Wed, 22 Mar 2023 16:45:19 -0700 Subject: [PATCH 16/21] rdar://106898040 (Fix currency text field) --- .../Locale/Locale.swift | 14 ++++++++++++++ .../Locale/Locale_Wrappers.swift | 10 +++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index b729fa338..69cc8f812 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -321,6 +321,20 @@ public struct Locale : Hashable, Equatable, Sendable { } } + /// This exists in `NSLocale` via the `displayName` API, using the currency *symbol* key instead of *code*. + internal func localizedString(forCurrencySymbol currencySymbol: String) -> String? { + switch kind { + case .fixed(let l): + return l.currencySymbolDisplayName(for: currencySymbol) +#if FOUNDATION_FRAMEWORK + case .bridged(let l): + return l.currencySymbolDisplayName(for: currencySymbol) +#endif + case .autoupdating: + return LocaleCache.cache.current.currencySymbolDisplayName(for: currencySymbol) + } + } + /// Returns a localized string for a specified ICU collation identifier. /// /// For example, in the "en" locale, the result for `"phonebook"` is `"Phonebook Sort Order"`. diff --git a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift index 95d8466aa..4f9981b90 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift @@ -265,6 +265,10 @@ internal final class _NSLocaleSwiftWrapper: @unchecked Sendable, Hashable, Custo func currencyCodeDisplayName(for currencyCode: String) -> String? { return _wrapped.displayName(forKey: .currencyCode, value: currencyCode) } + + func currencySymbolDisplayName(for currencySymbol: String) -> String? { + return _wrapped.displayName(forKey: .currencySymbol, value: currencySymbol) + } func collationIdentifierDisplayName(for collationIdentifier: String) -> String? { return _wrapped.displayName(forKey: .collationIdentifier, value: collationIdentifier) @@ -502,7 +506,7 @@ internal class _NSSwiftLocale: _NSLocaleBridge { case .measurementSystem: return nil case .decimalSeparator: return nil case .groupingSeparator: return nil - case .currencySymbol: return self.localizedString(forCurrencyCode: value) + case .currencySymbol: return self.localizedString(forCurrencySymbol: value) case .currencyCode: return self.localizedString(forCurrencyCode: value) case .collatorIdentifier: return self.localizedString(forCollatorIdentifier: value) case .quotationBeginDelimiterKey: return nil @@ -640,6 +644,10 @@ internal class _NSSwiftLocale: _NSLocaleBridge { locale.localizedString(forCurrencyCode: currencyCode) } + override func localizedString(forCurrencySymbol currencySymbol: String) -> String? { + locale.localizedString(forCurrencySymbol: currencySymbol) + } + override func localizedString(forCollatorIdentifier collatorIdentifier: String) -> String? { locale.localizedString(forCollatorIdentifier: collatorIdentifier) } From 55669e4dac964e4178bdbb281419928813fb22ad Mon Sep 17 00:00:00 2001 From: Charles Hu Date: Wed, 15 Mar 2023 13:16:51 -0700 Subject: [PATCH 17/21] rdar://107156343 (Move JSONEncoder to FoundationPreview) --- .../CharacterSet+Stub.swift | 8 + .../JSON/JSON5Scanner.swift | 1309 ++++++ .../JSON/JSONDecoder.swift | 1548 +++++++ .../JSON/JSONEncoder.swift | 1428 ++++++ .../JSON/JSONScanner.swift | 1453 ++++++ .../JSON/JSONWriter.swift | 333 ++ .../String/String+Comparison.swift | 746 +++ .../String/String+Encoding.swift | 139 + .../String/String+Extensions.swift | 22 + .../String/StringAPIs.swift | 267 ++ .../{ => String}/StringBlocks.swift | 5 - .../String/UnicodeScalar.swift | 85 + .../Calendar/Calendar_Enumerate.swift | 26 +- .../Locale/Locale.swift | 4 + Sources/TestSupport/TestSupport.swift | 4 + .../JSONEncoderTests.swift | 4017 +++++++++++++++++ .../LocaleComponentsTests.swift | 185 +- .../LocaleTests.swift | 54 +- 18 files changed, 11502 insertions(+), 131 deletions(-) rename Sources/{FoundationInternationalization => FoundationEssentials}/CharacterSet+Stub.swift (82%) create mode 100644 Sources/FoundationEssentials/JSON/JSON5Scanner.swift create mode 100644 Sources/FoundationEssentials/JSON/JSONDecoder.swift create mode 100644 Sources/FoundationEssentials/JSON/JSONEncoder.swift create mode 100644 Sources/FoundationEssentials/JSON/JSONScanner.swift create mode 100644 Sources/FoundationEssentials/JSON/JSONWriter.swift create mode 100644 Sources/FoundationEssentials/String/String+Comparison.swift create mode 100644 Sources/FoundationEssentials/String/String+Encoding.swift create mode 100644 Sources/FoundationEssentials/String/String+Extensions.swift create mode 100644 Sources/FoundationEssentials/String/StringAPIs.swift rename Sources/FoundationEssentials/{ => String}/StringBlocks.swift (98%) create mode 100644 Sources/FoundationEssentials/String/UnicodeScalar.swift create mode 100644 Tests/FoundationEssentialsTests/JSONEncoderTests.swift diff --git a/Sources/FoundationInternationalization/CharacterSet+Stub.swift b/Sources/FoundationEssentials/CharacterSet+Stub.swift similarity index 82% rename from Sources/FoundationInternationalization/CharacterSet+Stub.swift rename to Sources/FoundationEssentials/CharacterSet+Stub.swift index 5d4b57d1b..35cbd95c8 100644 --- a/Sources/FoundationInternationalization/CharacterSet+Stub.swift +++ b/Sources/FoundationEssentials/CharacterSet+Stub.swift @@ -25,6 +25,14 @@ public struct CharacterSet : Equatable, Hashable { public mutating func insert(charactersIn string: String) {} public mutating func insert(charactersIn range: Range) {} public mutating func insert(charactersIn range: ClosedRange) {} + + public func contains(_ member: Unicode.Scalar) -> Bool { return false } +} + +// MARK: - Exported Character Sets +extension CharacterSet { + public static let uppercaseLetters: CharacterSet = CharacterSet() + public static let lowercaseLetters: CharacterSet = CharacterSet() } #endif diff --git a/Sources/FoundationEssentials/JSON/JSON5Scanner.swift b/Sources/FoundationEssentials/JSON/JSON5Scanner.swift new file mode 100644 index 000000000..f23577127 --- /dev/null +++ b/Sources/FoundationEssentials/JSON/JSON5Scanner.swift @@ -0,0 +1,1309 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + + +internal struct JSON5Scanner { + let options: Options + var reader: DocumentReader + var depth: Int = 0 + var partialMap = JSONPartialMapData() + + internal struct Options { + var assumesTopLevelDictionary = false + } + + struct JSONPartialMapData { + var mapData: [Int] = [] + var prevMapDataSize = 0 + + mutating func resizeIfNecessary(with reader: DocumentReader) { + let currentCount = mapData.count + if currentCount > 0, currentCount.isMultiple(of: 2048) { + // Time to predict how big these arrays are going to be based on the current rate of consumption per processed bytes. + // total objects = (total bytes / current bytes) * current objects + let totalBytes = reader.bytes.count + let consumedBytes = reader.byteOffset(at: reader.readPtr) + let ratio = (Double(totalBytes) / Double(consumedBytes)) + let totalExpectedMapSize = Int( Double(mapData.count) * ratio ) + if prevMapDataSize == 0 || Double(totalExpectedMapSize) / Double(prevMapDataSize) > 1.25 { + mapData.reserveCapacity(totalExpectedMapSize) + prevMapDataSize = totalExpectedMapSize + } + + // print("Ratio is \(ratio). Reserving \(totalExpectedObjects) objects and \(totalExpectedMapSize) scratch space") + } + } + + mutating func recordStartCollection(tagType: JSONMap.TypeDescriptor, with reader: DocumentReader) -> Int { + resizeIfNecessary(with: reader) + + mapData.append(tagType.mapMarker) + + // Reserve space for the next object index and object count. + let startIdx = mapData.count + mapData.append(contentsOf: [0, 0]) + return startIdx + } + + mutating func recordEndCollection(count: Int, atStartOffset startOffset: Int, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + mapData.append(JSONMap.TypeDescriptor.collectionEnd.rawValue) + + let nextValueOffset = mapData.count + mapData.withUnsafeMutableBufferPointer { + $0[startOffset] = nextValueOffset + $0[startOffset &+ 1] = count + } + } + + mutating func recordEmptyCollection(tagType: JSONMap.TypeDescriptor, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + let nextValueOffset = mapData.count + 4 + mapData.append(contentsOf: [tagType.mapMarker, nextValueOffset, 0, JSONMap.TypeDescriptor.collectionEnd.mapMarker]) + } + + mutating func record(tagType: JSONMap.TypeDescriptor, count: Int, dataOffset: Int, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + mapData.append(contentsOf: [tagType.mapMarker, count, dataOffset]) + } + + mutating func record(tagType: JSONMap.TypeDescriptor, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + mapData.append(tagType.mapMarker) + } + } + + init(bytes: UnsafeBufferPointer, options: Options) { + self.options = options + self.reader = DocumentReader(bytes: bytes) + } + + mutating func scan() throws -> JSONMap { + if options.assumesTopLevelDictionary { + switch try reader.consumeWhitespace(allowingEOF: true) { + case ._openbrace?: + // If we've got the open brace anyway, just do a normal object scan. + try self.scanObject() + default: + try self.scanObject(withoutBraces: true) + } + } else { + try self.scanValue() + } +#if DEBUG + defer { + guard self.depth == 0 else { + preconditionFailure("Expected to end parsing with a depth of 0") + } + } +#endif + + // ensure only white space is remaining + if let char = try reader.consumeWhitespace(allowingEOF: true) { + throw JSONError.unexpectedCharacter(context: "after top-level value", ascii: char, location: reader.sourceLocation) + } + + return JSONMap(mapBuffer: partialMap.mapData, dataBuffer: self.reader.bytes) + } + + // MARK: Generic Value Scanning + + mutating func scanValue() throws { + let byte = try reader.consumeWhitespace() + switch byte { + case ._quote: + try scanString(withQuote: ._quote) + case ._singleQuote: + try scanString(withQuote: ._singleQuote) + case ._openbrace: + try scanObject() + case ._openbracket: + try scanArray() + case UInt8(ascii: "f"), UInt8(ascii: "t"): + try scanBool() + case UInt8(ascii: "n"): + try scanNull() + case UInt8(ascii: "-"), UInt8(ascii: "+"), _asciiNumbers, UInt8(ascii: "N"), UInt8(ascii: "I"), UInt8(ascii: "."): + try scanNumber() + case ._space, ._return, ._newline, ._tab: + preconditionFailure("Expected that all white space is consumed") + default: + throw JSONError.unexpectedCharacter(ascii: byte, location: reader.sourceLocation) + } + } + + + // MARK: - Scan Array - + + mutating func scanArray() throws { + let firstChar = reader.read() + precondition(firstChar == ._openbracket) + guard self.depth < 512 else { + throw JSONError.tooManyNestedArraysOrDictionaries(location: reader.sourceLocation(atOffset: -1)) + } + self.depth &+= 1 + defer { depth &-= 1 } + + // parse first value or end immediatly + switch try reader.consumeWhitespace() { + case ._space, ._return, ._newline, ._tab: + preconditionFailure("Expected that all white space is consumed") + case ._closebracket: + // if the first char after whitespace is a closing bracket, we found an empty array + reader.moveReaderIndex(forwardBy: 1) + return partialMap.recordEmptyCollection(tagType: .array, with: reader) + default: + break + } + + var count = 0 + let startOffset = partialMap.recordStartCollection(tagType: .array, with: reader) + defer { + partialMap.recordEndCollection(count: count, atStartOffset: startOffset, with: reader) + } + + ScanValues: + while true { + try scanValue() + count &+= 1 + + // consume the whitespace after the value before the comma + let ascii = try reader.consumeWhitespace() + switch ascii { + case ._space, ._return, ._newline, ._tab: + preconditionFailure("Expected that all white space is consumed") + case ._closebracket: + reader.moveReaderIndex(forwardBy: 1) + break ScanValues + case ._comma: + // consume the comma + reader.moveReaderIndex(forwardBy: 1) + // consume the whitespace before the next value + if try reader.consumeWhitespace() == ._closebracket { + // the foundation json implementation does support trailing commas + reader.moveReaderIndex(forwardBy: 1) + break ScanValues + } + continue + default: + throw JSONError.unexpectedCharacter(context: "in array", ascii: ascii, location: reader.sourceLocation) + } + } + } + + // MARK: - Scan Object - + + mutating func scanObject() throws { + let firstChar = self.reader.read() + precondition(firstChar == ._openbrace) + guard self.depth < 512 else { + throw JSONError.tooManyNestedArraysOrDictionaries(location: reader.sourceLocation(atOffset: -1)) + } + try scanObject(withoutBraces: false) + } + + @inline(never) + mutating func _scanObjectLoop(withoutBraces: Bool, count: inout Int, done: inout Bool) throws { + try scanKey() + + let colon = try reader.consumeWhitespace() + guard colon == ._colon else { + throw JSONError.unexpectedCharacter(context: "in object", ascii: colon, location: reader.sourceLocation) + } + reader.moveReaderIndex(forwardBy: 1) + + try self.scanValue() + count &+= 2 + + let commaOrBrace = try reader.consumeWhitespace(allowingEOF: withoutBraces) + switch commaOrBrace { + case ._comma?: + reader.moveReaderIndex(forwardBy: 1) + switch try reader.consumeWhitespace(allowingEOF: withoutBraces) { + case ._closebrace?: + if withoutBraces { + throw JSONError.unexpectedCharacter(ascii: ._closebrace, location: reader.sourceLocation) + } + + // the foundation json implementation does support trailing commas + reader.moveReaderIndex(forwardBy: 1) + done = true + case .none: + done = true + default: + return + } + case ._closebrace?: + if withoutBraces { + throw JSONError.unexpectedCharacter(ascii: ._closebrace, location: reader.sourceLocation) + } + reader.moveReaderIndex(forwardBy: 1) + done = true + case .none: + // If withoutBraces was false, then reaching EOF here would have thrown. + precondition(withoutBraces == true) + done = true + + default: + throw JSONError.unexpectedCharacter(context: "in object", ascii: commaOrBrace.unsafelyUnwrapped, location: reader.sourceLocation) + } + } + + mutating func scanObject(withoutBraces: Bool) throws { + self.depth &+= 1 + defer { depth &-= 1 } + + // parse first value or end immediatly + switch try reader.consumeWhitespace(allowingEOF: withoutBraces) { + case ._closebrace?: + if withoutBraces { + throw JSONError.unexpectedCharacter(ascii: ._closebrace, location: reader.sourceLocation) + } + + // if the first char after whitespace is a closing bracket, we found an empty object + self.reader.moveReaderIndex(forwardBy: 1) + return partialMap.recordEmptyCollection(tagType: .object, with: reader) + case .none: + // If withoutBraces was false, then reaching EOF here would have thrown. + precondition(withoutBraces == true) + return partialMap.recordEmptyCollection(tagType: .object, with: reader) + default: + break + } + + var count = 0 + let startOffset = partialMap.recordStartCollection(tagType: .object, with: reader) + defer { + partialMap.recordEndCollection(count: count, atStartOffset: startOffset, with: reader) + } + + var done = false + while !done { + try _scanObjectLoop(withoutBraces: withoutBraces, count: &count, done: &done) + } + } + + mutating func scanKey() throws { + guard let firstChar = reader.peek() else { + throw JSONError.unexpectedEndOfFile + } + + switch firstChar { + case ._quote: + try scanString(withQuote: ._quote) + case ._singleQuote: + try scanString(withQuote: ._singleQuote) + case _hexCharsUpper, _hexCharsLower, ._dollar, ._underscore, ._backslash: + try scanString(withQuote: nil) + default: + // Validate that the initial character is within the rules specified by JSON5. + guard let (unicodeScalar, _) = try reader.peekU32() else { + throw JSONError.unexpectedEndOfFile + } + guard unicodeScalar.isJSON5UnquotedKeyStartingCharacter else { + throw JSONError.unexpectedCharacter(context: "at beginning of JSON5 unquoted key", ascii: firstChar, location: reader.sourceLocation) + } + try scanString(withQuote: nil) + } + } + + mutating func scanString(withQuote quote: UInt8?) throws { + let quoteStart = reader.readPtr + var isSimple = false + try reader.skipUTF8StringTillNextUnescapedQuote(isSimple: &isSimple, quote: quote) + let stringStart = quote != nil ? quoteStart + 1 : quoteStart + let end = reader.readPtr + + // skipUTF8StringTillNextUnescapedQuote will have either thrown an error, or already peek'd the quote. + if let quote { + let shouldBeQuote = reader.read() + precondition(shouldBeQuote == quote) + } + + // skip initial quote + return partialMap.record(tagType: isSimple ? .simpleString : .string, count: end - stringStart, dataOffset: reader.byteOffset(at: stringStart), with: reader) + } + + mutating func scanNumber() throws { + let start = reader.readPtr + reader.skipNumber() + let end = reader.readPtr + return partialMap.record(tagType: .number, count: end - start, dataOffset: reader.byteOffset(at: start), with: reader) + } + + mutating func scanBool() throws { + if try reader.readBool() { + return partialMap.record(tagType: .true, with: reader) + } else { + return partialMap.record(tagType: .false, with: reader) + } + } + + mutating func scanNull() throws { + try reader.readNull() + return partialMap.record(tagType: .null, with: reader) + } + +} + +extension JSON5Scanner { + + struct DocumentReader { + let bytes: UnsafeBufferPointer + private(set) var readPtr : UnsafePointer + private let endPtr : UnsafePointer + + @inline(__always) + func checkRemainingBytes(_ count: Int) -> Bool { + return endPtr - readPtr >= count + } + + @inline(__always) + func requireRemainingBytes(_ count: Int) throws { + guard checkRemainingBytes(count) else { + throw JSONError.unexpectedEndOfFile + } + } + + var sourceLocation : JSONError.SourceLocation { + self.sourceLocation(atOffset: 0) + } + + func sourceLocation(atOffset offset: Int) -> JSONError.SourceLocation { + .sourceLocation(at: readPtr + offset, docStart: bytes.baseAddress.unsafelyUnwrapped) + } + + @inline(__always) + var isEOF: Bool { + readPtr == endPtr + } + + @inline(__always) + func byteOffset(at ptr: UnsafePointer) -> Int { + ptr - bytes.baseAddress.unsafelyUnwrapped + } + + init(bytes: UnsafeBufferPointer) { + self.bytes = bytes + self.readPtr = bytes.baseAddress.unsafelyUnwrapped + self.endPtr = self.readPtr + bytes.count + } + + @inline(__always) + mutating func read() -> UInt8? { + guard !isEOF else { + return nil + } + + defer { self.readPtr += 1 } + + return readPtr.pointee + } + + @inline(__always) + func peek(offset: Int = 0) -> UInt8? { + precondition(offset >= 0) + guard checkRemainingBytes(offset + 1) else { + return nil + } + + return (self.readPtr + offset).pointee + } + + // These UTF-8 decoding functions are cribbed and specialized from the stdlib. + + @inline(__always) + internal func _utf8ScalarLength(_ x: UInt8) -> Int? { + guard !UTF8.isContinuation(x) else { return nil } + if UTF8.isASCII(x) { return 1 } + return (~x).leadingZeroBitCount + } + + @inline(__always) + internal func _continuationPayload(_ x: UInt8) -> UInt32 { + return UInt32(x & 0x3F) + } + + @inline(__always) + internal func _decodeUTF8(_ x: UInt8) -> Unicode.Scalar? { + guard UTF8.isASCII(x) else { return nil } + return Unicode.Scalar(x) + } + + @inline(__always) + internal func _decodeUTF8(_ x: UInt8, _ y: UInt8) -> Unicode.Scalar? { + assert(_utf8ScalarLength(x) == 2) + guard UTF8.isContinuation(y) else { return nil } + let x = UInt32(x) + let value = ((x & 0b0001_1111) &<< 6) | _continuationPayload(y) + return Unicode.Scalar(value).unsafelyUnwrapped + } + + @inline(__always) + internal func _decodeUTF8( + _ x: UInt8, _ y: UInt8, _ z: UInt8 + ) -> Unicode.Scalar? { + assert(_utf8ScalarLength(x) == 3) + guard UTF8.isContinuation(y), UTF8.isContinuation(z) else { return nil } + let x = UInt32(x) + let value = ((x & 0b0000_1111) &<< 12) + | (_continuationPayload(y) &<< 6) + | _continuationPayload(z) + return Unicode.Scalar(value).unsafelyUnwrapped + } + + @inline(__always) + internal func _decodeUTF8( + _ x: UInt8, _ y: UInt8, _ z: UInt8, _ w: UInt8 + ) -> Unicode.Scalar? { + assert(_utf8ScalarLength(x) == 4) + guard UTF8.isContinuation(y), UTF8.isContinuation(z), UTF8.isContinuation(w) else { return nil } + let x = UInt32(x) + let value = ((x & 0b0000_1111) &<< 18) + | (_continuationPayload(y) &<< 12) + | (_continuationPayload(z) &<< 6) + | _continuationPayload(w) + return Unicode.Scalar(value).unsafelyUnwrapped + } + + internal func _decodeScalar( + _ utf8: UnsafeBufferPointer, startingAt i: Int + ) -> (Unicode.Scalar?, scalarLength: Int) { + let cu0 = utf8[i] + guard let len = _utf8ScalarLength(cu0), checkRemainingBytes(len) else { return (nil, 0) } + switch len { + case 1: return (_decodeUTF8(cu0), len) + case 2: return (_decodeUTF8(cu0, utf8[i &+ 1]), len) + case 3: return (_decodeUTF8(cu0, utf8[i &+ 1], utf8[i &+ 2]), len) + case 4: + return (_decodeUTF8( + cu0, + utf8[i &+ 1], + utf8[i &+ 2], + utf8[i &+ 3]), + len) + default: fatalError() + } + } + + func peekU32() throws -> (UnicodeScalar, Int)? { + guard let firstChar = peek() else { + return nil + } + + // This might be an escaped character. + if firstChar == ._backslash { + guard let char = peek(offset: 1) else { + throw JSONError.unexpectedEndOfFile + } + + switch char { + case UInt8(ascii: "u"): + try requireRemainingBytes(6) // 6 bytes for \, u, and 4 hex digits + var ptr = readPtr + 2 // Skip \u + let u16 = try JSONScanner.parseUnicodeHexSequence(cursor: &ptr, end: endPtr, docStart: bytes.baseAddress.unsafelyUnwrapped, allowNulls: false) + guard let scalar = UnicodeScalar(u16) else { + throw JSONError.couldNotCreateUnicodeScalarFromUInt32(location: sourceLocation, unicodeScalarValue: UInt32(u16)) + } + return (scalar, 6) + case UInt8(ascii: "x"): + try requireRemainingBytes(4) // 4 byets for \, x, and 2 hex digits + var ptr = readPtr + 2 // Skip \x + let u8 = try JSON5Scanner.parseTwoByteUnicodeHexSequence(cursor: &ptr, end: endPtr, docStart: bytes.baseAddress.unsafelyUnwrapped) + return (UnicodeScalar(u8), 4) + default: + throw JSONError.unexpectedCharacter(ascii: char, location: sourceLocation(atOffset: 1)) + } + } + + let (scalar, length) = _decodeScalar(self.bytes, startingAt: readPtr - self.bytes.baseAddress.unsafelyUnwrapped) + guard let scalar else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: sourceLocation) + } + return (scalar, length) + } + + @inline(__always) + mutating func moveReaderIndex(forwardBy offset: Int) { + self.readPtr += offset + } + + static let whitespaceBitmap: UInt64 = 1 << UInt8._space | 1 << UInt8._return | 1 << UInt8._newline | 1 << UInt8._tab | 1 << UInt8._verticalTab | 1 << UInt8._formFeed + + @inline(__always) + @discardableResult + mutating func consumeWhitespace() throws -> UInt8 { + var ptr = self.readPtr + while ptr < endPtr { + let ascii = ptr.pointee + if Self.whitespaceBitmap & (1 << ascii) != 0 { + ptr += 1 + continue + } else if ascii == ._nbsp { + ptr += 1 + continue + } else if ascii == ._slash { + guard try consumePossibleComment(from: &ptr) else { + self.readPtr = ptr + return ascii + } + continue + } else { + self.readPtr = ptr + return ascii + } + } + + throw JSONError.unexpectedEndOfFile + } + + @inline(__always) + @discardableResult + mutating func consumeWhitespace(allowingEOF: Bool) throws -> UInt8? { + var ptr = self.readPtr + while ptr < endPtr { + let ascii = ptr.pointee + if Self.whitespaceBitmap & (1 << ascii) != 0 { + ptr += 1 + continue + } else if ascii == ._nbsp { + ptr += 1 + continue + } else if ascii == ._slash { + guard try consumePossibleComment(from: &ptr) else { + self.readPtr = ptr + return ascii + } + continue + } else { + self.readPtr = ptr + return ascii + } + } + guard allowingEOF else { + throw JSONError.unexpectedEndOfFile + } + return nil + } + + @inline(__always) + func consumePossibleComment(from ptr: inout UnsafePointer) throws -> Bool { + // ptr still points to the first / + guard ptr + 1 < endPtr else { + return false + } + + switch (ptr + 1).pointee { + case ._slash: + ptr += 2 + consumeSingleLineComment(from: &ptr) + return true + case ._asterisk: + ptr += 2 + try consumeMultiLineComment(from: &ptr) + return true + default: + return false + } + } + + @inline(__always) + func consumeSingleLineComment(from ptr: inout UnsafePointer) { + // No need to bother getting fancy about CR-LF. These only get called in the process of skipping whitespace, and a trailing LF will be picked up by that. We also don't track line number information during nominal parsing. + var localPtr = ptr + while localPtr < endPtr { + let ascii = localPtr.pointee + switch ascii { + case ._newline, ._return: + ptr = localPtr + 1 + return + default: + localPtr += 1 + continue + } + } + ptr = endPtr + // Reaching EOF is fine. + } + + @inline(__always) + func consumeMultiLineComment(from ptr: inout UnsafePointer) throws { + var localPtr = ptr + while (localPtr+1) < endPtr { + switch (localPtr.pointee, (localPtr+1).pointee) { + case (._asterisk, ._slash): + ptr = localPtr + 2 + return + case (_, ._asterisk): + // Check the next asterisk. + localPtr += 1 + continue + default: + // We don't need to check the second byte again. + localPtr += 2 + continue + } + } + ptr = endPtr + throw JSONError.unterminatedBlockComment + } + + @inline(__always) + mutating func readExpectedString(_ str: StaticString, typeDescriptor: String) throws { + try requireRemainingBytes(str.utf8CodeUnitCount) + guard memcmp(readPtr, str.utf8Start, str.utf8CodeUnitCount) == 0 else { + // Figure out the exact character that is wrong. + var badOffset = 0 + for i in 0 ..< str.utf8CodeUnitCount { + if (readPtr + i).pointee != (str.utf8Start + i).pointee { + badOffset = i + break + } + } + throw JSONError.unexpectedCharacter(context: "in expected \(typeDescriptor) value", ascii: self.peek(offset: badOffset).unsafelyUnwrapped, location: sourceLocation(atOffset: badOffset)) + } + + // If all looks good, advance past the string. + self.moveReaderIndex(forwardBy: str.utf8CodeUnitCount) + } + + @inline(__always) + mutating func readBool() throws -> Bool { + switch self.read() { + case UInt8(ascii: "t"): + try readExpectedString("rue", typeDescriptor: "boolean") + return true + case UInt8(ascii: "f"): + try readExpectedString("alse", typeDescriptor: "boolean") + return false + default: + preconditionFailure("Expected to have `t` or `f` as first character") + } + } + + @inline(__always) + mutating func readNull() throws { + try readExpectedString("null", typeDescriptor: "null") + } + + // MARK: - Private Methods - + + // MARK: String + + mutating func skipUTF8StringTillQuoteOrBackslashOrInvalidCharacter(quote: UInt8) throws -> UInt8 { + while let byte = self.peek() { + switch byte { + case quote, ._backslash: + return byte + default: + // Any control characters in the 0-31 range are invalid. Any other characters will have at least one bit in a 0xe0 mask. + guard _fastPath(byte & 0xe0 != 0) else { + return byte + } + self.moveReaderIndex(forwardBy: 1) + } + } + throw JSONError.unexpectedEndOfFile + } + + @discardableResult + mutating func skipUTF8StringTillEndOfUnquotedKey(orEscapeSequence stopOnEscapeSequence: Bool) throws -> UnicodeScalar { + while let (scalar, len) = try peekU32() { + if scalar.isJSON5UnquotedKeyCharacter { + moveReaderIndex(forwardBy: len) + } else { + return scalar + } + } + throw JSONError.unexpectedEndOfFile + } + + mutating func skipUTF8StringTillNextUnescapedQuote(isSimple: inout Bool, quote: UInt8?) throws { + if let quote { + // Skip the open quote. + guard let shouldBeQuote = self.read() else { + throw JSONError.unexpectedEndOfFile + } + guard shouldBeQuote == quote else { + throw JSONError.unexpectedCharacter(ascii: shouldBeQuote, location: sourceLocation) + } + + // If there aren't any escapes, then this is a simple case and we can exit early. + if try skipUTF8StringTillQuoteOrBackslashOrInvalidCharacter(quote: quote) == quote { + isSimple = true + return + } + + isSimple = false + + while let byte = self.peek() { + // Checking for invalid control characters deferred until parse time. + switch byte { + case quote: + return + case ._backslash: + try skipEscapeSequence(quote: quote) + default: + moveReaderIndex(forwardBy: 1) + continue + } + } + throw JSONError.unexpectedEndOfFile + } else { + if try skipUTF8StringTillEndOfUnquotedKey(orEscapeSequence: true) == UnicodeScalar(._backslash) { + // The presence of a backslash means this isn't a "simple" key. Continue skipping until we reach the end of the key, this time ignoring backslashes. + isSimple = false + try skipUTF8StringTillEndOfUnquotedKey(orEscapeSequence: false) + } else { + // No backslashes. The string can be decoded directly as UTF8. + isSimple = true + } + } + } + + private mutating func skipEscapeSequence(quote: UInt8) throws { + let firstChar = self.read() + precondition(firstChar == ._backslash, "Expected to have an backslash first") + + guard let ascii = self.read() else { + throw JSONError.unexpectedEndOfFile + } + + // Invalid escaped characters checking deferred to parse time. + if ascii == UInt8(ascii: "u") { + try skipUnicodeHexSequence(quote: quote) + } else if ascii == UInt8(ascii: "x") { + try skipTwoByteUnicodeHexSequence(quote: quote) + } + } + + private mutating func skipUnicodeHexSequence(quote: UInt8) throws { + // As stated in RFC-8259 an escaped unicode character is 4 HEXDIGITs long + // https://tools.ietf.org/html/rfc8259#section-7 + try requireRemainingBytes(4) + + // We'll validate the actual characters following the '\u' escape during parsing. Just make sure that the string doesn't end prematurely. + guard readPtr.pointee != quote, + (readPtr+1).pointee != quote, + (readPtr+2).pointee != quote, + (readPtr+3).pointee != quote + else { + let hexString = String(decoding: UnsafeBufferPointer(start: readPtr, count: 4), as: UTF8.self) + throw JSONError.invalidHexDigitSequence(hexString, location: sourceLocation) + } + self.moveReaderIndex(forwardBy: 4) + } + + private mutating func skipTwoByteUnicodeHexSequence(quote: UInt8) throws { + try requireRemainingBytes(2) + + // We'll validate the actual characters following the '\u' escape during parsing. Just make sure that the string doesn't end prematurely. + guard readPtr.pointee != quote, + (readPtr+1).pointee != quote + else { + let hexString = String(decoding: UnsafeBufferPointer(start: readPtr, count: 2), as: UTF8.self) + throw JSONError.invalidHexDigitSequence(hexString, location: sourceLocation) + } + self.moveReaderIndex(forwardBy: 2) + } + + // MARK: Numbers + + mutating func skipNumber() { + guard let ascii = read() else { + preconditionFailure("Why was this function called, if there is no 0...9 or +/-") + } + switch ascii { + case _asciiNumbers, UInt8(ascii: "-"), UInt8(ascii: "+"), UInt8(ascii: "I"), UInt8(ascii: "N"), UInt8(ascii: "."): + break + default: + preconditionFailure("Why was this function called, if there is no 0...9 or +/-") + } + while let byte = peek() { + if _fastPath(_asciiNumbers.contains(byte)) { + moveReaderIndex(forwardBy: 1) + continue + } + switch byte { + case UInt8(ascii: "."), UInt8(ascii: "+"), UInt8(ascii: "-"): + moveReaderIndex(forwardBy: 1) + case _allLettersLower, _allLettersUpper: + // Extra permissive, to quickly allow literals like 'Infinity' and 'NaN', as well as 'e/E' for exponents and 'x/X' for hex numbers. Actual validation will be performed on parse. + moveReaderIndex(forwardBy: 1) + default: + return + } + } + } + } +} + +// MARK: - Deferred Parsing Methods - + +extension JSON5Scanner { + + // MARK: String + + static func stringValue(from jsonBytes: UnsafeBufferPointer, docStart: UnsafePointer) throws -> String { + let stringStartPtr = jsonBytes.baseAddress.unsafelyUnwrapped + let stringEndPtr = stringStartPtr + jsonBytes.count + + // Assume easy path first -- no escapes, no characters requiring escapes. + var cursor = stringStartPtr + while cursor < stringEndPtr { + let byte = cursor.pointee + if byte != ._backslash && _fastPath(byte & 0xe0 != 0) { + cursor += 1 + } else { + break + } + } + if cursor == stringEndPtr { + // We went through all the characters! Easy peasy. + guard let result = String._tryFromUTF8(jsonBytes) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: stringStartPtr, docStart: docStart)) + } + return result + } + return try _slowpath_stringValue(from: cursor, stringStartPtr: stringStartPtr, stringEndPtr: stringEndPtr, docStart: docStart) + } + + static func _slowpath_stringValue(from prevCursor: UnsafePointer, stringStartPtr: UnsafePointer, stringEndPtr: UnsafePointer, docStart: UnsafePointer) throws -> String { + var cursor = prevCursor + var chunkStart = cursor + guard var output = String._tryFromUTF8(UnsafeBufferPointer(start: stringStartPtr, count: cursor - stringStartPtr)) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: chunkStart, docStart: docStart)) + } + + // A reasonable guess as to the resulting capacity of the string is 1/4 the length of the remaining buffer. With this scheme, input full of 4 byte UTF-8 sequences won't waste a bunch of extra capacity and predominantly 1 byte UTF-8 sequences will only need to resize the buffer 1x or 2x. + output.reserveCapacity(output.underestimatedCount + (stringEndPtr - cursor) / 4) + + while cursor < stringEndPtr { + let byte = cursor.pointee + switch byte { + case ._backslash: + guard let stringChunk = String._tryFromUTF8(UnsafeBufferPointer(start: chunkStart, count: cursor - chunkStart)) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: chunkStart, docStart: docStart)) + } + output += stringChunk + + // Advance past the backslash + cursor += 1 + + try parseEscapeSequence(into: &output, cursor: &cursor, end: stringEndPtr, docStart: docStart) + chunkStart = cursor + + default: + guard _fastPath(byte & 0xe0 != 0) else { + // All Unicode characters may be placed within the quotation marks, except for the characters that must be escaped: quotation mark, reverse solidus, and the control characters (U+0000 through U+001F). + throw JSONError.unescapedControlCharacterInString(ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + cursor += 1 + continue + } + } + + guard let stringChunk = String._tryFromUTF8(UnsafeBufferPointer(start: chunkStart, count: cursor - chunkStart)) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: chunkStart, docStart: docStart)) + } + output += stringChunk + + return output + } + + private static func parseEscapeSequence(into string: inout String, cursor: inout UnsafePointer, end: UnsafePointer, docStart: UnsafePointer) throws { + precondition(end > cursor, "Scanning should have ensured that all escape sequences are valid shape") + + let ascii = cursor.pointee + cursor += 1 + switch ascii { + case UInt8(ascii:"\""): string.append("\"") + case UInt8(ascii:"'"): string.append("'") + case UInt8(ascii:"\\"): string.append("\\") + case UInt8(ascii:"/"): string.append("/") + case UInt8(ascii:"b"): string.append("\u{08}") // \b + case UInt8(ascii:"f"): string.append("\u{0C}") // \f + case UInt8(ascii:"n"): string.append("\u{0A}") // \n + case UInt8(ascii:"r"): string.append("\u{0D}") // \r + case UInt8(ascii:"t"): string.append("\u{09}") // \t + case ._newline: string.append("\n") + case ._return: + if cursor < end, cursor.pointee == ._newline { + cursor += 1 + string.append("\r\n") + } else { + string.append("\r") + } + case UInt8(ascii:"u"): + try JSONScanner.parseUnicodeSequence(into: &string, cursor: &cursor, end: end, docStart: docStart, allowNulls: false) + case UInt8(ascii:"x"): + let scalar = UnicodeScalar(try parseTwoByteUnicodeHexSequence(cursor: &cursor, end: end, docStart: docStart)) + string.unicodeScalars.append(scalar) + default: + throw JSONError.unexpectedEscapedCharacter(ascii: ascii, location: .sourceLocation(at: cursor, docStart: docStart)) + } + } + + private static func parseTwoByteUnicodeHexSequence(cursor: inout UnsafePointer, end: UnsafePointer, docStart: UnsafePointer) throws -> UInt8 { + precondition(end - cursor >= 2, "Scanning should have ensured that all escape sequences are valid shape") + + guard let first = cursor.pointee.hexDigitValue, + let second = (cursor+1).pointee.hexDigitValue + else { + let hexString = String(decoding: UnsafeBufferPointer(start: cursor, count: 2), as: Unicode.UTF8.self) + throw JSONError.invalidHexDigitSequence(hexString, location: .sourceLocation(at: cursor, docStart: docStart)) + } + let result = UInt8(first) << 4 | UInt8(second) + guard result != 0 else { + throw JSONError.invalidEscapedNullValue(location: .sourceLocation(at: cursor, docStart: docStart)) + } + cursor += 2 + return result + } + + // MARK: Numbers + + static func validateLeadingZero(in jsonBytes: UnsafeBufferPointer, following cursor: inout UnsafePointer, docStart: UnsafePointer, isHex: inout Bool) throws { + let endPtr = jsonBytes.baseAddress.unsafelyUnwrapped + jsonBytes.count + + // Leading zeros are very restricted. + let next = cursor+1 + if next == endPtr { + // Yep, this is valid. + return + } + switch next.pointee { + case UInt8(ascii: "."), UInt8(ascii: "e"), UInt8(ascii: "E"): + // We need to parse the fractional part. + break + case UInt8(ascii: "x"), UInt8(ascii: "X"): + // We have to further validate that there is another digit following this one. + let firstHexDigitPtr = cursor+2 + guard firstHexDigitPtr <= endPtr else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: next.pointee, location: .sourceLocation(at: next, docStart: docStart)) + } + guard firstHexDigitPtr.pointee.isValidHexDigit else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: (cursor+2).pointee, location: .sourceLocation(at: firstHexDigitPtr, docStart: docStart)) + } + isHex = true + cursor += 2 + case _asciiNumbers: + throw JSONError.numberWithLeadingZero(location: .sourceLocation(at: next, docStart: docStart)) + default: + throw JSONError.unexpectedCharacter(context: "in number", ascii: next.pointee, location: .sourceLocation(at: next, docStart: docStart)) + } + } + + static func validateInfinity(from jsonBytes: UnsafeBufferPointer, docStart: UnsafePointer) throws { + guard jsonBytes.count >= _json5Infinity.utf8CodeUnitCount else { + throw JSONError.invalidSpecialValue(expected: "Infinity", location: .sourceLocation(at: jsonBytes.baseAddress.unsafelyUnwrapped, docStart: docStart)) + } + guard strncmp(jsonBytes.baseAddress, _json5Infinity.utf8Start, _json5Infinity.utf8CodeUnitCount) == 0 else { + throw JSONError.invalidSpecialValue(expected: "Infinity", location: .sourceLocation(at: jsonBytes.baseAddress.unsafelyUnwrapped, docStart: docStart)) + } + } + + static func validateNaN(from jsonBytes: UnsafeBufferPointer, docStart: UnsafePointer) throws { + guard jsonBytes.count >= _json5NaN.utf8CodeUnitCount else { + throw JSONError.invalidSpecialValue(expected: "NaN", location: .sourceLocation(at: jsonBytes.baseAddress.unsafelyUnwrapped, docStart: docStart)) + } + guard strncmp(jsonBytes.baseAddress, _json5NaN.utf8Start, _json5NaN.utf8CodeUnitCount) == 0 else { + throw JSONError.invalidSpecialValue(expected: "NaN", location: .sourceLocation(at: jsonBytes.baseAddress.unsafelyUnwrapped, docStart: docStart)) + } + } + + static func validateLeadingDecimal(from jsonBytes: UnsafeBufferPointer, docStart: UnsafePointer) throws { + let cursor = jsonBytes.baseAddress.unsafelyUnwrapped + guard jsonBytes.count > 1 else { + throw JSONError.unexpectedCharacter(ascii: cursor.pointee, location: .sourceLocation(at: cursor, docStart: docStart)) + } + guard case _asciiNumbers = (cursor+1).pointee else { + throw JSONError.unexpectedCharacter(context: "after '.' in number", ascii: (cursor+1).pointee, location: .sourceLocation(at: cursor+1, docStart: docStart)) + } + } + + // Returns the pointer at which the number's digits begin. If there are no digits, the function throws. + static func prevalidateJSONNumber(from jsonBytes: UnsafeBufferPointer, docStart: UnsafePointer) throws -> (UnsafePointer, isHex: Bool, isSpecialDoubleValue: Bool) { + // Just make sure we (A) don't have a leading zero, and (B) We have at least one digit. + guard !jsonBytes.isEmpty else { + preconditionFailure("Why was this function called, if there is no 0...9 or +/-") + } + var cursor = jsonBytes.baseAddress.unsafelyUnwrapped + let endPtr = cursor + jsonBytes.count + let digitsBeginPtr : UnsafePointer + var isHex = false + var isSpecialValue = false + switch cursor.pointee { + case UInt8(ascii: "0"): + try validateLeadingZero(in: jsonBytes, following: &cursor, docStart: docStart, isHex: &isHex) + digitsBeginPtr = cursor + case UInt8(ascii: "1") ... UInt8(ascii: "9"): + digitsBeginPtr = cursor + case UInt8(ascii: "-"), UInt8(ascii: "+"): + cursor += 1 + guard cursor < endPtr else { + throw JSONError.unexpectedCharacter(context: "at end of number", ascii: cursor.pointee, location: .sourceLocation(at: endPtr-1, docStart: docStart)) + } + switch cursor.pointee { + case UInt8(ascii: "0"): + try validateLeadingZero(in: jsonBytes, following: &cursor, docStart: docStart, isHex: &isHex) + case UInt8(ascii: "1") ... UInt8(ascii: "9"): + // Good, we need at least one digit following the '-' + break + case UInt8(ascii: "I"): + let offsetBuffer = UnsafeBufferPointer(rebasing: jsonBytes.suffix(from: 1)) + try validateInfinity(from: offsetBuffer, docStart: docStart) + isSpecialValue = true + case UInt8(ascii: "N"): + let offsetBuffer = UnsafeBufferPointer(rebasing: jsonBytes.suffix(from: 1)) + try validateNaN(from: offsetBuffer, docStart: docStart) + isSpecialValue = true + case UInt8(ascii: "."): + let offsetBuffer = UnsafeBufferPointer(rebasing: jsonBytes.suffix(from: 1)) + try validateLeadingDecimal(from: offsetBuffer, docStart: docStart) + default: + // Any other character is invalid. + throw JSONError.unexpectedCharacter(context: "after '\(String(UnicodeScalar(cursor.pointee)))' in number", ascii: cursor.pointee, location: .sourceLocation(at: cursor, docStart: docStart)) + } + digitsBeginPtr = cursor + case UInt8(ascii: "I"): + try validateInfinity(from: jsonBytes, docStart: docStart) + digitsBeginPtr = cursor + isSpecialValue = true + case UInt8(ascii: "N"): + try validateNaN(from: jsonBytes, docStart: docStart) + digitsBeginPtr = cursor + isSpecialValue = true + case UInt8(ascii: "."): + // Leading decimals MUST be followed by a number, unlike trailing deciamls. + try validateLeadingDecimal(from: jsonBytes, docStart: docStart) + digitsBeginPtr = cursor + default: + preconditionFailure("Why was this function called, if there is no 0...9 or +/-") + } + + // Explicitly exclude a trailing 'e'. JSON5 and strtod both disallow it, but Decimal unfortunately accepts it so we need to prevent it in advance. + switch jsonBytes.last.unsafelyUnwrapped { + case UInt8(ascii: "e"), UInt8(ascii: "E"): + throw JSONError.unexpectedCharacter(context: "at end of number", ascii: jsonBytes.last.unsafelyUnwrapped, location: .sourceLocation(at: endPtr-1, docStart: docStart)) + default: + break + } + + return (digitsBeginPtr, isHex, isSpecialValue) + } + + // This function is intended to be called after prevalidateJSONNumber() (which provides the digitsBeginPtr) and after parsing fails. It will provide more useful information about the invalid input. + static func validateNumber(from jsonBytes: UnsafeBufferPointer, withDigitsBeginningAt digitsBeginPtr: UnsafePointer, docStart: UnsafePointer) throws { + enum ControlCharacter { + case operand + case decimalPoint + case exp + case expOperator + } + + var cursor = jsonBytes.baseAddress.unsafelyUnwrapped + let endPtr = cursor + jsonBytes.count + + // Any checks performed during pre-validation can be skipped. Proceed to the beginning of the actual number contents. + if jsonBytes[0] == UInt8(ascii: "+") || jsonBytes[0] == UInt8(ascii: "-") { + cursor += 1 + } + + if endPtr - cursor >= 2, strncasecmp_l(cursor, "0x", 2, nil) == 0 { + cursor += 2 + + while cursor < endPtr { + if cursor.pointee.isValidHexDigit { + cursor += 1 + } else { + throw JSONError.unexpectedCharacter(context: "in hex number", ascii: cursor.pointee, location: .sourceLocation(at: cursor, docStart: docStart)) + } + } + return + } + + var pastControlChar: ControlCharacter = .operand + var numbersSinceControlChar = 0 + + // parse everything else + while cursor < endPtr { + let byte = cursor.pointee + switch byte { + case _asciiNumbers: + numbersSinceControlChar += 1 + case UInt8(ascii: "."): + guard pastControlChar == .operand else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + pastControlChar = .decimalPoint + numbersSinceControlChar = 0 + + case UInt8(ascii: "e"), UInt8(ascii: "E"): + guard (pastControlChar == .operand && numbersSinceControlChar > 0) || pastControlChar == .decimalPoint + else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + pastControlChar = .exp + numbersSinceControlChar = 0 + case UInt8(ascii: "+"), UInt8(ascii: "-"): + guard numbersSinceControlChar == 0, pastControlChar == .exp else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + pastControlChar = .expOperator + numbersSinceControlChar = 0 + default: + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + cursor += 1 + } + + // prevalidateJSONNumber() already checks for trailing `e`/`E` characters. + } +} + +internal func _parseJSONHexIntegerDigits( + _ codeUnits: UnsafeBufferPointer, isNegative: Bool +) -> Result? { + guard _fastPath(!codeUnits.isEmpty) else { return nil } + + // ASCII constants, named for clarity: + let _0 = 48 as UInt8, _A = 65 as UInt8, _a = 97 as UInt8 + + let numericalUpperBound = _0 &+ 10 + let uppercaseUpperBound = _A &+ 6 + let lowercaseUpperBound = _a &+ 6 + let multiplicand: Result = 16 + + var result = 0 as Result + for digit in codeUnits { + let digitValue: Result + if _fastPath(digit >= _0 && digit < numericalUpperBound) { + digitValue = Result(truncatingIfNeeded: digit &- _0) + } else if _fastPath(digit >= _A && digit < uppercaseUpperBound) { + digitValue = Result(truncatingIfNeeded: digit &- _A &+ 10) + } else if _fastPath(digit >= _a && digit < lowercaseUpperBound) { + digitValue = Result(truncatingIfNeeded: digit &- _a &+ 10) + } else { + return nil + } + + let overflow1: Bool + (result, overflow1) = result.multipliedReportingOverflow(by: multiplicand) + let overflow2: Bool + (result, overflow2) = isNegative + ? result.subtractingReportingOverflow(digitValue) + : result.addingReportingOverflow(digitValue) + guard _fastPath(!overflow1 && !overflow2) else { return nil } + } + return result +} + +internal func _parseJSON5Integer(_ codeUnits: UnsafeBufferPointer, isHex: Bool) -> Result? { + guard _fastPath(!codeUnits.isEmpty) else { return nil } + + // ASCII constants, named for clarity: + let _plus = 43 as UInt8, _minus = 45 as UInt8 + + let first = codeUnits[0] + var isNegative = false + var digitsToParse = codeUnits + if first == _minus { + digitsToParse = UnsafeBufferPointer(rebasing: digitsToParse.suffix(from: 1)) + isNegative = true + } else if first == _plus { + digitsToParse = UnsafeBufferPointer(rebasing: digitsToParse.suffix(from: 1)) + } + + // Trust the caller regarding whether this is valid hex data. + if isHex { + digitsToParse = UnsafeBufferPointer(rebasing: digitsToParse.suffix(from: 2)) + return _parseJSONHexIntegerDigits(digitsToParse, isNegative: isNegative) + } else { + return _parseIntegerDigits(codeUnits, isNegative: isNegative) + } +} + +extension FixedWidthInteger { + init?(prevalidatedJSON5Buffer buffer: UnsafeBufferPointer, isHex: Bool) { + guard let val : Self = _parseJSON5Integer(buffer, isHex: isHex) else { + return nil + } + self = val + } +} + +internal extension UInt8 { + static let _verticalTab = UInt8(0x0b) + static let _formFeed = UInt8(0x0c) + static let _nbsp = UInt8(0xa0) + static let _asterisk = UInt8(ascii: "*") + static let _slash = UInt8(ascii: "/") + static let _singleQuote = UInt8(ascii: "'") + static let _dollar = UInt8(ascii: "$") + static let _underscore = UInt8(ascii: "_") +} + +let _json5Infinity: StaticString = "Infinity" +let _json5NaN: StaticString = "NaN" + +extension UnicodeScalar { + + @inline(__always) + var isJSON5UnquotedKeyStartingCharacter : Bool { + switch self.properties.generalCategory { + case .uppercaseLetter, .lowercaseLetter, .titlecaseLetter, .modifierLetter, .otherLetter, .letterNumber: + return true + default: + return false + } + } + + @inline(__always) + var isJSON5UnquotedKeyCharacter : Bool { + switch self.properties.generalCategory { + case .uppercaseLetter, .lowercaseLetter, .titlecaseLetter, .modifierLetter, .otherLetter, .letterNumber: + return true + case .nonspacingMark, .spacingMark: + return true + case .decimalNumber: + return true + case .connectorPunctuation: + return true + default: + switch self { + case UnicodeScalar(._underscore), UnicodeScalar(._dollar), UnicodeScalar(._backslash): + return true + case UnicodeScalar(0x200c): // ZWNJ + return true + case UnicodeScalar(0x200d): // ZWJ + return true + default: + return false + } + } + } +} diff --git a/Sources/FoundationEssentials/JSON/JSONDecoder.swift b/Sources/FoundationEssentials/JSON/JSONDecoder.swift new file mode 100644 index 000000000..1c9c8a003 --- /dev/null +++ b/Sources/FoundationEssentials/JSON/JSONDecoder.swift @@ -0,0 +1,1548 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) +import Darwin +#elseif canImport(Gibc) +import Glibc +#endif + +/// A marker protocol used to determine whether a value is a `String`-keyed `Dictionary` +/// containing `Decodable` values (in which case it should be exempt from key conversion strategies). +/// +/// The marker protocol also provides access to the type of the `Decodable` values, +/// which is needed for the implementation of the key conversion strategy exemption. +/// +/// NOTE: Please see comment above regarding SR-8276 +#if arch(i386) || arch(arm) +internal protocol _JSONStringDictionaryDecodableMarker { + static var elementType: Decodable.Type { get } +} +#else +private protocol _JSONStringDictionaryDecodableMarker { + static var elementType: Decodable.Type { get } +} +#endif + +extension Dictionary : _JSONStringDictionaryDecodableMarker where Key == String, Value: Decodable { + static var elementType: Decodable.Type { return Value.self } +} + +//===----------------------------------------------------------------------===// +// JSON Decoder +//===----------------------------------------------------------------------===// + +/// `JSONDecoder` facilitates the decoding of JSON into semantic `Decodable` types. +// NOTE: older overlays had Foundation.JSONDecoder as the ObjC name. +// The two must coexist, so it was renamed. The old name must not be +// used in the new runtime. _TtC10Foundation13__JSONDecoder is the +// mangled name for Foundation.__JSONDecoder. +#if FOUNDATION_FRAMEWORK +@_objcRuntimeName(_TtC10Foundation13__JSONDecoder) +#endif +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +open class JSONDecoder { + // MARK: Options + + /// The strategy to use for decoding `Date` values. + public enum DateDecodingStrategy : Sendable { + /// Defer to `Date` for decoding. This is the default strategy. + case deferredToDate + + /// Decode the `Date` as a UNIX timestamp from a JSON number. + case secondsSince1970 + + /// Decode the `Date` as UNIX millisecond timestamp from a JSON number. + case millisecondsSince1970 + +#if FOUNDATION_FRAMEWORK // TODO: Reenable once DateFormatStyle has been moved + /// Decode the `Date` as an ISO-8601-formatted string (in RFC 3339 format). + @available(macOS 10.12, iOS 10.0, watchOS 3.0, tvOS 10.0, *) + case iso8601 + + /// Decode the `Date` as a string parsed by the given formatter. + case formatted(DateFormatter) +#endif // FOUNDATION_FRAMEWORK + + /// Decode the `Date` as a custom value decoded by the given closure. + @preconcurrency + case custom(@Sendable (_ decoder: Decoder) throws -> Date) + } + + /// The strategy to use for decoding `Data` values. + public enum DataDecodingStrategy : Sendable { + /// Defer to `Data` for decoding. + case deferredToData + + /// Decode the `Data` from a Base64-encoded string. This is the default strategy. + case base64 + + /// Decode the `Data` as a custom value decoded by the given closure. + @preconcurrency + case custom(@Sendable (_ decoder: Decoder) throws -> Data) + } + + /// The strategy to use for non-JSON-conforming floating-point values (IEEE 754 infinity and NaN). + public enum NonConformingFloatDecodingStrategy : Sendable { + /// Throw upon encountering non-conforming values. This is the default strategy. + case `throw` + + /// Decode the values from the given representation strings. + case convertFromString(positiveInfinity: String, negativeInfinity: String, nan: String) + } + + /// The strategy to use for automatically changing the value of keys before decoding. + public enum KeyDecodingStrategy : Sendable { + /// Use the keys specified by each type. This is the default strategy. + case useDefaultKeys + + /// Convert from "snake_case_keys" to "camelCaseKeys" before attempting to match a key with the one specified by each type. + /// + /// The conversion to upper case uses `Locale.system`, also known as the ICU "root" locale. This means the result is consistent regardless of the current user's locale and language preferences. + /// + /// Converting from snake case to camel case: + /// 1. Capitalizes the word starting after each `_` + /// 2. Removes all `_` + /// 3. Preserves starting and ending `_` (as these are often used to indicate private variables or other metadata). + /// For example, `one_two_three` becomes `oneTwoThree`. `_one_two_three_` becomes `_oneTwoThree_`. + /// + /// - Note: Using a key decoding strategy has a nominal performance cost, as each string key has to be inspected for the `_` character. + case convertFromSnakeCase + + /// Provide a custom conversion from the key in the encoded JSON to the keys specified by the decoded types. + /// The full path to the current decoding position is provided for context (in case you need to locate this key within the payload). The returned key is used in place of the last component in the coding path before decoding. + /// If the result of the conversion is a duplicate key, then only one value will be present in the container for the type to decode from. + @preconcurrency + case custom(@Sendable (_ codingPath: [CodingKey]) -> CodingKey) + + fileprivate static func _convertFromSnakeCase(_ stringKey: String) -> String { + guard !stringKey.isEmpty else { return stringKey } + + // Find the first non-underscore character + guard let firstNonUnderscore = stringKey.firstIndex(where: { $0 != "_" }) else { + // Reached the end without finding an _ + return stringKey + } + + // Find the last non-underscore character + var lastNonUnderscore = stringKey.index(before: stringKey.endIndex) + while lastNonUnderscore > firstNonUnderscore && stringKey[lastNonUnderscore] == "_" { + stringKey.formIndex(before: &lastNonUnderscore) + } + + let keyRange = firstNonUnderscore...lastNonUnderscore + let leadingUnderscoreRange = stringKey.startIndex..() + + // MARK: - Constructing a JSON Decoder + + /// Initializes `self` with default strategies. + public init() {} + + private var scannerOptions : JSONScanner.Options { + .init(assumesTopLevelDictionary: self.assumesTopLevelDictionary) + } + + private var json5ScannerOptions : JSON5Scanner.Options { + .init(assumesTopLevelDictionary: self.assumesTopLevelDictionary) + } + + // MARK: - Decoding Values + + /// Decodes a top-level value of the given type from the given JSON representation. + /// + /// - parameter type: The type of the value to decode. + /// - parameter data: The data to decode from. + /// - returns: A value of the requested type. + /// - throws: `DecodingError.dataCorrupted` if values requested from the payload are corrupted, or if the given data is not valid JSON. + /// - throws: An error if any value throws an error during decoding. + open func decode(_ type: T.Type, from data: Data) throws -> T { + do { + return try Self.withUTF8Representation(of: data) { utf8Buffer in + + var impl: JSONDecoderImpl + let topValue: JSONMap.Value + do { + // JSON5 is implemented with a separate scanner to allow regular JSON scanning to achieve higher performance without compromising for `allowsJSON5` checks throughout. + // Since the resulting JSONMap is identical, the decoder implementation is mostly shared between the two, with only a few branches to handle different methods of parsing strings and numbers. Strings and numbers are not completely parsed until decoding time. + let map: JSONMap + if allowsJSON5 { + var scanner = JSON5Scanner(bytes: utf8Buffer, options: self.json5ScannerOptions) + map = try scanner.scan() + } else { + var scanner = JSONScanner(bytes: utf8Buffer, options: self.scannerOptions) + map = try scanner.scan() + } + topValue = map.loadValue(at: 0)! + impl = JSONDecoderImpl(userInfo: self.userInfo, from: map, codingPathNode: .root, options: self.options) + } + impl.push(value: topValue) // This is something the old implementation did and apps started relying on. Weird. + let result = try impl.unwrap(topValue, as: type, for: .root, _JSONKey?.none) + let uniquelyReferenced = isKnownUniquelyReferenced(&impl) + impl.takeOwnershipOfBackingDataIfNeeded(selfIsUniquelyReferenced: uniquelyReferenced) + return result + } + } catch let error as JSONError { + #if FOUNDATION_FRAMEWORK + let underlyingError: Error? = error.nsError + #else + let underlyingError: Error? = nil + #endif + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: [], debugDescription: "The given data was not valid JSON.", underlyingError: underlyingError)) + } catch { + throw error + } + } + + // Input: Data of any encoding specified by RFC4627 section 3, with or without BOM. + // Output: The closure is invoked with a UInt8 buffer containing the valid UTF-8 representation. If the input contained a BOM, that BOM will be excluded in the resulting buffer. + // If the input cannot be fully decoded by the detected encoding or cannot be converted to UTF-8, the function will throw a JSONError.cannotConvertEntireInputDataToUTF8 error. + // If the input is detected to already be UTF-8, the Data's buffer will be passed through without copying. + static func withUTF8Representation(of jsonData: Data, _ closure: (UnsafeBufferPointer) throws -> T ) throws -> T { + let length = jsonData.count + return try jsonData.withUnsafeBytes { (origPtr: UnsafeRawBufferPointer) -> T in + // RFC4627 section 3 + // The first two characters of a JSON text will always be ASCII. We can determine encoding by looking at the first four bytes. + let origBytes = origPtr.assumingMemoryBound(to: UInt8.self) + let byte0 = (length > 0) ? origBytes[0] : nil + let byte1 = (length > 1) ? origBytes[1] : nil + let byte2 = (length > 2) ? origBytes[2] : nil + let byte3 = (length > 3) ? origBytes[3] : nil + + // Check for explicit BOM first, then check the first two bytes. Note that if there is a BOM, we have to create our string without it. + // This isn't strictly part of the JSON spec but it's useful to do anyway. + let sourceEncoding : String._Encoding + let bomLength : Int + switch (byte0, byte1, byte2, byte3) { + case (0, 0, 0xFE, 0xFF): + sourceEncoding = .utf32BigEndian + bomLength = 4 + case (0xFE, 0xFF, 0, 0): + sourceEncoding = .utf32LittleEndian + bomLength = 4 + case (0xFE, 0xFF, _, _): + sourceEncoding = .utf16BigEndian + bomLength = 2 + case (0xFF, 0xFE, _, _): + sourceEncoding = .utf16LittleEndian + bomLength = 2 + case (0xEF, 0xBB, 0xBF, _): + sourceEncoding = .utf8 + bomLength = 3 + case let (0, 0, 0, .some(nz)) where nz != 0: + sourceEncoding = .utf32BigEndian + bomLength = 0 + case let (0, .some(nz1), 0, .some(nz2)) where nz1 != 0 && nz2 != 0: + sourceEncoding = .utf16BigEndian + bomLength = 0 + case let (.some(nz), 0, 0, 0) where nz != 0: + sourceEncoding = .utf32LittleEndian + bomLength = 0 + case let (.some(nz1), 0, .some(nz2), 0) where nz1 != 0 && nz2 != 0: + sourceEncoding = .utf16LittleEndian + bomLength = 0 + + // These cases technically aren't specified by RFC4627, since it only covers cases where the input has at least 4 octets. However, when parsing JSON with fragments allowed, it's possible to have a valid UTF-16 input that is a single digit, which is 2 octets. To properly support these inputs, we'll extend the pattern described above for 4 octets of UTF-16. + case let (0, .some(nz), nil, nil) where nz != 0: + sourceEncoding = .utf16BigEndian + bomLength = 0 + case let (.some(nz), 0, nil, nil) where nz != 0: + sourceEncoding = .utf16LittleEndian + bomLength = 0 + + default: + sourceEncoding = .utf8 + bomLength = 0 + } + let postBOMBuffer = UnsafeBufferPointer(rebasing: origBytes[bomLength ..< length]) + if sourceEncoding == .utf8 { + return try closure(postBOMBuffer) + } else { + guard var string = String(bytes: postBOMBuffer, encoding: sourceEncoding) else { + throw JSONError.cannotConvertEntireInputDataToUTF8 + } + return try string.withUTF8(closure) + } + } + } +} + +// MARK: - JSONDecoderImpl + +// NOTE: older overlays called this class _JSONDecoder. The two must +// coexist without a conflicting ObjC class name, so it was renamed. +// The old name must not be used in the new runtime. +fileprivate class JSONDecoderImpl { + var values: [JSONMap.Value] = [] + let userInfo: [CodingUserInfoKey: Any] + var jsonMap: JSONMap + let options: JSONDecoder._Options + + var codingPathNode: _JSONCodingPathNode + public var codingPath: [CodingKey] { + codingPathNode.path + } + + var topValue : JSONMap.Value { self.values.last! } + func push(value: __owned JSONMap.Value) { + self.values.append(value) + } + func popValue() { + self.values.removeLast() + } + + init(userInfo: [CodingUserInfoKey: Any], from map: JSONMap, codingPathNode: _JSONCodingPathNode, options: JSONDecoder._Options) { + self.userInfo = userInfo + self.codingPathNode = codingPathNode + self.jsonMap = map + self.options = options + } + + @inline(__always) + func withBuffer(for region: JSONMap.Region, perform closure: (UnsafeBufferPointer, UnsafePointer) throws -> T) rethrows -> T { + try jsonMap.withBuffer(for: region, perform: closure) + } + + // This JSONDecoderImpl may have multiple references if an init(from: Decoder) implementation allows the Decoder (this object) to escape, or if a container escapes. + // The JSONMap might have multiple references if a superDecoder, which creates a different JSONDecoderImpl instance but references the same JSONMap, is allowed to escape. + // In either case, we need to copy-in the input buffer since it's about to go out of scope. + func takeOwnershipOfBackingDataIfNeeded(selfIsUniquelyReferenced: Bool) { + if !selfIsUniquelyReferenced || !isKnownUniquelyReferenced(&jsonMap) { + jsonMap.copyInBuffer() + } + } +} + +extension JSONDecoderImpl: Decoder { + func container(keyedBy _: Key.Type) throws -> KeyedDecodingContainer { + switch topValue { + case let .object(region): + let container = try KeyedContainer( + impl: self, + codingPathNode: codingPathNode, + region: region + ) + return KeyedDecodingContainer(container) + case .null: + throw DecodingError.valueNotFound([String: Any].self, DecodingError.Context( + codingPath: self.codingPath, + debugDescription: "Cannot get keyed decoding container -- found null value instead" + )) + default: + throw DecodingError.typeMismatch([String: Any].self, DecodingError.Context( + codingPath: self.codingPath, + debugDescription: "Expected to decode \([String: Any].self) but found \(topValue.debugDataTypeDescription) instead." + )) + } + } + + func unkeyedContainer() throws -> UnkeyedDecodingContainer { + switch topValue { + case let .array(region): + return UnkeyedContainer( + impl: self, + codingPathNode: codingPathNode, + region: region + ) + case .null: + throw DecodingError.valueNotFound([Any].self, DecodingError.Context( + codingPath: self.codingPath, + debugDescription: "Cannot get unkeyed decoding container -- found null value instead" + )) + default: + throw DecodingError.typeMismatch([Any].self, DecodingError.Context( + codingPath: self.codingPath, + debugDescription: "Expected to decode \([Any].self) but found \(topValue.debugDataTypeDescription) instead." + )) + } + } + + func singleValueContainer() throws -> SingleValueDecodingContainer { + return self + } + + // MARK: Special case handling + + @inline(__always) + func checkNotNull(_ value: JSONMap.Value, expectedType: T.Type, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws { + if case .null = value { + throw DecodingError.valueNotFound(expectedType, DecodingError.Context( + codingPath: codingPathNode.path(with: additionalKey), + debugDescription: "Cannot get unkeyed decoding container -- found null value instead" + )) + } + } + + // Instead of creating a new JSONDecoderImpl for passing to methods that take Decoder arguments, wrap the access in this method, which temporarily mutates this JSONDecoderImpl instance with the nesteed value and its coding path. + @inline(__always) + func with(value: JSONMap.Value, path: _JSONCodingPathNode?, perform closure: () throws -> T) rethrows -> T { + let oldPath = self.codingPathNode + if let path { + self.codingPathNode = path + } + self.push(value: value) + + defer { + if path != nil { + self.codingPathNode = oldPath + } + self.popValue() + } + + return try closure() + } + + func unwrap(_ mapValue: JSONMap.Value, as type: T.Type, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> T { + if type == Date.self { + return try self.unwrapDate(from: mapValue, for: codingPathNode, additionalKey) as! T + } + if type == Data.self { + return try self.unwrapData(from: mapValue, for: codingPathNode, additionalKey) as! T + } +#if FOUNDATION_FRAMEWORK // TODO: Reenable once URL and Decimal are moved + if type == URL.self { + return try self.unwrapURL(from: mapValue, for: codingPathNode, additionalKey) as! T + } + if type == Decimal.self { + return try self.unwrapDecimal(from: mapValue, for: codingPathNode, additionalKey) as! T + } +#endif// FOUNDATION_FRAMEWORK + if T.self is _JSONStringDictionaryDecodableMarker.Type { + return try self.unwrapDictionary(from: mapValue, as: type, for: codingPathNode, additionalKey) + } + + return try self.with(value: mapValue, path: codingPathNode.pushing(additionalKey)) { + try type.init(from: self) + } + } + + private func unwrapDate(from mapValue: JSONMap.Value, for codingPathNode: _JSONCodingPathNode, _ additionalKey: K? = nil) throws -> Date { + try checkNotNull(mapValue, expectedType: Date.self, for: codingPathNode, additionalKey) + + switch self.options.dateDecodingStrategy { + case .deferredToDate: + return try self.with(value: mapValue, path: codingPathNode.pushing(additionalKey)) { + try Date(from: self) + } + + case .secondsSince1970: + let double = try self.unwrapFloatingPoint(from: mapValue, as: Double.self, for: codingPathNode, additionalKey) + return Date(timeIntervalSince1970: double) + + case .millisecondsSince1970: + let double = try self.unwrapFloatingPoint(from: mapValue, as: Double.self, for: codingPathNode, additionalKey) + return Date(timeIntervalSince1970: double / 1000.0) +#if FOUNDATION_FRAMEWORK // TODO: Reenable once DateFormatStyle has been moved + case .iso8601: + let string = try self.unwrapString(from: mapValue, for: codingPathNode, additionalKey) + guard let date = try? Date.ISO8601FormatStyle().parse(string) else { + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: self.codingPath, debugDescription: "Expected date string to be ISO8601-formatted.")) + } + return date + + case .formatted(let formatter): + let string = try self.unwrapString(from: mapValue, for: codingPathNode, additionalKey) + guard let date = formatter.date(from: string) else { + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: codingPathNode.path(with: additionalKey), debugDescription: "Date string does not match format expected by formatter.")) + } + return date +#endif // FOUNDATION_FRAMEWORK + case .custom(let closure): + return try self.with(value: mapValue, path: codingPathNode.pushing(additionalKey)) { + try closure(self) + } + } + } + + private func unwrapData(from mapValue: JSONMap.Value, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> Data { + try checkNotNull(mapValue, expectedType: Data.self, for: codingPathNode, additionalKey) + + switch self.options.dataDecodingStrategy { + case .deferredToData: + return try self.with(value: mapValue, path: codingPathNode.pushing(additionalKey)) { + try Data(from: self) + } + + case .base64: + let string = try self.unwrapString(from: mapValue, for: codingPathNode, additionalKey) + guard let data = Data(base64Encoded: string) else { + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: codingPathNode.path(with: additionalKey), debugDescription: "Encountered Data is not valid Base64.")) + } + + return data + + case .custom(let closure): + return try self.with(value: mapValue, path: codingPathNode.pushing(additionalKey)) { + try closure(self) + } + } + } + +#if FOUNDATION_FRAMEWORK // TODO: Reenable once DateFormatStyle has been moved + private func unwrapURL(from mapValue: JSONMap.Value, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> URL { + try checkNotNull(mapValue, expectedType: URL.self, for: codingPathNode, additionalKey) + + let string = try self.unwrapString(from: mapValue, for: codingPathNode, additionalKey) + guard let url = URL(string: string) else { + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: codingPathNode.path(with: additionalKey), + debugDescription: "Invalid URL string.")) + } + return url + } + + private func unwrapDecimal(from mapValue: JSONMap.Value, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> Decimal { + try checkNotNull(mapValue, expectedType: Decimal.self, for: codingPathNode, additionalKey) + + guard case .number(let region, let hasExponent) = mapValue else { + throw DecodingError.typeMismatch(Decimal.self, DecodingError.Context(codingPath: codingPathNode.path(with: additionalKey), debugDescription: "")) + } + + return try withBuffer(for: region) { numberBuffer, docStart in + if options.json5 { + let (digitsStartPtr, isHex, isSpecialJSON5DoubleValue) = try JSON5Scanner.prevalidateJSONNumber(from: numberBuffer, docStart: docStart) + + // Use our integer parsers for hex data, because the underlying strtod() implementation of T(prevalidatedBuffer:) is too permissive (e.g. it accepts decimals and 'p' exponents) which otherwise would require prevalidation of the entire string before calling it. + if isHex { + if numberBuffer.first! == UInt8(ascii: "-") { + guard let int = Int64(prevalidatedJSON5Buffer: numberBuffer, isHex: isHex), let decimal = Decimal(exactly: int) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return decimal + } else { + guard let int = UInt64(prevalidatedJSON5Buffer: numberBuffer, isHex: isHex), let decimal = Decimal(exactly: int) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return decimal + } + } else if isSpecialJSON5DoubleValue { + // Decimal itself doesn't have support for Infinity values yet. Even the part of the old NSJSONSerialization implementation that would try to reinterpret an NaN or Infinity value as an NSDecimalNumber did not have very predictable behavior. + // TODO: Proper handling of Infinity and NaN Decimal values. + return Decimal.quietNaN + } else { + let numberString = String(decoding: numberBuffer, as: UTF8.self) + if let decimal = Decimal(entire: numberString) { + return decimal + } + try JSON5Scanner.validateNumber(from: numberBuffer, withDigitsBeginningAt: digitsStartPtr, docStart: docStart) + } + + } else { + let digitsStartPtr = try JSONScanner.prevalidateJSONNumber(from: numberBuffer, hasExponent: hasExponent, docStart: docStart) + let numberString = String(decoding: numberBuffer, as: UTF8.self) + if let decimal = Decimal(entire: numberString) { + return decimal + } + try JSONScanner.validateNumber(from: numberBuffer, withDigitsBeginningAt: digitsStartPtr, docStart: docStart) + } + throw DecodingError.dataCorrupted(.init( + codingPath: codingPathNode.path(with: additionalKey), + debugDescription: "Parsed JSON number <\(String(decoding: numberBuffer, as: UTF8.self))> does not fit in \(Decimal.self).")) + } + } +#endif // FOUNDATION_FRAMEWORK + + private func unwrapDictionary(from mapValue: JSONMap.Value, as type: T.Type, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> T { + try checkNotNull(mapValue, expectedType: [String:Any].self, for: codingPathNode, additionalKey) + + guard let dictType = type as? (_JSONStringDictionaryDecodableMarker & Decodable).Type else { + preconditionFailure("Must only be called if T implements __JSONStringDictionaryDecodableMarker") + } + + guard case let .object(region) = mapValue else { + throw DecodingError.typeMismatch([String: Any].self, DecodingError.Context( + codingPath: codingPathNode.path(with: additionalKey), + debugDescription: "Expected to decode \([String: Any].self) but found \(mapValue.debugDataTypeDescription) instead." + )) + } + + var result = [String: Any]() + result.reserveCapacity(region.count / 2) + + let dictCodingPathNode = codingPathNode.pushing(additionalKey) + + var iter = jsonMap.makeObjectIterator(from: region.startOffset) + while let (keyValue, value) = iter.next() { + // Failing to unwrap a string here is impossible, as scanning already guarantees that dictionary keys are strings. + let key = try! self.unwrapString(from: keyValue, for: dictCodingPathNode, _JSONKey?.none) + let value = try self.unwrap(value, as: dictType.elementType, for: dictCodingPathNode, _JSONKey(stringValue: key)!) + result[key]._setIfNil(to: value) + } + + return result as! T + } + + private func unwrapString(from value: JSONMap.Value, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> String { + try checkNotNull(value, expectedType: [String].self, for: codingPathNode, additionalKey) + + guard case .string(let region, let isSimple) = value else { + throw self.createTypeMismatchError(type: String.self, for: codingPathNode.path(with: additionalKey), value: value) + } + return try withBuffer(for: region) { stringBuffer, docStart in + if isSimple { + guard let result = String._tryFromUTF8(stringBuffer) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: stringBuffer.baseAddress.unsafelyUnwrapped, docStart: docStart)) + } + return result + } + if options.json5 { + return try JSON5Scanner.stringValue(from: stringBuffer, docStart: docStart) + } else { + return try JSONScanner.stringValue(from: stringBuffer, docStart: docStart) + } + } + } + + func isTrueZero(_ buffer: UnsafeBufferPointer) -> Bool { + var remainingBuffer = buffer + + let nonZeroRange = UInt8(ascii: "1") ... UInt8(ascii: "9") + while remainingBuffer.count >= 4 { + if case nonZeroRange = remainingBuffer[0] { return false } + if case nonZeroRange = remainingBuffer[1] { return false } + if case nonZeroRange = remainingBuffer[2] { return false } + if case nonZeroRange = remainingBuffer[3] { return false } + remainingBuffer = UnsafeBufferPointer(rebasing: remainingBuffer.suffix(from: 4)) + } + + switch remainingBuffer.count { + case 3: + if case nonZeroRange = remainingBuffer[2] { return false } + fallthrough + case 2: + if case nonZeroRange = remainingBuffer[1] { return false } + fallthrough + case 1: + if case nonZeroRange = remainingBuffer[0] { return false } + default: + break + } + + return true + } + + private func unwrapFloatingPoint( + from value: JSONMap.Value, + as type: T.Type, + for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> T + { + try checkNotNull(value, expectedType: type, for: codingPathNode, additionalKey) + + if case .number(let region, let hasExponent) = value { + return try withBuffer(for: region) { numberBuffer, docStart in + if options.json5 { + let (digitsStartPtr, isHex, isSpecialJSON5DoubleValue) = try JSON5Scanner.prevalidateJSONNumber(from: numberBuffer, docStart: docStart) + + // Use our integer parsers for hex data, because the underlying strtod() implementation of T(prevalidatedBuffer:) is too permissive (e.g. it accepts decimals and 'p' exponents) which otherwise would require prevalidation of the entire string before calling it. + if isHex { + if numberBuffer.first.unsafelyUnwrapped == UInt8(ascii: "-") { + guard let int = Int64(prevalidatedJSON5Buffer: numberBuffer, isHex: isHex), let float = T(exactly: int) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return float + } else { + guard let int = UInt64(prevalidatedJSON5Buffer: numberBuffer, isHex: isHex), let float = T(exactly: int) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return float + } + } // else, fall through to the T(prevalidatedBuffer:) invocation, which is otherwise compatible with JSON5 after our pre-validation. + + if let floatingPoint = T(prevalidatedBuffer: numberBuffer) { + // Check for overflow/underflow, which can result in "rounding" to infinty or zero. + // While strtod does set ERANGE in the either case, we don't rely on it because setting errno to 0 first and then check the result is surprisingly expensive. For values "rounded" to infinity, we reject those out of hand, unless it's an explicit JSON5 infinity/nan value. For values "rounded" down to zero, we perform check for any non-zero digits in the input, which turns out to be much faster. + if floatingPoint.isFinite { + guard floatingPoint != 0 || isTrueZero(numberBuffer) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return floatingPoint + } else { + if options.json5, isSpecialJSON5DoubleValue { + return floatingPoint + } else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + } + } + + // We failed to parse the number. Is that because it was malformed? + try JSON5Scanner.validateNumber(from: numberBuffer, withDigitsBeginningAt: digitsStartPtr, docStart: docStart) + + } else { + let digitsStartPtr = try JSONScanner.prevalidateJSONNumber(from: numberBuffer, hasExponent: hasExponent, docStart: docStart) + + if let floatingPoint = T(prevalidatedBuffer: numberBuffer) { + // Check for overflow (which results in an infinite result), or rounding to zero. + // While strtod does set ERANGE in the either case, we don't rely on it because setting errno to 0 first and then check the result is surprisingly expensive. For values "rounded" to infinity, we reject those out of hand. For values "rounded" down to zero, we perform check for any non-zero digits in the input, which turns out to be much faster. + if floatingPoint.isFinite { + guard floatingPoint != 0 || isTrueZero(numberBuffer) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return floatingPoint + } else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + } + + try JSONScanner.validateNumber(from: numberBuffer, withDigitsBeginningAt: digitsStartPtr, docStart: docStart) + } + + // If that didn't throw, we'll say "it didn't fit" + throw DecodingError.dataCorrupted(.init( + codingPath: codingPathNode.path(with: additionalKey), + debugDescription: "Parsed JSON number <\(String(decoding: numberBuffer, as: Unicode.ASCII.self))> does not fit in \(type).")) + } + } + + if case .string(let region, let isSimple) = value, isSimple, + case .convertFromString(var posInfString, var negInfString, var nanString) = + self.options.nonConformingFloatDecodingStrategy + { + let result = withBuffer(for: region) { (stringBuffer, _) -> T? in + if posInfString.withUTF8({ stringBuffer.count == $0.count && memcmp(stringBuffer.baseAddress, $0.baseAddress, $0.count) == 0 }) { + return T.infinity + } else if negInfString.withUTF8({ stringBuffer.count == $0.count && memcmp(stringBuffer.baseAddress, $0.baseAddress, $0.count) == 0 }) { + return -T.infinity + } else if nanString.withUTF8({ stringBuffer.count == $0.count && memcmp(stringBuffer.baseAddress, $0.baseAddress, $0.count) == 0 }) { + return T.nan + } + return nil + } + if let result { return result } + } + + throw self.createTypeMismatchError(type: type, for: codingPathNode.path(with: additionalKey), value: value) + } + + private func unwrapFixedWidthInteger( + from value: JSONMap.Value, + as type: T.Type, + for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> T + { + try checkNotNull(value, expectedType: type, for: codingPathNode, additionalKey) + + guard case .number(let region, let hasExponent) = value else { + throw self.createTypeMismatchError(type: type, for: codingPathNode.path(with: additionalKey), value: value) + } + return try withBuffer(for: region) { numberBuffer, docStart in + let digitBeginning: UnsafePointer + if options.json5 { + let isHex : Bool + let isSpecialFloatValue: Bool + (digitBeginning, isHex, isSpecialFloatValue) = try JSON5Scanner.prevalidateJSONNumber(from: numberBuffer, docStart: docStart) + + // This is the fast pass. Number directly convertible to desired integer type. + if let integer = T(prevalidatedJSON5Buffer: numberBuffer, isHex: isHex) { + return integer + } + + // NaN and Infinity values are not representable as Integers. + if isSpecialFloatValue { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + } else { + digitBeginning = try JSONScanner.prevalidateJSONNumber(from: numberBuffer, hasExponent: hasExponent, docStart: docStart) + + // This is the fast pass. Number directly convertible to Integer. + if let integer = T(prevalidatedBuffer: numberBuffer) { + return integer + } + } + + return try _slowpath_unwrapFixedWidthInteger(as: type, numberBuffer: numberBuffer, docStart: docStart, digitBeginning: digitBeginning, for: codingPathNode, additionalKey) + } + } + + private func _slowpath_unwrapFixedWidthInteger(as type: T.Type, numberBuffer: UnsafeBufferPointer, docStart: UnsafePointer, digitBeginning: UnsafePointer, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)?) throws -> T { + // This is the slow path... If the fast path has failed. For example for "34.0" as an integer, we try to parse as either a Decimal or a Double and then convert back, losslessly. + if let double = Double(prevalidatedBuffer: numberBuffer) { + guard let value = T(exactly: double) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return value + } + + let number = String(decoding: numberBuffer, as: Unicode.ASCII.self) +#if FOUNDATION_FRAMEWORK // TODO: Reenable once Decimal is moved + if let decimal = Decimal(entire: number) { + guard let value = T(decimal) else { + throw JSONError.numberIsNotRepresentableInSwift(parsed: String(decoding: numberBuffer, as: UTF8.self)) + } + return value + } +#endif // FOUNDATION_FRAMEWORK + // Maybe it was just an unreadable sequence? + if options.json5 { + try JSON5Scanner.validateNumber(from: numberBuffer, withDigitsBeginningAt: digitBeginning, docStart: docStart) + } else { + try JSONScanner.validateNumber(from: numberBuffer, withDigitsBeginningAt: digitBeginning, docStart: docStart) + } + + throw DecodingError.dataCorrupted(.init( + codingPath: codingPathNode.path(with: additionalKey), + debugDescription: "Parsed JSON number <\(number)> does not fit in \(type).")) + } + + private func createTypeMismatchError(type: Any.Type, for path: [CodingKey], value: JSONMap.Value) -> DecodingError { + return DecodingError.typeMismatch(type, .init( + codingPath: path, + debugDescription: "Expected to decode \(type) but found \(value.debugDataTypeDescription) instead." + )) + } +} + +#if FOUNDATION_FRAMEWORK // TODO: Reenable once Decimal is moved +extension FixedWidthInteger { + init?(_ decimal: Decimal) { + let isNegative = decimal._isNegative != 0 + if decimal._length == 0 && isNegative { + return nil + } + if isNegative { + guard Self.isSigned else { + return nil + } + } + + var d : UInt64 = 0 + for i in (0.. Bool { + switch topValue { + case .null: + return true + default: + return false + } + } + + func decode(_: Bool.Type) throws -> Bool { + guard case .bool(let bool) = self.topValue else { + throw self.createTypeMismatchError(type: Bool.self, for: self.codingPath, value: self.topValue) + } + + return bool + } + + func decode(_: String.Type) throws -> String { + try self.unwrapString(from: self.topValue, for: self.codingPathNode, _JSONKey?.none) + } + + func decode(_: Double.Type) throws -> Double { + try decodeFloatingPoint() + } + + func decode(_: Float.Type) throws -> Float { + try decodeFloatingPoint() + } + + func decode(_: Int.Type) throws -> Int { + try decodeFixedWidthInteger() + } + + func decode(_: Int8.Type) throws -> Int8 { + try decodeFixedWidthInteger() + } + + func decode(_: Int16.Type) throws -> Int16 { + try decodeFixedWidthInteger() + } + + func decode(_: Int32.Type) throws -> Int32 { + try decodeFixedWidthInteger() + } + + func decode(_: Int64.Type) throws -> Int64 { + try decodeFixedWidthInteger() + } + + func decode(_: UInt.Type) throws -> UInt { + try decodeFixedWidthInteger() + } + + func decode(_: UInt8.Type) throws -> UInt8 { + try decodeFixedWidthInteger() + } + + func decode(_: UInt16.Type) throws -> UInt16 { + try decodeFixedWidthInteger() + } + + func decode(_: UInt32.Type) throws -> UInt32 { + try decodeFixedWidthInteger() + } + + func decode(_: UInt64.Type) throws -> UInt64 { + try decodeFixedWidthInteger() + } + + func decode(_ type: T.Type) throws -> T { + try self.unwrap(self.topValue, as: type, for: codingPathNode, _JSONKey?.none) + } + + @inline(__always) private func decodeFixedWidthInteger() throws -> T { + try self.unwrapFixedWidthInteger(from: self.topValue, as: T.self, for: codingPathNode, _JSONKey?.none) + } + + @inline(__always) private func decodeFloatingPoint() throws -> T { + try self.unwrapFloatingPoint(from: self.topValue, as: T.self, for: codingPathNode, _JSONKey?.none) + } +} + +extension JSONDecoderImpl { + struct KeyedContainer: KeyedDecodingContainerProtocol { + typealias Key = K + + let impl: JSONDecoderImpl + let codingPathNode: _JSONCodingPathNode + let dictionary: [String:JSONMap.Value] + + static func stringify(objectRegion: JSONMap.Region, using impl: JSONDecoderImpl, codingPathNode: _JSONCodingPathNode, keyDecodingStrategy: JSONDecoder.KeyDecodingStrategy) throws -> [String:JSONMap.Value] { + var result = [String:JSONMap.Value]() + result.reserveCapacity(objectRegion.count / 2) + + var iter = impl.jsonMap.makeObjectIterator(from: objectRegion.startOffset) + switch keyDecodingStrategy { + case .useDefaultKeys: + while let (keyValue, value) = iter.next() { + // Failing to unwrap a string here is impossible, as scanning already guarantees that dictionary keys are strings. + let key = try! impl.unwrapString(from: keyValue, for: codingPathNode, _JSONKey?.none) + result[key]._setIfNil(to: value) + } + case .convertFromSnakeCase: + while let (keyValue, value) = iter.next() { + // Failing to unwrap a string here is impossible, as scanning already guarantees that dictionary keys are strings. + let key = try! impl.unwrapString(from: keyValue, for: codingPathNode, _JSONKey?.none) + + // Convert the snake case keys in the container to camel case. + // If we hit a duplicate key after conversion, then we'll use the first one we saw. + // Effectively an undefined behavior with JSON dictionaries. + result[JSONDecoder.KeyDecodingStrategy._convertFromSnakeCase(key)]._setIfNil(to: value) + } + case .custom(let converter): + let codingPathForCustomConverter = codingPathNode.path + while let (keyValue, value) = iter.next() { + // Failing to unwrap a string here is impossible, as scanning already guarantees that dictionary keys are strings. + let key = try! impl.unwrapString(from: keyValue, for: codingPathNode, _JSONKey?.none) + + var pathForKey = codingPathForCustomConverter + pathForKey.append(_JSONKey(stringValue: key)!) + result[converter(pathForKey).stringValue]._setIfNil(to: value) + } + } + + return result + } + + init(impl: JSONDecoderImpl, codingPathNode: _JSONCodingPathNode, region: JSONMap.Region) throws { + self.impl = impl + self.codingPathNode = codingPathNode + self.dictionary = try Self.stringify(objectRegion: region, using: impl, codingPathNode: codingPathNode, keyDecodingStrategy: impl.options.keyDecodingStrategy) + } + + public var codingPath : [CodingKey] { + codingPathNode.path + } + + var allKeys: [K] { + self.dictionary.keys.compactMap { K(stringValue: $0) } + } + + func contains(_ key: K) -> Bool { + dictionary.keys.contains(key.stringValue) + } + + func decodeNil(forKey key: K) throws -> Bool { + guard case .null = try getValue(forKey: key) else { + return false + } + return true + } + + func decode(_ type: Bool.Type, forKey key: K) throws -> Bool { + let value = try getValue(forKey: key) + + guard case .bool(let bool) = value else { + throw createTypeMismatchError(type: type, forKey: key, value: value) + } + + return bool + } + + func decode(_ type: String.Type, forKey key: K) throws -> String { + let value = try getValue(forKey: key) + return try impl.unwrapString(from: value, for: self.codingPathNode, key) + } + + func decode(_: Double.Type, forKey key: K) throws -> Double { + try decodeFloatingPoint(key: key) + } + + func decode(_: Float.Type, forKey key: K) throws -> Float { + try decodeFloatingPoint(key: key) + } + + func decode(_: Int.Type, forKey key: K) throws -> Int { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: Int8.Type, forKey key: K) throws -> Int8 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: Int16.Type, forKey key: K) throws -> Int16 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: Int32.Type, forKey key: K) throws -> Int32 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: Int64.Type, forKey key: K) throws -> Int64 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: UInt.Type, forKey key: K) throws -> UInt { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: UInt8.Type, forKey key: K) throws -> UInt8 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: UInt16.Type, forKey key: K) throws -> UInt16 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: UInt32.Type, forKey key: K) throws -> UInt32 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_: UInt64.Type, forKey key: K) throws -> UInt64 { + try decodeFixedWidthInteger(key: key) + } + + func decode(_ type: T.Type, forKey key: K) throws -> T { + try self.impl.unwrap(try getValue(forKey: key), as: type, for: codingPathNode, key) + } + + func nestedContainer(keyedBy type: NestedKey.Type, forKey key: K) throws -> KeyedDecodingContainer { + let value = try getValue(forKey: key) + return try impl.with(value: value, path: codingPathNode.pushing(key)) { + try impl.container(keyedBy: type) + } + } + + func nestedUnkeyedContainer(forKey key: K) throws -> UnkeyedDecodingContainer { + let value = try getValue(forKey: key) + return try impl.with(value: value, path: codingPathNode.pushing(key)) { + try impl.unkeyedContainer() + } + } + + func superDecoder() throws -> Decoder { + return decoderForKeyNoThrow(_JSONKey.super) + } + + func superDecoder(forKey key: K) throws -> Decoder { + return decoderForKeyNoThrow(key) + } + + private func decoderForKeyNoThrow(_ key: some CodingKey) -> JSONDecoderImpl { + let value: JSONMap.Value + do { + value = try getValue(forKey: key) + } catch { + // if there no value for this key then return a null value + value = .null + } + let impl = JSONDecoderImpl(userInfo: self.impl.userInfo, from: self.impl.jsonMap, codingPathNode: self.codingPathNode.pushing(key), options: self.impl.options) + impl.push(value: value) + return impl + } + + @inline(__always) private func getValue(forKey key: some CodingKey) throws -> JSONMap.Value { + guard let value = dictionary[key.stringValue] else { + throw DecodingError.keyNotFound(key, .init( + codingPath: self.codingPath, + debugDescription: "No value associated with key \(key) (\"\(key.stringValue)\")." + )) + } + return value + } + + private func createTypeMismatchError(type: Any.Type, forKey key: K, value: JSONMap.Value) -> DecodingError { + return DecodingError.typeMismatch(type, .init( + codingPath: self.codingPathNode.path(with: key), debugDescription: "Expected to decode \(type) but found \(value.debugDataTypeDescription) instead." + )) + } + + @inline(__always) private func decodeFixedWidthInteger(key: Self.Key) throws -> T { + let value = try getValue(forKey: key) + return try self.impl.unwrapFixedWidthInteger(from: value, as: T.self, for: codingPathNode, key) + } + + @inline(__always) private func decodeFloatingPoint(key: K) throws -> T { + let value = try getValue(forKey: key) + return try self.impl.unwrapFloatingPoint(from: value, as: T.self, for: codingPathNode, key) + } + } +} + +extension JSONDecoderImpl { + struct UnkeyedContainer: UnkeyedDecodingContainer { + let impl: JSONDecoderImpl + var valueIterator: JSONMap.ArrayIterator + var peekedValue: JSONMap.Value? + let count: Int? + + var isAtEnd: Bool { self.currentIndex >= (self.count!) } + var currentIndex = 0 + + init(impl: JSONDecoderImpl, codingPathNode: _JSONCodingPathNode, region: JSONMap.Region) { + self.impl = impl + self.codingPathNode = codingPathNode + self.valueIterator = impl.jsonMap.makeArrayIterator(from: region.startOffset) + self.count = region.count + } + + let codingPathNode: _JSONCodingPathNode + public var codingPath: [CodingKey] { + codingPathNode.path + } + + @inline(__always) + var currentIndexKey : _JSONKey { + .init(index: currentIndex) + } + + @inline(__always) + var currentCodingPath: [CodingKey] { + codingPathNode.path(with: currentIndexKey) + } + + private mutating func advanceToNextValue() { + currentIndex += 1 + peekedValue = nil + } + + mutating func decodeNil() throws -> Bool { + let value = try self.peekNextValue(ofType: Never.self) + switch value { + case .null: + advanceToNextValue() + return true + default: + // The protocol states: + // If the value is not null, does not increment currentIndex. + return false + } + } + + mutating func decode(_ type: Bool.Type) throws -> Bool { + let value = try self.peekNextValue(ofType: Bool.self) + guard case .bool(let bool) = value else { + throw impl.createTypeMismatchError(type: type, for: self.currentCodingPath, value: value) + } + + advanceToNextValue() + return bool + } + + mutating func decode(_ type: String.Type) throws -> String { + let value = try self.peekNextValue(ofType: String.self) + let string = try impl.unwrapString(from: value, for: codingPathNode, currentIndexKey) + advanceToNextValue() + return string + } + + mutating func decode(_: Double.Type) throws -> Double { + try decodeFloatingPoint() + } + + mutating func decode(_: Float.Type) throws -> Float { + try decodeFloatingPoint() + } + + mutating func decode(_: Int.Type) throws -> Int { + try decodeFixedWidthInteger() + } + + mutating func decode(_: Int8.Type) throws -> Int8 { + try decodeFixedWidthInteger() + } + + mutating func decode(_: Int16.Type) throws -> Int16 { + try decodeFixedWidthInteger() + } + + mutating func decode(_: Int32.Type) throws -> Int32 { + try decodeFixedWidthInteger() + } + + mutating func decode(_: Int64.Type) throws -> Int64 { + try decodeFixedWidthInteger() + } + + mutating func decode(_: UInt.Type) throws -> UInt { + try decodeFixedWidthInteger() + } + + mutating func decode(_: UInt8.Type) throws -> UInt8 { + try decodeFixedWidthInteger() + } + + mutating func decode(_: UInt16.Type) throws -> UInt16 { + try decodeFixedWidthInteger() + } + + mutating func decode(_: UInt32.Type) throws -> UInt32 { + try decodeFixedWidthInteger() + } + + mutating func decode(_: UInt64.Type) throws -> UInt64 { + try decodeFixedWidthInteger() + } + + mutating func decode(_ type: T.Type) throws -> T { + let value = try self.peekNextValue(ofType: type) + let result = try impl.unwrap(value, as: type, for: codingPathNode, currentIndexKey) + + advanceToNextValue() + return result + } + + mutating func nestedContainer(keyedBy type: NestedKey.Type) throws -> KeyedDecodingContainer { + let value = try self.peekNextValue(ofType: KeyedDecodingContainer.self) + let container = try impl.with(value: value, path: codingPathNode.pushing(currentIndexKey)) { + try impl.container(keyedBy: type) + } + + advanceToNextValue() + return container + } + + mutating func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer { + let value = try self.peekNextValue(ofType: UnkeyedDecodingContainer.self) + let container = try impl.with(value: value, path: codingPathNode.pushing(currentIndexKey)) { + try impl.unkeyedContainer() + } + + advanceToNextValue() + return container + } + + mutating func superDecoder() throws -> Decoder { + let decoder = try decoderForNextElement(ofType: Decoder.self) + advanceToNextValue() + return decoder + } + + private mutating func decoderForNextElement(ofType type: T.Type) throws -> JSONDecoderImpl { + let value = try self.peekNextValue(ofType: type) + let impl = JSONDecoderImpl( + userInfo: self.impl.userInfo, + from: self.impl.jsonMap, + codingPathNode: self.codingPathNode.pushing(_JSONKey(index: self.currentIndex)), + options: self.impl.options + ) + impl.push(value: value) + return impl + } + + @inline(__always) + private mutating func peekNextValue(ofType type: T.Type) throws -> JSONMap.Value { + if let value = peekedValue { + return value + } + guard let nextValue = valueIterator.next() else { + var message = "Unkeyed container is at end." + if T.self == UnkeyedContainer.self { + message = "Cannot get nested unkeyed container -- unkeyed container is at end." + } + if T.self == Decoder.self { + message = "Cannot get superDecoder() -- unkeyed container is at end." + } + + var path = self.codingPath + path.append(_JSONKey(index: self.currentIndex)) + + throw DecodingError.valueNotFound( + type, + .init(codingPath: path, + debugDescription: message, + underlyingError: nil)) + } + peekedValue = nextValue + return nextValue + } + + @inline(__always) private mutating func decodeFixedWidthInteger() throws -> T { + let value = try self.peekNextValue(ofType: T.self) + let key = _JSONKey(index: self.currentIndex) + let result = try self.impl.unwrapFixedWidthInteger(from: value, as: T.self, for: codingPathNode, key) + advanceToNextValue() + return result + } + + @inline(__always) private mutating func decodeFloatingPoint() throws -> T { + let value = try self.peekNextValue(ofType: T.self) + let key = _JSONKey(index: self.currentIndex) + let result = try self.impl.unwrapFloatingPoint(from: value, as: T.self, for: codingPathNode, key) + advanceToNextValue() + return result + } + } +} + +// This is a workaround for the lack of a "set value only if absent" function for Dictionary. + extension Optional { + fileprivate mutating func _setIfNil(to value: Wrapped) { + guard _fastPath(self == nil) else { return } + self = value + } + } + +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) +extension JSONDecoder : @unchecked Sendable {} diff --git a/Sources/FoundationEssentials/JSON/JSONEncoder.swift b/Sources/FoundationEssentials/JSON/JSONEncoder.swift new file mode 100644 index 000000000..b5e13dfbe --- /dev/null +++ b/Sources/FoundationEssentials/JSON/JSONEncoder.swift @@ -0,0 +1,1428 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// A marker protocol used to determine whether a value is a `String`-keyed `Dictionary` +/// containing `Encodable` values (in which case it should be exempt from key conversion strategies). +/// +/// NOTE: The architecture and environment check is due to a bug in the current (2018-08-08) Swift 4.2 +/// runtime when running on i386 simulator. The issue is tracked in https://bugs.swift.org/browse/SR-8276 +/// Making the protocol `internal` instead of `private` works around this issue. +/// Once SR-8276 is fixed, this check can be removed and the protocol always be made private. +#if arch(i386) || arch(arm) +internal protocol _JSONStringDictionaryEncodableMarker { } +#else +private protocol _JSONStringDictionaryEncodableMarker { } +#endif + +extension Dictionary : _JSONStringDictionaryEncodableMarker where Key == String, Value: Encodable { } + +//===----------------------------------------------------------------------===// +// JSON Encoder +//===----------------------------------------------------------------------===// + +/// `JSONEncoder` facilitates the encoding of `Encodable` values into JSON. +// NOTE: older overlays had Foundation.JSONEncoder as the ObjC name. +// The two must coexist, so it was renamed. The old name must not be +// used in the new runtime. _TtC10Foundation13__JSONEncoder is the +// mangled name for Foundation.__JSONEncoder. +@_objcRuntimeName(_TtC10Foundation13__JSONEncoder) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +open class JSONEncoder { + // MARK: Options + + /// The formatting of the output JSON data. + public struct OutputFormatting : OptionSet, Sendable { + /// The format's default value. + public let rawValue: UInt + + /// Creates an OutputFormatting value with the given raw value. + public init(rawValue: UInt) { + self.rawValue = rawValue + } + + /// Produce human-readable JSON with indented output. + public static let prettyPrinted = OutputFormatting(rawValue: 1 << 0) + +#if FOUNDATION_FRAMEWORK + // TODO: Reenable once String.compare is implemented + + /// Produce JSON with dictionary keys sorted in lexicographic order. + @available(macOS 10.13, iOS 11.0, watchOS 4.0, tvOS 11.0, *) + public static let sortedKeys = OutputFormatting(rawValue: 1 << 1) +#endif // FOUNDATION_FRAMEWORK + + /// By default slashes get escaped ("/" → "\/", "http://apple.com/" → "http:\/\/apple.com\/") + /// for security reasons, allowing outputted JSON to be safely embedded within HTML/XML. + /// In contexts where this escaping is unnecessary, the JSON is known to not be embedded, + /// or is intended only for display, this option avoids this escaping. + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public static let withoutEscapingSlashes = OutputFormatting(rawValue: 1 << 3) + } + + /// The strategy to use for encoding `Date` values. + public enum DateEncodingStrategy : Sendable { + /// Defer to `Date` for choosing an encoding. This is the default strategy. + case deferredToDate + + /// Encode the `Date` as a UNIX timestamp (as a JSON number). + case secondsSince1970 + + /// Encode the `Date` as UNIX millisecond timestamp (as a JSON number). + case millisecondsSince1970 + +#if FOUNDATION_FRAMEWORK // TODO: Reenable once DateFormatStyle has been ported + /// Encode the `Date` as an ISO-8601-formatted string (in RFC 3339 format). + @available(macOS 10.12, iOS 10.0, watchOS 3.0, tvOS 10.0, *) + case iso8601 + + /// Encode the `Date` as a string formatted by the given formatter. + case formatted(DateFormatter) +#endif // FOUNDATION_FRAMEWORK + + /// Encode the `Date` as a custom value encoded by the given closure. + /// + /// If the closure fails to encode a value into the given encoder, the encoder will encode an empty automatic container in its place. + @preconcurrency + case custom(@Sendable (Date, Encoder) throws -> Void) + } + + /// The strategy to use for encoding `Data` values. + public enum DataEncodingStrategy : Sendable { + /// Defer to `Data` for choosing an encoding. + case deferredToData + + /// Encoded the `Data` as a Base64-encoded string. This is the default strategy. + case base64 + + /// Encode the `Data` as a custom value encoded by the given closure. + /// + /// If the closure fails to encode a value into the given encoder, the encoder will encode an empty automatic container in its place. + @preconcurrency + case custom(@Sendable (Data, Encoder) throws -> Void) + } + + /// The strategy to use for non-JSON-conforming floating-point values (IEEE 754 infinity and NaN). + public enum NonConformingFloatEncodingStrategy : Sendable { + /// Throw upon encountering non-conforming values. This is the default strategy. + case `throw` + + /// Encode the values using the given representation strings. + case convertToString(positiveInfinity: String, negativeInfinity: String, nan: String) + } + + /// The strategy to use for automatically changing the value of keys before encoding. + public enum KeyEncodingStrategy : Sendable { + /// Use the keys specified by each type. This is the default strategy. + case useDefaultKeys + +#if FOUNDATION_FRAMEWORK + // TODO: Reenable this option once String.capitalize() is moved + + /// Convert from "camelCaseKeys" to "snake_case_keys" before writing a key to JSON payload. + /// + /// Capital characters are determined by testing membership in `CharacterSet.uppercaseLetters` and `CharacterSet.lowercaseLetters` (Unicode General Categories Lu and Lt). + /// The conversion to lower case uses `Locale.system`, also known as the ICU "root" locale. This means the result is consistent regardless of the current user's locale and language preferences. + /// + /// Converting from camel case to snake case: + /// 1. Splits words at the boundary of lower-case to upper-case + /// 2. Inserts `_` between words + /// 3. Lowercases the entire string + /// 4. Preserves starting and ending `_`. + /// + /// For example, `oneTwoThree` becomes `one_two_three`. `_oneTwoThree_` becomes `_one_two_three_`. + /// + /// - Note: Using a key encoding strategy has a nominal performance cost, as each string key has to be converted. + case convertToSnakeCase +#endif + + /// Provide a custom conversion to the key in the encoded JSON from the keys specified by the encoded types. + /// The full path to the current encoding position is provided for context (in case you need to locate this key within the payload). The returned key is used in place of the last component in the coding path before encoding. + /// If the result of the conversion is a duplicate key, then only one value will be present in the result. + @preconcurrency + case custom(@Sendable (_ codingPath: [CodingKey]) -> CodingKey) + + fileprivate static func _convertToSnakeCase(_ stringKey: String) -> String { + guard !stringKey.isEmpty else { return stringKey } + + var words : [Range] = [] + // The general idea of this algorithm is to split words on transition from lower to upper case, then on transition of >1 upper case characters to lowercase + // + // myProperty -> my_property + // myURLProperty -> my_url_property + // + // We assume, per Swift naming conventions, that the first character of the key is lowercase. + var wordStart = stringKey.startIndex + var searchRange = stringKey.index(after: wordStart)..1 capital letters. Turn those into a word, stopping at the capital before the lower case character. + let beforeLowerIndex = stringKey.index(before: lowerCaseRange.lowerBound) + words.append(upperCaseRange.lowerBound..() + + // MARK: - Constructing a JSON Encoder + + /// Initializes `self` with default strategies. + public init() {} + + + // MARK: - Encoding Values + + /// Encodes the given top-level value and returns its JSON representation. + /// + /// - parameter value: The value to encode. + /// - returns: A new `Data` value containing the encoded JSON data. + /// - throws: `EncodingError.invalidValue` if a non-conforming floating-point value is encountered during encoding, and the encoding strategy is `.throw`. + /// - throws: An error if any value throws an error during encoding. + open func encode(_ value: T) throws -> Data { + let encoder = __JSONEncoder(options: self.options, initialDepth: 0) + + guard let topLevel = try encoder.wrapGeneric(value, for: .root) else { + throw EncodingError.invalidValue(value, + EncodingError.Context(codingPath: [], debugDescription: "Top-level \(T.self) did not encode any values.")) + } + + let writingOptions = JSONWriter.WritingOptions(rawValue: self.outputFormatting.rawValue).union(.fragmentsAllowed) + do { + var writer = JSONWriter(options: writingOptions) + try writer.serializeJSON(topLevel) + return writer.data + } catch let error as JSONError { + #if FOUNDATION_FRAMEWORK + let underlyingError: Error? = error.nsError + #else + let underlyingError: Error? = nil + #endif // FOUNDATION_FRAMEWORK + throw EncodingError.invalidValue(value, + EncodingError.Context(codingPath: [], debugDescription: "Unable to encode the given top-level value to JSON.", underlyingError: underlyingError)) + } + } +} + +// MARK: - __JSONEncoder + +// NOTE: older overlays called this class _JSONEncoder. +// The two must coexist without a conflicting ObjC class name, so it +// was renamed. The old name must not be used in the new runtime. +private class __JSONEncoder : Encoder { + // MARK: Properties + + /// The encoder's storage. + var storage: _JSONEncodingStorage + + /// Options set on the top-level encoder. + let options: JSONEncoder._Options + + var encoderCodingPathNode: _JSONCodingPathNode + var codingPathDepth: Int + + /// Contextual user-provided information for use during encoding. + public var userInfo: [CodingUserInfoKey : Any] { + return self.options.userInfo + } + + /// The path to the current point in encoding. + public var codingPath: [CodingKey] { + encoderCodingPathNode.path + } + + // MARK: - Initialization + + /// Initializes `self` with the given top-level encoder options. + init(options: JSONEncoder._Options, codingPathNode: _JSONCodingPathNode = .root, initialDepth: Int) { + self.options = options + self.storage = _JSONEncodingStorage() + self.encoderCodingPathNode = codingPathNode + self.codingPathDepth = initialDepth + } + + /// Returns whether a new element can be encoded at this coding path. + /// + /// `true` if an element has not yet been encoded at this coding path; `false` otherwise. + var canEncodeNewValue: Bool { + // Every time a new value gets encoded, the key it's encoded for is pushed onto the coding path (even if it's a nil key from an unkeyed container). + // At the same time, every time a container is requested, a new value gets pushed onto the storage stack. + // If there are more values on the storage stack than on the coding path, it means the value is requesting more than one container, which violates the precondition. + // + // This means that anytime something that can request a new container goes onto the stack, we MUST push a key onto the coding path. + // Things which will not request containers do not need to have the coding path extended for them (but it doesn't matter if it is, because they will not reach here). + return self.storage.count == self.codingPathDepth + } + + // MARK: - Encoder Methods + public func container(keyedBy: Key.Type) -> KeyedEncodingContainer { + // If an existing keyed container was already requested, return that one. + let topWritable: _JSONEncodingStorage.Writable + if self.canEncodeNewValue { + // We haven't yet pushed a container at this level; do so here. + topWritable = self.storage.pushKeyedContainer() + } else { + guard let writable = self.storage.writables.last, writable.isObject else { + preconditionFailure("Attempt to push new keyed encoding container when already previously encoded at this path.") + } + topWritable = writable + } + + let container = _JSONKeyedEncodingContainer(referencing: self, codingPathNode: self.encoderCodingPathNode, wrapping: topWritable) + return KeyedEncodingContainer(container) + } + + public func unkeyedContainer() -> UnkeyedEncodingContainer { + // If an existing unkeyed container was already requested, return that one. + let topWritable: _JSONEncodingStorage.Writable + if self.canEncodeNewValue { + // We haven't yet pushed a container at this level; do so here. + topWritable = self.storage.pushUnkeyedContainer() + } else { + guard let writable = self.storage.writables.last, writable.isArray else { + preconditionFailure("Attempt to push new unkeyed encoding container when already previously encoded at this path.") + } + topWritable = writable + } + + return _JSONUnkeyedEncodingContainer(referencing: self, codingPathNode: self.encoderCodingPathNode, wrapping: topWritable) + } + + public func singleValueContainer() -> SingleValueEncodingContainer { + return self + } + + // Instead of creating a new __JSONEncoder for passing to methods that take Encoder arguments, wrap the access in this method, which temporarily mutates this __JSONEncoder instance with the additional nesting depth and its coding path. + @inline(__always) + func with(path: _JSONCodingPathNode?, perform closure: () throws -> T) rethrows -> T { + let oldPath = self.encoderCodingPathNode + let oldDepth = self.codingPathDepth + if let path { + self.encoderCodingPathNode = path + self.codingPathDepth = path.depth + } + + defer { + if path != nil { + self.encoderCodingPathNode = oldPath + self.codingPathDepth = oldDepth + } + } + + return try closure() + } +} + +// MARK: - Encoding Storage and Containers + +private struct _JSONEncodingStorage { + // MARK: Properties + + class Writable { + enum Entry { + case value(JSONValue) + case writableReference(Writable) + + @inline(__always) + var value: JSONValue { + switch self { + case .value(let v): + return v + case .writableReference(let writable): + return writable.value + } + } + } + + enum Backing { + case array([Entry]) + case object([String:Entry]) + case singleValue(JSONValue) + } + var backing: Backing + + @inline(__always) + func encode(_ value: JSONValue, for key: String) { + switch backing { + case .object(var dict): + // Newly encoded values ALWAYS take precedence over any collection references that might have been inserted previously. + dict[key] = .value(value) + self.backing = .object(dict) + default: + preconditionFailure("Wrong underlying JSON writable type") + } + } + + @inline(__always) + func insert(_ writable: Writable, for key: String) { + switch backing { + case .object(var object): + if let _ = object.updateValue(.writableReference(writable), forKey: key) { + preconditionFailure("Previous entry replaced by reference for key \(key)") + } + backing = .object(object) + default: + preconditionFailure("Wrong underlying JSON writable type") + } + } + + @inline(__always) + func encode(_ value: JSONValue) { + switch backing { + case .array(var array): + array.append(.value(value)) + backing = .array(array) + default: + preconditionFailure("Wrong undlying JSON writable type") + } + } + + @inline(__always) + func encode(_ value: JSONValue, insertedAt index: Int) { + switch backing { + case .array(var array): + array.insert(.value(value), at: index) + backing = .array(array) + default: + preconditionFailure("Wrong undlying JSON writable type") + } + } + + @inline(__always) + func insert(_ writable: Writable) { + switch backing { + case .array(var array): + array.append(.writableReference(writable)) + backing = .array(array) + default: + preconditionFailure("Wrong undlying JSON writable type") + } + } + + @inline(__always) + var count: Int { + switch backing { + case .array(let array): return array.count + case .object(let dict): return dict.count + case .singleValue: return 1 + } + } + + @inline(__always) + init(_ backing: Backing) { + self.backing = backing + } + + @inline(__always) + internal var value: JSONValue { + switch backing { + case .object(let dict): + var valueDict = [String:JSONValue]() + for (key, entry) in dict { + valueDict[key] = entry.value + } + return .object(valueDict) + case .array(let array): + return .array(array.map(\.value)) + case .singleValue(let value): + return value + } + } + + // This mutates the backing because we might need to turn an object or array value back into a Writable reference. + @inline(__always) + func getWritable(for key: String) -> Writable? { + switch backing { + case .object(var backingDict): + switch backingDict[key] { + case .writableReference(let writable): + return writable + case .value(let value): + switch value { + case .array(let arrayValue): + let newWritable = Writable(.array(arrayValue.map { Entry.value($0) })) + backingDict[key] = .writableReference(newWritable) + backing = .object(backingDict) + return newWritable + case .object(let dictValue): + var newDict = [String:Entry](minimumCapacity: dictValue.count) + for (key, value) in dictValue { + newDict[key] = Entry.value(value) + } + let newWritable = Writable(.object(newDict)) + backingDict[key] = .writableReference(newWritable) + backing = .object(backingDict) + return newWritable + default: return nil + } + case .none: + return nil + } + default: + preconditionFailure("Wrong undlying JSON writable type") + } + } + + @inline(__always) + subscript (_ index: Int) -> Writable? { + switch backing { + case .array(let array): + guard case let .writableReference(writable) = array[index] else { + return nil + } + return writable + default: + preconditionFailure("Wrong undlying JSON writable type") + } + } + + @inline(__always) + var isObject: Bool { + guard case .object = backing else { + return false + } + return true + } + + @inline(__always) + var isArray: Bool { + guard case .array = backing else { + return false + } + return true + } + } + + var writables = [Writable]() + + // MARK: - Initialization + + /// Initializes `self` with no containers. + init() {} + + // MARK: - Modifying the Stack + + var count: Int { + return self.writables.count + } + + mutating func pushKeyedContainer() -> Writable { + let object = Writable(.object([:])) + self.writables.append(object) + return object + } + + mutating func pushUnkeyedContainer() -> Writable { + let object = Writable(.array([])) + self.writables.append(object) + return object + } + + mutating func push(writable: __owned Writable) { + self.writables.append(writable) + } + + mutating func popWritable() -> Writable { + precondition(!self.writables.isEmpty, "Empty writable stack.") + return self.writables.popLast().unsafelyUnwrapped + } +} + +// MARK: - Encoding Containers + +private struct _JSONKeyedEncodingContainer : KeyedEncodingContainerProtocol { + typealias Key = K + + // MARK: Properties + + /// A reference to the encoder we're writing to. + private let encoder: __JSONEncoder + + private let writable: _JSONEncodingStorage.Writable + private let codingPathNode: _JSONCodingPathNode + + /// The path of coding keys taken to get to this point in encoding. + public var codingPath: [CodingKey] { + codingPathNode.path + } + + // MARK: - Initialization + + /// Initializes `self` with the given references. + init(referencing encoder: __JSONEncoder, codingPathNode: _JSONCodingPathNode, wrapping writable: _JSONEncodingStorage.Writable) { + self.encoder = encoder + self.codingPathNode = codingPathNode + self.writable = writable + } + + // MARK: - Coding Path Operations + + private func _converted(_ key: CodingKey) -> String { + switch encoder.options.keyEncodingStrategy { + case .useDefaultKeys: + return key.stringValue +#if FOUNDATION_FRAMEWORK + case .convertToSnakeCase: + let newKeyString = JSONEncoder.KeyEncodingStrategy._convertToSnakeCase(key.stringValue) + return newKeyString +#endif // FOUNDATION_FRAMEWORK + case .custom(let converter): + return converter(codingPathNode.path(with: key)).stringValue + } + } + + // MARK: - KeyedEncodingContainerProtocol Methods + + public mutating func encodeNil(forKey key: Key) throws { + writable.encode(.null, for: _converted(key)) + } + public mutating func encode(_ value: Bool, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: Int, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: Int8, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: Int16, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: Int32, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: Int64, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: UInt, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: UInt8, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: UInt16, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: UInt32, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: UInt64, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + public mutating func encode(_ value: String, forKey key: Key) throws { + writable.encode(self.encoder.wrap(value), for: _converted(key)) + } + + public mutating func encode(_ value: Float, forKey key: Key) throws { + let wrapped = try self.encoder.wrap(value, for: self.encoder.encoderCodingPathNode, key) + writable.encode(wrapped, for: _converted(key)) + } + + public mutating func encode(_ value: Double, forKey key: Key) throws { + let wrapped = try self.encoder.wrap(value, for: self.encoder.encoderCodingPathNode, key) + writable.encode(wrapped, for: _converted(key)) + } + + public mutating func encode(_ value: T, forKey key: Key) throws { + let wrapped = try self.encoder.wrap(value, for: self.encoder.encoderCodingPathNode, key) + writable.encode(wrapped, for: _converted(key)) + } + + public mutating func nestedContainer(keyedBy keyType: NestedKey.Type, forKey key: Key) -> KeyedEncodingContainer { + let containerKey = _converted(key) + let writable: _JSONEncodingStorage.Writable + if let existingWritable = self.writable.getWritable(for: containerKey) { + precondition( + existingWritable.isObject, + "Attempt to re-encode into nested KeyedEncodingContainer<\(Key.self)> for key \"\(containerKey)\" is invalid: non-keyed container already encoded for this key" + ) + writable = existingWritable + } else { + writable = _JSONEncodingStorage.Writable(.object([:])) + self.writable.insert(writable, for: containerKey) + } + + let container = _JSONKeyedEncodingContainer(referencing: self.encoder, codingPathNode: self.codingPathNode.pushing(key), wrapping: writable) + return KeyedEncodingContainer(container) + } + + public mutating func nestedUnkeyedContainer(forKey key: Key) -> UnkeyedEncodingContainer { + let containerKey = _converted(key) + let writable: _JSONEncodingStorage.Writable + if let existingWritable = self.writable.getWritable(for: containerKey) { + precondition( + existingWritable.isArray, + "Attempt to re-encode into nested UnkeyedEncodingContainer for key \"\(containerKey)\" is invalid: keyed container/single value already encoded for this key" + ) + writable = existingWritable + } else { + writable = _JSONEncodingStorage.Writable(.array([])) + self.writable.insert(writable, for: containerKey) + } + + return _JSONUnkeyedEncodingContainer(referencing: self.encoder, codingPathNode: self.codingPathNode.pushing(key), wrapping: writable) + } + + public mutating func superEncoder() -> Encoder { + return __JSONReferencingEncoder(referencing: self.encoder, key: _JSONKey.super, convertedKey: _converted(_JSONKey.super), codingPathNode: self.encoder.encoderCodingPathNode, wrapping: self.writable) + } + + public mutating func superEncoder(forKey key: Key) -> Encoder { + return __JSONReferencingEncoder(referencing: self.encoder, key: key, convertedKey: _converted(key), codingPathNode: self.encoder.encoderCodingPathNode, wrapping: self.writable) + } +} + +private struct _JSONUnkeyedEncodingContainer : UnkeyedEncodingContainer { + // MARK: Properties + + /// A reference to the encoder we're writing to. + private let encoder: __JSONEncoder + + private let writable: _JSONEncodingStorage.Writable + private let codingPathNode: _JSONCodingPathNode + + /// The path of coding keys taken to get to this point in encoding. + public var codingPath: [CodingKey] { + codingPathNode.path + } + + /// The number of elements encoded into the container. + public var count: Int { + self.writable.count + } + + // MARK: - Initialization + + /// Initializes `self` with the given references. + init(referencing encoder: __JSONEncoder, codingPathNode: _JSONCodingPathNode, wrapping writable: _JSONEncodingStorage.Writable) { + self.encoder = encoder + self.codingPathNode = codingPathNode + self.writable = writable + } + + // MARK: - UnkeyedEncodingContainer Methods + + public mutating func encodeNil() throws { self.writable.encode(.null) } + public mutating func encode(_ value: Bool) throws { self.writable.encode(.bool(value)) } + public mutating func encode(_ value: Int) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: Int8) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: Int16) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: Int32) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: Int64) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: UInt) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: UInt8) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: UInt16) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: UInt32) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: UInt64) throws { self.writable.encode(self.encoder.wrap(value)) } + public mutating func encode(_ value: String) throws { self.writable.encode(self.encoder.wrap(value)) } + + public mutating func encode(_ value: Float) throws { + self.writable.encode(try JSONValue.number(from: value, with: encoder.options.nonConformingFloatEncodingStrategy, for: self.encoder.encoderCodingPathNode, _JSONKey(index: self.count))) + } + + public mutating func encode(_ value: Double) throws { + self.writable.encode(try JSONValue.number(from: value, with: encoder.options.nonConformingFloatEncodingStrategy, for: self.encoder.encoderCodingPathNode, _JSONKey(index: self.count))) + } + + public mutating func encode(_ value: T) throws { + let wrapped = try self.encoder.wrap(value, for: self.encoder.encoderCodingPathNode, _JSONKey(index: self.count)) + self.writable.encode(wrapped) + } + + public mutating func nestedContainer(keyedBy keyType: NestedKey.Type) -> KeyedEncodingContainer { + let key = _JSONKey(index: self.count) + let writable = _JSONEncodingStorage.Writable(.object([:])) + self.writable.insert(writable) + let container = _JSONKeyedEncodingContainer(referencing: self.encoder, codingPathNode: self.codingPathNode.pushing(key), wrapping: writable) + return KeyedEncodingContainer(container) + } + + public mutating func nestedUnkeyedContainer() -> UnkeyedEncodingContainer { + let key = _JSONKey(index: self.count) + let writable = _JSONEncodingStorage.Writable(.array([])) + self.writable.insert(writable) + return _JSONUnkeyedEncodingContainer(referencing: self.encoder, codingPathNode: self.codingPathNode.pushing(key), wrapping: writable) + } + + public mutating func superEncoder() -> Encoder { + return __JSONReferencingEncoder(referencing: self.encoder, at: self.writable.count, codingPathNode: self.encoder.encoderCodingPathNode, wrapping: self.writable) + } +} + +extension __JSONEncoder : SingleValueEncodingContainer { + // MARK: - SingleValueEncodingContainer Methods + + private func assertCanEncodeNewValue() { + precondition(self.canEncodeNewValue, "Attempt to encode value through single value container when previously value already encoded.") + } + + public func encodeNil() throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(.null))) + } + + public func encode(_ value: Bool) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(.bool(value)))) + } + + public func encode(_ value: Int) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: Int8) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: Int16) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: Int32) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: Int64) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: UInt) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: UInt8) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: UInt16) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: UInt32) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: UInt64) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: String) throws { + assertCanEncodeNewValue() + self.storage.push(writable: .init(.singleValue(wrap(value)))) + } + + public func encode(_ value: Float) throws { + assertCanEncodeNewValue() + let wrapped = try self.wrap(value, for: self.encoderCodingPathNode) + self.storage.push(writable: .init(.singleValue(wrapped))) + } + + public func encode(_ value: Double) throws { + assertCanEncodeNewValue() + let wrapped = try self.wrap(value, for: self.encoderCodingPathNode) + self.storage.push(writable: .init(.singleValue(wrapped))) + } + + public func encode(_ value: T) throws { + assertCanEncodeNewValue() + try self.storage.push(writable: .init(.singleValue(self.wrap(value, for: self.encoderCodingPathNode)))) + } +} + +// MARK: - Concrete Value Representations + +private extension __JSONEncoder { + /// Returns the given value boxed in a container appropriate for pushing onto the container stack. + @inline(__always) func wrap(_ value: Bool) -> JSONValue { JSONValue.bool(value) } + @inline(__always) func wrap(_ value: Int) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: Int8) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: Int16) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: Int32) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: Int64) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: UInt) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: UInt8) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: UInt16) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: UInt32) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: UInt64) -> JSONValue { JSONValue.number(from: value) } + @inline(__always) func wrap(_ value: String) -> JSONValue { .string(value) } + + @inline(__always) + func wrap(_ float: Float, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = _JSONKey?.none) throws -> JSONValue { + try JSONValue.number(from: float, with: self.options.nonConformingFloatEncodingStrategy, for: codingPathNode, additionalKey) + } + + @inline(__always) + func wrap(_ double: Double, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = _JSONKey?.none) throws -> JSONValue { + try JSONValue.number(from: double, with: self.options.nonConformingFloatEncodingStrategy, for: codingPathNode, additionalKey) + } + + func wrap(_ date: Date, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = _JSONKey?.none) throws -> JSONValue { + switch self.options.dateEncodingStrategy { + case .deferredToDate: + // Dates encode as single-value objects; this can't both throw and push a container, so no need to catch the error. + try self.with(path: codingPathNode.pushing(additionalKey)) { + try date.encode(to: self) + } + return self.storage.popWritable().value + + case .secondsSince1970: + return try JSONValue.number(from: date.timeIntervalSince1970, with: .throw, for: codingPathNode, additionalKey) + + case .millisecondsSince1970: + return try JSONValue.number(from: 1000.0 * date.timeIntervalSince1970, with: .throw, for: codingPathNode, additionalKey) + +#if FOUNDATION_FRAMEWORK + case .iso8601: + return self.wrap(date.formatted(.iso8601)) + + case .formatted(let formatter): + return self.wrap(formatter.string(from: date)) +#endif + + case .custom(let closure): + let depth = self.storage.count + do { + try self.with(path: codingPathNode.pushing(additionalKey)) { + try closure(date, self) + } + } catch { + // If the value pushed a container before throwing, pop it back off to restore state. + if self.storage.count > depth { + let _ = self.storage.popWritable() + } + + throw error + } + + guard self.storage.count > depth else { + // The closure didn't encode anything. Return the default keyed container. + return .object([:]) + } + + // We can pop because the closure encoded something. + return self.storage.popWritable().value + } + } + + func wrap(_ data: Data, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = _JSONKey?.none) throws -> JSONValue { + switch self.options.dataEncodingStrategy { + case .deferredToData: + let depth = self.storage.count + do { + try self.with(path: codingPathNode.pushing(additionalKey)) { + try data.encode(to: self) + } + } catch { + // If the value pushed a container before throwing, pop it back off to restore state. + // This shouldn't be possible for Data (which encodes as an array of bytes), but it can't hurt to catch a failure. + if self.storage.count > depth { + let _ = self.storage.popWritable() + } + + throw error + } + + return self.storage.popWritable().value + + case .base64: + return self.wrap(data.base64EncodedString()) + + case .custom(let closure): + let depth = self.storage.count + do { + try self.with(path: codingPathNode.pushing(additionalKey)) { + try closure(data, self) + } + } catch { + // If the value pushed a container before throwing, pop it back off to restore state. + if self.storage.count > depth { + let _ = self.storage.popWritable() + } + + throw error + } + + guard self.storage.count > depth else { + // The closure didn't encode anything. Return the default keyed container. + return .object([:]) + } + + // We can pop because the closure encoded something. + return self.storage.popWritable().value + } + } + + func wrap(_ dict: [String : Encodable], for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = _JSONKey?.none) throws -> JSONValue? { + let depth = self.storage.count + let result = self.storage.pushKeyedContainer() + let rootPath = codingPathNode.pushing(additionalKey) + do { + for (key, value) in dict { + result.encode(try wrap(value, for: rootPath, _JSONKey(stringValue: key)), for: key) + } + } catch { + // If the value pushed a container before throwing, pop it back off to restore state. + if self.storage.count > depth { + let _ = self.storage.popWritable() + } + + throw error + } + + // The top container should be a new container. + guard self.storage.count > depth else { + return nil + } + + return self.storage.popWritable().value + } + + func wrap(_ value: Encodable, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = _JSONKey?.none) throws -> JSONValue { + return try self.wrapGeneric(value, for: codingPathNode, additionalKey) ?? JSONValue.object([:]) + } + + func wrapGeneric(_ value: Encodable, for node: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = _JSONKey?.none) throws -> JSONValue? { + switch value { + case let date as Date: + // Respect Date encoding strategy + return try self.wrap(date, for: node, additionalKey) + case let data as Data: + // Respect Data encoding strategy + return try self.wrap(data, for: node, additionalKey) +#if FOUNDATION_FRAMEWORK // TODO: Reenable once URL and Decimal are moved + case let url as URL: + // Encode URLs as single strings. + return self.wrap(url.absoluteString) + case let decimal as Decimal: + return JSONValue.number(decimal.description) +#endif // FOUNDATION_FRAMEWORK + case let dict as _JSONStringDictionaryEncodableMarker: + return try self.wrap(dict as! [String : Encodable], for: node, additionalKey) + default: + break + } + + // The value should request a container from the __JSONEncoder. + let depth = self.storage.count + do { + try self.with(path: node.pushing(additionalKey)) { + try value.encode(to: self) + } + } catch { + // If the value pushed a container before throwing, pop it back off to restore state. + if self.storage.count > depth { + let _ = self.storage.popWritable() + } + + throw error + } + + // The top container should be a new container. + guard self.storage.count > depth else { + return nil + } + + return self.storage.popWritable().value + } +} + +// MARK: - __JSONReferencingEncoder + +/// __JSONReferencingEncoder is a special subclass of __JSONEncoder which has its own storage, but references the contents of a different encoder. +/// It's used in superEncoder(), which returns a new encoder for encoding a superclass -- the lifetime of the encoder should not escape the scope it's created in, but it doesn't necessarily know when it's done being used (to write to the original container). +// NOTE: older overlays called this class _JSONReferencingEncoder. +// The two must coexist without a conflicting ObjC class name, so it +// was renamed. The old name must not be used in the new runtime. +private class __JSONReferencingEncoder : __JSONEncoder { + // MARK: Reference types. + + /// The type of container we're referencing. + private enum Reference { + /// Referencing a specific index in an array container. + case array(_JSONEncodingStorage.Writable, Int) + + /// Referencing a specific key in a dictionary container. + case dictionary(_JSONEncodingStorage.Writable, String) + } + + // MARK: - Properties + + /// The encoder we're referencing. + let encoder: __JSONEncoder + + /// The container reference itself. + private let reference: Reference + + // MARK: - Initialization + + /// Initializes `self` by referencing the given array container in the given encoder. + init(referencing encoder: __JSONEncoder, at index: Int, codingPathNode: _JSONCodingPathNode, wrapping writable: _JSONEncodingStorage.Writable) { + self.encoder = encoder + self.reference = .array(writable, index) + super.init(options: encoder.options, codingPathNode: codingPathNode.pushing(_JSONKey(index: index)), initialDepth: codingPathNode.depth) + } + + /// Initializes `self` by referencing the given dictionary container in the given encoder. + init(referencing encoder: __JSONEncoder, key: CodingKey, convertedKey: String, codingPathNode: _JSONCodingPathNode, wrapping dictionary: _JSONEncodingStorage.Writable) { + self.encoder = encoder + self.reference = .dictionary(dictionary, convertedKey) + super.init(options: encoder.options, codingPathNode: codingPathNode.pushing(key), initialDepth: codingPathNode.depth) + } + + // MARK: - Coding Path Operations + + override var canEncodeNewValue: Bool { + // With a regular encoder, the storage and coding path grow together. + // A referencing encoder, however, inherits its parents coding path, as well as the key it was created for. + // We have to take this into account. + return self.storage.count == self.codingPath.count - self.encoder.codingPath.count - 1 + } + + // MARK: - Deinitialization + + // Finalizes `self` by writing the contents of our storage to the referenced encoder's storage. + deinit { + let value: JSONValue + switch self.storage.count { + case 0: value = .object([:]) + case 1: value = self.storage.popWritable().value + default: fatalError("Referencing encoder deallocated with multiple containers on stack.") + } + + switch self.reference { + case .array(let arrayRef, let index): + arrayRef.encode(value, insertedAt: index) + case .dictionary(let dictionaryRef, let key): + dictionaryRef.encode(value, for: key) + } + } +} + +// MARK: - Shared Key Types +internal struct _JSONKey: CodingKey { + enum Rep { + case string(String) + case int(Int) + case index(Int) + case both(String, Int) + } + let rep : Rep + + public init?(stringValue: String) { + self.rep = .string(stringValue) + } + + public init?(intValue: Int) { + self.rep = .int(intValue) + } + + internal init(index: Int) { + self.rep = .index(index) + } + + public init(stringValue: String, intValue: Int?) { + if let intValue { + self.rep = .both(stringValue, intValue) + } else { + self.rep = .string(stringValue) + } + } + + var stringValue: String { + switch rep { + case let .string(str): return str + case let .int(int): return "\(int)" + case let .index(index): return "Index \(index)" + case let .both(str, _): return str + } + } + + var intValue: Int? { + switch rep { + case .string: return nil + case let .int(int): return int + case let .index(index): return index + case let .both(_, int): return int + } + } + + internal static let `super` = _JSONKey(stringValue: "super")! +} + +//===----------------------------------------------------------------------===// +// Coding Path Node +//===----------------------------------------------------------------------===// + +// This construction allows overall fewer and smaller allocations as the coding path is modified. +internal enum _JSONCodingPathNode { + case root + indirect case node(CodingKey, _JSONCodingPathNode) + + var path : [CodingKey] { + switch self { + case .root: + return [] + case let .node(key, parent): + return parent.path + [key] + } + } + + mutating func push(_ key: __owned some CodingKey) { + self = .node(key, self) + } + + mutating func pop() { + guard case let .node(_, parent) = self else { + preconditionFailure("Can't pop the root node") + } + self = parent + } + + func pushing(_ key: __owned (some CodingKey)?) -> _JSONCodingPathNode { + if let key { + return .node(key, self) + } else { + return self + } + } + + func path(with key: __owned (some CodingKey)?) -> [CodingKey] { + if let key { + return self.path + [key] + } + return self.path + } + + var depth : Int { + switch self { + case .root: return 0 + case let .node(_, parent): return parent.depth + 1 + } + } +} + +//===----------------------------------------------------------------------===// +// Error Utilities +//===----------------------------------------------------------------------===// + +extension EncodingError { + /// Returns a `.invalidValue` error describing the given invalid floating-point value. + /// + /// + /// - parameter value: The value that was invalid to encode. + /// - parameter path: The path of `CodingKey`s taken to encode this value. + /// - returns: An `EncodingError` with the appropriate path and debug description. + fileprivate static func _invalidFloatingPointValue(_ value: T, at codingPath: [CodingKey]) -> EncodingError { + let valueDescription: String + if value == T.infinity { + valueDescription = "\(T.self).infinity" + } else if value == -T.infinity { + valueDescription = "-\(T.self).infinity" + } else { + valueDescription = "\(T.self).nan" + } + + let debugDescription = "Unable to encode \(valueDescription) directly in JSON. Use JSONEncoder.NonConformingFloatEncodingStrategy.convertToString to specify how the value should be encoded." + return .invalidValue(value, EncodingError.Context(codingPath: codingPath, debugDescription: debugDescription)) + } +} + +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) +extension JSONEncoder : @unchecked Sendable {} diff --git a/Sources/FoundationEssentials/JSON/JSONScanner.swift b/Sources/FoundationEssentials/JSON/JSONScanner.swift new file mode 100644 index 000000000..07cae0765 --- /dev/null +++ b/Sources/FoundationEssentials/JSON/JSONScanner.swift @@ -0,0 +1,1453 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/* + A JSONMap is created by a JSON scanner to describe the values found in a JSON payload, including their size, location, and contents, without copying any of the non-structural data. It is used by the JSONDecoder, which will fully parse only those values that are required to decode the requested Decodable types. + + To minimize the number of allocations required during scanning, the map's contents are implemented using a array of integers, whose values are a serialization of the JSON payload's full structure. Each type has its own unique marker value, which is followed by zero or more other integers that describe that contents of that type, if any. + + Due to the complexity and additional allocations required to parse JSON string values into Swift Strings or JSON number values into the requested integer or floating-point types, their map contents are captured as lengths of bytes and byte offsets into the input. This allows the full parsing to occur at decode time, or to be skipped if the value is not desired. A partial, imperfect parsing is performed by the scanner, simply "skipping" characters which are valid in their given contexts without interpeting or further validating them relative to the other inputs. This incomplete scanning process does however guarnatee that the structure of the JSON input is correctly interpreted. + + The JSONMap representation of JSON arrays and objects is a sequence of integers that is delimited by their starting marker and a shared "collection end" marker. Their contents are nested in between those two markers. To facilitate skipping over unwanted elements of a collection, which is especially useful for JSON objects, the map encodes the offset in the map array to the next object after the end of the collection. + + For instance, a JSON payload such as the following: + + ``` + {"array":[1,2,3],"number":42} + ``` + + will be scanned into a map buffer looking like this: + + ``` + Key: + == Object Marker + == Array Marker + == Simple String (a variant of String that can has no escpaes and can be passed directly to a UTF-8 parser) + == Number Marker + == Collection End + (See JSONMap.TypeDescriptor comments below for more details) + + Map offset: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 13, 14, 15 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 + Map contents: [ , 26, 2, , 5, 2, , 19, 3, , 1, 10, , 1, 12, , 1, 14, , , 6, 18, , 2, 26, ] + Description: | -- key2 -- ------------------------- value1 -------------------------- --- key2 --- -- value2 -- + | | | --arr elm 0- --arr elm 0- --arr elm 0- + | > Byte offset from the beginning of the input to the contents of the string + | > Offset to the next entry after this array, which is key2 + > Offset to next entry after this object, which is the endIndex of the array, as this is the top level value + + A Decodable type that wishes only to decode the "number" key of this object as an Int will be able to entirely skip the decoding of the "array" value by doing the following. + 1. Find the type of the value at index 0 (object), and its size at index 2. + 2. Begin parsing keys at index 3. It decodes the string, and finds "array", which is not a match for "number". + 3. Skip the key's value by finding its type (array), and then its nextSiblingOffset index (19) + 4. Parse the next key at index 4. It decodes the string and finds "number", which is a match. + 5. Decode the value by findings its type (number), its length (2) and the byte offset from the beginning of the input (26). + 6. Pass that byte offset + length into the number parser to produce the correspomding Swift Int value. +*/ + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif // canImport(Darwin) + +internal class JSONMap { + enum TypeDescriptor : Int { + case string // [marker, count, sourceByteOffset] + case number // [marker, count, sourceByteOffset] + case null // [marker] + case `true` // [marker] + case `false` // [marker] + + case object // [marker, nextSiblingOffset, count, , .collectionEnd] + case array // [marker, nextSiblingOffset, count, , .collectionEnd] + case collectionEnd + + case simpleString // [marker, count, sourceByteOffset] + case numberContainingExponent // [marker, count, sourceByteOffset] + + @inline(__always) + var mapMarker: Int { + self.rawValue + } + } + + struct Region { + let startOffset: Int + let count: Int + } + + enum Value { + case string(Region, isSimple: Bool) + case number(Region, containsExponent: Bool) + case bool(Bool) + case null + + case object(Region) + case array(Region) + } + + let mapBuffer : [Int] + var dataLock : LockedState<(buffer: UnsafeBufferPointer, owned: Bool)> + + init(mapBuffer: [Int], dataBuffer: UnsafeBufferPointer) { + self.mapBuffer = mapBuffer + self.dataLock = .init(initialState: (buffer: dataBuffer, owned: false)) + } + + func copyInBuffer() { + dataLock.withLock { state in + guard !state.owned else { + return + } + + // Allocate an additional byte to ensure we have a trailing NUL byte which is important for cases like a floating point number fragment. + let buffer = UnsafeMutableBufferPointer.allocate(capacity: state.buffer.count + 1) + let (_, lastIndex) = buffer.initialize(from: state.buffer) + buffer[lastIndex] = 0 + + state = (buffer: UnsafeBufferPointer(buffer), owned: true) + } + } + + + @inline(__always) + func withBuffer(for region: Region, perform closure: (UnsafeBufferPointer, UnsafePointer) throws -> T) rethrows -> T { + try dataLock.withLock { + let start = $0.buffer.baseAddress.unsafelyUnwrapped + let buffer = UnsafeBufferPointer(start: start.advanced(by: region.startOffset), count: region.count) + return try closure(buffer, start) + } + } + + deinit { + dataLock.withLock { + if $0.owned { + $0.buffer.deallocate() + } + } + } + + func loadValue(at mapOffset: Int) -> Value? { + let marker = mapBuffer[mapOffset] + let type = JSONMap.TypeDescriptor(rawValue: marker) + switch type { + case .string, .simpleString: + let length = mapBuffer[mapOffset + 1] + let dataOffset = mapBuffer[mapOffset + 2] + return .string(.init(startOffset: dataOffset, count: length), isSimple: type == .simpleString) + case .number, .numberContainingExponent: + let length = mapBuffer[mapOffset + 1] + let dataOffset = mapBuffer[mapOffset + 2] + return .number(.init(startOffset: dataOffset, count: length), containsExponent: type == .numberContainingExponent) + case .object: + // Skip the offset to the next sibling value. + let count = mapBuffer[mapOffset + 2] + return .object(.init(startOffset: mapOffset + 3, count: count)) + case .array: + // Skip the offset to the next sibling value. + let count = mapBuffer[mapOffset + 2] + return .array(.init(startOffset: mapOffset + 3, count: count)) + case .null: + return .null + case .true: + return .bool(true) + case .false: + return .bool(false) + case .collectionEnd: + return nil + default: + fatalError("Invalid JSON value type code in mapping: \(marker))") + } + } + + func offset(after previousValueOffset: Int) -> Int { + let marker = mapBuffer[previousValueOffset] + let type = JSONMap.TypeDescriptor(rawValue: marker) + switch type { + case .string, .simpleString, .number, .numberContainingExponent: + return previousValueOffset + 3 // Skip marker, length, and data offset + case .null, .true, .false: + return previousValueOffset + 1 // Skip only the marker. + case .object, .array: + // The collection records the offset to the next sibling. + return mapBuffer[previousValueOffset + 1] + case .collectionEnd: + fatalError("Attempt to find next object past the end of collection at offset \(previousValueOffset))") + default: + fatalError("Invalid JSON value type code in mapping: \(marker))") + } + } + + struct ArrayIterator { + var currentOffset: Int + let map : JSONMap + + mutating func next() -> JSONMap.Value? { + guard let next = peek() else { + return nil + } + advance() + return next + } + + func peek() -> JSONMap.Value? { + guard let next = map.loadValue(at: currentOffset) else { + return nil + } + return next + } + + mutating func advance() { + currentOffset = map.offset(after: currentOffset) + } + } + + func makeArrayIterator(from offset: Int) -> ArrayIterator { + return .init(currentOffset: offset, map: self) + } + + struct ObjectIterator { + var currentOffset: Int + let map : JSONMap + + mutating func next() -> (key: JSONMap.Value, value: JSONMap.Value)? { + let keyOffset = currentOffset + guard let key = map.loadValue(at: currentOffset) else { + return nil + } + let valueOffset = map.offset(after: keyOffset) + guard let value = map.loadValue(at: valueOffset) else { + preconditionFailure("JSONMap object constructed incorrectly. No value found for key") + } + currentOffset = map.offset(after: valueOffset) + return (key, value) + } + } + + func makeObjectIterator(from offset: Int) -> ObjectIterator { + return .init(currentOffset: offset, map: self) + } +} + +extension JSONMap.Value { + var debugDataTypeDescription : String { + switch self { + case .string: return "a string" + case .number: return "number" + case .bool: return "bool" + case .null: return "null" + case .object: return "a dictionary" + case .array: return "an array" + } + } +} + + + +internal struct JSONScanner { + let options: Options + var reader: DocumentReader + var depth: Int = 0 + var partialMap = JSONPartialMapData() + + internal struct Options { + var assumesTopLevelDictionary = false + } + + struct JSONPartialMapData { + var mapData: [Int] = [] + var prevMapDataSize = 0 + + mutating func resizeIfNecessary(with reader: DocumentReader) { + let currentCount = mapData.count + if currentCount > 0, currentCount.isMultiple(of: 2048) { + // Time to predict how big these arrays are going to be based on the current rate of consumption per processed bytes. + // total objects = (total bytes / current bytes) * current objects + let totalBytes = reader.bytes.count + let consumedBytes = reader.byteOffset(at: reader.readPtr) + let ratio = (Double(totalBytes) / Double(consumedBytes)) + let totalExpectedMapSize = Int( Double(mapData.count) * ratio ) + if prevMapDataSize == 0 || Double(totalExpectedMapSize) / Double(prevMapDataSize) > 1.25 { + mapData.reserveCapacity(totalExpectedMapSize) + prevMapDataSize = totalExpectedMapSize + } + + // print("Ratio is \(ratio). Reserving \(totalExpectedObjects) objects and \(totalExpectedMapSize) scratch space") + } + } + + mutating func recordStartCollection(tagType: JSONMap.TypeDescriptor, with reader: DocumentReader) -> Int { + resizeIfNecessary(with: reader) + + mapData.append(tagType.mapMarker) + + // Reserve space for the next object index and object count. + let startIdx = mapData.count + mapData.append(contentsOf: [0, 0]) + return startIdx + } + + mutating func recordEndCollection(count: Int, atStartOffset startOffset: Int, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + mapData.append(JSONMap.TypeDescriptor.collectionEnd.rawValue) + + let nextValueOffset = mapData.count + mapData.withUnsafeMutableBufferPointer { + $0[startOffset] = nextValueOffset + $0[startOffset + 1] = count + } + } + + mutating func recordEmptyCollection(tagType: JSONMap.TypeDescriptor, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + let nextValueOffset = mapData.count + 4 + mapData.append(contentsOf: [tagType.mapMarker, nextValueOffset, 0, JSONMap.TypeDescriptor.collectionEnd.mapMarker]) + } + + mutating func record(tagType: JSONMap.TypeDescriptor, count: Int, dataOffset: Int, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + mapData.append(contentsOf: [tagType.mapMarker, count, dataOffset]) + } + + mutating func record(tagType: JSONMap.TypeDescriptor, with reader: DocumentReader) { + resizeIfNecessary(with: reader) + + mapData.append(tagType.mapMarker) + } + } + + init(bytes: UnsafeBufferPointer, options: Options) { + self.options = options + self.reader = DocumentReader(bytes: bytes) + } + + mutating func scan() throws -> JSONMap { + if options.assumesTopLevelDictionary { + switch try reader.consumeWhitespace(allowingEOF: true) { + case ._openbrace?: + // If we've got the open brace anyway, just do a normal object scan. + try self.scanObject() + default: + try self.scanObject(withoutBraces: true) + } + } else { + try self.scanValue() + } +#if DEBUG + defer { + guard self.depth == 0 else { + preconditionFailure("Expected to end parsing with a depth of 0") + } + } +#endif + + // ensure only white space is remaining + var whitespace = 0 + while let next = reader.peek(offset: whitespace) { + switch next { + case ._space, ._tab, ._return, ._newline: + whitespace += 1 + continue + default: + throw JSONError.unexpectedCharacter(context: "after top-level value", ascii: next, location: reader.sourceLocation(atOffset: whitespace)) + } + } + + let map = JSONMap(mapBuffer: partialMap.mapData, dataBuffer: self.reader.bytes) + + // If the input contains only a number, we have to copy the input into a form that is guaranteed to have a trailing NUL byte so that strtod doesn't perform a buffer overrun. + if case .number = map.loadValue(at: 0)! { + map.copyInBuffer() + } + + return map + } + + // MARK: Generic Value Scanning + + mutating func scanValue() throws { + let byte = try reader.consumeWhitespace() + switch byte { + case ._quote: + try scanString() + case ._openbrace: + try scanObject() + case ._openbracket: + try scanArray() + case UInt8(ascii: "f"), UInt8(ascii: "t"): + try scanBool() + case UInt8(ascii: "n"): + try scanNull() + case UInt8(ascii: "-"), _asciiNumbers: + try scanNumber() + case ._space, ._return, ._newline, ._tab: + preconditionFailure("Expected that all white space is consumed") + default: + throw JSONError.unexpectedCharacter(ascii: byte, location: reader.sourceLocation) + } + } + + + // MARK: - Scan Array - + + mutating func scanArray() throws { + let firstChar = reader.read() + precondition(firstChar == ._openbracket) + guard self.depth < 512 else { + throw JSONError.tooManyNestedArraysOrDictionaries(location: reader.sourceLocation(atOffset: 1)) + } + self.depth &+= 1 + defer { depth &-= 1 } + + // parse first value or end immediatly + switch try reader.consumeWhitespace() { + case ._space, ._return, ._newline, ._tab: + preconditionFailure("Expected that all white space is consumed") + case ._closebracket: + // if the first char after whitespace is a closing bracket, we found an empty array + reader.moveReaderIndex(forwardBy: 1) + return partialMap.recordEmptyCollection(tagType: .array, with: reader) + default: + break + } + + var count = 0 + let startOffset = partialMap.recordStartCollection(tagType: .array, with: reader) + defer { + partialMap.recordEndCollection(count: count, atStartOffset: startOffset, with: reader) + } + + ScanValues: + while true { + try scanValue() + count += 1 + + // consume the whitespace after the value before the comma + let ascii = try reader.consumeWhitespace() + switch ascii { + case ._space, ._return, ._newline, ._tab: + preconditionFailure("Expected that all white space is consumed") + case ._closebracket: + reader.moveReaderIndex(forwardBy: 1) + break ScanValues + case ._comma: + // consume the comma + reader.moveReaderIndex(forwardBy: 1) + // consume the whitespace before the next value + if try reader.consumeWhitespace() == ._closebracket { + // the foundation json implementation does support trailing commas + reader.moveReaderIndex(forwardBy: 1) + break ScanValues + } + continue + default: + throw JSONError.unexpectedCharacter(context: "in array", ascii: ascii, location: reader.sourceLocation) + } + } + } + + // MARK: - Scan Object - + + mutating func scanObject() throws { + let firstChar = self.reader.read() + precondition(firstChar == ._openbrace) + guard self.depth < 512 else { + throw JSONError.tooManyNestedArraysOrDictionaries(location: reader.sourceLocation(atOffset: -1)) + } + try scanObject(withoutBraces: false) + } + + @inline(never) + mutating func _scanObjectLoop(withoutBraces: Bool, count: inout Int, done: inout Bool) throws { + try scanString() + + let colon = try reader.consumeWhitespace() + guard colon == ._colon else { + throw JSONError.unexpectedCharacter(context: "in object", ascii: colon, location: reader.sourceLocation) + } + reader.moveReaderIndex(forwardBy: 1) + + try self.scanValue() + count += 2 + + let commaOrBrace = try reader.consumeWhitespace(allowingEOF: withoutBraces) + switch commaOrBrace { + case ._comma?: + reader.moveReaderIndex(forwardBy: 1) + switch try reader.consumeWhitespace(allowingEOF: withoutBraces) { + case ._closebrace?: + if withoutBraces { + throw JSONError.unexpectedCharacter(ascii: ._closebrace, location: reader.sourceLocation) + } + + // the foundation json implementation does support trailing commas + reader.moveReaderIndex(forwardBy: 1) + done = true + case .none: + done = true + default: + return + } + case ._closebrace?: + if withoutBraces { + throw JSONError.unexpectedCharacter(ascii: ._closebrace, location: reader.sourceLocation) + } + reader.moveReaderIndex(forwardBy: 1) + done = true + case .none: + // If withoutBraces was false, then reaching EOF here would have thrown. + precondition(withoutBraces == true) + done = true + + default: + throw JSONError.unexpectedCharacter(context: "in object", ascii: commaOrBrace.unsafelyUnwrapped, location: reader.sourceLocation) + } + } + + mutating func scanObject(withoutBraces: Bool) throws { + self.depth &+= 1 + defer { depth &-= 1 } + + // parse first value or end immediatly + switch try reader.consumeWhitespace(allowingEOF: withoutBraces) { + case ._closebrace?: + if withoutBraces { + throw JSONError.unexpectedCharacter(ascii: ._closebrace, location: reader.sourceLocation) + } + + // if the first char after whitespace is a closing bracket, we found an empty object + self.reader.moveReaderIndex(forwardBy: 1) + return partialMap.recordEmptyCollection(tagType: .object, with: reader) + case .none: + // If withoutBraces was false, then reaching EOF here would have thrown. + precondition(withoutBraces == true) + return partialMap.recordEmptyCollection(tagType: .object, with: reader) + default: + break + } + + var count = 0 + let startOffset = partialMap.recordStartCollection(tagType: .object, with: reader) + defer { + partialMap.recordEndCollection(count: count, atStartOffset: startOffset, with: reader) + } + + var done = false + while !done { + try _scanObjectLoop(withoutBraces: withoutBraces, count: &count, done: &done) + } + } + + mutating func scanString() throws { + let quoteStart = reader.readPtr + var isSimple = false + try reader.skipUTF8StringTillNextUnescapedQuote(isSimple: &isSimple) + let stringStart = quoteStart + 1 + let end = reader.readPtr + + // skipUTF8StringTillNextUnescapedQuote will have either thrown an error, or already peek'd the quote. + let shouldBePostQuote = reader.read() + precondition(shouldBePostQuote == ._quote) + + // skip initial quote + return partialMap.record(tagType: isSimple ? .simpleString : .string, count: end - stringStart, dataOffset: reader.byteOffset(at: stringStart), with: reader) + } + + mutating func scanNumber() throws { + let start = reader.readPtr + var containsExponent = false + reader.skipNumber(containsExponent: &containsExponent) + let end = reader.readPtr + return partialMap.record(tagType: containsExponent ? .numberContainingExponent : .number, count: end - start, dataOffset: reader.byteOffset(at: start), with: reader) + } + + mutating func scanBool() throws { + if try reader.readBool() { + return partialMap.record(tagType: .true, with: reader) + } else { + return partialMap.record(tagType: .false, with: reader) + } + } + + mutating func scanNull() throws { + try reader.readNull() + return partialMap.record(tagType: .null, with: reader) + } + +} + +extension JSONScanner { + + struct DocumentReader { + let bytes: UnsafeBufferPointer + private(set) var readPtr : UnsafePointer + private let endPtr : UnsafePointer + + @inline(__always) + func checkRemainingBytes(_ count: Int) -> Bool { + return endPtr - readPtr >= count + } + + @inline(__always) + func requireRemainingBytes(_ count: Int) throws { + guard checkRemainingBytes(count) else { + throw JSONError.unexpectedEndOfFile + } + } + + var sourceLocation : JSONError.SourceLocation { + self.sourceLocation(atOffset: 0) + } + + func sourceLocation(atOffset offset: Int) -> JSONError.SourceLocation { + .sourceLocation(at: readPtr + offset, docStart: bytes.baseAddress.unsafelyUnwrapped) + } + + @inline(__always) + var isEOF: Bool { + readPtr == endPtr + } + + @inline(__always) + func byteOffset(at ptr: UnsafePointer) -> Int { + ptr - bytes.baseAddress.unsafelyUnwrapped + } + + init(bytes: UnsafeBufferPointer) { + self.bytes = bytes + self.readPtr = bytes.baseAddress.unsafelyUnwrapped + self.endPtr = self.readPtr + bytes.count + } + + @inline(__always) + mutating func read() -> UInt8? { + guard !isEOF else { + return nil + } + + defer { self.readPtr += 1 } + + return readPtr.pointee + } + + @inline(__always) + func peek(offset: Int = 0) -> UInt8? { + precondition(offset >= 0) + guard checkRemainingBytes(offset + 1) else { + return nil + } + + return (self.readPtr + offset).pointee + } + + @inline(__always) + mutating func moveReaderIndex(forwardBy offset: Int) { + self.readPtr += offset + } + + static var whitespaceBitmap: UInt64 { 1 << UInt8._space | 1 << UInt8._return | 1 << UInt8._newline | 1 << UInt8._tab } + + @inline(__always) + @discardableResult + mutating func consumeWhitespace() throws -> UInt8 { + var ptr = self.readPtr + while ptr < endPtr { + let ascii = ptr.pointee + if Self.whitespaceBitmap & (1 << ascii) != 0 { + ptr += 1 + continue + } else { + self.readPtr = ptr + return ascii + } + } + + throw JSONError.unexpectedEndOfFile + } + + @inline(__always) + @discardableResult + mutating func consumeWhitespace(allowingEOF: Bool) throws -> UInt8? { + var ptr = self.readPtr + while ptr < endPtr { + let ascii = ptr.pointee + if Self.whitespaceBitmap & (1 << ascii) != 0 { + ptr += 1 + continue + } else { + self.readPtr = ptr + return ascii + } + } + guard allowingEOF else { + throw JSONError.unexpectedEndOfFile + } + return nil + } + + @inline(__always) + mutating func readExpectedString(_ str: StaticString, typeDescriptor: String) throws { + try requireRemainingBytes(str.utf8CodeUnitCount) + guard memcmp(readPtr, str.utf8Start, str.utf8CodeUnitCount) == 0 else { + // Figure out the exact character that is wrong. + var badOffset = 0 + for i in 0 ..< str.utf8CodeUnitCount { + if (readPtr + i).pointee != (str.utf8Start + i).pointee { + badOffset = i + break + } + } + throw JSONError.unexpectedCharacter(context: "in expected \(typeDescriptor) value", ascii: self.peek(offset: badOffset).unsafelyUnwrapped, location: sourceLocation(atOffset: badOffset)) + } + + // If all looks good, advance past the string. + self.moveReaderIndex(forwardBy: str.utf8CodeUnitCount) + } + + @inline(__always) + mutating func readBool() throws -> Bool { + switch self.read() { + case UInt8(ascii: "t"): + try readExpectedString("rue", typeDescriptor: "boolean") + return true + case UInt8(ascii: "f"): + try readExpectedString("alse", typeDescriptor: "boolean") + return false + default: + preconditionFailure("Expected to have `t` or `f` as first character") + } + } + + @inline(__always) + mutating func readNull() throws { + try readExpectedString("null", typeDescriptor: "null") + } + + // MARK: - Private Methods - + + // MARK: String + + mutating func skipUTF8StringTillQuoteOrBackslashOrInvalidCharacter() throws -> UInt8 { + while let byte = self.peek() { + switch byte { + case ._quote, ._backslash: + return byte + default: + // Any control characters in the 0-31 range are invalid. Any other characters will have at least one bit in a 0xe0 mask. + guard _fastPath(byte & 0xe0 != 0) else { + return byte + } + self.moveReaderIndex(forwardBy: 1) + } + } + throw JSONError.unexpectedEndOfFile + } + + mutating func skipUTF8StringTillNextUnescapedQuote(isSimple: inout Bool) throws { + // Skip the open quote. + guard let shouldBeQuote = self.read() else { + throw JSONError.unexpectedEndOfFile + } + guard shouldBeQuote == ._quote else { + throw JSONError.unexpectedCharacter(ascii: shouldBeQuote, location: sourceLocation) + } + + // If there aren't any escapes, then this is a simple case and we can exit early. + if try skipUTF8StringTillQuoteOrBackslashOrInvalidCharacter() == ._quote { + isSimple = true + return + } + + while let byte = self.peek() { + // Checking for invalid control characters deferred until parse time. + switch byte { + case ._quote: + isSimple = false + return + case ._backslash: + try skipEscapeSequence() + default: + moveReaderIndex(forwardBy: 1) + continue + } + } + throw JSONError.unexpectedEndOfFile + } + + private mutating func skipEscapeSequence() throws { + let firstChar = self.read() + precondition(firstChar == ._backslash, "Expected to have an backslash first") + + guard let ascii = self.read() else { + throw JSONError.unexpectedEndOfFile + } + + // Invalid escaped characters checking deferred to parse time. + if ascii == UInt8(ascii: "u") { + try skipUnicodeHexSequence() + } + } + + private mutating func skipUnicodeHexSequence() throws { + // As stated in RFC-8259 an escaped unicode character is 4 HEXDIGITs long + // https://tools.ietf.org/html/rfc8259#section-7 + try requireRemainingBytes(4) + + // We'll validate the actual characters following the '\u' escape during parsing. Just make sure that the string doesn't end prematurely. + guard readPtr.pointee != ._quote, + (readPtr+1).pointee != ._quote, + (readPtr+2).pointee != ._quote, + (readPtr+3).pointee != ._quote + else { + let hexString = String(decoding: UnsafeBufferPointer(start: readPtr, count: 4), as: UTF8.self) + throw JSONError.invalidHexDigitSequence(hexString, location: sourceLocation) + } + self.moveReaderIndex(forwardBy: 4) + } + + // MARK: Numbers + + mutating func skipNumber(containsExponent: inout Bool) { + guard let ascii = read() else { + preconditionFailure("Why was this function called, if there is no 0...9 or -") + } + switch ascii { + case _asciiNumbers, UInt8(ascii: "-"): + break + default: + preconditionFailure("Why was this function called, if there is no 0...9 or -") + } + while let byte = peek() { + if _fastPath(_asciiNumbers.contains(byte)) { + moveReaderIndex(forwardBy: 1) + continue + } + switch byte { + case UInt8(ascii: "."), UInt8(ascii: "+"), UInt8(ascii: "-"): + moveReaderIndex(forwardBy: 1) + case UInt8(ascii: "e"), UInt8(ascii: "E"): + moveReaderIndex(forwardBy: 1) + containsExponent = true + default: + return + } + } + } + } +} + +// MARK: - Deferred Parsing Methods - + +extension JSONScanner { + + // MARK: String + + static func stringValue(from jsonBytes: UnsafeBufferPointer, docStart: UnsafePointer) throws -> String { + let stringStartPtr = jsonBytes.baseAddress.unsafelyUnwrapped + let stringEndPtr = stringStartPtr + jsonBytes.count + + // Assume easy path first -- no escapes, no characters requiring escapes. + var cursor = stringStartPtr + while cursor < stringEndPtr { + let byte = cursor.pointee + if byte != ._backslash && _fastPath(byte & 0xe0 != 0) { + cursor += 1 + } else { + break + } + } + if cursor == stringEndPtr { + // We went through all the characters! Easy peasy. + guard let result = String._tryFromUTF8(jsonBytes) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: stringStartPtr, docStart: docStart)) + } + return result + } + return try _slowpath_stringValue(from: cursor, stringStartPtr: stringStartPtr, stringEndPtr: stringEndPtr, docStart: docStart) + } + + static func _slowpath_stringValue(from prevCursor: UnsafePointer, stringStartPtr: UnsafePointer, stringEndPtr: UnsafePointer, docStart: UnsafePointer) throws -> String { + var cursor = prevCursor + var chunkStart = cursor + guard var output = String._tryFromUTF8(UnsafeBufferPointer(start: stringStartPtr, count: cursor - stringStartPtr)) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: chunkStart, docStart: docStart)) + } + + while cursor < stringEndPtr { + let byte = cursor.pointee + switch byte { + case ._backslash: + guard let stringChunk = String._tryFromUTF8(UnsafeBufferPointer(start: chunkStart, count: cursor - chunkStart)) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: chunkStart, docStart: docStart)) + } + output += stringChunk + + // Advance past the backslash + cursor += 1 + + try parseEscapeSequence(into: &output, cursor: &cursor, end: stringEndPtr, docStart: docStart) + chunkStart = cursor + + default: + guard _fastPath(byte & 0xe0 != 0) else { + // All Unicode characters may be placed within the quotation marks, except for the characters that must be escaped: quotation mark, reverse solidus, and the control characters (U+0000 through U+001F). + throw JSONError.unescapedControlCharacterInString(ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + cursor += 1 + continue + } + } + + guard let stringChunk = String._tryFromUTF8(UnsafeBufferPointer(start: chunkStart, count: cursor - chunkStart)) else { + throw JSONError.cannotConvertInputStringDataToUTF8(location: .sourceLocation(at: chunkStart, docStart: docStart)) + } + output += stringChunk + + return output + } + + private static func parseEscapeSequence(into string: inout String, cursor: inout UnsafePointer, end: UnsafePointer, docStart: UnsafePointer) throws { + precondition(end > cursor, "Scanning should have ensured that all escape sequences are valid shape") + + let ascii = cursor.pointee + cursor += 1 + switch ascii { + case UInt8(ascii:"\""): string.append("\"") + case UInt8(ascii:"\\"): string.append("\\") + case UInt8(ascii:"/"): string.append("/") + case UInt8(ascii:"b"): string.append("\u{08}") // \b + case UInt8(ascii:"f"): string.append("\u{0C}") // \f + case UInt8(ascii:"n"): string.append("\u{0A}") // \n + case UInt8(ascii:"r"): string.append("\u{0D}") // \r + case UInt8(ascii:"t"): string.append("\u{09}") // \t + case UInt8(ascii:"u"): + try parseUnicodeSequence(into: &string, cursor: &cursor, end: end, docStart: docStart) + default: + throw JSONError.unexpectedEscapedCharacter(ascii: ascii, location: .sourceLocation(at: cursor, docStart: docStart)) + } + } + + // Shared with JSON5, which requires allowNulls = false for compatibility. + internal static func parseUnicodeSequence(into string: inout String, cursor: inout UnsafePointer, end: UnsafePointer, docStart: UnsafePointer, allowNulls: Bool = true) throws { + // we build this for utf8 only for now. + let bitPattern = try parseUnicodeHexSequence(cursor: &cursor, end: end, docStart: docStart, allowNulls: allowNulls) + + // check if lead surrogate + if UTF16.isLeadSurrogate(bitPattern) { + // if we have a lead surrogate we expect a trailing surrogate next + let leadSurrogateBitPattern = bitPattern + guard end - cursor >= 2 else { + throw JSONError.expectedLowSurrogateUTF8SequenceAfterHighSurrogate(location: .sourceLocation(at: cursor, docStart: docStart)) + } + let escapeChar = cursor.pointee + let uChar = (cursor+1).pointee + guard escapeChar == ._backslash, uChar == UInt8(ascii: "u") else { + throw JSONError.expectedLowSurrogateUTF8SequenceAfterHighSurrogate(location: .sourceLocation(at: cursor, docStart: docStart)) + } + cursor += 2 + + let trailSurrogateBitPattern = try parseUnicodeHexSequence(cursor: &cursor, end: end, docStart: docStart) + guard UTF16.isTrailSurrogate(trailSurrogateBitPattern) else { + throw JSONError.expectedLowSurrogateUTF8SequenceAfterHighSurrogate(location: .sourceLocation(at: cursor-6, docStart: docStart)) + } + + let encodedScalar = UTF16.EncodedScalar([leadSurrogateBitPattern, trailSurrogateBitPattern]) + let unicode = UTF16.decode(encodedScalar) + string.unicodeScalars.append(unicode) + } else { + guard let unicode = Unicode.Scalar(bitPattern) else { + throw JSONError.couldNotCreateUnicodeScalarFromUInt32(location: .sourceLocation(at: cursor-6, docStart: docStart), unicodeScalarValue: UInt32(bitPattern)) + } + string.unicodeScalars.append(unicode) + } + } + + internal static func parseUnicodeHexSequence(cursor: inout UnsafePointer, end: UnsafePointer, docStart: UnsafePointer, allowNulls: Bool = true) throws -> UInt16 { + precondition(end - cursor >= 4, "Scanning should have ensured that all escape sequences are valid shape") + + guard let first = cursor.pointee.hexDigitValue, + let second = (cursor+1).pointee.hexDigitValue, + let third = (cursor+2).pointee.hexDigitValue, + let fourth = (cursor+3).pointee.hexDigitValue + else { + let hexString = String(decoding: UnsafeBufferPointer(start: cursor, count: 4), as: Unicode.UTF8.self) + throw JSONError.invalidHexDigitSequence(hexString, location: .sourceLocation(at: cursor, docStart: docStart)) + } + let firstByte = UInt16(first) << 4 | UInt16(second) + let secondByte = UInt16(third) << 4 | UInt16(fourth) + + let result = UInt16(firstByte) << 8 | UInt16(secondByte) + guard allowNulls || result != 0 else { + throw JSONError.invalidEscapedNullValue(location: .sourceLocation(at: cursor, docStart: docStart)) + } + cursor += 4 + return result + } + + + // MARK: Numbers + + static func validateLeadingZero(in jsonBytes: UnsafeBufferPointer, following cursor: UnsafePointer, docStart: UnsafePointer) throws { + let endPtr = jsonBytes.baseAddress.unsafelyUnwrapped + jsonBytes.count + + // Leading zeros are very restricted. + let next = cursor+1 + if next == endPtr { + // Yep, this is valid. + return + } + switch next.pointee { + case UInt8(ascii: "."), UInt8(ascii: "e"), UInt8(ascii: "E"): + // We need to parse the fractional part. + break + case _asciiNumbers: + throw JSONError.numberWithLeadingZero(location: .sourceLocation(at: next, docStart: docStart)) + default: + throw JSONError.unexpectedCharacter(context: "in number", ascii: next.pointee, location: .sourceLocation(at: next, docStart: docStart)) + } + } + + // Returns the pointer at which the number's digits begin. If there are no digits, the function throws. + static func prevalidateJSONNumber(from jsonBytes: UnsafeBufferPointer, hasExponent: Bool, docStart: UnsafePointer) throws -> UnsafePointer { + // Just make sure we (A) don't have a leading zero, and (B) We have at least one digit. + guard !jsonBytes.isEmpty else { + preconditionFailure("Why was this function called, if there is no 0...9 or -") + } + let startPtr = jsonBytes.baseAddress.unsafelyUnwrapped + let endPtr = startPtr + jsonBytes.count + let digitsBeginPtr : UnsafePointer + switch startPtr.pointee { + case UInt8(ascii: "0"): + try validateLeadingZero(in: jsonBytes, following: startPtr, docStart: docStart) + digitsBeginPtr = startPtr + case UInt8(ascii: "1") ... UInt8(ascii: "9"): + digitsBeginPtr = startPtr + case UInt8(ascii: "-"): + let next = startPtr+1 + guard next < endPtr else { + throw JSONError.unexpectedCharacter(context: "at end of number", ascii: startPtr.pointee, location: .sourceLocation(at: endPtr-1, docStart: docStart)) + } + switch next.pointee { + case UInt8(ascii: "0"): + try validateLeadingZero(in: jsonBytes, following: next, docStart: docStart) + case UInt8(ascii: "1") ... UInt8(ascii: "9"): + // Good, we need at least one digit following the '-' + break + default: + // Any other character is invalid. + throw JSONError.unexpectedCharacter(context: "after '-' in number", ascii: startPtr.pointee, location: .sourceLocation(at: next, docStart: docStart)) + } + digitsBeginPtr = next + default: + preconditionFailure("Why was this function called, if there is no 0...9 or -") + } + + // If the number was found to have an exponent, we have to guarantee that there are digits preceding it, which is a JSON requirement. If we don't, then our floating point parser, which is tolerant of that construction, will succeed and not produce the required error. + if hasExponent { + var cursor = digitsBeginPtr+1 + while cursor < endPtr { + switch cursor.pointee { + case UInt8(ascii: "e"), UInt8(ascii: "E"): + guard case _asciiNumbers = (cursor-1).pointee else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: cursor.pointee, location: .sourceLocation(at: cursor, docStart: docStart)) + } + // We're done iterating. + cursor = endPtr + default: + cursor += 1 + } + } + } + + switch jsonBytes.last.unsafelyUnwrapped { + case _asciiNumbers: + break // In JSON, all numbers must end with a digit. + default: + throw JSONError.unexpectedCharacter(context: "at end of number", ascii: jsonBytes.last.unsafelyUnwrapped, location: .sourceLocation(at: endPtr-1, docStart: docStart)) + } + return digitsBeginPtr + } + + // This function is intended to be called after prevalidateJSONNumber() (which provides the digitsBeginPtr) and after parsing fails. It will provide more useful information about the invalid input. + static func validateNumber(from jsonBytes: UnsafeBufferPointer, withDigitsBeginningAt digitsBeginPtr: UnsafePointer, docStart: UnsafePointer) throws { + enum ControlCharacter { + case operand + case decimalPoint + case exp + case expOperator + } + + var cursor = jsonBytes.baseAddress.unsafelyUnwrapped + let endPtr = cursor + jsonBytes.count + + cursor = digitsBeginPtr + 1 + + // Fast-path, assume all digits. + while cursor < endPtr { + let byte = cursor.pointee + if _asciiNumbers.contains(byte) { + cursor += 1 + } else { + break + } + } + if cursor == endPtr { + // They were all digits. We're done! + // TODO: This should preconditionFailure() I think. Same for regular JSON. + return + } + + var pastControlChar: ControlCharacter = .operand + var numbersSinceControlChar = cursor - digitsBeginPtr + + // parse everything else + while cursor < endPtr { + let byte = cursor.pointee + switch byte { + case UInt8(ascii: "0"): + numbersSinceControlChar += 1 + case UInt8(ascii: "1") ... UInt8(ascii: "9"): + numbersSinceControlChar += 1 + case UInt8(ascii: "."): + guard numbersSinceControlChar > 0, pastControlChar == .operand else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + pastControlChar = .decimalPoint + numbersSinceControlChar = 0 + + case UInt8(ascii: "e"), UInt8(ascii: "E"): + guard numbersSinceControlChar > 0, + pastControlChar == .operand || pastControlChar == .decimalPoint + else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + pastControlChar = .exp + numbersSinceControlChar = 0 + case UInt8(ascii: "+"), UInt8(ascii: "-"): + guard numbersSinceControlChar == 0, pastControlChar == .exp else { + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + + pastControlChar = .expOperator + numbersSinceControlChar = 0 + default: + throw JSONError.unexpectedCharacter(context: "in number", ascii: byte, location: .sourceLocation(at: cursor, docStart: docStart)) + } + cursor += 1 + } + + guard numbersSinceControlChar > 0 else { + preconditionFailure("Found trailing non-digit. Number character buffer was not validated with prevalidateJSONNumber()") + } + } +} + +// Protocol conformed to by the numeric types we parse. For each of them, the +protocol PrevalidatedJSONNumberBufferConvertible { + init?(prevalidatedBuffer buffer: UnsafeBufferPointer) +} + +extension Double : PrevalidatedJSONNumberBufferConvertible { + init?(prevalidatedBuffer buffer: UnsafeBufferPointer) { + let bufferEnd = buffer.baseAddress.unsafelyUnwrapped + buffer.count + var endPtr : UnsafeMutablePointer? = nil + let result = withUnsafeMutablePointer(to: &endPtr) { + strtod_l(buffer.baseAddress, $0, nil) + } + guard let endPtr, endPtr == bufferEnd else { + return nil + } + self = result + } +} + +extension Float : PrevalidatedJSONNumberBufferConvertible { + init?(prevalidatedBuffer buffer: UnsafeBufferPointer) { + let bufferEnd = buffer.baseAddress.unsafelyUnwrapped + buffer.count + var endPtr : UnsafeMutablePointer? = nil + let result = withUnsafeMutablePointer(to: &endPtr) { + strtof_l(buffer.baseAddress, $0, nil) + } + guard let endPtr, endPtr == bufferEnd else { + return nil + } + self = result + } +} + +@_alwaysEmitIntoClient +internal func _parseIntegerDigits( + _ codeUnits: UnsafeBufferPointer, isNegative: Bool +) -> Result? { + guard _fastPath(!codeUnits.isEmpty) else { return nil } + + // ASCII constants, named for clarity: + let _0 = 48 as UInt8 + + let numericalUpperBound: UInt8 = _0 &+ 10 + let multiplicand : Result = 10 + var result : Result = 0 + + var iter = codeUnits.makeIterator() + while let digit = iter.next() { + let digitValue: Result + if _fastPath(digit >= _0 && digit < numericalUpperBound) { + digitValue = Result(truncatingIfNeeded: digit &- _0) + } else { + return nil + } + let overflow1: Bool + (result, overflow1) = result.multipliedReportingOverflow(by: multiplicand) + let overflow2: Bool + (result, overflow2) = isNegative + ? result.subtractingReportingOverflow(digitValue) + : result.addingReportingOverflow(digitValue) + guard _fastPath(!overflow1 && !overflow2) else { return nil } + } + return result +} + +@_alwaysEmitIntoClient +internal func _parseInteger(_ codeUnits: UnsafeBufferPointer) -> Result? { + guard _fastPath(!codeUnits.isEmpty) else { return nil } + + // ASCII constants, named for clarity: + let _plus = 43 as UInt8, _minus = 45 as UInt8 + + let first = codeUnits[0] + if first == _minus { + return _parseIntegerDigits(UnsafeBufferPointer(rebasing: codeUnits[1...]), isNegative: true) + } + if first == _plus { + return _parseIntegerDigits(UnsafeBufferPointer(rebasing: codeUnits[1...]), isNegative: false) + } + return _parseIntegerDigits(codeUnits, isNegative: false) +} + +extension FixedWidthInteger { + init?(prevalidatedBuffer buffer: UnsafeBufferPointer) { + guard let val : Self = _parseInteger(buffer) else { + return nil + } + self = val + } +} + +extension UInt8 { + + internal static var _space: UInt8 { UInt8(ascii: " ") } + internal static var _return: UInt8 { UInt8(ascii: "\r") } + internal static var _newline: UInt8 { UInt8(ascii: "\n") } + internal static var _tab: UInt8 { UInt8(ascii: "\t") } + + internal static var _colon: UInt8 { UInt8(ascii: ":") } + internal static var _comma: UInt8 { UInt8(ascii: ",") } + + internal static var _openbrace: UInt8 { UInt8(ascii: "{") } + internal static var _closebrace: UInt8 { UInt8(ascii: "}") } + + internal static var _openbracket: UInt8 { UInt8(ascii: "[") } + internal static var _closebracket: UInt8 { UInt8(ascii: "]") } + + internal static var _quote: UInt8 { UInt8(ascii: "\"") } + internal static var _backslash: UInt8 { UInt8(ascii: "\\") } + +} + +internal var _asciiNumbers: ClosedRange { UInt8(ascii: "0") ... UInt8(ascii: "9") } +internal var _hexCharsUpper: ClosedRange { UInt8(ascii: "A") ... UInt8(ascii: "F") } +internal var _hexCharsLower: ClosedRange { UInt8(ascii: "a") ... UInt8(ascii: "f") } +internal var _allLettersUpper: ClosedRange { UInt8(ascii: "A") ... UInt8(ascii: "Z") } +internal var _allLettersLower: ClosedRange { UInt8(ascii: "a") ... UInt8(ascii: "z") } + +extension UInt8 { + internal var hexDigitValue: UInt8? { + switch self { + case _asciiNumbers: + return self - _asciiNumbers.lowerBound + case _hexCharsUpper: + // uppercase letters + return self - _hexCharsUpper.lowerBound &+ 10 + case _hexCharsLower: + // lowercase letters + return self - _hexCharsLower.lowerBound &+ 10 + default: + return nil + } + } + + internal var isValidHexDigit: Bool { + switch self { + case _asciiNumbers, _hexCharsUpper, _hexCharsLower: + return true + default: + return false + } + } +} + +enum JSONError: Swift.Error, Equatable { + struct SourceLocation: Equatable { + let line: Int + let column: Int + let index: Int + + static func sourceLocation(at location: UnsafePointer, docStart: UnsafePointer) -> SourceLocation { + precondition(docStart <= location) + var cursor = docStart + var line = 1 + var col = 0 + while cursor <= location { + switch cursor.pointee { + case ._return: + if cursor+1 <= location, cursor.pointee == ._newline { + cursor += 1 + } + line += 1 + col = 0 + case ._newline: + line += 1 + col = 0 + default: + col += 1 + } + cursor += 1 + } + return SourceLocation(line: line, column: col, index: location - docStart) + } + } + + case cannotConvertEntireInputDataToUTF8 + case cannotConvertInputStringDataToUTF8(location: SourceLocation) + case unexpectedCharacter(context: String? = nil, ascii: UInt8, location: SourceLocation) + case unexpectedEndOfFile + case tooManyNestedArraysOrDictionaries(location: SourceLocation? = nil) + case invalidHexDigitSequence(String, location: SourceLocation) + case invalidEscapedNullValue(location: SourceLocation) + case invalidSpecialValue(expected: String, location: SourceLocation) + case unexpectedEscapedCharacter(ascii: UInt8, location: SourceLocation) + case unescapedControlCharacterInString(ascii: UInt8, location: SourceLocation) + case expectedLowSurrogateUTF8SequenceAfterHighSurrogate(location: SourceLocation) + case couldNotCreateUnicodeScalarFromUInt32(location: SourceLocation, unicodeScalarValue: UInt32) + case numberWithLeadingZero(location: SourceLocation) + case numberIsNotRepresentableInSwift(parsed: String) + case singleFragmentFoundButNotAllowed + + // JSON5 + + case unterminatedBlockComment + + var debugDescription : String { + switch self { + case .cannotConvertEntireInputDataToUTF8: + return "Unable to convert data to a string using the detected encoding. The data may be corrupt." + case let .cannotConvertInputStringDataToUTF8(location): + let line = location.line + let col = location.column + return "Unable to convert data to a string around line \(line), column \(col)." + case let .unexpectedCharacter(context, ascii, location): + let line = location.line + let col = location.column + if let context { + return "Unexpected character '\(String(UnicodeScalar(ascii)))' \(context) around line \(line), column \(col)." + } else { + return "Unexpected character '\(String(UnicodeScalar(ascii)))' around line \(line), column \(col)." + } + case .unexpectedEndOfFile: + return "Unexpected end of file" + case .tooManyNestedArraysOrDictionaries(let location): + if let location { + let line = location.line + let col = location.column + return "Too many nested arrays or dictionaries around line \(line), column \(col)." + } else { + return "Too many nested arrays or dictionaries." + } + case let .invalidHexDigitSequence(hexSequence, location): + let line = location.line + let col = location.column + return "Invalid hex digit in unicode escape sequence '\(hexSequence)' around line \(line), column \(col)." + case let .invalidEscapedNullValue(location): + let line = location.line + let col = location.column + return "Unsupported escaped null around line \(line), column \(col)." + case let .invalidSpecialValue(expected, location): + let line = location.line + let col = location.column + return "Invalid \(expected) value around line \(line), column \(col)." + case let .unexpectedEscapedCharacter(ascii, location): + let line = location.line + let col = location.column + return "Invalid escape sequence '\(String(UnicodeScalar(ascii)))' around line \(line), column \(col)." + case let .unescapedControlCharacterInString(ascii, location): + let line = location.line + let col = location.column + return "Unescaped control character '0x\(String(ascii, radix: 16))' around line \(line), column \(col)." + case let .expectedLowSurrogateUTF8SequenceAfterHighSurrogate(location): + let line = location.line + let col = location.column + return "Missing low code point in surrogate pair around line \(line), column \(col)." + case let .couldNotCreateUnicodeScalarFromUInt32(location, unicodeScalarValue): + let line = location.line + let col = location.column + return "Invalid unicode scalar value '0x\(String(unicodeScalarValue, radix: 16))' around line \(line), column \(col)." + case let .numberWithLeadingZero(location): + let line = location.line + let col = location.column + return "Number with leading zero around line \(line), column \(col)." + case let .numberIsNotRepresentableInSwift(parsed): + return "Number \(parsed) is not representable in Swift." + case .singleFragmentFoundButNotAllowed: + return "JSON input did not start with array or object as required by options." + + // JSON5 + + case .unterminatedBlockComment: + return "Unterminated block comment" + } + } + + var sourceLocation: SourceLocation? { + switch self { + case let .cannotConvertInputStringDataToUTF8(location), let .unexpectedCharacter(_, _, location): + return location + case let .tooManyNestedArraysOrDictionaries(location): + return location + case let .invalidHexDigitSequence(_, location), let .invalidEscapedNullValue(location), let .invalidSpecialValue(_, location): + return location + case let .unexpectedEscapedCharacter(_, location), let .unescapedControlCharacterInString(_, location), let .expectedLowSurrogateUTF8SequenceAfterHighSurrogate(location): + return location + case let .couldNotCreateUnicodeScalarFromUInt32(location, _), let .numberWithLeadingZero(location): + return location + default: + return nil + } + } + +#if FOUNDATION_FRAMEWORK + var nsError: NSError { + var userInfo : [String: Any] = [ + NSDebugDescriptionErrorKey : self.debugDescription + ] + if let location = self.sourceLocation { + userInfo["NSJSONSerializationErrorIndex"] = location.index + } + return .init(domain: NSCocoaErrorDomain, code: CocoaError.propertyListReadCorrupt.rawValue, userInfo: userInfo) + } +#endif // FOUNDATION_FRAMEWORK +} diff --git a/Sources/FoundationEssentials/JSON/JSONWriter.swift b/Sources/FoundationEssentials/JSON/JSONWriter.swift new file mode 100644 index 000000000..82b875958 --- /dev/null +++ b/Sources/FoundationEssentials/JSON/JSONWriter.swift @@ -0,0 +1,333 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +enum JSONValue: Equatable { + case string(String) + case number(String) + case bool(Bool) + case null + + case array([JSONValue]) + case object([String: JSONValue]) +} + +extension JSONValue { + static func number(from num: any (FixedWidthInteger & CustomStringConvertible)) -> JSONValue { + return .number(num.description) + } + + static func number(from float: T, with options: JSONEncoder.NonConformingFloatEncodingStrategy, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = Optional<_JSONKey>.none) throws -> JSONValue { + guard !float.isNaN, !float.isInfinite else { + if case .convertToString(let posInfString, let negInfString, let nanString) = options { + switch float { + case T.infinity: + return .string(posInfString) + case -T.infinity: + return .string(negInfString) + default: + // must be nan in this case + return .string(nanString) + } + } + + let path = codingPathNode.path(with: additionalKey) + throw EncodingError.invalidValue(float, .init( + codingPath: path, + debugDescription: "Unable to encode \(T.self).\(float) directly in JSON." + )) + } + + var string = float.description + if string.hasSuffix(".0") { + string.removeLast(2) + } + return .number(string) + } +} + +internal struct JSONWriter { + + // Structures with container nesting deeper than this limit are not valid. + private static let maximumRecursionDepth = 512 + + private var indent = 0 + private let pretty: Bool + private let sortedKeys: Bool + private let withoutEscapingSlashes: Bool + + var data = Data() + + init(options: WritingOptions) { + pretty = options.contains(.prettyPrinted) +#if FOUNDATION_FRAMEWORK + sortedKeys = options.contains(.sortedKeys) +#else + sortedKeys = false +#endif + withoutEscapingSlashes = options.contains(.withoutEscapingSlashes) + data = Data() + } + + mutating func serializeJSON(_ value: JSONValue, depth: Int = 0) throws { + switch value { + case .string(let str): + try serializeString(str) + case .bool(let boolValue): + writer(boolValue.description) + case .number(let numberStr): + writer(numberStr) + case .array(let array): + try serializeArray(array, depth: depth + 1) + case .object(let object): + try serializeObject(object, depth: depth + 1) + case .null: + serializeNull() + } + } + + @inline(__always) + mutating func writer(_ string: StaticString) { + string.withUTF8Buffer { + data.append($0.baseAddress.unsafelyUnwrapped, count: $0.count) + } + } + + @inline(__always) + mutating func writer(_ string: String) { + var localString = string + localString.withUTF8 { + data.append($0.baseAddress.unsafelyUnwrapped, count: $0.count) + } + } + + @inline(__always) + mutating func writer(ascii: UInt8) { + data.append(ascii) + } + + @inline(__always) + mutating func writer(pointer: UnsafePointer, count: Int) { + data.append(pointer, count: count) + } + + mutating func serializeString(_ str: String) throws { + writer("\"") + + str.withCString { + $0.withMemoryRebound(to: UInt8.self, capacity: 1) { + var cursor = $0 + var mark = cursor + while cursor.pointee != 0 { + let escapeString: String + switch cursor.pointee { + case ._quote: + escapeString = "\\\"" + break + case ._backslash: + escapeString = "\\\\" + break + case ._slash where !withoutEscapingSlashes: + escapeString = "\\/" + break + case 0x8: + escapeString = "\\b" + break + case 0xc: + escapeString = "\\f" + break + case ._newline: + escapeString = "\\n" + break + case ._return: + escapeString = "\\r" + break + case ._tab: + escapeString = "\\t" + break + case 0x0...0xf: + escapeString = "\\u000\(String(cursor.pointee, radix: 16))" + break + case 0x10...0x1f: + escapeString = "\\u00\(String(cursor.pointee, radix: 16))" + break + default: + // Accumulate this byte + cursor += 1 + continue + } + + // Append accumulated bytes + if cursor > mark { + writer(pointer: mark, count: cursor-mark) + } + writer(escapeString) + + cursor += 1 + mark = cursor // Start accumulating bytes starting after this escaped byte. + } + + // Append accumulated bytes + if cursor > mark { + writer(pointer: mark, count: cursor-mark) + } + } + } + writer("\"") + } + + mutating func serializeArray(_ array: [JSONValue], depth: Int) throws { + guard depth < Self.maximumRecursionDepth else { + throw JSONError.tooManyNestedArraysOrDictionaries() + } + + writer("[") + if pretty { + writer("\n") + incIndent() + } + + var first = true + for elem in array { + if first { + first = false + } else if pretty { + writer(",\n") + } else { + writer(",") + } + if pretty { + writeIndent() + } + try serializeJSON(elem, depth: depth) + } + if pretty { + writer("\n") + decAndWriteIndent() + } + writer("]") + } + + mutating func serializeObject(_ dict: [String:JSONValue], depth: Int) throws { + guard depth < Self.maximumRecursionDepth else { + throw JSONError.tooManyNestedArraysOrDictionaries() + } + + writer("{") + if pretty { + writer("\n") + incIndent() + if dict.count > 0 { + writeIndent() + } + } + + var first = true + + func serializeObjectElement(key: String, value: JSONValue, depth: Int) throws { + if first { + first = false + } else if pretty { + writer(",\n") + writeIndent() + } else { + writer(",") + } + try serializeString(key) + pretty ? writer(" : ") : writer(":") + try serializeJSON(value, depth: depth) + } + + if sortedKeys { + let elems = dict.sorted(by: { a, b in + let options: String.CompareOptions = [.numeric, .caseInsensitive, .forcedOrdering] + let range: Range = a.key.startIndex.. "AAA" with `.caseInsensitive` specified) + static let forcedOrdering = CompareOptions(rawValue: 512) + /// The search string is treated as an ICU-compatible regular expression; + /// if set, no other options can apply except `.caseInsensitive` and `.anchored` + static let regularExpression = CompareOptions(rawValue: 1024) + } +#endif // FOUNDATION_FRAMEWORK +} + +extension UTF8.CodeUnit { + static let newline: Self = 0x0A + static let carriageReturn: Self = 0x0D + + var _numericValue: Int? { + if self >= 48 && self <= 57 { + return Int(self - 48) + } + return nil + } + + // Copied from std; see comment in String.swift _uppercaseASCII() and _lowercaseASCII() + var _lowercased: Self { + let _uppercaseTable: UInt64 = + 0b0000_0000_0000_0000_0001_1111_1111_1111 &<< 32 + let isUpper = _uppercaseTable &>> UInt64(((self &- 1) & 0b0111_1111) &>> 1) + let toAdd = (isUpper & 0x1) &<< 5 + return self &+ UInt8(truncatingIfNeeded: toAdd) + } + + var _uppercased: Self { + let _lowercaseTable: UInt64 = + 0b0001_1111_1111_1111_0000_0000_0000_0000 &<< 32 + let isLower = _lowercaseTable &>> UInt64(((self &- 1) & 0b0111_1111) &>> 1) + let toSubtract = (isLower & 0x1) &<< 5 + return self &- UInt8(truncatingIfNeeded: toSubtract) + } +} + +// MARK: - _StringCompareOptionsIterable Methods +// Internal protocols to share the implementation for iterating BidirectionalCollections of String family and process their elements according to String.CompareOptions. +internal protocol _StringCompareOptionsConvertible : Comparable & Equatable { + associatedtype IterableType: _StringCompareOptionsIterable + func _transform(toHalfWidth: Bool, stripDiacritics: Bool, caseFolding: Bool) -> IterableType + var intValue: Int? { get } + var isExtendCharacter: Bool { get } +} + +internal protocol _StringCompareOptionsIterable : BidirectionalCollection where Element: _StringCompareOptionsConvertible, Element.IterableType.SubSequence == Self.SubSequence, Element == SubSequence.Element { + init() + var first: Element? { get } + func _consumeExtendCharacters(from i: inout Index) + func consumeNumbers(from i: inout Index, initialValue: Int) -> Int +} + +extension _StringCompareOptionsIterable { + func consumeNumbers(from i: inout Index, initialValue: Int) -> Int { + guard i < endIndex else { + return initialValue + } + + var value = initialValue + while i < endIndex { + let c = self[i] + guard let num = c.intValue else { + break + } + // equivalent to `value = value * 10 + num` but considering overflow + let multiplied = value.multipliedReportingOverflow(by: 10) + guard !multiplied.overflow else { break } + + let added = multiplied.partialValue.addingReportingOverflow(num) + guard !added.overflow else { break } + + value = added.partialValue + self.formIndex(after: &i) + } + + return value + } + + func _consumeExtendCharacters(from i: inout Index) { + while i < endIndex, self[i].isExtendCharacter { + formIndex(after: &i) + } + } + + func _compare(_ other: S, toHalfWidth: Bool, diacriticsInsensitive: Bool, caseFold: Bool, numeric: Bool, forceOrdering: Bool) -> ComparisonResult where S.Element == Element { + + var idx1 = self.startIndex + var idx2 = other.startIndex + + var compareResult: ComparisonResult = .orderedSame + + var norm1 = _StringCompareOptionsIterableBuffer() + var norm2 = _StringCompareOptionsIterableBuffer() + + while idx1 < self.endIndex && idx2 < other.endIndex { + var c1: Element + var c2: Element + if norm1.isEmpty { + c1 = self[idx1] + } else { + c1 = norm1.current + norm1.advance() + } + + if norm2.isEmpty { + c2 = other[idx2] + } else { + c2 = norm2.current + norm2.advance() + } + + if numeric, norm1.isEmpty, norm2.isEmpty, c1.intValue != nil, c2.intValue != nil { + let value1 = self.consumeNumbers(from: &idx1, initialValue: 0) + let value2 = other.consumeNumbers(from: &idx2, initialValue: 0) + + if value1 == value2 { + if forceOrdering { + let dist1 = self.distance(from: startIndex, to: idx1) + let dist2 = other.distance(from: other.startIndex, to: idx2) + if dist1 != dist2 { + compareResult = ComparisonResult(dist1, dist2) + } + } + continue + } else { + return ComparisonResult(value1, value2) + } + } + + if diacriticsInsensitive && idx1 > startIndex { + var str1Skip = false + var str2Skip = false + if norm1.isEmpty && c1.isExtendCharacter { + c1 = c2 + str1Skip = true + } + + if norm2.isEmpty && c2.isExtendCharacter { + c2 = c1 + str2Skip = true + } + + if str1Skip != str2Skip { + if str1Skip { + other.formIndex(before: &idx2) + } else { + formIndex(before: &idx1) + } + } + } + + if c1 != c2 { + if !(toHalfWidth || diacriticsInsensitive || caseFold) { + return ComparisonResult(c1, c2) + } + + if forceOrdering && compareResult == .orderedSame { + compareResult = ComparisonResult(c1, c2) + } + + if norm1.isEmpty { + let t1 = c1._transform(toHalfWidth: toHalfWidth, stripDiacritics: diacriticsInsensitive, caseFolding: caseFold) + if let first = t1.first { + c1 = first + norm1 = .init(t1) + norm1.advance() + } + } + + if norm1.isEmpty && !norm2.isEmpty { + return ComparisonResult(c1, c2) + } + + if norm2.isEmpty && (norm1.isEmpty || c1 != c2) { + let t2 = c2._transform(toHalfWidth: toHalfWidth, stripDiacritics: diacriticsInsensitive, caseFolding: caseFold) + if let first = t2.first { + c2 = first + norm2 = .init(t2) + norm2.advance() + } + + if norm2.isEmpty || c1 != c2 { + return ComparisonResult(c1, c2) + } + } + + if !norm1.isEmpty && !norm2.isEmpty { + while !norm1.isEnd && !norm2.isEnd { + if norm1.current != norm2.current { + break + } + norm1.advance() + norm2.advance() + } + + if !norm1.isEnd && !norm2.isEnd { + return ComparisonResult(norm1.current, norm2.current) + } + } + } + + if !norm1.isEmpty && norm1.isEnd { + norm1.clear() + } + + if !norm2.isEmpty && norm2.isEnd { + norm2.clear() + } + + if norm1.isEmpty { + formIndex(after: &idx1) + } + + if norm2.isEmpty { + other.formIndex(after: &idx2) + } + } + + // Process the trailing diacritics, if there's any + if diacriticsInsensitive { + self._consumeExtendCharacters(from: &idx1) + other._consumeExtendCharacters(from: &idx2) + } + + let result = ComparisonResult(stringIndex: idx1, idx2: idx2, endIndex1: endIndex, endIndex2: other.endIndex) + return result == .orderedSame ? compareResult : result + } + + func _range(of strToFind: S, toHalfWidth: Bool, diacriticsInsensitive: Bool, caseFold: Bool, anchored: Bool, backwards: Bool) -> Range? where S.Index == Index, S.Element == Element { + + if !toHalfWidth && !diacriticsInsensitive && !caseFold { + return _range(of: strToFind, anchored: anchored, backwards: backwards) + } + + // These options may cause the string to change their count + let lengthVariants = caseFold || diacriticsInsensitive + + var fromLoc: Index + var toLoc: Index + if backwards { + if lengthVariants { + fromLoc = index(endIndex, offsetBy: -1) + } else { + guard let idx = _index(endIndex, backwardsOffsetByCountOf: strToFind) else { + return nil + } + fromLoc = idx + } + toLoc = (anchored && !lengthVariants) ? fromLoc : startIndex + } else { + fromLoc = startIndex + if anchored { + toLoc = fromLoc + } else if lengthVariants { + toLoc = index(endIndex, offsetBy: -1) + } else { + guard let idx = _index(endIndex, backwardsOffsetByCountOf: strToFind) else { + return nil + } + toLoc = idx + } + } + + let delta = fromLoc <= toLoc ? 1 : -1 + var result: Range? = nil + + while true { + // Outer loop: loops through `self` + + var str1Char: Element + var str2Char: Element + + var str1Index = fromLoc + var str2Index = strToFind.startIndex + + var useStrBuf1 = false + var useStrBuf2 = false + + var strBuf1 = _StringCompareOptionsIterableBuffer() + var strBuf2 = _StringCompareOptionsIterableBuffer() + + while str2Index < strToFind.endIndex { + // Inner loop: loops through `strToFind` + if !useStrBuf1 { + if str1Index == endIndex { + break + } + str1Char = self[str1Index] + } else { + str1Char = strBuf1.current + strBuf1.advance() + } + + if !useStrBuf2 { + str2Char = strToFind[str2Index] + } else { + str2Char = strBuf2.current + strBuf2.advance() + } + + if str1Char != str2Char { + if !useStrBuf1 { + let transformed = str1Char._transform(toHalfWidth: toHalfWidth, stripDiacritics: diacriticsInsensitive, caseFolding: caseFold) + + if let c = transformed.first { + str1Char = c + strBuf1 = .init(transformed) + strBuf1.advance() + useStrBuf1 = true + } + } + + if !useStrBuf1 && useStrBuf2 { break } + + if !useStrBuf2 && (!useStrBuf1 || str1Char != str2Char) { + let transformed = str2Char._transform(toHalfWidth: toHalfWidth, stripDiacritics: diacriticsInsensitive, caseFolding: caseFold) + if let c = transformed.first { + str2Char = c + strBuf2 = .init(transformed) + strBuf2.advance() + useStrBuf2 = true + } + + if str1Char != transformed.first { + break + } + } + } + + if useStrBuf1 && useStrBuf2 { + while !strBuf1.isEnd && !strBuf2.isEnd { + if strBuf1.current != strBuf2.current { + break + } + strBuf1.advance() + strBuf2.advance() + } + + if !strBuf1.isEnd && !strBuf2.isEnd { + break + } + + } + + if useStrBuf1 && strBuf1.isEnd { + useStrBuf1 = false + } + + if useStrBuf2 && strBuf2.isEnd { + useStrBuf2 = false + } + + if !useStrBuf1 { + formIndex(after: &str1Index) + } + + if !useStrBuf2 { + strToFind.formIndex(after: &str2Index) + } + } + + if str2Index == strToFind.endIndex { + // If `self` has extended characters following the lastly matched character, consume these + var match = true + if useStrBuf1 { + // if strToFind matches the string after transformed (strBuf1), try consuming extended characters from the buffer first + match = false + if diacriticsInsensitive { + strBuf1._consumeExtendCharacters() + } + + if strBuf1.isEnd { + formIndex(after: &str1Index) + match = true + } + } + + // After using up strBuf1, inspect the rest of original strings in `self` + if match && diacriticsInsensitive && str1Index < endIndex { + _consumeExtendCharacters(from: &str1Index) + } + + if match { + if !(anchored && backwards) || str1Index == endIndex { + result = fromLoc..(of strToFind: S, anchored: Bool, backwards: Bool) -> Range? where S.Element == Element { + var result: Range? = nil + var fromLoc: Index + var toLoc: Index + if backwards { + guard let idx = _index(endIndex, backwardsOffsetByCountOf: strToFind) else { + // strToFind.count > string.count: bail + return nil + } + fromLoc = idx + + toLoc = anchored ? fromLoc : startIndex + } else { + fromLoc = startIndex + if anchored { + toLoc = fromLoc + } else { + guard let idx = _index(endIndex, backwardsOffsetByCountOf: strToFind) else { + return nil + } + toLoc = idx + } + } + + let delta = fromLoc <= toLoc ? 1 : -1 + + while true { + var str1Index = fromLoc + var str2Index = strToFind.startIndex + + while str2Index < strToFind.endIndex && str1Index < endIndex { + if self[str1Index] != strToFind[str2Index] { + break + } + formIndex(after: &str1Index) + strToFind.formIndex(after: &str2Index) + } + + if str2Index == strToFind.endIndex { + result = fromLoc.. String.UTF8View { + String(unsafeUninitializedCapacity: 1) { + $0[0] = caseFolding ? self._lowercased : self + return 1 + }.utf8 + } + + var intValue: Int? { + return (self >= 48 || self <= 57) ? Int(self - 48) : nil + } + + var isExtendCharacter: Bool { + // This won't really get called and will be removed in a future PR + return false + } +} + +extension Character : _StringCompareOptionsConvertible { + + func _transform(toHalfWidth: Bool, stripDiacritics: Bool, caseFolding: Bool) -> String { + if isASCII { + // we only need to handle case folding, in which case is just lower case + return caseFolding ? lowercased() : String(self) + } + + var new = "" + for scalar in unicodeScalars { + var tmp = scalar + if toHalfWidth { + tmp = scalar._toHalfWidth() + } + + if stripDiacritics { + if scalar._isGraphemeExtend { + // skip this + continue + } else { + tmp = tmp._stripDiacritics() + } + } + + if caseFolding { + new += tmp._caseFoldMapping + } else { + new += String(tmp) + } + } + + return String(new) + } + + var intValue: Int? { + return wholeNumberValue + } + + var isExtendCharacter: Bool { + return _isExtendCharacter + } + +} + +extension UnicodeScalar : _StringCompareOptionsConvertible { + func _transform(toHalfWidth: Bool, stripDiacritics: Bool, caseFolding: Bool) -> String.UnicodeScalarView { + + var new = self + if toHalfWidth { + new = new._toHalfWidth() + } + + if stripDiacritics { + if new._isGraphemeExtend { + return String.UnicodeScalarView() + } else { + new = new._stripDiacritics() + } + } + + if caseFolding { + return new._caseFoldMapping.unicodeScalars + } else { + return String(new).unicodeScalars + } + } + + var intValue: Int? { + guard let v = properties.numericValue else { + return nil + } + return Int(v) + } + + var isExtendCharacter: Bool { + return _isGraphemeExtend + } +} + +// MARK: - _StringCompareOptionsIterableBuffer +internal struct _StringCompareOptionsIterableBuffer { + var _buf: StorageType + var _index: StorageType.Index + + init() { + _buf = StorageType() + _index = _buf.startIndex + } + + init(_ content: StorageType) { + _buf = content + _index = _buf.startIndex + } + + var current: StorageType.Element { + return _buf[_index] + } + + mutating func advance() { + _buf.formIndex(after: &_index) + } + + var isEnd: Bool { + return _index == _buf.endIndex + } + + var isEmpty: Bool { + return _buf.isEmpty + } + + mutating func _consumeExtendCharacters() { + _buf._consumeExtendCharacters(from: &_index) + } + + mutating func clear() { + self = .init() + } +} + +// MARK: Comparison Implementations +extension Substring { + func _unlocalizedCompare(other: Substring, options: String.CompareOptions) -> ComparisonResult { + if options.isEmpty { + return ComparisonResult(self, other) + } + + let diacriticInsensitive = options.contains(.diacriticInsensitive) + let toHalfWidth = options.contains(.widthInsensitive) + let caseFold = options.contains(.caseInsensitive) + let numeric = options.contains(.numeric) + let forceOrdering = options.contains(.forcedOrdering) + + var result: ComparisonResult + if options.contains(.literal) { + // Per documentation, literal means "Performs a byte-for-byte comparison. Differing literal sequences (such as composed character sequences) that would otherwise be considered equivalent are considered not to match." Therefore we're comparing the scalars rather than characters + result = unicodeScalars._compare(other.unicodeScalars, toHalfWidth: toHalfWidth, diacriticsInsensitive: diacriticInsensitive, caseFold: caseFold, numeric: numeric, forceOrdering: forceOrdering) + } else { + result = _compare(other, toHalfWidth: toHalfWidth, diacriticsInsensitive: diacriticInsensitive, caseFold: caseFold, numeric: numeric, forceOrdering: forceOrdering) + } + + if result == .orderedSame && forceOrdering { + result = unicodeScalars._compare(other.unicodeScalars) + } + + return result + } + + func _rangeOfCharacter(from set: CharacterSet, options: String.CompareOptions) -> Range? { + guard !isEmpty else { return nil } + + return unicodeScalars._rangeOfCharacter(from: set, anchored: options.contains(.anchored), backwards: options.contains(.backwards)) + } +} + +extension Substring.UnicodeScalarView { + func _compare(_ other: Substring.UnicodeScalarView) -> ComparisonResult { + var idx1 = startIndex + var idx2 = other.startIndex + + var scalar1: Unicode.Scalar + var scalar2: Unicode.Scalar + while idx1 < endIndex && idx2 < other.endIndex { + scalar1 = self[idx1] + scalar2 = other[idx2] + + if scalar1 == scalar2 { + self.formIndex(after: &idx1) + other.formIndex(after: &idx2) + continue + } else { + return ComparisonResult(scalar1, scalar2) + } + } + + return ComparisonResult(stringIndex: idx1, idx2: idx2, endIndex1: endIndex, endIndex2: other.endIndex) + } +} + +// MARK: - ComparisonResult Extension +extension ComparisonResult { + init(stringIndex idx1: Index, idx2: Index, endIndex1: Index, endIndex2: Index) { + if idx1 == endIndex1 && idx2 == endIndex2 { + self = .orderedSame + } else if idx1 == endIndex1 { + self = .orderedAscending + } else { + self = .orderedDescending + } + } + + init(_ t1: T, _ t2: T) { + if t1 < t2 { + self = .orderedAscending + } else if t1 > t2 { + self = .orderedDescending + } else { + self = .orderedSame + } + } +} + +extension BidirectionalCollection { + // Equal to calling `index(&idx, offsetBy: -other.count)` with just one loop + func _index(_ index: Index, backwardsOffsetByCountOf other: S) -> Index? { + var idx = index + var otherIdx = other.endIndex + while otherIdx > other.startIndex { + guard idx > startIndex else { + // other.count > self.count: bail + return nil + } + other.formIndex(before: &otherIdx) + formIndex(before: &idx) + } + return idx + } +} diff --git a/Sources/FoundationEssentials/String/String+Encoding.swift b/Sources/FoundationEssentials/String/String+Encoding.swift new file mode 100644 index 000000000..a960706ce --- /dev/null +++ b/Sources/FoundationEssentials/String/String+Encoding.swift @@ -0,0 +1,139 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if FOUNDATION_FRAMEWORK +// FIXME: one day this will be bridged from CoreFoundation and we +// should drop it here. (need support +// for CF bridging) +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +public var kCFStringEncodingASCII: CFStringEncoding { return 0x0600 } +#endif // FOUNDATION_FRAMEWORK + +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +extension String { + +#if !FOUNDATION_FRAMEWORK + // This is a workaround for Clang importer's ambiguous lookup issue since + // - Swift doesn't allow typealias to nested type + // - Swift doesn't allow typealias to builtin types like String + // We therefore rename String.Encoding to String._Encoding for package + // internal use so we can use `String._Encoding` to disambiguate. + public typealias Encoding = _Encoding + public struct _Encoding : RawRepresentable, Sendable, Equatable { + public var rawValue: UInt + public init(rawValue: UInt) { self.rawValue = rawValue } + + public static let ascii = Self(rawValue: 1) + public static let nextstep = Self(rawValue: 2) + public static let japaneseEUC = Self(rawValue: 3) + public static let utf8 = Self(rawValue: 4) + public static let isoLatin1 = Self(rawValue: 5) + public static let symbol = Self(rawValue: 6) + public static let nonLossyASCII = Self(rawValue: 7) + public static let shiftJIS = Self(rawValue: 8) + public static let isoLatin2 = Self(rawValue: 9) + public static let unicode = Self(rawValue: 10) + public static let windowsCP1251 = Self(rawValue: 11) + public static let windowsCP1252 = Self(rawValue: 12) + public static let windowsCP1253 = Self(rawValue: 13) + public static let windowsCP1254 = Self(rawValue: 14) + public static let windowsCP1250 = Self(rawValue: 15) + public static let iso2022JP = Self(rawValue: 21) + public static let macOSRoman = Self(rawValue: 30) + public static let utf16 = Self.unicode + public static let utf16BigEndian = Self(rawValue: 0x90000100) + public static let utf16LittleEndian = Self(rawValue: 0x94000100) + public static let utf32 = Self(rawValue: 0x8c000100) + public static let utf32BigEndian = Self(rawValue: 0x98000100) + public static let utf32LittleEndian = Self(rawValue: 0x9c000100) + } +#else + public struct Encoding : RawRepresentable, Sendable { + public var rawValue: UInt + public init(rawValue: UInt) { self.rawValue = rawValue } + + public static let ascii = Encoding(rawValue: 1) + public static let nextstep = Encoding(rawValue: 2) + public static let japaneseEUC = Encoding(rawValue: 3) + public static let utf8 = Encoding(rawValue: 4) + public static let isoLatin1 = Encoding(rawValue: 5) + public static let symbol = Encoding(rawValue: 6) + public static let nonLossyASCII = Encoding(rawValue: 7) + public static let shiftJIS = Encoding(rawValue: 8) + public static let isoLatin2 = Encoding(rawValue: 9) + public static let unicode = Encoding(rawValue: 10) + public static let windowsCP1251 = Encoding(rawValue: 11) + public static let windowsCP1252 = Encoding(rawValue: 12) + public static let windowsCP1253 = Encoding(rawValue: 13) + public static let windowsCP1254 = Encoding(rawValue: 14) + public static let windowsCP1250 = Encoding(rawValue: 15) + public static let iso2022JP = Encoding(rawValue: 21) + public static let macOSRoman = Encoding(rawValue: 30) + public static let utf16 = Encoding.unicode + public static let utf16BigEndian = Encoding(rawValue: 0x90000100) + public static let utf16LittleEndian = Encoding(rawValue: 0x94000100) + public static let utf32 = Encoding(rawValue: 0x8c000100) + public static let utf32BigEndian = Encoding(rawValue: 0x98000100) + public static let utf32LittleEndian = Encoding(rawValue: 0x9c000100) + } + + // For internal usage to disambiguate + internal typealias _Encoding = Encoding + + public typealias EncodingConversionOptions = NSString.EncodingConversionOptions + public typealias EnumerationOptions = NSString.EnumerationOptions +#endif // FOUNDATION_FRAMEWORK +} + +#if FOUNDATION_FRAMEWORK +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +extension String.Encoding : Hashable { + public var hashValue: Int { + // Note: This is effectively the same hashValue definition that + // RawRepresentable provides on its own. We only need to keep this to + // ensure ABI compatibility with 5.0. + return rawValue.hashValue + } + + @_alwaysEmitIntoClient // Introduced in 5.1 + public func hash(into hasher: inout Hasher) { + // Note: `hash(only:)` is only defined here because we also define + // `hashValue`. + // + // In 5.0, `hash(into:)` was resolved to RawRepresentable's functionally + // equivalent definition; we added this definition in 5.1 to make it + // clear this `hash(into:)` isn't synthesized by the compiler. + // (Otherwise someone may be tempted to define it, possibly breaking the + // hash encoding and thus the ABI. RawRepresentable's definition is + // inlinable.) + hasher.combine(rawValue) + } + + public static func ==(lhs: String.Encoding, rhs: String.Encoding) -> Bool { + // Note: This is effectively the same == definition that + // RawRepresentable provides on its own. We only need to keep this to + // ensure ABI compatibility with 5.0. + return lhs.rawValue == rhs.rawValue + } +} + +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +extension String.Encoding : CustomStringConvertible { + public var description: String { +#if FOUNDATION_FRAMEWORK + return String.localizedName(of: self) +#else + return "\(self)" +#endif + } +} +#endif diff --git a/Sources/FoundationEssentials/String/String+Extensions.swift b/Sources/FoundationEssentials/String/String+Extensions.swift new file mode 100644 index 000000000..194c2be3d --- /dev/null +++ b/Sources/FoundationEssentials/String/String+Extensions.swift @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension Character { + var _isExtendCharacter: Bool { + guard !self.isASCII else { + return false + } + + return unicodeScalars.allSatisfy { $0._isGraphemeExtend } + } + +} diff --git a/Sources/FoundationEssentials/String/StringAPIs.swift b/Sources/FoundationEssentials/String/StringAPIs.swift new file mode 100644 index 000000000..230195f23 --- /dev/null +++ b/Sources/FoundationEssentials/String/StringAPIs.swift @@ -0,0 +1,267 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if canImport(_ForSwiftFoundation) +@_implementationOnly import _ForSwiftFoundation +#endif + +#if !FOUNDATION_FRAMEWORK +fileprivate func _foundation_essentials_feature_enabled() -> Bool { return true } +#endif + +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +extension StringProtocol { + // - (NSRange)rangeOfCharacterFromSet:(NSCharacterSet *)aSet + // + // - (NSRange) + // rangeOfCharacterFromSet:(NSCharacterSet *)aSet + // options:(StringCompareOptions)mask + // + // - (NSRange) + // rangeOfCharacterFromSet:(NSCharacterSet *)aSet + // options:(StringCompareOptions)mask + // range:(NSRange)aRange + + /// Finds and returns the range in the `String` of the first + /// character from a given character set found in a given range with + /// given options. + public func rangeOfCharacter(from aSet: CharacterSet, options mask: String.CompareOptions = [], range aRange: Range? = nil) -> Range? { + if _foundation_essentials_feature_enabled() { + var subStr = Substring(self) + if let aRange { + subStr = subStr[aRange] + } + return subStr._rangeOfCharacter(from: aSet, options: mask) + } + +#if FOUNDATION_FRAMEWORK + return aSet.withUnsafeImmutableStorage { + return _optionalRange(_ns._rangeOfCharacter(from: $0, options: mask, range: _toRelativeNSRange(aRange ?? startIndex.. Data? { + switch encoding { + case .utf8: + return Data(self.utf8) + default: +#if FOUNDATION_FRAMEWORK // TODO: Implement data(using:allowLossyConversion:) in Swift + return _ns.data( + using: encoding.rawValue, + allowLossyConversion: allowLossyConversion) +#else + return nil +#endif + } + } +} + +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +extension String { + //===--- Initializers that can fail -------------------------------------===// + // - (instancetype) + // initWithBytes:(const void *)bytes + // length:(NSUInteger)length + // encoding:(NSStringEncoding)encoding + + /// Creates a new string equivalent to the given bytes interpreted in the + /// specified encoding. + /// + /// - Parameters: + /// - bytes: A sequence of bytes to interpret using `encoding`. + /// - encoding: The ecoding to use to interpret `bytes`. + public init?(bytes: __shared S, encoding: Encoding) + where S.Iterator.Element == UInt8 + { +#if FOUNDATION_FRAMEWORK // TODO: Move init?(bytes:encoding) to Swift + func makeString(bytes: UnsafeBufferPointer) -> String? { + if encoding == .utf8 || encoding == .ascii, + let str = String._tryFromUTF8(bytes) { + if encoding == .utf8 || (encoding == .ascii && str._guts._isContiguousASCII) { + return str + } + } + + if let ns = NSString( + bytes: bytes.baseAddress.unsafelyUnwrapped, length: bytes.count, encoding: encoding.rawValue) { + return String._unconditionallyBridgeFromObjectiveC(ns) + } else { + return nil + } + } + if let string = (bytes.withContiguousStorageIfAvailable(makeString) ?? + Array(bytes).withUnsafeBufferPointer(makeString)) { + self = string + } else { + return nil + } +#else + guard encoding == .utf8 || encoding == .ascii else { + return nil + } + func makeString(buffer: UnsafeBufferPointer) -> String? { + if let string = String._tryFromUTF8(buffer), + (encoding == .utf8 || (encoding == .ascii && string._guts._isContiguousASCII)) { + return string + } + + return buffer.withMemoryRebound(to: CChar.self) { ptr in + guard let address = ptr.baseAddress else { + return nil + } + return String(validatingUTF8: address) + } + } + + if let string = bytes.withContiguousStorageIfAvailable(makeString) ?? + Array(bytes).withUnsafeBufferPointer(makeString) { + self = string + } else { + return nil + } +#endif // FOUNDATION_FRAMEWORK + } + + // - (instancetype) + // initWithData:(NSData *)data + // encoding:(NSStringEncoding)encoding + + /// Returns a `String` initialized by converting given `data` into + /// Unicode characters using a given `encoding`. + public init?(data: __shared Data, encoding: Encoding) { + if encoding == .utf8 || encoding == .ascii, + let str = data.withUnsafeBytes({ + String._tryFromUTF8($0.bindMemory(to: UInt8.self)) + }) { + if encoding == .utf8 || (encoding == .ascii && str._guts._isContiguousASCII) { + self = str + return + } + } +#if FOUNDATION_FRAMEWORK + guard let s = NSString(data: data, encoding: encoding.rawValue) else { return nil } + self = String._unconditionallyBridgeFromObjectiveC(s) +#else + return nil +#endif // FOUNDATION_FRAMEWORK + } +} + +// MARK: - Stubbed Methods +extension StringProtocol { +#if !FOUNDATION_FRAMEWORK + // - (NSComparisonResult) + // compare:(NSString *)aString + // + // - (NSComparisonResult) + // compare:(NSString *)aString options:(StringCompareOptions)mask + // + // - (NSComparisonResult) + // compare:(NSString *)aString options:(StringCompareOptions)mask + // range:(NSRange)range + // + // - (NSComparisonResult) + // compare:(NSString *)aString options:(StringCompareOptions)mask + // range:(NSRange)range locale:(id)locale + + /// Compares the string using the specified options and + /// returns the lexical ordering for the range. + internal func compare(_ aString: T, options mask: String.CompareOptions = [], range: Range? = nil) -> ComparisonResult { + // TODO: This method is modified from `public func compare(_ aString: T, options mask: String.CompareOptions = [], range: Range? = nil, locale: Locale? = nil) -> ComparisonResult`. Move that method here once `Locale` can be staged in `FoundationEssentials`. + var substr = Substring(self) + if let range { + substr = substr[range] + } + return substr._unlocalizedCompare(other: Substring(aString), options: mask) + } +#endif +} + + +extension Substring.UnicodeScalarView { + func _rangeOfCharacter(from set: CharacterSet, anchored: Bool, backwards: Bool) -> Range? { + guard !isEmpty else { return nil } + + let fromLoc: String.Index + let toLoc: String.Index + let step: Int + if backwards { + fromLoc = index(before: endIndex) + toLoc = anchored ? fromLoc : startIndex + step = -1 + } else { + fromLoc = startIndex + toLoc = anchored ? fromLoc : index(before: endIndex) + step = 1 + } + + var done = false + var found = false + + var idx = fromLoc + while !done { + let ch = self[idx] + if set.contains(ch) { + done = true + found = true + } else if idx == toLoc { + done = true + } else { + formIndex(&idx, offsetBy: step) + } + } + + guard found else { return nil } + return idx.. Self { +#if FOUNDATION_FRAMEWORK // TODO: Implement `CFUniCharCompatibilityDecompose` in Swift + if value >= 0xFF00 && value < 0xFFEF { + var halfWidth = value + CFUniCharCompatibilityDecompose(&halfWidth, 1, 1) + return UnicodeScalar(halfWidth)! + } else { + return self + } +#else + return self +#endif + } + + var _isGraphemeExtend: Bool { +#if FOUNDATION_FRAMEWORK // TODO: Implement `CFUniCharGetBitmapPtrForPlane` in Swift + let truncated = UInt16(truncatingIfNeeded: value) // intentionally truncated + let bitmap = CFUniCharGetBitmapPtrForPlane(UInt32(kCFUniCharGraphemeExtendCharacterSet), (value < 0x10000) ? 0 : (value >> 16)) + return CFUniCharIsMemberOfBitmap(truncated, bitmap) +#else + return false +#endif + } + + var _isCanonicalDecomposable: Bool { +#if FOUNDATION_FRAMEWORK // TODO: Implement `CFUniCharGetBitmapPtrForPlane` in Swift + let truncated = UInt16(truncatingIfNeeded: value) + let bitmap = CFUniCharGetBitmapPtrForPlane(UInt32(kCFUniCharCanonicalDecomposableCharacterSet), value >> 16) + return CFUniCharIsMemberOfBitmap(truncated, bitmap) +#else + return false +#endif + } + + func _stripDiacritics() -> Self { + guard _isCanonicalDecomposable else { + return self + } + +#if FOUNDATION_FRAMEWORK // TODO: Implement `CFUniCharDecomposeCharacter` in Swift + var stripped: UInt32? = nil + withUnsafeTemporaryAllocation(of: UTF32Char.self, capacity: 64) { ptr in + guard let base = ptr.baseAddress else { + return + } + let len = CFUniCharDecomposeCharacter(value, base, ptr.count) + if len > 0 { + if ptr[0] < 0x0510 { + stripped = ptr[0] + } + } + } + + return stripped != nil ? UnicodeScalar(stripped!)! : self +#else + return self +#endif // FOUNDATION_FRAMEWORK + } + + var _caseFoldMapping : String { +#if FOUNDATION_FRAMEWORK // TODO: Expose Case Mapping Data without @_spi(_Unicode) + return self.properties._caseFolded +#else + return "" +#endif + } +} diff --git a/Sources/FoundationInternationalization/Calendar/Calendar_Enumerate.swift b/Sources/FoundationInternationalization/Calendar/Calendar_Enumerate.swift index 8231d5888..3f8e122e8 100644 --- a/Sources/FoundationInternationalization/Calendar/Calendar_Enumerate.swift +++ b/Sources/FoundationInternationalization/Calendar/Calendar_Enumerate.swift @@ -14,18 +14,6 @@ import FoundationEssentials #endif -extension ComparisonResult { - init(_ t1: T, _ t2: T) { - if t1 < t2 { - self = .orderedAscending - } else if t1 > t2 { - self = .orderedDescending - } else { - self = .orderedSame - } - } -} - extension Range where Bound: Comparable { func extended(to other: Range) -> Range { Swift.min(self.lowerBound, other.lowerBound)..(_ t1: T, _ t2: T) { + if t1 < t2 { + self = .orderedAscending + } else if t1 > t2 { + self = .orderedDescending + } else { + self = .orderedSame + } + } +} +#endif diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index 69cc8f812..e2734d9be 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -10,6 +10,10 @@ // //===----------------------------------------------------------------------===// +#if canImport(FoundationEssentials) +import FoundationEssentials +#endif // canImport(FoundationEssentials) + /** `Locale` encapsulates information about linguistic, cultural, and technological conventions and standards. Examples of information encapsulated by a locale include the symbol used for the decimal separator in numbers and the way dates are formatted. diff --git a/Sources/TestSupport/TestSupport.swift b/Sources/TestSupport/TestSupport.swift index ae5a01e90..766ab40bc 100644 --- a/Sources/TestSupport/TestSupport.swift +++ b/Sources/TestSupport/TestSupport.swift @@ -23,6 +23,8 @@ public typealias UUID = Foundation.UUID public typealias Date = Foundation.Date public typealias DateComponents = Foundation.DateComponents public typealias TimeInterval = Foundation.TimeInterval +public typealias JSONEncoder = Foundation.JSONEncoder +public typealias JSONDecoder = Foundation.JSONDecoder // XCTest implicitly imports Foundation @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) @@ -81,6 +83,8 @@ public typealias Data = FoundationEssentials.Data public typealias UUID = FoundationEssentials.UUID public typealias Date = FoundationEssentials.Date public typealias TimeInterval = FoundationEssentials.TimeInterval +public typealias JSONEncoder = FoundationEssentials.JSONEncoder +public typealias JSONDecoder = FoundationEssentials.JSONDecoder @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) public typealias ListFormatStyle = FoundationInternationalization.ListFormatStyle diff --git a/Tests/FoundationEssentialsTests/JSONEncoderTests.swift b/Tests/FoundationEssentialsTests/JSONEncoderTests.swift new file mode 100644 index 000000000..c864fcaeb --- /dev/null +++ b/Tests/FoundationEssentialsTests/JSONEncoderTests.swift @@ -0,0 +1,4017 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// RUN: %target-run-simple-swift +// REQUIRES: executable_test +// REQUIRES: objc_interop +// REQUIRES: rdar49634697 +// REQUIRES: rdar55727144 + +#if canImport(TestSupport) +import TestSupport +#endif // canImport(TestSupport) + +#if canImport(FoundationEssentials) +@testable import FoundationEssentials +#endif + +#if FOUNDATION_FRAMEWORK +@testable import Foundation +#endif + +// MARK: - Test Suite + +final class JSONEncoderTests : XCTestCase { + // MARK: - Encoding Top-Level Empty Types + func testEncodingTopLevelEmptyStruct() { + let empty = EmptyStruct() + _testRoundTrip(of: empty, expectedJSON: _jsonEmptyDictionary) + } + + func testEncodingTopLevelEmptyClass() { + let empty = EmptyClass() + _testRoundTrip(of: empty, expectedJSON: _jsonEmptyDictionary) + } + + // MARK: - Encoding Top-Level Single-Value Types + func testEncodingTopLevelSingleValueEnum() { + _testRoundTrip(of: Switch.off) + _testRoundTrip(of: Switch.on) + } + + func testEncodingTopLevelSingleValueStruct() { + _testRoundTrip(of: Timestamp(3141592653)) + } + + func testEncodingTopLevelSingleValueClass() { + _testRoundTrip(of: Counter()) + } + + // MARK: - Encoding Top-Level Structured Types + func testEncodingTopLevelStructuredStruct() { + // Address is a struct type with multiple fields. + let address = Address.testValue + _testRoundTrip(of: address) + } + + func testEncodingTopLevelStructuredSingleStruct() { + // Numbers is a struct which encodes as an array through a single value container. + let numbers = Numbers.testValue + _testRoundTrip(of: numbers) + } + + func testEncodingTopLevelStructuredSingleClass() { + // Mapping is a class which encodes as a dictionary through a single value container. + let mapping = Mapping.testValue + _testRoundTrip(of: mapping) + } + + func testEncodingTopLevelDeepStructuredType() { + // Company is a type with fields which are Codable themselves. + let company = Company.testValue + _testRoundTrip(of: company) + } + + func testEncodingClassWhichSharesEncoderWithSuper() { + // Employee is a type which shares its encoder & decoder with its superclass, Person. + let employee = Employee.testValue + _testRoundTrip(of: employee) + } + + func testEncodingTopLevelNullableType() { + // EnhancedBool is a type which encodes either as a Bool or as nil. + _testRoundTrip(of: EnhancedBool.true, expectedJSON: "true".data(using: String._Encoding.utf8)!) + _testRoundTrip(of: EnhancedBool.false, expectedJSON: "false".data(using: String._Encoding.utf8)!) + _testRoundTrip(of: EnhancedBool.fileNotFound, expectedJSON: "null".data(using: String._Encoding.utf8)!) + } + +#if false // FIXME: XCTest doesn't support crash tests yet rdar://20195010&22387653 + func testEncodingConflictedTypeNestedContainersWithTheSameTopLevelKey() { + struct Model : Encodable, Equatable { + let first: String + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: TopLevelCodingKeys.self) + + var firstNestedContainer = container.nestedContainer(keyedBy: FirstNestedCodingKeys.self, forKey: .top) + try firstNestedContainer.encode(self.first, forKey: .first) + + // The following line would fail as it attempts to re-encode into already encoded container is invalid. This will always fail + var secondNestedContainer = container.nestedUnkeyedContainer(forKey: .top) + try secondNestedContainer.encode("second") + } + + init(first: String) { + self.first = first + } + + static var testValue: Model { + return Model(first: "Johnny Appleseed") + } + + enum TopLevelCodingKeys : String, CodingKey { + case top + } + enum FirstNestedCodingKeys : String, CodingKey { + case first + } + } + + let model = Model.testValue + // This following test would fail as it attempts to re-encode into already encoded container is invalid. This will always fail + expectCrashLater() + _testEncodeFailure(of: model) + } +#endif + + // MARK: - Date Strategy Tests + + // Disabled for now till we resolve rdar://52618414 + func x_testEncodingDate() { + + func formattedLength(of value: Double) -> Int { + let empty = UnsafeMutablePointer.allocate(capacity: 0) + defer { empty.deallocate() } + let length = snprintf(ptr: empty, 0, "%0.*g", DBL_DECIMAL_DIG, value) + return Int(length) + } + + // Duplicated to handle a special case + func localTestRoundTrip(of value: T) { + var payload: Data! = nil + do { + let encoder = JSONEncoder() + payload = try encoder.encode(value) + } catch { + XCTFail("Failed to encode \(T.self) to JSON: \(error)") + } + + do { + let decoder = JSONDecoder() + let decoded = try decoder.decode(T.self, from: payload) + + /// `snprintf`'s `%g`, which `JSONSerialization` uses internally for double values, does not respect + /// our precision requests in every case. This bug effects Darwin, FreeBSD, and Linux currently + /// causing this test (which uses the current time) to fail occasionally. + if formattedLength(of: (decoded as! Date).timeIntervalSinceReferenceDate) > DBL_DECIMAL_DIG + 2 { + let adjustedTimeIntervalSinceReferenceDate: (Date) -> Double = { date in + let adjustment = pow(10, Double(DBL_DECIMAL_DIG)) + return Double(floor(adjustment * date.timeIntervalSinceReferenceDate).rounded() / adjustment) + } + + let decodedAprox = adjustedTimeIntervalSinceReferenceDate(decoded as! Date) + let valueAprox = adjustedTimeIntervalSinceReferenceDate(value as! Date) + XCTAssertEqual(decodedAprox, valueAprox, "\(T.self) did not round-trip to an equal value after DBL_DECIMAL_DIG adjustment \(decodedAprox) != \(valueAprox).") + return + } + + XCTAssertEqual(decoded, value, "\(T.self) did not round-trip to an equal value. \((decoded as! Date).timeIntervalSinceReferenceDate) != \((value as! Date).timeIntervalSinceReferenceDate)") + } catch { + XCTFail("Failed to decode \(T.self) from JSON: \(error)") + } + } + + // Test the above `snprintf` edge case evaluation with a known triggering case + let knownBadDate = Date(timeIntervalSinceReferenceDate: 0.0021413276231263384) + localTestRoundTrip(of: knownBadDate) + + localTestRoundTrip(of: Date()) + + // Optional dates should encode the same way. + localTestRoundTrip(of: Optional(Date())) + } + + func testEncodingDateSecondsSince1970() { + // Cannot encode an arbitrary number of seconds since we've lost precision since 1970. + let seconds = 1000.0 + let expectedJSON = "1000".data(using: String._Encoding.utf8)! + + _testRoundTrip(of: Date(timeIntervalSince1970: seconds), + expectedJSON: expectedJSON, + dateEncodingStrategy: .secondsSince1970, + dateDecodingStrategy: .secondsSince1970) + + // Optional dates should encode the same way. + _testRoundTrip(of: Optional(Date(timeIntervalSince1970: seconds)), + expectedJSON: expectedJSON, + dateEncodingStrategy: .secondsSince1970, + dateDecodingStrategy: .secondsSince1970) + } + + func testEncodingDateMillisecondsSince1970() { + // Cannot encode an arbitrary number of seconds since we've lost precision since 1970. + let seconds = 1000.0 + let expectedJSON = "1000000".data(using: String._Encoding.utf8)! + + _testRoundTrip(of: Date(timeIntervalSince1970: seconds), + expectedJSON: expectedJSON, + dateEncodingStrategy: .millisecondsSince1970, + dateDecodingStrategy: .millisecondsSince1970) + + // Optional dates should encode the same way. + _testRoundTrip(of: Optional(Date(timeIntervalSince1970: seconds)), + expectedJSON: expectedJSON, + dateEncodingStrategy: .millisecondsSince1970, + dateDecodingStrategy: .millisecondsSince1970) + } + + func testEncodingDateCustom() { + let timestamp = Date() + + // We'll encode a number instead of a date. + let encode = { (_ data: Date, _ encoder: Encoder) throws -> Void in + var container = encoder.singleValueContainer() + try container.encode(42) + } + let decode = { (_: Decoder) throws -> Date in return timestamp } + + let expectedJSON = "42".data(using: String._Encoding.utf8)! + _testRoundTrip(of: timestamp, + expectedJSON: expectedJSON, + dateEncodingStrategy: .custom(encode), + dateDecodingStrategy: .custom(decode)) + + // Optional dates should encode the same way. + _testRoundTrip(of: Optional(timestamp), + expectedJSON: expectedJSON, + dateEncodingStrategy: .custom(encode), + dateDecodingStrategy: .custom(decode)) + } + + func testEncodingDateCustomEmpty() { + let timestamp = Date() + + // Encoding nothing should encode an empty keyed container ({}). + let encode = { (_: Date, _: Encoder) throws -> Void in } + let decode = { (_: Decoder) throws -> Date in return timestamp } + + let expectedJSON = "{}".data(using: String._Encoding.utf8)! + _testRoundTrip(of: timestamp, + expectedJSON: expectedJSON, + dateEncodingStrategy: .custom(encode), + dateDecodingStrategy: .custom(decode)) + + // Optional dates should encode the same way. + _testRoundTrip(of: Optional(timestamp), + expectedJSON: expectedJSON, + dateEncodingStrategy: .custom(encode), + dateDecodingStrategy: .custom(decode)) + } + + // MARK: - Data Strategy Tests + func testEncodingData() { + let data = Data([0xDE, 0xAD, 0xBE, 0xEF]) + + let expectedJSON = "[222,173,190,239]".data(using: String._Encoding.utf8)! + _testRoundTrip(of: data, + expectedJSON: expectedJSON, + dataEncodingStrategy: .deferredToData, + dataDecodingStrategy: .deferredToData) + + // Optional data should encode the same way. + _testRoundTrip(of: Optional(data), + expectedJSON: expectedJSON, + dataEncodingStrategy: .deferredToData, + dataDecodingStrategy: .deferredToData) + } + + func testEncodingDataCustom() { + // We'll encode a number instead of data. + let encode = { (_ data: Data, _ encoder: Encoder) throws -> Void in + var container = encoder.singleValueContainer() + try container.encode(42) + } + let decode = { (_: Decoder) throws -> Data in return Data() } + + let expectedJSON = "42".data(using: String._Encoding.utf8)! + _testRoundTrip(of: Data(), + expectedJSON: expectedJSON, + dataEncodingStrategy: .custom(encode), + dataDecodingStrategy: .custom(decode)) + + // Optional data should encode the same way. + _testRoundTrip(of: Optional(Data()), + expectedJSON: expectedJSON, + dataEncodingStrategy: .custom(encode), + dataDecodingStrategy: .custom(decode)) + } + + func testEncodingDataCustomEmpty() { + // Encoding nothing should encode an empty keyed container ({}). + let encode = { (_: Data, _: Encoder) throws -> Void in } + let decode = { (_: Decoder) throws -> Data in return Data() } + + let expectedJSON = "{}".data(using: String._Encoding.utf8)! + _testRoundTrip(of: Data(), + expectedJSON: expectedJSON, + dataEncodingStrategy: .custom(encode), + dataDecodingStrategy: .custom(decode)) + + // Optional Data should encode the same way. + _testRoundTrip(of: Optional(Data()), + expectedJSON: expectedJSON, + dataEncodingStrategy: .custom(encode), + dataDecodingStrategy: .custom(decode)) + } + + // MARK: - Non-Conforming Floating Point Strategy Tests + func testEncodingNonConformingFloats() { + _testEncodeFailure(of: Float.infinity) + _testEncodeFailure(of: Float.infinity) + _testEncodeFailure(of: -Float.infinity) + _testEncodeFailure(of: Float.nan) + + _testEncodeFailure(of: Double.infinity) + _testEncodeFailure(of: -Double.infinity) + _testEncodeFailure(of: Double.nan) + + // Optional Floats/Doubles should encode the same way. + _testEncodeFailure(of: Float.infinity) + _testEncodeFailure(of: -Float.infinity) + _testEncodeFailure(of: Float.nan) + + _testEncodeFailure(of: Double.infinity) + _testEncodeFailure(of: -Double.infinity) + _testEncodeFailure(of: Double.nan) + } + + func testEncodingNonConformingFloatStrings() { + let encodingStrategy: JSONEncoder.NonConformingFloatEncodingStrategy = .convertToString(positiveInfinity: "INF", negativeInfinity: "-INF", nan: "NaN") + let decodingStrategy: JSONDecoder.NonConformingFloatDecodingStrategy = .convertFromString(positiveInfinity: "INF", negativeInfinity: "-INF", nan: "NaN") + + _testRoundTrip(of: Float.infinity, + expectedJSON: "\"INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + _testRoundTrip(of: -Float.infinity, + expectedJSON: "\"-INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + + // Since Float.nan != Float.nan, we have to use a placeholder that'll encode NaN but actually round-trip. + _testRoundTrip(of: FloatNaNPlaceholder(), + expectedJSON: "\"NaN\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + + _testRoundTrip(of: Double.infinity, + expectedJSON: "\"INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + _testRoundTrip(of: -Double.infinity, + expectedJSON: "\"-INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + + // Since Double.nan != Double.nan, we have to use a placeholder that'll encode NaN but actually round-trip. + _testRoundTrip(of: DoubleNaNPlaceholder(), + expectedJSON: "\"NaN\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + + // Optional Floats and Doubles should encode the same way. + _testRoundTrip(of: Optional(Float.infinity), + expectedJSON: "\"INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + _testRoundTrip(of: Optional(-Float.infinity), + expectedJSON: "\"-INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + _testRoundTrip(of: Optional(Double.infinity), + expectedJSON: "\"INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + _testRoundTrip(of: Optional(-Double.infinity), + expectedJSON: "\"-INF\"".data(using: String._Encoding.utf8)!, + nonConformingFloatEncodingStrategy: encodingStrategy, + nonConformingFloatDecodingStrategy: decodingStrategy) + } + + // MARK: - Key Strategy Tests + private struct EncodeMe : Encodable { + var keyName: String + func encode(to coder: Encoder) throws { + var c = coder.container(keyedBy: _TestKey.self) + try c.encode("test", forKey: _TestKey(stringValue: keyName)!) + } + } + + func testEncodingKeyStrategyCustom() { + let expected = "{\"QQQhello\":\"test\"}" + let encoded = EncodeMe(keyName: "hello") + + let encoder = JSONEncoder() + let customKeyConversion = { (_ path : [CodingKey]) -> CodingKey in + let key = _TestKey(stringValue: "QQQ" + path.last!.stringValue)! + return key + } + encoder.keyEncodingStrategy = .custom(customKeyConversion) + let resultData = try! encoder.encode(encoded) + let resultString = String(bytes: resultData, encoding: String._Encoding.utf8) + + XCTAssertEqual(expected, resultString) + } + + private struct EncodeFailure : Encodable { + var someValue: Double + } + + private struct EncodeFailureNested : Encodable { + var nestedValue: EncodeFailure + } + + private struct EncodeNested : Encodable { + let nestedValue: EncodeMe + } + + private struct EncodeNestedNested : Encodable { + let outerValue: EncodeNested + } + + func testEncodingKeyStrategyPath() { + // Make sure a more complex path shows up the way we want + // Make sure the path reflects keys in the Swift, not the resulting ones in the JSON + let expected = "{\"QQQouterValue\":{\"QQQnestedValue\":{\"QQQhelloWorld\":\"test\"}}}" + let encoded = EncodeNestedNested(outerValue: EncodeNested(nestedValue: EncodeMe(keyName: "helloWorld"))) + + let encoder = JSONEncoder() + var callCount = 0 + + let customKeyConversion = { (_ path : [CodingKey]) -> CodingKey in + // This should be called three times: + // 1. to convert 'outerValue' to something + // 2. to convert 'nestedValue' to something + // 3. to convert 'helloWorld' to something + callCount = callCount + 1 + + if path.count == 0 { + XCTFail("The path should always have at least one entry") + } else if path.count == 1 { + XCTAssertEqual(["outerValue"], path.map { $0.stringValue }) + } else if path.count == 2 { + XCTAssertEqual(["outerValue", "nestedValue"], path.map { $0.stringValue }) + } else if path.count == 3 { + XCTAssertEqual(["outerValue", "nestedValue", "helloWorld"], path.map { $0.stringValue }) + } else { + XCTFail("The path mysteriously had more entries") + } + + let key = _TestKey(stringValue: "QQQ" + path.last!.stringValue)! + return key + } + encoder.keyEncodingStrategy = .custom(customKeyConversion) + let resultData = try! encoder.encode(encoded) + let resultString = String(bytes: resultData, encoding: String._Encoding.utf8) + + XCTAssertEqual(expected, resultString) + XCTAssertEqual(3, callCount) + } + + private struct DecodeMe : Decodable { + let found: Bool + init(from coder: Decoder) throws { + let c = try coder.container(keyedBy: _TestKey.self) + // Get the key that we expect to be passed in (camel case) + let camelCaseKey = try c.decode(String.self, forKey: _TestKey(stringValue: "camelCaseKey")!) + + // Use the camel case key to decode from the JSON. The decoder should convert it to snake case to find it. + found = try c.decode(Bool.self, forKey: _TestKey(stringValue: camelCaseKey)!) + } + } + + private struct DecodeMe2 : Decodable { var hello: String } + + func testDecodingKeyStrategyCustom() { + let input = "{\"----hello\":\"test\"}".data(using: String._Encoding.utf8)! + let decoder = JSONDecoder() + let customKeyConversion = { (_ path: [CodingKey]) -> CodingKey in + // This converter removes the first 4 characters from the start of all string keys, if it has more than 4 characters + let string = path.last!.stringValue + guard string.count > 4 else { return path.last! } + let newString = String(string.dropFirst(4)) + return _TestKey(stringValue: newString)! + } + decoder.keyDecodingStrategy = .custom(customKeyConversion) + let result = try! decoder.decode(DecodeMe2.self, from: input) + + XCTAssertEqual("test", result.hello) + } + + func testDecodingDictionaryStringKeyConversionUntouched() { + let input = "{\"leave_me_alone\":\"test\"}".data(using: String._Encoding.utf8)! + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + let result = try! decoder.decode([String: String].self, from: input) + + XCTAssertEqual(["leave_me_alone": "test"], result) + } + + func testDecodingDictionaryFailureKeyPath() { + let input = "{\"leave_me_alone\":\"test\"}".data(using: String._Encoding.utf8)! + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + do { + _ = try decoder.decode([String: Int].self, from: input) + } catch DecodingError.typeMismatch(_, let context) { + XCTAssertEqual(1, context.codingPath.count) + XCTAssertEqual("leave_me_alone", context.codingPath[0].stringValue) + } catch { + XCTFail("Unexpected error: \(String(describing: error))") + } + } + + private struct DecodeFailure : Decodable { + var intValue: Int + } + + private struct DecodeFailureNested : Decodable { + var nestedValue: DecodeFailure + } + + private struct DecodeMe3 : Codable { + var thisIsCamelCase : String + } + + func testKeyStrategyDuplicateKeys() { + // This test is mostly to make sure we don't assert on duplicate keys + struct DecodeMe5 : Codable { + var oneTwo : String + var numberOfKeys : Int + + enum CodingKeys : String, CodingKey { + case oneTwo + case oneTwoThree + } + + init() { + oneTwo = "test" + numberOfKeys = 0 + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + oneTwo = try container.decode(String.self, forKey: .oneTwo) + numberOfKeys = container.allKeys.count + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(oneTwo, forKey: .oneTwo) + try container.encode("test2", forKey: .oneTwoThree) + } + } + + let customKeyConversion = { (_ path: [CodingKey]) -> CodingKey in + // All keys are the same! + return _TestKey(stringValue: "oneTwo")! + } + + // Decoding + // This input has a dictionary with two keys, but only one will end up in the container + let input = "{\"unused key 1\":\"test1\",\"unused key 2\":\"test2\"}".data(using: String._Encoding.utf8)! + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .custom(customKeyConversion) + + let decodingResult = try! decoder.decode(DecodeMe5.self, from: input) + // There will be only one result for oneTwo. + XCTAssertEqual(1, decodingResult.numberOfKeys) + // While the order in which these values should be taken is NOT defined by the JSON spec in any way, the historical behavior has been to select the *first* value for a given key. + XCTAssertEqual(decodingResult.oneTwo, "test1") + + // Encoding + let encoded = DecodeMe5() + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .custom(customKeyConversion) + let decodingResultData = try! encoder.encode(encoded) + let decodingResultString = String(bytes: decodingResultData, encoding: String._Encoding.utf8) + + // There will be only one value in the result (the second one encoded) + XCTAssertEqual("{\"oneTwo\":\"test2\"}", decodingResultString) + } + + // MARK: - Encoder Features + func testNestedContainerCodingPaths() { + let encoder = JSONEncoder() + do { + let _ = try encoder.encode(NestedContainersTestType()) + } catch let error as NSError { + XCTFail("Caught error during encoding nested container types: \(error)") + } + } + + func testSuperEncoderCodingPaths() { + let encoder = JSONEncoder() + do { + let _ = try encoder.encode(NestedContainersTestType(testSuperEncoder: true)) + } catch let error as NSError { + XCTFail("Caught error during encoding nested container types: \(error)") + } + } + + // MARK: - Type coercion + func testTypeCoercion() { + _testRoundTripTypeCoercionFailure(of: [false, true], as: [Int].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [Int8].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [Int16].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [Int32].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [Int64].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [UInt].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [UInt8].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [UInt16].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [UInt32].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [UInt64].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [Float].self) + _testRoundTripTypeCoercionFailure(of: [false, true], as: [Double].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [Int], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [Int8], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [Int16], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [Int32], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [Int64], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [UInt], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [UInt8], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [UInt16], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [UInt32], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0, 1] as [UInt64], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0.0, 1.0] as [Float], as: [Bool].self) + _testRoundTripTypeCoercionFailure(of: [0.0, 1.0] as [Double], as: [Bool].self) + } + + func testDecodingConcreteTypeParameter() { + let encoder = JSONEncoder() + guard let json = try? encoder.encode(Employee.testValue) else { + XCTFail("Unable to encode Employee.") + return + } + + let decoder = JSONDecoder() + guard let decoded = try? decoder.decode(Employee.self as Person.Type, from: json) else { + XCTFail("Failed to decode Employee as Person from JSON.") + return + } + + expectEqual(type(of: decoded), Employee.self, "Expected decoded value to be of type Employee; got \(type(of: decoded)) instead.") + } + + // MARK: - Encoder State + // SR-6078 + func testEncoderStateThrowOnEncode() { + struct ReferencingEncoderWrapper : Encodable { + let value: T + init(_ value: T) { self.value = value } + + func encode(to encoder: Encoder) throws { + // This approximates a subclass calling into its superclass, where the superclass encodes a value that might throw. + // The key here is that getting the superEncoder creates a referencing encoder. + var container = encoder.unkeyedContainer() + let superEncoder = container.superEncoder() + + // Pushing a nested container on leaves the referencing encoder with multiple containers. + var nestedContainer = superEncoder.unkeyedContainer() + try nestedContainer.encode(value) + } + } + + // The structure that would be encoded here looks like + // + // [[[Float.infinity]]] + // + // The wrapper asks for an unkeyed container ([^]), gets a super encoder, and creates a nested container into that ([[^]]). + // We then encode an array into that ([[[^]]]), which happens to be a value that causes us to throw an error. + // + // The issue at hand reproduces when you have a referencing encoder (superEncoder() creates one) that has a container on the stack (unkeyedContainer() adds one) that encodes a value going through box_() (Array does that) that encodes something which throws (Float.infinity does that). + // When reproducing, this will cause a test failure via fatalError(). + _ = try? JSONEncoder().encode(ReferencingEncoderWrapper([Float.infinity])) + } + + func testEncoderStateThrowOnEncodeCustomDate() { + // This test is identical to testEncoderStateThrowOnEncode, except throwing via a custom Date closure. + struct ReferencingEncoderWrapper : Encodable { + let value: T + init(_ value: T) { self.value = value } + func encode(to encoder: Encoder) throws { + var container = encoder.unkeyedContainer() + let superEncoder = container.superEncoder() + var nestedContainer = superEncoder.unkeyedContainer() + try nestedContainer.encode(value) + } + } + + // The closure needs to push a container before throwing an error to trigger. + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .custom({ _, encoder in + let _ = encoder.unkeyedContainer() + enum CustomError : Error { case foo } + throw CustomError.foo + }) + + _ = try? encoder.encode(ReferencingEncoderWrapper(Date())) + } + + func testEncoderStateThrowOnEncodeCustomData() { + // This test is identical to testEncoderStateThrowOnEncode, except throwing via a custom Data closure. + struct ReferencingEncoderWrapper : Encodable { + let value: T + init(_ value: T) { self.value = value } + func encode(to encoder: Encoder) throws { + var container = encoder.unkeyedContainer() + let superEncoder = container.superEncoder() + var nestedContainer = superEncoder.unkeyedContainer() + try nestedContainer.encode(value) + } + } + + // The closure needs to push a container before throwing an error to trigger. + let encoder = JSONEncoder() + encoder.dataEncodingStrategy = .custom({ _, encoder in + let _ = encoder.unkeyedContainer() + enum CustomError : Error { case foo } + throw CustomError.foo + }) + + _ = try? encoder.encode(ReferencingEncoderWrapper(Data())) + } + + func test_106506794() throws { + struct Level1: Codable, Equatable { + let level2: Level2 + + enum CodingKeys: String, CodingKey { + case level2 + } + + func encode(to encoder: Encoder) throws { + var keyed = encoder.container(keyedBy: Self.CodingKeys.self) + var nested = keyed.nestedUnkeyedContainer(forKey: .level2) + try nested.encode(level2) + } + + init(from decoder: Decoder) throws { + let keyed = try decoder.container(keyedBy: Self.CodingKeys.self) + var nested = try keyed.nestedUnkeyedContainer(forKey: .level2) + self.level2 = try nested.decode(Level2.self) + } + + struct Level2: Codable, Equatable { + let name : String + } + + init(level2: Level2) { + self.level2 = level2 + } + } + + let value = Level1.init(level2: .init(name: "level2")) + let data = try JSONEncoder().encode(value) + + do { + let decodedValue = try JSONDecoder().decode(Level1.self, from: data) + XCTAssertEqual(value, decodedValue) + } catch { + XCTFail("Decode should not have failed with error: \(error))") + } + } + + // MARK: - Decoder State + // SR-6048 + func testDecoderStateThrowOnDecode() { + // The container stack here starts as [[1,2,3]]. Attempting to decode as [String] matches the outer layer (Array), and begins decoding the array. + // Once Array decoding begins, 1 is pushed onto the container stack ([[1,2,3], 1]), and 1 is attempted to be decoded as String. This throws a .typeMismatch, but the container is not popped off the stack. + // When attempting to decode [Int], the container stack is still ([[1,2,3], 1]), and 1 fails to decode as [Int]. + let json = "[1,2,3]".data(using: String._Encoding.utf8)! + let _ = try! JSONDecoder().decode(EitherDecodable<[String], [Int]>.self, from: json) + } + + func testDecoderStateThrowOnDecodeCustomDate() { + // This test is identical to testDecoderStateThrowOnDecode, except we're going to fail because our closure throws an error, not because we hit a type mismatch. + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .custom({ decoder in + enum CustomError : Error { case foo } + throw CustomError.foo + }) + + let json = "1".data(using: String._Encoding.utf8)! + let _ = try! decoder.decode(EitherDecodable.self, from: json) + } + + func testDecoderStateThrowOnDecodeCustomData() { + // This test is identical to testDecoderStateThrowOnDecode, except we're going to fail because our closure throws an error, not because we hit a type mismatch. + let decoder = JSONDecoder() + decoder.dataDecodingStrategy = .custom({ decoder in + enum CustomError : Error { case foo } + throw CustomError.foo + }) + + let json = "1".data(using: String._Encoding.utf8)! + let _ = try! decoder.decode(EitherDecodable.self, from: json) + } + + + func testDecodingFailure() { + struct DecodeFailure : Decodable { + var invalid: String + } + let toDecode = "{\"invalid\": json}"; + _testDecodeFailure(of: DecodeFailure.self, data: toDecode.data(using: String._Encoding.utf8)!) + } + + func testDecodingFailureThrowInInitKeyedContainer() { + struct DecodeFailure : Decodable { + private enum CodingKeys: String, CodingKey { + case checkedString + } + + private enum Error: Swift.Error { + case expectedError + } + + var checkedString: String + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let string = try container.decode(String.self, forKey: .checkedString) + guard string == "foo" else { + throw Error.expectedError + } + self.checkedString = string // shouldn't happen + } + } + + let toDecode = "{ \"checkedString\" : \"baz\" }" + _testDecodeFailure(of: DecodeFailure.self, data: toDecode.data(using: String._Encoding.utf8)!) + } + + func testDecodingFailureThrowInInitSingleContainer() { + struct DecodeFailure : Decodable { + private enum Error: Swift.Error { + case expectedError + } + + var checkedString: String + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let string = try container.decode(String.self) + guard string == "foo" else { + throw Error.expectedError + } + self.checkedString = string // shouldn't happen + } + } + + let toDecode = "{ \"checkedString\" : \"baz\" }" + _testDecodeFailure(of: DecodeFailure.self, data: toDecode.data(using: String._Encoding.utf8)!) + } + + func testInvalidFragment() { + struct DecodeFailure: Decodable { + var foo: String + } + let toDecode = "\"foo" + _testDecodeFailure(of: DecodeFailure.self, data: toDecode.data(using: String._Encoding.utf8)!) + } + + func testRepeatedFailedNilChecks() { + struct RepeatNilCheckDecodable : Decodable { + enum Failure : Error { + case badNil + case badValue(expected: Int, actual: Int) + } + + init(from decoder: Decoder) throws { + var unkeyedContainer = try decoder.unkeyedContainer() + + guard try unkeyedContainer.decodeNil() == false else { + throw Failure.badNil + } + guard try unkeyedContainer.decodeNil() == false else { + throw Failure.badNil + } + let value = try unkeyedContainer.decode(Int.self) + guard value == 1 else { + throw Failure.badValue(expected: 1, actual: value) + } + + guard try unkeyedContainer.decodeNil() == false else { + throw Failure.badNil + } + guard try unkeyedContainer.decodeNil() == false else { + throw Failure.badNil + } + let value2 = try unkeyedContainer.decode(Int.self) + guard value2 == 2 else { + throw Failure.badValue(expected: 2, actual: value2) + } + + guard try unkeyedContainer.decodeNil() == false else { + throw Failure.badNil + } + guard try unkeyedContainer.decodeNil() == false else { + throw Failure.badNil + } + let value3 = try unkeyedContainer.decode(Int.self) + guard value3 == 3 else { + throw Failure.badValue(expected: 3, actual: value3) + } + } + } + let json = "[1, 2, 3]".data(using: String._Encoding.utf8)! + XCTAssertNoThrow(try JSONDecoder().decode(RepeatNilCheckDecodable.self, from: json)) + } + + func testDelayedDecoding() throws { + + // One variation is deferring the use of a container. + struct DelayedDecodable_ContainerVersion : Codable { + var _i : Int? = nil + init(_ i: Int) { + self._i = i + } + + func encode(to encoder: Encoder) throws { + var c = encoder.unkeyedContainer() + try c.encode(_i!) + } + + var cont : UnkeyedDecodingContainer? = nil + init(from decoder: Decoder) throws { + cont = try decoder.unkeyedContainer() + } + + var i : Int { + get throws { + if let i = _i { + return i + } + var contCopy = cont! + return try contCopy.decode(Int.self) + } + } + } + + let before = DelayedDecodable_ContainerVersion(42) + let data = try JSONEncoder().encode(before) + + let decoded = try JSONDecoder().decode(DelayedDecodable_ContainerVersion.self, from: data) + XCTAssertNoThrow(try decoded.i) + + // The other variant is deferring the use of the *top-level* decoder. This does NOT work for non-top level decoders. + struct DelayedDecodable_DecoderVersion : Codable { + var _i : Int? = nil + init(_ i: Int) { + self._i = i + } + + func encode(to encoder: Encoder) throws { + var c = encoder.unkeyedContainer() + try c.encode(_i!) + } + + var decoder : Decoder? = nil + init(from decoder: Decoder) throws { + self.decoder = decoder + } + + var i : Int { + get throws { + if let i = _i { + return i + } + var unkeyed = try decoder!.unkeyedContainer() + return try unkeyed.decode(Int.self) + } + } + } + // Reuse the same data. + let decoded2 = try JSONDecoder().decode(DelayedDecodable_DecoderVersion.self, from: data) + XCTAssertNoThrow(try decoded2.i) + } + + // MARK: - Helper Functions + private var _jsonEmptyDictionary: Data { + return "{}".data(using: String._Encoding.utf8)! + } + + private func _testEncodeFailure(of value: T) { + do { + let _ = try JSONEncoder().encode(value) + XCTFail("Encode of top-level \(T.self) was expected to fail.") + } catch { + XCTAssertNotNil(error); + } + } + + private func _testDecodeFailure(of value: T.Type, data: Data) { + do { + let _ = try JSONDecoder().decode(value, from: data) + XCTFail("Decode of top-level \(value) was expected to fail.") + } catch { + XCTAssertNotNil(error); + } + } + + private func _testRoundTrip(of value: T, + expectedJSON json: Data? = nil, + outputFormatting: JSONEncoder.OutputFormatting = [], + dateEncodingStrategy: JSONEncoder.DateEncodingStrategy = .deferredToDate, + dateDecodingStrategy: JSONDecoder.DateDecodingStrategy = .deferredToDate, + dataEncodingStrategy: JSONEncoder.DataEncodingStrategy = .base64, + dataDecodingStrategy: JSONDecoder.DataDecodingStrategy = .base64, + keyEncodingStrategy: JSONEncoder.KeyEncodingStrategy = .useDefaultKeys, + keyDecodingStrategy: JSONDecoder.KeyDecodingStrategy = .useDefaultKeys, + nonConformingFloatEncodingStrategy: JSONEncoder.NonConformingFloatEncodingStrategy = .throw, + nonConformingFloatDecodingStrategy: JSONDecoder.NonConformingFloatDecodingStrategy = .throw) where T : Codable, T : Equatable { + var payload: Data! = nil + do { + let encoder = JSONEncoder() + encoder.outputFormatting = outputFormatting + encoder.dateEncodingStrategy = dateEncodingStrategy + encoder.dataEncodingStrategy = dataEncodingStrategy + encoder.nonConformingFloatEncodingStrategy = nonConformingFloatEncodingStrategy + encoder.keyEncodingStrategy = keyEncodingStrategy + payload = try encoder.encode(value) + } catch { + XCTFail("Failed to encode \(T.self) to JSON: \(error)") + } + + if let expectedJSON = json { + XCTAssertEqual(expectedJSON, payload, "Produced JSON not identical to expected JSON.") + } + + do { + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = dateDecodingStrategy + decoder.dataDecodingStrategy = dataDecodingStrategy + decoder.nonConformingFloatDecodingStrategy = nonConformingFloatDecodingStrategy + decoder.keyDecodingStrategy = keyDecodingStrategy + let decoded = try decoder.decode(T.self, from: payload) + XCTAssertEqual(decoded, value, "\(T.self) did not round-trip to an equal value.") + } catch { + XCTFail("Failed to decode \(T.self) from JSON: \(error)") + } + } + + private func _testRoundTripTypeCoercionFailure(of value: T, as type: U.Type) where T : Codable, U : Codable { + do { + let data = try JSONEncoder().encode(value) + let _ = try JSONDecoder().decode(U.self, from: data) + XCTFail("Coercion from \(T.self) to \(U.self) was expected to fail.") + } catch {} + } + + private func _test(JSONString: String, to object: T) { +#if FOUNDATION_FRAMEWORK + let encs : [String._Encoding] = [.utf8, .utf16BigEndian, .utf16LittleEndian, .utf32BigEndian, .utf32LittleEndian] +#else + // TODO: Reenable other encoding once string.data(using:) is fully implemented. + let encs: [String._Encoding] = [.utf8] +#endif + let decoder = JSONDecoder() + for enc in encs { + let data = JSONString.data(using: enc)! + let parsed : T + do { + parsed = try decoder.decode(T.self, from: data) + } catch { + XCTFail("Failed to decode \(JSONString) with encoding \(enc): Error: \(error)") + continue + } + XCTAssertEqual(object, parsed) + } + } + + func test_JSONEscapedSlashes() { + _test(JSONString: "\"\\/test\\/path\"", to: "/test/path") + _test(JSONString: "\"\\\\/test\\\\/path\"", to: "\\/test\\/path") + } + + func test_JSONEscapedForwardSlashes() { + _testRoundTrip(of: ["/":1], expectedJSON: +""" +{"\\/":1} +""".data(using: String._Encoding.utf8)!) + } + + func test_JSONUnicodeCharacters() { + // UTF8: + // E9 96 86 E5 B4 AC EB B0 BA EB 80 AB E9 A2 92 + // 閆崬밺뀫颒 + _test(JSONString: "[\"閆崬밺뀫颒\"]", to: ["閆崬밺뀫颒"]) + _test(JSONString: "[\"本日\"]", to: ["本日"]) + } + + func test_JSONUnicodeEscapes() { + let testCases = [ + // e-acute and greater-than-or-equal-to + "\"\\u00e9\\u2265\"" : "é≥", + + // e-acute and greater-than-or-equal-to, surrounded by 42 + "\"42\\u00e942\\u226542\"" : "42é42≥42", + + // e-acute with upper-case hex + "\"\\u00E9\"" : "é", + + // G-clef (UTF16 surrogate pair) 0x1D11E + "\"\\uD834\\uDD1E\"" : "𝄞" + ] + for (input, expectedOutput) in testCases { + _test(JSONString: input, to: expectedOutput) + } + } + + func test_JSONBadUnicodeEscapes() { + let badCases = ["\\uD834", "\\uD834hello", "hello\\uD834", "\\uD834\\u1221", "\\uD8", "\\uD834x\\uDD1E"] + for str in badCases { + let data = str.data(using: String._Encoding.utf8)! + XCTAssertThrowsError(try JSONDecoder().decode(String.self, from: data)) + } + } + + func test_superfluouslyEscapedCharacters() { + let json = "[\"\\h\\e\\l\\l\\o\"]" + XCTAssertThrowsError(try JSONDecoder().decode([String].self, from: json.data(using: String._Encoding.utf8)!)) + } + + func test_equivalentUTF8Sequences() { + let json = +""" +{ + "caf\\u00e9" : true, + "cafe\\u0301" : false +} +""".data(using: String._Encoding.utf8)! + + do { + let dict = try JSONDecoder().decode([String:Bool].self, from: json) + XCTAssertEqual(dict.count, 1) + } catch { + XCTFail("Unexpected error: \(error)") + } + } + + func test_JSONControlCharacters() { + let array = [ + "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", + "\\u0005", "\\u0006", "\\u0007", "\\b", "\\t", + "\\n", "\\u000b", "\\f", "\\r", "\\u000e", + "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", + "\\u0014", "\\u0015", "\\u0016", "\\u0017", "\\u0018", + "\\u0019", "\\u001a", "\\u001b", "\\u001c", "\\u001d", + "\\u001e", "\\u001f", " " + ] + for (ascii, json) in zip(0...0x20, array) { + let quotedJSON = "\"\(json)\"" + let expectedResult = String(Character(UnicodeScalar(ascii)!)) + _test(JSONString: quotedJSON, to: expectedResult) + } + } + + func test_JSONNumberFragments() { + let array = ["0 ", "1.0 ", "0.1 ", "1e3 ", "-2.01e-3 ", "0", "1.0", "1e3", "-2.01e-3"] + let expected = [0, 1.0, 0.1, 1000, -0.00201, 0, 1.0, 1000, -0.00201] + for (json, expected) in zip(array, expected) { + _test(JSONString: json, to: expected) + } + } + + func test_invalidJSONNumbersFailAsExpected() { + let array = ["0.", "1e ", "-2.01e- ", "+", "2.01e-1234", "+2.0q", "2s", "NaN", "nan", "Infinity", "inf", "-", "0x42", "1.e2"] + for json in array { + let data = json.data(using: String._Encoding.utf8)! + XCTAssertThrowsError(try JSONDecoder().decode(Float.self, from: data), "Expected error for input \"\(json)\"") + } + } + + func _checkExpectedThrownDataCorruptionUnderlyingError(contains substring: String, closure: () throws -> Void) { + do { + try closure() + XCTFail("Expected failure containing string: \"\(substring)\"") + } catch let error as DecodingError { + guard case let .dataCorrupted(context) = error else { + XCTFail("Unexpected DecodingError type: \(error)") + return + } +#if FOUNDATION_FRAMEWORK + let nsError = context.underlyingError! as NSError + XCTAssertTrue(nsError.debugDescription.contains(substring), "Description \"\(nsError.debugDescription)\" doesn't contain substring \"\(substring)\"") +#endif + } catch { + XCTFail("Unexpected error type: \(error)") + } + } + + func test_topLevelFragmentsWithGarbage() { + _checkExpectedThrownDataCorruptionUnderlyingError(contains: "Unexpected character") { + let _ = try JSONDecoder().decode(Bool.self, from: "tru_".data(using: String._Encoding.utf8)!) + let _ = try json5Decoder.decode(Bool.self, from: "tru_".data(using: String._Encoding.utf8)!) + } + _checkExpectedThrownDataCorruptionUnderlyingError(contains: "Unexpected character") { + let _ = try JSONDecoder().decode(Bool.self, from: "fals_".data(using: String._Encoding.utf8)!) + let _ = try json5Decoder.decode(Bool.self, from: "fals_".data(using: String._Encoding.utf8)!) + } + _checkExpectedThrownDataCorruptionUnderlyingError(contains: "Unexpected character") { + let _ = try JSONDecoder().decode(Bool?.self, from: "nul_".data(using: String._Encoding.utf8)!) + let _ = try json5Decoder.decode(Bool?.self, from: "nul_".data(using: String._Encoding.utf8)!) + } + } + + func test_topLevelNumberFragmentsWithJunkDigitCharacters() { + let fullData = "3.141596".data(using: .utf8)! + let partialData = fullData[0..<4] + + XCTAssertEqual(3.14, try JSONDecoder().decode(Double.self, from: partialData)) + } + + func test_depthTraversal() { + struct SuperNestedArray : Decodable { + init(from decoder: Decoder) throws { + var container = try decoder.unkeyedContainer() + while container.count! > 0 { + container = try container.nestedUnkeyedContainer() + } + } + } + + let MAX_DEPTH = 512 + let jsonGood = String(repeating: "[", count: MAX_DEPTH / 2) + String(repeating: "]", count: MAX_DEPTH / 2) + let jsonBad = String(repeating: "[", count: MAX_DEPTH + 1) + String(repeating: "]", count: MAX_DEPTH + 1) + + XCTAssertNoThrow(try JSONDecoder().decode(SuperNestedArray.self, from: jsonGood.data(using: String._Encoding.utf8)!)) + XCTAssertThrowsError(try JSONDecoder().decode(SuperNestedArray.self, from: jsonBad.data(using: String._Encoding.utf8)!)) + + } + + func test_JSONPermitsTrailingCommas() { + // Trailing commas aren't valid JSON and should never be emitted, but are syntactically unambiguous and are allowed by + // most parsers for ease of use. + let json = "{\"key\" : [ true, ],}" + let data = json.data(using: String._Encoding.utf8)! + + let result = try! JSONDecoder().decode([String:[Bool]].self, from: data) + let expected = ["key" : [true]] + XCTAssertEqual(result, expected) + } + + func test_whitespaceOnlyData() { + let data = " ".data(using: String._Encoding.utf8)! + XCTAssertThrowsError(try JSONDecoder().decode(Int.self, from: data)) + } + + func test_smallFloatNumber() { + _testRoundTrip(of: [["magic_number" : 7.45673334164903e-115]]) + } + + func test_largeIntegerNumber() { + let num : UInt64 = 6032314514195021674 + let json = "{\"a\":\(num)}" + let data = json.data(using: String._Encoding.utf8)! + + let result = try! JSONDecoder().decode([String:UInt64].self, from: data) + let number = result["a"]! + XCTAssertEqual(number, num) + } + + func test_roundTrippingExtremeValues() { + struct Numbers : Codable, Equatable { + let floats : [Float] + let doubles : [Double] + } + let testValue = Numbers(floats: [.greatestFiniteMagnitude, .leastNormalMagnitude], doubles: [.greatestFiniteMagnitude, .leastNormalMagnitude]) + _testRoundTrip(of: testValue) + } + + func test_roundTrippingDoubleValues() { + struct Numbers : Codable, Equatable { + let doubles : [String:Double] + let decimals : [String:Decimal] + } + let testValue = Numbers( + doubles: [ + "-55.66" : -55.66, + "-9.81" : -9.81, + "-0.284" : -0.284, + "-3.4028234663852886e+38" : Double(-Float.greatestFiniteMagnitude), + "-1.1754943508222875e-38" : Double(-Float.leastNormalMagnitude), + "-1.7976931348623157e+308" : -.greatestFiniteMagnitude, + "-2.2250738585072014e-308" : -.leastNormalMagnitude, + "0.000001" : 0.000001, + ], + decimals: [ + "1.234567891011121314" : Decimal(string: "1.234567891011121314")!, + "-1.234567891011121314" : Decimal(string: "-1.234567891011121314")!, + "0.1234567891011121314" : Decimal(string: "0.1234567891011121314")!, + "-0.1234567891011121314" : Decimal(string: "-0.1234567891011121314")!, + "123.4567891011121314e-100" : Decimal(string: "123.4567891011121314e-100")!, + "-123.4567891011121314e-100" : Decimal(string: "-123.4567891011121314e-100")!, + "11234567891011121314e-100" : Decimal(string: "1234567891011121314e-100")!, + "-1234567891011121314e-100" : Decimal(string: "-1234567891011121314e-100")!, + "0.1234567891011121314e-100" : Decimal(string: "0.1234567891011121314e-100")!, + "-0.1234567891011121314e-100" : Decimal(string: "-0.1234567891011121314e-100")!, + "3.14159265358979323846264338327950288419" : Decimal(string: "3.14159265358979323846264338327950288419")!, + "2.71828182845904523536028747135266249775" : Decimal(string: "2.71828182845904523536028747135266249775")!, + "440474310512876335.18692524723746578303827301433673643795" : Decimal(string: "440474310512876335.18692524723746578303827301433673643795")! + ] + ) + _testRoundTrip(of: testValue) + } + + func test_localeDecimalPolicyIndependence() { + let currentLocale = setlocale(LC_ALL, nil) + + let orig = ["decimalValue" : 1.1] + + do { + setlocale(LC_ALL, "fr_FR") + let data = try JSONEncoder().encode(orig) + + setlocale(LC_ALL, "en_US_POSIX") + let decoded = try JSONDecoder().decode(type(of: orig).self, from: data) + + XCTAssertEqual(orig, decoded) + } catch { + XCTFail("Error: \(error)") + } + + setlocale(LC_ALL, currentLocale) + } + + func test_whitespace() { + let tests : [(json: String, expected: [String:Bool])] = [ + ("{\"v\"\n : true}", ["v":true]), + ("{\"v\"\r\n : true}", ["v":true]), + ("{\"v\"\r : true}", ["v":true]) + ] + for test in tests { + let data = test.json.data(using: String._Encoding.utf8)! + let decoded = try! JSONDecoder().decode([String:Bool].self, from: data) + XCTAssertEqual(test.expected, decoded) + } + } + + func test_assumesTopLevelDictionary() { + let decoder = JSONDecoder() + decoder.assumesTopLevelDictionary = true + + let json = "\"x\" : 42" + do { + let result = try decoder.decode([String:Int].self, from: json.data(using: String._Encoding.utf8)!) + XCTAssertEqual(result, ["x" : 42]) + } catch { + XCTFail("Error thrown while decoding assumed top-level dictionary: \(error)") + } + + let jsonWithBraces = "{\"x\" : 42}" + do { + let result = try decoder.decode([String:Int].self, from: jsonWithBraces.data(using: String._Encoding.utf8)!) + XCTAssertEqual(result, ["x" : 42]) + } catch { + XCTFail("Error thrown while decoding assumed top-level dictionary: \(error)") + } + + do { + let result = try decoder.decode([String:Int].self, from: Data()) + XCTAssertEqual(result, [:]) + } catch { + XCTFail("Error thrown while decoding empty assumed top-level dictionary: \(error)") + } + + let jsonWithEndBraceOnly = "\"x\" : 42}" + XCTAssertThrowsError(try decoder.decode([String:Int].self, from: jsonWithEndBraceOnly.data(using: String._Encoding.utf8)!)) + + let jsonWithStartBraceOnly = "{\"x\" : 42" + XCTAssertThrowsError(try decoder.decode([String:Int].self, from: jsonWithStartBraceOnly.data(using: String._Encoding.utf8)!)) + + } + + func test_BOMPrefixes() { + let json = "\"👍🏻\"" + let decoder = JSONDecoder() + + // UTF-8 BOM + let utf8_BOM = Data([0xEF, 0xBB, 0xBF]) + XCTAssertEqual("👍🏻", try decoder.decode(String.self, from: utf8_BOM + json.data(using: String._Encoding.utf8)!)) + +#if FOUNDATION_FRAMEWORK + // TODO: Reenable these once string.data(using:) is fully implemented + + // UTF-16 BE + let utf16_BE_BOM = Data([0xFE, 0xFF]) + XCTAssertEqual("👍🏻", try decoder.decode(String.self, from: utf16_BE_BOM + json.data(using: String._Encoding.utf16BigEndian)!)) + + // UTF-16 LE + let utf16_LE_BOM = Data([0xFF, 0xFE]) + XCTAssertEqual("👍🏻", try decoder.decode(String.self, from: utf16_LE_BOM + json.data(using: String._Encoding.utf16LittleEndian)!)) + + // UTF-32 BE + let utf32_BE_BOM = Data([0x0, 0x0, 0xFE, 0xFF]) + XCTAssertEqual("👍🏻", try decoder.decode(String.self, from: utf32_BE_BOM + json.data(using: String._Encoding.utf32BigEndian)!)) + + // UTF-32 LE + let utf32_LE_BOM = Data([0xFE, 0xFF, 0, 0]) + XCTAssertEqual("👍🏻", try decoder.decode(String.self, from: utf32_LE_BOM + json.data(using: String._Encoding.utf32LittleEndian)!)) + + // Try some mismatched BOMs + XCTAssertThrowsError(try decoder.decode(String.self, from: utf32_LE_BOM + json.data(using: String._Encoding.utf32BigEndian)!)) + XCTAssertThrowsError(try decoder.decode(String.self, from: utf16_BE_BOM + json.data(using: String._Encoding.utf32LittleEndian)!)) + XCTAssertThrowsError(try decoder.decode(String.self, from: utf8_BOM + json.data(using: String._Encoding.utf16BigEndian)!)) +#endif // FOUNDATION_FRAMEWORK + } + + func test_valueNotFoundError() { + struct ValueNotFound : Decodable { + let a: Bool + let nope: String? + + enum CodingKeys: String, CodingKey { + case a, nope + } + + init(from decoder: Decoder) throws { + let keyed = try decoder.container(keyedBy: CodingKeys.self) + self.a = try keyed.decode(Bool.self, forKey: .a) + + do { + let sup = try keyed.superDecoder(forKey: .nope) + self.nope = try sup.singleValueContainer().decode(String.self) + } catch DecodingError.valueNotFound { + // This is fine. + self.nope = nil + } + } + } + let json = "{\"a\":true}".data(using: String._Encoding.utf8)! + + // The expected valueNotFound error is swalled by the init(from:) implementation. + XCTAssertNoThrow(try JSONDecoder().decode(ValueNotFound.self, from: json)) + } + + func test_infiniteDate() { + let date = Date(timeIntervalSince1970: .infinity) + + let encoder = JSONEncoder() + + encoder.dateEncodingStrategy = .deferredToDate + XCTAssertThrowsError(try encoder.encode([date])) + + encoder.dateEncodingStrategy = .secondsSince1970 + XCTAssertThrowsError(try encoder.encode([date])) + + encoder.dateEncodingStrategy = .millisecondsSince1970 + XCTAssertThrowsError(try encoder.encode([date])) + } + + func test_typeEncodesNothing() { + struct EncodesNothing : Encodable { + func encode(to encoder: Encoder) throws { + // Intentionally nothing. + } + } + let enc = JSONEncoder() + + XCTAssertThrowsError(try enc.encode(EncodesNothing())) + + // Unknown if the following behavior is strictly correct, but it's what the prior implementation does, so this test exists to make sure we maintain compatibility. + + let arrayData = try! enc.encode([EncodesNothing()]) + XCTAssertEqual("[{}]", String(data: arrayData, encoding: .utf8)) + + let objectData = try! enc.encode(["test" : EncodesNothing()]) + XCTAssertEqual("{\"test\":{}}", String(data: objectData, encoding: .utf8)) + } + + func test_superEncoders() { + struct SuperEncoding : Encodable { + enum CodingKeys: String, CodingKey { + case firstSuper + case secondSuper + case unkeyed + } + func encode(to encoder: Encoder) throws { + var keyed = encoder.container(keyedBy: CodingKeys.self) + + let keyedSuper1 = keyed.superEncoder(forKey: .firstSuper) + let keyedSuper2 = keyed.superEncoder(forKey: .secondSuper) + var keyedSVC1 = keyedSuper1.singleValueContainer() + var keyedSVC2 = keyedSuper2.singleValueContainer() + try keyedSVC1.encode("First") + try keyedSVC2.encode("Second") + + var unkeyed = keyed.nestedUnkeyedContainer(forKey: .unkeyed) + try unkeyed.encode(0) + let unkeyedSuper1 = unkeyed.superEncoder() + let unkeyedSuper2 = unkeyed.superEncoder() + try unkeyed.encode(42) + var unkeyedSVC1 = unkeyedSuper1.singleValueContainer() + var unkeyedSVC2 = unkeyedSuper2.singleValueContainer() + try unkeyedSVC1.encode("First") + try unkeyedSVC2.encode("Second") + + // NOTE!!! At pressent, the order in which the values in the unkeyed container's superEncoders above get inserted into the resulting array depends on the order in which the superEncoders are deinit'd!! This can result in some very unexpected results, and this pattern is not recommended. This test exists just to verify compatibility. + } + } + let data = try! JSONEncoder().encode(SuperEncoding()) + let string = String(data: data, encoding: .utf8)! + + XCTAssertTrue(string.contains("\"firstSuper\":\"First\"")) + XCTAssertTrue(string.contains("\"secondSuper\":\"Second\"")) + XCTAssertTrue(string.contains("[0,\"First\",\"Second\",42]")) + } + + func testRedundantKeys() { + // Last encoded key wins. + + struct RedundantEncoding : Encodable { + enum ReplacedType { + case value + case keyedContainer + case unkeyedContainer + } + let replacedType: ReplacedType + let useSuperEncoder: Bool + + enum CodingKeys: String, CodingKey { + case key + } + func encode(to encoder: Encoder) throws { + var keyed = encoder.container(keyedBy: CodingKeys.self) + switch replacedType { + case .value: + try keyed.encode(0, forKey: .key) + case .keyedContainer: + let _ = keyed.nestedContainer(keyedBy: CodingKeys.self, forKey: .key) + case .unkeyedContainer: + let _ = keyed.nestedUnkeyedContainer(forKey: .key) + } + if useSuperEncoder { + var svc = keyed.superEncoder(forKey: .key).singleValueContainer() + try svc.encode(42) + } else { + try keyed.encode(42, forKey: .key) + } + } + } + var data = try! JSONEncoder().encode(RedundantEncoding(replacedType: .value, useSuperEncoder: false)) + XCTAssertEqual(String(data: data, encoding: .utf8), ("{\"key\":42}")) + + data = try! JSONEncoder().encode(RedundantEncoding(replacedType: .value, useSuperEncoder: true)) + XCTAssertEqual(String(data: data, encoding: .utf8), ("{\"key\":42}")) + + data = try! JSONEncoder().encode(RedundantEncoding(replacedType: .keyedContainer, useSuperEncoder: false)) + XCTAssertEqual(String(data: data, encoding: .utf8), ("{\"key\":42}")) + + data = try! JSONEncoder().encode(RedundantEncoding(replacedType: .keyedContainer, useSuperEncoder: true)) + XCTAssertEqual(String(data: data, encoding: .utf8), ("{\"key\":42}")) + + data = try! JSONEncoder().encode(RedundantEncoding(replacedType: .unkeyedContainer, useSuperEncoder: false)) + XCTAssertEqual(String(data: data, encoding: .utf8), ("{\"key\":42}")) + + data = try! JSONEncoder().encode(RedundantEncoding(replacedType: .unkeyedContainer, useSuperEncoder: true)) + XCTAssertEqual(String(data: data, encoding: .utf8), ("{\"key\":42}")) + } + + // None of these tests can be run in our automatic test suites right now, because they are expected to hit a preconditionFailure. They can only be verified manually. + func disabled_testPreconditionFailuresForContainerReplacement() { + struct RedundantEncoding : Encodable { + enum Subcase { + case replaceValueWithKeyedContainer + case replaceValueWithUnkeyedContainer + case replaceKeyedContainerWithUnkeyed + case replaceUnkeyedContainerWithKeyed + } + let subcase : Subcase + + enum CodingKeys: String, CodingKey { + case key + } + func encode(to encoder: Encoder) throws { + switch subcase { + case .replaceValueWithKeyedContainer: + var keyed = encoder.container(keyedBy: CodingKeys.self) + try keyed.encode(42, forKey: .key) + let _ = keyed.nestedContainer(keyedBy: CodingKeys.self, forKey: .key) + case .replaceValueWithUnkeyedContainer: + var keyed = encoder.container(keyedBy: CodingKeys.self) + try keyed.encode(42, forKey: .key) + let _ = keyed.nestedUnkeyedContainer(forKey: .key) + case .replaceKeyedContainerWithUnkeyed: + var keyed = encoder.container(keyedBy: CodingKeys.self) + let _ = keyed.nestedContainer(keyedBy: CodingKeys.self, forKey: .key) + let _ = keyed.nestedUnkeyedContainer(forKey: .key) + case .replaceUnkeyedContainerWithKeyed: + var keyed = encoder.container(keyedBy: CodingKeys.self) + let _ = keyed.nestedUnkeyedContainer(forKey: .key) + let _ = keyed.nestedContainer(keyedBy: CodingKeys.self, forKey: .key) + } + } + } + let _ = try! JSONEncoder().encode(RedundantEncoding(subcase: .replaceValueWithKeyedContainer)) +// let _ = try! JSONEncoder().encode(RedundantEncoding(subcase: .replaceValueWithUnkeyedContainer)) +// let _ = try! JSONEncoder().encode(RedundantEncoding(subcase: .replaceKeyedContainerWithUnkeyed)) +// let _ = try! JSONEncoder().encode(RedundantEncoding(subcase: .replaceUnkeyedContainerWithKeyed)) + } + + var json5Decoder: JSONDecoder { + let decoder = JSONDecoder() + decoder.allowsJSON5 = true + return decoder + } + + func test_json5Numbers() { + let decoder = json5Decoder + + let successfulIntegers: [(String,Int)] = [ + ("1", 1), + ("11", 11), + ("99887766", 99887766), + ("-1", -1), + ("-10", -10), + ("0", 0), + ("+0", +0), + ("-0", -0), + ("+1", +1), + ("+10", +10), + ("0x1F", 0x1F), + ("-0X1f", -0x1f), + ("+0X1f", +0x1f), + ("1.", 1), + ("1.e2", 100), + ("1e2", 100), + ("1E2", 100), + ("1e+2", 100), + ("1E+2", 100), + ("1e+02", 100), + ("1E+02", 100), + ] + for (json, expected) in successfulIntegers { + do { + let val = try decoder.decode(Int.self, from: json.data(using: String._Encoding.utf8)!) + XCTAssertEqual(val, expected, "Wrong value parsed from input \"\(json)\"") + } catch { + XCTFail("Error when parsing input \"\(json)\": \(error)") + } + } + + let successfulDoubles: [(String,Double)] = [ + ("1", 1), + ("11", 11), + ("99887766", 99887766), + ("-1", -1), + ("-10", -10), + ("0", 0), + ("+0", +0), + ("-0", -0), + ("+1", +1), + ("+10", +10), + ("Infinity", Double.infinity), + ("-Infinity", -Double.infinity), + ("+Infinity", Double.infinity), + ("-NaN", -Double.nan), + ("+NaN", Double.nan), + ("NaN", Double.nan), + (".1", 0.1), + ("1.", 1.0), + ("-.1", -0.1), + ("+.1", +0.1), + ("1e-2", 1e-2), + ("1E-2", 1E-2), + ("1e-02", 1e-02), + ("1E-02", 1E-02), + ("1e2", 1e2), + ("1E2", 1E2), + ("1e+2", 1e+2), + ("1E+2", 1E+2), + ("1e+02", 1e+02), + ("1E+02", 1E+02), + ("0x1F", Double(0x1F)), + ("-0X1f", Double(-0x1f)), + ("+0X1f", Double(+0x1f)), + ] + for (json, expected) in successfulDoubles { + do { + let val = try decoder.decode(Double.self, from: json.data(using: String._Encoding.utf8)!) + if expected.isNaN { + XCTAssertTrue(val.isNaN, "Wrong value \(val) parsed from input \"\(json)\"") + } else { + XCTAssertEqual(val, expected, "Wrong value parsed from input \"\(json)\"") + } + } catch { + XCTFail("Error when parsing input \"\(json)\": \(error)") + } + } + + let unsuccessfulIntegers = [ + "-", // single - + "+", // single + + "-a", // - followed by non-digit + "+a", // + followed by non-digit + "-0x", + "+0x", + "-0x ", + "+0x ", + "-0xAFFFFFAFFFFFAFFFFFAFFFFFAFFFFFAFFFFFAFFFFFAFFFFF", + "0xABC.DEF", + "0xABCpD", + "1e", + "1E", + "1e ", + "1E ", + "+1e ", + "+1e", + "-1e ", + "-1E ", + ] + for json in unsuccessfulIntegers { + do { + let _ = try decoder.decode(Int.self, from: json.data(using: String._Encoding.utf8)!) + XCTFail("Expected failure for input \"\(json)\"") + } catch { } + } + + let unsuccessfulDoubles = [ + "-Inf", + "-Inf ", + "+Inf", + "+Inf ", + "+Na", + "+Na ", + "-Na", + "-Na ", + "-infinity", + "-infinity ", + "+infinity", + "+infinity ", + "+NAN", + "+NA ", + "-NA", + "-NA ", + "-NAN", + "-NAN ", + "0x2.", + "0x2.2", + ".e1", + "0xFFFFFFFFFFFFFFFFFFFFFF", + ]; + for json in unsuccessfulDoubles { + do { + let _ = try decoder.decode(Double.self, from: json.data(using: String._Encoding.utf8)!) + XCTFail("Expected failure for input \"\(json)\"") + } catch { } + } + } + + func test_json5Null() { + let validJSON = "null" + let invalidJSON = [ + "Null", + "nul", + "nu", + "n", + "n ", + "nu " + ] + + XCTAssertNoThrow(try json5Decoder.decode(NullReader.self, from: validJSON.data(using: String._Encoding.utf8)!)) + + for json in invalidJSON { + XCTAssertThrowsError(try json5Decoder.decode(NullReader.self, from: json.data(using: String._Encoding.utf8)!), "Expected failure while decoding input \"\(json)\"") + } + } + + func test_json5EsotericErrors() { + // All of the following should fail + let arrayStrings = [ + "[", + "[ ", + "[\n\n", + "['hi',", + "['hi', ", + "['hi',\n" + ] + let objectStrings = [ + "{", + "{ ", + "{k ", + "{k :", + "{k : ", + "{k : true", + "{k : true ", + "{k : true\n\n", + "{k : true ", + "{k : true ", + ] + let objectCharacterArrays: [[UInt8]] = [ + [.init(ascii: "{"), 0x80], // Invalid UTF-8: Unexpected continuation byte + [.init(ascii: "{"), 0xc0], // Invalid UTF-8: Initial byte of 2-byte sequence without continuation + [.init(ascii: "{"), 0xe0, 0x80], // Invalid UTF-8: Initial byte of 3-byte sequence with only one continuation + [.init(ascii: "{"), 0xf0, 0x80, 0x80], // Invalid UTF-8: Initial byte of 3-byte sequence with only one continuation + ] + for json in arrayStrings { + XCTAssertThrowsError(try json5Decoder.decode([String].self, from: json.data(using: String._Encoding.utf8)!), "Expected error for input \"\(json)\"") + } + for json in objectStrings { + XCTAssertThrowsError(try json5Decoder.decode([String:Bool].self, from: json.data(using: String._Encoding.utf8)!), "Expected error for input \(json)") + } + for json in objectCharacterArrays { + XCTAssertThrowsError(try json5Decoder.decode([String:Bool].self, from: Data(json)), "Expected error for input \(json)") + } + } + + func test_json5Strings() { + let stringsToTrues = [ + "{v\n : true}", + "{v \n : true}", + "{ v \n : true,\nv\n:true,}", + "{v\r : true}", + "{v \r : true}", + "{ v \r : true,\rv\r:true,}", + "{v\r\n : true}", + "{v \r\n : true}", + "{ v \r\n : true,\r\nv\r\n:true,}", + "{v// comment \n : true}", + "{v // comment \n : true}", + "{v/* comment*/ \n : true}", + "{v/* comment */\n: true}", + "{v/* comment */:/*comment*/\ntrue}", + "{v// comment \r : true}", + "{v // comment \r : true}", + "{v/* comment*/ \r : true}", + "{v/* comment */\r: true}", + "{v/* comment */:/*comment*/\rtrue}", + "{v// comment \r\n : true}", + "{v // comment \r\n : true}", + "{v/* comment*/ \r\n : true}", + "{v/* comment */\r\n: true}", + "{v/* comment */:/*comment*/\r\ntrue}", + "// start with a comment\r\n{v:true}", + ] + + let stringsToStrings = [ + "{v : \"hi\\x20there\"}" : "hi there", + "{v : \"hi\\xthere\"}" : nil, + "{v : \"hi\\x2there\"}" : nil, + "{v : \"hi\\0there\"}" : nil, + "{v : \"hi\\x00there\"}" : nil, + "{v : \"hi\\u0000there\"}" : nil, // disallowed in JSON5 mode only + "{v:\"hello\\uA\"}" : nil, + "{v:\"hello\\uA \"}" : nil + ] + + for json in stringsToTrues { + XCTAssertNoThrow(try json5Decoder.decode([String:Bool].self, from: json.data(using: String._Encoding.utf8)!), "Failed to parse \"\(json)\"") + } + for (json, expected) in stringsToStrings { + do { + let decoded = try json5Decoder.decode([String:String].self, from: json.data(using: String._Encoding.utf8)!) + XCTAssertEqual(expected, decoded["v"]) + } catch { + if let expected { + XCTFail("Expected \(expected) for input \"\(json)\", but failed with \(error)") + } + } + } + } + + func test_json5AssumedDictionary() { + let decoder = json5Decoder + decoder.assumesTopLevelDictionary = true + + let stringsToString = [ + "hello: \"world\"" : [ "hello" : "world" ], + "{hello: \"world\"}" : [ "hello" : "world" ], // Still has markers + "hello: \"world\", goodbye: \"42\"" : [ "hello" : "world", "goodbye" : "42" ], // more than one value + "hello: \"world\"," : [ "hello" : "world" ], // Trailing comma + "hello: \"world\" " : [ "hello" : "world" ], // Trailing whitespace + "hello: \"world\", " : [ "hello" : "world" ], // Trailing whitespace and comma + "hello: \"world\" , " : [ "hello" : "world" ], // Trailing whitespace and comma + " hello : \"world\" " : [ "hello" : "world" ], // Before and after whitespace + "{hello: \"world\"" : nil, // Partial dictionary 1 + "hello: \"world\"}" : nil, // Partial dictionary 2 + "hello: \"world\" x " : nil, // Junk at end + "hello: \"world\" x" : nil, // Junk at end + "hello: \"world\"x" : nil, // Junk at end + "" : [:], // empty but valid + " " : [:], // empty but valid + "{ }" : [:], // empty but valid + "{}" : [:], // empty but valid + "," : nil, // Invalid + " , " : nil, // Invalid + ", " : nil, // Invalid + " ," : nil, // Invalid + ] + for (json, expected) in stringsToString { + do { + let decoded = try decoder.decode([String:String].self, from: json.data(using: String._Encoding.utf8)!) + XCTAssertEqual(expected, decoded) + } catch { + if let expected { + XCTFail("Expected \(expected) for input \"\(json)\", but failed with \(error)") + } + } + } + + struct HelloGoodbye : Decodable, Equatable { + let hello: String + let goodbye: [String:String] + } + let helloGoodbyeExpectedValue = HelloGoodbye( + hello: "world", + goodbye: ["hi" : "there"]) + let stringsToNestedDictionary = [ + "hello: \"world\", goodbye: {\"hi\":\"there\"}", // more than one value, nested dictionary + "hello: \"world\", goodbye: {\"hi\":\"there\"},", // more than one value, nested dictionary, trailing comma 1 + "hello: \"world\", goodbye: {\"hi\":\"there\",},", // more than one value, nested dictionary, trailing comma 2 + ] + for json in stringsToNestedDictionary { + do { + let decoded = try decoder.decode(HelloGoodbye.self, from: json.data(using: String._Encoding.utf8)!) + XCTAssertEqual(helloGoodbyeExpectedValue, decoded) + } catch { + XCTFail("Expected \(helloGoodbyeExpectedValue) for input \"\(json)\", but failed with \(error)") + } + } + + let arrayJSON = "[1,2,3]".data(using: String._Encoding.utf8)! // Assumed dictionary can't be an array + XCTAssertThrowsError(try decoder.decode([Int].self, from: arrayJSON)) + + let strFragmentJSON = "fragment".data(using: String._Encoding.utf8)! // Assumed dictionary can't be a fragment + XCTAssertThrowsError(try decoder.decode(String.self, from: strFragmentJSON)) + + let numFragmentJSON = "42".data(using: String._Encoding.utf8)! // Assumed dictionary can't be a fragment + XCTAssertThrowsError(try decoder.decode(Int.self, from: numFragmentJSON)) + } + + enum JSON5SpecTestType { + case json5 + case json5_foundationPermissiveJSON + case json + case js + case malformed + + var fileExtension : String { + switch self { + case .json5: return "json5" + case .json5_foundationPermissiveJSON: return "json5" + case .json: return "json" + case .js: return "js" + case .malformed: return "txt" + } + } + } +} + +// MARK: - FoundationPreview Disabled Tests +#if FOUNDATION_FRAMEWORK +extension JSONEncoderTests { + // TODO: Reenable once .iso8601 formatter is moved + func testEncodingDateISO8601() { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = .withInternetDateTime + + let timestamp = Date(timeIntervalSince1970: 1000) + let expectedJSON = "\"\(formatter.string(from: timestamp))\"".data(using: String._Encoding.utf8)! + + _testRoundTrip(of: timestamp, + expectedJSON: expectedJSON, + dateEncodingStrategy: .iso8601, + dateDecodingStrategy: .iso8601) + + + // Optional dates should encode the same way. + _testRoundTrip(of: Optional(timestamp), + expectedJSON: expectedJSON, + dateEncodingStrategy: .iso8601, + dateDecodingStrategy: .iso8601) + } + + // TODO: Reenable once DateFormatStyle is moved + func testEncodingDateFormatted() { + let formatter = DateFormatter() + formatter.dateStyle = .full + formatter.timeStyle = .full + + let timestamp = Date(timeIntervalSince1970: 1000) + let expectedJSON = "\"\(formatter.string(from: timestamp))\"".data(using: String._Encoding.utf8)! + + _testRoundTrip(of: timestamp, + expectedJSON: expectedJSON, + dateEncodingStrategy: .formatted(formatter), + dateDecodingStrategy: .formatted(formatter)) + + // Optional dates should encode the same way. + _testRoundTrip(of: Optional(timestamp), + expectedJSON: expectedJSON, + dateEncodingStrategy: .formatted(formatter), + dateDecodingStrategy: .formatted(formatter)) + } + + // TODO: Reenable once string.data(using:) is fully implemented + func test_twoByteUTF16Inputs() { + let json = "7" + let decoder = JSONDecoder() + + XCTAssertEqual(7, try decoder.decode(Int.self, from: json.data(using: .utf16BigEndian)!)) + XCTAssertEqual(7, try decoder.decode(Int.self, from: json.data(using: .utf16LittleEndian)!)) + } + + private func _run_passTest(name: String, json5: Bool = false, type: T.Type) { + let bundle = Bundle(for: Self.self) + let jsonURL = bundle.url(forResource: name, withExtension: json5 ? "json5" : "json" , subdirectory: json5 ? "JSON5/pass" : "JSON/pass")! + let jsonData = try! Data(contentsOf: jsonURL) + + let plistData : Data? + if let plistURL = bundle.url(forResource: name, withExtension: "plist", subdirectory: "JSON/pass") { + plistData = try! Data(contentsOf: plistURL) + } else { + plistData = nil + } + + let decoder = json5Decoder + + let decoded: T + do { + decoded = try decoder.decode(T.self, from: jsonData) + } catch { + XCTFail("Pass test \"\(name)\" failed with error: \(error)") + return + } + + let prettyPrintEncoder = JSONEncoder() + prettyPrintEncoder.outputFormatting = .prettyPrinted + + for encoder in [JSONEncoder(), prettyPrintEncoder] { + let reencodedData = try! encoder.encode(decoded) + let redecodedObjects = try! decoder.decode(T.self, from: reencodedData) + XCTAssertEqual(decoded, redecodedObjects) + + if let plistData { + let decodedPlistObjects = try! PropertyListDecoder().decode(T.self, from: plistData) + XCTAssertEqual(decoded, decodedPlistObjects) + } + } + } + + func test_JSONPassTests() { + _run_passTest(name: "pass1-utf8", type: JSONPass.Test1.self) + _run_passTest(name: "pass1-utf16be", type: JSONPass.Test1.self) + _run_passTest(name: "pass1-utf16le", type: JSONPass.Test1.self) + _run_passTest(name: "pass1-utf32be", type: JSONPass.Test1.self) + _run_passTest(name: "pass1-utf32le", type: JSONPass.Test1.self) + + _run_passTest(name: "pass2", type: JSONPass.Test2.self) + _run_passTest(name: "pass3", type: JSONPass.Test3.self) + _run_passTest(name: "pass4", type: JSONPass.Test4.self) + _run_passTest(name: "pass5", type: JSONPass.Test5.self) + _run_passTest(name: "pass6", type: JSONPass.Test6.self) + _run_passTest(name: "pass7", type: JSONPass.Test7.self) + _run_passTest(name: "pass8", type: JSONPass.Test8.self) + _run_passTest(name: "pass9", type: JSONPass.Test9.self) + _run_passTest(name: "pass10", type: JSONPass.Test10.self) + _run_passTest(name: "pass11", type: JSONPass.Test11.self) + _run_passTest(name: "pass12", type: JSONPass.Test12.self) + _run_passTest(name: "pass13", type: JSONPass.Test13.self) + _run_passTest(name: "pass14", type: JSONPass.Test14.self) + _run_passTest(name: "pass15", type: JSONPass.Test15.self) + } + + func test_json5PassJSONFiles() { + _run_passTest(name: "example", json5: true, type: JSON5Pass.Example.self) + _run_passTest(name: "hex", json5: true, type: JSON5Pass.Hex.self) + _run_passTest(name: "numbers", json5: true, type: JSON5Pass.Numbers.self) + _run_passTest(name: "strings", json5: true, type: JSON5Pass.Strings.self) + _run_passTest(name: "whitespace", json5: true, type: JSON5Pass.Whitespace.self) + } + + private func _run_failTest(name: String, type: T.Type) { + let bundle = Bundle(for: Self.self) + let jsonURL = bundle.url(forResource: name, withExtension: "json", subdirectory: "JSON/fail")! + let jsonData = try! Data(contentsOf: jsonURL) + + let decoder = JSONDecoder() + decoder.assumesTopLevelDictionary = true + do { + let _ = try decoder.decode(T.self, from: jsonData) + XCTFail("Decoding should have failed for invalid JSON data (test name: \(name))") + } catch { + print(error as NSError) + } + } + + func test_JSONFailTests() { + _run_failTest(name: "fail1", type: JSONFail.Test1.self) + _run_failTest(name: "fail2", type: JSONFail.Test2.self) + _run_failTest(name: "fail3", type: JSONFail.Test3.self) + _run_failTest(name: "fail4", type: JSONFail.Test4.self) + _run_failTest(name: "fail5", type: JSONFail.Test5.self) + _run_failTest(name: "fail6", type: JSONFail.Test6.self) + _run_failTest(name: "fail7", type: JSONFail.Test7.self) + _run_failTest(name: "fail8", type: JSONFail.Test8.self) + _run_failTest(name: "fail9", type: JSONFail.Test9.self) + _run_failTest(name: "fail10", type: JSONFail.Test10.self) + _run_failTest(name: "fail11", type: JSONFail.Test11.self) + _run_failTest(name: "fail12", type: JSONFail.Test12.self) + _run_failTest(name: "fail13", type: JSONFail.Test13.self) + _run_failTest(name: "fail14", type: JSONFail.Test14.self) + _run_failTest(name: "fail15", type: JSONFail.Test15.self) + _run_failTest(name: "fail16", type: JSONFail.Test16.self) + _run_failTest(name: "fail17", type: JSONFail.Test17.self) + _run_failTest(name: "fail18", type: JSONFail.Test18.self) + _run_failTest(name: "fail19", type: JSONFail.Test19.self) + _run_failTest(name: "fail21", type: JSONFail.Test21.self) + _run_failTest(name: "fail22", type: JSONFail.Test22.self) + _run_failTest(name: "fail23", type: JSONFail.Test23.self) + _run_failTest(name: "fail24", type: JSONFail.Test24.self) + _run_failTest(name: "fail25", type: JSONFail.Test25.self) + _run_failTest(name: "fail26", type: JSONFail.Test26.self) + _run_failTest(name: "fail27", type: JSONFail.Test27.self) + _run_failTest(name: "fail28", type: JSONFail.Test28.self) + _run_failTest(name: "fail29", type: JSONFail.Test29.self) + _run_failTest(name: "fail30", type: JSONFail.Test30.self) + _run_failTest(name: "fail31", type: JSONFail.Test31.self) + _run_failTest(name: "fail32", type: JSONFail.Test32.self) + _run_failTest(name: "fail33", type: JSONFail.Test33.self) + _run_failTest(name: "fail34", type: JSONFail.Test34.self) + _run_failTest(name: "fail35", type: JSONFail.Test35.self) + _run_failTest(name: "fail36", type: JSONFail.Test36.self) + _run_failTest(name: "fail37", type: JSONFail.Test37.self) + _run_failTest(name: "fail38", type: JSONFail.Test38.self) + _run_failTest(name: "fail39", type: JSONFail.Test39.self) + _run_failTest(name: "fail40", type: JSONFail.Test40.self) + _run_failTest(name: "fail41", type: JSONFail.Test41.self) + + } + + func _run_json5SpecTest(_ category: String, _ name: String, testType: JSON5SpecTestType, type: T.Type) { + let bundle = Bundle(for: Self.self) + let subdirectory = "/JSON5/spec/\(category)" + let ext = testType.fileExtension + let jsonURL = bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory)! + let jsonData = try! Data(contentsOf: jsonURL) + + let json5 = json5Decoder + let json = JSONDecoder() + + switch testType { + case .json, .json5_foundationPermissiveJSON: + // Valid JSON should remain valid JSON5 + XCTAssertNoThrow(try json5.decode(type, from: jsonData)) + + // Repeat with non-JSON5-compliant decoder. + XCTAssertNoThrow(try json.decode(type, from: jsonData)) + case .json5: + XCTAssertNoThrow(try json5.decode(type, from: jsonData)) + + // Regular JSON decoder should throw. + do { + let val = try json.decode(type, from: jsonData) + XCTFail("Expected decode failure (original JSON)for test \(name).\(ext), but got: \(val)") + } catch { } + case .js: + // Valid ES5 that's explicitly disallowed by JSON5 is also invalid JSON. + do { + let val = try json5.decode(type, from: jsonData) + XCTFail("Expected decode failure (JSON5) for test \(name).\(ext), but got: \(val)") + } catch { } + + // Regular JSON decoder should also throw. + do { + let val = try json.decode(type, from: jsonData) + XCTFail("Expected decode failure (original JSON) for test \(name).\(ext), but got: \(val)") + } catch { } + case .malformed: + // Invalid ES5 should remain invalid JSON5 + do { + let val = try json5.decode(type, from: jsonData) + XCTFail("Expected decode failure (JSON5) for test \(name).\(ext), but got: \(val)") + } catch { } + + // Regular JSON decoder should also throw. + do { + let val = try json.decode(type, from: jsonData) + XCTFail("Expected decode failure (original JSON) for test \(name).\(ext), but got: \(val)") + } catch { } + } + } + + // Also tests non-JSON5 decoder against the non-JSON5 tests in this test suite. + func test_json5Spec() { + // Expected successes: + _run_json5SpecTest("arrays", "empty-array", testType: .json, type: [Bool].self) + _run_json5SpecTest("arrays", "regular-array", testType: .json, type: [Bool?].self) + _run_json5SpecTest("arrays", "trailing-comma-array", testType: .json5_foundationPermissiveJSON, type: [NullReader].self) + + _run_json5SpecTest("comments", "block-comment-following-array-element", testType: .json5, type: [Bool].self) + _run_json5SpecTest("comments", "block-comment-following-top-level-value", testType: .json5, type: NullReader.self) + _run_json5SpecTest("comments", "block-comment-in-string", testType: .json, type: String.self) + _run_json5SpecTest("comments", "block-comment-preceding-top-level-value", testType: .json5, type: NullReader.self) + _run_json5SpecTest("comments", "block-comment-with-asterisks", testType: .json5, type: Bool.self) + _run_json5SpecTest("comments", "inline-comment-following-array-element", testType: .json5, type: [Bool].self) + _run_json5SpecTest("comments", "inline-comment-following-top-level-value", testType: .json5, type: NullReader.self) + _run_json5SpecTest("comments", "inline-comment-in-string", testType: .json, type: String.self) + _run_json5SpecTest("comments", "inline-comment-preceding-top-level-value", testType: .json5, type: NullReader.self) + + _run_json5SpecTest("misc", "npm-package", testType: .json, type: JSON5Spec.NPMPackage.self) + _run_json5SpecTest("misc", "npm-package", testType: .json5, type: JSON5Spec.NPMPackage.self) + _run_json5SpecTest("misc", "readme-example", testType: .json5, type: JSON5Spec.ReadmeExample.self) + _run_json5SpecTest("misc", "valid-whitespace", testType: .json5, type: [String:Bool].self) + + _run_json5SpecTest("new-lines", "comment-cr", testType: .json5, type: [String:String].self) + _run_json5SpecTest("new-lines", "comment-crlf", testType: .json5, type: [String:String].self) + _run_json5SpecTest("new-lines", "comment-lf", testType: .json5, type: [String:String].self) + _run_json5SpecTest("new-lines", "escaped-cr", testType: .json5, type: [String:String].self) + _run_json5SpecTest("new-lines", "escaped-crlf", testType: .json5, type: [String:String].self) + _run_json5SpecTest("new-lines", "escaped-lf", testType: .json5, type: [String:String].self) + + _run_json5SpecTest("numbers", "float-leading-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "float-leading-zero", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "float-trailing-decimal-point-with-integer-exponent", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "float-trailing-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "float-with-integer-exponent", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "float", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "hexadecimal-lowercase-letter", testType: .json5, type: UInt.self) + _run_json5SpecTest("numbers", "hexadecimal-uppercase-x", testType: .json5, type: UInt.self) + _run_json5SpecTest("numbers", "hexadecimal-with-integer-exponent", testType: .json5, type: UInt.self) + _run_json5SpecTest("numbers", "hexadecimal", testType: .json5, type: UInt.self) + _run_json5SpecTest("numbers", "infinity", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-integer-exponent", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-negative-integer-exponent", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-negative-zero-integer-exponent", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "integer-with-positive-integer-exponent", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "integer-with-positive-zero-integer-exponent", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "integer-with-zero-integer-exponent", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "integer", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "nan", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "negative-float-leading-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "negative-float-leading-zero", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "negative-float-trailing-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "negative-float", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "negative-hexadecimal", testType: .json5, type: Int.self) + _run_json5SpecTest("numbers", "negative-infinity", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "negative-integer", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "negative-zero-float-leading-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "negative-zero-float-trailing-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "negative-zero-hexadecimal", testType: .json5, type: Int.self) + _run_json5SpecTest("numbers", "negative-zero-integer", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "positive-integer", testType: .json5, type: Int.self) + _run_json5SpecTest("numbers", "positive-zero-float-leading-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "positive-zero-float-trailing-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "positive-zero-float", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "positive-zero-hexadecimal", testType: .json5, type: Int.self) + _run_json5SpecTest("numbers", "positive-zero-integer", testType: .json5, type: Int.self) + _run_json5SpecTest("numbers", "zero-float-leading-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "zero-float-trailing-decimal-point", testType: .json5, type: Double.self) + _run_json5SpecTest("numbers", "zero-float", testType: .json, type: Double.self) + _run_json5SpecTest("numbers", "zero-hexadecimal", testType: .json5, type: Int.self) + _run_json5SpecTest("numbers", "zero-integer-with-integer-exponent", testType: .json, type: Int.self) + _run_json5SpecTest("numbers", "zero-integer", testType: .json, type: Int.self) + + _run_json5SpecTest("objects", "duplicate-keys", testType: .json, type: [String:Bool].self) + _run_json5SpecTest("objects", "empty-object", testType: .json, type: [String:Bool].self) + _run_json5SpecTest("objects", "reserved-unquoted-key", testType: .json5, type: [String:Bool].self) + _run_json5SpecTest("objects", "single-quoted-key", testType: .json5, type: [String:String].self) + _run_json5SpecTest("objects", "trailing-comma-object", testType: .json5_foundationPermissiveJSON, type: [String:String].self) + _run_json5SpecTest("objects", "unquoted-keys", testType: .json5, type: [String:String].self) + + _run_json5SpecTest("strings", "escaped-single-quoted-string", testType: .json5, type: String.self) + _run_json5SpecTest("strings", "multi-line-string", testType: .json5, type: String.self) + _run_json5SpecTest("strings", "single-quoted-string", testType: .json5, type: String.self) + + _run_json5SpecTest("todo", "unicode-escaped-unquoted-key", testType: .json5, type: [String:String].self) + _run_json5SpecTest("todo", "unicode-unquoted-key", testType: .json5, type: [String:String].self) + + // Expected failures: + _run_json5SpecTest("arrays", "leading-comma-array", testType: .js, type: [Bool].self) + _run_json5SpecTest("arrays", "lone-trailing-comma-array", testType: .js, type: [Bool].self) + _run_json5SpecTest("arrays", "no-comma-array", testType: .malformed, type: [Bool].self) + + _run_json5SpecTest("comments", "top-level-block-comment", testType: .malformed, type: Bool.self) + _run_json5SpecTest("comments", "top-level-inline-comment", testType: .malformed, type: Bool.self) + _run_json5SpecTest("comments", "unterminated-block-comment", testType: .malformed, type: Bool.self) + + _run_json5SpecTest("misc", "empty", testType: .malformed, type: Bool.self) + + _run_json5SpecTest("numbers", "hexadecimal-empty", testType: .malformed, type: UInt.self) + _run_json5SpecTest("numbers", "integer-with-float-exponent", testType: .malformed, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-hexadecimal-exponent", testType: .malformed, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-negative-float-exponent", testType: .malformed, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-negative-hexadecimal-exponent", testType: .malformed, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-positive-float-exponent", testType: .malformed, type: Double.self) + _run_json5SpecTest("numbers", "integer-with-positive-hexadecimal-exponent", testType: .malformed, type: Double.self) + _run_json5SpecTest("numbers", "lone-decimal-point", testType: .malformed, type: Double.self) + _run_json5SpecTest("numbers", "negative-noctal", testType: .js, type: Int.self) + _run_json5SpecTest("numbers", "negative-octal", testType: .malformed, type: Int.self) + _run_json5SpecTest("numbers", "noctal-with-leading-octal-digit", testType: .js, type: Int.self) + _run_json5SpecTest("numbers", "noctal", testType: .js, type: Int.self) + _run_json5SpecTest("numbers", "octal", testType: .malformed, type: Int.self) + _run_json5SpecTest("numbers", "positive-noctal", testType: .js, type: Int.self) + _run_json5SpecTest("numbers", "positive-octal", testType: .malformed, type: Int.self) + _run_json5SpecTest("numbers", "positive-zero-octal", testType: .malformed, type: Int.self) + _run_json5SpecTest("numbers", "zero-octal", testType: .malformed, type: Int.self) + + _run_json5SpecTest("objects", "illegal-unquoted-key-number", testType: .malformed, type: [String:String].self) + + // The spec test disallows this case, but historically NSJSONSerialization has allowed it. Our new implementation is more up-to-spec. + _run_json5SpecTest("objects", "illegal-unquoted-key-symbol", testType: .malformed, type: [String:String].self) + + _run_json5SpecTest("objects", "leading-comma-object", testType: .malformed, type: [String:String].self) + _run_json5SpecTest("objects", "lone-trailing-comma-object", testType: .malformed, type: [String:String].self) + _run_json5SpecTest("objects", "no-comma-object", testType: .malformed, type: [String:String].self) + + _run_json5SpecTest("strings", "unescaped-multi-line-string", testType: .malformed, type: String.self) + + } + + // TODO: Reenable once Data.base64EncodedString() is implemented + func testEncodingDataBase64() { + let data = Data([0xDE, 0xAD, 0xBE, 0xEF]) + + let expectedJSON = "\"3q2+7w==\"".data(using: String._Encoding.utf8)! + _testRoundTrip(of: data, expectedJSON: expectedJSON) + + // Optional data should encode the same way. + _testRoundTrip(of: Optional(data), expectedJSON: expectedJSON) + } +} + +// MARK: - .convertFromSnakeCase Tests +// TODO: Reenable these tests once convertFromSnakeCase is implemented +extension JSONEncoderTests { + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testDecodingKeyStrategyCamel() { + let fromSnakeCaseTests = [ + ("", ""), // don't die on empty string + ("a", "a"), // single character + ("ALLCAPS", "ALLCAPS"), // If no underscores, we leave the word as-is + ("ALL_CAPS", "allCaps"), // Conversion from screaming snake case + ("single", "single"), // do not capitalize anything with no underscore + ("snake_case", "snakeCase"), // capitalize a character + ("one_two_three", "oneTwoThree"), // more than one word + ("one_2_three", "one2Three"), // numerics + ("one2_three", "one2Three"), // numerics, part 2 + ("snake_Ćase", "snakeĆase"), // do not further modify a capitalized diacritic + ("snake_ćase", "snakeĆase"), // capitalize a diacritic + ("alreadyCamelCase", "alreadyCamelCase"), // do not modify already camel case + ("__this_and_that", "__thisAndThat"), + ("_this_and_that", "_thisAndThat"), + ("this__and__that", "thisAndThat"), + ("this_and_that__", "thisAndThat__"), + ("this_aNd_that", "thisAndThat"), + ("_one_two_three", "_oneTwoThree"), + ("one_two_three_", "oneTwoThree_"), + ("__one_two_three", "__oneTwoThree"), + ("one_two_three__", "oneTwoThree__"), + ("_one_two_three_", "_oneTwoThree_"), + ("__one_two_three", "__oneTwoThree"), + ("__one_two_three__", "__oneTwoThree__"), + ("_test", "_test"), + ("_test_", "_test_"), + ("__test", "__test"), + ("test__", "test__"), + ("_", "_"), + ("__", "__"), + ("___", "___"), + ("m͉̟̹y̦̳G͍͚͎̳r̤͉̤͕ͅea̲͕t͇̥̼͖U͇̝̠R͙̻̥͓̣L̥̖͎͓̪̫ͅR̩͖̩eq͈͓u̞e̱s̙t̤̺ͅ", "m͉̟̹y̦̳G͍͚͎̳r̤͉̤͕ͅea̲͕t͇̥̼͖U͇̝̠R͙̻̥͓̣L̥̖͎͓̪̫ͅR̩͖̩eq͈͓u̞e̱s̙t̤̺ͅ"), // because Itai wanted to test this + ("🐧_🐟", "🐧🐟") // fishy emoji example? + ] + + for test in fromSnakeCaseTests { + // This JSON contains the camel case key that the test object should decode with, then it uses the snake case key (test.0) as the actual key for the boolean value. + let input = "{\"camelCaseKey\":\"\(test.1)\",\"\(test.0)\":true}".data(using: String._Encoding.utf8)! + + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + + let result = try! decoder.decode(DecodeMe.self, from: input) + + XCTAssertTrue(result.found) + } + } + + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testEncodingDictionaryStringKeyConversionUntouched() { + let expected = "{\"leaveMeAlone\":\"test\"}" + let toEncode: [String: String] = ["leaveMeAlone": "test"] + + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + let resultData = try! encoder.encode(toEncode) + let resultString = String(bytes: resultData, encoding: String._Encoding.utf8) + + XCTAssertEqual(expected, resultString) + } + + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testKeyStrategySnakeGeneratedAndCustom() { + // Test that this works with a struct that has automatically generated keys + struct DecodeMe4 : Codable { + var thisIsCamelCase : String + var thisIsCamelCaseToo : String + private enum CodingKeys : String, CodingKey { + case thisIsCamelCase = "fooBar" + case thisIsCamelCaseToo + } + } + + // Decoding + let input = "{\"foo_bar\":\"test\",\"this_is_camel_case_too\":\"test2\"}".data(using: String._Encoding.utf8)! + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + let decodingResult = try! decoder.decode(DecodeMe4.self, from: input) + + XCTAssertEqual("test", decodingResult.thisIsCamelCase) + XCTAssertEqual("test2", decodingResult.thisIsCamelCaseToo) + + // Encoding + let encoded = DecodeMe4(thisIsCamelCase: "test", thisIsCamelCaseToo: "test2") + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + let encodingResultData = try! encoder.encode(encoded) + let encodingResultString = String(bytes: encodingResultData, encoding: String._Encoding.utf8) + XCTAssertTrue(encodingResultString!.contains("foo_bar")) + XCTAssertTrue(encodingResultString!.contains("this_is_camel_case_too")) + } + + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testDecodingDictionaryFailureKeyPathNested() { + let input = "{\"top_level\": {\"sub_level\": {\"nested_value\": {\"int_value\": \"not_an_int\"}}}}".data(using: String._Encoding.utf8)! + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + do { + _ = try decoder.decode([String: [String : DecodeFailureNested]].self, from: input) + } catch DecodingError.typeMismatch(_, let context) { + XCTAssertEqual(4, context.codingPath.count) + XCTAssertEqual("top_level", context.codingPath[0].stringValue) + XCTAssertEqual("sub_level", context.codingPath[1].stringValue) + XCTAssertEqual("nestedValue", context.codingPath[2].stringValue) + XCTAssertEqual("intValue", context.codingPath[3].stringValue) + } catch { + XCTFail("Unexpected error: \(String(describing: error))") + } + } + + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testDecodingKeyStrategyCamelGenerated() { + let encoded = DecodeMe3(thisIsCamelCase: "test") + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + let resultData = try! encoder.encode(encoded) + let resultString = String(bytes: resultData, encoding: String._Encoding.utf8) + XCTAssertEqual("{\"this_is_camel_case\":\"test\"}", resultString) + } + + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testEncodingKeyStrategySnakeGenerated() { + // Test that this works with a struct that has automatically generated keys + let input = "{\"this_is_camel_case\":\"test\"}".data(using: String._Encoding.utf8)! + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + let result = try! decoder.decode(DecodeMe3.self, from: input) + + XCTAssertEqual("test", result.thisIsCamelCase) + } + + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testEncodingDictionaryFailureKeyPath() { + let toEncode: [String: EncodeFailure] = ["key": EncodeFailure(someValue: Double.nan)] + + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + do { + _ = try encoder.encode(toEncode) + } catch EncodingError.invalidValue(_, let context) { + XCTAssertEqual(2, context.codingPath.count) + XCTAssertEqual("key", context.codingPath[0].stringValue) + XCTAssertEqual("someValue", context.codingPath[1].stringValue) + } catch { + XCTFail("Unexpected error: \(String(describing: error))") + } + } + + // TODO: Reenable once `convertFromSnakeCase` is implemented + func testEncodingDictionaryFailureKeyPathNested() { + let toEncode: [String: [String: EncodeFailureNested]] = ["key": ["sub_key": EncodeFailureNested(nestedValue: EncodeFailure(someValue: Double.nan))]] + + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + do { + _ = try encoder.encode(toEncode) + } catch EncodingError.invalidValue(_, let context) { + XCTAssertEqual(4, context.codingPath.count) + XCTAssertEqual("key", context.codingPath[0].stringValue) + XCTAssertEqual("sub_key", context.codingPath[1].stringValue) + XCTAssertEqual("nestedValue", context.codingPath[2].stringValue) + XCTAssertEqual("someValue", context.codingPath[3].stringValue) + } catch { + XCTFail("Unexpected error: \(String(describing: error))") + } + } + + // TODO: Reenable once `convertFromSnakeCase` is properly implemented + func testEncodingKeyStrategySnake() { + let toSnakeCaseTests = [ + ("simpleOneTwo", "simple_one_two"), + ("myURL", "my_url"), + ("singleCharacterAtEndX", "single_character_at_end_x"), + ("thisIsAnXMLProperty", "this_is_an_xml_property"), + ("single", "single"), // no underscore + ("", ""), // don't die on empty string + ("a", "a"), // single character + ("aA", "a_a"), // two characters + ("version4Thing", "version4_thing"), // numerics + ("partCAPS", "part_caps"), // only insert underscore before first all caps + ("partCAPSLowerAGAIN", "part_caps_lower_again"), // switch back and forth caps. + ("manyWordsInThisThing", "many_words_in_this_thing"), // simple lowercase + underscore + more + ("asdfĆqer", "asdf_ćqer"), + ("already_snake_case", "already_snake_case"), + ("dataPoint22", "data_point22"), + ("dataPoint22Word", "data_point22_word"), + ("_oneTwoThree", "_one_two_three"), + ("oneTwoThree_", "one_two_three_"), + ("__oneTwoThree", "__one_two_three"), + ("oneTwoThree__", "one_two_three__"), + ("_oneTwoThree_", "_one_two_three_"), + ("__oneTwoThree", "__one_two_three"), + ("__oneTwoThree__", "__one_two_three__"), + ("_test", "_test"), + ("_test_", "_test_"), + ("__test", "__test"), + ("test__", "test__"), + ("m͉̟̹y̦̳G͍͚͎̳r̤͉̤͕ͅea̲͕t͇̥̼͖U͇̝̠R͙̻̥͓̣L̥̖͎͓̪̫ͅR̩͖̩eq͈͓u̞e̱s̙t̤̺ͅ", "m͉̟̹y̦̳_g͍͚͎̳r̤͉̤͕ͅea̲͕t͇̥̼͖_u͇̝̠r͙̻̥͓̣l̥̖͎͓̪̫ͅ_r̩͖̩eq͈͓u̞e̱s̙t̤̺ͅ"), // because Itai wanted to test this + ("🐧🐟", "🐧🐟") // fishy emoji example? + ] + + for test in toSnakeCaseTests { + let expected = "{\"\(test.1)\":\"test\"}" + let encoded = EncodeMe(keyName: test.0) + + let encoder = JSONEncoder() + encoder.keyEncodingStrategy = .convertToSnakeCase + let resultData = try! encoder.encode(encoded) + let resultString = String(bytes: resultData, encoding: String._Encoding.utf8) + + XCTAssertEqual(expected, resultString) + } + } +} + +// MARK: - .sortedKeys Tests +// TODO: Reenable these tests once .sortedKeys is implemented +extension JSONEncoderTests { + func testEncodingTopLevelStructuredClass() { + // Person is a class with multiple fields. + let expectedJSON = "{\"email\":\"appleseed@apple.com\",\"name\":\"Johnny Appleseed\"}".data(using: String._Encoding.utf8)! + let person = Person.testValue + _testRoundTrip(of: person, expectedJSON: expectedJSON, outputFormatting: [.sortedKeys]) + } + + func testEncodingOutputFormattingSortedKeys() { + let expectedJSON = "{\"email\":\"appleseed@apple.com\",\"name\":\"Johnny Appleseed\"}".data(using: String._Encoding.utf8)! + let person = Person.testValue + _testRoundTrip(of: person, expectedJSON: expectedJSON, outputFormatting: [.sortedKeys]) + } + + func testEncodingOutputFormattingPrettyPrintedSortedKeys() { + let expectedJSON = "{\n \"email\" : \"appleseed@apple.com\",\n \"name\" : \"Johnny Appleseed\"\n}".data(using: String._Encoding.utf8)! + let person = Person.testValue + _testRoundTrip(of: person, expectedJSON: expectedJSON, outputFormatting: [.prettyPrinted, .sortedKeys]) + } + + func testEncodingSortedKeys() { + // When requesting sorted keys, dictionary keys are sorted prior to being written out. + // This sort should be stable, numeric, and follow human-readable sorting rules as defined by the system locale. + let dict = [ + // These three keys should appear in a stable, deterministic ordering relative to one another, regardless of their order in the dictionary. + // For example, if the sort were naively case-insensitive, these three would be `NSOrderedSame`, and would not swap with one another in the sort, maintaining their relative ordering based on their position in the dictionary. The inclusion of other keys in the dictionary can alter their relative ordering (because of hashing), producing non-stable output. + "Foo" : 1, + "FOO" : 2, + "foo" : 3, + + // These keys should output in numeric order (1, 2, 3, 11, 12) rather than literal string order (1, 11, 12, 2, 3). + "foo1" : 4, + "Foo2" : 5, + "foo3" : 6, + "foo12" : 7, + "Foo11" : 8, + + // This key should be sorted in a human-readable way (e.g. among the other "foo" keys, not after them just because the binary value of 'ø' > 'o'). + "føo" : 9, + "bar" : 10 + ] + + _testRoundTrip(of: dict, expectedJSON: "{\"bar\":10,\"foo\":3,\"Foo\":1,\"FOO\":2,\"føo\":9,\"foo1\":4,\"Foo2\":5,\"foo3\":6,\"Foo11\":8,\"foo12\":7}".data(using: String._Encoding.utf8)!, outputFormatting: [.sortedKeys]) + } + + func testEncodingSortedKeysStableOrdering() { + // We want to make sure that keys of different length (but with identical prefixes) always sort in a stable way, regardless of their hash ordering. + var dict = ["AAA" : 1, "AAAAAAB" : 2] + var expectedJSONString = "{\"AAA\":1,\"AAAAAAB\":2}" + _testRoundTrip(of: dict, expectedJSON: expectedJSONString.data(using: String._Encoding.utf8)!, outputFormatting: [.sortedKeys]) + + // We don't want this test to rely on the hashing of Strings or how Dictionary uses that hash. + // We'll insert a large number of keys into this dictionary and guarantee that the ordering of the above keys has indeed not changed. + // To ensure that we don't accidentally test the same (passing) case every time these keys will be shuffled. + let testSize = 256 + var Ns = Array(0 ..< testSize) + + // Simple Fisher-Yates shuffle. + for i in Ns.indices.reversed() { + let index = Int(Double.random(in: 0.0 ..< Double(i+1))) + let N = Ns[i] + Ns[i] = Ns[index] + + // Normally we'd set Ns[index] = N, but since all we need this value for is for inserting into the dictionary later, we can do it right here and not even write back to the source array. + // No need to do an O(n) loop over Ns again. + dict["key\(N)"] = N + } + + for i in 0 ..< testSize { + let insertedKeyJSON = ",\"key\(i)\":\(i)" + expectedJSONString.insert(contentsOf: insertedKeyJSON, at: expectedJSONString.index(before: expectedJSONString.endIndex)) + } + + _testRoundTrip(of: dict, expectedJSON: expectedJSONString.data(using: String._Encoding.utf8)!, outputFormatting: [.sortedKeys]) + } + + // TODO: Reenable once .sortedKeys is implemented + func testEncodingMultipleNestedContainersWithTheSameTopLevelKey() { + struct Model : Codable, Equatable { + let first: String + let second: String + + init(from coder: Decoder) throws { + let container = try coder.container(keyedBy: TopLevelCodingKeys.self) + + let firstNestedContainer = try container.nestedContainer(keyedBy: FirstNestedCodingKeys.self, forKey: .top) + self.first = try firstNestedContainer.decode(String.self, forKey: .first) + + let secondNestedContainer = try container.nestedContainer(keyedBy: SecondNestedCodingKeys.self, forKey: .top) + self.second = try secondNestedContainer.decode(String.self, forKey: .second) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: TopLevelCodingKeys.self) + + var firstNestedContainer = container.nestedContainer(keyedBy: FirstNestedCodingKeys.self, forKey: .top) + try firstNestedContainer.encode(self.first, forKey: .first) + + var secondNestedContainer = container.nestedContainer(keyedBy: SecondNestedCodingKeys.self, forKey: .top) + try secondNestedContainer.encode(self.second, forKey: .second) + } + + init(first: String, second: String) { + self.first = first + self.second = second + } + + static var testValue: Model { + return Model(first: "Johnny Appleseed", + second: "appleseed@apple.com") + } + + enum TopLevelCodingKeys : String, CodingKey { + case top + } + + enum FirstNestedCodingKeys : String, CodingKey { + case first + } + enum SecondNestedCodingKeys : String, CodingKey { + case second + } + } + + let model = Model.testValue + let expectedJSON = "{\"top\":{\"first\":\"Johnny Appleseed\",\"second\":\"appleseed@apple.com\"}}".data(using: String._Encoding.utf8)! + _testRoundTrip(of: model, expectedJSON: expectedJSON, outputFormatting: [.sortedKeys]) + } + + func test_redundantKeyedContainer() { + struct EncodesTwice: Encodable { + enum CodingKeys: String, CodingKey { + case container + case somethingElse + } + + struct Nested: Encodable { + let foo = "Test" + } + + func encode(to encoder: Encoder) throws { + var topLevel = encoder.container(keyedBy: CodingKeys.self) + try topLevel.encode("Foo", forKey: .somethingElse) + + // Encode an object-like JSON value for the key "container" + try topLevel.encode(Nested(), forKey: .container) + + // A nested container for the same "container" key should reuse the previous container, appending to it, instead of asserting. 106648746. + var secondAgain = topLevel.nestedContainer(keyedBy: CodingKeys.self, forKey: .container) + try secondAgain.encode("SecondAgain", forKey: .somethingElse) + } + } + + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + let data = try! encoder.encode(EncodesTwice()) + let string = String(data: data, encoding: .utf8)! + + XCTAssertEqual(string, "{\"container\":{\"foo\":\"Test\",\"somethingElse\":\"SecondAgain\"},\"somethingElse\":\"Foo\"}") + } +} + +// MARK: - Decimal Tests +// TODO: Reenable these tests once Decimal is moved +extension JSONEncoderTests { + func testInterceptDecimal() { + let expectedJSON = "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000".data(using: String._Encoding.utf8)! + + // Want to make sure we write out a JSON number, not the keyed encoding here. + // 1e127 is too big to fit natively in a Double, too, so want to make sure it's encoded as a Decimal. + let decimal = Decimal(sign: .plus, exponent: 127, significand: Decimal(1)) + _testRoundTrip(of: decimal, expectedJSON: expectedJSON) + + // Optional Decimals should encode the same way. + _testRoundTrip(of: Optional(decimal), expectedJSON: expectedJSON) + } + + func test_hugeNumbers() { + let json = "23456789012000000000000000000000000000000000000000000000000000000000000000000 " + let data = json.data(using: String._Encoding.utf8)! + + let decimal = try! JSONDecoder().decode(Decimal.self, from: data) + let expected = Decimal(string: json) + XCTAssertEqual(decimal, expected) + } + + func testInterceptLargeDecimal() { + struct TestBigDecimal: Codable, Equatable { + var uint64Max: Decimal = Decimal(UInt64.max) + var unit64MaxPlus1: Decimal = Decimal(UInt64.max) + Decimal(1) + var int64Min: Decimal = Decimal(Int64.min) + var int64MinMinus1: Decimal = Decimal(Int64.min) - Decimal(1) + } + + let testBigDecimal = TestBigDecimal() + _testRoundTrip(of: testBigDecimal) + } +} + +// MARK: - URL Tests +// TODO: Reenable these tests once URL is moved +extension JSONEncoderTests { + func testInterceptURL() { + // Want to make sure JSONEncoder writes out single-value URLs, not the keyed encoding. + let expectedJSON = "\"http:\\/\\/swift.org\"".data(using: String._Encoding.utf8)! + let url = URL(string: "http://swift.org")! + _testRoundTrip(of: url, expectedJSON: expectedJSON) + + // Optional URLs should encode the same way. + _testRoundTrip(of: Optional(url), expectedJSON: expectedJSON) + } + + func testInterceptURLWithoutEscapingOption() { + // Want to make sure JSONEncoder writes out single-value URLs, not the keyed encoding. + let expectedJSON = "\"http://swift.org\"".data(using: String._Encoding.utf8)! + let url = URL(string: "http://swift.org")! + _testRoundTrip(of: url, expectedJSON: expectedJSON, outputFormatting: [.withoutEscapingSlashes]) + + // Optional URLs should encode the same way. + _testRoundTrip(of: Optional(url), expectedJSON: expectedJSON, outputFormatting: [.withoutEscapingSlashes]) + } +} +#endif // FOUNDATION_FRAMEWORK + +// MARK: - Helper Global Functions +func expectEqualPaths(_ lhs: [CodingKey], _ rhs: [CodingKey], _ prefix: String) { + if lhs.count != rhs.count { + XCTFail("\(prefix) [CodingKey].count mismatch: \(lhs.count) != \(rhs.count)") + return + } + + for (key1, key2) in zip(lhs, rhs) { + switch (key1.intValue, key2.intValue) { + case (.none, .none): break + case (.some(let i1), .none): + XCTFail("\(prefix) CodingKey.intValue mismatch: \(type(of: key1))(\(i1)) != nil") + return + case (.none, .some(let i2)): + XCTFail("\(prefix) CodingKey.intValue mismatch: nil != \(type(of: key2))(\(i2))") + return + case (.some(let i1), .some(let i2)): + guard i1 == i2 else { + XCTFail("\(prefix) CodingKey.intValue mismatch: \(type(of: key1))(\(i1)) != \(type(of: key2))(\(i2))") + return + } + } + + XCTAssertEqual(key1.stringValue, key2.stringValue, "\(prefix) CodingKey.stringValue mismatch: \(type(of: key1))('\(key1.stringValue)') != \(type(of: key2))('\(key2.stringValue)')") + } +} + +// MARK: - Test Types +/* FIXME: Import from %S/Inputs/Coding/SharedTypes.swift somehow. */ + +// MARK: - Empty Types +fileprivate struct EmptyStruct : Codable, Equatable { + static func ==(_ lhs: EmptyStruct, _ rhs: EmptyStruct) -> Bool { + return true + } +} + +fileprivate class EmptyClass : Codable, Equatable { + static func ==(_ lhs: EmptyClass, _ rhs: EmptyClass) -> Bool { + return true + } +} + +// MARK: - Single-Value Types +/// A simple on-off switch type that encodes as a single Bool value. +fileprivate enum Switch : Codable { + case off + case on + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + switch try container.decode(Bool.self) { + case false: self = .off + case true: self = .on + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .off: try container.encode(false) + case .on: try container.encode(true) + } + } +} + +/// A simple timestamp type that encodes as a single Double value. +fileprivate struct Timestamp : Codable, Equatable { + let value: Double + + init(_ value: Double) { + self.value = value + } + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + value = try container.decode(Double.self) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(self.value) + } + + static func ==(_ lhs: Timestamp, _ rhs: Timestamp) -> Bool { + return lhs.value == rhs.value + } +} + +/// A simple referential counter type that encodes as a single Int value. +fileprivate final class Counter : Codable, Equatable { + var count: Int = 0 + + init() {} + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + count = try container.decode(Int.self) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(self.count) + } + + static func ==(_ lhs: Counter, _ rhs: Counter) -> Bool { + return lhs === rhs || lhs.count == rhs.count + } +} + +// MARK: - Structured Types +/// A simple address type that encodes as a dictionary of values. +fileprivate struct Address : Codable, Equatable { + let street: String + let city: String + let state: String + let zipCode: Int + let country: String + + init(street: String, city: String, state: String, zipCode: Int, country: String) { + self.street = street + self.city = city + self.state = state + self.zipCode = zipCode + self.country = country + } + + static func ==(_ lhs: Address, _ rhs: Address) -> Bool { + return lhs.street == rhs.street && + lhs.city == rhs.city && + lhs.state == rhs.state && + lhs.zipCode == rhs.zipCode && + lhs.country == rhs.country + } + + static var testValue: Address { + return Address(street: "1 Infinite Loop", + city: "Cupertino", + state: "CA", + zipCode: 95014, + country: "United States") + } +} + +/// A simple person class that encodes as a dictionary of values. +fileprivate class Person : Codable, Equatable { + let name: String + let email: String +#if FOUNDATION_FRAMEWORK + let website: URL? + + + init(name: String, email: String, website: URL? = nil) { + self.name = name + self.email = email + self.website = website + } +#else + init(name: String, email: String) { + self.name = name + self.email = email + } +#endif + + func isEqual(_ other: Person) -> Bool { +#if FOUNDATION_FRAMEWORK + return self.name == other.name && + self.email == other.email && + self.website == other.website +#else + return self.name == other.name && + self.email == other.email +#endif + } + + static func ==(_ lhs: Person, _ rhs: Person) -> Bool { + return lhs.isEqual(rhs) + } + + class var testValue: Person { + return Person(name: "Johnny Appleseed", email: "appleseed@apple.com") + } +} + +/// A class which shares its encoder and decoder with its superclass. +fileprivate class Employee : Person { + let id: Int + +#if FOUNDATION_FRAMEWORK + init(name: String, email: String, website: URL? = nil, id: Int) { + self.id = id + super.init(name: name, email: email, website: website) + } +#else + init(name: String, email: String, id: Int) { + self.id = id + super.init(name: name, email: email) + } +#endif + + enum CodingKeys : String, CodingKey { + case id + } + + required init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + id = try container.decode(Int.self, forKey: .id) + try super.init(from: decoder) + } + + override func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(id, forKey: .id) + try super.encode(to: encoder) + } + + override func isEqual(_ other: Person) -> Bool { + if let employee = other as? Employee { + guard self.id == employee.id else { return false } + } + + return super.isEqual(other) + } + + override class var testValue: Employee { + return Employee(name: "Johnny Appleseed", email: "appleseed@apple.com", id: 42) + } +} + +/// A simple company struct which encodes as a dictionary of nested values. +fileprivate struct Company : Codable, Equatable { + let address: Address + var employees: [Employee] + + init(address: Address, employees: [Employee]) { + self.address = address + self.employees = employees + } + + static func ==(_ lhs: Company, _ rhs: Company) -> Bool { + return lhs.address == rhs.address && lhs.employees == rhs.employees + } + + static var testValue: Company { + return Company(address: Address.testValue, employees: [Employee.testValue]) + } +} + +/// An enum type which decodes from Bool?. +fileprivate enum EnhancedBool : Codable { + case `true` + case `false` + case fileNotFound + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { + self = .fileNotFound + } else { + let value = try container.decode(Bool.self) + self = value ? .true : .false + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .true: try container.encode(true) + case .false: try container.encode(false) + case .fileNotFound: try container.encodeNil() + } + } +} + +/// A type which encodes as an array directly through a single value container. +private struct Numbers : Codable, Equatable { + let values = [4, 8, 15, 16, 23, 42] + + init() {} + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let decodedValues = try container.decode([Int].self) + guard decodedValues == values else { + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: decoder.codingPath, debugDescription: "The Numbers are wrong!")) + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(values) + } + + static func ==(_ lhs: Numbers, _ rhs: Numbers) -> Bool { + return lhs.values == rhs.values + } + + static var testValue: Numbers { + return Numbers() + } +} + +/// A type which encodes as a dictionary directly through a single value container. +fileprivate final class Mapping : Codable, Equatable { + let values: [String : Int] + + init(values: [String : Int]) { + self.values = values + } + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + values = try container.decode([String : Int].self) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(values) + } + + static func ==(_ lhs: Mapping, _ rhs: Mapping) -> Bool { + return lhs === rhs || lhs.values == rhs.values + } + + static var testValue: Mapping { + return Mapping(values: ["Apple": 42, + "localhost": 127]) + } +} + +private struct NestedContainersTestType : Encodable { + let testSuperEncoder: Bool + + init(testSuperEncoder: Bool = false) { + self.testSuperEncoder = testSuperEncoder + } + + enum TopLevelCodingKeys : Int, CodingKey { + case a + case b + case c + } + + enum IntermediateCodingKeys : Int, CodingKey { + case one + case two + } + + func encode(to encoder: Encoder) throws { + if self.testSuperEncoder { + var topLevelContainer = encoder.container(keyedBy: TopLevelCodingKeys.self) + expectEqualPaths(encoder.codingPath, [], "Top-level Encoder's codingPath changed.") + expectEqualPaths(topLevelContainer.codingPath, [], "New first-level keyed container has non-empty codingPath.") + + let superEncoder = topLevelContainer.superEncoder(forKey: .a) + expectEqualPaths(encoder.codingPath, [], "Top-level Encoder's codingPath changed.") + expectEqualPaths(topLevelContainer.codingPath, [], "First-level keyed container's codingPath changed.") + expectEqualPaths(superEncoder.codingPath, [TopLevelCodingKeys.a], "New superEncoder had unexpected codingPath.") + _testNestedContainers(in: superEncoder, baseCodingPath: [TopLevelCodingKeys.a]) + } else { + _testNestedContainers(in: encoder, baseCodingPath: []) + } + } + + func _testNestedContainers(in encoder: Encoder, baseCodingPath: [CodingKey]) { + expectEqualPaths(encoder.codingPath, baseCodingPath, "New encoder has non-empty codingPath.") + + // codingPath should not change upon fetching a non-nested container. + var firstLevelContainer = encoder.container(keyedBy: TopLevelCodingKeys.self) + expectEqualPaths(encoder.codingPath, baseCodingPath, "Top-level Encoder's codingPath changed.") + expectEqualPaths(firstLevelContainer.codingPath, baseCodingPath, "New first-level keyed container has non-empty codingPath.") + + // Nested Keyed Container + do { + // Nested container for key should have a new key pushed on. + var secondLevelContainer = firstLevelContainer.nestedContainer(keyedBy: IntermediateCodingKeys.self, forKey: .a) + expectEqualPaths(encoder.codingPath, baseCodingPath, "Top-level Encoder's codingPath changed.") + expectEqualPaths(firstLevelContainer.codingPath, baseCodingPath, "First-level keyed container's codingPath changed.") + expectEqualPaths(secondLevelContainer.codingPath, baseCodingPath + [TopLevelCodingKeys.a], "New second-level keyed container had unexpected codingPath.") + + // Inserting a keyed container should not change existing coding paths. + let thirdLevelContainerKeyed = secondLevelContainer.nestedContainer(keyedBy: IntermediateCodingKeys.self, forKey: .one) + expectEqualPaths(encoder.codingPath, baseCodingPath, "Top-level Encoder's codingPath changed.") + expectEqualPaths(firstLevelContainer.codingPath, baseCodingPath, "First-level keyed container's codingPath changed.") + expectEqualPaths(secondLevelContainer.codingPath, baseCodingPath + [TopLevelCodingKeys.a], "Second-level keyed container's codingPath changed.") + expectEqualPaths(thirdLevelContainerKeyed.codingPath, baseCodingPath + [TopLevelCodingKeys.a, IntermediateCodingKeys.one], "New third-level keyed container had unexpected codingPath.") + + // Inserting an unkeyed container should not change existing coding paths. + let thirdLevelContainerUnkeyed = secondLevelContainer.nestedUnkeyedContainer(forKey: .two) + expectEqualPaths(encoder.codingPath, baseCodingPath + [], "Top-level Encoder's codingPath changed.") + expectEqualPaths(firstLevelContainer.codingPath, baseCodingPath + [], "First-level keyed container's codingPath changed.") + expectEqualPaths(secondLevelContainer.codingPath, baseCodingPath + [TopLevelCodingKeys.a], "Second-level keyed container's codingPath changed.") + expectEqualPaths(thirdLevelContainerUnkeyed.codingPath, baseCodingPath + [TopLevelCodingKeys.a, IntermediateCodingKeys.two], "New third-level unkeyed container had unexpected codingPath.") + } + + // Nested Unkeyed Container + do { + // Nested container for key should have a new key pushed on. + var secondLevelContainer = firstLevelContainer.nestedUnkeyedContainer(forKey: .b) + expectEqualPaths(encoder.codingPath, baseCodingPath, "Top-level Encoder's codingPath changed.") + expectEqualPaths(firstLevelContainer.codingPath, baseCodingPath, "First-level keyed container's codingPath changed.") + expectEqualPaths(secondLevelContainer.codingPath, baseCodingPath + [TopLevelCodingKeys.b], "New second-level keyed container had unexpected codingPath.") + + // Appending a keyed container should not change existing coding paths. + let thirdLevelContainerKeyed = secondLevelContainer.nestedContainer(keyedBy: IntermediateCodingKeys.self) + expectEqualPaths(encoder.codingPath, baseCodingPath, "Top-level Encoder's codingPath changed.") + expectEqualPaths(firstLevelContainer.codingPath, baseCodingPath, "First-level keyed container's codingPath changed.") + expectEqualPaths(secondLevelContainer.codingPath, baseCodingPath + [TopLevelCodingKeys.b], "Second-level unkeyed container's codingPath changed.") + expectEqualPaths(thirdLevelContainerKeyed.codingPath, baseCodingPath + [TopLevelCodingKeys.b, _TestKey(index: 0)], "New third-level keyed container had unexpected codingPath.") + + // Appending an unkeyed container should not change existing coding paths. + let thirdLevelContainerUnkeyed = secondLevelContainer.nestedUnkeyedContainer() + expectEqualPaths(encoder.codingPath, baseCodingPath, "Top-level Encoder's codingPath changed.") + expectEqualPaths(firstLevelContainer.codingPath, baseCodingPath, "First-level keyed container's codingPath changed.") + expectEqualPaths(secondLevelContainer.codingPath, baseCodingPath + [TopLevelCodingKeys.b], "Second-level unkeyed container's codingPath changed.") + expectEqualPaths(thirdLevelContainerUnkeyed.codingPath, baseCodingPath + [TopLevelCodingKeys.b, _TestKey(index: 1)], "New third-level unkeyed container had unexpected codingPath.") + } + } +} + +// MARK: - Helper Types + +/// A key type which can take on any string or integer value. +/// This needs to mirror _JSONKey. +fileprivate struct _TestKey : CodingKey { + var stringValue: String + var intValue: Int? + + init?(stringValue: String) { + self.stringValue = stringValue + self.intValue = nil + } + + init?(intValue: Int) { + self.stringValue = "\(intValue)" + self.intValue = intValue + } + + init(index: Int) { + self.stringValue = "Index \(index)" + self.intValue = index + } +} + +fileprivate struct FloatNaNPlaceholder : Codable, Equatable { + init() {} + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(Float.nan) + } + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let float = try container.decode(Float.self) + if !float.isNaN { + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: decoder.codingPath, debugDescription: "Couldn't decode NaN.")) + } + } + + static func ==(_ lhs: FloatNaNPlaceholder, _ rhs: FloatNaNPlaceholder) -> Bool { + return true + } +} + +fileprivate struct DoubleNaNPlaceholder : Codable, Equatable { + init() {} + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(Double.nan) + } + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let double = try container.decode(Double.self) + if !double.isNaN { + throw DecodingError.dataCorrupted(DecodingError.Context(codingPath: decoder.codingPath, debugDescription: "Couldn't decode NaN.")) + } + } + + static func ==(_ lhs: DoubleNaNPlaceholder, _ rhs: DoubleNaNPlaceholder) -> Bool { + return true + } +} + +fileprivate enum EitherDecodable : Decodable { + case t(T) + case u(U) + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + do { + self = .t(try container.decode(T.self)) + } catch { + self = .u(try container.decode(U.self)) + } + } +} + +struct NullReader : Decodable, Equatable { + enum NullError : String, Error { + case expectedNull = "Expected a null value" + } + init(from decoder: Decoder) throws { + let c = try decoder.singleValueContainer() + guard c.decodeNil() else { + throw NullError.expectedNull + } + } +} + +enum JSONPass { } + +extension JSONPass { + struct Test1: Codable, Equatable { + let glossary: Glossary + + struct Glossary: Codable, Equatable { + let title: String + let glossDiv: GlossDiv + + enum CodingKeys: String, CodingKey { + case title + case glossDiv = "GlossDiv" + } + } + + struct GlossDiv: Codable, Equatable { + let title: String + let glossList: GlossList + + enum CodingKeys: String, CodingKey { + case title + case glossList = "GlossList" + } + } + + struct GlossList: Codable, Equatable { + let glossEntry: GlossEntry + + enum CodingKeys: String, CodingKey { + case glossEntry = "GlossEntry" + } + } + + struct GlossEntry: Codable, Equatable { + let id, sortAs, glossTerm, acronym: String + let abbrev: String + let glossDef: GlossDef + let glossSee: String + + enum CodingKeys: String, CodingKey { + case id = "ID" + case sortAs = "SortAs" + case glossTerm = "GlossTerm" + case acronym = "Acronym" + case abbrev = "Abbrev" + case glossDef = "GlossDef" + case glossSee = "GlossSee" + } + } + + struct GlossDef: Codable, Equatable { + let para: String + let glossSeeAlso: [String] + + enum CodingKeys: String, CodingKey { + case para + case glossSeeAlso = "GlossSeeAlso" + } + } + } +} + +extension JSONPass { + struct Test2: Codable, Equatable { + let menu: Menu + + struct Menu: Codable, Equatable { + let id, value: String + let popup: Popup + } + + struct Popup: Codable, Equatable { + let menuitem: [Menuitem] + } + + struct Menuitem: Codable, Equatable { + let value, onclick: String + } + } +} + +extension JSONPass { + struct Test3: Codable, Equatable { + let widget: Widget + + struct Widget: Codable, Equatable { + let debug: String + let window: Window + let image: Image + let text: Text + } + + struct Image: Codable, Equatable { + let src, name: String + let hOffset, vOffset: Int + let alignment: String + } + + struct Text: Codable, Equatable { + let data: String + let size: Int + let style, name: String + let hOffset, vOffset: Int + let alignment, onMouseUp: String + } + + struct Window: Codable, Equatable { + let title, name: String + let width, height: Int + } + } +} + +extension JSONPass { + struct Test4: Codable, Equatable { + let webApp: WebApp + + enum CodingKeys: String, CodingKey { + case webApp = "web-app" + } + + struct WebApp: Codable, Equatable { + let servlet: [Servlet] + let servletMapping: ServletMapping + let taglib: Taglib + + enum CodingKeys: String, CodingKey { + case servlet + case servletMapping = "servlet-mapping" + case taglib + } + } + + struct Servlet: Codable, Equatable { + let servletName, servletClass: String + let initParam: InitParam? + + enum CodingKeys: String, CodingKey { + case servletName = "servlet-name" + case servletClass = "servlet-class" + case initParam = "init-param" + } + } + + struct InitParam: Codable, Equatable { + let configGlossaryInstallationAt, configGlossaryAdminEmail, configGlossaryPoweredBy, configGlossaryPoweredByIcon: String? + let configGlossaryStaticPath, templateProcessorClass, templateLoaderClass, templatePath: String? + let templateOverridePath, defaultListTemplate, defaultFileTemplate: String? + let useJSP: Bool? + let jspListTemplate, jspFileTemplate: String? + let cachePackageTagsTrack, cachePackageTagsStore, cachePackageTagsRefresh, cacheTemplatesTrack: Int? + let cacheTemplatesStore, cacheTemplatesRefresh, cachePagesTrack, cachePagesStore: Int? + let cachePagesRefresh, cachePagesDirtyRead: Int? + let searchEngineListTemplate, searchEngineFileTemplate, searchEngineRobotsDB: String? + let useDataStore: Bool? + let dataStoreClass, redirectionClass, dataStoreName, dataStoreDriver: String? + let dataStoreURL, dataStoreUser, dataStorePassword, dataStoreTestQuery: String? + let dataStoreLogFile: String? + let dataStoreInitConns, dataStoreMaxConns, dataStoreConnUsageLimit: Int? + let dataStoreLogLevel: String? + let maxURLLength: Int? + let mailHost, mailHostOverride: String? + let log: Int? + let logLocation, logMaxSize: String? + let dataLog: Int? + let dataLogLocation, dataLogMaxSize, removePageCache, removeTemplateCache: String? + let fileTransferFolder: String? + let lookInContext, adminGroupID: Int? + let betaServer: Bool? + + enum CodingKeys: String, CodingKey { + case configGlossaryInstallationAt + case configGlossaryAdminEmail + case configGlossaryPoweredBy + case configGlossaryPoweredByIcon + case configGlossaryStaticPath + case templateProcessorClass, templateLoaderClass, templatePath, templateOverridePath, defaultListTemplate, defaultFileTemplate, useJSP, jspListTemplate, jspFileTemplate, cachePackageTagsTrack, cachePackageTagsStore, cachePackageTagsRefresh, cacheTemplatesTrack, cacheTemplatesStore, cacheTemplatesRefresh, cachePagesTrack, cachePagesStore, cachePagesRefresh, cachePagesDirtyRead, searchEngineListTemplate, searchEngineFileTemplate + case searchEngineRobotsDB + case useDataStore, dataStoreClass, redirectionClass, dataStoreName, dataStoreDriver + case dataStoreURL + case dataStoreUser, dataStorePassword, dataStoreTestQuery, dataStoreLogFile, dataStoreInitConns, dataStoreMaxConns, dataStoreConnUsageLimit, dataStoreLogLevel + case maxURLLength + case mailHost, mailHostOverride, log, logLocation, logMaxSize, dataLog, dataLogLocation, dataLogMaxSize, removePageCache, removeTemplateCache, fileTransferFolder, lookInContext, adminGroupID, betaServer + } + } + + struct ServletMapping: Codable, Equatable { + let cofaxCDS, cofaxEmail, cofaxAdmin, fileServlet: String + let cofaxTools: String + } + + struct Taglib: Codable, Equatable { + let taglibURI, taglibLocation: String + + enum CodingKeys: String, CodingKey { + case taglibURI = "taglib-uri" + case taglibLocation = "taglib-location" + } + } + } +} + +extension JSONPass { + struct Test5: Codable, Equatable { + let image: Image + + enum CodingKeys: String, CodingKey { + case image = "Image" + } + + struct Image: Codable, Equatable { + let width, height: Int + let title: String + let thumbnail: Thumbnail + let ids: [Int] + + enum CodingKeys: String, CodingKey { + case width = "Width" + case height = "Height" + case title = "Title" + case thumbnail = "Thumbnail" + case ids = "IDs" + } + } + + struct Thumbnail: Codable, Equatable { + let url: String + let height: Int + let width: String + + enum CodingKeys: String, CodingKey { + case url = "Url" + case height = "Height" + case width = "Width" + } + } + } +} + +extension JSONPass { + typealias Test6 = [Test6Element] + + struct Test6Element: Codable, Equatable { + let precision: String + let latitude, longitude: Double + let address, city, state, zip: String + let country: String + + enum CodingKeys: String, CodingKey { + case precision + case latitude = "Latitude" + case longitude = "Longitude" + case address = "Address" + case city = "City" + case state = "State" + case zip = "Zip" + case country = "Country" + } + + static func == (lhs: Self, rhs: Self) -> Bool { + guard lhs.precision == rhs.precision, lhs.address == rhs.address, lhs.city == rhs.city, lhs.zip == rhs.zip, lhs.country == rhs.country else { + return false + } + guard fabs(lhs.longitude - rhs.longitude) <= 1e-10 else { + return false + } + guard fabs(lhs.latitude - rhs.latitude) <= 1e-10 else { + return false + } + return true + } + } +} + +extension JSONPass { + struct Test7: Codable, Equatable { + let menu: Menu + + struct Menu: Codable, Equatable { + let header: String + let items: [Item] + } + + struct Item: Codable, Equatable { + let id: String + let label: String? + } + } +} + +extension JSONPass { + typealias Test8 = [[[[[[[[[[[[[[[[[[[String]]]]]]]]]]]]]]]]]]] +} + +extension JSONPass { + struct Test9: Codable, Equatable { + let objects : [AnyHashable] + + init(from decoder: Decoder) throws { + var container = try decoder.unkeyedContainer() + var decodedObjects = [AnyHashable]() + + decodedObjects.append(try container.decode(String.self)) + decodedObjects.append(try container.decode([String:[String]].self)) + decodedObjects.append(try container.decode([String:String].self)) + decodedObjects.append(try container.decode([String].self)) + decodedObjects.append(try container.decode(Int.self)) + decodedObjects.append(try container.decode(Bool.self)) + decodedObjects.append(try container.decode(Bool.self)) + if try container.decodeNil() { + decodedObjects.append("") + } + decodedObjects.append(try container.decode(SpecialCases.self)) + decodedObjects.append(try container.decode(Float.self)) + decodedObjects.append(try container.decode(Float.self)) + decodedObjects.append(try container.decode(Float.self)) + decodedObjects.append(try container.decode(Int.self)) + decodedObjects.append(try container.decode(Double.self)) + decodedObjects.append(try container.decode(Double.self)) + decodedObjects.append(try container.decode(Double.self)) + decodedObjects.append(try container.decode(Double.self)) + decodedObjects.append(try container.decode(Double.self)) + decodedObjects.append(try container.decode(Double.self)) + decodedObjects.append(try container.decode(String.self)) + + self.objects = decodedObjects + } + + func encode(to encoder: Encoder) throws { + var container = encoder.unkeyedContainer() + + try container.encode(objects[ 0] as! String) + try container.encode(objects[ 1] as! [String:[String]]) + try container.encode(objects[ 2] as! [String:String]) + try container.encode(objects[ 3] as! [String]) + try container.encode(objects[ 4] as! Int) + try container.encode(objects[ 5] as! Bool) + try container.encode(objects[ 6] as! Bool) + try container.encodeNil() + try container.encode(objects[ 8] as! SpecialCases) + try container.encode(objects[ 9] as! Float) + try container.encode(objects[10] as! Float) + try container.encode(objects[11] as! Float) + try container.encode(objects[12] as! Int) + try container.encode(objects[13] as! Double) + try container.encode(objects[14] as! Double) + try container.encode(objects[15] as! Double) + try container.encode(objects[16] as! Double) + try container.encode(objects[17] as! Double) + try container.encode(objects[18] as! Double) + try container.encode(objects[19] as! String) + } + + struct SpecialCases : Codable, Hashable { + let integer : UInt64 + let real : Double + let e : Double + let E : Double + let empty_key : Double + let zero : UInt8 + let one : UInt8 + let space : String + let quote : String + let backslash : String + let controls : String + let slash : String + let alpha : String + let ALPHA : String + let digit : String + let _0123456789 : String + let special : String + let hex: String + let `true` : Bool + let `false` : Bool + let null : Bool? + let array : [String] + let object : [String:String] + let address : String + #if FOUNDATION_FRAMEWORK + let url : URL + #endif + let comment : String + let special_sequences_key : String + let spaced : [Int] + let compact : [Int] + let jsontext : String + let quotes : String + let escapedKey : String + + enum CodingKeys: String, CodingKey { + case integer + case real + case e + case E + case empty_key = "" + case zero + case one + case space + case quote + case backslash + case controls + case slash + case alpha + case ALPHA + case digit + case _0123456789 = "0123456789" + case special + case hex + case `true` + case `false` + case null + case array + case object + case address + #if FOUNDATION_FRAMEWORK + case url + #endif + case comment + case special_sequences_key = "# -- --> */" + case spaced = " s p a c e d " + case compact + case jsontext + case quotes + case escapedKey = "/\\\"\u{CAFE}\u{BABE}\u{AB98}\u{FCDE}\u{bcda}\u{ef4A}\u{08}\u{0C}\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" + } + } + } +} + +extension JSONPass { + typealias Test10 = [String:[String:String]] + typealias Test11 = [String:String] +} + +extension JSONPass { + struct Test12: Codable, Equatable { + let query: Query + + struct Query: Codable, Equatable { + let pages: Pages + } + + struct Pages: Codable, Equatable { + let the80348: The80348 + + enum CodingKeys: String, CodingKey { + case the80348 = "80348" + } + } + + struct The80348: Codable, Equatable { + let pageid, ns: Int + let title: String + let langlinks: [Langlink] + } + + struct Langlink: Codable, Equatable { + let lang, asterisk: String + + enum CodingKeys: String, CodingKey { + case lang + case asterisk = "*" + } + } + } +} + +extension JSONPass { + typealias Test13 = [String:Int] + typealias Test14 = [String:[String:[String:String]]] +} + +extension JSONPass { + struct Test15: Codable, Equatable { + let attached: Bool + let klass: String + let errors: [String:[String]] + let gid: Int + let id: ID + let mpid, name: String + let properties: Properties + let state: State + let type: String + let version: Int + + enum CodingKeys: String, CodingKey { + case attached + case klass = "class" + case errors, gid, id, mpid, name, properties, state, type, version + } + + struct ID: Codable, Equatable { + let klass: String + let inc: Int + let machine: Int + let new: Bool + let time: UInt64 + let timeSecond: UInt64 + + enum CodingKeys: String, CodingKey { + case klass = "class" + case inc, machine, new, time, timeSecond + } + } + + class Properties: Codable, Equatable { + let mpid, type: String + let dbo: DBO? + let gid: Int + let name: String? + let state: State? + let apiTimestamp: String? + let gatewayTimestamp: String? + let eventData: [String:Float]? + + static func == (lhs: Properties, rhs: Properties) -> Bool { + return lhs.mpid == rhs.mpid && lhs.type == rhs.type && lhs.dbo == rhs.dbo && lhs.gid == rhs.gid && lhs.name == rhs.name && lhs.state == rhs.state && lhs.apiTimestamp == rhs.apiTimestamp && lhs.gatewayTimestamp == rhs.gatewayTimestamp && lhs.eventData == rhs.eventData + } + } + + struct DBO: Codable, Equatable { + let id: ID + let gid: Int + let mpid: String + let name: String + let type: String + let version: Int + + enum CodingKeys: String, CodingKey { + case id = "_id" + case gid, mpid, name, type, version + } + } + + struct State: Codable, Equatable { + let apiTimestamp: String + let attached: Bool + let klass : String + let errors: [String:[String]] + let eventData: [String:Float] + let gatewayTimestamp: String + let gid: Int + let id: ID + let mpid: String + let properties: Properties + let type: String + let version: Int? + + enum CodingKeys: String, CodingKey { + case apiTimestamp, attached + case klass = "class" + case errors, eventData, gatewayTimestamp, gid, id, mpid, properties, type, version + } + } + } +} + +enum JSONFail { + typealias Test1 = String + typealias Test2 = [String] + typealias Test3 = [String:String] + typealias Test4 = [String] + typealias Test5 = [String] + typealias Test6 = [String] + typealias Test7 = [String] + typealias Test8 = [String] + typealias Test9 = [String] + typealias Test10 = [String:Bool] + typealias Test11 = [String:Int] + typealias Test12 = [String:String] + typealias Test13 = [String:Int] + typealias Test14 = [String:Int] + typealias Test15 = [String] + typealias Test16 = [String] + typealias Test17 = [String] + typealias Test18 = [String] + typealias Test19 = [String:String?] + typealias Test21 = [String:String?] + typealias Test22 = [String] + typealias Test23 = [String] + typealias Test24 = [String] + typealias Test25 = [String] + typealias Test26 = [String] + typealias Test27 = [String] + typealias Test28 = [String] + typealias Test29 = [Float] + typealias Test30 = [Float] + typealias Test31 = [Float] + typealias Test32 = [String:Bool] + typealias Test33 = [String] + typealias Test34 = [String] + typealias Test35 = [String:String] + typealias Test36 = [String:Int] + typealias Test37 = [String:Int] + typealias Test38 = [String:Float] + typealias Test39 = [String:String] + typealias Test40 = [String:String] + typealias Test41 = [String:String] +} + +enum JSON5Pass { } + +extension JSON5Pass { + struct Example : Codable, Equatable { + let unquoted: String + let singleQuotes: String + let lineBreaks: String + let hexadecimal: UInt + let leadingDecimalPoint: Double + let andTrailing: Double + let positiveSign: Int + let trailingComma: String + let andIn: [String] + let backwardsCompatible: String + } +} + +extension JSON5Pass { + struct Hex : Codable, Equatable { + let `in`: [Int] + let out: [Int] + } +} + +extension JSON5Pass { + struct Numbers : Codable, Equatable { + let a: Double + let b: Double + let c: Double + let d: Int + } +} + +extension JSON5Pass { + struct Strings : Codable, Equatable { + let Hello: String + let Hello2: String + let Hello3: String + let hex1: String + let hex2: String + } +} + +extension JSON5Pass { + struct Whitespace : Codable, Equatable { + let Hello: String + } +} + +enum JSON5Spec { } + +extension JSON5Spec { + struct NPMPackage: Codable { + let name: String + let publishConfig: PublishConfig + let `description`: String + let keywords: [String] + let version: String + let preferGlobal: Bool + let config: Config + let homepage: String + let author: String + let repository: Repository + let bugs: Bugs + let directories: [String:String] + let main, bin: String + let dependencies: [String:String] + let bundleDependencies: [String] + let devDependencies: [String:String] + let engines: [String:String] + let scripts: [String:String] + let licenses: [License] + + struct PublishConfig: Codable { + let proprietaryAttribs: Bool + + enum CodingKeys: String, CodingKey { + case proprietaryAttribs = "proprietary-attribs" + } + } + + struct Config: Codable { + let publishtest: Bool + } + + struct Repository: Codable { + let type: String + let url: String + } + + struct Bugs: Codable { + let email: String + let url: String + } + + struct License: Codable { + let type: String + let url: String + } + } +} + +extension JSON5Spec { + struct ReadmeExample: Codable { + let foo: String + let `while`: Bool + let this: String + let here: String + let hex: UInt + let half: Double + let delta: Int + let to: Double + let finally: String + let oh: [String] + } +} diff --git a/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift b/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift index e2f34dab3..e4256cb36 100644 --- a/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift @@ -17,6 +17,7 @@ import TestSupport #if FOUNDATION_FRAMEWORK @testable import Foundation #else +@testable import FoundationEssentials @testable import FoundationInternationalization #endif // FOUNDATION_FRAMEWORK @@ -115,21 +116,6 @@ final class LocaleComponentsTests: XCTestCase { XCTAssertTrue(Locale.NumberingSystem.availableNumberingSystems.contains(Locale.NumberingSystem("java"))) } - // Locale components are considered equal regardless of the identifier's case - func testCaseInsensitiveEquality() { - XCTAssertEqual(Locale.Collation("search"), Locale.Collation("SEARCH")) - XCTAssertEqual(Locale.NumberingSystem("latn"), Locale.NumberingSystem("Latn")) - XCTAssertEqual( - [ Locale.NumberingSystem("latn"), Locale.NumberingSystem("ARAB") ], - [ Locale.NumberingSystem("Latn"), Locale.NumberingSystem("arab") ]) - XCTAssertEqual( - Set([ Locale.NumberingSystem("latn"), Locale.NumberingSystem("ARAB") ]), - Set([ Locale.NumberingSystem("arab"), Locale.NumberingSystem("Latn") ])) - XCTAssertEqual(Locale.Region("US"), Locale.Region("us")) - XCTAssertEqual(Locale.Script("Hant"), Locale.Script("hant")) - XCTAssertEqual(Locale.LanguageCode("EN"), Locale.LanguageCode("en")) - } - // The internal identifier getter would ignore invalid keywords and returns ICU-style identifier func testInternalIdentifier() { // In previous versions Locale.Components(identifier:) would not include @va=posix and en_US_POSIX would result in simply en_US_POSIX. We now return the @va=posix for compatibility with CFLocale. @@ -347,7 +333,7 @@ final class LocaleCodableTests: XCTestCase { // Test types that used to encode both `identifier` and `normalizdIdentifier` now only encodes `identifier` func _testRoundtripCoding(_ obj: T, identifier: String, normalizedIdentifier: String, file: StaticString = #file, line: UInt = #line) -> T? { let previousEncoded = "{\"_identifier\":\"\(identifier)\",\"_normalizedIdentifier\":\"\(normalizedIdentifier)\"}" - let previousEncodedData = previousEncoded.data(using: .utf8)! + let previousEncodedData = previousEncoded.data(using: String._Encoding.utf8)! let decoder = JSONDecoder() guard let decoded = try? decoder.decode(T.self, from: previousEncodedData) else { XCTFail("Decoding \(obj) failed", file: file, line: line) @@ -501,6 +487,96 @@ final class LocaleCodableTests: XCTestCase { } + func test_decode_compatible_localeComponents() { + func expectDecode(_ encoded: String, _ expected: Locale.Components, file: StaticString = #file, line: UInt = #line) { + guard let data = encoded.data(using: String._Encoding.utf8), let decoded = try? JSONDecoder().decode(Locale.Components.self, from: data) else { + XCTFail(file: file, line: line) + return + } + XCTAssertEqual(decoded, expected, file: file, line: line) + } + + do { + var expected = Locale.Components(identifier: "") + expected.region = "HK" + expected.firstDayOfWeek = .monday + expected.languageComponents.region = "TW" + expected.languageComponents.languageCode = "zh" + expected.hourCycle = .oneToTwelve + expected.timeZone = .gmt + expected.calendar = .buddhist + expected.currency = "GBP" + expected.measurementSystem = .us + + expectDecode(""" + {"region":{"_identifier":"HK","_normalizedIdentifier":"HK"},"firstDayOfWeek":"mon","languageComponents":{"region":{"_identifier":"TW","_normalizedIdentifier":"TW"},"languageCode":{"_identifier":"zh","_normalizedIdentifier":"zh"}},"hourCycle":"h12","timeZone":{"identifier":"GMT"},"calendar":{"buddhist":{}},"currency":{"_identifier":"GBP","_normalizedIdentifier":"gbp"},"measurementSystem":{"_identifier":"ussystem","_normalizedIdentifier":"ussystem"}} + """, expected) + } + + do { + expectDecode(""" + {"languageComponents":{}} + """, Locale.Components(identifier: "")) + } + } + + func test_decode_compatible_language() { + + func expectDecode(_ encoded: String, _ expected: Locale.Language, file: StaticString = #file, line: UInt = #line) { + guard let data = encoded.data(using: String._Encoding.utf8), let decoded = try? JSONDecoder().decode(Locale.Language.self, from: data) else { + XCTFail(file: file, line: line) + return + } + XCTAssertEqual(decoded, expected, file: file, line: line) + } + + expectDecode(""" + {"components":{"script":{"_identifier":"Hans","_normalizedIdentifier":"Hans"},"languageCode":{"_identifier":"zh","_normalizedIdentifier":"zh"},"region":{"_identifier":"HK","_normalizedIdentifier":"HK"}}} + """, Locale.Language(identifier: "zh-Hans-HK")) + + expectDecode(""" + {"components":{}} + """, Locale.Language(identifier: "")) + } + + func test_decode_compatible_languageComponents() { + func expectDecode(_ encoded: String, _ expected: Locale.Language.Components, file: StaticString = #file, line: UInt = #line) { + guard let data = encoded.data(using: String._Encoding.utf8), let decoded = try? JSONDecoder().decode(Locale.Language.Components.self, from: data) else { + XCTFail(file: file, line: line) + return + } + XCTAssertEqual(decoded, expected, file: file, line: line) + } + + expectDecode(""" + {"script":{"_identifier":"Hans","_normalizedIdentifier":"Hans"},"languageCode":{"_identifier":"zh","_normalizedIdentifier":"zh"},"region":{"_identifier":"HK","_normalizedIdentifier":"HK"}} + """, Locale.Language.Components(identifier: "zh-Hans-HK")) + + expectDecode("{}", Locale.Language.Components(identifier: "")) + } +} + +// MARK: - FoundationPreview Disabled Tests +#if FOUNDATION_FRAMEWORK +extension LocaleComponentsTests { + // TODO: Reenable once String.capitalized is implemented in Swift + // Locale components are considered equal regardless of the identifier's case + func testCaseInsensitiveEquality() { + XCTAssertEqual(Locale.Collation("search"), Locale.Collation("SEARCH")) + XCTAssertEqual(Locale.NumberingSystem("latn"), Locale.NumberingSystem("Latn")) + XCTAssertEqual( + [ Locale.NumberingSystem("latn"), Locale.NumberingSystem("ARAB") ], + [ Locale.NumberingSystem("Latn"), Locale.NumberingSystem("arab") ]) + XCTAssertEqual( + Set([ Locale.NumberingSystem("latn"), Locale.NumberingSystem("ARAB") ]), + Set([ Locale.NumberingSystem("arab"), Locale.NumberingSystem("Latn") ])) + XCTAssertEqual(Locale.Region("US"), Locale.Region("us")) + XCTAssertEqual(Locale.Script("Hant"), Locale.Script("hant")) + XCTAssertEqual(Locale.LanguageCode("EN"), Locale.LanguageCode("en")) + } +} + +extension LocaleCodableTests { func _encodeAsJSON(_ t: T) -> String? { let encoder = JSONEncoder() encoder.outputFormatting = [ .sortedKeys ] @@ -519,7 +595,7 @@ final class LocaleCodableTests: XCTestCase { XCTAssertEqual(encoded, expectedEncoded, file: file, line: line) - let data = encoded.data(using: .utf8) + let data = encoded.data(using: String._Encoding.utf8) guard let data, let decoded = try? JSONDecoder().decode(Locale.Language.self, from: data) else { XCTFail(file: file, line: line) return @@ -564,7 +640,7 @@ final class LocaleCodableTests: XCTestCase { XCTAssertEqual(encoded, expectedEncoded, file: file, line: line) - let data = encoded.data(using: .utf8) + let data = encoded.data(using: String._Encoding.utf8) guard let data, let decoded = try? JSONDecoder().decode(Locale.Language.Components.self, from: data) else { XCTFail(file: file, line: line) return @@ -604,7 +680,7 @@ final class LocaleCodableTests: XCTestCase { XCTAssertEqual(encoded, expectedEncoded, file: file, line: line) - let data = encoded.data(using: .utf8) + let data = encoded.data(using: String._Encoding.utf8) guard let data, let decoded = try? JSONDecoder().decode(Locale.Components.self, from: data) else { XCTFail(file: file, line: line) return @@ -634,74 +710,5 @@ final class LocaleCodableTests: XCTestCase { {"languageComponents":{}} """) } - - - func test_decode_compatible_localeComponents() { - func expectDecode(_ encoded: String, _ expected: Locale.Components, file: StaticString = #file, line: UInt = #line) { - guard let data = encoded.data(using: .utf8), let decoded = try? JSONDecoder().decode(Locale.Components.self, from: data) else { - XCTFail(file: file, line: line) - return - } - XCTAssertEqual(decoded, expected, file: file, line: line) - } - - do { - var expected = Locale.Components(identifier: "") - expected.region = "HK" - expected.firstDayOfWeek = .monday - expected.languageComponents.region = "TW" - expected.languageComponents.languageCode = "zh" - expected.hourCycle = .oneToTwelve - expected.timeZone = .gmt - expected.calendar = .buddhist - expected.currency = "GBP" - expected.measurementSystem = .us - - expectDecode(""" - {"region":{"_identifier":"HK","_normalizedIdentifier":"HK"},"firstDayOfWeek":"mon","languageComponents":{"region":{"_identifier":"TW","_normalizedIdentifier":"TW"},"languageCode":{"_identifier":"zh","_normalizedIdentifier":"zh"}},"hourCycle":"h12","timeZone":{"identifier":"GMT"},"calendar":{"buddhist":{}},"currency":{"_identifier":"GBP","_normalizedIdentifier":"gbp"},"measurementSystem":{"_identifier":"ussystem","_normalizedIdentifier":"ussystem"}} - """, expected) - } - - do { - expectDecode(""" - {"languageComponents":{}} - """, Locale.Components(identifier: "")) - } - } - - func test_decode_compatible_language() { - - func expectDecode(_ encoded: String, _ expected: Locale.Language, file: StaticString = #file, line: UInt = #line) { - guard let data = encoded.data(using: .utf8), let decoded = try? JSONDecoder().decode(Locale.Language.self, from: data) else { - XCTFail(file: file, line: line) - return - } - XCTAssertEqual(decoded, expected, file: file, line: line) - } - - expectDecode(""" - {"components":{"script":{"_identifier":"Hans","_normalizedIdentifier":"Hans"},"languageCode":{"_identifier":"zh","_normalizedIdentifier":"zh"},"region":{"_identifier":"HK","_normalizedIdentifier":"HK"}}} - """, Locale.Language(identifier: "zh-Hans-HK")) - - expectDecode(""" - {"components":{}} - """, Locale.Language(identifier: "")) - } - - func test_decode_compatible_languageComponents() { - func expectDecode(_ encoded: String, _ expected: Locale.Language.Components, file: StaticString = #file, line: UInt = #line) { - guard let data = encoded.data(using: .utf8), let decoded = try? JSONDecoder().decode(Locale.Language.Components.self, from: data) else { - XCTFail(file: file, line: line) - return - } - XCTAssertEqual(decoded, expected, file: file, line: line) - } - - expectDecode(""" - {"script":{"_identifier":"Hans","_normalizedIdentifier":"Hans"},"languageCode":{"_identifier":"zh","_normalizedIdentifier":"zh"},"region":{"_identifier":"HK","_normalizedIdentifier":"HK"}} - """, Locale.Language.Components(identifier: "zh-Hans-HK")) - - expectDecode("{}", Locale.Language.Components(identifier: "")) - } - } +#endif diff --git a/Tests/FoundationInternationalizationTests/LocaleTests.swift b/Tests/FoundationInternationalizationTests/LocaleTests.swift index 143dbee01..5cab816d0 100644 --- a/Tests/FoundationInternationalizationTests/LocaleTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleTests.swift @@ -271,28 +271,6 @@ final class LocalePropertiesTests : XCTestCase { _verify(locale: loc, expectedLanguage: language, script: script, languageRegion: languageRegion, region: region, subdivision: subdivision, measurementSystem: measurementSystem, calendar: calendar, hourCycle: hourCycle, currency: currency, numberingSystem: numberingSystem, numberingSystems: numberingSystems, firstDayOfWeek: firstDayOfWeek, collation: collation, variant: variant, file: file, line: line) } - func test_defaultValue() { - verify("en_US", expectedLanguage: "en", script: "Latn", languageRegion: "US", region: "US", measurementSystem: .us, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "USD", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) - - verify("en_GB", expectedLanguage: "en", script: "Latn", languageRegion: "GB", region: "GB", measurementSystem: .uk, calendar: .gregorian, hourCycle: .zeroToTwentyThree, currency: "GBP", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .monday, collation: .standard, variant: nil) - - verify("zh_TW", expectedLanguage: "zh", script: "Hant", languageRegion: "TW", region: "TW", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "TWD", numberingSystem: "latn", numberingSystems: [ "latn", "hantfin", "hanidec", "hant" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) - - verify("ar_EG", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "arab", numberingSystems: [ "latn", "arab" ], firstDayOfWeek: .saturday, collation: .standard, variant: nil) - } - - func test_keywordOverrides() { - - verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") - - // With legacy values - verify("ar_EG@calendar=ethiopic-amete-alem;collation=dictionary;measure=imperial;numbers=traditional", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .saturday, collation: "dictionary", variant: nil) - - verify("ar-EG-u-ca-ethioaa-co-dict-cu-frf-fw-fri-hc-h11-ms-uksystem-nu-traditio-rg-uszzzz",expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dictionary") - - verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz;sd=usca", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: "usca", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") - } - func test_localeComponentsAndLocale() { func verify(components: Locale.Components, identifier: String, file: StaticString = #file, line: UInt = #line) { let locFromComponents = Locale(components: components) @@ -548,7 +526,7 @@ extension LocaleTests { verify("en-GB-u-ca-islamic-rg-uszzzz", .sunday, shouldRespectUserPrefForGregorian: false, shouldRespectUserPrefForIslamic: true) } - // Reenable once (Locale.canonicalIdentifier) is implemented + // TODO: Reenable once (Locale.canonicalIdentifier) is implemented func test_identifierTypesFromICUIdentifier() throws { verify("und_ZZ", cldr: "und_ZZ", bcp47: "und-ZZ", icu: "und_ZZ") @@ -570,7 +548,7 @@ extension LocaleTests { verify("en_Hant_IL_FOO_BAR@ currency = EUR; calendar = Japanese ;", cldr: "en_Hant_IL_u_ca_japanese_cu_eur_x_lvariant_foo_bar", bcp47: "en-Hant-IL-u-ca-japanese-cu-eur-x-lvariant-foo-bar", icu: "en_Hant_IL_BAR_FOO@calendar=Japanese;currency=EUR") } - // Reimplement once (Locale.canonicalIdentifier) is implemented + // TODO: Reenable once (Locale.canonicalIdentifier) is implemented func test_identifierTypesFromBCP47Identifier() throws { verify("fr-FR-1606nict-u-ca-gregory-x-test", cldr: "fr_FR_1606nict_u_ca_gregory_x_test", bcp47: "fr-FR-1606nict-u-ca-gregory-x-test", icu: "fr_FR_1606NICT@calendar=gregorian;x=test") @@ -584,7 +562,7 @@ extension LocaleTests { verify("zh-cmn-CH-u-co-pinyin", cldr: "zh_CH_u_co_pinyin", bcp47: "zh-CH-u-co-pinyin", icu: "zh_CH@collation=pinyin") } - // Reimplemented once (Locale.canonicalIdentifier) is implemented + // TODO: Reenable once (Locale.canonicalIdentifier) is implemented func test_identifierTypesFromSpecialIdentifier() throws { verify("", cldr: "root", bcp47: "und", icu: "en_US_POSIX") verify("root", cldr: "root", bcp47: "root", icu: "root") @@ -616,6 +594,32 @@ extension LocaleTests { verify("Hant", cldr: "hant", bcp47: "hant", icu: "hant") } } + +extension LocalePropertiesTests { + // TODO: Reenable once String.capitalize is implemented + func test_defaultValue() { + verify("en_US", expectedLanguage: "en", script: "Latn", languageRegion: "US", region: "US", measurementSystem: .us, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "USD", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) + + verify("en_GB", expectedLanguage: "en", script: "Latn", languageRegion: "GB", region: "GB", measurementSystem: .uk, calendar: .gregorian, hourCycle: .zeroToTwentyThree, currency: "GBP", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .monday, collation: .standard, variant: nil) + + verify("zh_TW", expectedLanguage: "zh", script: "Hant", languageRegion: "TW", region: "TW", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "TWD", numberingSystem: "latn", numberingSystems: [ "latn", "hantfin", "hanidec", "hant" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) + + verify("ar_EG", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "arab", numberingSystems: [ "latn", "arab" ], firstDayOfWeek: .saturday, collation: .standard, variant: nil) + } + + // TODO: Reenable once String.capitalize is implemented + func test_keywordOverrides() { + + verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") + + // With legacy values + verify("ar_EG@calendar=ethiopic-amete-alem;collation=dictionary;measure=imperial;numbers=traditional", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .saturday, collation: "dictionary", variant: nil) + + verify("ar-EG-u-ca-ethioaa-co-dict-cu-frf-fw-fri-hc-h11-ms-uksystem-nu-traditio-rg-uszzzz",expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dictionary") + + verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz;sd=usca", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: "usca", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") + } +} #endif // FOUNDATION_FRAMEWORK // MARK: - Disabled Tests From 2b3a84d1f068ffdfd7ea9e82a0f62ea6d1699fba Mon Sep 17 00:00:00 2001 From: Tony Parker Date: Wed, 8 Mar 2023 17:28:37 -0800 Subject: [PATCH 18/21] rdar://106190030 (Fix crash) --- .../Locale/Locale.swift | 18 +- .../Locale/Locale_Cache.swift | 190 +++++++++-- .../Locale/Locale_ICU.swift | 318 ++++++------------ .../Locale/Locale_Wrappers.swift | 6 +- 4 files changed, 285 insertions(+), 247 deletions(-) diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index e2734d9be..81fcaad19 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -122,26 +122,19 @@ public struct Locale : Hashable, Equatable, Sendable { #if FOUNDATION_FRAMEWORK /// This returns an instance of `Locale` that's set up exactly like it would be if the user changed the current locale to that identifier, set the preferences keys in the overrides dictionary, then called `current`. internal static func localeAsIfCurrent(name: String?, cfOverrides: CFDictionary? = nil, disableBundleMatching: Bool = false) -> Locale { - let (inner, _) = _Locale._currentLocaleWithCFOverrides(name: name, overrides: cfOverrides, disableBundleMatching: disableBundleMatching) - return Locale(.fixed(inner)) + return LocaleCache.cache.localeAsIfCurrent(name: name, cfOverrides: cfOverrides, disableBundleMatching: disableBundleMatching) } #endif /// This returns an instance of `Locale` that's set up exactly like it would be if the user changed the current locale to that identifier, set the preferences keys in the overrides dictionary, then called `current`. internal static func localeAsIfCurrent(name: String?, overrides: LocalePreferences? = nil, disableBundleMatching: Bool = false) -> Locale { // On Darwin, this overrides are applied on top of CFPreferences. - let (inner, _) = _Locale._currentLocaleWithOverrides(name: name, overrides: overrides, disableBundleMatching: disableBundleMatching) - return Locale(.fixed(inner)) + return LocaleCache.cache.localeAsIfCurrent(name: name, overrides: overrides, disableBundleMatching: disableBundleMatching) } - internal static func localeAsIfCurrentWithBundleLocalizations(_ availableLocalizations: [String], allowsMixedLocalizations: Bool) -> Locale? { - guard let inner = _Locale._currentLocaleWithBundleLocalizations(availableLocalizations, allowsMixedLocalizations: allowsMixedLocalizations) else { - return nil - } - return Locale(.fixed(inner)) + return LocaleCache.cache.localeAsIfCurrentWithBundleLocalizations(availableLocalizations, allowsMixedLocalizations: allowsMixedLocalizations) } - // MARK: - // @@ -168,7 +161,8 @@ public struct Locale : Hashable, Equatable, Sendable { self = .init(components: comps) } - private init(_ kind: Kind) { + /// To be used only by `LocaleCache`. + internal init(_ kind: Kind) { self.kind = kind } @@ -930,7 +924,7 @@ public struct Locale : Hashable, Equatable, Sendable { /// - seealso: `Bundle.preferredLocalizations(from:)` /// - seealso: `Bundle.preferredLocalizations(from:forPreferences:)` public static var preferredLanguages: [String] { - _Locale.preferredLanguages(forCurrentUser: false) + LocaleCache.cache.preferredLanguages(forCurrentUser: false) } diff --git a/Sources/FoundationInternationalization/Locale/Locale_Cache.swift b/Sources/FoundationInternationalization/Locale/Locale_Cache.swift index 93680dc17..44fc961fc 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Cache.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Cache.swift @@ -60,18 +60,26 @@ struct LocaleCache : Sendable { } } - mutating func current() -> _Locale { + mutating func current(preferences: LocalePreferences?, cache: Bool) -> _Locale? { resetCurrentIfNeeded() if let cachedCurrentLocale { return cachedCurrentLocale - } else { - let (locale, doCache) = _Locale._currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: false) - if doCache { - self.cachedCurrentLocale = locale - } - return locale } + + // At this point we know we need to create, or re-create, the Locale instance. + // If we do not have a set of preferences to use, we have to return nil. + guard let preferences else { + return nil + } + + let locale = _Locale(name: nil, prefs: preferences, disableBundleMatching: false) + if cache { + // It's possible this was an 'incomplete locale', in which case we will want to calculate it again later. + self.cachedCurrentLocale = locale + } + + return locale } mutating func fixed(_ id: String) -> _Locale { @@ -98,7 +106,7 @@ struct LocaleCache : Sendable { } } - mutating func currentNSLocale() -> _NSSwiftLocale { + mutating func currentNSLocale(preferences: LocalePreferences?, cache: Bool) -> _NSSwiftLocale? { resetCurrentIfNeeded() if let currentNSLocale = cachedCurrentNSLocale { @@ -108,17 +116,25 @@ struct LocaleCache : Sendable { let nsLocale = _NSSwiftLocale(Locale(inner: current)) cachedCurrentNSLocale = nsLocale return nsLocale - } else { - // We have neither a Swift Locale nor an NSLocale. Recalculate and set both. - let (locale, doCache) = _Locale._currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: false) - let nsLocale = _NSSwiftLocale(Locale(inner: locale)) - if doCache { - // It's possible this was an 'incomplete locale', in which case we will want to calculate it again later. - self.cachedCurrentLocale = locale - cachedCurrentNSLocale = nsLocale - } - return nsLocale } + + // At this point we know we need to create, or re-create, the Locale instance. + + // If we do not have a set of preferences to use, we have to return nil. + guard let preferences else { + return nil + } + + // We have neither a Swift Locale nor an NSLocale. Recalculate and set both. + let locale = _Locale(name: nil, prefs: preferences, disableBundleMatching: false) + let nsLocale = _NSSwiftLocale(Locale(inner: locale)) + if cache { + // It's possible this was an 'incomplete locale', in which case we will want to calculate it again later. + self.cachedCurrentLocale = locale + cachedCurrentNSLocale = nsLocale + } + + return nsLocale } mutating func autoupdatingNSLocale() -> _NSSwiftLocale { @@ -179,18 +195,30 @@ struct LocaleCache : Sendable { } var current: _Locale { - lock.withLock { $0.current() } + var result = lock.withLock { + $0.current(preferences: nil, cache: false) + } + + if let result { return result } + + // We need to fetch prefs and try again + let (prefs, doCache) = preferences() + + result = lock.withLock { + $0.current(preferences: prefs, cache: doCache) + } + + guard let result else { + fatalError("Nil result getting current Locale with preferences") + } + + return result } var system: _Locale { lock.withLock { $0.system() } } - var preferred: _Locale { - let (locale, _) = _Locale._currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: false) - return locale - } - func fixed(_ id: String) -> _Locale { lock.withLock { $0.fixed(id) } } @@ -205,7 +233,24 @@ struct LocaleCache : Sendable { } func currentNSLocale() -> _NSSwiftLocale { - lock.withLock { $0.currentNSLocale() } + var result = lock.withLock { + $0.currentNSLocale(preferences: nil, cache: false) + } + + if let result { return result } + + // We need to fetch prefs and try again. Don't do this inside a lock (106190030). On Darwin it is possible to get a KVO callout from fetching the preferences, which could ask for the current Locale, which could cause a reentrant lock. + let (prefs, doCache) = preferences() + + result = lock.withLock { + $0.currentNSLocale(preferences: prefs, cache: doCache) + } + + guard let result else { + fatalError("Nil result getting current NSLocale with preferences") + } + + return result } func systemNSLocale() -> _NSSwiftLocale { @@ -216,4 +261,99 @@ struct LocaleCache : Sendable { func fixedComponents(_ comps: Locale.Components) -> _Locale { lock.withLock { $0.fixedComponents(comps) } } + +#if FOUNDATION_FRAMEWORK + func preferences() -> (LocalePreferences, Bool) { + // On Darwin, we check the current user preferences for Locale values + var wouldDeadlock: DarwinBoolean = false + let cfPrefs = __CFXPreferencesCopyCurrentApplicationStateWithDeadlockAvoidance(&wouldDeadlock).takeRetainedValue() + + var prefs = LocalePreferences() + prefs.apply(cfPrefs) + + if wouldDeadlock.boolValue { + // Don't cache a locale built with incomplete prefs + return (prefs, false) + } else { + return (prefs, true) + } + } + + func preferredLanguages(forCurrentUser: Bool) -> [String] { + var languages: [String] = [] + if forCurrentUser { + languages = CFPreferencesCopyValue("AppleLanguages" as CFString, kCFPreferencesAnyApplication, kCFPreferencesCurrentUser, kCFPreferencesAnyHost) as? [String] ?? [] + } else { + languages = CFPreferencesCopyAppValue("AppleLanguages" as CFString, kCFPreferencesCurrentApplication) as? [String] ?? [] + } + + return languages.compactMap { + Locale.canonicalLanguageIdentifier(from: $0) + } + } + + func preferredLocale() -> String? { + guard let preferredLocaleID = CFPreferencesCopyAppValue("AppleLocale" as CFString, kCFPreferencesCurrentApplication) as? String else { + return nil + } + return preferredLocaleID + } +#else + func preferences() -> (LocalePreferences, Bool) { + var prefs = LocalePreferences() + prefs.locale = "en_US" + prefs.languages = ["en-US"] + return (prefs, true) + } + + func preferredLanguages(forCurrentUser: Bool) -> [String] { + [Locale.canonicalLanguageIdentifier(from: "en-US")] + } + + func preferredLocale() -> String? { + "en_US" + } +#endif + +#if FOUNDATION_FRAMEWORK + /// This returns an instance of `Locale` that's set up exactly like it would be if the user changed the current locale to that identifier, set the preferences keys in the overrides dictionary, then called `current`. + func localeAsIfCurrent(name: String?, cfOverrides: CFDictionary? = nil, disableBundleMatching: Bool = false) -> Locale { + + var (prefs, _) = preferences() + if let cfOverrides { prefs.apply(cfOverrides) } + + let inner = _Locale(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) + return Locale(.fixed(inner)) + } +#endif + + /// This returns an instance of `Locale` that's set up exactly like it would be if the user changed the current locale to that identifier, set the preferences keys in the overrides dictionary, then called `current`. + func localeAsIfCurrent(name: String?, overrides: LocalePreferences? = nil, disableBundleMatching: Bool = false) -> Locale { + var (prefs, _) = preferences() + if let overrides { prefs.apply(overrides) } + + let inner = _Locale(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) + return Locale(.fixed(inner)) + } + + + func localeAsIfCurrentWithBundleLocalizations(_ availableLocalizations: [String], allowsMixedLocalizations: Bool) -> Locale? { + guard !allowsMixedLocalizations else { + let (prefs, _) = preferences() + let inner = _Locale(name: nil, prefs: prefs, disableBundleMatching: true) + return Locale(.fixed(inner)) + } + + let preferredLanguages = preferredLanguages(forCurrentUser: false) + guard let preferredLocaleID = preferredLocale() else { return nil } + + let canonicalizedLocalizations = availableLocalizations.compactMap { Locale.canonicalLanguageIdentifier(from: $0) } + let identifier = _Locale.localeIdentifierForCanonicalizedLocalizations(canonicalizedLocalizations, preferredLanguages: preferredLanguages, preferredLocaleID: preferredLocaleID) + guard let identifier else { + return nil + } + + let inner = _Locale(identifier: identifier) + return Locale(.fixed(inner)) + } } diff --git a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift index a3a58444d..380bf9146 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_ICU.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_ICU.swift @@ -108,6 +108,14 @@ internal final class _Locale: Sendable, Hashable { internal let prefs: LocalePreferences? private let lock: LockedState + // MARK: - Logging +#if FOUNDATION_FRAMEWORK + static private let log: OSLog = { + OSLog(subsystem: "com.apple.foundation", category: "locale") + }() +#endif // FOUNDATION_FRAMEWORK + + // MARK: - init init(identifier: String, prefs: LocalePreferences? = nil) { @@ -140,6 +148,108 @@ internal final class _Locale: Sendable, Hashable { lock = LockedState(initialState: state) } + /// Use to create a current-like Locale, with preferences. + init(name: String?, prefs: LocalePreferences, disableBundleMatching: Bool) { + var ident: String? + if let name { + ident = Locale._canonicalLocaleIdentifier(from: name) +#if FOUNDATION_FRAMEWORK + if Self.log.isEnabled(type: .debug) { + if let ident { + let components = Locale.Components(identifier: ident) + if components.languageComponents.region == nil { + Logger(Self.log).debug("Current locale fetched with overriding locale identifier '\(ident, privacy: .public)' which does not have a country code") + } + } + } +#endif // FOUNDATION_FRAMEWORK + } + + if let identSet = ident { + ident = Locale._canonicalLocaleIdentifier(from: identSet) + } else { + let preferredLocale = prefs.locale + + // If CFBundleAllowMixedLocalizations is set, don't do any checking of the user's preferences for locale-matching purposes (32264371) +#if FOUNDATION_FRAMEWORK + let allowMixed = Bundle.main.infoDictionary?["CFBundleAllowMixedLocalizations"] as? Bool ?? false +#else + let allowMixed = false +#endif + let performBundleMatching = !disableBundleMatching && !allowMixed + + let preferredLanguages = prefs.languages + + #if FOUNDATION_FRAMEWORK + if preferredLanguages == nil && (preferredLocale == nil || performBundleMatching) { + Logger(Self.log).debug("Lookup of 'AppleLanguages' from current preferences failed lookup (app preferences do not contain the key); likely falling back to default locale identifier as current") + } + #endif + + // Since localizations can contains legacy lproj names such as `English`, `French`, etc. we need to canonicalize these into language identifiers such as `en`, `fr`, etc. Otherwise the logic that later compares these to language identifiers will fail. () + // `preferredLanguages` has not yet been canonicalized, and if we won't perform the bundle matching below (and have a preferred locale), we don't need to canonicalize the list up-front. We'll do so below on demand. + var canonicalizedLocalizations: [String]? + + if let preferredLocale, let preferredLanguages, performBundleMatching { + let mainBundle = Bundle.main + let availableLocalizations = mainBundle.localizations + canonicalizedLocalizations = Self.canonicalizeLocalizations(availableLocalizations) + + ident = Self.localeIdentifierForCanonicalizedLocalizations(canonicalizedLocalizations!, preferredLanguages: preferredLanguages, preferredLocaleID: preferredLocale) + } + + if ident == nil { + // Either we didn't need to match the locale identifier against the main bundle's localizations, or were unable to. + if let preferredLocale { + ident = Locale._canonicalLocaleIdentifier(from: preferredLocale) + } else if let preferredLanguages { + if canonicalizedLocalizations == nil { + canonicalizedLocalizations = Self.canonicalizeLocalizations(preferredLanguages) + } + + if canonicalizedLocalizations!.count > 0 { + let languageName = canonicalizedLocalizations![0] + + // This variable name is a bit confusing, but we do indeed mean to call the canonicalLocaleIdentifier function here and not canonicalLanguageIdentifier. + let languageIdentifier = Locale._canonicalLocaleIdentifier(from: languageName) + // Country??? + if let countryCode = prefs.country { + #if FOUNDATION_FRAMEWORK + Logger(Self.log).debug("Locale.current constructing a locale identifier from preferred languages by combining with set country code '\(countryCode, privacy: .public)'") + #endif // FOUNDATION_FRAMEWORK + ident = Locale._canonicalLocaleIdentifier(from: "\(languageIdentifier)_\(countryCode)") + } else { + #if FOUNDATION_FRAMEWORK + Logger(Self.log).debug("Locale.current constructing a locale identifier from preferred languages without a set country code") + #endif // FOUNDATION_FRAMEWORK + ident = Locale._canonicalLocaleIdentifier(from: languageIdentifier) + } + } else { + #if FOUNDATION_FRAMEWORK + Logger(Self.log).debug("Value for 'AppleLanguages' found in preferences contains no valid entries; falling back to default locale identifier as current") + #endif // FOUNDATION_FRAMEWORK + } + } else { + // We're going to fall back below. + // At this point, we've logged about both `preferredLocale` and `preferredLanguages` being missing, so no need to log again. + } + } + } + + if ident == nil { + #if os(macOS) + ident = "" + #else + ident = "en_US" + #endif + } + + self.identifier = Locale._canonicalLocaleIdentifier(from: ident!) + doesNotRequireSpecialCaseHandling = Self.identifierDoesNotRequireSpecialCaseHandling(self.identifier) + self.prefs = prefs + lock = LockedState(initialState: State()) + } + deinit { lock.withLock { state in state.cleanup() @@ -1211,7 +1321,7 @@ internal final class _Locale: Sendable, Hashable { if let prefs, let override = prefs.languages { langs = override } else { - langs = _Locale.preferredLanguages(forCurrentUser: false) + langs = LocaleCache.cache.preferredLanguages(forCurrentUser: false) } for l in langs { @@ -1379,212 +1489,6 @@ internal final class _Locale: Sendable, Hashable { Locale.canonicalLanguageIdentifier(from: $0) } } - - // MARK: - - - static func preferredLanguages(forCurrentUser: Bool) -> [String] { -#if FOUNDATION_FRAMEWORK // TODO: (_Locale.preferredLanguages) Implement `preferredLanguages` for Linux - var languages: [String] = [] - if forCurrentUser { - languages = CFPreferencesCopyValue("AppleLanguages" as CFString, kCFPreferencesAnyApplication, kCFPreferencesCurrentUser, kCFPreferencesAnyHost) as? [String] ?? [] - } else { - languages = CFPreferencesCopyAppValue("AppleLanguages" as CFString, kCFPreferencesCurrentApplication) as? [String] ?? [] - } -#else - // Fallback to en - let languages: [String] = ["en"] -#endif // FOUNDATION_FRAMEWORK - return canonicalizeLocalizations(languages) - } - - // MARK: - -#if FOUNDATION_FRAMEWORK - static private let log: OSLog = { - OSLog(subsystem: "com.apple.foundation", category: "locale") - }() -#endif // FOUNDATION_FRAMEWORK - -#if FOUNDATION_FRAMEWORK - static func _prefsFromCFPrefs() -> (LocalePreferences, Bool) { - // On Darwin, we check the current user preferences for Locale values - var wouldDeadlock: DarwinBoolean = false - let cfPrefs = __CFXPreferencesCopyCurrentApplicationStateWithDeadlockAvoidance(&wouldDeadlock).takeRetainedValue() - - var prefs = LocalePreferences() - prefs.apply(cfPrefs) - - if wouldDeadlock.boolValue { - // Don't cache a locale built with incomplete prefs - return (prefs, false) - } else { - return (prefs, true) - } - } - - /// Create a `Locale` that acts like a `currentLocale`, using `CFPreferences` values with `CFDictionary` overrides. - internal static func _currentLocaleWithCFOverrides(name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> (_Locale, Bool) { - var (prefs, wouldDeadlock) = _prefsFromCFPrefs() - if let overrides { - prefs.apply(overrides) - } - let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) - - let suitableForCache = !disableBundleMatching && !wouldDeadlock - return (result, suitableForCache) - } - - /// Create a `Locale` that acts like a `currentLocale`, using `CFPreferences` values and `LocalePreferences` overrides. - internal static func _currentLocaleWithOverrides(name: String?, overrides: LocalePreferences?, disableBundleMatching: Bool) -> (_Locale, Bool) { - var (prefs, wouldDeadlock) = _prefsFromCFPrefs() - if let overrides { - prefs.apply(overrides) - } - let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) - - let suitableForCache = !disableBundleMatching && !wouldDeadlock - return (result, suitableForCache) - } -#else - /// Create a `Locale` that acts like a `currentLocale`, using default values and `LocalePreferences` overrides. - internal static func _currentLocaleWithOverrides(name: String?, overrides: LocalePreferences?, disableBundleMatching: Bool) -> (_Locale, Bool) { - let suitableForCache = disableBundleMatching ? false : true - - // On this platform, preferences start with default values - var prefs = LocalePreferences() - prefs.locale = "en_US" - prefs.languages = ["en-US"] - - // Apply overrides - if let overrides { prefs.apply(overrides) } - - let result = _localeWithPreferences(name: name, prefs: prefs, disableBundleMatching: disableBundleMatching) - return (result, suitableForCache) - } -#endif - - private static func _localeWithPreferences(name: String?, prefs: LocalePreferences, disableBundleMatching: Bool) -> _Locale { - var ident: String? - if let name { - ident = Locale._canonicalLocaleIdentifier(from: name) -#if FOUNDATION_FRAMEWORK - if log.isEnabled(type: .debug) { - if let ident { - let components = Locale.Components(identifier: ident) - if components.languageComponents.region == nil { - Logger(log).debug("Current locale fetched with overriding locale identifier '\(ident, privacy: .public)' which does not have a country code") - } - } - } -#endif // FOUNDATION_FRAMEWORK - } - - if let identSet = ident { - ident = Locale._canonicalLocaleIdentifier(from: identSet) - } else { - let preferredLocale = prefs.locale - - // If CFBundleAllowMixedLocalizations is set, don't do any checking of the user's preferences for locale-matching purposes (32264371) -#if FOUNDATION_FRAMEWORK - let allowMixed = Bundle.main.infoDictionary?["CFBundleAllowMixedLocalizations"] as? Bool ?? false -#else - let allowMixed = false -#endif - let performBundleMatching = !disableBundleMatching && !allowMixed - - let preferredLanguages = prefs.languages - - #if FOUNDATION_FRAMEWORK - if preferredLanguages == nil && (preferredLocale == nil || performBundleMatching) { - Logger(log).debug("Lookup of 'AppleLanguages' from current preferences failed lookup (app preferences do not contain the key); likely falling back to default locale identifier as current") - } - #endif - - // Since localizations can contains legacy lproj names such as `English`, `French`, etc. we need to canonicalize these into language identifiers such as `en`, `fr`, etc. Otherwise the logic that later compares these to language identifiers will fail. () - // `preferredLanguages` has not yet been canonicalized, and if we won't perform the bundle matching below (and have a preferred locale), we don't need to canonicalize the list up-front. We'll do so below on demand. - var canonicalizedLocalizations: [String]? - - if let preferredLocale, let preferredLanguages, performBundleMatching { - let mainBundle = Bundle.main - let availableLocalizations = mainBundle.localizations - canonicalizedLocalizations = canonicalizeLocalizations(availableLocalizations) - - ident = localeIdentifierForCanonicalizedLocalizations(canonicalizedLocalizations!, preferredLanguages: preferredLanguages, preferredLocaleID: preferredLocale) - } - - if ident == nil { - // Either we didn't need to match the locale identifier against the main bundle's localizations, or were unable to. - if let preferredLocale { - ident = Locale._canonicalLocaleIdentifier(from: preferredLocale) - } else if let preferredLanguages { - if canonicalizedLocalizations == nil { - canonicalizedLocalizations = canonicalizeLocalizations(preferredLanguages) - } - - if canonicalizedLocalizations!.count > 0 { - let languageName = canonicalizedLocalizations![0] - - // This variable name is a bit confusing, but we do indeed mean to call the canonicalLocaleIdentifier function here and not canonicalLanguageIdentifier. - let languageIdentifier = Locale._canonicalLocaleIdentifier(from: languageName) - // Country??? - if let countryCode = prefs.country { - #if FOUNDATION_FRAMEWORK - Logger(log).debug("Locale.current constructing a locale identifier from preferred languages by combining with set country code '\(countryCode, privacy: .public)'") - #endif // FOUNDATION_FRAMEWORK - ident = Locale._canonicalLocaleIdentifier(from: "\(languageIdentifier)_\(countryCode)") - } else { - #if FOUNDATION_FRAMEWORK - Logger(log).debug("Locale.current constructing a locale identifier from preferred languages without a set country code") - #endif // FOUNDATION_FRAMEWORK - ident = Locale._canonicalLocaleIdentifier(from: languageIdentifier) - } - } else { - #if FOUNDATION_FRAMEWORK - Logger(log).debug("Value for 'AppleLanguages' found in preferences contains no valid entries; falling back to default locale identifier as current") - #endif // FOUNDATION_FRAMEWORK - } - } else { - // We're going to fall back below. - // At this point, we've logged about both `preferredLocale` and `preferredLanguages` being missing, so no need to log again. - } - } - } - - if ident == nil { - #if os(macOS) - ident = "" - #else - ident = "en_US" - #endif - } - let locale = _Locale(identifier: ident!, prefs: prefs) - return locale - } - - internal static func _currentLocaleWithBundleLocalizations(_ availableLocalizations: [String], allowsMixedLocalizations: Bool) -> _Locale? { -#if FOUNDATION_FRAMEWORK - guard !allowsMixedLocalizations else { - let (result, _) = _currentLocaleWithOverrides(name: nil, overrides: nil, disableBundleMatching: true) - return result - } - - let preferredLanguages = preferredLanguages(forCurrentUser: false) - guard let preferredLocaleID = CFPreferencesCopyAppValue("AppleLocale" as CFString, kCFPreferencesCurrentApplication) as? String else { - return nil - } - - let canonicalizedLocalizations = canonicalizeLocalizations(availableLocalizations) - let identifier = localeIdentifierForCanonicalizedLocalizations(canonicalizedLocalizations, preferredLanguages: preferredLanguages, preferredLocaleID: preferredLocaleID) - guard let identifier else { - return nil - } - - return LocaleCache.cache.fixed(identifier) -#else - // TODO: Implement preferred locale once UserDefaults is moved - return nil -#endif // FOUNDATION_FRAMEWORK - } - } // MARK: - diff --git a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift index 4f9981b90..03a2d6614 100644 --- a/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift +++ b/Sources/FoundationInternationalization/Locale/Locale_Wrappers.swift @@ -43,13 +43,13 @@ extension NSLocale { @objc private class func _newLocaleAsIfCurrent(_ name: String?, overrides: CFDictionary?, disableBundleMatching: Bool) -> NSLocale? { - let inner = Locale.localeAsIfCurrent(name: name, cfOverrides: overrides, disableBundleMatching: disableBundleMatching) + let inner = LocaleCache.cache.localeAsIfCurrent(name: name, cfOverrides: overrides, disableBundleMatching: disableBundleMatching) return _NSSwiftLocale(inner) } @objc(_currentLocaleWithBundleLocalizations:disableBundleMatching:) private class func _currentLocaleWithBundleLocalizations(_ availableLocalizations: [String], allowsMixedLocalizations: Bool) -> NSLocale? { - guard let inner = Locale.localeAsIfCurrentWithBundleLocalizations(availableLocalizations, allowsMixedLocalizations: allowsMixedLocalizations) else { + guard let inner = LocaleCache.cache.localeAsIfCurrentWithBundleLocalizations(availableLocalizations, allowsMixedLocalizations: allowsMixedLocalizations) else { return nil } return _NSSwiftLocale(inner) @@ -62,7 +62,7 @@ extension NSLocale { @objc private class func _preferredLanguagesForCurrentUser(_ forCurrentUser: Bool) -> [String] { - _Locale.preferredLanguages(forCurrentUser: forCurrentUser) + LocaleCache.cache.preferredLanguages(forCurrentUser: forCurrentUser) } @objc From ec6077b4e7bee3bfe3e76f25e9e232f69cb1ca5e Mon Sep 17 00:00:00 2001 From: Tina Liu Date: Thu, 23 Mar 2023 15:56:40 -0700 Subject: [PATCH 19/21] rdar://107156343 (Rebased JSONEncoder on top of new String changes) --- .../String/UnicodeScalar.swift | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/Sources/FoundationEssentials/String/UnicodeScalar.swift b/Sources/FoundationEssentials/String/UnicodeScalar.swift index 269830ffc..c71de7f15 100644 --- a/Sources/FoundationEssentials/String/UnicodeScalar.swift +++ b/Sources/FoundationEssentials/String/UnicodeScalar.swift @@ -31,23 +31,11 @@ extension UnicodeScalar { } var _isGraphemeExtend: Bool { -#if FOUNDATION_FRAMEWORK // TODO: Implement `CFUniCharGetBitmapPtrForPlane` in Swift - let truncated = UInt16(truncatingIfNeeded: value) // intentionally truncated - let bitmap = CFUniCharGetBitmapPtrForPlane(UInt32(kCFUniCharGraphemeExtendCharacterSet), (value < 0x10000) ? 0 : (value >> 16)) - return CFUniCharIsMemberOfBitmap(truncated, bitmap) -#else - return false -#endif + return BuiltInUnicodeScalarSet.graphemeExtend.contains(self) } var _isCanonicalDecomposable: Bool { -#if FOUNDATION_FRAMEWORK // TODO: Implement `CFUniCharGetBitmapPtrForPlane` in Swift - let truncated = UInt16(truncatingIfNeeded: value) - let bitmap = CFUniCharGetBitmapPtrForPlane(UInt32(kCFUniCharCanonicalDecomposableCharacterSet), value >> 16) - return CFUniCharIsMemberOfBitmap(truncated, bitmap) -#else - return false -#endif + return BuiltInUnicodeScalarSet.canonicalDecomposable.contains(self) } func _stripDiacritics() -> Self { From f4d36057403727ef99a0f3cfd621a811addf326f Mon Sep 17 00:00:00 2001 From: Charles Hu Date: Thu, 23 Mar 2023 18:34:09 -0700 Subject: [PATCH 20/21] Rebased on top of the new String APIs --- .../JSON/JSONDecoder.swift | 10 +- .../JSON/JSONEncoder.swift | 13 +- .../String+Essentials.swift | 43 --- .../String/String+Essentials.swift | 121 ++++++++ .../String/String+Extensions.swift | 38 +++ .../String/StringAPIs.swift | 267 ------------------ .../StringProtocol+Essentials.swift | 40 +++ .../String/StringProtocol+Stub.swift | 28 ++ .../String/UnicodeScalar.swift | 6 +- .../{ => String}/String+Locale.swift | 0 .../{ => String}/StringProtocol+Locale.swift | 0 11 files changed, 234 insertions(+), 332 deletions(-) delete mode 100644 Sources/FoundationEssentials/String+Essentials.swift create mode 100644 Sources/FoundationEssentials/String/String+Essentials.swift delete mode 100644 Sources/FoundationEssentials/String/StringAPIs.swift rename Sources/FoundationEssentials/{ => String}/StringProtocol+Essentials.swift (56%) create mode 100644 Sources/FoundationEssentials/String/StringProtocol+Stub.swift rename Sources/FoundationInternationalization/{ => String}/String+Locale.swift (100%) rename Sources/FoundationInternationalization/{ => String}/StringProtocol+Locale.swift (100%) diff --git a/Sources/FoundationEssentials/JSON/JSONDecoder.swift b/Sources/FoundationEssentials/JSON/JSONDecoder.swift index 1c9c8a003..1773b5f2d 100644 --- a/Sources/FoundationEssentials/JSON/JSONDecoder.swift +++ b/Sources/FoundationEssentials/JSON/JSONDecoder.swift @@ -21,17 +21,9 @@ import Glibc /// /// The marker protocol also provides access to the type of the `Decodable` values, /// which is needed for the implementation of the key conversion strategy exemption. -/// -/// NOTE: Please see comment above regarding SR-8276 -#if arch(i386) || arch(arm) -internal protocol _JSONStringDictionaryDecodableMarker { - static var elementType: Decodable.Type { get } -} -#else private protocol _JSONStringDictionaryDecodableMarker { static var elementType: Decodable.Type { get } } -#endif extension Dictionary : _JSONStringDictionaryDecodableMarker where Key == String, Value: Decodable { static var elementType: Decodable.Type { return Value.self } @@ -667,7 +659,7 @@ extension JSONDecoderImpl: Decoder { } } -#if FOUNDATION_FRAMEWORK // TODO: Reenable once DateFormatStyle has been moved +#if FOUNDATION_FRAMEWORK // TODO: Reenable once URL and Decimal has been moved private func unwrapURL(from mapValue: JSONMap.Value, for codingPathNode: _JSONCodingPathNode, _ additionalKey: (some CodingKey)? = nil) throws -> URL { try checkNotNull(mapValue, expectedType: URL.self, for: codingPathNode, additionalKey) diff --git a/Sources/FoundationEssentials/JSON/JSONEncoder.swift b/Sources/FoundationEssentials/JSON/JSONEncoder.swift index b5e13dfbe..048fe96bb 100644 --- a/Sources/FoundationEssentials/JSON/JSONEncoder.swift +++ b/Sources/FoundationEssentials/JSON/JSONEncoder.swift @@ -12,16 +12,7 @@ /// A marker protocol used to determine whether a value is a `String`-keyed `Dictionary` /// containing `Encodable` values (in which case it should be exempt from key conversion strategies). -/// -/// NOTE: The architecture and environment check is due to a bug in the current (2018-08-08) Swift 4.2 -/// runtime when running on i386 simulator. The issue is tracked in https://bugs.swift.org/browse/SR-8276 -/// Making the protocol `internal` instead of `private` works around this issue. -/// Once SR-8276 is fixed, this check can be removed and the protocol always be made private. -#if arch(i386) || arch(arm) -internal protocol _JSONStringDictionaryEncodableMarker { } -#else private protocol _JSONStringDictionaryEncodableMarker { } -#endif extension Dictionary : _JSONStringDictionaryEncodableMarker where Key == String, Value: Encodable { } @@ -34,7 +25,9 @@ extension Dictionary : _JSONStringDictionaryEncodableMarker where Key == String, // The two must coexist, so it was renamed. The old name must not be // used in the new runtime. _TtC10Foundation13__JSONEncoder is the // mangled name for Foundation.__JSONEncoder. +#if FOUNDATION_FRAMEWORK @_objcRuntimeName(_TtC10Foundation13__JSONEncoder) +#endif @available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) open class JSONEncoder { // MARK: Options @@ -125,7 +118,7 @@ open class JSONEncoder { case useDefaultKeys #if FOUNDATION_FRAMEWORK - // TODO: Reenable this option once String.capitalize() is moved + // TODO: Reenable this option once String.rangeOfCharacter(from:) is moved /// Convert from "camelCaseKeys" to "snake_case_keys" before writing a key to JSON payload. /// diff --git a/Sources/FoundationEssentials/String+Essentials.swift b/Sources/FoundationEssentials/String+Essentials.swift deleted file mode 100644 index d043d26cb..000000000 --- a/Sources/FoundationEssentials/String+Essentials.swift +++ /dev/null @@ -1,43 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -extension String { - func _capitalized() -> String { - var new = "" - new.reserveCapacity(utf8.count) - - let uppercaseSet = BuiltInUnicodeScalarSet.uppercaseLetter - let lowercaseSet = BuiltInUnicodeScalarSet.lowercaseLetter - let cfcaseIgnorableSet = BuiltInUnicodeScalarSet.caseIgnorable - - var isLastCased = false - for scalar in unicodeScalars { - let properties = scalar.properties - if uppercaseSet.contains(scalar) { - new += isLastCased ? properties.lowercaseMapping : String(scalar) - isLastCased = true - } else if lowercaseSet.contains(scalar) { - new += isLastCased ? String(scalar) : properties.titlecaseMapping - isLastCased = true - } else if !cfcaseIgnorableSet.contains(scalar) { - // We only use a subset of case-ignorable characters as defined in CF instead of the full set of characters satisfying `property.isCaseIgnorable` for compatibility reasons - new += String(scalar) - isLastCased = false - } else { - new += String(scalar) - } - } - - return new - } - -} diff --git a/Sources/FoundationEssentials/String/String+Essentials.swift b/Sources/FoundationEssentials/String/String+Essentials.swift new file mode 100644 index 000000000..096453957 --- /dev/null +++ b/Sources/FoundationEssentials/String/String+Essentials.swift @@ -0,0 +1,121 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +extension String { + func _capitalized() -> String { + var new = "" + new.reserveCapacity(utf8.count) + + let uppercaseSet = BuiltInUnicodeScalarSet.uppercaseLetter + let lowercaseSet = BuiltInUnicodeScalarSet.lowercaseLetter + let cfcaseIgnorableSet = BuiltInUnicodeScalarSet.caseIgnorable + + var isLastCased = false + for scalar in unicodeScalars { + let properties = scalar.properties + if uppercaseSet.contains(scalar) { + new += isLastCased ? properties.lowercaseMapping : String(scalar) + isLastCased = true + } else if lowercaseSet.contains(scalar) { + new += isLastCased ? String(scalar) : properties.titlecaseMapping + isLastCased = true + } else if !cfcaseIgnorableSet.contains(scalar) { + // We only use a subset of case-ignorable characters as defined in CF instead of the full set of characters satisfying `property.isCaseIgnorable` for compatibility reasons + new += String(scalar) + isLastCased = false + } else { + new += String(scalar) + } + } + + return new + } + + /// Creates a new string equivalent to the given bytes interpreted in the + /// specified encoding. + /// + /// - Parameters: + /// - bytes: A sequence of bytes to interpret using `encoding`. + /// - encoding: The ecoding to use to interpret `bytes`. + public init?(bytes: __shared S, encoding: Encoding) + where S.Iterator.Element == UInt8 + { +#if FOUNDATION_FRAMEWORK // TODO: Move init?(bytes:encoding) to Swift + func makeString(bytes: UnsafeBufferPointer) -> String? { + if encoding == .utf8 || encoding == .ascii, + let str = String._tryFromUTF8(bytes) { + if encoding == .utf8 || (encoding == .ascii && str._guts._isContiguousASCII) { + return str + } + } + + if let ns = NSString( + bytes: bytes.baseAddress.unsafelyUnwrapped, length: bytes.count, encoding: encoding.rawValue) { + return String._unconditionallyBridgeFromObjectiveC(ns) + } else { + return nil + } + } + if let string = (bytes.withContiguousStorageIfAvailable(makeString) ?? + Array(bytes).withUnsafeBufferPointer(makeString)) { + self = string + } else { + return nil + } +#else + guard encoding == .utf8 || encoding == .ascii else { + return nil + } + func makeString(buffer: UnsafeBufferPointer) -> String? { + if let string = String._tryFromUTF8(buffer), + (encoding == .utf8 || (encoding == .ascii && string._guts._isContiguousASCII)) { + return string + } + + return buffer.withMemoryRebound(to: CChar.self) { ptr in + guard let address = ptr.baseAddress else { + return nil + } + return String(validatingUTF8: address) + } + } + + if let string = bytes.withContiguousStorageIfAvailable(makeString) ?? + Array(bytes).withUnsafeBufferPointer(makeString) { + self = string + } else { + return nil + } +#endif // FOUNDATION_FRAMEWORK + } + + /// Returns a `String` initialized by converting given `data` into + /// Unicode characters using a given `encoding`. + public init?(data: __shared Data, encoding: Encoding) { + if encoding == .utf8 || encoding == .ascii, + let str = data.withUnsafeBytes({ + String._tryFromUTF8($0.bindMemory(to: UInt8.self)) + }) { + if encoding == .utf8 || (encoding == .ascii && str._guts._isContiguousASCII) { + self = str + return + } + } +#if FOUNDATION_FRAMEWORK + guard let s = NSString(data: data, encoding: encoding.rawValue) else { return nil } + self = String._unconditionallyBridgeFromObjectiveC(s) +#else + return nil +#endif // FOUNDATION_FRAMEWORK + } +} diff --git a/Sources/FoundationEssentials/String/String+Extensions.swift b/Sources/FoundationEssentials/String/String+Extensions.swift index 194c2be3d..0d732e616 100644 --- a/Sources/FoundationEssentials/String/String+Extensions.swift +++ b/Sources/FoundationEssentials/String/String+Extensions.swift @@ -20,3 +20,41 @@ extension Character { } } + +extension Substring.UnicodeScalarView { + func _rangeOfCharacter(from set: CharacterSet, anchored: Bool, backwards: Bool) -> Range? { + guard !isEmpty else { return nil } + + let fromLoc: String.Index + let toLoc: String.Index + let step: Int + if backwards { + fromLoc = index(before: endIndex) + toLoc = anchored ? fromLoc : startIndex + step = -1 + } else { + fromLoc = startIndex + toLoc = anchored ? fromLoc : index(before: endIndex) + step = 1 + } + + var done = false + var found = false + + var idx = fromLoc + while !done { + let ch = self[idx] + if set.contains(ch) { + done = true + found = true + } else if idx == toLoc { + done = true + } else { + formIndex(&idx, offsetBy: step) + } + } + + guard found else { return nil } + return idx.. Bool { return true } -#endif - -@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) -extension StringProtocol { - // - (NSRange)rangeOfCharacterFromSet:(NSCharacterSet *)aSet - // - // - (NSRange) - // rangeOfCharacterFromSet:(NSCharacterSet *)aSet - // options:(StringCompareOptions)mask - // - // - (NSRange) - // rangeOfCharacterFromSet:(NSCharacterSet *)aSet - // options:(StringCompareOptions)mask - // range:(NSRange)aRange - - /// Finds and returns the range in the `String` of the first - /// character from a given character set found in a given range with - /// given options. - public func rangeOfCharacter(from aSet: CharacterSet, options mask: String.CompareOptions = [], range aRange: Range? = nil) -> Range? { - if _foundation_essentials_feature_enabled() { - var subStr = Substring(self) - if let aRange { - subStr = subStr[aRange] - } - return subStr._rangeOfCharacter(from: aSet, options: mask) - } - -#if FOUNDATION_FRAMEWORK - return aSet.withUnsafeImmutableStorage { - return _optionalRange(_ns._rangeOfCharacter(from: $0, options: mask, range: _toRelativeNSRange(aRange ?? startIndex.. Data? { - switch encoding { - case .utf8: - return Data(self.utf8) - default: -#if FOUNDATION_FRAMEWORK // TODO: Implement data(using:allowLossyConversion:) in Swift - return _ns.data( - using: encoding.rawValue, - allowLossyConversion: allowLossyConversion) -#else - return nil -#endif - } - } -} - -@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) -extension String { - //===--- Initializers that can fail -------------------------------------===// - // - (instancetype) - // initWithBytes:(const void *)bytes - // length:(NSUInteger)length - // encoding:(NSStringEncoding)encoding - - /// Creates a new string equivalent to the given bytes interpreted in the - /// specified encoding. - /// - /// - Parameters: - /// - bytes: A sequence of bytes to interpret using `encoding`. - /// - encoding: The ecoding to use to interpret `bytes`. - public init?(bytes: __shared S, encoding: Encoding) - where S.Iterator.Element == UInt8 - { -#if FOUNDATION_FRAMEWORK // TODO: Move init?(bytes:encoding) to Swift - func makeString(bytes: UnsafeBufferPointer) -> String? { - if encoding == .utf8 || encoding == .ascii, - let str = String._tryFromUTF8(bytes) { - if encoding == .utf8 || (encoding == .ascii && str._guts._isContiguousASCII) { - return str - } - } - - if let ns = NSString( - bytes: bytes.baseAddress.unsafelyUnwrapped, length: bytes.count, encoding: encoding.rawValue) { - return String._unconditionallyBridgeFromObjectiveC(ns) - } else { - return nil - } - } - if let string = (bytes.withContiguousStorageIfAvailable(makeString) ?? - Array(bytes).withUnsafeBufferPointer(makeString)) { - self = string - } else { - return nil - } -#else - guard encoding == .utf8 || encoding == .ascii else { - return nil - } - func makeString(buffer: UnsafeBufferPointer) -> String? { - if let string = String._tryFromUTF8(buffer), - (encoding == .utf8 || (encoding == .ascii && string._guts._isContiguousASCII)) { - return string - } - - return buffer.withMemoryRebound(to: CChar.self) { ptr in - guard let address = ptr.baseAddress else { - return nil - } - return String(validatingUTF8: address) - } - } - - if let string = bytes.withContiguousStorageIfAvailable(makeString) ?? - Array(bytes).withUnsafeBufferPointer(makeString) { - self = string - } else { - return nil - } -#endif // FOUNDATION_FRAMEWORK - } - - // - (instancetype) - // initWithData:(NSData *)data - // encoding:(NSStringEncoding)encoding - - /// Returns a `String` initialized by converting given `data` into - /// Unicode characters using a given `encoding`. - public init?(data: __shared Data, encoding: Encoding) { - if encoding == .utf8 || encoding == .ascii, - let str = data.withUnsafeBytes({ - String._tryFromUTF8($0.bindMemory(to: UInt8.self)) - }) { - if encoding == .utf8 || (encoding == .ascii && str._guts._isContiguousASCII) { - self = str - return - } - } -#if FOUNDATION_FRAMEWORK - guard let s = NSString(data: data, encoding: encoding.rawValue) else { return nil } - self = String._unconditionallyBridgeFromObjectiveC(s) -#else - return nil -#endif // FOUNDATION_FRAMEWORK - } -} - -// MARK: - Stubbed Methods -extension StringProtocol { -#if !FOUNDATION_FRAMEWORK - // - (NSComparisonResult) - // compare:(NSString *)aString - // - // - (NSComparisonResult) - // compare:(NSString *)aString options:(StringCompareOptions)mask - // - // - (NSComparisonResult) - // compare:(NSString *)aString options:(StringCompareOptions)mask - // range:(NSRange)range - // - // - (NSComparisonResult) - // compare:(NSString *)aString options:(StringCompareOptions)mask - // range:(NSRange)range locale:(id)locale - - /// Compares the string using the specified options and - /// returns the lexical ordering for the range. - internal func compare(_ aString: T, options mask: String.CompareOptions = [], range: Range? = nil) -> ComparisonResult { - // TODO: This method is modified from `public func compare(_ aString: T, options mask: String.CompareOptions = [], range: Range? = nil, locale: Locale? = nil) -> ComparisonResult`. Move that method here once `Locale` can be staged in `FoundationEssentials`. - var substr = Substring(self) - if let range { - substr = substr[range] - } - return substr._unlocalizedCompare(other: Substring(aString), options: mask) - } -#endif -} - - -extension Substring.UnicodeScalarView { - func _rangeOfCharacter(from set: CharacterSet, anchored: Bool, backwards: Bool) -> Range? { - guard !isEmpty else { return nil } - - let fromLoc: String.Index - let toLoc: String.Index - let step: Int - if backwards { - fromLoc = index(before: endIndex) - toLoc = anchored ? fromLoc : startIndex - step = -1 - } else { - fromLoc = startIndex - toLoc = anchored ? fromLoc : index(before: endIndex) - step = 1 - } - - var done = false - var found = false - - var idx = fromLoc - while !done { - let ch = self[idx] - if set.contains(ch) { - done = true - found = true - } else if idx == toLoc { - done = true - } else { - formIndex(&idx, offsetBy: step) - } - } - - guard found else { return nil } - return idx.. Bool { return true } #endif @available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) @@ -45,4 +47,42 @@ extension StringProtocol { return String(self)._capitalized() #endif } + + /// Finds and returns the range in the `String` of the first + /// character from a given character set found in a given range with + /// given options. + public func rangeOfCharacter(from aSet: CharacterSet, options mask: String.CompareOptions = [], range aRange: Range? = nil) -> Range? { + if _foundation_essentials_feature_enabled() { + var subStr = Substring(self) + if let aRange { + subStr = subStr[aRange] + } + return subStr._rangeOfCharacter(from: aSet, options: mask) + } + +#if FOUNDATION_FRAMEWORK + return aSet.withUnsafeImmutableStorage { + return _optionalRange(_ns._rangeOfCharacter(from: $0, options: mask, range: _toRelativeNSRange(aRange ?? startIndex.. Data? { + switch encoding { + case .utf8: + return Data(self.utf8) + default: +#if FOUNDATION_FRAMEWORK // TODO: Implement data(using:allowLossyConversion:) in Swift + return _ns.data( + using: encoding.rawValue, + allowLossyConversion: allowLossyConversion) +#else + return nil +#endif + } + } } diff --git a/Sources/FoundationEssentials/String/StringProtocol+Stub.swift b/Sources/FoundationEssentials/String/StringProtocol+Stub.swift new file mode 100644 index 000000000..3ff0304c1 --- /dev/null +++ b/Sources/FoundationEssentials/String/StringProtocol+Stub.swift @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if !FOUNDATION_FRAMEWORK +@available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) +extension StringProtocol { + /// Compares the string using the specified options and + /// returns the lexical ordering for the range. + internal func compare(_ aString: T, options mask: String.CompareOptions = [], range: Range? = nil) -> ComparisonResult { + // TODO: This method is modified from `public func compare(_ aString: T, options mask: String.CompareOptions = [], range: Range? = nil, locale: Locale? = nil) -> ComparisonResult`. Move that method here once `Locale` can be staged in `FoundationEssentials`. + var substr = Substring(self) + if let range { + substr = substr[range] + } + return substr._unlocalizedCompare(other: Substring(aString), options: mask) + } +} + +#endif // !FOUNDATION_FRAMEWORK diff --git a/Sources/FoundationEssentials/String/UnicodeScalar.swift b/Sources/FoundationEssentials/String/UnicodeScalar.swift index c71de7f15..3c351b4d1 100644 --- a/Sources/FoundationEssentials/String/UnicodeScalar.swift +++ b/Sources/FoundationEssentials/String/UnicodeScalar.swift @@ -26,7 +26,7 @@ extension UnicodeScalar { return self } #else - return self + fatalError("_toHalfWidth is not implemented yet") #endif } @@ -59,7 +59,7 @@ extension UnicodeScalar { return stripped != nil ? UnicodeScalar(stripped!)! : self #else - return self + fatalError("_stripDiacritics is not implemented yet") #endif // FOUNDATION_FRAMEWORK } @@ -67,7 +67,7 @@ extension UnicodeScalar { #if FOUNDATION_FRAMEWORK // TODO: Expose Case Mapping Data without @_spi(_Unicode) return self.properties._caseFolded #else - return "" + fatalError("_caseFoldMapping is not implemented yet") #endif } } diff --git a/Sources/FoundationInternationalization/String+Locale.swift b/Sources/FoundationInternationalization/String/String+Locale.swift similarity index 100% rename from Sources/FoundationInternationalization/String+Locale.swift rename to Sources/FoundationInternationalization/String/String+Locale.swift diff --git a/Sources/FoundationInternationalization/StringProtocol+Locale.swift b/Sources/FoundationInternationalization/String/StringProtocol+Locale.swift similarity index 100% rename from Sources/FoundationInternationalization/StringProtocol+Locale.swift rename to Sources/FoundationInternationalization/String/StringProtocol+Locale.swift From e2fb807ee1b3d78b83658033917243cf0e12cc22 Mon Sep 17 00:00:00 2001 From: Charles Hu Date: Mon, 27 Mar 2023 09:02:13 -0700 Subject: [PATCH 21/21] Enabled more tests --- Package.swift | 3 +- .../JSON/JSONWriter.swift | 6 ++- .../Locale/Locale.swift | 4 +- .../JSONEncoderTests.swift | 2 +- .../LocaleComponentsTests.swift | 7 +-- .../LocaleTests.swift | 47 +++++++++---------- 6 files changed, 35 insertions(+), 34 deletions(-) diff --git a/Package.swift b/Package.swift index ce87e044b..285c48bd8 100644 --- a/Package.swift +++ b/Package.swift @@ -14,7 +14,8 @@ let package = Package( .library(name: "FoundationNetworking", targets: ["FoundationNetworking"]) ], dependencies: [ - .package(url: "git@github.com:apple/swift-foundation-icu.git", branch: "main") + // Intentionally use this deprecated version so we can control the package name. + .package(name: "swift-foundation-icu", url: "git@github.com:apple/swift-foundation-icu.git", branch: "main") ], targets: [ // Foundation (umbrella) diff --git a/Sources/FoundationEssentials/JSON/JSONWriter.swift b/Sources/FoundationEssentials/JSON/JSONWriter.swift index 82b875958..6ac8c45d2 100644 --- a/Sources/FoundationEssentials/JSON/JSONWriter.swift +++ b/Sources/FoundationEssentials/JSON/JSONWriter.swift @@ -312,8 +312,11 @@ internal struct JSONWriter { } } -// MARK: - Exported Types +// MARK: - WritingOptions extension JSONWriter { +#if FOUNDATION_FRAMEWORK + typealias WritingOptions = JSONSerialization.WritingOptions +#else struct WritingOptions : OptionSet, Sendable { let rawValue: UInt @@ -330,4 +333,5 @@ extension JSONWriter { /// Specifies that the output doesn’t prefix slash characters with escape characters. static let withoutEscapingSlashes = WritingOptions(rawValue: 1 << 3) } +#endif // FOUNDATION_FRAMEWORK } diff --git a/Sources/FoundationInternationalization/Locale/Locale.swift b/Sources/FoundationInternationalization/Locale/Locale.swift index 81fcaad19..5599542a4 100644 --- a/Sources/FoundationInternationalization/Locale/Locale.swift +++ b/Sources/FoundationInternationalization/Locale/Locale.swift @@ -890,11 +890,13 @@ public struct Locale : Hashable, Equatable, Sendable { switch kind { case .autoupdating: return LocaleCache.cache.current.prefs case .fixed(let l): return l.prefs +#if FOUNDATION_FRAMEWORK case .bridged(_): return nil +#endif } } - #if FOUNDATION_FRAMEWORK +#if FOUNDATION_FRAMEWORK internal func pref(for key: String) -> Any? { switch kind { case .autoupdating: return LocaleCache.cache.current.pref(for: key) diff --git a/Tests/FoundationEssentialsTests/JSONEncoderTests.swift b/Tests/FoundationEssentialsTests/JSONEncoderTests.swift index c864fcaeb..9907c20d0 100644 --- a/Tests/FoundationEssentialsTests/JSONEncoderTests.swift +++ b/Tests/FoundationEssentialsTests/JSONEncoderTests.swift @@ -1223,7 +1223,7 @@ final class JSONEncoderTests : XCTestCase { } func test_topLevelNumberFragmentsWithJunkDigitCharacters() { - let fullData = "3.141596".data(using: .utf8)! + let fullData = "3.141596".data(using: String._Encoding.utf8)! let partialData = fullData[0..<4] XCTAssertEqual(3.14, try JSONDecoder().decode(Double.self, from: partialData)) diff --git a/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift b/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift index e4256cb36..4cbec57c3 100644 --- a/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleComponentsTests.swift @@ -554,12 +554,7 @@ final class LocaleCodableTests: XCTestCase { expectDecode("{}", Locale.Language.Components(identifier: "")) } -} -// MARK: - FoundationPreview Disabled Tests -#if FOUNDATION_FRAMEWORK -extension LocaleComponentsTests { - // TODO: Reenable once String.capitalized is implemented in Swift // Locale components are considered equal regardless of the identifier's case func testCaseInsensitiveEquality() { XCTAssertEqual(Locale.Collation("search"), Locale.Collation("SEARCH")) @@ -576,6 +571,8 @@ extension LocaleComponentsTests { } } +// MARK: - FoundationPreview Disabled Tests +#if FOUNDATION_FRAMEWORK extension LocaleCodableTests { func _encodeAsJSON(_ t: T) -> String? { let encoder = JSONEncoder() diff --git a/Tests/FoundationInternationalizationTests/LocaleTests.swift b/Tests/FoundationInternationalizationTests/LocaleTests.swift index 5cab816d0..bf9a18ca5 100644 --- a/Tests/FoundationInternationalizationTests/LocaleTests.swift +++ b/Tests/FoundationInternationalizationTests/LocaleTests.swift @@ -426,6 +426,28 @@ final class LocalePropertiesTests : XCTestCase { XCTAssertEqual(customizedLocale.decimalSeparator, "*") XCTAssertEqual(customizedLocale.groupingSeparator, "-") } + + func test_defaultValue() { + verify("en_US", expectedLanguage: "en", script: "Latn", languageRegion: "US", region: "US", measurementSystem: .us, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "USD", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) + + verify("en_GB", expectedLanguage: "en", script: "Latn", languageRegion: "GB", region: "GB", measurementSystem: .uk, calendar: .gregorian, hourCycle: .zeroToTwentyThree, currency: "GBP", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .monday, collation: .standard, variant: nil) + + verify("zh_TW", expectedLanguage: "zh", script: "Hant", languageRegion: "TW", region: "TW", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "TWD", numberingSystem: "latn", numberingSystems: [ "latn", "hantfin", "hanidec", "hant" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) + + verify("ar_EG", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "arab", numberingSystems: [ "latn", "arab" ], firstDayOfWeek: .saturday, collation: .standard, variant: nil) + } + + func test_keywordOverrides() { + + verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") + + // With legacy values + verify("ar_EG@calendar=ethiopic-amete-alem;collation=dictionary;measure=imperial;numbers=traditional", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .saturday, collation: "dictionary", variant: nil) + + verify("ar-EG-u-ca-ethioaa-co-dict-cu-frf-fw-fri-hc-h11-ms-uksystem-nu-traditio-rg-uszzzz",expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dictionary") + + verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz;sd=usca", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: "usca", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") + } } // MARK: - Bridging Tests @@ -595,31 +617,6 @@ extension LocaleTests { } } -extension LocalePropertiesTests { - // TODO: Reenable once String.capitalize is implemented - func test_defaultValue() { - verify("en_US", expectedLanguage: "en", script: "Latn", languageRegion: "US", region: "US", measurementSystem: .us, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "USD", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) - - verify("en_GB", expectedLanguage: "en", script: "Latn", languageRegion: "GB", region: "GB", measurementSystem: .uk, calendar: .gregorian, hourCycle: .zeroToTwentyThree, currency: "GBP", numberingSystem: "latn", numberingSystems: [ "latn" ], firstDayOfWeek: .monday, collation: .standard, variant: nil) - - verify("zh_TW", expectedLanguage: "zh", script: "Hant", languageRegion: "TW", region: "TW", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "TWD", numberingSystem: "latn", numberingSystems: [ "latn", "hantfin", "hanidec", "hant" ], firstDayOfWeek: .sunday, collation: .standard, variant: nil) - - verify("ar_EG", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .metric, calendar: .gregorian, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "arab", numberingSystems: [ "latn", "arab" ], firstDayOfWeek: .saturday, collation: .standard, variant: nil) - } - - // TODO: Reenable once String.capitalize is implemented - func test_keywordOverrides() { - - verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") - - // With legacy values - verify("ar_EG@calendar=ethiopic-amete-alem;collation=dictionary;measure=imperial;numbers=traditional", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "EG", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .oneToTwelve, currency: "EGP", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .saturday, collation: "dictionary", variant: nil) - - verify("ar-EG-u-ca-ethioaa-co-dict-cu-frf-fw-fri-hc-h11-ms-uksystem-nu-traditio-rg-uszzzz",expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: nil, measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditional", numberingSystems: [ "traditional", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dictionary") - - verify("ar_EG@calendar=ethioaa;collation=dict;currency=frf;fw=fri;hours=h11;measure=uksystem;numbers=traditio;rg=uszzzz;sd=usca", expectedLanguage: "ar", script: "arab", languageRegion: "EG", region: "us", subdivision: "usca", measurementSystem: .uk, calendar: .ethiopicAmeteAlem, hourCycle: .zeroToEleven, currency: "FRF", numberingSystem: "traditio", numberingSystems: [ "traditio", "latn", "arab" ], firstDayOfWeek: .friday, collation: "dict") - } -} #endif // FOUNDATION_FRAMEWORK // MARK: - Disabled Tests