From f2ea71680b89f95c322d72705e626167f72afca8 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 18 Nov 2024 22:21:38 -0600 Subject: [PATCH 001/234] Fix a non-portable target reference in CMake helper function (#829) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes an oversight in a helper function in the project’s CMake rules: `_swift_testing_install_target()` has a hard-coded reference to the target named ”Testing” but it should instead refer to whatever target was passed in via the `module` argument to the function. Discovered while working on #825. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- cmake/modules/SwiftModuleInstallation.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/SwiftModuleInstallation.cmake b/cmake/modules/SwiftModuleInstallation.cmake index 6122cb71a..b291f1ff0 100644 --- a/cmake/modules/SwiftModuleInstallation.cmake +++ b/cmake/modules/SwiftModuleInstallation.cmake @@ -43,7 +43,7 @@ function(get_swift_testing_install_lib_dir type result_var_name) endfunction() function(_swift_testing_install_target module) - target_compile_options(Testing PRIVATE "-no-toolchain-stdlib-rpath") + target_compile_options(${module} PRIVATE "-no-toolchain-stdlib-rpath") if(APPLE) set_target_properties(${module} PROPERTIES From ab18b73ffc68dbefbf4c508294f059f4b36684d0 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 19 Nov 2024 12:04:23 -0600 Subject: [PATCH 002/234] Expand the 'Known Issues' documentation article (#823) This expands the [Known issues](https://swiftpackageindex.com/swiftlang/swift-testing/main/documentation/testing/known-issues) DocC article to give examples and describe several different ways it can be used. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Resolves rdar://137961874 --- Sources/Testing/Issues/KnownIssue.swift | 24 +++- Sources/Testing/Testing.docc/known-issues.md | 141 ++++++++++++++++++- 2 files changed, 157 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Issues/KnownIssue.swift b/Sources/Testing/Issues/KnownIssue.swift index 4d7c16739..f59e9d422 100644 --- a/Sources/Testing/Issues/KnownIssue.swift +++ b/Sources/Testing/Issues/KnownIssue.swift @@ -96,7 +96,7 @@ public typealias KnownIssueMatcher = @Sendable (_ issue: Issue) -> Bool /// be attributed. /// - body: The function to invoke. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail. For example: /// /// ```swift @@ -112,6 +112,10 @@ public typealias KnownIssueMatcher = @Sendable (_ issue: Issue) -> Bool /// while others should continue to cause test failures, use /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` /// instead. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, @@ -143,7 +147,7 @@ public func withKnownIssue( /// - Throws: Whatever is thrown by `body`, unless it is matched by /// `issueMatcher`. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail, or if a precondition affects whether /// issues are known to occur. For example: /// @@ -165,6 +169,10 @@ public func withKnownIssue( /// instead. /// /// - Note: `issueMatcher` may be invoked more than once for the same issue. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, @@ -205,7 +213,7 @@ public func withKnownIssue( /// be attributed. /// - body: The function to invoke. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail. For example: /// /// ```swift @@ -221,6 +229,10 @@ public func withKnownIssue( /// while others should continue to cause test failures, use /// ``withKnownIssue(_:isIntermittent:isolation:sourceLocation:_:when:matching:)`` /// instead. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, @@ -254,7 +266,7 @@ public func withKnownIssue( /// - Throws: Whatever is thrown by `body`, unless it is matched by /// `issueMatcher`. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail, or if a precondition affects whether /// issues are known to occur. For example: /// @@ -276,6 +288,10 @@ public func withKnownIssue( /// instead. /// /// - Note: `issueMatcher` may be invoked more than once for the same issue. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, diff --git a/Sources/Testing/Testing.docc/known-issues.md b/Sources/Testing/Testing.docc/known-issues.md index 31906a5df..e400e3032 100644 --- a/Sources/Testing/Testing.docc/known-issues.md +++ b/Sources/Testing/Testing.docc/known-issues.md @@ -10,13 +10,146 @@ See https://swift.org/LICENSE.txt for license information See https://swift.org/CONTRIBUTORS.txt for Swift project authors --> -Highlight known issues when running tests. +Mark issues as known when running tests. ## Overview -The testing library provides a function, `withKnownIssue()`, that you -use to mark issues as known. Use this function to inform the testing library -at runtime not to mark the test as failing when issues occur. +The testing library provides several functions named `withKnownIssue()` that +you can use to mark issues as known. Use them to inform the testing library that +a test should not be marked as failing if only known issues are recorded. + +### Mark an expectation failure as known + +Consider a test function with a single expectation: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + #expect(foodTruck.grill.isHeating) // ❌ Expectation failed +} +``` + +If the value of the `isHeating` property is `false`, `#expect` will record an +issue. If you cannot fix the underlying problem, you can surround the failing +code in a closure passed to `withKnownIssue()`: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + withKnownIssue("Propane tank is empty") { + #expect(foodTruck.grill.isHeating) // Known issue + } +} +``` + +The issue recorded by `#expect` will then be considered "known" and the test +will not be marked as a failure. You may include an optional comment to explain +the problem or provide context. + +### Mark a thrown error as known + +If an `Error` is caught by the closure passed to `withKnownIssue()`, the issue +representing that caught error will be marked as known. Continuing the previous +example, suppose the problem is that the `startGrill()` function is throwing an +error. You can apply `withKnownIssue()` to this situation as well: + +```swift +@Test func grillHeating() { + var foodTruck = FoodTruck() + withKnownIssue { + try foodTruck.startGrill() // Known issue + #expect(foodTruck.grill.isHeating) + } +} +``` + +Because all errors thrown from the closure are caught and interpreted as known +issues, the `withKnownIssue()` function is not throwing. Consequently, any +subsequent code which depends on the throwing call having succeeded (such as the +`#expect` after `startGrill()`) must be included in the closure to avoid +additional issues. + +- Note: `withKnownIssue()` is recommended instead of `#expect(throws:)` for any + error which is considered a known issue so that the test status and results + will reflect the situation more accurately. + +### Match a specific issue + +By default, `withKnownIssue()` considers all issues recorded while invoking the +body closure known. If multiple issues may be recorded, you can pass a trailing +closure labeled `matching:` which will be called once for each recorded issue +to determine whether it should be treated as known: + +```swift +@Test func batteryLevel() throws { + var foodTruck = FoodTruck() + try withKnownIssue { + let batteryLevel = try #require(foodTruck.batteryLevel) // Known + #expect(batteryLevel >= 0.8) // Not considered known + } matching: { issue in + guard case .expectationFailed(let expectation) = issue.kind else { + return false + } + return expectation.isRequired + } +} +``` + +### Resolve a known issue + +If there are no issues recorded while calling `function`, `withKnownIssue()` +will record a distinct issue about the lack of any issues having been recorded. +This notifies you that the underlying problem may have been resolved so that you +can investigate and consider removing `withKnownIssue()` if it's no longer +necessary. + +### Handle a nondeterministic failure + +If `withKnownIssue()` sometimes succeeds but other times records an issue +indicating there were no known issues, this may indicate a nondeterministic +failure or a "flaky" test. + +The first step in resolving a nondeterministic test failure is to analyze the +code being tested and determine the source of the unpredictable behavior. If +you discover a bug such as a race condition, the ideal resolution is to fix +the underlying problem so that the code always behaves consistently even if +it continues to exhibit the known issue. + +If the underlying problem only occurs in certain circumstances, consider +including a precondition. For example, if the grill only fails to heat when +there's no propane, you can pass a trailing closure labeled `when:` which +determines whether issues recorded in the body closure should be considered +known: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + withKnownIssue { + // Only considered known when hasPropane == false + #expect(foodTruck.grill.isHeating) + } when: { + !hasPropane + } +} +``` + +If the underlying problem is unpredictable and fails at random, you can pass +`isIntermittent: true` to let the testing library know that it will not always +occur. Then, the testing library will not record an issue when zero known issues +are recorded: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + withKnownIssue(isIntermittent: true) { + #expect(foodTruck.grill.isHeating) + } +} +``` ## Topics From 1df9545d99b80d8b3ede1c6f3dda1c4087e829af Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 19 Nov 2024 14:19:49 -0500 Subject: [PATCH 003/234] [6.1] Remove '-dev' from package version (#831) --- cmake/modules/LibraryVersion.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/LibraryVersion.cmake b/cmake/modules/LibraryVersion.cmake index 720d2c85c..9b3dbedf3 100644 --- a/cmake/modules/LibraryVersion.cmake +++ b/cmake/modules/LibraryVersion.cmake @@ -8,7 +8,7 @@ # The current version of the Swift Testing release. For release branches, # remember to remove -dev. -set(SWT_TESTING_LIBRARY_VERSION "6.1-dev") +set(SWT_TESTING_LIBRARY_VERSION "6.1") find_package(Git QUIET) if(Git_FOUND) From 271bff1a1ef7f31bd9f3ef4d3650746d08ce62f9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 2 Dec 2024 11:43:04 -0600 Subject: [PATCH 004/234] Configure CMake to build and install the Foundation cross-import overlay (#825) This augments the project's CMake rules to begin building and installing the Foundation cross-import overlay (`_Testing_Foundation`), including the associated `.swiftcrossimport` directory which causes the overlay to be applied to clients. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Resolves rdar://139877808 --- Sources/CMakeLists.txt | 1 + Sources/Overlays/CMakeLists.txt | 9 +++++ .../_Testing_Foundation/CMakeLists.txt | 39 +++++++++++++++++++ Sources/Testing/CMakeLists.txt | 4 ++ .../Foundation.swiftoverlay | 3 ++ cmake/modules/SwiftModuleInstallation.cmake | 20 ++++++++++ 6 files changed, 76 insertions(+) create mode 100644 Sources/Overlays/CMakeLists.txt create mode 100644 Sources/Overlays/_Testing_Foundation/CMakeLists.txt create mode 100644 Sources/Testing/Testing.swiftcrossimport/Foundation.swiftoverlay diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 83a9a2b23..56c28cba1 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -104,4 +104,5 @@ endif() include(AvailabilityDefinitions) include(CompilerSettings) add_subdirectory(_TestingInternals) +add_subdirectory(Overlays) add_subdirectory(Testing) diff --git a/Sources/Overlays/CMakeLists.txt b/Sources/Overlays/CMakeLists.txt new file mode 100644 index 000000000..2120d680f --- /dev/null +++ b/Sources/Overlays/CMakeLists.txt @@ -0,0 +1,9 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2024 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +add_subdirectory(_Testing_Foundation) diff --git a/Sources/Overlays/_Testing_Foundation/CMakeLists.txt b/Sources/Overlays/_Testing_Foundation/CMakeLists.txt new file mode 100644 index 000000000..0c942cfa3 --- /dev/null +++ b/Sources/Overlays/_Testing_Foundation/CMakeLists.txt @@ -0,0 +1,39 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2024 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +add_library(_Testing_Foundation + Attachments/EncodingFormat.swift + Attachments/Attachment+URL.swift + Attachments/Attachable+NSSecureCoding.swift + Attachments/Data+Attachable.swift + Attachments/Attachable+Encodable+NSSecureCoding.swift + Attachments/Attachable+Encodable.swift + Events/Clock+Date.swift + ReexportTesting.swift) + +target_link_libraries(_Testing_Foundation PUBLIC + Testing) + +# Although this library links Foundation on all platforms, it only does so using +# `target_link_libraries()` when building for non-Apple platforms. This is +# because that command uses the `-lFoundation` linker flag, but on Apple +# platforms Foundation is a .framework and requires a different flag. However, +# we don't need to explicitly pass any linker flag since it's handled +# automatically on Apple platforms via auto-linking. +if(NOT APPLE) + target_link_libraries(_Testing_Foundation PUBLIC + Foundation) +endif() + +# Note: This does not enable Library Evolution, despite emitting a module +# interface, because Foundation does not have Library Evolution enabled for all +# platforms. +target_compile_options(_Testing_Foundation PRIVATE + -emit-module-interface -emit-module-interface-path $/_Testing_Foundation.swiftinterface) + +_swift_testing_install_target(_Testing_Foundation) diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 8efe7816c..f7728ac49 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -125,3 +125,7 @@ target_compile_options(Testing PRIVATE -emit-module-interface -emit-module-interface-path $/Testing.swiftinterface) _swift_testing_install_target(Testing) + +# Install the Swift cross-import overlay directory. +_swift_testing_install_swiftcrossimport(Testing + Testing.swiftcrossimport) diff --git a/Sources/Testing/Testing.swiftcrossimport/Foundation.swiftoverlay b/Sources/Testing/Testing.swiftcrossimport/Foundation.swiftoverlay new file mode 100644 index 000000000..8c0f90b84 --- /dev/null +++ b/Sources/Testing/Testing.swiftcrossimport/Foundation.swiftoverlay @@ -0,0 +1,3 @@ +version: 1 +modules: +- name: _Testing_Foundation diff --git a/cmake/modules/SwiftModuleInstallation.cmake b/cmake/modules/SwiftModuleInstallation.cmake index b291f1ff0..ba4f6fec4 100644 --- a/cmake/modules/SwiftModuleInstallation.cmake +++ b/cmake/modules/SwiftModuleInstallation.cmake @@ -96,3 +96,23 @@ function(_swift_testing_install_target module) RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftinterface) endif() endfunction() + +# Install the specified .swiftcrossimport directory for the specified declaring +# module. +# +# Usage: +# _swift_testing_install_swiftcrossimport(module swiftcrossimport_dir) +# +# Arguments: +# module: The name of the declaring module. This is used to determine where +# the .swiftcrossimport directory should be installed, since it must be +# adjacent to the declaring module's .swiftmodule directory. +# swiftcrossimport_dir: The path to the source .swiftcrossimport directory +# which will be installed. +function(_swift_testing_install_swiftcrossimport module swiftcrossimport_dir) + get_target_property(type ${module} TYPE) + get_swift_testing_install_lib_dir(${type} lib_destination_dir) + + install(DIRECTORY "${swiftcrossimport_dir}" + DESTINATION "${lib_destination_dir}") +endfunction() From 8db108c11fdc972ef7a7b7d782e29e47124aeda4 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 2 Dec 2024 11:48:57 -0600 Subject: [PATCH 005/234] Document the steps for building & installing the project for macOS using CMake (#835) This adds documentation to `CONTRIBUTING.md` describing steps for building and installing the project for macOS using CMake. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- CONTRIBUTING.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 97c4ede9b..c63bd0ce5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,6 +98,48 @@ To learn how to run only specific tests or other testing options, run `swift test --help` to view the usage documentation. --> +## Using CMake to build the project for macOS + +1. Install [CMake](https://cmake.org/) and [Ninja](https://ninja-build.org/). + - See the [Installing Dependencies](https://github.com/swiftlang/swift/blob/main/docs/HowToGuides/GettingStarted.md#macos) + section of the Swift [Getting Started](https://github.com/swiftlang/swift/blob/main/docs/HowToGuides/GettingStarted.md) + guide for instructions. + +1. Run the following command from the root of this repository to configure the + project to build using CMake (using the Ninja generator): + + ```bash + cmake -G Ninja -B build + ``` + +1. Run the following command to perform the build: + + ```bash + cmake --build build + ``` + +### Installing built content using CMake + +You can use the steps in this section to perform an install. This is primarily +useful to validate the built content from this project which will be included in +a Swift toolchain. + +1. Run the following command to (re-)configure the project with an install + prefix specified: + + ```bash + cmake -G Ninja --install-prefix "$(pwd)/build/install" -B build + ``` + +1. Perform the CMake build step as described in the previous section. + +1. Run the following command to install the built content into the + `build/install/` subdirectory: + + ```bash + cmake --install build + ``` + ## Using Docker on macOS to test for Linux 1. Install [Docker Desktop for Mac](https://www.docker.com/products/docker-desktop). From 71971b2404c29229123837d85c3a90f92a362fda Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 2 Dec 2024 16:05:45 -0500 Subject: [PATCH 006/234] Revert "[6.1] Remove '-dev' from package version" (#838) Reverts swiftlang/swift-testing#831 --- cmake/modules/LibraryVersion.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/LibraryVersion.cmake b/cmake/modules/LibraryVersion.cmake index 9b3dbedf3..720d2c85c 100644 --- a/cmake/modules/LibraryVersion.cmake +++ b/cmake/modules/LibraryVersion.cmake @@ -8,7 +8,7 @@ # The current version of the Swift Testing release. For release branches, # remember to remove -dev. -set(SWT_TESTING_LIBRARY_VERSION "6.1") +set(SWT_TESTING_LIBRARY_VERSION "6.1-dev") find_package(Git QUIET) if(Git_FOUND) From 3c71e00573c3e4913e74e8429078c0ce7c2df9cd Mon Sep 17 00:00:00 2001 From: Mishal Shah Date: Mon, 2 Dec 2024 13:07:34 -0800 Subject: [PATCH 007/234] Bump the Swift version to 6.2 (#833) https://github.com/swiftlang/swift/pull/77799 --- cmake/modules/LibraryVersion.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/LibraryVersion.cmake b/cmake/modules/LibraryVersion.cmake index 720d2c85c..e50e51182 100644 --- a/cmake/modules/LibraryVersion.cmake +++ b/cmake/modules/LibraryVersion.cmake @@ -8,7 +8,7 @@ # The current version of the Swift Testing release. For release branches, # remember to remove -dev. -set(SWT_TESTING_LIBRARY_VERSION "6.1-dev") +set(SWT_TESTING_LIBRARY_VERSION "6.2-dev") find_package(Git QUIET) if(Git_FOUND) From 8fb3f68a522e9d6dc00dcc90828bf39224327e8c Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 3 Dec 2024 23:10:06 -0600 Subject: [PATCH 008/234] Don't install the binary Swift module when building for Apple platforms (#837) This modifies the CMake rules to stop installing binary .swiftmodule files when building for Apple platforms. ### Motivation: Apple platforms have a stable ABI and modules for those platforms should instead use the textual .swiftinterface which is already installed by CMake. The binary .swiftmodule permits access to [SPI](https://github.com/swiftlang/swift-testing/blob/main/Documentation/SPI.md) declarations from the testing library, and these are not intended to be exposed in distribution builds. Although this PR only removes access to these on macOS, since it's the only platform which has a textual .swiftinterface currently, we intend to investigate ways to match this behavior for other platforms in the future. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Resolves rdar://136083081 --- cmake/modules/SwiftModuleInstallation.cmake | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmake/modules/SwiftModuleInstallation.cmake b/cmake/modules/SwiftModuleInstallation.cmake index ba4f6fec4..15537250f 100644 --- a/cmake/modules/SwiftModuleInstallation.cmake +++ b/cmake/modules/SwiftModuleInstallation.cmake @@ -86,14 +86,18 @@ function(_swift_testing_install_target module) install(FILES $/${module_name}.swiftdoc DESTINATION "${module_dir}" RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftdoc) - install(FILES $/${module_name}.swiftmodule - DESTINATION "${module_dir}" - RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftmodule) if(APPLE) # Only Darwin has stable ABI. install(FILES $/${module_name}.swiftinterface DESTINATION "${module_dir}" RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftinterface) + else() + # Only install the binary .swiftmodule on platforms which do not have a + # stable ABI. Other platforms will use the textual .swiftinterface + # (installed above) and this limits access to this module's SPIs. + install(FILES $/${module_name}.swiftmodule + DESTINATION "${module_dir}" + RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftmodule) endif() endfunction() From a4ed760ce69261101fefffd1abf78ea663a4a44c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 5 Dec 2024 19:55:43 -0800 Subject: [PATCH 009/234] Change archive format for directories to .zip and add iOS/etc. support. (#826) This PR switches from .tar.gz as the preferred archive format for compressed directories to .zip and uses `NSFileCoordinator` on Darwin to enable support for iOS, watchOS, tvOS, and visionOS. This feature remains experimental. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Attachments/Attachment+URL.swift | 217 +++++++++++++----- Tests/TestingTests/AttachmentTests.swift | 8 +- 2 files changed, 164 insertions(+), 61 deletions(-) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index d70641c69..815fcfd18 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -16,6 +16,10 @@ public import Foundation private import UniformTypeIdentifiers #endif +#if !SWT_NO_PROCESS_SPAWNING && os(Windows) +private import WinSDK +#endif + #if !SWT_NO_FILE_IO extension URL { /// The file system path of the URL, equivalent to `path`. @@ -32,17 +36,13 @@ extension URL { } } -#if SWT_TARGET_OS_APPLE && canImport(UniformTypeIdentifiers) -@available(_uttypesAPI, *) -extension UTType { - /// A type that represents a `.tgz` archive, or `nil` if the system does not - /// recognize that content type. - fileprivate static let tgz = UTType("org.gnu.gnu-zip-tar-archive") -} -#endif - @_spi(Experimental) extension Attachment where AttachableValue == Data { +#if SWT_TARGET_OS_APPLE + /// An operation queue to use for asynchronously reading data from disk. + private static let _operationQueue = OperationQueue() +#endif + /// Initialize an instance of this type with the contents of the given URL. /// /// - Parameters: @@ -65,8 +65,6 @@ extension Attachment where AttachableValue == Data { throw CocoaError(.featureUnsupported, userInfo: [NSLocalizedDescriptionKey: "Attaching downloaded files is not supported"]) } - // FIXME: use NSFileCoordinator on Darwin? - let url = url.resolvingSymlinksInPath() let isDirectory = try url.resourceValues(forKeys: [.isDirectoryKey]).isDirectory! @@ -83,79 +81,178 @@ extension Attachment where AttachableValue == Data { // Ensure the preferred name of the archive has an appropriate extension. preferredName = { #if SWT_TARGET_OS_APPLE && canImport(UniformTypeIdentifiers) - if #available(_uttypesAPI, *), let tgz = UTType.tgz { - return (preferredName as NSString).appendingPathExtension(for: tgz) + if #available(_uttypesAPI, *) { + return (preferredName as NSString).appendingPathExtension(for: .zip) } #endif - return (preferredName as NSString).appendingPathExtension("tgz") ?? preferredName + return (preferredName as NSString).appendingPathExtension("zip") ?? preferredName }() + } - try await self.init(Data(compressedContentsOfDirectoryAt: url), named: preferredName, sourceLocation: sourceLocation) +#if SWT_TARGET_OS_APPLE + let data: Data = try await withCheckedThrowingContinuation { continuation in + let fileCoordinator = NSFileCoordinator() + let fileAccessIntent = NSFileAccessIntent.readingIntent(with: url, options: [.forUploading]) + + fileCoordinator.coordinate(with: [fileAccessIntent], queue: Self._operationQueue) { error in + let result = Result { + if let error { + throw error + } + return try Data(contentsOf: fileAccessIntent.url, options: [.mappedIfSafe]) + } + continuation.resume(with: result) + } + } +#else + let data = if isDirectory { + try await _compressContentsOfDirectory(at: url) } else { // Load the file. - try self.init(Data(contentsOf: url, options: [.mappedIfSafe]), named: preferredName, sourceLocation: sourceLocation) + try Data(contentsOf: url, options: [.mappedIfSafe]) } +#endif + + self.init(data, named: preferredName, sourceLocation: sourceLocation) } } -// MARK: - Attaching directories +#if !SWT_NO_PROCESS_SPAWNING && os(Windows) +/// The filename of the archiver tool. +private let _archiverName = "tar.exe" -extension Data { - /// Initialize an instance of this type by compressing the contents of a - /// directory. - /// - /// - Parameters: - /// - directoryURL: A URL referring to the directory to attach. - /// - /// - Throws: Any error encountered trying to compress the directory, or if - /// directories cannot be compressed on this platform. - /// - /// This initializer asynchronously compresses the contents of `directoryURL` - /// into an archive (currently of `.tgz` format, although this is subject to - /// change) and stores a mapped copy of that archive. - init(compressedContentsOfDirectoryAt directoryURL: URL) async throws { - let temporaryName = "\(UUID().uuidString).tgz" - let temporaryURL = FileManager.default.temporaryDirectory.appendingPathComponent(temporaryName) +/// The path to the archiver tool. +/// +/// This path refers to a file (named `_archiverName`) within the `"System32"` +/// folder of the current system, which is not always located in `"C:\Windows."` +/// +/// If the path cannot be determined, the value of this property is `nil`. +private let _archiverPath: String? = { + let bufferCount = GetSystemDirectoryW(nil, 0) + guard bufferCount > 0 else { + return nil + } + + return withUnsafeTemporaryAllocation(of: wchar_t.self, capacity: Int(bufferCount)) { buffer -> String? in + let bufferCount = GetSystemDirectoryW(buffer.baseAddress!, UINT(buffer.count)) + guard bufferCount > 0 && bufferCount < buffer.count else { + return nil + } + return _archiverName.withCString(encodedAs: UTF16.self) { archiverName -> String? in + var result: UnsafeMutablePointer? + + let flags = ULONG(PATHCCH_ALLOW_LONG_PATHS.rawValue) + guard S_OK == PathAllocCombine(buffer.baseAddress!, archiverName, flags, &result) else { + return nil + } + defer { + LocalFree(result) + } + + return result.flatMap { String.decodeCString($0, as: UTF16.self)?.result } + } + } +}() +#endif + +/// Compress the contents of a directory to an archive, then map that archive +/// back into memory. +/// +/// - Parameters: +/// - directoryURL: A URL referring to the directory to attach. +/// +/// - Returns: An instance of `Data` containing the compressed contents of the +/// given directory. +/// +/// - Throws: Any error encountered trying to compress the directory, or if +/// directories cannot be compressed on this platform. +/// +/// This function asynchronously compresses the contents of `directoryURL` into +/// an archive (currently of `.zip` format, although this is subject to change.) +private func _compressContentsOfDirectory(at directoryURL: URL) async throws -> Data { #if !SWT_NO_PROCESS_SPAWNING -#if os(Windows) - let tarPath = #"C:\Windows\System32\tar.exe"# + let temporaryName = "\(UUID().uuidString).zip" + let temporaryURL = FileManager.default.temporaryDirectory.appendingPathComponent(temporaryName) + defer { + try? FileManager().removeItem(at: temporaryURL) + } + + // The standard version of tar(1) does not (appear to) support writing PKZIP + // archives. FreeBSD's (AKA bsdtar) was long ago rebased atop libarchive and + // knows how to write PKZIP archives, while Windows inherited FreeBSD's tar + // tool in Windows 10 Build 17063 (per https://techcommunity.microsoft.com/blog/containers/tar-and-curl-come-to-windows/382409). + // + // On Linux (which does not have FreeBSD's version of tar(1)), we can use + // zip(1) instead. +#if os(Linux) + let archiverPath = "/usr/bin/zip" +#elseif SWT_TARGET_OS_APPLE || os(FreeBSD) + let archiverPath = "/usr/bin/tar" +#elseif os(Windows) + guard let archiverPath = _archiverPath else { + throw CocoaError(.fileWriteUnknown, userInfo: [ + NSLocalizedDescriptionKey: "Could not determine the path to '\(_archiverName)'.", + ]) + } #else - let tarPath = "/usr/bin/tar" +#warning("Platform-specific implementation missing: tar or zip tool unavailable") + let archiverPath = "" + throw CocoaError(.featureUnsupported, userInfo: [NSLocalizedDescriptionKey: "This platform does not support attaching directories to tests."]) #endif + + try await withCheckedThrowingContinuation { continuation in + let process = Process() + + process.executableURL = URL(fileURLWithPath: archiverPath, isDirectory: false) + let sourcePath = directoryURL.fileSystemPath let destinationPath = temporaryURL.fileSystemPath - defer { - try? FileManager().removeItem(at: temporaryURL) - } +#if os(Linux) + // The zip command constructs relative paths from the current working + // directory rather than from command-line arguments. + process.arguments = [destinationPath, "--recurse-paths", "."] + process.currentDirectoryURL = directoryURL +#elseif SWT_TARGET_OS_APPLE || os(FreeBSD) + process.arguments = ["--create", "--auto-compress", "--directory", sourcePath, "--file", destinationPath, "."] +#elseif os(Windows) + // The Windows version of bsdtar can handle relative paths for other archive + // formats, but produces empty archives when inferring the zip format with + // --auto-compress, so archive with absolute paths here. + // + // An alternative may be to use PowerShell's Compress-Archive command, + // however that comes with a security risk as we'd be responsible for two + // levels of command-line argument escaping. + process.arguments = ["--create", "--auto-compress", "--file", destinationPath, sourcePath] +#endif - try await withCheckedThrowingContinuation { continuation in - do { - _ = try Process.run( - URL(fileURLWithPath: tarPath, isDirectory: false), - arguments: ["--create", "--gzip", "--directory", sourcePath, "--file", destinationPath, "."] - ) { process in - let terminationReason = process.terminationReason - let terminationStatus = process.terminationStatus - if terminationReason == .exit && terminationStatus == EXIT_SUCCESS { - continuation.resume() - } else { - let error = CocoaError(.fileWriteUnknown, userInfo: [ - NSLocalizedDescriptionKey: "The directory at '\(sourcePath)' could not be compressed.", - ]) - continuation.resume(throwing: error) - } - } - } catch { + process.standardOutput = nil + process.standardError = nil + + process.terminationHandler = { process in + let terminationReason = process.terminationReason + let terminationStatus = process.terminationStatus + if terminationReason == .exit && terminationStatus == EXIT_SUCCESS { + continuation.resume() + } else { + let error = CocoaError(.fileWriteUnknown, userInfo: [ + NSLocalizedDescriptionKey: "The directory at '\(sourcePath)' could not be compressed (\(terminationStatus)).", + ]) continuation.resume(throwing: error) } } - try self.init(contentsOf: temporaryURL, options: [.mappedIfSafe]) + do { + try process.run() + } catch { + continuation.resume(throwing: error) + } + } + + return try Data(contentsOf: temporaryURL, options: [.mappedIfSafe]) #else - throw CocoaError(.featureUnsupported, userInfo: [NSLocalizedDescriptionKey: "This platform does not support attaching directories to tests."]) + throw CocoaError(.featureUnsupported, userInfo: [NSLocalizedDescriptionKey: "This platform does not support attaching directories to tests."]) #endif - } } #endif #endif diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 431e08d24..e9a50f0d5 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -275,7 +275,13 @@ struct AttachmentTests { return } - #expect(attachment.preferredName == "\(temporaryDirectoryName).tgz") + #expect(attachment.preferredName == "\(temporaryDirectoryName).zip") + try! attachment.withUnsafeBufferPointer { buffer in + #expect(buffer.count > 32) + #expect(buffer[0] == UInt8(ascii: "P")) + #expect(buffer[1] == UInt8(ascii: "K")) + #expect(buffer.contains("loremipsum.txt".utf8)) + } valueAttached() } From 906092d5b8b151abd3f14db95bc13a7e2a488944 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 9 Dec 2024 21:20:09 -0500 Subject: [PATCH 010/234] Add a CoreGraphics cross-import overlay with support for attaching `CGImage`s. (#827) This PR adds a new cross-import overlay target with Apple's Core Graphics framework that allows attaching a `CGImage` as an attachment in an arbitrary image format (PNG, JPEG, etc.) Because `CGImage` is imported into Swift as a non-final class, it cannot conform directly to `Attachable`, so an `AttachableContainer` type acts as a proxy. This type is not meant to be used directly, so its name is underscored. Initializers on `Attachment` are provided so that this abstraction is almost entirely transparent to test authors. A new protocol, `AttachableAsCGImage`, is introduced that abstracts away the relationship between the attached image and Core Graphics; in the future, I intend to make additional image types like `NSImage` and `UIImage` conform to this protocol too. Example usage: ```swift let sparklyDiamonds: CGImage = ... let attachment = Attachment(image, named: "sparkly-diamonds", as: .tiff, encodingQuality: 0.75) ... attachment.attach() ``` The code in this PR is, by definition, specific to Apple's platforms. In the future, I'd be interested in adding Windows/Linux equivalents (`HBITMAP`? Whatever Gnome/KDE/Qt use?) but that's beyond the scope of this PR. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 9 + .../Attachments/AttachableAsCGImage.swift | 89 ++++++++++ .../Attachment+AttachableAsCGImage.swift | 147 ++++++++++++++++ .../CGImage+AttachableAsCGImage.swift | 20 +++ .../Attachments/ImageAttachmentError.swift | 41 +++++ .../_AttachableImageContainer.swift | 163 ++++++++++++++++++ .../ReexportTesting.swift | 11 ++ Tests/TestingTests/AttachmentTests.swift | 88 ++++++++++ 8 files changed, 568 insertions(+) create mode 100644 Sources/Overlays/_Testing_CoreGraphics/Attachments/AttachableAsCGImage.swift create mode 100644 Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift create mode 100644 Sources/Overlays/_Testing_CoreGraphics/Attachments/CGImage+AttachableAsCGImage.swift create mode 100644 Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift create mode 100644 Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift create mode 100644 Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift diff --git a/Package.swift b/Package.swift index f840a21cb..b109c0447 100644 --- a/Package.swift +++ b/Package.swift @@ -51,6 +51,7 @@ let package = Package( name: "TestingTests", dependencies: [ "Testing", + "_Testing_CoreGraphics", "_Testing_Foundation", ], swiftSettings: .packageSettings @@ -91,6 +92,14 @@ let package = Package( ), // Cross-import overlays (not supported by Swift Package Manager) + .target( + name: "_Testing_CoreGraphics", + dependencies: [ + "Testing", + ], + path: "Sources/Overlays/_Testing_CoreGraphics", + swiftSettings: .packageSettings + ), .target( name: "_Testing_Foundation", dependencies: [ diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/AttachableAsCGImage.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/AttachableAsCGImage.swift new file mode 100644 index 000000000..14df843c6 --- /dev/null +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/AttachableAsCGImage.swift @@ -0,0 +1,89 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) +public import CoreGraphics +private import ImageIO + +/// A protocol describing images that can be converted to instances of +/// ``Testing/Attachment``. +/// +/// Instances of types conforming to this protocol do not themselves conform to +/// ``Testing/Attachable``. Instead, the testing library provides additional +/// initializers on ``Testing/Attachment`` that take instances of such types and +/// handle converting them to image data when needed. +/// +/// The following system-provided image types conform to this protocol and can +/// be attached to a test: +/// +/// - [`CGImage`](https://developer.apple.com/documentation/coregraphics/cgimage) +/// +/// You do not generally need to add your own conformances to this protocol. If +/// you have an image in another format that needs to be attached to a test, +/// first convert it to an instance of one of the types above. +@_spi(Experimental) +public protocol AttachableAsCGImage { + /// An instance of `CGImage` representing this image. + /// + /// - Throws: Any error that prevents the creation of an image. + var attachableCGImage: CGImage { get throws } + + /// The orientation of the image. + /// + /// The value of this property is the raw value of an instance of + /// `CGImagePropertyOrientation`. The default value of this property is + /// `.up`. + /// + /// This property is not part of the public interface of the testing + /// library. It may be removed in a future update. + var _attachmentOrientation: UInt32 { get } + + /// The scale factor of the image. + /// + /// The value of this property is typically greater than `1.0` when an image + /// originates from a Retina Display screenshot or similar. The default value + /// of this property is `1.0`. + /// + /// This property is not part of the public interface of the testing + /// library. It may be removed in a future update. + var _attachmentScaleFactor: CGFloat { get } + + /// Make a copy of this instance to pass to an attachment. + /// + /// - Returns: A copy of `self`, or `self` if no copy is needed. + /// + /// Several system image types do not conform to `Sendable`; use this + /// function to make copies of such images that will not be shared outside + /// of an attachment and so can be generally safely stored. + /// + /// The default implementation of this function when `Self` conforms to + /// `Sendable` simply returns `self`. + /// + /// This function is not part of the public interface of the testing library. + /// It may be removed in a future update. + func _makeCopyForAttachment() -> Self +} + +extension AttachableAsCGImage { + public var _attachmentOrientation: UInt32 { + CGImagePropertyOrientation.up.rawValue + } + + public var _attachmentScaleFactor: CGFloat { + 1.0 + } +} + +extension AttachableAsCGImage where Self: Sendable { + public func _makeCopyForAttachment() -> Self { + self + } +} +#endif diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift new file mode 100644 index 000000000..f93afb7f7 --- /dev/null +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift @@ -0,0 +1,147 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) +@_spi(ForSwiftTestingOnly) @_spi(Experimental) public import Testing + +public import UniformTypeIdentifiers + +extension Attachment { + /// Initialize an instance of this type that encloses the given image. + /// + /// - Parameters: + /// - attachableValue: The value that will be attached to the output of + /// the test run. + /// - preferredName: The preferred name of the attachment when writing it + /// to a test report or to disk. If `nil`, the testing library attempts + /// to derive a reasonable filename for the attached value. + /// - contentType: The image format with which to encode `attachableValue`. + /// If this type does not conform to [`UTType.image`](https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/image), + /// the result is undefined. Pass `nil` to let the testing library decide + /// which image format to use. + /// - encodingQuality: The encoding quality to use when encoding the image. + /// If the image format used for encoding (specified by the `contentType` + /// argument) does not support variable-quality encoding, the value of + /// this argument is ignored. + /// - sourceLocation: The source location of the call to this initializer. + /// This value is used when recording issues associated with the + /// attachment. + /// + /// This is the designated initializer for this type when attaching an image + /// that conforms to ``AttachableAsCGImage``. + fileprivate init( + attachableValue: T, + named preferredName: String?, + contentType: (any Sendable)?, + encodingQuality: Float, + sourceLocation: SourceLocation + ) where AttachableValue == _AttachableImageContainer { + var imageContainer = _AttachableImageContainer(image: attachableValue, encodingQuality: encodingQuality) + + // Update the preferred name to include an extension appropriate for the + // given content type. (Note the `else` branch duplicates the logic in + // `preferredContentType(forEncodingQuality:)` but will go away once our + // minimum deployment targets include the UniformTypeIdentifiers framework.) + var preferredName = preferredName ?? Self.defaultPreferredName + if #available(_uttypesAPI, *) { + let contentType: UTType = contentType + .map { $0 as! UTType } + .flatMap { contentType in + if UTType.image.conforms(to: contentType) { + // This type is an abstract base type of .image (or .image itself.) + // We'll infer the concrete type based on other arguments. + return nil + } + return contentType + } ?? .preferred(forEncodingQuality: encodingQuality) + preferredName = (preferredName as NSString).appendingPathExtension(for: contentType) + imageContainer.contentType = contentType + } else { + // The caller can't provide a content type, so we'll pick one for them. + let ext = if encodingQuality < 1.0 { + "jpg" + } else { + "png" + } + if (preferredName as NSString).pathExtension.caseInsensitiveCompare(ext) != .orderedSame { + preferredName = (preferredName as NSString).appendingPathExtension(ext) ?? preferredName + } + } + + self.init(imageContainer, named: preferredName, sourceLocation: sourceLocation) + } + + /// Initialize an instance of this type that encloses the given image. + /// + /// - Parameters: + /// - attachableValue: The value that will be attached to the output of + /// the test run. + /// - preferredName: The preferred name of the attachment when writing it + /// to a test report or to disk. If `nil`, the testing library attempts + /// to derive a reasonable filename for the attached value. + /// - contentType: The image format with which to encode `attachableValue`. + /// If this type does not conform to [`UTType.image`](https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/image), + /// the result is undefined. Pass `nil` to let the testing library decide + /// which image format to use. + /// - encodingQuality: The encoding quality to use when encoding the image. + /// If the image format used for encoding (specified by the `contentType` + /// argument) does not support variable-quality encoding, the value of + /// this argument is ignored. + /// - sourceLocation: The source location of the call to this initializer. + /// This value is used when recording issues associated with the + /// attachment. + /// + /// The following system-provided image types conform to the + /// ``AttachableAsCGImage`` protocol and can be attached to a test: + /// + /// - [`CGImage`](https://developer.apple.com/documentation/coregraphics/cgimage) + @_spi(Experimental) + @available(_uttypesAPI, *) + public init( + _ attachableValue: T, + named preferredName: String? = nil, + as contentType: UTType?, + encodingQuality: Float = 1.0, + sourceLocation: SourceLocation = #_sourceLocation + ) where AttachableValue == _AttachableImageContainer { + self.init(attachableValue: attachableValue, named: preferredName, contentType: contentType, encodingQuality: encodingQuality, sourceLocation: sourceLocation) + } + + /// Initialize an instance of this type that encloses the given image. + /// + /// - Parameters: + /// - attachableValue: The value that will be attached to the output of + /// the test run. + /// - preferredName: The preferred name of the attachment when writing it + /// to a test report or to disk. If `nil`, the testing library attempts + /// to derive a reasonable filename for the attached value. + /// - encodingQuality: The encoding quality to use when encoding the image. + /// If the image format used for encoding (specified by the `contentType` + /// argument) does not support variable-quality encoding, the value of + /// this argument is ignored. + /// - sourceLocation: The source location of the call to this initializer. + /// This value is used when recording issues associated with the + /// attachment. + /// + /// The following system-provided image types conform to the + /// ``AttachableAsCGImage`` protocol and can be attached to a test: + /// + /// - [`CGImage`](https://developer.apple.com/documentation/coregraphics/cgimage) + @_spi(Experimental) + public init( + _ attachableValue: T, + named preferredName: String? = nil, + encodingQuality: Float = 1.0, + sourceLocation: SourceLocation = #_sourceLocation + ) where AttachableValue == _AttachableImageContainer { + self.init(attachableValue: attachableValue, named: preferredName, contentType: nil, encodingQuality: encodingQuality, sourceLocation: sourceLocation) + } +} +#endif diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/CGImage+AttachableAsCGImage.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/CGImage+AttachableAsCGImage.swift new file mode 100644 index 000000000..944798d39 --- /dev/null +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/CGImage+AttachableAsCGImage.swift @@ -0,0 +1,20 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) +public import CoreGraphics + +@_spi(Experimental) +extension CGImage: AttachableAsCGImage { + public var attachableCGImage: CGImage { + self + } +} +#endif diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift new file mode 100644 index 000000000..fdd0b3f3e --- /dev/null +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift @@ -0,0 +1,41 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) +/// A type representing an error that can occur when attaching an image. +@_spi(ForSwiftTestingOnly) +public enum ImageAttachmentError: Error, CustomStringConvertible { + /// The specified content type did not conform to `.image`. + case contentTypeDoesNotConformToImage + + /// The image could not be converted to an instance of `CGImage`. + case couldNotCreateCGImage + + /// The image destination could not be created. + case couldNotCreateImageDestination + + /// The image could not be converted. + case couldNotConvertImage + + @_spi(ForSwiftTestingOnly) + public var description: String { + switch self { + case .contentTypeDoesNotConformToImage: + "The specified type does not represent an image format." + case .couldNotCreateCGImage: + "Could not create the corresponding Core Graphics image." + case .couldNotCreateImageDestination: + "Could not create the Core Graphics image destination to encode this image." + case .couldNotConvertImage: + "Could not convert the image to the specified format." + } + } +} +#endif diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift new file mode 100644 index 000000000..5e8fcd227 --- /dev/null +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift @@ -0,0 +1,163 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) +@_spi(Experimental) public import Testing +private import CoreGraphics + +private import ImageIO +import UniformTypeIdentifiers + +/// ## Why can't images directly conform to Attachable? +/// +/// Three reasons: +/// +/// 1. Several image classes are not marked `Sendable`, which means that as far +/// as Swift is concerned, they cannot be safely passed to Swift Testing's +/// event handler (primarily because `Event` is `Sendable`.) So we would have +/// to eagerly serialize them, which is unnecessarily expensive if we know +/// they're actually concurrency-safe. +/// 2. We would have no place to store metadata such as the encoding quality +/// (although in the future we may introduce a "metadata" associated type to +/// `Attachable` that could store that info.) +/// 3. `Attachable` has a requirement with `Self` in non-parameter, non-return +/// position. As far as Swift is concerned, a non-final class cannot satisfy +/// such a requirement, and all image types we care about are non-final +/// classes. Thus, the compiler will steadfastly refuse to allow non-final +/// classes to conform to the `Attachable` protocol. We could get around this +/// by changing the signature of `withUnsafeBufferPointer()` so that the +/// generic parameter to `Attachment` is not `Self`, but that would defeat +/// much of the purpose of making `Attachment` generic in the first place. +/// (And no, the language does not let us write `where T: Self` anywhere +/// useful.) + +/// A wrapper type for image types such as `CGImage` and `NSImage` that can be +/// attached indirectly. +/// +/// You do not need to use this type directly. Instead, initialize an instance +/// of ``Attachment`` using an instance of an image type that conforms to +/// ``AttachableAsCGImage``. The following system-provided image types conform +/// to the ``AttachableAsCGImage`` protocol and can be attached to a test: +/// +/// - [`CGImage`](https://developer.apple.com/documentation/coregraphics/cgimage) +@_spi(Experimental) +public struct _AttachableImageContainer: Sendable where Image: AttachableAsCGImage { + /// The underlying image. + /// + /// `CGImage` and `UIImage` are sendable, but `NSImage` is not. `NSImage` + /// instances can be created from closures that are run at rendering time. + /// The AppKit cross-import overlay is responsible for ensuring that any + /// instances of this type it creates hold "safe" `NSImage` instances. + nonisolated(unsafe) var image: Image + + /// The encoding quality to use when encoding the represented image. + public var encodingQuality: Float + + /// Storage for ``contentType``. + private var _contentType: (any Sendable)? + + /// The content type to use when encoding the image. + /// + /// This property should eventually move up to ``Attachment``. It is not part + /// of the public interface of the testing library. + @available(_uttypesAPI, *) + var contentType: UTType? { + get { + _contentType as? UTType + } + set { + _contentType = newValue + } + } + + init(image: Image, encodingQuality: Float) { + self.image = image._makeCopyForAttachment() + self.encodingQuality = encodingQuality + } +} + +// MARK: - + +@available(_uttypesAPI, *) +extension UTType { + /// Determine the preferred content type to encode this image as for a given + /// encoding quality. + /// + /// - Parameters: + /// - encodingQuality: The encoding quality to use when encoding the image. + /// + /// - Returns: The type to encode this image as. + static func preferred(forEncodingQuality encodingQuality: Float) -> Self { + // If the caller wants lossy encoding, use JPEG. + if encodingQuality < 1.0 { + return .jpeg + } + + // Lossless encoding implies PNG. + return .png + } +} + +extension _AttachableImageContainer: AttachableContainer { + public var attachableValue: Image { + image + } + + public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + let data = NSMutableData() + + // Convert the image to a CGImage. + let attachableCGImage = try image.attachableCGImage + + // Get the type to encode as. (Note the `else` branches duplicate the logic + // in `preferredContentType(forEncodingQuality:)` but will go away once our + // minimum deployment targets include the UniformTypeIdentifiers framework.) + let typeIdentifier: CFString + if #available(_uttypesAPI, *), let contentType { + guard contentType.conforms(to: .image) else { + throw ImageAttachmentError.contentTypeDoesNotConformToImage + } + typeIdentifier = contentType.identifier as CFString + } else if encodingQuality < 1.0 { + typeIdentifier = kUTTypeJPEG + } else { + typeIdentifier = kUTTypePNG + } + + // Create the image destination. + guard let dest = CGImageDestinationCreateWithData(data as CFMutableData, typeIdentifier, 1, nil) else { + throw ImageAttachmentError.couldNotCreateImageDestination + } + + // Configure the properties of the image conversion operation. + let orientation = image._attachmentOrientation + let scaleFactor = image._attachmentScaleFactor + let properties: [CFString: Any] = [ + kCGImageDestinationLossyCompressionQuality: CGFloat(encodingQuality), + kCGImagePropertyOrientation: orientation, + kCGImagePropertyDPIWidth: 72.0 * scaleFactor, + kCGImagePropertyDPIHeight: 72.0 * scaleFactor, + ] + + // Perform the image conversion. + CGImageDestinationAddImage(dest, attachableCGImage, properties as CFDictionary) + guard CGImageDestinationFinalize(dest) else { + throw ImageAttachmentError.couldNotConvertImage + } + + // Pass the bits of the image out to the body. Note that we have an + // NSMutableData here so we have to use slightly different API than we would + // with an instance of Data. + return try withExtendedLifetime(data) { + try body(UnsafeRawBufferPointer(start: data.bytes, count: data.length)) + } + } +} +#endif diff --git a/Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift b/Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift new file mode 100644 index 000000000..3faa622d7 --- /dev/null +++ b/Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift @@ -0,0 +1,11 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_exported public import Testing diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index e9a50f0d5..8d38603e0 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -14,6 +14,13 @@ private import _TestingInternals import Foundation @_spi(Experimental) import _Testing_Foundation #endif +#if canImport(CoreGraphics) +import CoreGraphics +@_spi(Experimental) @_spi(ForSwiftTestingOnly) import _Testing_CoreGraphics +#endif +#if canImport(UniformTypeIdentifiers) +import UniformTypeIdentifiers +#endif @Suite("Attachment Tests") struct AttachmentTests { @@ -452,6 +459,87 @@ extension AttachmentTests { } } +extension AttachmentTests { + @Suite("Image tests") + struct ImageTests { + enum ImageTestError: Error { + case couldNotCreateCGContext + case couldNotCreateCGGradient + case couldNotCreateCGImage + } + +#if canImport(CoreGraphics) + static let cgImage = Result { + let size = CGSize(width: 32.0, height: 32.0) + let rgb = CGColorSpaceCreateDeviceRGB() + let bitmapInfo = CGImageAlphaInfo.premultipliedFirst.rawValue + guard let context = CGContext( + data: nil, + width: Int(size.width), + height: Int(size.height), + bitsPerComponent: 8, + bytesPerRow: Int(size.width) * 4, + space: rgb, + bitmapInfo: bitmapInfo + ) else { + throw ImageTestError.couldNotCreateCGContext + } + guard let gradient = CGGradient( + colorsSpace: rgb, + colors: [ + CGColor(red: 1.0, green: 0.0, blue: 0.0, alpha: 1.0), + CGColor(red: 0.0, green: 1.0, blue: 0.0, alpha: 1.0), + CGColor(red: 0.0, green: 0.0, blue: 1.0, alpha: 1.0), + ] as CFArray, + locations: nil + ) else { + throw ImageTestError.couldNotCreateCGGradient + } + context.drawLinearGradient( + gradient, + start: .zero, + end: CGPoint(x: size.width, y: size.height), + options: [.drawsBeforeStartLocation, .drawsAfterEndLocation] + ) + guard let image = context.makeImage() else { + throw ImageTestError.couldNotCreateCGImage + } + return image + } + + @available(_uttypesAPI, *) + @Test func attachCGImage() throws { + let image = try Self.cgImage.get() + let attachment = Attachment(image, named: "diamond") + #expect(attachment.attachableValue === image) + try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { buffer in + #expect(buffer.count > 32) + } + attachment.attach() + } + + @available(_uttypesAPI, *) + @Test(arguments: [Float(0.0).nextUp, 0.25, 0.5, 0.75, 1.0], [.png as UTType?, .jpeg, .gif, .image, .data, nil]) + func attachCGImage(quality: Float, type: UTType?) throws { + let image = try Self.cgImage.get() + let attachment = Attachment(image, named: "diamond", as: type, encodingQuality: quality) + #expect(attachment.attachableValue === image) + try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { buffer in + #expect(buffer.count > 32) + } + } + + @available(_uttypesAPI, *) + @Test func cannotAttachCGImageWithNonImageType() async { + #expect(throws: ImageAttachmentError.contentTypeDoesNotConformToImage) { + let attachment = Attachment(try Self.cgImage.get(), named: "diamond", as: .mp3) + try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { _ in } + } + } +#endif + } +} + // MARK: - Fixtures struct MyAttachable: Attachable, ~Copyable { From 0fd04d18edbea17baf7fc79b09d1656afdabe148 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 9 Dec 2024 19:30:25 -0800 Subject: [PATCH 011/234] Fix DocC warnings and errors, including those from SPI (#836) This includes a variety of cleanups and fixes to resolve DocC warnings and errors, including those which show up only when you manually include SPI declarations (by passing the `-include-spi-symbols` Swift compiler flag). I extracted some of these from #295, and found new ones. I still plan to pursue that PR, but reduce its focus to just enabling the SPI documentation flag in some local development workflows. In the mean time, this PR contains only documentation fixes. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Attachments/Attachment.swift | 10 ++-- Sources/Testing/Events/Event.swift | 7 ++- .../Event.HumanReadableOutputRecorder.swift | 6 +-- Sources/Testing/ExitTests/ExitTest.swift | 4 +- Sources/Testing/Issues/Issue.swift | 31 ++++++------ .../Test.Case.Generator.swift | 30 +++++------ .../Running/Configuration.TestFilter.swift | 2 +- Sources/Testing/Running/Configuration.swift | 7 +-- .../Testing/Running/Runner.Plan+Dumping.swift | 1 - Sources/Testing/Running/Runner.Plan.swift | 2 +- .../Testing/Support/CartesianProduct.swift | 20 ++++---- Sources/Testing/Test+Macro.swift | 50 +++++++++---------- .../Testing.docc/LimitingExecutionTime.md | 5 +- Sources/Testing/Testing.docc/Traits.md | 2 +- Sources/Testing/Testing.docc/Traits/Trait.md | 2 +- Sources/Testing/Traits/ConditionTrait.swift | 24 ++++----- Sources/Testing/Traits/TimeLimitTrait.swift | 4 +- .../TestSupport/TestingAdditions.swift | 20 ++++---- 18 files changed, 117 insertions(+), 110 deletions(-) diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index d79a43fcb..93ae53c73 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -111,11 +111,11 @@ extension Attachment where AttachableValue == AnyAttachable { /// events of kind ``Event/Kind/valueAttached(_:)``. Test tools authors who use /// `@_spi(ForToolsIntegrationOnly)` will see instances of this type when /// handling those events. -/// -/// @Comment { -/// Swift's type system requires that this type be at least as visible as -/// `Event.Kind.valueAttached(_:)`, otherwise it would be declared private. -/// } +// +// @Comment { +// Swift's type system requires that this type be at least as visible as +// `Event.Kind.valueAttached(_:)`, otherwise it would be declared private. +// } @_spi(Experimental) @_spi(ForToolsIntegrationOnly) public struct AnyAttachable: AttachableContainer, Copyable, Sendable { #if !SWT_NO_LAZY_ATTACHMENTS diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index d76346623..60e564d5a 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -462,8 +462,11 @@ extension Event.Kind { /// This is the last event posted before ``Runner/run()`` returns. case runEnded - /// Snapshots an ``Event.Kind``. - /// - Parameter kind: The original ``Event.Kind`` to snapshot. + /// Initialize an instance of this type by snapshotting the specified event + /// kind. + /// + /// - Parameters: + /// - kind: The original event kind to snapshot. public init(snapshotting kind: Event.Kind) { switch kind { case .testDiscovered: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index ab1f56702..9468d55ae 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -74,9 +74,9 @@ extension Event { /// Initialize a new human-readable event recorder. /// /// Output from the testing library is converted to "messages" using the - /// ``Event/HumanReadableOutputRecorder/record(_:)`` function. The format of - /// those messages is, as the type's name suggests, not meant to be - /// machine-readable and is subject to change. + /// ``Event/HumanReadableOutputRecorder/record(_:in:verbosity:)`` function. + /// The format of those messages is, as the type's name suggests, not meant + /// to be machine-readable and is subject to change. public init() {} } } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index f21104f31..39a0ea550 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -53,8 +53,8 @@ public struct ExitTest: Sendable, ~Copyable { /// this property to determine what information you need to preserve from your /// child process. /// - /// The value of this property always includes ``Result/exitCondition`` even - /// if the test author does not specify it. + /// The value of this property always includes ``ExitTestArtifacts/exitCondition`` + /// even if the test author does not specify it. /// /// Within a child process running an exit test, the value of this property is /// otherwise unspecified. diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 68f9bb7b4..731a240d1 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -49,12 +49,12 @@ public struct Issue: Sendable { /// /// - Parameters: /// - timeLimitComponents: The time limit reached by the test. - /// - /// @Comment { - /// - Bug: The associated value of this enumeration case should be an - /// instance of `Duration`, but the testing library's deployment target - /// predates the introduction of that type. - /// } + // + // @Comment { + // - Bug: The associated value of this enumeration case should be an + // instance of `Duration`, but the testing library's deployment target + // predates the introduction of that type. + // } indirect case timeLimitExceeded(timeLimitComponents: (seconds: Int64, attoseconds: Int64)) /// A known issue was expected, but was not recorded. @@ -337,12 +337,12 @@ extension Issue.Kind { /// /// - Parameters: /// - timeLimitComponents: The time limit reached by the test. - /// - /// @Comment { - /// - Bug: The associated value of this enumeration case should be an - /// instance of `Duration`, but the testing library's deployment target - /// predates the introduction of that type. - /// } + // + // @Comment { + // - Bug: The associated value of this enumeration case should be an + // instance of `Duration`, but the testing library's deployment target + // predates the introduction of that type. + // } indirect case timeLimitExceeded(timeLimitComponents: (seconds: Int64, attoseconds: Int64)) /// A known issue was expected, but was not recorded. @@ -355,8 +355,11 @@ extension Issue.Kind { /// within the tests being run. case system - /// Snapshots an ``Issue.Kind``. - /// - Parameter kind: The original ``Issue.Kind`` to snapshot. + /// Initialize an instance of this type by snapshotting the specified issue + /// kind. + /// + /// - Parameters: + /// - kind: The original issue kind to snapshot. public init(snapshotting kind: Issue.Kind) { self = switch kind { case .unconditional: diff --git a/Sources/Testing/Parameterization/Test.Case.Generator.swift b/Sources/Testing/Parameterization/Test.Case.Generator.swift index 123f5c712..d4d583e48 100644 --- a/Sources/Testing/Parameterization/Test.Case.Generator.swift +++ b/Sources/Testing/Parameterization/Test.Case.Generator.swift @@ -13,11 +13,11 @@ extension Test.Case { /// a known collection of argument values. /// /// Instances of this type can be iterated over multiple times. - /// - /// @Comment { - /// - Bug: The testing library should support variadic generics. - /// ([103416861](rdar://103416861)) - /// } + // + // @Comment { + // - Bug: The testing library should support variadic generics. + // ([103416861](rdar://103416861)) + // } struct Generator: Sendable where S: Sequence & Sendable, S.Element: Sendable { /// The underlying sequence of argument values. /// @@ -146,11 +146,11 @@ extension Test.Case { /// /// This initializer overload is specialized for sequences of 2-tuples to /// efficiently de-structure their elements when appropriate. - /// - /// @Comment { - /// - Bug: The testing library should support variadic generics. - /// ([103416861](rdar://103416861)) - /// } + // + // @Comment { + // - Bug: The testing library should support variadic generics. + // ([103416861](rdar://103416861)) + // } private init( sequence: S, parameters: [Test.Parameter], @@ -184,11 +184,11 @@ extension Test.Case { /// /// This initializer overload is specialized for collections of 2-tuples to /// efficiently de-structure their elements when appropriate. - /// - /// @Comment { - /// - Bug: The testing library should support variadic generics. - /// ([103416861](rdar://103416861)) - /// } + // + // @Comment { + // - Bug: The testing library should support variadic generics. + // ([103416861](rdar://103416861)) + // } init( arguments collection: S, parameters: [Test.Parameter], diff --git a/Sources/Testing/Running/Configuration.TestFilter.swift b/Sources/Testing/Running/Configuration.TestFilter.swift index ec2427348..7ef1fb08c 100644 --- a/Sources/Testing/Running/Configuration.TestFilter.swift +++ b/Sources/Testing/Running/Configuration.TestFilter.swift @@ -116,7 +116,7 @@ extension Configuration.TestFilter { /// of test IDs. /// /// - Parameters: - /// - selection: A set of test IDs to be excluded. + /// - testIDs: A set of test IDs to be excluded. public init(excluding testIDs: some Collection) { self.init(_kind: .testIDs(Set(testIDs), membership: .excluding)) } diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index be9101d24..3f9ed67f4 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -69,8 +69,8 @@ public struct Configuration: Sendable { /// The conditions under which test iterations should continue. /// /// If the value of this property is `nil`, a test plan will be run - /// ``count`` times regardless of whether or not issues are encountered - /// while running. + /// ``maximumIterationCount`` times regardless of whether or not issues are + /// encountered while running. public var continuationCondition: ContinuationCondition? /// The maximum number of times the test run should iterate. @@ -88,7 +88,8 @@ public struct Configuration: Sendable { /// - continuationCondition: The conditions under which test iterations /// should continue. If `nil`, the iterations should continue /// unconditionally `count` times. - /// - count: The maximum number of times the test run should iterate. + /// - maximumIterationCount: The maximum number of times the test run + /// should iterate. public static func repeating(_ continuationCondition: ContinuationCondition? = nil, maximumIterationCount: Int) -> Self { Self(continuationCondition: continuationCondition, maximumIterationCount: maximumIterationCount) } diff --git a/Sources/Testing/Running/Runner.Plan+Dumping.swift b/Sources/Testing/Running/Runner.Plan+Dumping.swift index dc8d0e23c..1303fcb91 100644 --- a/Sources/Testing/Running/Runner.Plan+Dumping.swift +++ b/Sources/Testing/Running/Runner.Plan+Dumping.swift @@ -96,7 +96,6 @@ extension Runner.Plan { /// `true`, `Swift.dump(_:to:name:indent:maxDepth:maxItems:)` is called /// instead of the testing library's implementation. /// - indent: How many spaces to indent each level of text in the dump. - /// - depth: How many levels deep `stepGraph` is in the total graph. /// /// This function produces a detailed dump of the runner plan suitable for /// inclusion in diagnostics or for display as part of a command-line diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index 33398be59..7553acf6e 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -445,7 +445,7 @@ extension Runner.Plan.Step { } extension Runner.Plan.Action { - /// A serializable snapshot of a ``Runner/Plan-swift.struct/Step/Action`` + /// A serializable snapshot of a ``Runner/Plan-swift.struct/Action`` /// instance. @_spi(ForToolsIntegrationOnly) public enum Snapshot: Sendable, Codable { diff --git a/Sources/Testing/Support/CartesianProduct.swift b/Sources/Testing/Support/CartesianProduct.swift index 43d92e462..07b164eb5 100644 --- a/Sources/Testing/Support/CartesianProduct.swift +++ b/Sources/Testing/Support/CartesianProduct.swift @@ -17,11 +17,11 @@ /// `[(1, "a"), (1, "b"), (1, "c"), (2, "a"), (2, "b"), ... (3, "c")]`. /// /// This type is not part of the public interface of the testing library. -/// -/// @Comment { -/// - Bug: The testing library should support variadic generics. -/// ([103416861](rdar://103416861)) -/// } +// +// @Comment { +// - Bug: The testing library should support variadic generics. +// ([103416861](rdar://103416861)) +// } struct CartesianProduct: LazySequenceProtocol where C1: Collection, C2: Collection { fileprivate var collection1: C1 fileprivate var collection2: C2 @@ -63,11 +63,11 @@ extension CartesianProduct: Sendable where C1: Sendable, C2: Sendable {} /// while `collection2` is iterated `collection1.count` times. /// /// For more information on Cartesian products, see ``CartesianProduct``. -/// -/// @Comment { -/// - Bug: The testing library should support variadic generics. -/// ([103416861](rdar://103416861)) -/// } +// +// @Comment { +// - Bug: The testing library should support variadic generics. +// ([103416861](rdar://103416861)) +// } func cartesianProduct(_ collection1: C1, _ collection2: C2) -> CartesianProduct where C1: Collection, C2: Collection { CartesianProduct(collection1: collection1, collection2: collection2) } diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index 891b37fe3..6f8536ac1 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -220,14 +220,14 @@ public macro Test( /// During testing, the associated test function is called once for each element /// in `collection`. /// -/// @Comment { -/// - Bug: The testing library should support variadic generics. -/// ([103416861](rdar://103416861)) -/// } -/// /// ## See Also /// /// - +// +// @Comment { +// - Bug: The testing library should support variadic generics. +// ([103416861](rdar://103416861)) +// } @attached(peer) public macro Test( _ displayName: _const String? = nil, _ traits: any TestTrait..., @@ -273,14 +273,14 @@ extension Test { /// During testing, the associated test function is called once for each pair of /// elements in `collection1` and `collection2`. /// -/// @Comment { -/// - Bug: The testing library should support variadic generics. -/// ([103416861](rdar://103416861)) -/// } -/// /// ## See Also /// /// - +// +// @Comment { +// - Bug: The testing library should support variadic generics. +// ([103416861](rdar://103416861)) +// } @attached(peer) @_documentation(visibility: private) public macro Test( @@ -301,14 +301,14 @@ public macro Test( /// During testing, the associated test function is called once for each pair of /// elements in `collection1` and `collection2`. /// -/// @Comment { -/// - Bug: The testing library should support variadic generics. -/// ([103416861](rdar://103416861)) -/// } -/// /// ## See Also /// /// - +// +// @Comment { +// - Bug: The testing library should support variadic generics. +// ([103416861](rdar://103416861)) +// } @attached(peer) public macro Test( _ displayName: _const String? = nil, _ traits: any TestTrait..., @@ -327,14 +327,14 @@ public macro Test( /// During testing, the associated test function is called once for each element /// in `zippedCollections`. /// -/// @Comment { -/// - Bug: The testing library should support variadic generics. -/// ([103416861](rdar://103416861)) -/// } -/// /// ## See Also /// /// - +// +// @Comment { +// - Bug: The testing library should support variadic generics. +// ([103416861](rdar://103416861)) +// } @attached(peer) @_documentation(visibility: private) public macro Test( @@ -355,14 +355,14 @@ public macro Test( /// During testing, the associated test function is called once for each element /// in `zippedCollections`. /// -/// @Comment { -/// - Bug: The testing library should support variadic generics. -/// ([103416861](rdar://103416861)) -/// } -/// /// ## See Also /// /// - +// +// @Comment { +// - Bug: The testing library should support variadic generics. +// ([103416861](rdar://103416861)) +// } @attached(peer) public macro Test( _ displayName: _const String? = nil, _ traits: any TestTrait..., diff --git a/Sources/Testing/Testing.docc/LimitingExecutionTime.md b/Sources/Testing/Testing.docc/LimitingExecutionTime.md index 2b992b108..151b52028 100644 --- a/Sources/Testing/Testing.docc/LimitingExecutionTime.md +++ b/Sources/Testing/Testing.docc/LimitingExecutionTime.md @@ -19,8 +19,9 @@ resources to complete, may rely on downloaded data from a server, or may otherwise be dependent on external factors. If a test may hang indefinitely or may consume too many system resources to -complete effectively, consider setting a time limit for it so that it's marked as failing if it runs for an excessive amount of time. Use the -``Trait/timeLimit(_:)`` trait as an upper bound: +complete effectively, consider setting a time limit for it so that it's marked +as failing if it runs for an excessive amount of time. Use the +``Trait/timeLimit(_:)-4kzjp`` trait as an upper bound: ```swift @Test(.timeLimit(.minutes(60)) diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index db8455fc5..3fe181bb9 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -30,7 +30,7 @@ behavior of test functions. - ``Trait/disabled(_:sourceLocation:)`` - ``Trait/disabled(if:_:sourceLocation:)`` - ``Trait/disabled(_:sourceLocation:_:)`` -- ``Trait/timeLimit(_:)`` +- ``Trait/timeLimit(_:)-4kzjp`` ### Running tests serially or in parallel diff --git a/Sources/Testing/Testing.docc/Traits/Trait.md b/Sources/Testing/Testing.docc/Traits/Trait.md index 1528ec1b4..d5a110602 100644 --- a/Sources/Testing/Testing.docc/Traits/Trait.md +++ b/Sources/Testing/Testing.docc/Traits/Trait.md @@ -22,7 +22,7 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors ### Limiting the running time of tests -- ``Trait/timeLimit(_:)`` +- ``Trait/timeLimit(_:)-4kzjp`` ### Running tests serially or in parallel diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index 09c8909dc..8e1117f8a 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -122,12 +122,12 @@ extension Trait where Self == ConditionTrait { /// /// - Returns: An instance of ``ConditionTrait`` that will evaluate the /// specified closure. - /// - /// @Comment { - /// - Bug: `condition` cannot be `async` without making this function - /// `async` even though `condition` is not evaluated locally. - /// ([103037177](rdar://103037177)) - /// } + // + // @Comment { + // - Bug: `condition` cannot be `async` without making this function + // `async` even though `condition` is not evaluated locally. + // ([103037177](rdar://103037177)) + // } public static func enabled( if condition: @autoclosure @escaping @Sendable () throws -> Bool, _ comment: Comment? = nil, @@ -183,12 +183,12 @@ extension Trait where Self == ConditionTrait { /// /// - Returns: An instance of ``ConditionTrait`` that will evaluate the /// specified closure. - /// - /// @Comment { - /// - Bug: `condition` cannot be `async` without making this function - /// `async` even though `condition` is not evaluated locally. - /// ([103037177](rdar://103037177)) - /// } + // + // @Comment { + // - Bug: `condition` cannot be `async` without making this function + // `async` even though `condition` is not evaluated locally. + // ([103037177](rdar://103037177)) + // } public static func disabled( if condition: @autoclosure @escaping @Sendable () throws -> Bool, _ comment: Comment? = nil, diff --git a/Sources/Testing/Traits/TimeLimitTrait.swift b/Sources/Testing/Traits/TimeLimitTrait.swift index fe1d7f787..2dc86de20 100644 --- a/Sources/Testing/Traits/TimeLimitTrait.swift +++ b/Sources/Testing/Traits/TimeLimitTrait.swift @@ -12,7 +12,7 @@ /// /// To add this trait to a test, use one of the following functions: /// -/// - ``Trait/timeLimit(_:)`` +/// - ``Trait/timeLimit(_:)-4kzjp`` @available(_clockAPI, *) public struct TimeLimitTrait: TestTrait, SuiteTrait { /// A type representing the duration of a time limit applied to a test. @@ -189,7 +189,7 @@ extension Test { /// /// Time limits are associated with tests using this trait: /// - /// - ``Trait/timeLimit(_:)`` + /// - ``Trait/timeLimit(_:)-4kzjp`` /// /// If a test has more than one time limit associated with it, the value of /// this property is the shortest one. If a test has no time limits associated diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 0f0d4641a..906bf737a 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -157,11 +157,11 @@ extension Test { /// - testFunction: The function to call when running this test. During /// testing, this function is called once for each element in /// `collection`. - /// - /// @Comment { - /// - Bug: The testing library should support variadic generics. - /// ([103416861](rdar://103416861)) - /// } + // + // @Comment { + // - Bug: The testing library should support variadic generics. + // ([103416861](rdar://103416861)) + // } init( _ traits: any TestTrait..., arguments collection: C, @@ -186,11 +186,11 @@ extension Test { /// - testFunction: The function to call when running this test. During /// testing, this function is called once for each pair of elements in /// `collection1` and `collection2`. - /// - /// @Comment { - /// - Bug: The testing library should support variadic generics. - /// ([103416861](rdar://103416861)) - /// } + // + // @Comment { + // - Bug: The testing library should support variadic generics. + // ([103416861](rdar://103416861)) + // } init( _ traits: any TestTrait..., arguments collection1: C1, _ collection2: C2, From 7ae364735607df035faa2638a25dc5404feb0edb Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 10 Dec 2024 14:12:54 -0500 Subject: [PATCH 012/234] Make `Attachment` conform to `CustomStringConvertible`. (#849) This PR adds `CustomStringConvertible` conformance to `Attachment`. Note that if the attachable value is a move-only type, the conformance is not available in the stdlib, but we still provide a `description` property. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Attachments/Attachment.swift | 15 +++++++++++++++ Tests/TestingTests/AttachmentTests.swift | 14 ++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index 93ae53c73..034f5e03c 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -148,6 +148,21 @@ public struct AnyAttachable: AttachableContainer, Copyable, Sendable { } } +// MARK: - Describing an attachment + +extension Attachment where AttachableValue: ~Copyable { + public var description: String { + let typeInfo = TypeInfo(describing: AttachableValue.self) + return #""\#(preferredName)": instance of '\#(typeInfo.unqualifiedName)'"# + } +} + +extension Attachment: CustomStringConvertible { + public var description: String { + #""\#(preferredName)": \#(String(describingForTest: attachableValue))"# + } +} + // MARK: - Getting an attachable value from an attachment @_spi(Experimental) diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 8d38603e0..4013882a1 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -30,6 +30,20 @@ struct AttachmentTests { attachment.attach() } + @Test func description() { + let attachableValue = MySendableAttachable(string: "") + let attachment = Attachment(attachableValue, named: "AttachmentTests.saveValue.html") + #expect(String(describing: attachment).contains(#""\#(attachment.preferredName)""#)) + #expect(attachment.description.contains("MySendableAttachable(")) + } + + @Test func moveOnlyDescription() { + let attachableValue = MyAttachable(string: "") + let attachment = Attachment(attachableValue, named: "AttachmentTests.saveValue.html") + #expect(attachment.description.contains(#""\#(attachment.preferredName)""#)) + #expect(attachment.description.contains("'MyAttachable'")) + } + #if !SWT_NO_FILE_IO func compare(_ attachableValue: borrowing MySendableAttachable, toContentsOfFileAtPath filePath: String) throws { let file = try FileHandle(forReadingAtPath: filePath) From 8a78407d567271dcfa8afbcff7bae7bb95b04d48 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 10 Dec 2024 14:35:39 -0500 Subject: [PATCH 013/234] Adopt prerelease tags of swift-syntax-601. (#847) This PR moves our main branch's swift-syntax dependency from 600.0.0 to 601. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 2 +- Sources/TestingMacros/CMakeLists.txt | 2 +- Sources/TestingMacros/SuiteDeclarationMacro.swift | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index b109c0447..4d9112c72 100644 --- a/Package.swift +++ b/Package.swift @@ -33,7 +33,7 @@ let package = Package( ], dependencies: [ - .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "600.0.0"), + .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "601.0.0-latest"), ], targets: [ diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index c91620449..ad58fc35b 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -31,7 +31,7 @@ if(SwiftTesting_BuildMacrosAsExecutables) set(FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/_d) FetchContent_Declare(SwiftSyntax GIT_REPOSITORY https://github.com/swiftlang/swift-syntax - GIT_TAG cb53fa1bd3219b0b23ded7dfdd3b2baff266fd25) # 600.0.0 + GIT_TAG 1cd35348b089ff8966588742c69727205d99f8ed) # 601.0.0-prerelease-2024-11-18 FetchContent_MakeAvailable(SwiftSyntax) endif() diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index 3b193bb65..c9fb6bb08 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -19,6 +19,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { public static func expansion( of node: AttributeSyntax, providingMembersOf declaration: some DeclGroupSyntax, + conformingTo protocols: [TypeSyntax], in context: some MacroExpansionContext ) throws -> [DeclSyntax] { guard _diagnoseIssues(with: declaration, suiteAttribute: node, in: context) else { From 0098422b89be902cd7b0076dc1846b0f114ae9c1 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 10 Dec 2024 13:41:33 -0600 Subject: [PATCH 014/234] Simplify the logic for parsing and handling test arguments in the @Test macro (#834) This is a minor refactor of the logic and data structures in the `@Test` macro implementation responsible for parsing and handling test function argument expressions. ### Motivation: I discovered there was a way to simplify this logic while working on #808, and since this isn't strictly pertinent to that PR I wanted to extract it and land it separately here. ### Modifications: - Remove the need to hardcode the parameter label `arguments:` in three places. - Remove the need to iterate a Collection using raw indices. - Resolve one "TODO" comment. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Support/AttributeDiscovery.swift | 53 +++++++++---------- .../DiagnosticMessage+Diagnosing.swift | 5 +- .../TestingMacros/TestDeclarationMacro.swift | 3 +- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/Sources/TestingMacros/Support/AttributeDiscovery.swift b/Sources/TestingMacros/Support/AttributeDiscovery.swift index f1cfd665f..77b2b174e 100644 --- a/Sources/TestingMacros/Support/AttributeDiscovery.swift +++ b/Sources/TestingMacros/Support/AttributeDiscovery.swift @@ -66,17 +66,19 @@ struct AttributeInfo { /// The traits applied to the attribute, if any. var traits = [ExprSyntax]() + /// Test arguments passed to a parameterized test function, if any. + /// + /// When non-`nil`, the value of this property is an array beginning with the + /// argument passed to this attribute for the parameter labeled `arguments:` + /// followed by all of the remaining, unlabeled arguments. + var testFunctionArguments: [Argument]? + /// Whether or not this attribute specifies arguments to the associated test /// function. var hasFunctionArguments: Bool { - otherArguments.lazy - .compactMap(\.label?.tokenKind) - .contains(.identifier("arguments")) + testFunctionArguments != nil } - /// Additional arguments passed to the attribute, if any. - var otherArguments = [Argument]() - /// The source location of the attribute. /// /// When parsing, the testing library uses the start of the attribute's name @@ -98,6 +100,7 @@ struct AttributeInfo { init(byParsing attribute: AttributeSyntax, on declaration: some SyntaxProtocol, in context: some MacroExpansionContext) { self.attribute = attribute + var nonDisplayNameArguments: [Argument] = [] if let arguments = attribute.arguments, case let .argumentList(argumentList) = arguments { // If the first argument is an unlabelled string literal, it's the display // name of the test or suite. If it's anything else, including a nil @@ -106,11 +109,11 @@ struct AttributeInfo { let firstArgumentHasLabel = (firstArgument.label != nil) if !firstArgumentHasLabel, let stringLiteral = firstArgument.expression.as(StringLiteralExprSyntax.self) { displayName = stringLiteral - otherArguments = argumentList.dropFirst().map(Argument.init) + nonDisplayNameArguments = argumentList.dropFirst().map(Argument.init) } else if !firstArgumentHasLabel, firstArgument.expression.is(NilLiteralExprSyntax.self) { - otherArguments = argumentList.dropFirst().map(Argument.init) + nonDisplayNameArguments = argumentList.dropFirst().map(Argument.init) } else { - otherArguments = argumentList.map(Argument.init) + nonDisplayNameArguments = argumentList.map(Argument.init) } } } @@ -119,7 +122,7 @@ struct AttributeInfo { // See _SelfRemover for more information. Rewriting a syntax tree discards // location information from the copy, so only invoke the rewriter if the // `Self` keyword is present somewhere. - otherArguments = otherArguments.map { argument in + nonDisplayNameArguments = nonDisplayNameArguments.map { argument in var expr = argument.expression if argument.expression.tokens(viewMode: .sourceAccurate).map(\.tokenKind).contains(.keyword(.Self)) { let selfRemover = _SelfRemover(in: context) @@ -131,15 +134,14 @@ struct AttributeInfo { // Look for any traits in the remaining arguments and slice them off. Traits // are the remaining unlabelled arguments. The first labelled argument (if // present) is the start of subsequent context-specific arguments. - if !otherArguments.isEmpty { - if let labelledArgumentIndex = otherArguments.firstIndex(where: { $0.label != nil }) { + if !nonDisplayNameArguments.isEmpty { + if let labelledArgumentIndex = nonDisplayNameArguments.firstIndex(where: { $0.label != nil }) { // There is an argument with a label, so splice there. - traits = otherArguments[otherArguments.startIndex ..< labelledArgumentIndex].map(\.expression) - otherArguments = Array(otherArguments[labelledArgumentIndex...]) + traits = nonDisplayNameArguments[.. Date: Tue, 10 Dec 2024 15:19:10 -0500 Subject: [PATCH 015/234] Fix multiline comments not rendering correctly at CLI. (#850) This PR fixes a bug where a multi-line comment or message written to `stderr` would only have its first line in grey and would not write atomically, resulting in odd/incorrect output. All messages generated by a test event are now written to `stderr` atomically. Resolves rdar://134519515. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Event.ConsoleOutputRecorder.swift | 36 +++++++++++++++---- .../Event.HumanReadableOutputRecorder.swift | 14 +------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index a0a6a8ee3..b1e90c535 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -304,18 +304,40 @@ extension Event.ConsoleOutputRecorder { /// destination. @discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool { let messages = _humanReadableOutputRecorder.record(event, in: context) - for message in messages { - let symbol = message.symbol?.stringValue(options: options) ?? " " - if case .details = message.symbol, options.useANSIEscapeCodes, options.ansiColorBitDepth > 1 { + // Padding to use in place of a symbol for messages that don't have one. + var padding = " " +#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) + if options.useSFSymbols { + padding = " " + } +#endif + + let lines = messages.lazy.map { [test = context.test] message in + let symbol = message.symbol?.stringValue(options: options) ?? padding + + if case .details = message.symbol { // Special-case the detail symbol to apply grey to the entire line of - // text instead of just the symbol. - write("\(_ansiEscapeCodePrefix)90m\(symbol) \(message.stringValue)\(_resetANSIEscapeCode)\n") + // text instead of just the symbol. Details may be multi-line messages, + // so split the message on newlines and indent all lines to align them + // to the indentation provided by the symbol. + var lines = message.stringValue.split(whereSeparator: \.isNewline) + lines = CollectionOfOne(lines[0]) + lines.dropFirst().map { line in + "\(padding) \(line)" + } + let stringValue = lines.joined(separator: "\n") + if options.useANSIEscapeCodes, options.ansiColorBitDepth > 1 { + return "\(_ansiEscapeCodePrefix)90m\(symbol) \(stringValue)\(_resetANSIEscapeCode)\n" + } else { + return "\(symbol) \(stringValue)\n" + } } else { - let colorDots = context.test.map(\.tags).map { self.colorDots(for: $0) } ?? "" - write("\(symbol) \(colorDots)\(message.stringValue)\n") + let colorDots = test.map { self.colorDots(for: $0.tags) } ?? "" + return "\(symbol) \(colorDots)\(message.stringValue)\n" } } + + write(lines.joined()) return !messages.isEmpty } diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 9468d55ae..98303f11c 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -92,19 +92,7 @@ extension Event.HumanReadableOutputRecorder { /// - Returns: A formatted string representing `comments`, or `nil` if there /// are none. private func _formattedComments(_ comments: [Comment]) -> [Message] { - // Insert an arrow character at the start of each comment, then indent any - // additional lines in the comment to align them with the arrow. - comments.lazy - .flatMap { comment in - let lines = comment.rawValue.split(whereSeparator: \.isNewline) - if let firstLine = lines.first { - let remainingLines = lines.dropFirst() - return CollectionOfOne(Message(symbol: .details, stringValue: String(firstLine))) + remainingLines.lazy - .map(String.init) - .map { Message(stringValue: $0) } - } - return [] - } + comments.map { Message(symbol: .details, stringValue: $0.rawValue) } } /// Get a string representing the comments attached to a test, formatted for From a239222dc58c66dc3ebad3155b3ba4aa58049fbd Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 12 Dec 2024 12:54:01 -0500 Subject: [PATCH 016/234] Fix `posix_spawn()` error handling. (#855) `posix_spawn()` does not (portably) set `errno`. Instead, it returns its error code. Fix our call. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/SpawnProcess.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 7dd845c67..2d13b1955 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -175,8 +175,9 @@ func spawnExecutable( } var pid = pid_t() - guard 0 == posix_spawn(&pid, executablePath, fileActions, attrs, argv, environ) else { - throw CError(rawValue: swt_errno()) + let processSpawned = posix_spawn(&pid, executablePath, fileActions, attrs, argv, environ) + guard 0 == processSpawned else { + throw CError(rawValue: processSpawned) } return pid } From 97cda258191fca6219444525b739d6014c9f9a83 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Thu, 12 Dec 2024 12:32:18 -0800 Subject: [PATCH 017/234] build: tweak for portability with exceptions Teach the build system how to spell `-fno-exceptions` in a foreign grammar. This allows building with `clang-cl` without a warning for the invalid argument. The spelling is documented at https://learn.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=msvc-170 --- Sources/_TestingInternals/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Sources/_TestingInternals/CMakeLists.txt b/Sources/_TestingInternals/CMakeLists.txt index 8841ba50b..e72143e63 100644 --- a/Sources/_TestingInternals/CMakeLists.txt +++ b/Sources/_TestingInternals/CMakeLists.txt @@ -17,8 +17,14 @@ add_library(_TestingInternals STATIC WillThrow.cpp) target_include_directories(_TestingInternals PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) -target_compile_options(_TestingInternals PRIVATE - -fno-exceptions) +if("${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "MSVC" OR + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + target_compile_options(_TestingInternals PRIVATE + /EHa-c) +else() + target_compile_options(_TestingInternals PRIVATE + -fno-exceptions) +endif() if(NOT BUILD_SHARED_LIBS) # When building a static library, install the internal library archive From 993c7cb8d6cd9f6f16ca0bc6c72ac35596a94c1f Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 12 Dec 2024 16:23:15 -0500 Subject: [PATCH 018/234] Add a `preferredName(for:basedOn:)` member to `Attachable` to allow customizing filenames. (#854) This PR introduces a new optional member of `Attachable`, `preferredName(for:basedOn:)`, that we use when writing the corresponding attachment to disk/test reports/etc. in order to allow attachable types to customize the name independently of what the user specifies. For example: ```swift let a = Attachment(x) let b = Attachment(y, named: "hello") ``` In both the attachments created above, the file name is incompletely specified. `a` has a default name, and both `a` and `b` have no path extension (which is important for the OS to correctly recognize the produced file's type.) By adding this new function to `Attachable`, we give `x` and `y` the opportunity to say "this is JPEG data" or "this is plain text" (and so forth.) The new function is implemented by `_AttachableImageContainer`. I've also created `_AttachableURLContainer` to represent files mapped from disk for attachment (instead of directly passing them around as `Data`.) Third-party conforming types will generally want to use Foundation's `NSString` or `URL` API to append path extensions (etc.) > [!NOTE] > Attachments remain experimental. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/StyleGuide.md | 17 ++++ .../Attachment+AttachableAsCGImage.swift | 33 +------ .../Attachments/ImageAttachmentError.swift | 6 -- .../_AttachableImageContainer.swift | 93 +++++++++++-------- .../Attachments/Attachable+Encodable.swift | 5 - .../Attachable+NSSecureCoding.swift | 4 - .../Attachments/Attachment+URL.swift | 29 ++---- .../Attachments/EncodingFormat.swift | 28 ------ .../Attachments/_AttachableURLContainer.swift | 68 ++++++++++++++ .../_Testing_Foundation/CMakeLists.txt | 1 + Sources/Testing/Attachments/Attachable.swift | 20 ++++ Sources/Testing/Attachments/Attachment.swift | 38 +++++++- Tests/TestingTests/AttachmentTests.swift | 9 +- 13 files changed, 207 insertions(+), 144 deletions(-) create mode 100644 Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift diff --git a/Documentation/StyleGuide.md b/Documentation/StyleGuide.md index 8002463f1..f20c4d7e1 100644 --- a/Documentation/StyleGuide.md +++ b/Documentation/StyleGuide.md @@ -46,6 +46,23 @@ Symbols marked `private` should be given a leading underscore to emphasize that they are private. Symbols marked `fileprivate`, `internal`, etc. should not have a leading underscore (except for those `public` symbols mentioned above.) +Symbols that provide storage for higher-visibility symbols can be underscored if +their preferred names would otherwise conflict. For example: + +```swift +private var _errorCount: Int + +public var errorCount: Int { + get { + _errorCount + } + set { + precondition(newValue >= 0, "Error count cannot be negative") + _errorCount = newValue + } +} +``` + Exported C and C++ symbols that are exported should be given the prefix `swt_` and should otherwise be named using the same lowerCamelCase naming rules as in Swift. Use the `SWT_EXTERN` macro to ensure that symbols are consistently diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift index f93afb7f7..ddc876442 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift @@ -43,38 +43,7 @@ extension Attachment { encodingQuality: Float, sourceLocation: SourceLocation ) where AttachableValue == _AttachableImageContainer { - var imageContainer = _AttachableImageContainer(image: attachableValue, encodingQuality: encodingQuality) - - // Update the preferred name to include an extension appropriate for the - // given content type. (Note the `else` branch duplicates the logic in - // `preferredContentType(forEncodingQuality:)` but will go away once our - // minimum deployment targets include the UniformTypeIdentifiers framework.) - var preferredName = preferredName ?? Self.defaultPreferredName - if #available(_uttypesAPI, *) { - let contentType: UTType = contentType - .map { $0 as! UTType } - .flatMap { contentType in - if UTType.image.conforms(to: contentType) { - // This type is an abstract base type of .image (or .image itself.) - // We'll infer the concrete type based on other arguments. - return nil - } - return contentType - } ?? .preferred(forEncodingQuality: encodingQuality) - preferredName = (preferredName as NSString).appendingPathExtension(for: contentType) - imageContainer.contentType = contentType - } else { - // The caller can't provide a content type, so we'll pick one for them. - let ext = if encodingQuality < 1.0 { - "jpg" - } else { - "png" - } - if (preferredName as NSString).pathExtension.caseInsensitiveCompare(ext) != .orderedSame { - preferredName = (preferredName as NSString).appendingPathExtension(ext) ?? preferredName - } - } - + let imageContainer = _AttachableImageContainer(image: attachableValue, encodingQuality: encodingQuality, contentType: contentType) self.init(imageContainer, named: preferredName, sourceLocation: sourceLocation) } diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift index fdd0b3f3e..b63831317 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift @@ -12,9 +12,6 @@ /// A type representing an error that can occur when attaching an image. @_spi(ForSwiftTestingOnly) public enum ImageAttachmentError: Error, CustomStringConvertible { - /// The specified content type did not conform to `.image`. - case contentTypeDoesNotConformToImage - /// The image could not be converted to an instance of `CGImage`. case couldNotCreateCGImage @@ -24,11 +21,8 @@ public enum ImageAttachmentError: Error, CustomStringConvertible { /// The image could not be converted. case couldNotConvertImage - @_spi(ForSwiftTestingOnly) public var description: String { switch self { - case .contentTypeDoesNotConformToImage: - "The specified type does not represent an image format." case .couldNotCreateCGImage: "Could not create the corresponding Core Graphics image." case .couldNotCreateImageDestination: diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift index 5e8fcd227..9db225826 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift @@ -58,53 +58,75 @@ public struct _AttachableImageContainer: Sendable where Image: Attachable nonisolated(unsafe) var image: Image /// The encoding quality to use when encoding the represented image. - public var encodingQuality: Float + var encodingQuality: Float /// Storage for ``contentType``. private var _contentType: (any Sendable)? /// The content type to use when encoding the image. /// - /// This property should eventually move up to ``Attachment``. It is not part - /// of the public interface of the testing library. + /// The testing library uses this property to determine which image format to + /// encode the associated image as when it is attached to a test. + /// + /// If the value of this property does not conform to [`UTType.image`](https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/image), + /// the result is undefined. @available(_uttypesAPI, *) - var contentType: UTType? { + var contentType: UTType { get { - _contentType as? UTType + if let contentType = _contentType as? UTType { + return contentType + } else { + return encodingQuality < 1.0 ? .jpeg : .png + } } set { + precondition( + newValue.conforms(to: .image), + "An image cannot be attached as an instance of type '\(newValue.identifier)'. Use a type that conforms to 'public.image' instead." + ) _contentType = newValue } } - init(image: Image, encodingQuality: Float) { - self.image = image._makeCopyForAttachment() - self.encodingQuality = encodingQuality + /// The content type to use when encoding the image, substituting a concrete + /// type for `UTType.image`. + /// + /// This property is not part of the public interface of the testing library. + @available(_uttypesAPI, *) + var computedContentType: UTType { + if let contentType = _contentType as? UTType, contentType != .image { + contentType + } else { + encodingQuality < 1.0 ? .jpeg : .png + } } -} - -// MARK: - -@available(_uttypesAPI, *) -extension UTType { - /// Determine the preferred content type to encode this image as for a given - /// encoding quality. + /// The type identifier (as a `CFString`) corresponding to this instance's + /// ``computedContentType`` property. /// - /// - Parameters: - /// - encodingQuality: The encoding quality to use when encoding the image. + /// The value of this property is used by ImageIO when serializing an image. /// - /// - Returns: The type to encode this image as. - static func preferred(forEncodingQuality encodingQuality: Float) -> Self { - // If the caller wants lossy encoding, use JPEG. - if encodingQuality < 1.0 { - return .jpeg + /// This property is not part of the public interface of the testing library. + /// It is used by ImageIO below. + var typeIdentifier: CFString { + if #available(_uttypesAPI, *) { + computedContentType.identifier as CFString + } else { + encodingQuality < 1.0 ? kUTTypeJPEG : kUTTypePNG } + } - // Lossless encoding implies PNG. - return .png + init(image: Image, encodingQuality: Float, contentType: (any Sendable)?) { + self.image = image._makeCopyForAttachment() + self.encodingQuality = encodingQuality + if #available(_uttypesAPI, *), let contentType = contentType as? UTType { + self.contentType = contentType + } } } +// MARK: - + extension _AttachableImageContainer: AttachableContainer { public var attachableValue: Image { image @@ -116,21 +138,6 @@ extension _AttachableImageContainer: AttachableContainer { // Convert the image to a CGImage. let attachableCGImage = try image.attachableCGImage - // Get the type to encode as. (Note the `else` branches duplicate the logic - // in `preferredContentType(forEncodingQuality:)` but will go away once our - // minimum deployment targets include the UniformTypeIdentifiers framework.) - let typeIdentifier: CFString - if #available(_uttypesAPI, *), let contentType { - guard contentType.conforms(to: .image) else { - throw ImageAttachmentError.contentTypeDoesNotConformToImage - } - typeIdentifier = contentType.identifier as CFString - } else if encodingQuality < 1.0 { - typeIdentifier = kUTTypeJPEG - } else { - typeIdentifier = kUTTypePNG - } - // Create the image destination. guard let dest = CGImageDestinationCreateWithData(data as CFMutableData, typeIdentifier, 1, nil) else { throw ImageAttachmentError.couldNotCreateImageDestination @@ -159,5 +166,13 @@ extension _AttachableImageContainer: AttachableContainer { try body(UnsafeRawBufferPointer(start: data.bytes, count: data.length)) } } + + public borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String { + if #available(_uttypesAPI, *) { + return (suggestedName as NSString).appendingPathExtension(for: computedContentType) + } + + return suggestedName + } } #endif diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift index 3e26f7ead..cfae97ca7 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift @@ -86,11 +86,6 @@ extension Attachable where Self: Encodable { /// _and_ [`NSSecureCoding`](https://developer.apple.com/documentation/foundation/nssecurecoding), /// the default implementation of this function uses the value's conformance /// to `Encodable`. - /// - /// - Note: On Apple platforms, if the attachment's preferred name includes - /// some other path extension, that path extension must represent a type - /// that conforms to [`UTType.propertyList`](https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/propertylist) - /// or to [`UTType.json`](https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/json). public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try _Testing_Foundation.withUnsafeBufferPointer(encoding: self, for: attachment, body) } diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift index 622787384..c6916ec39 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift @@ -46,10 +46,6 @@ extension Attachable where Self: NSSecureCoding { /// _and_ [`NSSecureCoding`](https://developer.apple.com/documentation/foundation/nssecurecoding), /// the default implementation of this function uses the value's conformance /// to `Encodable`. - /// - /// - Note: On Apple platforms, if the attachment's preferred name includes - /// some other path extension, that path extension must represent a type - /// that conforms to [`UTType.propertyList`](https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/propertylist). public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { let format = try EncodingFormat(for: attachment) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index 815fcfd18..9bfa027d9 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -37,7 +37,7 @@ extension URL { } @_spi(Experimental) -extension Attachment where AttachableValue == Data { +extension Attachment where AttachableValue == _AttachableURLContainer { #if SWT_TARGET_OS_APPLE /// An operation queue to use for asynchronously reading data from disk. private static let _operationQueue = OperationQueue() @@ -65,30 +65,12 @@ extension Attachment where AttachableValue == Data { throw CocoaError(.featureUnsupported, userInfo: [NSLocalizedDescriptionKey: "Attaching downloaded files is not supported"]) } + // If the user did not provide a preferred name, derive it from the URL. + let preferredName = preferredName ?? url.lastPathComponent + let url = url.resolvingSymlinksInPath() let isDirectory = try url.resourceValues(forKeys: [.isDirectoryKey]).isDirectory! - // Determine the preferred name of the attachment if one was not provided. - var preferredName = if let preferredName { - preferredName - } else if case let lastPathComponent = url.lastPathComponent, !lastPathComponent.isEmpty { - lastPathComponent - } else { - Self.defaultPreferredName - } - - if isDirectory { - // Ensure the preferred name of the archive has an appropriate extension. - preferredName = { -#if SWT_TARGET_OS_APPLE && canImport(UniformTypeIdentifiers) - if #available(_uttypesAPI, *) { - return (preferredName as NSString).appendingPathExtension(for: .zip) - } -#endif - return (preferredName as NSString).appendingPathExtension("zip") ?? preferredName - }() - } - #if SWT_TARGET_OS_APPLE let data: Data = try await withCheckedThrowingContinuation { continuation in let fileCoordinator = NSFileCoordinator() @@ -113,7 +95,8 @@ extension Attachment where AttachableValue == Data { } #endif - self.init(data, named: preferredName, sourceLocation: sourceLocation) + let urlContainer = _AttachableURLContainer(url: url, data: data, isCompressedDirectory: isDirectory) + self.init(urlContainer, named: preferredName, sourceLocation: sourceLocation) } } diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift b/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift index b60a54882..bbbe934ab 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift @@ -12,10 +12,6 @@ @_spi(Experimental) import Testing import Foundation -#if SWT_TARGET_OS_APPLE && canImport(UniformTypeIdentifiers) -private import UniformTypeIdentifiers -#endif - /// An enumeration describing the encoding formats we support for `Encodable` /// and `NSSecureCoding` types that conform to `Attachable`. enum EncodingFormat { @@ -43,30 +39,6 @@ enum EncodingFormat { /// - Throws: If the attachment's content type or media type is unsupported. init(for attachment: borrowing Attachment) throws { let ext = (attachment.preferredName as NSString).pathExtension - -#if SWT_TARGET_OS_APPLE && canImport(UniformTypeIdentifiers) - // If the caller explicitly wants to encode their data as either XML or as a - // property list, use PropertyListEncoder. Otherwise, we'll fall back to - // JSONEncoder below. - if #available(_uttypesAPI, *), let contentType = UTType(filenameExtension: ext) { - if contentType == .data { - self = .default - } else if contentType.conforms(to: .json) { - self = .json - } else if contentType.conforms(to: .xml) { - self = .propertyListFormat(.xml) - } else if contentType.conforms(to: .binaryPropertyList) || contentType == .propertyList { - self = .propertyListFormat(.binary) - } else if contentType.conforms(to: .propertyList) { - self = .propertyListFormat(.openStep) - } else { - let contentTypeDescription = contentType.localizedDescription ?? contentType.identifier - throw CocoaError(.propertyListWriteInvalid, userInfo: [NSLocalizedDescriptionKey: "The content type '\(contentTypeDescription)' cannot be used to attach an instance of \(type(of: self)) to a test."]) - } - return - } -#endif - if ext.isEmpty { // No path extension? No problem! Default data. self = .default diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift b/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift new file mode 100644 index 000000000..38f21d4d3 --- /dev/null +++ b/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift @@ -0,0 +1,68 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if canImport(Foundation) +@_spi(Experimental) public import Testing +public import Foundation + +/// A wrapper type representing file system objects and URLs that can be +/// attached indirectly. +/// +/// You do not need to use this type directly. Instead, initialize an instance +/// of ``Attachment`` using a file URL. +@_spi(Experimental) +public struct _AttachableURLContainer: Sendable { + /// The underlying URL. + var url: URL + + /// The data contained at ``url``. + var data: Data + + /// Whether or not this instance represents a compressed directory. + var isCompressedDirectory: Bool +} + +// MARK: - + +extension _AttachableURLContainer: AttachableContainer { + public var attachableValue: URL { + url + } + + public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + try data.withUnsafeBytes(body) + } + + public borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String { + // What extension should we have on the filename so that it has the same + // type as the original file (or, in the case of a compressed directory, is + // a zip file?) + let preferredPathExtension = if isCompressedDirectory { + "zip" + } else { + url.pathExtension + } + + // What path extension is on the suggested name already? + let nsSuggestedName = suggestedName as NSString + let suggestedPathExtension = nsSuggestedName.pathExtension + + // If the suggested name's extension isn't what we would prefer, append the + // preferred extension. + if !preferredPathExtension.isEmpty, + suggestedPathExtension.caseInsensitiveCompare(preferredPathExtension) != .orderedSame, + let result = nsSuggestedName.appendingPathExtension(preferredPathExtension) { + return result + } + + return suggestedName + } +} +#endif diff --git a/Sources/Overlays/_Testing_Foundation/CMakeLists.txt b/Sources/Overlays/_Testing_Foundation/CMakeLists.txt index 0c942cfa3..54a340323 100644 --- a/Sources/Overlays/_Testing_Foundation/CMakeLists.txt +++ b/Sources/Overlays/_Testing_Foundation/CMakeLists.txt @@ -7,6 +7,7 @@ # See http://swift.org/CONTRIBUTORS.txt for Swift project authors add_library(_Testing_Foundation + Attachments/_AttachableURLContainer.swift Attachments/EncodingFormat.swift Attachments/Attachment+URL.swift Attachments/Attachable+NSSecureCoding.swift diff --git a/Sources/Testing/Attachments/Attachable.swift b/Sources/Testing/Attachments/Attachable.swift index 990f80dee..4a1d775a5 100644 --- a/Sources/Testing/Attachments/Attachable.swift +++ b/Sources/Testing/Attachments/Attachable.swift @@ -64,6 +64,22 @@ public protocol Attachable: ~Copyable { /// would not be idiomatic for the buffer to contain a textual description of /// the image. borrowing func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R + + /// Generate a preferred name for the given attachment. + /// + /// - Parameters: + /// - attachment: The attachment that needs to be named. + /// - suggestedName: A suggested name to use as the basis of the preferred + /// name. This string was provided by the developer when they initialized + /// `attachment`. + /// + /// - Returns: The preferred name for `attachment`. + /// + /// The testing library uses this function to determine the best name to use + /// when adding `attachment` to a test report or persisting it to storage. The + /// default implementation of this function returns `suggestedName` without + /// any changes. + borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String } // MARK: - Default implementations @@ -72,6 +88,10 @@ extension Attachable where Self: ~Copyable { public var estimatedAttachmentByteCount: Int? { nil } + + public borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String { + suggestedName + } } extension Attachable where Self: Collection, Element == UInt8 { diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index 034f5e03c..f69df3679 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -42,6 +42,9 @@ public struct Attachment: ~Copyable where AttachableValue: Atta "untitled" } + /// Storage for ``preferredName``. + fileprivate var _preferredName: String? + /// A filename to use when writing this attachment to a test report or to a /// file on disk. /// @@ -49,7 +52,14 @@ public struct Attachment: ~Copyable where AttachableValue: Atta /// testing library may substitute a different filename as needed. If the /// value of this property has not been explicitly set, the testing library /// will attempt to generate its own value. - public var preferredName: String + public var preferredName: String { + let suggestedName = if let _preferredName, !_preferredName.isEmpty { + _preferredName + } else { + Self.defaultPreferredName + } + return attachableValue.preferredName(for: self, basedOn: suggestedName) + } /// The source location of this instance. /// @@ -83,7 +93,7 @@ extension Attachment where AttachableValue: ~Copyable { /// attachment. public init(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { self._attachableValue = attachableValue - self.preferredName = preferredName ?? Self.defaultPreferredName + self._preferredName = preferredName self.sourceLocation = sourceLocation } } @@ -98,7 +108,7 @@ extension Attachment where AttachableValue == AnyAttachable { self.init( _attachableValue: AnyAttachable(attachableValue: attachment.attachableValue), fileSystemPath: attachment.fileSystemPath, - preferredName: attachment.preferredName, + _preferredName: attachment._preferredName, sourceLocation: attachment.sourceLocation ) } @@ -139,13 +149,26 @@ public struct AnyAttachable: AttachableContainer, Copyable, Sendable { let temporaryAttachment = Attachment( _attachableValue: attachableValue, fileSystemPath: attachment.fileSystemPath, - preferredName: attachment.preferredName, + _preferredName: attachment._preferredName, sourceLocation: attachment.sourceLocation ) return try temporaryAttachment.withUnsafeBufferPointer(body) } return try open(attachableValue, for: attachment) } + + public borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String { + func open(_ attachableValue: T, for attachment: borrowing Attachment) -> String where T: Attachable & Sendable & Copyable { + let temporaryAttachment = Attachment( + _attachableValue: attachableValue, + fileSystemPath: attachment.fileSystemPath, + _preferredName: attachment._preferredName, + sourceLocation: attachment.sourceLocation + ) + return temporaryAttachment.preferredName + } + return open(attachableValue, for: attachment) + } } // MARK: - Describing an attachment @@ -232,7 +255,12 @@ extension Attachment where AttachableValue: ~Copyable { do { let attachmentCopy = try withUnsafeBufferPointer { buffer in let attachableContainer = AnyAttachable(attachableValue: Array(buffer)) - return Attachment(_attachableValue: attachableContainer, fileSystemPath: fileSystemPath, preferredName: preferredName, sourceLocation: sourceLocation) + return Attachment( + _attachableValue: attachableContainer, + fileSystemPath: fileSystemPath, + _preferredName: preferredName, // invokes preferredName(for:basedOn:) + sourceLocation: sourceLocation + ) } Event.post(.valueAttached(attachmentCopy)) } catch { diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 4013882a1..98ecc668d 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -533,7 +533,7 @@ extension AttachmentTests { } @available(_uttypesAPI, *) - @Test(arguments: [Float(0.0).nextUp, 0.25, 0.5, 0.75, 1.0], [.png as UTType?, .jpeg, .gif, .image, .data, nil]) + @Test(arguments: [Float(0.0).nextUp, 0.25, 0.5, 0.75, 1.0], [.png as UTType?, .jpeg, .gif, .image, nil]) func attachCGImage(quality: Float, type: UTType?) throws { let image = try Self.cgImage.get() let attachment = Attachment(image, named: "diamond", as: type, encodingQuality: quality) @@ -541,15 +541,20 @@ extension AttachmentTests { try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { buffer in #expect(buffer.count > 32) } + if let ext = type?.preferredFilenameExtension { + #expect(attachment.preferredName == ("diamond" as NSString).appendingPathExtension(ext)) + } } +#if !SWT_NO_EXIT_TESTS @available(_uttypesAPI, *) @Test func cannotAttachCGImageWithNonImageType() async { - #expect(throws: ImageAttachmentError.contentTypeDoesNotConformToImage) { + await #expect(exitsWith: .failure) { let attachment = Attachment(try Self.cgImage.get(), named: "diamond", as: .mp3) try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { _ in } } } +#endif #endif } } From 5fd1afade12d56b9c676cedef274f4053758a99c Mon Sep 17 00:00:00 2001 From: Joseph Heck Date: Fri, 13 Dec 2024 09:12:07 -0800 Subject: [PATCH 019/234] Add example for how to test any error is thrown in the error testing documentation article (#853) I was stumped earlier when trying to find the details for how to test that any error is thrown. This detail _is_ in the migrating from XCTest content, but I landed on this page, and it seemed an odd miss, especially given the history of ignoring the kind of error in many cases. (I found my solution in [a post by Jonathan the forums](https://forums.swift.org/t/swift-testing-whats-the-recommend-approach-to-test-throwing-function/70806/2).) While it is duplicated content, this would have helped me - so I'm proposing it for a documentation update. Happy to edit as y'all see fit. --------- Co-authored-by: Stuart Montgomery --- .../testing-for-errors-in-swift-code.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md index 5113202d0..244b7bd98 100644 --- a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md +++ b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md @@ -44,6 +44,22 @@ test that the code throws an error of a given type, or matches an arbitrary Boolean test. Similar overloads of ``require(_:_:sourceLocation:)-5l63q`` stop running your test if the code doesn't throw the expected error. +### Validate that your code throws any error + +To check that the code under test throws an error of any type, pass +`(any Error).self` as the first argument to either +``expect(throws:_:sourceLocation:performing:)-1xr34`` or +``require(_:_:sourceLocation:)-5l63q``: + +```swift +@Test func cannotAddToppingToPizzaBeforeStartOfList() { + var order = PizzaToppings(bases: [.calzone, .deepCrust]) + #expect(throws: (any Error).self) { + try order.add(topping: .mozarella, toPizzasIn: -1..<0) + } +} +``` + ### Validate that your code doesn't throw an error A test function that throws an error fails, which is usually sufficient for From 0b771191d75682d9990c05f5a52f11a57fbf8e69 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 13 Dec 2024 15:25:40 -0500 Subject: [PATCH 020/234] [SWT-0006] Return the thrown error from `#expect(throws:)` and `#require(throws:)`. (#780) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR changes the signatures of the various `throws:` overloads of `#expect` and `#require` so that on success they return the error that was thrown rather than `Void`. This then allows more ergonomic inspection of the error's properties: ```swift let error = try #require(throws: MyError.self) { try f() } #expect(error.hasWidget) #expect(error.userName == "John Smith") ``` For more information, see [the proposal document](https://github.com/swiftlang/swift-testing/blob/jgrynspan/return-errors-from-expect-throws/Documentation/Proposals/0006-return-errors-from-expect-throws.md). Resolves rdar://138235250.
Further PR Details It is not possible to overload a macro or function solely by return type without the compiler reporting `Ambiguous use of 'f()'`, so we are not able to stage this change in using `@_spi(Experimental)` without breaking test code that already imports our SPI. This change is potentially source-breaking for tests that inadvertently forward the result of these macro invocations to an enclosing scope. For example, the compiler will start emitting a warning here: ```swift func bar(_ pfoo: UnsafePointer) throws { ... } withUnsafePointer(to: foo) { pfoo in // ⚠️ Result of call to 'withUnsafePointer(to:_:)' is unused #expect(throws: BadFooError.self) { try bar(pfoo) } } ``` This warning can be suppressed by assigning the result of `#expect` (or of `withUnsafePointer(to:_:)`) to `_`: ```swift func bar(_ pfoo: UnsafePointer) throws { ... } withUnsafePointer(to: foo) { pfoo in _ = #expect(throws: BadFooError.self) { try bar(pfoo) } } ``` Because `#expect` and `#require` are macros, they cannot be referenced by name like functions, so you cannot assign them to variables (and then run into trouble with the types of those variables.) Finally, this change deprecates the variants of `#expect` and `#require` that take two closures: ```swift #expect { // ⚠️ 'expect(_:sourceLocation:performing:throws:)' is deprecated: Examine the result of '#expect(throws:)' instead. ... } throws: { error in guard let error = error as? FoodTruckError else { return false } return error.napkinCount == 0 } ``` These variants are no longer needed because you can simply examine the result of the other variants: ```swift let error = #expect(throws: FoodTruckError.self) { ... } #expect(error?.napkinCount == 0) ```
### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../0006-return-errors-from-expect-throws.md | 267 ++++++++++++++++++ .../Expectations/Expectation+Macro.swift | 55 +++- .../ExpectationChecking+Macro.swift | 42 +-- Sources/Testing/Issues/Issue+Recording.swift | 18 +- .../Support/Additions/ResultAdditions.swift | 19 +- Sources/Testing/Support/SystemError.swift | 13 + .../AvailabilityStubs/ExpectComplexThrows.md | 28 ++ .../AvailabilityStubs/RequireComplexThrows.md | 28 ++ Sources/Testing/Testing.docc/Expectations.md | 8 +- .../Testing.docc/MigratingFromXCTest.md | 2 +- .../testing-for-errors-in-swift-code.md | 23 +- Sources/TestingMacros/ConditionMacro.swift | 29 ++ .../MacroExpansionContextAdditions.swift | 52 +++- Sources/TestingMacros/TestingMacrosMain.swift | 1 + .../ConditionMacroTests.swift | 2 + .../TestSupport/Parse.swift | 1 + Tests/TestingTests/IssueTests.swift | 79 ++++++ Tests/TestingTests/Traits/TagListTests.swift | 2 +- 18 files changed, 609 insertions(+), 60 deletions(-) create mode 100644 Documentation/Proposals/0006-return-errors-from-expect-throws.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md diff --git a/Documentation/Proposals/0006-return-errors-from-expect-throws.md b/Documentation/Proposals/0006-return-errors-from-expect-throws.md new file mode 100644 index 000000000..2f15c02ba --- /dev/null +++ b/Documentation/Proposals/0006-return-errors-from-expect-throws.md @@ -0,0 +1,267 @@ +# Return errors from `#expect(throws:)` + +* Proposal: [SWT-0006](0006-filename.md) +* Authors: [Jonathan Grynspan](https://github.com/grynspan) +* Status: **Awaiting review** +* Bug: rdar://138235250 +* Implementation: [swiftlang/swift-testing#780](https://github.com/swiftlang/swift-testing/pull/780) +* Review: ([pitch](https://forums.swift.org/t/pitch-returning-errors-from-expect-throws/75567)) + +## Introduction + +Swift Testing includes overloads of `#expect()` and `#require()` that can be +used to assert that some code throws an error. They are useful when validating +that your code's failure cases are correctly detected and handled. However, for +more complex validation cases, they aren't particularly ergonomic. This proposal +seeks to resolve that issue by having these overloads return thrown errors for +further inspection. + +## Motivation + +We offer three variants of `#expect(throws:)`: + +- One that takes an error type, and matches any error of the same type; +- One that takes an error _instance_ (conforming to `Equatable`) and matches any + error that compares equal to it; and +- One that takes a trailing closure and allows test authors to write arbitrary + validation logic. + +The third overload has proven to be somewhat problematic. First, it yields the +error to its closure as an instance of `any Error`, which typically forces the +developer to cast it before doing any useful comparisons. Second, the test +author must return `true` to indicate the error matched and `false` to indicate +it didn't, which can be both logically confusing and difficult to express +concisely: + +```swift +try #require { + let potato = try Sack.randomPotato() + try potato.turnIntoFrenchFries() +} throws: { error in + guard let error = error as PotatoError else { + return false + } + guard case .potatoNotPeeled = error else { + return false + } + return error.variety != .russet +} +``` + +The first impulse many test authors have here is to use `#expect()` in the +second closure, but it doesn't return the necessary boolean value _and_ it can +result in multiple issues being recorded in a test when there's really only one. + +## Proposed solution + +I propose deprecating [`#expect(_:sourceLocation:performing:throws:)`](https://developer.apple.com/documentation/testing/expect(_:sourcelocation:performing:throws:)) +and [`#require(_:sourceLocation:performing:throws:)`](https://developer.apple.com/documentation/testing/require(_:sourcelocation:performing:throws:)) +and modifying the other overloads so that, on success, they return the errors +that were thrown. + +## Detailed design + +All overloads of `#expect(throws:)` and `#require(throws:)` will be updated to +return an instance of the error type specified by their arguments, with the +problematic overloads returning `any Error` since more precise type information +is not statically available. The problematic overloads will also be deprecated: + +```diff +--- a/Sources/Testing/Expectations/Expectation+Macro.swift ++++ b/Sources/Testing/Expectations/Expectation+Macro.swift ++@discardableResult + @freestanding(expression) public macro expect( + throws errorType: E.Type, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) ++) -> E? where E: Error + ++@discardableResult + @freestanding(expression) public macro require( + throws errorType: E.Type, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) where E: Error ++) -> E where E: Error + ++@discardableResult + @freestanding(expression) public macro expect( + throws error: E, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) where E: Error & Equatable ++) -> E? where E: Error & Equatable + ++@discardableResult + @freestanding(expression) public macro require( + throws error: E, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R +-) where E: Error & Equatable ++) -> E where E: Error & Equatable + ++@available(*, deprecated, message: "Examine the result of '#expect(throws:)' instead.") ++@discardableResult + @freestanding(expression) public macro expect( + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R, + throws errorMatcher: (any Error) async throws -> Bool +-) ++) -> (any Error)? + ++@available(*, deprecated, message: "Examine the result of '#require(throws:)' instead.") ++@discardableResult + @freestanding(expression) public macro require( + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () async throws -> R, + throws errorMatcher: (any Error) async throws -> Bool +-) ++) -> any Error +``` + +(More detailed information about the deprecations will be provided via DocC.) + +The `#expect(throws:)` overloads return an optional value that is `nil` if the +expectation failed, while the `#require(throws:)` overloads return non-optional +values and throw instances of `ExpectationFailedError` on failure (as before.) + +> [!NOTE] +> Instances of `ExpectationFailedError` thrown by `#require(throws:)` on failure +> are not returned as that would defeat the purpose of using `#require(throws:)` +> instead of `#expect(throws:)`. + +Test authors will be able to use the result of the above functions to verify +that the thrown error is correct: + +```swift +let error = try #require(throws: PotatoError.self) { + let potato = try Sack.randomPotato() + try potato.turnIntoFrenchFries() +} +#expect(error == .potatoNotPeeled) +#expect(error.variety != .russet) +``` + +The new code is more concise than the old code and avoids boilerplate casting +from `any Error`. + +## Source compatibility + +In most cases, this change does not affect source compatibility. Swift does not +allow forming references to macros at runtime, so we don't need to worry about +type mismatches assigning one to some local variable. + +We have identified two scenarios where a new warning will be emitted. + +### Inferred return type from macro invocation + +The return type of the macro may be used by the compiler to infer the return +type of an enclosing closure. If the return value is then discarded, the +compiler may emit a warning: + +```swift +func pokePotato(_ pPotato: UnsafePointer) throws { ... } + +let potato = Potato() +try await Task.sleep(for: .months(3)) +withUnsafePointer(to: potato) { pPotato in + // ^ ^ ^ ⚠️ Result of call to 'withUnsafePointer(to:_:)' is unused + #expect(throws: PotatoError.rotten) { + try pokePotato(pPotato) + } +} +``` + +This warning can be suppressed by assigning the result of the macro invocation +or the result of the function call to `_`: + +```swift +withUnsafePointer(to: potato) { pPotato in + _ = #expect(throws: PotatoError.rotten) { + try pokePotato(pPotato) + } +} +``` + +### Use of `#require(throws:)` in a generic context with `Never.self` + +If `#require(throws:)` (but not `#expect(throws:)`) is used in a generic context +where the type of thrown error is a generic parameter, and the type is resolved +to `Never`, there is no valid value for the invocation to return: + +```swift +func wrapper(throws type: E.Type, _ body: () throws -> Void) throws -> E { + return try #require(throws: type) { + try body() + } +} +let error = try #require(throws: Never.self) { ... } +``` + +We don't think this particular pattern is common (and outside of our own test +target, I'd be surprised if anybody's attempted it yet.) However, we do need to +handle it gracefully. If this pattern is encountered, Swift Testing will record +an "API Misused" issue for the current test and advise the test author to switch +to `#expect(throws:)` or to not pass `Never.self` here. + +## Integration with supporting tools + +N/A + +## Future directions + +- Adopting [typed throws](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0413-typed-throws.md) + to statically require that the error thrown from test code is of the correct + type. + + If we adopted typed throws in the signatures of these macros, it would force + adoption of typed throws in the code under test even when it may not be + appropriate. For example, if we adopted typed throws, the following code would + not compile: + + ```swift + func cook(_ food: consuming some Food) throws { ... } + + let error: PotatoError? = #expect(throws: PotatoError.self) { + var potato = Potato() + potato.fossilize() + try cook(potato) // 🛑 ERROR: Invalid conversion of thrown error type + // 'any Error' to 'PotatoError' + } + ``` + + We believe it may be possible to overload these macros or their expansions so + that the code sample above _does_ compile and behave as intended. We intend to + experiment further with this idea and potentially revisit typed throws support + in a future proposal. + +## Alternatives considered + +- Leaving the existing implementation and signatures in place. We've had + sufficient feedback about the ergonomics of this API that we want to address + the problem. + +- Having the return type of the macros be `any Error` and returning _any_ error + that was thrown even on mismatch. This would make the ergonomics of the + subsequent test code less optimal because the test author would need to cast + the error to the appropriate type before inspecting it. + + There's a philosophical argument to be made here that if a mismatched error is + thrown, then the test has already failed and is in an inconsistent state, so + we should allow the test to fail rather than return what amounts to "bad + output". + + If the test author wants to inspect any arbitrary thrown error, they can + specify `(any Error).self` instead of a concrete error type. + +## Acknowledgments + +Thanks to the team and to [@jakepetroules](https://github.com/jakepetroules) for +starting the discussion that ultimately led to this proposal. diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 5012c93ca..c8a691e85 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -142,6 +142,9 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: If the expectation passes, the instance of `errorType` that was +/// thrown by `expression`. If the expectation fails, the result is `nil`. +/// /// Use this overload of `#expect()` when the expression `expression` _should_ /// throw an error of a given type: /// @@ -158,7 +161,7 @@ public macro require( /// discarded. /// /// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``expect(throws:_:sourceLocation:performing:)-1xr34`` instead. +/// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. /// /// ## Expressions that should never throw /// @@ -181,12 +184,13 @@ public macro require( /// fail when an error is thrown by `expression`, rather than to explicitly /// check that an error is _not_ thrown by it, do not use this macro. Instead, /// simply call the code in question and allow it to throw an error naturally. +@discardableResult @freestanding(expression) public macro expect( throws errorType: E.Type, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error +) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error /// Check that an expression always throws an error of a given type, and throw /// an error if it does not. @@ -200,6 +204,8 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: The instance of `errorType` that was thrown by `expression`. +/// /// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not /// throw a matching error. The error thrown by `expression` is not rethrown. /// @@ -219,16 +225,17 @@ public macro require( /// is thrown. Any value returned by `expression` is discarded. /// /// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``require(throws:_:sourceLocation:performing:)-7v83e`` instead. +/// use ``require(throws:_:sourceLocation:performing:)-4djuw`` instead. /// /// If `expression` should _never_ throw, simply invoke the code without using /// this macro. The test will then fail if an error is thrown. +@discardableResult @freestanding(expression) public macro require( throws errorType: E.Type, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") where E: Error +) -> E = #externalMacro(module: "TestingMacros", type: "RequireThrowsMacro") where E: Error /// Check that an expression never throws an error, and throw an error if it /// does. @@ -261,6 +268,10 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: If the expectation passes, the instance of `E` that was thrown by +/// `expression` and is equal to `error`. If the expectation fails, the result +/// is `nil`. +/// /// Use this overload of `#expect()` when the expression `expression` _should_ /// throw a specific error: /// @@ -276,13 +287,14 @@ public macro require( /// in the current task. Any value returned by `expression` is discarded. /// /// If the thrown error need only be an instance of a particular type, use -/// ``expect(throws:_:sourceLocation:performing:)-79piu`` instead. +/// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. +@discardableResult @freestanding(expression) public macro expect( throws error: E, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error & Equatable +) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error & Equatable /// Check that an expression always throws a specific error, and throw an error /// if it does not. @@ -293,6 +305,9 @@ public macro require( /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. /// - expression: The expression to be evaluated. + +/// - Returns: The instance of `E` that was thrown by `expression` and is equal +/// to `error`. /// /// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not /// throw a matching error. The error thrown by `expression` is not rethrown. @@ -313,13 +328,14 @@ public macro require( /// Any value returned by `expression` is discarded. /// /// If the thrown error need only be an instance of a particular type, use -/// ``require(throws:_:sourceLocation:performing:)-76bjn`` instead. +/// ``require(throws:_:sourceLocation:performing:)-7n34r`` instead. +@discardableResult @freestanding(expression) public macro require( throws error: E, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") where E: Error & Equatable +) -> E = #externalMacro(module: "TestingMacros", type: "RequireMacro") where E: Error & Equatable // MARK: - Arbitrary error matching @@ -333,6 +349,9 @@ public macro require( /// - errorMatcher: A closure to invoke when `expression` throws an error that /// indicates if it matched or not. /// +/// - Returns: If the expectation passes, the error that was thrown by +/// `expression`. If the expectation fails, the result is `nil`. +/// /// Use this overload of `#expect()` when the expression `expression` _should_ /// throw an error, but the logic to determine if the error matches is complex: /// @@ -353,15 +372,17 @@ public macro require( /// discarded. /// /// If the thrown error need only be an instance of a particular type, use -/// ``expect(throws:_:sourceLocation:performing:)-79piu`` instead. If the thrown +/// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. If the thrown /// error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``expect(throws:_:sourceLocation:performing:)-1xr34`` instead. +/// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. +@available(*, deprecated, message: "Examine the result of '#expect(throws:)' instead.") +@discardableResult @freestanding(expression) public macro expect( _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R, throws errorMatcher: (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") +) -> (any Error)? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") /// Check that an expression always throws an error matching some condition, and /// throw an error if it does not. @@ -374,6 +395,8 @@ public macro require( /// - errorMatcher: A closure to invoke when `expression` throws an error that /// indicates if it matched or not. /// +/// - Returns: The error that was thrown by `expression`. +/// /// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not /// throw a matching error. The error thrown by `expression` is not rethrown. /// @@ -398,18 +421,20 @@ public macro require( /// discarded. /// /// If the thrown error need only be an instance of a particular type, use -/// ``require(throws:_:sourceLocation:performing:)-76bjn`` instead. If the thrown error need +/// ``require(throws:_:sourceLocation:performing:)-7n34r`` instead. If the thrown error need /// only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), -/// use ``require(throws:_:sourceLocation:performing:)-7v83e`` instead. +/// use ``require(throws:_:sourceLocation:performing:)-4djuw`` instead. /// /// If `expression` should _never_ throw, simply invoke the code without using /// this macro. The test will then fail if an error is thrown. +@available(*, deprecated, message: "Examine the result of '#require(throws:)' instead.") +@discardableResult @freestanding(expression) public macro require( _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: () async throws -> R, throws errorMatcher: (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") +) -> any Error = #externalMacro(module: "TestingMacros", type: "RequireMacro") // MARK: - Exit tests @@ -425,7 +450,7 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// -/// - Returns: If the exit test passed, an instance of ``ExitTestArtifacts`` +/// - Returns: If the exit test passes, an instance of ``ExitTestArtifacts`` /// describing the state of the exit test when it exited. If the exit test /// fails, the result is `nil`. /// diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index eff01e5bf..ca452e2f8 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -824,7 +824,8 @@ public func __checkCast( /// Check that an expression always throws an error. /// /// This overload is used for `#expect(throws:) { }` invocations that take error -/// types. +/// types. It is disfavored so that `#expect(throws: Never.self)` preferentially +/// returns `Void`. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. @@ -835,7 +836,7 @@ public func __checkClosureCall( comments: @autoclosure () -> [Comment], isRequired: Bool, sourceLocation: SourceLocation -) -> Result where E: Error { +) -> Result where E: Error { if errorType == Never.self { __checkClosureCall( throws: Never.self, @@ -844,7 +845,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { _ in nil } } else { __checkClosureCall( performing: body, @@ -854,14 +855,15 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } } /// Check that an expression always throws an error. /// /// This overload is used for `await #expect(throws:) { }` invocations that take -/// error types. +/// error types. It is disfavored so that `#expect(throws: Never.self)` +/// preferentially returns `Void`. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. @@ -873,7 +875,7 @@ public func __checkClosureCall( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result where E: Error { +) async -> Result where E: Error { if errorType == Never.self { await __checkClosureCall( throws: Never.self, @@ -883,7 +885,7 @@ public func __checkClosureCall( isRequired: isRequired, isolation: isolation, sourceLocation: sourceLocation - ) + ).map { _ in nil } } else { await __checkClosureCall( performing: body, @@ -894,7 +896,7 @@ public func __checkClosureCall( isRequired: isRequired, isolation: isolation, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } } @@ -932,7 +934,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { _ in } } /// Check that an expression never throws an error. @@ -969,7 +971,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { _ in } } // MARK: - Matching instances of equatable errors @@ -988,7 +990,7 @@ public func __checkClosureCall( comments: @autoclosure () -> [Comment], isRequired: Bool, sourceLocation: SourceLocation -) -> Result where E: Error & Equatable { +) -> Result where E: Error & Equatable { __checkClosureCall( performing: body, throws: { true == (($0 as? E) == error) }, @@ -997,7 +999,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } /// Check that an expression always throws an error. @@ -1015,7 +1017,7 @@ public func __checkClosureCall( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result where E: Error & Equatable { +) async -> Result where E: Error & Equatable { await __checkClosureCall( performing: body, throws: { true == (($0 as? E) == error) }, @@ -1025,7 +1027,7 @@ public func __checkClosureCall( isRequired: isRequired, isolation: isolation, sourceLocation: sourceLocation - ) + ).map { $0 as? E } } // MARK: - Arbitrary error matching @@ -1044,10 +1046,11 @@ public func __checkClosureCall( comments: @autoclosure () -> [Comment], isRequired: Bool, sourceLocation: SourceLocation -) -> Result { +) -> Result<(any Error)?, any Error> { var errorMatches = false var mismatchExplanationValue: String? = nil var expression = expression + var caughtError: (any Error)? do { let result = try body() @@ -1057,6 +1060,7 @@ public func __checkClosureCall( } mismatchExplanationValue = explanation } catch { + caughtError = error expression = expression.capturingRuntimeValues(error) let secondError = Issue.withErrorRecording(at: sourceLocation) { errorMatches = try errorMatcher(error) @@ -1075,7 +1079,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { caughtError } } /// Check that an expression always throws an error. @@ -1093,10 +1097,11 @@ public func __checkClosureCall( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result<(any Error)?, any Error> { var errorMatches = false var mismatchExplanationValue: String? = nil var expression = expression + var caughtError: (any Error)? do { let result = try await body() @@ -1106,6 +1111,7 @@ public func __checkClosureCall( } mismatchExplanationValue = explanation } catch { + caughtError = error expression = expression.capturingRuntimeValues(error) let secondError = await Issue.withErrorRecording(at: sourceLocation) { errorMatches = try await errorMatcher(error) @@ -1124,7 +1130,7 @@ public func __checkClosureCall( comments: comments(), isRequired: isRequired, sourceLocation: sourceLocation - ) + ).map { caughtError } } // MARK: - Exit tests diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index e13099eaf..8a80e4467 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -29,11 +29,19 @@ extension Issue { func record(configuration: Configuration? = nil) -> Self { // If this issue is a caught error of kind SystemError, reinterpret it as a // testing system issue instead (per the documentation for SystemError.) - if case let .errorCaught(error) = kind, let error = error as? SystemError { - var selfCopy = self - selfCopy.kind = .system - selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) - return selfCopy.record(configuration: configuration) + if case let .errorCaught(error) = kind { + // TODO: consider factoring this logic out into a protocol + if let error = error as? SystemError { + var selfCopy = self + selfCopy.kind = .system + selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) + return selfCopy.record(configuration: configuration) + } else if let error = error as? APIMisuseError { + var selfCopy = self + selfCopy.kind = .apiMisused + selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) + return selfCopy.record(configuration: configuration) + } } // If this issue matches via the known issue matcher, set a copy of it to be diff --git a/Sources/Testing/Support/Additions/ResultAdditions.swift b/Sources/Testing/Support/Additions/ResultAdditions.swift index f14f68c85..9a2e6ea5a 100644 --- a/Sources/Testing/Support/Additions/ResultAdditions.swift +++ b/Sources/Testing/Support/Additions/ResultAdditions.swift @@ -31,16 +31,27 @@ extension Result { /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. - @inlinable public func __expected() -> Success where Success == T? { + @discardableResult @inlinable public func __expected() -> Success where Success == T? { try? get() } /// Handle this instance as if it were returned from a call to `#require()`. /// + /// This overload of `__require()` assumes that the result cannot actually be + /// `nil` on success. The optionality is part of our ABI contract for the + /// `__check()` function family so that we can support uninhabited types and + /// "soft" failures. + /// + /// If the value really is `nil` (e.g. we're dealing with `Never`), the + /// testing library throws an error representing an issue of kind + /// ``Issue/Kind-swift.enum/apiMisused``. + /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. - @inlinable public func __required() throws -> T where Success == T? { - // TODO: handle edge case where the value is nil (see #780) - try get()! + @discardableResult public func __required() throws -> T where Success == T? { + guard let result = try get() else { + throw APIMisuseError(description: "Could not unwrap 'nil' value of type Optional<\(T.self)>. Consider using #expect() instead of #require() here.") + } + return result } } diff --git a/Sources/Testing/Support/SystemError.swift b/Sources/Testing/Support/SystemError.swift index d68b9c241..d2d4809e3 100644 --- a/Sources/Testing/Support/SystemError.swift +++ b/Sources/Testing/Support/SystemError.swift @@ -21,3 +21,16 @@ struct SystemError: Error, CustomStringConvertible { var description: String } + +/// A type representing misuse of testing library API. +/// +/// When an error of this type is thrown and caught by the testing library, it +/// is recorded as an issue of kind ``Issue/Kind/apiMisused`` rather than +/// ``Issue/Kind/errorCaught(_:)``. +/// +/// This type is not part of the public interface of the testing library. +/// External callers should generally record issues by throwing their own errors +/// or by calling ``Issue/record(_:sourceLocation:)``. +struct APIMisuseError: Error, CustomStringConvertible { + var description: String +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md new file mode 100644 index 000000000..755bf5089 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md @@ -0,0 +1,28 @@ +# ``expect(_:sourceLocation:performing:throws:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.0, deprecated: 999.0) + @Available(Xcode, introduced: 16.0, deprecated: 999.0) +} + +@DeprecationSummary { + Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` or + ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: + + ```swift + let error = #expect(throws: FoodTruckError.self) { + ... + } + #expect(error?.napkinCount == 0) + ``` +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md new file mode 100644 index 000000000..ff42dc40f --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md @@ -0,0 +1,28 @@ +# ``require(_:sourceLocation:performing:throws:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.0, deprecated: 999.0) + @Available(Xcode, introduced: 16.0, deprecated: 999.0) +} + +@DeprecationSummary { + Examine the result of ``require(throws:_:sourceLocation:performing:)-7n34r`` + or ``require(throws:_:sourceLocation:performing:)-4djuw`` instead: + + ```swift + let error = try #require(throws: FoodTruckError.self) { + ... + } + #expect(error.napkinCount == 0) + ``` +} diff --git a/Sources/Testing/Testing.docc/Expectations.md b/Sources/Testing/Testing.docc/Expectations.md index ce92824c1..fd3b0070d 100644 --- a/Sources/Testing/Testing.docc/Expectations.md +++ b/Sources/Testing/Testing.docc/Expectations.md @@ -65,11 +65,11 @@ the test when the code doesn't satisfy a requirement, use ### Checking that errors are thrown - -- ``expect(throws:_:sourceLocation:performing:)-79piu`` -- ``expect(throws:_:sourceLocation:performing:)-1xr34`` +- ``expect(throws:_:sourceLocation:performing:)-1hfms`` +- ``expect(throws:_:sourceLocation:performing:)-7du1h`` - ``expect(_:sourceLocation:performing:throws:)`` -- ``require(throws:_:sourceLocation:performing:)-76bjn`` -- ``require(throws:_:sourceLocation:performing:)-7v83e`` +- ``require(throws:_:sourceLocation:performing:)-7n34r`` +- ``require(throws:_:sourceLocation:performing:)-4djuw`` - ``require(_:sourceLocation:performing:throws:)`` ### Confirming that asynchronous events occur diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index bf0b43e34..133daa49c 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -326,7 +326,7 @@ their equivalents in the testing library: | `XCTAssertLessThanOrEqual(x, y)` | `#expect(x <= y)` | | `XCTAssertLessThan(x, y)` | `#expect(x < y)` | | `XCTAssertThrowsError(try f())` | `#expect(throws: (any Error).self) { try f() }` | -| `XCTAssertThrowsError(try f()) { error in … }` | `#expect { try f() } throws: { error in return … }` | +| `XCTAssertThrowsError(try f()) { error in … }` | `let error = #expect(throws: (any Error).self) { try f() }` | | `XCTAssertNoThrow(try f())` | `#expect(throws: Never.self) { try f() }` | | `try XCTUnwrap(x)` | `try #require(x)` | | `XCTFail("…")` | `Issue.record("…")` | diff --git a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md index 244b7bd98..2611f8835 100644 --- a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md +++ b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md @@ -26,7 +26,7 @@ If the code throws an error, then your test fails. To check that the code under test throws a specific error, or to continue a longer test function after the code throws an error, pass that error as the -first argument of ``expect(throws:_:sourceLocation:performing:)-1xr34``, and +first argument of ``expect(throws:_:sourceLocation:performing:)-7du1h``, and pass a closure that calls the code under test: ```swift @@ -81,4 +81,23 @@ the error to `Never`: If the closure throws _any_ error, the testing library records an issue. If you need the test to stop when the code throws an error, include the code inline in the test function instead of wrapping it in a call to -``expect(throws:_:sourceLocation:performing:)-1xr34``. +``expect(throws:_:sourceLocation:performing:)-7du1h``. + +## Inspect an error thrown by your code + +When you use `#expect(throws:)` or `#require(throws:)` and the error matches the +expectation, it is returned to the caller so that you can perform additional +validation. If the expectation fails because no error was thrown or an error of +a different type was thrown, `#expect(throws:)` returns `nil`: + +```swift +@Test func cannotAddMarshmallowsToPizza() throws { + let error = #expect(throws: PizzaToppings.InvalidToppingError.self) { + try Pizza.current.add(topping: .marshmallows) + } + #expect(error?.topping == .marshmallows) + #expect(error?.reason == .dessertToppingOnly) +} +``` + +If you aren't sure what type of error will be thrown, pass `(any Error).self`. diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 341b27d7d..3d2013c69 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -332,6 +332,35 @@ public struct NonOptionalRequireMacro: RefinedConditionMacro { } } +/// A type describing the expansion of the `#require(throws:)` macro. +/// +/// This macro makes a best effort to check if the type argument is `Never.self` +/// (as we only have the syntax tree here) and diagnoses it as redundant if so. +/// See also ``RequireThrowsNeverMacro`` which is used when full type checking +/// is contextually available. +/// +/// This type is otherwise exactly equivalent to ``RequireMacro``. +public struct RequireThrowsMacro: RefinedConditionMacro { + public typealias Base = RequireMacro + + public static func expansion( + of macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) throws -> ExprSyntax { + if let argument = macro.arguments.first { + let argumentTokens: [String] = argument.expression.tokens(viewMode: .fixedUp).lazy + .filter { $0.tokenKind != .period } + .map(\.textWithoutBackticks) + if argumentTokens == ["Swift", "Never", "self"] || argumentTokens == ["Never", "self"] { + context.diagnose(.requireThrowsNeverIsRedundant(argument.expression, in: macro)) + } + } + + // Perform the normal macro expansion for #require(). + return try RequireMacro.expansion(of: macro, in: context) + } +} + /// A type describing the expansion of the `#require(throws:)` macro when it is /// passed `Never.self`, which is redundant. /// diff --git a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift index 4539ed04d..ca0137b5d 100644 --- a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift @@ -80,6 +80,34 @@ extension MacroExpansionContext { // MARK: - extension MacroExpansionContext { + /// Whether or not our generated warnings are suppressed in the current + /// lexical context. + /// + /// The value of this property is `true` if the current lexical context + /// contains a node with the `@_semantics("testing.macros.nowarnings")` + /// attribute applied to it. + /// + /// - Warning: This functionality is not part of the public interface of the + /// testing library. It may be modified or removed in a future update. + var areWarningsSuppressed: Bool { +#if DEBUG + for lexicalContext in self.lexicalContext { + guard let lexicalContext = lexicalContext.asProtocol((any WithAttributesSyntax).self) else { + continue + } + for attribute in lexicalContext.attributes { + if case let .attribute(attribute) = attribute, + attribute.attributeNameText == "_semantics", + case let .string(argument) = attribute.arguments, + argument.representedLiteralValue == "testing.macros.nowarnings" { + return true + } + } + } +#endif + return false + } + /// Emit a diagnostic message. /// /// - Parameters: @@ -87,23 +115,27 @@ extension MacroExpansionContext { /// arguments to `Diagnostic.init()` are derived from the message's /// `syntax` property. func diagnose(_ message: DiagnosticMessage) { - diagnose( - Diagnostic( - node: message.syntax, - position: message.syntax.positionAfterSkippingLeadingTrivia, - message: message, - fixIts: message.fixIts - ) - ) + diagnose(CollectionOfOne(message)) } /// Emit a sequence of diagnostic messages. /// /// - Parameters: /// - messages: The diagnostic messages to emit. - func diagnose(_ messages: some Sequence) { + func diagnose(_ messages: some Collection) { + lazy var areWarningsSuppressed = areWarningsSuppressed for message in messages { - diagnose(message) + if message.severity == .warning && areWarningsSuppressed { + continue + } + diagnose( + Diagnostic( + node: message.syntax, + position: message.syntax.positionAfterSkippingLeadingTrivia, + message: message, + fixIts: message.fixIts + ) + ) } } diff --git a/Sources/TestingMacros/TestingMacrosMain.swift b/Sources/TestingMacros/TestingMacrosMain.swift index ebc62d660..c6904a6e7 100644 --- a/Sources/TestingMacros/TestingMacrosMain.swift +++ b/Sources/TestingMacros/TestingMacrosMain.swift @@ -24,6 +24,7 @@ struct TestingMacrosMain: CompilerPlugin { RequireMacro.self, AmbiguousRequireMacro.self, NonOptionalRequireMacro.self, + RequireThrowsMacro.self, RequireThrowsNeverMacro.self, ExitTestExpectMacro.self, ExitTestRequireMacro.self, diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index 7ede6233c..9f1201367 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -354,6 +354,8 @@ struct ConditionMacroTests { @Test("#require(throws: Never.self) produces a diagnostic", arguments: [ + "#requireThrows(throws: Swift.Never.self)", + "#requireThrows(throws: Never.self)", "#requireThrowsNever(throws: Never.self)", ] ) diff --git a/Tests/TestingMacrosTests/TestSupport/Parse.swift b/Tests/TestingMacrosTests/TestSupport/Parse.swift index fcb0215bc..e6b36e3b2 100644 --- a/Tests/TestingMacrosTests/TestSupport/Parse.swift +++ b/Tests/TestingMacrosTests/TestSupport/Parse.swift @@ -23,6 +23,7 @@ fileprivate let allMacros: [String: any Macro.Type] = [ "require": RequireMacro.self, "requireAmbiguous": AmbiguousRequireMacro.self, // different name needed only for unit testing "requireNonOptional": NonOptionalRequireMacro.self, // different name needed only for unit testing + "requireThrows": RequireThrowsMacro.self, // different name needed only for unit testing "requireThrowsNever": RequireThrowsNeverMacro.self, // different name needed only for unit testing "expectExitTest": ExitTestRequireMacro.self, // different name needed only for unit testing "requireExitTest": ExitTestRequireMacro.self, // different name needed only for unit testing diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 53fe92b84..631ff0c54 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -491,6 +491,7 @@ final class IssueTests: XCTestCase { }.run(configuration: .init()) } + @available(*, deprecated) func testErrorCheckingWithExpect() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.isInverted = true @@ -539,6 +540,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } + @available(*, deprecated) func testErrorCheckingWithExpect_Mismatching() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.expectedFulfillmentCount = 13 @@ -663,6 +665,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } + @available(*, deprecated) func testErrorCheckingWithExpectAsync() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.isInverted = true @@ -706,6 +709,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } + @available(*, deprecated) func testErrorCheckingWithExpectAsync_Mismatching() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.expectedFulfillmentCount = 13 @@ -822,6 +826,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } + @available(*, deprecated) func testErrorCheckingWithExpect_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") @@ -849,6 +854,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } + @available(*, deprecated) func testErrorCheckingWithExpectAsync_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") @@ -876,6 +882,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } + @available(*, deprecated) func testErrorCheckingWithRequire_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") @@ -904,6 +911,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } + @available(*, deprecated) func testErrorCheckingWithRequireAsync_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") @@ -932,6 +940,77 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } + func testErrorCheckingWithExpect_ResultValue() throws { + let error = #expect(throws: MyDescriptiveError.self) { + throw MyDescriptiveError(description: "abc123") + } + #expect(error?.description == "abc123") + } + + func testErrorCheckingWithRequire_ResultValue() async throws { + let error = try #require(throws: MyDescriptiveError.self) { + throw MyDescriptiveError(description: "abc123") + } + #expect(error.description == "abc123") + } + + func testErrorCheckingWithExpect_ResultValueIsNever() async throws { + let error: Never? = #expect(throws: Never.self) { + throw MyDescriptiveError(description: "abc123") + } + #expect(error == nil) + } + + func testErrorCheckingWithRequire_ResultValueIsNever() async throws { + let errorCaught = expectation(description: "Error caught") + errorCaught.isInverted = true + let apiMisused = expectation(description: "API misused") + let expectationFailed = expectation(description: "Expectation failed") + expectationFailed.isInverted = true + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + if case .errorCaught = issue.kind { + errorCaught.fulfill() + } else if case .apiMisused = issue.kind { + apiMisused.fulfill() + } else { + expectationFailed.fulfill() + } + } + + await Test { + func f(_ type: E.Type) throws -> E where E: Error { + try #require(throws: type) {} + } + try f(Never.self) + }.run(configuration: configuration) + + await fulfillment(of: [errorCaught, apiMisused, expectationFailed], timeout: 0.0) + } + + @_semantics("testing.macros.nowarnings") + func testErrorCheckingWithRequire_ResultValueIsNever_VariousSyntaxes() throws { + // Basic expressions succeed and don't diagnose. + #expect(throws: Never.self) {} + try #require(throws: Never.self) {} + + // Casting to specific types succeeds and doesn't diagnose. + let _: Void = try #require(throws: Never.self) {} + let _: Any = try #require(throws: Never.self) {} + + // Casting to any Error throws an API misuse error because Never cannot be + // instantiated. NOTE: inner function needed for lexical context. + @_semantics("testing.macros.nowarnings") + func castToAnyError() throws { + let _: any Error = try #require(throws: Never.self) {} + } + #expect(throws: APIMisuseError.self, performing: castToAnyError) + } + func testFail() async throws { var configuration = Configuration() configuration.eventHandler = { event, _ in diff --git a/Tests/TestingTests/Traits/TagListTests.swift b/Tests/TestingTests/Traits/TagListTests.swift index 29b8e3909..1ec8d1248 100644 --- a/Tests/TestingTests/Traits/TagListTests.swift +++ b/Tests/TestingTests/Traits/TagListTests.swift @@ -171,7 +171,7 @@ struct TagListTests { func noTagColorsReadFromBadPath(tagColorJSON: String) throws { var tagColorJSON = tagColorJSON tagColorJSON.withUTF8 { tagColorJSON in - #expect(throws: (any Error).self) { + _ = #expect(throws: (any Error).self) { _ = try JSON.decode(Tag.Color.self, from: .init(tagColorJSON)) } } From e2ec0411e5f7407fc2d325c9feea8f0ac10a60e2 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 13 Dec 2024 17:27:31 -0500 Subject: [PATCH 021/234] Change default behavior of exit tests to always succeed. (#858) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit tests simulate calling `main()` (or some similar thing), but if you don't explicitly exit from them, they force a failure. After much mulling and some discussion with colleagues a while back, we should change the behavior so that, if a test doesn't otherwise terminate, it acts as if `main()` returned naturally—that is, by exiting with `EXIT_SUCCESS` rather than by forcing `EXIT_FAILURE`. This behavior is more consistent with the feature's theory of operation. For example: ```swift await #expect(exitsWith: .success) { assert(2 > 1) // this is true and doesn't assert, so this test should pass, right? } ``` Exit tests remain an experimental feature. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 8 +++----- Tests/TestingTests/ExitTestTests.swift | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 39a0ea550..edb01c4dc 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -158,11 +158,9 @@ extension ExitTest { _errorInMain(error) } - // Run some glue code that terminates the process with an exit condition - // that does not match the expected one. If the exit test's body doesn't - // terminate, we'll manually call exit() and cause the test to fail. - let expectingFailure = expectedExitCondition == .failure - exit(expectingFailure ? EXIT_SUCCESS : EXIT_FAILURE) + // If we get to this point without terminating, then we simulate main()'s + // behavior which is to exit with EXIT_SUCCESS. + exit(EXIT_SUCCESS) } } diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 73fcfbb30..c82c4f73f 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -22,6 +22,7 @@ private import _TestingInternals exit(EXIT_FAILURE + 1) } } + await #expect(exitsWith: .success) {} await #expect(exitsWith: .success) { exit(EXIT_SUCCESS) } @@ -63,7 +64,7 @@ private import _TestingInternals } @Test("Exit tests (failing)") func failing() async { - await confirmation("Exit tests failed", expectedCount: 10) { failed in + await confirmation("Exit tests failed", expectedCount: 9) { failed in var configuration = Configuration() configuration.eventHandler = { event, _ in if case .issueRecorded = event.kind { @@ -410,7 +411,6 @@ private import _TestingInternals @Suite(.hidden) struct FailingExitTests { @Test(.hidden) func failingExitTests() async { - await #expect(exitsWith: .success) {} await #expect(exitsWith: .failure) {} await #expect(exitsWith: .exitCode(123)) {} await #expect(exitsWith: .failure) { From c61e5ada29a44d5e34b7e8dfb258a9bb1e71609c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 17 Dec 2024 17:22:44 -0500 Subject: [PATCH 022/234] Remove an import of UniformTypeIdentifiers from the Foundation overlay. (#864) An earlier iteration of this code needed to talk to the UniformTypeIdentifiers framework on Apple platforms. This is no longer the case. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../_Testing_Foundation/Attachments/Attachment+URL.swift | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index 9bfa027d9..9d0a0a5f1 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -12,10 +12,6 @@ @_spi(Experimental) @_spi(ForSwiftTestingOnly) public import Testing public import Foundation -#if SWT_TARGET_OS_APPLE && canImport(UniformTypeIdentifiers) -private import UniformTypeIdentifiers -#endif - #if !SWT_NO_PROCESS_SPAWNING && os(Windows) private import WinSDK #endif From 014d874e5a410868737e17d0cfe0fcf76d4eca1a Mon Sep 17 00:00:00 2001 From: michael-yuji Date: Wed, 18 Dec 2024 14:24:27 -0800 Subject: [PATCH 023/234] Additional FreeBSD fixes (#865) Add FreeBSD support for swift-testing. ### Result: With this PR, swift-testing can be built on FreeBSD (tested on x86_64 FreeBSD 14.1-RELEASE-p6) ### Known issue: There are some issue running `swift test` on this repo, but most likely due to bugs in my host toolchain. - Tests failed to link due to duplicated main symbols: ``` error: link command failed with exit code 1 (use -v to see invocation) ld: error: duplicate symbol: main >>> defined at TestingMacrosMain.swift:0 (/zdata/swift-main/swift-project/swift-testing/Sources/TestingMacros/TestingMacrosMain.swift:0) >>> /zdata/swift-main/swift-project/swift-testing/.build/x86_64-unknown-freebsd14.1/debug/TestingMacros-tool.build/TestingMacrosMain.swift.o:(main) >>> defined at runner.swift:0 (/zdata/swift-main/swift-project/swift-testing/.build/x86_64-unknown-freebsd14.1/debug/swift-testingPackageTests.derived/runner.swift:0) >>> /zdata/swift-main/swift-project/swift-testing/.build/x86_64-unknown-freebsd14.1/debug/swift_testingPackageTests-tool.build/runner.swift.o:(.text.main+0x0) clang: error: linker command failed with exit code 1 (use -v to see invocation) [25/26] Linking swift-testingPackageTests.xctest ``` - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 2 +- Sources/Testing/ExitTests/SpawnProcess.swift | 9 ++++++++- Sources/Testing/ExitTests/WaitFor.swift | 6 +++++- Sources/Testing/Support/Locked.swift | 18 ++++++++++-------- Sources/_TestingInternals/include/Includes.h | 8 ++++++++ 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index edb01c4dc..5c3179ae5 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -114,7 +114,7 @@ extension ExitTest { // As with Linux, disable the generation core files. FreeBSD does not, as // far as I can tell, special-case RLIMIT_CORE=1. var rl = rlimit(rlim_cur: 0, rlim_max: 0) - _ = setrlimit(CInt(RLIMIT_CORE.rawValue), &rl) + _ = setrlimit(RLIMIT_CORE, &rl) #elseif os(Windows) // On Windows, similarly disable Windows Error Reporting and the Windows // Error Reporting UI. Note we expect to be the first component to call diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 2d13b1955..91a2b48a3 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -143,12 +143,19 @@ func spawnExecutable( #if SWT_TARGET_OS_APPLE // Close all other file descriptors open in the parent. flags |= CShort(POSIX_SPAWN_CLOEXEC_DEFAULT) -#elseif os(Linux) || os(FreeBSD) +#elseif os(Linux) // This platform doesn't have POSIX_SPAWN_CLOEXEC_DEFAULT, but we can at // least close all file descriptors higher than the highest inherited one. // We are assuming here that the caller didn't set FD_CLOEXEC on any of // these file descriptors. _ = swt_posix_spawn_file_actions_addclosefrom_np(fileActions, highestFD + 1) +#elseif os(FreeBSD) + // Like Linux, this platfrom doesn't have POSIX_SPAWN_CLOEXEC_DEFAULT; + // However; unlike Linux, all non-EOL FreeBSD (>= 13.1) supports + // `posix_spawn_file_actions_addclosefrom_np` and therefore we don't need + // need `swt_posix_spawn_file_actions_addclosefrom_np` to guard the availability + // of this api. + _ = posix_spawn_file_actions_addclosefrom_np(fileActions, highestFD + 1) #else #warning("Platform-specific implementation missing: cannot close unused file descriptors") #endif diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index 521a092d3..287e26e4d 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -85,7 +85,11 @@ private let _childProcessContinuations = Locked<[pid_t: CheckedContinuation.allocate(capacity: 1) + #else let result = UnsafeMutablePointer.allocate(capacity: 1) + #endif _ = pthread_cond_init(result, nil) return result }() @@ -132,7 +136,7 @@ private let _createWaitThread: Void = { // Create the thread. It will run immediately; because it runs in an infinite // loop, we aren't worried about detaching or joining it. -#if SWT_TARGET_OS_APPLE +#if SWT_TARGET_OS_APPLE || os(FreeBSD) var thread: pthread_t? #else var thread = pthread_t() diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index df18319d0..61013cd33 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -36,20 +36,22 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { /// To keep the implementation of this type as simple as possible, /// `pthread_mutex_t` is used on Apple platforms instead of `os_unfair_lock` /// or `OSAllocatedUnfairLock`. -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) - private typealias _Lock = pthread_mutex_t +#if SWT_TARGET_OS_APPLE || os(Linux) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) + typealias PlatformLock = pthread_mutex_t +#elseif os(FreeBSD) + typealias PlatformLock = pthread_mutex_t? #elseif os(Windows) - private typealias _Lock = SRWLOCK + typealias PlatformLock = SRWLOCK #elseif os(WASI) // No locks on WASI without multithreaded runtime. - private typealias _Lock = Void + typealias PlatformLock = Void #else #warning("Platform-specific implementation missing: locking unavailable") - private typealias _Lock = Void + typealias PlatformLock = Void #endif /// A type providing heap-allocated storage for an instance of ``Locked``. - private final class _Storage: ManagedBuffer { + private final class _Storage: ManagedBuffer { deinit { withUnsafeMutablePointerToElements { lock in #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) @@ -66,7 +68,7 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { } /// Storage for the underlying lock and wrapped value. - private nonisolated(unsafe) var _storage: ManagedBuffer + private nonisolated(unsafe) var _storage: ManagedBuffer init(rawValue: T) { _storage = _Storage.create(minimumCapacity: 1, makingHeaderWith: { _ in rawValue }) @@ -142,7 +144,7 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { /// - Warning: Callers that unlock the lock _must_ lock it again before the /// closure returns. If the lock is not acquired when `body` returns, the /// effect is undefined. - nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { + nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { try withLock { value in try _storage.withUnsafeMutablePointerToElements { lock in try body(lock, value) diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index 4a621d5c2..51c02e277 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -80,6 +80,10 @@ #include #endif +#if __has_include() +#include +#endif + #if __has_include() #include #endif @@ -125,6 +129,10 @@ #endif #endif +#if defined(__FreeBSD__) +#include +#endif + #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #define NOMINMAX From 591dc82516477deb1584e79226f4c1e4aa55f238 Mon Sep 17 00:00:00 2001 From: michael-yuji Date: Wed, 18 Dec 2024 16:35:36 -0800 Subject: [PATCH 024/234] Fix more things on FreeBSD (#867) This PR contains a few more fixes for FreeBSD: Use threadsafe `strerror_r` on FreeBSD Define "SWT_NO_SNAPSHOT_TYPES" when host is FreeBSD Work around issue https://github.com/swiftlang/swift/issues/62985 ### Result: Passed all the tests on FreeBSD ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 11 +++++++---- Sources/Testing/SourceAttribution/Backtrace.swift | 2 +- Sources/Testing/Support/CError.swift | 6 ++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Package.swift b/Package.swift index 4d9112c72..266ea2a7d 100644 --- a/Package.swift +++ b/Package.swift @@ -45,7 +45,10 @@ let package = Package( ], exclude: ["CMakeLists.txt"], cxxSettings: .packageSettings, - swiftSettings: .packageSettings + swiftSettings: .packageSettings, + linkerSettings: [ + .linkedLibrary("execinfo", .when(platforms: [.custom("freebsd")])) + ] ), .testTarget( name: "TestingTests", @@ -114,7 +117,7 @@ let package = Package( ) // BUG: swift-package-manager-#6367 -#if !os(Windows) +#if !os(Windows) && !os(FreeBSD) package.targets.append(contentsOf: [ .testTarget( name: "TestingMacrosTests", @@ -143,7 +146,7 @@ extension Array where Element == PackageDescription.SwiftSetting { .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .windows, .wasi])), + .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .windows, .wasi])), .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] @@ -178,7 +181,7 @@ extension Array where Element == PackageDescription.CXXSetting { result += [ .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .windows, .wasi])), + .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .windows, .wasi])), .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] diff --git a/Sources/Testing/SourceAttribution/Backtrace.swift b/Sources/Testing/SourceAttribution/Backtrace.swift index a17aed21d..b8301c0a4 100644 --- a/Sources/Testing/SourceAttribution/Backtrace.swift +++ b/Sources/Testing/SourceAttribution/Backtrace.swift @@ -181,7 +181,7 @@ extension Backtrace { /// crash. To avoid said crash, we'll keep a strong reference to the /// object (abandoning memory until the process exits.) /// ([swift-#62985](https://github.com/swiftlang/swift/issues/62985)) -#if os(Windows) +#if os(Windows) || os(FreeBSD) nonisolated(unsafe) var errorObject: AnyObject? #else nonisolated(unsafe) weak var errorObject: AnyObject? diff --git a/Sources/Testing/Support/CError.swift b/Sources/Testing/Support/CError.swift index 5dfdeac6a..47b9d6612 100644 --- a/Sources/Testing/Support/CError.swift +++ b/Sources/Testing/Support/CError.swift @@ -47,6 +47,12 @@ func strerror(_ errorCode: CInt) -> String { _ = strerror_s(buffer.baseAddress!, buffer.count, errorCode) return strnlen(buffer.baseAddress!, buffer.count) } +#elseif os(FreeBSD) + // FreeBSD's implementation of strerror() is not thread-safe. + String(unsafeUninitializedCapacity: 1024) { buffer in + _ = strerror_r(errorCode, buffer.baseAddress!, buffer.count) + return strnlen(buffer.baseAddress!, buffer.count) + } #else String(cString: _TestingInternals.strerror(errorCode)) #endif From de25f64f5c2e04624264594dfb8b77e18c3a7311 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 19 Dec 2024 17:59:58 -0500 Subject: [PATCH 025/234] Fix some typos in the new FreeBSD-specific code. (#871) Fixes some typos/code style bits and renames `withUnsafeUnderlyingLock()` to `withUnsafePlatformLock()` to make it more consistent with the newly-exposed `PlatformLock` typealias. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/SpawnProcess.swift | 10 +++++----- Sources/Testing/ExitTests/WaitFor.swift | 8 ++++---- Sources/Testing/Support/Locked.swift | 4 +--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 91a2b48a3..204fd9bbe 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -150,11 +150,11 @@ func spawnExecutable( // these file descriptors. _ = swt_posix_spawn_file_actions_addclosefrom_np(fileActions, highestFD + 1) #elseif os(FreeBSD) - // Like Linux, this platfrom doesn't have POSIX_SPAWN_CLOEXEC_DEFAULT; - // However; unlike Linux, all non-EOL FreeBSD (>= 13.1) supports - // `posix_spawn_file_actions_addclosefrom_np` and therefore we don't need - // need `swt_posix_spawn_file_actions_addclosefrom_np` to guard the availability - // of this api. + // Like Linux, this platform doesn't have POSIX_SPAWN_CLOEXEC_DEFAULT. + // Unlike Linux, all non-EOL FreeBSD versions (≥13.1) support + // `posix_spawn_file_actions_addclosefrom_np`. Therefore, we don't need + // `swt_posix_spawn_file_actions_addclosefrom_np` to guard the + // availability of this function. _ = posix_spawn_file_actions_addclosefrom_np(fileActions, highestFD + 1) #else #warning("Platform-specific implementation missing: cannot close unused file descriptors") diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index 287e26e4d..668fe8dcb 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -85,11 +85,11 @@ private let _childProcessContinuations = Locked<[pid_t: CheckedContinuation.allocate(capacity: 1) - #else +#else let result = UnsafeMutablePointer.allocate(capacity: 1) - #endif +#endif _ = pthread_cond_init(result, nil) return result }() @@ -126,7 +126,7 @@ private let _createWaitThread: Void = { // newly-scheduled waiter process. (If this condition is spuriously // woken, we'll just loop again, which is fine.) Note that we read errno // outside the lock in case acquiring the lock perturbs it. - _childProcessContinuations.withUnsafeUnderlyingLock { lock, childProcessContinuations in + _childProcessContinuations.withUnsafePlatformLock { lock, childProcessContinuations in if childProcessContinuations.isEmpty { _ = pthread_cond_wait(_waitThreadNoChildrenCondition, lock) } diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index 61013cd33..11c4c4c86 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -123,7 +123,6 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { } } -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) /// Acquire the lock and invoke a function while it is held, yielding both the /// protected value and a reference to the lock itself. /// @@ -144,14 +143,13 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { /// - Warning: Callers that unlock the lock _must_ lock it again before the /// closure returns. If the lock is not acquired when `body` returns, the /// effect is undefined. - nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { + nonmutating func withUnsafePlatformLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { try withLock { value in try _storage.withUnsafeMutablePointerToElements { lock in try body(lock, value) } } } -#endif } extension Locked where T: AdditiveArithmetic { From 0e9bcba2f90ca3b04c14d5909362bd1b3d3146af Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 20 Dec 2024 12:17:46 -0500 Subject: [PATCH 026/234] Add `@_spi` annotations to reexport decls in cross-import overlays. (#870) This PR attempts to make sure that SPI is reexported from our cross-import overlays. This change is speculative and I'm not actually sure if it'll do what I want. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift | 2 +- Sources/Overlays/_Testing_Foundation/ReexportTesting.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift b/Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift index 3faa622d7..5b28faa77 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/ReexportTesting.swift @@ -8,4 +8,4 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@_exported public import Testing +@_exported @_spi(Experimental) @_spi(ForToolsIntegrationOnly) public import Testing diff --git a/Sources/Overlays/_Testing_Foundation/ReexportTesting.swift b/Sources/Overlays/_Testing_Foundation/ReexportTesting.swift index 3faa622d7..5b28faa77 100644 --- a/Sources/Overlays/_Testing_Foundation/ReexportTesting.swift +++ b/Sources/Overlays/_Testing_Foundation/ReexportTesting.swift @@ -8,4 +8,4 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@_exported public import Testing +@_exported @_spi(Experimental) @_spi(ForToolsIntegrationOnly) public import Testing From aa96a7903a71f53a77c08ef3de1b7c2da0b3a1e0 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 20 Dec 2024 12:32:45 -0500 Subject: [PATCH 027/234] Remove the workaround for swift-package-manager-#8111. (#872) Remove the workaround for https://github.com/swiftlang/swift-package-manager/issues/8111. The issue has been resolved and Windows correctly exports symbols from dynamic library targets. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 1 + Tests/TestingTests/ABIEntryPointTests.swift | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 266ea2a7d..3cf7a1e12 100644 --- a/Package.swift +++ b/Package.swift @@ -28,6 +28,7 @@ let package = Package( products: [ .library( name: "Testing", + type: .dynamic, // needed so Windows exports ABI entry point symbols targets: ["Testing"] ), ], diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index db9440033..f259fc8cf 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -129,7 +129,6 @@ struct ABIEntryPointTests { passing arguments: __CommandLineArguments_v0, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void = { _ in } ) async throws -> Bool { -#if !(!SWT_FIXED_SWIFTPM_8111 && os(Windows)) #if !os(Linux) && !os(FreeBSD) && !os(Android) && !SWT_NO_DYNAMIC_LINKING // Get the ABI entry point by dynamically looking it up at runtime. // @@ -143,7 +142,6 @@ struct ABIEntryPointTests { } ) } -#endif #endif let abiEntryPoint = unsafeBitCast(abiv0_getEntryPoint(), to: ABIv0.EntryPoint.self) From 840c82289e2c0ecc98605c7fc9fda787053c95b8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 20 Dec 2024 17:56:00 -0500 Subject: [PATCH 028/234] Amend #872 to avoid loader failures when hosted in Xcode. --- Package.swift | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Package.swift b/Package.swift index 3cf7a1e12..13f517976 100644 --- a/Package.swift +++ b/Package.swift @@ -26,11 +26,20 @@ let package = Package( ], products: [ - .library( - name: "Testing", - type: .dynamic, // needed so Windows exports ABI entry point symbols - targets: ["Testing"] - ), + { +#if os(Windows) + .library( + name: "Testing", + type: .dynamic, // needed so Windows exports ABI entry point symbols + targets: ["Testing"] + ) +#else + .library( + name: "Testing", + targets: ["Testing"] + ) +#endif + }() ], dependencies: [ From 5b4d6d6f7d4e0dbca4dd6593e0c8862022388d7c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 20 Dec 2024 20:01:00 -0500 Subject: [PATCH 029/234] Don't use `@_semantics`, use our own attribute instead. (#874) This PR replaces our use of `@_semantics` with a custom attribute macro. `@_semantics` is reserved for use by the standard library and runtime and, while useful, can be emulated on our side of the module barrier without much difficulty. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --------- Co-authored-by: Stuart Montgomery --- Sources/Testing/Test+Macro.swift | 21 +++++ Sources/TestingMacros/CMakeLists.txt | 1 + Sources/TestingMacros/PragmaMacro.swift | 79 +++++++++++++++++++ .../MacroExpansionContextAdditions.swift | 24 ++---- Sources/TestingMacros/TestingMacrosMain.swift | 1 + .../TestingMacrosTests/PragmaMacroTests.swift | 29 +++++++ Tests/TestingTests/IssueTests.swift | 4 +- 7 files changed, 141 insertions(+), 18 deletions(-) create mode 100644 Sources/TestingMacros/PragmaMacro.swift create mode 100644 Tests/TestingMacrosTests/PragmaMacroTests.swift diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index 6f8536ac1..0fb29562d 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -484,6 +484,27 @@ extension Test { } } +// MARK: - Test pragmas + +/// A macro used similarly to `#pragma` in C or `@_semantics` in the standard +/// library. +/// +/// - Parameters: +/// - arguments: Zero or more context-specific arguments. +/// +/// The use cases for this macro are subject to change over time as the needs of +/// the testing library change. The implementation of this macro in the +/// TestingMacros target determines how different arguments are handled. +/// +/// - Note: This macro has compile-time effects _only_ and should not affect a +/// compiled test target. +/// +/// - Warning: This macro is used to implement other macros declared by the testing +/// library. Do not use it directly. +@attached(peer) public macro __testing( + semantics arguments: _const String... +) = #externalMacro(module: "TestingMacros", type: "PragmaMacro") + // MARK: - Helper functions /// A function that abstracts away whether or not the `try` keyword is needed on diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index ad58fc35b..acf09f339 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -81,6 +81,7 @@ endif() target_sources(TestingMacros PRIVATE ConditionMacro.swift + PragmaMacro.swift SourceLocationMacro.swift SuiteDeclarationMacro.swift Support/Additions/DeclGroupSyntaxAdditions.swift diff --git a/Sources/TestingMacros/PragmaMacro.swift b/Sources/TestingMacros/PragmaMacro.swift new file mode 100644 index 000000000..48027b213 --- /dev/null +++ b/Sources/TestingMacros/PragmaMacro.swift @@ -0,0 +1,79 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +public import SwiftSyntax +public import SwiftSyntaxMacros + +/// A type describing the expansion of the `@__testing` attribute macro. +/// +/// Supported uses: +/// +/// - `@__testing(semantics: "nomacrowarnings")`: suppress warning diagnostics +/// generated by macros. (The implementation of this use case is held in trust +/// at ``MacroExpansionContext/areWarningsSuppressed``. +/// +/// This type is used to implement the `@__testing` attribute macro. Do not use +/// it directly. +public struct PragmaMacro: PeerMacro, Sendable { + public static func expansion( + of node: AttributeSyntax, + providingPeersOf declaration: some DeclSyntaxProtocol, + in context: some MacroExpansionContext + ) throws -> [DeclSyntax] { + return [] + } + + public static var formatMode: FormatMode { + .disabled + } +} + +/// Get all pragma attributes (`@__testing`) associated with a syntax node. +/// +/// - Parameters: +/// - node: The syntax node to inspect. +/// +/// - Returns: The set of pragma attributes strings associated with `node`. +/// +/// Attributes conditionally applied with `#if` are ignored. +func pragmas(on node: some WithAttributesSyntax) -> [AttributeSyntax] { + node.attributes + .compactMap { attribute in + if case let .attribute(attribute) = attribute { + return attribute + } + return nil + }.filter { attribute in + attribute.attributeName.isNamed("__testing", inModuleNamed: "Testing") + } +} + +/// Get all "semantics" attributed to a syntax node using the +/// `@__testing(semantics:)` attribute. +/// +/// - Parameters: +/// - node: The syntax node to inspect. +/// +/// - Returns: The set of "semantics" strings associated with `node`. +/// +/// Attributes conditionally applied with `#if` are ignored. +func semantics(of node: some WithAttributesSyntax) -> [String] { + pragmas(on: node) + .compactMap { attribute in + if case let .argumentList(arguments) = attribute.arguments { + return arguments + } + return nil + }.filter { arguments in + arguments.first?.label?.textWithoutBackticks == "semantics" + }.flatMap { argument in + argument.compactMap { $0.expression.as(StringLiteralExprSyntax.self)?.representedLiteralValue } + } +} diff --git a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift index ca0137b5d..8bcf2522a 100644 --- a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift @@ -84,28 +84,20 @@ extension MacroExpansionContext { /// lexical context. /// /// The value of this property is `true` if the current lexical context - /// contains a node with the `@_semantics("testing.macros.nowarnings")` - /// attribute applied to it. + /// contains a node with the `@__testing(semantics: "nowarnings")` attribute + /// applied to it. /// /// - Warning: This functionality is not part of the public interface of the /// testing library. It may be modified or removed in a future update. var areWarningsSuppressed: Bool { #if DEBUG - for lexicalContext in self.lexicalContext { - guard let lexicalContext = lexicalContext.asProtocol((any WithAttributesSyntax).self) else { - continue - } - for attribute in lexicalContext.attributes { - if case let .attribute(attribute) = attribute, - attribute.attributeNameText == "_semantics", - case let .string(argument) = attribute.arguments, - argument.representedLiteralValue == "testing.macros.nowarnings" { - return true - } - } - } -#endif + return lexicalContext + .compactMap { $0.asProtocol((any WithAttributesSyntax).self) } + .flatMap { semantics(of: $0) } + .contains("nomacrowarnings") +#else return false +#endif } /// Emit a diagnostic message. diff --git a/Sources/TestingMacros/TestingMacrosMain.swift b/Sources/TestingMacros/TestingMacrosMain.swift index c6904a6e7..1894f4282 100644 --- a/Sources/TestingMacros/TestingMacrosMain.swift +++ b/Sources/TestingMacros/TestingMacrosMain.swift @@ -30,6 +30,7 @@ struct TestingMacrosMain: CompilerPlugin { ExitTestRequireMacro.self, TagMacro.self, SourceLocationMacro.self, + PragmaMacro.self, ] } } diff --git a/Tests/TestingMacrosTests/PragmaMacroTests.swift b/Tests/TestingMacrosTests/PragmaMacroTests.swift new file mode 100644 index 000000000..9e85419da --- /dev/null +++ b/Tests/TestingMacrosTests/PragmaMacroTests.swift @@ -0,0 +1,29 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +import Testing +@testable import TestingMacros + +import SwiftParser +import SwiftSyntax + +@Suite("PragmaMacro Tests") +struct PragmaMacroTests { + @Test func findSemantics() throws { + let node = """ + @Testing.__testing(semantics: "abc123") + @__testing(semantics: "def456") + let x = 0 + """ as DeclSyntax + let nodeWithAttributes = try #require(node.asProtocol((any WithAttributesSyntax).self)) + let semantics = semantics(of: nodeWithAttributes) + #expect(semantics == ["abc123", "def456"]) + } +} diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 631ff0c54..a73f0706d 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -992,7 +992,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, apiMisused, expectationFailed], timeout: 0.0) } - @_semantics("testing.macros.nowarnings") + @__testing(semantics: "nomacrowarnings") func testErrorCheckingWithRequire_ResultValueIsNever_VariousSyntaxes() throws { // Basic expressions succeed and don't diagnose. #expect(throws: Never.self) {} @@ -1004,7 +1004,7 @@ final class IssueTests: XCTestCase { // Casting to any Error throws an API misuse error because Never cannot be // instantiated. NOTE: inner function needed for lexical context. - @_semantics("testing.macros.nowarnings") + @__testing(semantics: "nomacrowarnings") func castToAnyError() throws { let _: any Error = try #require(throws: Never.self) {} } From 878df301de557e6845d31a947a5679f5e58ecd8d Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 2 Jan 2025 15:56:46 -0500 Subject: [PATCH 030/234] Mark `#expect(_:throws:)` and `#require(_:throws:)` as to-be-deprecated. (#875) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These symbols should go through a transition period where they are documented as to-be-deprecated rather than being deprecated outright. My mistake. With this change applied, Xcode treats these macros as deprecated (deemphasizing them in autocomplete, for instance) and DocC marks them as deprecated in an unspecified Swift version, but no warning is emitted if they are used. This is consistent with API marked `API_TO_BE_DEPRECATED` or `deprecated: 100000.0` in Apple's SDKs. > [!NOTE] > There is a bug in the DocC compiler that emits a diagnostic of the form: > > > ⚠️ 'require(_:sourceLocation:performing:throws:)' isn't unconditionally deprecated > > This issue is being tracked already with rdar://141785948. Resolves #873. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Proposals/0006-return-errors-from-expect-throws.md | 4 ++-- Sources/Testing/Expectations/Expectation+Macro.swift | 4 ++-- .../Testing.docc/AvailabilityStubs/ExpectComplexThrows.md | 6 +++--- .../AvailabilityStubs/RequireComplexThrows.md | 6 +++--- .../Testing.docc/testing-for-errors-in-swift-code.md | 4 ++-- Tests/TestingTests/IssueTests.swift | 8 -------- 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/Documentation/Proposals/0006-return-errors-from-expect-throws.md b/Documentation/Proposals/0006-return-errors-from-expect-throws.md index 2f15c02ba..94f25bef8 100644 --- a/Documentation/Proposals/0006-return-errors-from-expect-throws.md +++ b/Documentation/Proposals/0006-return-errors-from-expect-throws.md @@ -105,7 +105,7 @@ is not statically available. The problematic overloads will also be deprecated: -) where E: Error & Equatable +) -> E where E: Error & Equatable -+@available(*, deprecated, message: "Examine the result of '#expect(throws:)' instead.") ++@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") +@discardableResult @freestanding(expression) public macro expect( _ comment: @autoclosure () -> Comment? = nil, @@ -115,7 +115,7 @@ is not statically available. The problematic overloads will also be deprecated: -) +) -> (any Error)? -+@available(*, deprecated, message: "Examine the result of '#require(throws:)' instead.") ++@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") +@discardableResult @freestanding(expression) public macro require( _ comment: @autoclosure () -> Comment? = nil, diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index c8a691e85..83bf045db 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -375,7 +375,7 @@ public macro require( /// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. If the thrown /// error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), /// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. -@available(*, deprecated, message: "Examine the result of '#expect(throws:)' instead.") +@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") @discardableResult @freestanding(expression) public macro expect( _ comment: @autoclosure () -> Comment? = nil, @@ -427,7 +427,7 @@ public macro require( /// /// If `expression` should _never_ throw, simply invoke the code without using /// this macro. The test will then fail if an error is thrown. -@available(*, deprecated, message: "Examine the result of '#require(throws:)' instead.") +@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") @discardableResult @freestanding(expression) public macro require( _ comment: @autoclosure () -> Comment? = nil, diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md index 755bf5089..3114002d2 100644 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md @@ -11,11 +11,11 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors --> @Metadata { - @Available(Swift, introduced: 6.0, deprecated: 999.0) - @Available(Xcode, introduced: 16.0, deprecated: 999.0) + @Available(Swift, introduced: 6.0) + @Available(Xcode, introduced: 16.0) } -@DeprecationSummary { +@DeprecationSummary { Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` or ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md index ff42dc40f..291ac0d32 100644 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md @@ -11,11 +11,11 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors --> @Metadata { - @Available(Swift, introduced: 6.0, deprecated: 999.0) - @Available(Xcode, introduced: 16.0, deprecated: 999.0) + @Available(Swift, introduced: 6.0) + @Available(Xcode, introduced: 16.0) } -@DeprecationSummary { +@DeprecationSummary { Examine the result of ``require(throws:_:sourceLocation:performing:)-7n34r`` or ``require(throws:_:sourceLocation:performing:)-4djuw`` instead: diff --git a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md index 2611f8835..7053d0fa5 100644 --- a/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md +++ b/Sources/Testing/Testing.docc/testing-for-errors-in-swift-code.md @@ -48,8 +48,8 @@ running your test if the code doesn't throw the expected error. To check that the code under test throws an error of any type, pass `(any Error).self` as the first argument to either -``expect(throws:_:sourceLocation:performing:)-1xr34`` or -``require(_:_:sourceLocation:)-5l63q``: +``expect(throws:_:sourceLocation:performing:)-1hfms`` or +``require(throws:_:sourceLocation:performing:)-7n34r``: ```swift @Test func cannotAddToppingToPizzaBeforeStartOfList() { diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index a73f0706d..4a4fda631 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -491,7 +491,6 @@ final class IssueTests: XCTestCase { }.run(configuration: .init()) } - @available(*, deprecated) func testErrorCheckingWithExpect() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.isInverted = true @@ -540,7 +539,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } - @available(*, deprecated) func testErrorCheckingWithExpect_Mismatching() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.expectedFulfillmentCount = 13 @@ -665,7 +663,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } - @available(*, deprecated) func testErrorCheckingWithExpectAsync() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.isInverted = true @@ -709,7 +706,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } - @available(*, deprecated) func testErrorCheckingWithExpectAsync_Mismatching() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.expectedFulfillmentCount = 13 @@ -826,7 +822,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } - @available(*, deprecated) func testErrorCheckingWithExpect_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") @@ -854,7 +849,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } - @available(*, deprecated) func testErrorCheckingWithExpectAsync_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") @@ -882,7 +876,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } - @available(*, deprecated) func testErrorCheckingWithRequire_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") @@ -911,7 +904,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [errorCaught, expectationFailed], timeout: 0.0) } - @available(*, deprecated) func testErrorCheckingWithRequireAsync_ThrowingFromErrorMatcher() async throws { let errorCaught = expectation(description: "Error matcher's error caught") let expectationFailed = expectation(description: "Expectation failed") From 042581d3ff7b71b4b581917bae48cbcd5281a660 Mon Sep 17 00:00:00 2001 From: "LamTrinh.Dev" Date: Fri, 3 Jan 2025 23:55:52 +0700 Subject: [PATCH 031/234] [docs] Correct the link of Proposal SWT-0006 (#877) ### Motivation: Currently, we the link of `SWT-0006` at `0006-return-errors-from-expect-throws.md` that will display `404 Not Found` . Correct the link of Proposal SWT-0006. ### Modifications: + Documentation/Proposals/0006-return-errors-from-expect-throws.md ### Result: Correct the link of Proposal SWT-0006 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). --- .../Proposals/0006-return-errors-from-expect-throws.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Proposals/0006-return-errors-from-expect-throws.md b/Documentation/Proposals/0006-return-errors-from-expect-throws.md index 94f25bef8..ab1735ff3 100644 --- a/Documentation/Proposals/0006-return-errors-from-expect-throws.md +++ b/Documentation/Proposals/0006-return-errors-from-expect-throws.md @@ -1,6 +1,6 @@ # Return errors from `#expect(throws:)` -* Proposal: [SWT-0006](0006-filename.md) +* Proposal: [SWT-0006](0006-return-errors-from-expect-throws.md) * Authors: [Jonathan Grynspan](https://github.com/grynspan) * Status: **Awaiting review** * Bug: rdar://138235250 From b85decc502b0fc699ad23ef76fd3be7701db8b36 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 4 Jan 2025 09:29:05 -0500 Subject: [PATCH 032/234] Remove the `introduced: 6.0` argument from the to-be-deprecated attributes on `#expect(performing:throws:)` and `#require(performing:throws:)`. (#881) --- .../Proposals/0006-return-errors-from-expect-throws.md | 4 ++-- Sources/Testing/Expectations/Expectation+Macro.swift | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/Proposals/0006-return-errors-from-expect-throws.md b/Documentation/Proposals/0006-return-errors-from-expect-throws.md index ab1735ff3..7eee79538 100644 --- a/Documentation/Proposals/0006-return-errors-from-expect-throws.md +++ b/Documentation/Proposals/0006-return-errors-from-expect-throws.md @@ -105,7 +105,7 @@ is not statically available. The problematic overloads will also be deprecated: -) where E: Error & Equatable +) -> E where E: Error & Equatable -+@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") ++@available(swift, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") +@discardableResult @freestanding(expression) public macro expect( _ comment: @autoclosure () -> Comment? = nil, @@ -115,7 +115,7 @@ is not statically available. The problematic overloads will also be deprecated: -) +) -> (any Error)? -+@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") ++@available(swift, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") +@discardableResult @freestanding(expression) public macro require( _ comment: @autoclosure () -> Comment? = nil, diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 83bf045db..7df075475 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -375,7 +375,7 @@ public macro require( /// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. If the thrown /// error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), /// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. -@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") +@available(swift, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") @discardableResult @freestanding(expression) public macro expect( _ comment: @autoclosure () -> Comment? = nil, @@ -427,7 +427,7 @@ public macro require( /// /// If `expression` should _never_ throw, simply invoke the code without using /// this macro. The test will then fail if an error is thrown. -@available(swift, introduced: 6.0, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") +@available(swift, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") @discardableResult @freestanding(expression) public macro require( _ comment: @autoclosure () -> Comment? = nil, From 33e4d1baea06d3b074d120aa8a743e9acc33c20c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 6 Jan 2025 15:00:01 -0500 Subject: [PATCH 033/234] Add speculative support for OpenBSD. (#890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenBSD has partial support in the Swift toolchain. This PR adds speculative support for it based on the assumption that what works with FreeBSD will generally also work with OpenBSD. The big differences: 1. We need to include `` instead of ``, 2. `/usr/bin/tar` does not support writing PKZIP files (so we look for the optional `/usr/bin/zip` instead), and 3. OpenBSD has no way to determine the path to the current executable, so we naïvely assume `argv[0]` is correct. These changes are speculative and have not been tested. Partially resolves #888. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/Porting.md | 2 +- Package.swift | 8 ++++---- .../Attachments/Attachment+URL.swift | 15 ++++++++++++++- .../Testing/ABI/EntryPoints/EntryPoint.swift | 6 +++--- .../ABI/EntryPoints/SwiftPMEntryPoint.swift | 2 +- .../Events/Recorder/Event.Symbol.swift | 2 +- Sources/Testing/ExitTests/ExitCondition.swift | 10 ++++++---- Sources/Testing/ExitTests/ExitTest.swift | 18 +++++++++++++----- Sources/Testing/ExitTests/SpawnProcess.swift | 19 +++++++++++-------- Sources/Testing/ExitTests/WaitFor.swift | 14 ++++++++------ .../Backtrace+Symbolication.swift | 2 +- .../Testing/SourceAttribution/Backtrace.swift | 4 ++-- .../Additions/CommandLineAdditions.swift | 10 +++++++++- Sources/Testing/Support/CError.swift | 4 ++-- Sources/Testing/Support/Environment.swift | 6 +++--- Sources/Testing/Support/FileHandle.swift | 12 ++++++------ Sources/Testing/Support/GetSymbol.swift | 6 +++--- Sources/Testing/Support/Locked.swift | 8 ++++---- Sources/Testing/Support/Versions.swift | 2 +- .../Traits/Tags/Tag.Color+Loading.swift | 4 ++-- Sources/_TestingInternals/Discovery.cpp | 2 +- Sources/_TestingInternals/include/Includes.h | 4 ++++ Sources/_TestingInternals/include/Stubs.h | 2 +- .../Support/EnvironmentTests.swift | 2 +- .../Support/FileHandleTests.swift | 4 ++-- 25 files changed, 104 insertions(+), 64 deletions(-) diff --git a/Documentation/Porting.md b/Documentation/Porting.md index 90c49bded..39d1d8e88 100644 --- a/Documentation/Porting.md +++ b/Documentation/Porting.md @@ -256,7 +256,7 @@ platform conditional and provide a stub implementation: +++ b/Sources/Testing/Support/FileHandle.swift var isTTY: Bool { - #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) + #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) // ... +#elseif os(Classic) + return false diff --git a/Package.swift b/Package.swift index 13f517976..a72f7086a 100644 --- a/Package.swift +++ b/Package.swift @@ -57,7 +57,7 @@ let package = Package( cxxSettings: .packageSettings, swiftSettings: .packageSettings, linkerSettings: [ - .linkedLibrary("execinfo", .when(platforms: [.custom("freebsd")])) + .linkedLibrary("execinfo", .when(platforms: [.custom("freebsd"), .openbsd])) ] ), .testTarget( @@ -127,7 +127,7 @@ let package = Package( ) // BUG: swift-package-manager-#6367 -#if !os(Windows) && !os(FreeBSD) +#if !os(Windows) && !os(FreeBSD) && !os(OpenBSD) package.targets.append(contentsOf: [ .testTarget( name: "TestingMacrosTests", @@ -156,7 +156,7 @@ extension Array where Element == PackageDescription.SwiftSetting { .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .windows, .wasi])), + .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi])), .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] @@ -191,7 +191,7 @@ extension Array where Element == PackageDescription.CXXSetting { result += [ .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .windows, .wasi])), + .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi])), .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index 9d0a0a5f1..305bdd35b 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -164,10 +164,23 @@ private func _compressContentsOfDirectory(at directoryURL: URL) async throws -> // // On Linux (which does not have FreeBSD's version of tar(1)), we can use // zip(1) instead. + // + // OpenBSD's tar(1) does not support writing PKZIP archives, and /usr/bin/zip + // tool is an optional install, so we check if it's present before trying to + // execute it. #if os(Linux) let archiverPath = "/usr/bin/zip" #elseif SWT_TARGET_OS_APPLE || os(FreeBSD) let archiverPath = "/usr/bin/tar" +#elseif os(OpenBSD) + let archiverPath = "/usr/local/bin/zip" + var isDirectory = false + if !FileManager.default.fileExists(atPath: archiverPath, isDirectory: &isDirectory) || isDirectory { + throw CocoaError(.fileNoSuchFile, userInfo: [ + NSLocalizedDescriptionKey: "The 'zip' package is not installed.", + NSFilePathErrorKey: archiverPath + ]) + } #elseif os(Windows) guard let archiverPath = _archiverPath else { throw CocoaError(.fileWriteUnknown, userInfo: [ @@ -187,7 +200,7 @@ private func _compressContentsOfDirectory(at directoryURL: URL) async throws -> let sourcePath = directoryURL.fileSystemPath let destinationPath = temporaryURL.fileSystemPath -#if os(Linux) +#if os(Linux) || os(OpenBSD) // The zip command constructs relative paths from the current working // directory rather than from command-line arguments. process.arguments = [destinationPath, "--recurse-paths", "."] diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index ea37326d5..a0a5df2a0 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -656,7 +656,7 @@ extension Event.ConsoleOutputRecorder.Options { /// Whether or not the system terminal claims to support 16-color ANSI escape /// codes. private static var _terminalSupports16ColorANSIEscapeCodes: Bool { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) if let termVariable = Environment.variable(named: "TERM") { return termVariable != "dumb" } @@ -678,7 +678,7 @@ extension Event.ConsoleOutputRecorder.Options { /// Whether or not the system terminal claims to support 256-color ANSI escape /// codes. private static var _terminalSupports256ColorANSIEscapeCodes: Bool { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) if let termVariable = Environment.variable(named: "TERM") { return strstr(termVariable, "256") != nil } @@ -700,7 +700,7 @@ extension Event.ConsoleOutputRecorder.Options { /// Whether or not the system terminal claims to support true-color ANSI /// escape codes. private static var _terminalSupportsTrueColorANSIEscapeCodes: Bool { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) if let colortermVariable = Environment.variable(named: "COLORTERM") { return strstr(colortermVariable, "truecolor") != nil } diff --git a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift index 52e4d2a54..3c72e9f20 100644 --- a/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/SwiftPMEntryPoint.swift @@ -24,7 +24,7 @@ private import _TestingInternals /// /// This constant is not part of the public interface of the testing library. var EXIT_NO_TESTS_FOUND: CInt { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) EX_UNAVAILABLE #elseif os(Windows) CInt(ERROR_NOT_FOUND) diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index aed2bbc3c..0f50ed95c 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -106,7 +106,7 @@ extension Event.Symbol { /// be used to represent it in text-based output. The value of this property /// is platform-dependent. public var unicodeCharacter: Character { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) switch self { case .default: // Unicode: WHITE DIAMOND diff --git a/Sources/Testing/ExitTests/ExitCondition.swift b/Sources/Testing/ExitTests/ExitCondition.swift index d589b5367..19f884303 100644 --- a/Sources/Testing/ExitTests/ExitCondition.swift +++ b/Sources/Testing/ExitTests/ExitCondition.swift @@ -49,12 +49,13 @@ public enum ExitCondition: Sendable { /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | /// - /// On macOS, FreeBSD, and Windows, the full exit code reported by the process - /// is yielded to the parent process. Linux and other POSIX-like systems may - /// only reliably report the low unsigned 8 bits (0–255) of the exit - /// code. + /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by + /// the process is yielded to the parent process. Linux and other POSIX-like + /// systems may only reliably report the low unsigned 8 bits (0–255) of + /// the exit code. case exitCode(_ exitCode: CInt) /// The process terminated with the given signal. @@ -70,6 +71,7 @@ public enum ExitCondition: Sendable { /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | case signal(_ signal: CInt) } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 5c3179ae5..af7981297 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -110,9 +110,9 @@ extension ExitTest { // SEE: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/coredump.c#n610 var rl = rlimit(rlim_cur: 1, rlim_max: 1) _ = setrlimit(CInt(RLIMIT_CORE.rawValue), &rl) -#elseif os(FreeBSD) - // As with Linux, disable the generation core files. FreeBSD does not, as - // far as I can tell, special-case RLIMIT_CORE=1. +#elseif os(FreeBSD) || os(OpenBSD) + // As with Linux, disable the generation core files. The BSDs do not, as far + // as I can tell, special-case RLIMIT_CORE=1. var rl = rlimit(rlim_cur: 0, rlim_max: 0) _ = setrlimit(RLIMIT_CORE, &rl) #elseif os(Windows) @@ -152,6 +152,14 @@ extension ExitTest { } #endif +#if os(OpenBSD) + // OpenBSD does not have posix_spawn_file_actions_addclosefrom_np(). + // However, it does have closefrom(2), which we call here as a best effort. + if let from = Environment.variable(named: "SWT_CLOSEFROM").flatMap(CInt.init) { + _ = closefrom(from) + } +#endif + do { try await body() } catch { @@ -344,7 +352,7 @@ extension ExitTest { } var fd: CInt? -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) fd = CInt(backChannelEnvironmentVariable) #elseif os(Windows) if let handle = UInt(backChannelEnvironmentVariable).flatMap(HANDLE.init(bitPattern:)) { @@ -541,7 +549,7 @@ extension ExitTest { // known environment variable to the corresponding file descriptor // (HANDLE on Windows.) var backChannelEnvironmentVariable: String? -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) backChannelEnvironmentVariable = backChannelWriteEnd.withUnsafePOSIXFileDescriptor { fd in fd.map(String.init(describing:)) } diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 204fd9bbe..fd18aad8a 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -17,7 +17,7 @@ internal import _TestingInternals /// A platform-specific value identifying a process running on the current /// system. -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) typealias ProcessID = pid_t #elseif os(Windows) typealias ProcessID = HANDLE @@ -62,13 +62,13 @@ func spawnExecutable( ) throws -> ProcessID { // Darwin and Linux differ in their optionality for the posix_spawn types we // use, so use this typealias to paper over the differences. -#if SWT_TARGET_OS_APPLE || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(FreeBSD) || os(OpenBSD) typealias P = T? #elseif os(Linux) typealias P = T #endif -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) return try withUnsafeTemporaryAllocation(of: P.self, capacity: 1) { fileActions in let fileActions = fileActions.baseAddress! guard 0 == posix_spawn_file_actions_init(fileActions) else { @@ -105,9 +105,7 @@ func spawnExecutable( } // Forward standard I/O streams and any explicitly added file handles. -#if os(Linux) || os(FreeBSD) - var highestFD = CInt(-1) -#endif + var highestFD = max(STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO) func inherit(_ fileHandle: borrowing FileHandle, as standardFD: CInt? = nil) throws { try fileHandle.withUnsafePOSIXFileDescriptor { fd in guard let fd else { @@ -118,9 +116,8 @@ func spawnExecutable( } else { #if SWT_TARGET_OS_APPLE _ = posix_spawn_file_actions_addinherit_np(fileActions, fd) -#elseif os(Linux) || os(FreeBSD) - highestFD = max(highestFD, fd) #endif + highestFD = max(highestFD, fd) } } } @@ -156,6 +153,12 @@ func spawnExecutable( // `swt_posix_spawn_file_actions_addclosefrom_np` to guard the // availability of this function. _ = posix_spawn_file_actions_addclosefrom_np(fileActions, highestFD + 1) +#elseif os(OpenBSD) + // OpenBSD does not have posix_spawn_file_actions_addclosefrom_np(). + // However, it does have closefrom(2), which we can call from within the + // spawned child process if we control its execution. + var environment = environment + environment["SWT_CLOSEFROM"] = String(describing: highestFD + 1) #else #warning("Platform-specific implementation missing: cannot close unused file descriptors") #endif diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index 668fe8dcb..239b4a4ba 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -11,7 +11,7 @@ #if !SWT_NO_PROCESS_SPAWNING internal import _TestingInternals -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) /// Block the calling thread, wait for the target process to exit, and return /// a value describing the conditions under which it exited. /// @@ -78,14 +78,14 @@ func wait(for pid: consuming pid_t) async throws -> ExitCondition { return try _blockAndWait(for: pid) } -#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) +#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) /// A mapping of awaited child PIDs to their corresponding Swift continuations. private let _childProcessContinuations = Locked<[pid_t: CheckedContinuation]>() /// A condition variable used to suspend the waiter thread created by /// `_createWaitThread()` when there are no child processes to await. private nonisolated(unsafe) let _waitThreadNoChildrenCondition = { -#if os(FreeBSD) +#if os(FreeBSD) || os(OpenBSD) let result = UnsafeMutablePointer.allocate(capacity: 1) #else let result = UnsafeMutablePointer.allocate(capacity: 1) @@ -136,7 +136,7 @@ private let _createWaitThread: Void = { // Create the thread. It will run immediately; because it runs in an infinite // loop, we aren't worried about detaching or joining it. -#if SWT_TARGET_OS_APPLE || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(FreeBSD) || os(OpenBSD) var thread: pthread_t? #else var thread = pthread_t() @@ -147,14 +147,16 @@ private let _createWaitThread: Void = { { _ in // Set the thread name to help with diagnostics. Note that different // platforms support different thread name lengths. See MAXTHREADNAMESIZE - // on Darwin, TASK_COMM_LEN on Linux, and MAXCOMLEN on FreeBSD. We try to - // maximize legibility in the available space. + // on Darwin, TASK_COMM_LEN on Linux, MAXCOMLEN on FreeBSD, and _MAXCOMLEN + // on OpenBSD. We try to maximize legibility in the available space. #if SWT_TARGET_OS_APPLE _ = pthread_setname_np("Swift Testing exit test monitor") #elseif os(Linux) _ = swt_pthread_setname_np(pthread_self(), "SWT ExT monitor") #elseif os(FreeBSD) _ = pthread_set_name_np(pthread_self(), "SWT ex test monitor") +#elseif os(OpenBSD) + _ = pthread_set_name_np(pthread_self(), "SWT exit test monitor") #else #warning("Platform-specific implementation missing: thread naming unavailable") #endif diff --git a/Sources/Testing/SourceAttribution/Backtrace+Symbolication.swift b/Sources/Testing/SourceAttribution/Backtrace+Symbolication.swift index 833b95231..89fdffce0 100644 --- a/Sources/Testing/SourceAttribution/Backtrace+Symbolication.swift +++ b/Sources/Testing/SourceAttribution/Backtrace+Symbolication.swift @@ -71,7 +71,7 @@ extension Backtrace { result[i] = SymbolicatedAddress(address: address, offset: offset, symbolName: symbolName) } } -#elseif os(Linux) || os(FreeBSD) || os(Android) +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) // Although these platforms have dladdr(), they do not have symbol names // from DWARF binaries by default, only from shared libraries. The standard // library's backtracing functionality has implemented sufficient ELF/DWARF diff --git a/Sources/Testing/SourceAttribution/Backtrace.swift b/Sources/Testing/SourceAttribution/Backtrace.swift index b8301c0a4..a6019860c 100644 --- a/Sources/Testing/SourceAttribution/Backtrace.swift +++ b/Sources/Testing/SourceAttribution/Backtrace.swift @@ -69,7 +69,7 @@ public struct Backtrace: Sendable { initializedCount = addresses.withMemoryRebound(to: UnsafeMutableRawPointer.self) { addresses in .init(clamping: backtrace(addresses.baseAddress!, .init(clamping: addresses.count))) } -#elseif os(Linux) || os(FreeBSD) +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) initializedCount = .init(clamping: backtrace(addresses.baseAddress!, .init(clamping: addresses.count))) #elseif os(Windows) initializedCount = Int(clamping: RtlCaptureStackBackTrace(0, ULONG(clamping: addresses.count), addresses.baseAddress!, nil)) @@ -181,7 +181,7 @@ extension Backtrace { /// crash. To avoid said crash, we'll keep a strong reference to the /// object (abandoning memory until the process exits.) /// ([swift-#62985](https://github.com/swiftlang/swift/issues/62985)) -#if os(Windows) || os(FreeBSD) +#if os(Windows) || os(FreeBSD) || os(OpenBSD) nonisolated(unsafe) var errorObject: AnyObject? #else nonisolated(unsafe) weak var errorObject: AnyObject? diff --git a/Sources/Testing/Support/Additions/CommandLineAdditions.swift b/Sources/Testing/Support/Additions/CommandLineAdditions.swift index 762ab7290..0fda59839 100644 --- a/Sources/Testing/Support/Additions/CommandLineAdditions.swift +++ b/Sources/Testing/Support/Additions/CommandLineAdditions.swift @@ -55,6 +55,14 @@ extension CommandLine { return String(cString: buffer.baseAddress!) } } +#elseif os(OpenBSD) + // OpenBSD does not have API to get a path to the running executable. Use + // arguments[0]. We do a basic sniff test for a path-like string, but + // otherwise return argv[0] verbatim. + guard let argv0 = arguments.first, argv0.contains("/") else { + throw CError(rawValue: ENOEXEC) + } + return argv0 #elseif os(Windows) var result: String? #if DEBUG @@ -87,7 +95,7 @@ extension CommandLine { return arguments[0] #else #warning("Platform-specific implementation missing: executable path unavailable") - return "" + throw SystemError(description: "The executable path of the current process could not be determined.") #endif } } diff --git a/Sources/Testing/Support/CError.swift b/Sources/Testing/Support/CError.swift index 47b9d6612..a8462fda4 100644 --- a/Sources/Testing/Support/CError.swift +++ b/Sources/Testing/Support/CError.swift @@ -47,8 +47,8 @@ func strerror(_ errorCode: CInt) -> String { _ = strerror_s(buffer.baseAddress!, buffer.count, errorCode) return strnlen(buffer.baseAddress!, buffer.count) } -#elseif os(FreeBSD) - // FreeBSD's implementation of strerror() is not thread-safe. +#elseif os(FreeBSD) || os(OpenBSD) + // FreeBSD's/OpenBSD's implementation of strerror() is not thread-safe. String(unsafeUninitializedCapacity: 1024) { buffer in _ = strerror_r(errorCode, buffer.baseAddress!, buffer.count) return strnlen(buffer.baseAddress!, buffer.count) diff --git a/Sources/Testing/Support/Environment.swift b/Sources/Testing/Support/Environment.swift index e10505877..ec2ee9c74 100644 --- a/Sources/Testing/Support/Environment.swift +++ b/Sources/Testing/Support/Environment.swift @@ -42,7 +42,7 @@ enum Environment { } } -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) /// Get all environment variables from a POSIX environment block. /// /// - Parameters: @@ -103,7 +103,7 @@ enum Environment { } #endif return _get(fromEnviron: _NSGetEnviron()!.pointee!) -#elseif os(Linux) || os(FreeBSD) || os(Android) +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) _get(fromEnviron: swt_environ()) #elseif os(WASI) _get(fromEnviron: __wasilibc_get_environ()) @@ -170,7 +170,7 @@ enum Environment { } return nil } -#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) getenv(name).flatMap { String(validatingCString: $0) } #elseif os(Windows) name.withCString(encodedAs: UTF16.self) { name in diff --git a/Sources/Testing/Support/FileHandle.swift b/Sources/Testing/Support/FileHandle.swift index c234206f8..2a2bfe967 100644 --- a/Sources/Testing/Support/FileHandle.swift +++ b/Sources/Testing/Support/FileHandle.swift @@ -215,7 +215,7 @@ struct FileHandle: ~Copyable, Sendable { /// descriptor, `nil` is passed to `body`. borrowing func withUnsafePOSIXFileDescriptor(_ body: (CInt?) throws -> R) rethrows -> R { try withUnsafeCFILEHandle { handle in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) let fd = fileno(handle) #elseif os(Windows) let fd = _fileno(handle) @@ -274,7 +274,7 @@ struct FileHandle: ~Copyable, Sendable { /// other threads. borrowing func withLock(_ body: () throws -> R) rethrows -> R { try withUnsafeCFILEHandle { handle in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) flockfile(handle) defer { funlockfile(handle) @@ -309,7 +309,7 @@ extension FileHandle { // If possible, reserve enough space in the resulting buffer to contain // the contents of the file being read. var size: Int? -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) withUnsafePOSIXFileDescriptor { fd in var s = stat() if let fd, 0 == fstat(fd, &s) { @@ -505,7 +505,7 @@ extension FileHandle { extension FileHandle { /// Is this file handle a TTY or PTY? var isTTY: Bool { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) // If stderr is a TTY and TERM is set, that's good enough for us. withUnsafePOSIXFileDescriptor { fd in if let fd, 0 != isatty(fd), let term = Environment.variable(named: "TERM"), !term.isEmpty { @@ -532,7 +532,7 @@ extension FileHandle { #if !SWT_NO_PIPES /// Is this file handle a pipe or FIFO? var isPipe: Bool { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) withUnsafePOSIXFileDescriptor { fd in guard let fd else { return false @@ -607,7 +607,7 @@ func fileExists(atPath path: String) -> Bool { /// resolved, the resulting string may differ slightly but refers to the same /// file system object. If the path could not be resolved, returns `nil`. func canonicalizePath(_ path: String) -> String? { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) path.withCString { path in if let resolvedCPath = realpath(path, nil) { defer { diff --git a/Sources/Testing/Support/GetSymbol.swift b/Sources/Testing/Support/GetSymbol.swift index 3d4eb32d8..264bc0daa 100644 --- a/Sources/Testing/Support/GetSymbol.swift +++ b/Sources/Testing/Support/GetSymbol.swift @@ -13,7 +13,7 @@ internal import _TestingInternals #if !SWT_NO_DYNAMIC_LINKING /// The platform-specific type of a loaded image handle. -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) typealias ImageAddress = UnsafeMutableRawPointer #elseif os(Windows) typealias ImageAddress = HMODULE @@ -28,7 +28,7 @@ typealias ImageAddress = Never /// and cannot be imported directly into Swift. As well, `RTLD_DEFAULT` is only /// defined on Linux when `_GNU_SOURCE` is defined, so it is not sufficient to /// declare a wrapper function in the internal module's Stubs.h file. -#if SWT_TARGET_OS_APPLE || os(FreeBSD) +#if SWT_TARGET_OS_APPLE || os(FreeBSD) || os(OpenBSD) private nonisolated(unsafe) let RTLD_DEFAULT = ImageAddress(bitPattern: -2) #elseif os(Android) && _pointerBitWidth(_32) private nonisolated(unsafe) let RTLD_DEFAULT = ImageAddress(bitPattern: UInt(0xFFFFFFFF)) @@ -59,7 +59,7 @@ private nonisolated(unsafe) let RTLD_DEFAULT = ImageAddress(bitPattern: 0) /// calling `EnumProcessModules()` and iterating over the returned handles /// looking for one containing the given function. func symbol(in handle: ImageAddress? = nil, named symbolName: String) -> UnsafeRawPointer? { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) dlsym(handle ?? RTLD_DEFAULT, symbolName).map(UnsafeRawPointer.init) #elseif os(Windows) symbolName.withCString { symbolName in diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index 11c4c4c86..d0edbc801 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -38,7 +38,7 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { /// or `OSAllocatedUnfairLock`. #if SWT_TARGET_OS_APPLE || os(Linux) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) typealias PlatformLock = pthread_mutex_t -#elseif os(FreeBSD) +#elseif os(FreeBSD) || os(OpenBSD) typealias PlatformLock = pthread_mutex_t? #elseif os(Windows) typealias PlatformLock = SRWLOCK @@ -54,7 +54,7 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { private final class _Storage: ManagedBuffer { deinit { withUnsafeMutablePointerToElements { lock in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) _ = pthread_mutex_destroy(lock) #elseif os(Windows) // No deinitialization needed. @@ -73,7 +73,7 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { init(rawValue: T) { _storage = _Storage.create(minimumCapacity: 1, makingHeaderWith: { _ in rawValue }) _storage.withUnsafeMutablePointerToElements { lock in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) _ = pthread_mutex_init(lock, nil) #elseif os(Windows) InitializeSRWLock(lock) @@ -103,7 +103,7 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { /// concurrency tools. nonmutating func withLock(_ body: (inout T) throws -> R) rethrows -> R { try _storage.withUnsafeMutablePointers { rawValue, lock in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) _ = pthread_mutex_lock(lock) defer { _ = pthread_mutex_unlock(lock) diff --git a/Sources/Testing/Support/Versions.swift b/Sources/Testing/Support/Versions.swift index 592722486..5e974c6f1 100644 --- a/Sources/Testing/Support/Versions.swift +++ b/Sources/Testing/Support/Versions.swift @@ -31,7 +31,7 @@ let operatingSystemVersion: String = { default: return "\(productVersion) (\(buildNumber))" } -#elseif !SWT_NO_UNAME && (SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD)) +#elseif !SWT_NO_UNAME && (SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD)) var name = utsname() if 0 == uname(&name) { let release = withUnsafeBytes(of: name.release) { release in diff --git a/Sources/Testing/Traits/Tags/Tag.Color+Loading.swift b/Sources/Testing/Traits/Tags/Tag.Color+Loading.swift index 3e3682e6f..2ab35b107 100644 --- a/Sources/Testing/Traits/Tags/Tag.Color+Loading.swift +++ b/Sources/Testing/Traits/Tags/Tag.Color+Loading.swift @@ -11,7 +11,7 @@ private import _TestingInternals #if !SWT_NO_FILE_IO -#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) || os(Linux) || os(FreeBSD) +#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) || os(Linux) || os(FreeBSD) || os(OpenBSD) /// The path to the current user's home directory, if known. private var _homeDirectoryPath: String? { #if SWT_TARGET_OS_APPLE @@ -57,7 +57,7 @@ var swiftTestingDirectoryPath: String? { // The (default) name of the .swift-testing directory. let swiftTestingDirectoryName = ".swift-testing" -#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) || os(Linux) || os(FreeBSD) +#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) || os(Linux) || os(FreeBSD) || os(OpenBSD) if let homeDirectoryPath = _homeDirectoryPath { return appendPathComponent(swiftTestingDirectoryName, to: homeDirectoryPath) } diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index eac49c45d..baf4ebd90 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -412,7 +412,7 @@ static void enumerateTypeMetadataSections(const SectionEnumerator& body) { } } -#elif defined(__linux__) || defined(__FreeBSD__) || defined(__ANDROID__) +#elif defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__ANDROID__) #pragma mark - ELF implementation /// Specifies the address range corresponding to a section. diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index 51c02e277..b1f4c7973 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -133,6 +133,10 @@ #include #endif +#if defined(__OpenBSD__) +#include +#endif + #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #define NOMINMAX diff --git a/Sources/_TestingInternals/include/Stubs.h b/Sources/_TestingInternals/include/Stubs.h index 4e114f751..caeb7c493 100644 --- a/Sources/_TestingInternals/include/Stubs.h +++ b/Sources/_TestingInternals/include/Stubs.h @@ -93,7 +93,7 @@ static DWORD_PTR swt_PROC_THREAD_ATTRIBUTE_HANDLE_LIST(void) { } #endif -#if defined(__linux__) || defined(__FreeBSD__) || defined(__ANDROID__) +#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__ANDROID__) /// The environment block. /// /// By POSIX convention, the environment block variable is declared in client diff --git a/Tests/TestingTests/Support/EnvironmentTests.swift b/Tests/TestingTests/Support/EnvironmentTests.swift index a4fb8ddd9..512ebfe7b 100644 --- a/Tests/TestingTests/Support/EnvironmentTests.swift +++ b/Tests/TestingTests/Support/EnvironmentTests.swift @@ -90,7 +90,7 @@ extension Environment { environment[name] = value } return true -#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(Android) || os(WASI) +#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) if let value { return 0 == setenv(name, value, 1) } diff --git a/Tests/TestingTests/Support/FileHandleTests.swift b/Tests/TestingTests/Support/FileHandleTests.swift index c7f347357..c837ac7cf 100644 --- a/Tests/TestingTests/Support/FileHandleTests.swift +++ b/Tests/TestingTests/Support/FileHandleTests.swift @@ -14,7 +14,7 @@ private import _TestingInternals #if !SWT_NO_FILE_IO // NOTE: we don't run these tests on iOS (etc.) because processes on those // platforms are sandboxed and do not have arbitrary filesystem access. -#if os(macOS) || os(Linux) || os(FreeBSD) || os(Android) || os(Windows) +#if os(macOS) || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(Windows) @Suite("FileHandle Tests") struct FileHandleTests { // FileHandle is non-copyable, so it cannot yet be used as a test parameter. @@ -255,7 +255,7 @@ func temporaryDirectory() throws -> String { } return try #require(Environment.variable(named: "TMPDIR")) } -#elseif os(Linux) || os(FreeBSD) +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) "/tmp" #elseif os(Android) Environment.variable(named: "TMPDIR") ?? "/data/local/tmp" From 4ca12eeba0f8584f34d2df056c25dceaca11f10e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 6 Jan 2025 15:06:44 -0500 Subject: [PATCH 034/234] Revert "Work around a crash importing FoundationXML. (#786)" (#882) This reverts commit 405d8c9d60730228527dbd74d9ba75bf3fde5069. The issue should no longer be occurring. See #786. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Tests/TestingTests/EventRecorderTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 1922a7841..97619b755 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -15,7 +15,7 @@ import RegexBuilder #if canImport(Foundation) import Foundation #endif -#if SWT_FIXED_138761752 && canImport(FoundationXML) +#if canImport(FoundationXML) import FoundationXML #endif @@ -299,7 +299,7 @@ struct EventRecorderTests { } #endif -#if (SWT_TARGET_OS_APPLE && canImport(Foundation)) || (SWT_FIXED_138761752 && canImport(FoundationXML)) +#if canImport(Foundation) || canImport(FoundationXML) @Test( "JUnitXMLRecorder outputs valid XML", .bug("https://github.com/swiftlang/swift-testing/issues/254") From 65b7ef21b3d5425951b422cc3a0f6abd1f20b5a9 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 6 Jan 2025 16:41:33 -0500 Subject: [PATCH 035/234] Use the Toolhelp32 API to enumerate loaded Win32 modules. (#892) This PR replaces our call to `EnumProcessModules()` with one to `CreateToolhelp32Snapshot(TH32CS_SNAPMODULE)`. Three reasons: 1. `EnumProcessModules()` requires us to specify a large, fixed-size buffer to contain all the `HMODULE` handles; 2. `EnumProcessModules()` does not own any references to the handles it returns, meaning that a module can be unloaded while we are iterating over them (while `CreateToolhelp32Snapshot()` temporarily bumps the refcounts of the handles it produces); and 3. `CreateToolhelp32Snapshot()` lets us produce a lazy sequence of `HMODULE` values rather than an array, letting us write somewhat Swiftier code that uses it. The overhead of using `CreateToolhelp32Snapshot()` was negligible (below the noise level when measuring). ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Support/Additions/WinSDKAdditions.swift | 52 +++++++++++++++++++ Sources/Testing/Support/GetSymbol.swift | 23 ++------ 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/Sources/Testing/Support/Additions/WinSDKAdditions.swift b/Sources/Testing/Support/Additions/WinSDKAdditions.swift index 488d52dd6..9b902c5d1 100644 --- a/Sources/Testing/Support/Additions/WinSDKAdditions.swift +++ b/Sources/Testing/Support/Additions/WinSDKAdditions.swift @@ -50,4 +50,56 @@ let STATUS_SIGNAL_CAUGHT_BITS = { return result }() + +// MARK: - HMODULE members + +extension HMODULE { + /// A helper type that manages state for ``HMODULE/all``. + private final class _AllState { + /// The toolhelp snapshot. + var snapshot: HANDLE? + + /// The module iterator. + var me = MODULEENTRY32W() + + deinit { + if let snapshot { + CloseHandle(snapshot) + } + } + } + + /// All modules loaded in the current process. + /// + /// - Warning: It is possible for one or more modules in this sequence to be + /// unloaded while you are iterating over it. To minimize the risk, do not + /// discard the sequence until iteration is complete. Modules containing + /// Swift code can never be safely unloaded. + static var all: some Sequence { + sequence(state: _AllState()) { state in + if let snapshot = state.snapshot { + // We have already iterated over the first module. Return the next one. + if Module32NextW(snapshot, &state.me) { + return state.me.hModule + } + } else { + // Create a toolhelp snapshot that lists modules. + guard let snapshot = CreateToolhelp32Snapshot(DWORD(TH32CS_SNAPMODULE), 0) else { + return nil + } + state.snapshot = snapshot + + // Initialize the iterator for use by the resulting sequence and return + // the first module. + state.me.dwSize = DWORD(MemoryLayout.stride(ofValue: state.me)) + if Module32FirstW(snapshot, &state.me) { + return state.me.hModule + } + } + + // Reached the end of the iteration. + return nil + } + } +} #endif diff --git a/Sources/Testing/Support/GetSymbol.swift b/Sources/Testing/Support/GetSymbol.swift index 264bc0daa..b0f057088 100644 --- a/Sources/Testing/Support/GetSymbol.swift +++ b/Sources/Testing/Support/GetSymbol.swift @@ -70,25 +70,10 @@ func symbol(in handle: ImageAddress? = nil, named symbolName: String) -> UnsafeR } } - // Find all the modules loaded in the current process. We assume there - // aren't more than 1024 loaded modules (as does Microsoft sample code.) - return withUnsafeTemporaryAllocation(of: HMODULE?.self, capacity: 1024) { hModules in - let byteCount = DWORD(hModules.count * MemoryLayout.stride) - var byteCountNeeded: DWORD = 0 - guard K32EnumProcessModules(GetCurrentProcess(), hModules.baseAddress!, byteCount, &byteCountNeeded) else { - return nil - } - - // Enumerate all modules looking for one containing the given symbol. - let hModuleCount = min(hModules.count, Int(byteCountNeeded) / MemoryLayout.stride) - let hModulesEnd = hModules.index(hModules.startIndex, offsetBy: hModuleCount) - for hModule in hModules[.. Date: Tue, 7 Jan 2025 11:45:39 -0500 Subject: [PATCH 036/234] Discover test content stored in the test content section of loaded images. (#893) This PR implements discovery, _but not emission_, of test content that has been added to a loaded image's test content metadata section at compile time. Loading this data from a dedicated section has several benefits over our current model, which involves walking Swift's type metadata table looking for types that conform to a protocol: 1. We don't need to define that protocol as public API in Swift Testing, 1. We don't need to emit type metadata (much larger than what we really need) for every test function, 1. We don't need to duplicate a large chunk of the Swift ABI sources in order to walk the type metadata table correctly, and 1. Almost all the new code is written in Swift, whereas the code it is intended to replace could not be fully represented in Swift and needed to be written in C++. The change also opens up the possibility of supporting generic types in the future because we can emit metadata without needing to emit a nested type (which is not always valid in a generic context.) That's a "future direction" and not covered by this PR specifically. I've defined a layout for entries in the new `swift5_tests` section that should be flexible enough for us in the short-to-medium term and which lets us define additional arbitrary test content record types. The layout of this section is covered in depth in the new [TestContent.md](Documentation/ABI/TestContent.md) article. This PR does **not** include the code necessary to _emit_ test content records into images at compile time. That part of the change is covered by #880 and requires a new language feature to control which section data is emitted to. An experimental version of that language feature is currently available under the `"SymbolLinkageMarkers"` label. Because there is no test content in the test content section yet, this PR does not remove the "legacy" codepaths that discover tests in the type metadata section. > [!NOTE] > This change is experimental. ### See Also https://github.com/swiftlang/swift-testing/pull/880 https://github.com/swiftlang/swift-testing/issues/735 https://github.com/swiftlang/swift/issues/76698 https://github.com/swiftlang/swift/pull/78411 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 168 ++++++++++++++ Package.swift | 5 +- Sources/Testing/CMakeLists.txt | 3 + Sources/Testing/Discovery+Platform.swift | 217 ++++++++++++++++++ Sources/Testing/Discovery.swift | 173 ++++++++++++++ Sources/Testing/ExitTests/ExitTest.swift | 111 ++++++--- .../Support/Additions/WinSDKAdditions.swift | 27 +++ Sources/Testing/Test+Discovery+Legacy.swift | 81 +++++++ Sources/Testing/Test+Discovery.swift | 110 ++++----- Sources/_TestingInternals/Discovery.cpp | 25 ++ Sources/_TestingInternals/include/Discovery.h | 56 +++++ Sources/_TestingInternals/include/Includes.h | 4 + Sources/_TestingInternals/include/Stubs.h | 8 + Tests/TestingTests/ABIEntryPointTests.swift | 2 + Tests/TestingTests/MiscellaneousTests.swift | 74 ++++++ 15 files changed, 978 insertions(+), 86 deletions(-) create mode 100644 Documentation/ABI/TestContent.md create mode 100644 Sources/Testing/Discovery+Platform.swift create mode 100644 Sources/Testing/Discovery.swift create mode 100644 Sources/Testing/Test+Discovery+Legacy.swift diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md new file mode 100644 index 000000000..fd7b9f893 --- /dev/null +++ b/Documentation/ABI/TestContent.md @@ -0,0 +1,168 @@ +# Runtime-discoverable test content + + + +This document describes the format and location of test content that the testing +library emits at compile time and can discover at runtime. + +> [!WARNING] +> The content of this document is subject to change pending efforts to define a +> Swift-wide standard mechanism for runtime metadata emission and discovery. +> Treat the information in this document as experimental. + +## Basic format + +Swift Testing stores test content records in a dedicated platform-specific +section in built test products: + +| Platform | Binary Format | Section Name | +|-|:-:|-| +| macOS, iOS, watchOS, tvOS, visionOS | Mach-O | `__DATA_CONST,__swift5_tests` | +| Linux, FreeBSD, OpenBSD, Android | ELF | `swift5_tests` | +| WASI | WebAssembly | `swift5_tests` | +| Windows | PE/COFF | `.sw5test$B`[^windowsPadding] | + +[^windowsPadding]: On Windows, the Swift compiler [emits](https://github.com/swiftlang/swift/blob/main/stdlib/public/runtime/SwiftRT-COFF.cpp) + leading and trailing padding into this section, both zeroed and of size + `MemoryLayout.stride`. Code that walks this section must skip over this + padding. + +### Record layout + +Regardless of platform, all test content records created and discoverable by the +testing library have the following layout: + +```swift +typealias TestContentRecord = ( + kind: UInt32, + reserved1: UInt32, + accessor: (@convention(c) (_ outValue: UnsafeMutableRawPointer, _ hint: UnsafeRawPointer?) -> CBool)?, + context: UInt, + reserved2: UInt +) +``` + +This type has natural size, stride, and alignment. Its fields are native-endian. +If needed, this type can be represented in C as a structure: + +```c +struct SWTTestContentRecord { + uint32_t kind; + uint32_t reserved1; + bool (* _Nullable accessor)(void *outValue, const void *_Null_unspecified hint); + uintptr_t context; + uintptr_t reserved2; +}; +``` + +### Record content + +#### The kind field + +Each record's _kind_ determines how the record will be interpreted at runtime. A +record's kind is a 32-bit unsigned value. The following kinds are defined: + +| As Hexadecimal | As [FourCC](https://en.wikipedia.org/wiki/FourCC) | Interpretation | +|-:|:-:|-| +| `0x00000000` | – | Reserved (**do not use**) | +| `0x74657374` | `'test'` | Test or suite declaration | +| `0x65786974` | `'exit'` | Exit test | + + + +#### The accessor field + +The function `accessor` is a C function. When called, it initializes the memory +at its argument `outValue` to an instance of some Swift type and returns `true`, +or returns `false` if it could not generate the relevant content. On successful +return, the caller is responsible for deinitializing the memory at `outValue` +when done with it. + +If `accessor` is `nil`, the test content record is ignored. The testing library +may, in the future, define record kinds that do not provide an accessor function +(that is, they represent pure compile-time information only.) + +The second argument to this function, `hint`, is an optional input that can be +passed to help the accessor function determine if its corresponding test content +record matches what the caller is looking for. If the caller passes `nil` as the +`hint` argument, the accessor behaves as if it matched (that is, no additional +filtering is performed.) + +The concrete Swift type of the value written to `outValue` and the value pointed +to by `hint` depend on the kind of record: + +- For test or suite declarations (kind `0x74657374`), the accessor produces an + asynchronous Swift function that returns an instance of `Test`: + + ```swift + @Sendable () async -> Test + ``` + + This signature is not the signature of `accessor`, but of the Swift function + reference it writes to `outValue`. This level of indirection is necessary + because loading a test or suite declaration is an asynchronous operation, but + C functions cannot be `async`. + + Test content records of this kind do not specify a type for `hint`. Always + pass `nil`. + +- For exit test declarations (kind `0x65786974`), the accessor produces a + structure describing the exit test (of type `__ExitTest`.) + + Test content records of this kind accept a `hint` of type `SourceLocation`. + They only produce a result if they represent an exit test declared at the same + source location (or if the hint is `nil`.) + +#### The context field + +This field can be used by test content to store additional context for a test +content record that needs to be made available before the accessor is called: + +- For test or suite declarations (kind `0x74657374`), this field contains a bit + mask with the following flags currently defined: + + | Bit | Value | Description | + |-:|-:|-| + | `1 << 0` | `1` | This record contains a suite declaration | + | `1 << 1` | `2` | This record contains a parameterized test function declaration | + + Other bits are reserved for future use and must be set to `0`. + +- For exit test declarations (kind `0x65786974`), this field is reserved for + future use and must be set to `0`. + +#### The reserved1 and reserved2 fields + +These fields are reserved for future use. Always set them to `0`. + +## Third-party test content + +Testing tools may make use of the same storage and discovery mechanisms by +emitting their own test content records into the test record content section. + +Third-party test content should set the `kind` field to a unique value only used +by that tool, or used by that tool in collaboration with other compatible tools. +At runtime, Swift Testing ignores test content records with unrecognized `kind` +values. To reserve a new unique `kind` value, open a [GitHub issue](https://github.com/swiftlang/swift-testing/issues/new/choose) +against Swift Testing. + +The layout of third-party test content records must be compatible with that of +`TestContentRecord` as specified above. Third-party tools are ultimately +responsible for ensuring the values they emit into the test content section are +correctly aligned and have sufficient padding; failure to do so may render +downstream test code unusable. + + diff --git a/Package.swift b/Package.swift index a72f7086a..ab06f693d 100644 --- a/Package.swift +++ b/Package.swift @@ -67,7 +67,10 @@ let package = Package( "_Testing_CoreGraphics", "_Testing_Foundation", ], - swiftSettings: .packageSettings + swiftSettings: .packageSettings + [ + // For testing test content section discovery only + .enableExperimentalFeature("SymbolLinkageMarkers"), + ] ), .macro( diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index f7728ac49..f205561a8 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -81,10 +81,13 @@ add_library(Testing Support/Locked.swift Support/SystemError.swift Support/Versions.swift + Discovery.swift + Discovery+Platform.swift Test.ID.Selection.swift Test.ID.swift Test.swift Test+Discovery.swift + Test+Discovery+Legacy.swift Test+Macro.swift Traits/Bug.swift Traits/Comment.swift diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift new file mode 100644 index 000000000..db5594703 --- /dev/null +++ b/Sources/Testing/Discovery+Platform.swift @@ -0,0 +1,217 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +internal import _TestingInternals + +/// A structure describing the bounds of a Swift metadata section. +struct SectionBounds: Sendable { + /// The base address of the image containing the section, if known. + nonisolated(unsafe) var imageAddress: UnsafeRawPointer? + + /// The in-memory representation of the section. + nonisolated(unsafe) var buffer: UnsafeRawBufferPointer + + /// All test content section bounds found in the current process. + static var allTestContent: some RandomAccessCollection { + _testContentSectionBounds() + } +} + +#if !SWT_NO_DYNAMIC_LINKING +#if SWT_TARGET_OS_APPLE +// MARK: - Apple implementation + +/// An array containing all of the test content section bounds known to the +/// testing library. +private let _sectionBounds = Locked<[SectionBounds]>(rawValue: []) + +/// A call-once function that initializes `_sectionBounds` and starts listening +/// for loaded Mach headers. +private let _startCollectingSectionBounds: Void = { + // Ensure _sectionBounds is initialized before we touch libobjc or dyld. + _sectionBounds.withLock { sectionBounds in + sectionBounds.reserveCapacity(Int(_dyld_image_count())) + } + + func addSectionBounds(from mh: UnsafePointer) { +#if _pointerBitWidth(_64) + let mh = UnsafeRawPointer(mh).assumingMemoryBound(to: mach_header_64.self) +#endif + + // Ignore this Mach header if it is in the shared cache. On platforms that + // support it (Darwin), most system images are contained in this range. + // System images can be expected not to contain test declarations, so we + // don't need to walk them. + guard 0 == mh.pointee.flags & MH_DYLIB_IN_CACHE else { + return + } + + // If this image contains the Swift section we need, acquire the lock and + // store the section's bounds. + var size = CUnsignedLong(0) + if let start = getsectiondata(mh, "__DATA_CONST", "__swift5_tests", &size), size > 0 { + _sectionBounds.withLock { sectionBounds in + let buffer = UnsafeRawBufferPointer(start: start, count: Int(clamping: size)) + let sb = SectionBounds(imageAddress: mh, buffer: buffer) + sectionBounds.append(sb) + } + } + } + +#if _runtime(_ObjC) + objc_addLoadImageFunc { mh in + addSectionBounds(from: mh) + } +#else + _dyld_register_func_for_add_image { mh, _ in + addSectionBounds(from: mh) + } +#endif +}() + +/// The Apple-specific implementation of ``SectionBounds/all``. +/// +/// - Returns: An array of structures describing the bounds of all known test +/// content sections in the current process. +private func _testContentSectionBounds() -> [SectionBounds] { + _startCollectingSectionBounds + return _sectionBounds.rawValue +} + +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) +// MARK: - ELF implementation + +private import SwiftShims // For MetadataSections + +/// The ELF-specific implementation of ``SectionBounds/all``. +/// +/// - Returns: An array of structures describing the bounds of all known test +/// content sections in the current process. +private func _testContentSectionBounds() -> [SectionBounds] { + var result = [SectionBounds]() + + withUnsafeMutablePointer(to: &result) { result in + swift_enumerateAllMetadataSections({ sections, context in + let version = sections.load(as: UInt.self) + guard version >= 4 else { + // This structure is too old to contain the swift5_tests field. + return true + } + + let sections = sections.load(as: MetadataSections.self) + let result = context.assumingMemoryBound(to: [SectionBounds].self) + + let start = UnsafeRawPointer(bitPattern: sections.swift5_tests.start) + let size = Int(clamping: sections.swift5_tests.length) + if let start, size > 0 { + let buffer = UnsafeRawBufferPointer(start: start, count: size) + let sb = SectionBounds(imageAddress: sections.baseAddress, buffer: buffer) + result.pointee.append(sb) + } + + return true + }, result) + } + + return result +} + +#elseif os(Windows) +// MARK: - Windows implementation + +/// Find the section with the given name in the given module. +/// +/// - Parameters: +/// - sectionName: The name of the section to look for. Long section names are +/// not supported. +/// - hModule: The module to inspect. +/// +/// - Returns: A structure describing the given section, or `nil` if the section +/// could not be found. +private func _findSection(named sectionName: String, in hModule: HMODULE) -> SectionBounds? { + hModule.withNTHeader { ntHeader in + guard let ntHeader else { + return nil + } + + let sectionHeaders = UnsafeBufferPointer( + start: swt_IMAGE_FIRST_SECTION(ntHeader), + count: Int(clamping: max(0, ntHeader.pointee.FileHeader.NumberOfSections)) + ) + return sectionHeaders.lazy + .filter { sectionHeader in + // FIXME: Handle longer names ("/%u") from string table + withUnsafeBytes(of: sectionHeader.Name) { thisSectionName in + 0 == strncmp(sectionName, thisSectionName.baseAddress!, Int(IMAGE_SIZEOF_SHORT_NAME)) + } + }.compactMap { sectionHeader in + guard let virtualAddress = Int(exactly: sectionHeader.VirtualAddress), virtualAddress > 0 else { + return nil + } + + var buffer = UnsafeRawBufferPointer( + start: UnsafeRawPointer(hModule) + virtualAddress, + count: Int(clamping: min(max(0, sectionHeader.Misc.VirtualSize), max(0, sectionHeader.SizeOfRawData))) + ) + guard buffer.count > 2 * MemoryLayout.stride else { + return nil + } + + // Skip over the leading and trailing zeroed uintptr_t values. These + // values are always emitted by SwiftRT-COFF.cpp into all Swift images. +#if DEBUG + let firstPointerValue = buffer.baseAddress!.loadUnaligned(as: UInt.self) + assert(firstPointerValue == 0, "First pointer-width value in section '\(sectionName)' at \(buffer.baseAddress!) was expected to equal 0 (found \(firstPointerValue) instead)") + let lastPointerValue = ((buffer.baseAddress! + buffer.count) - MemoryLayout.stride).loadUnaligned(as: UInt.self) + assert(lastPointerValue == 0, "Last pointer-width value in section '\(sectionName)' at \(buffer.baseAddress!) was expected to equal 0 (found \(lastPointerValue) instead)") +#endif + buffer = UnsafeRawBufferPointer( + rebasing: buffer + .dropFirst(MemoryLayout.stride) + .dropLast(MemoryLayout.stride) + ) + + return SectionBounds(imageAddress: hModule, buffer: buffer) + }.first + } +} + +/// The Windows-specific implementation of ``SectionBounds/all``. +/// +/// - Returns: An array of structures describing the bounds of all known test +/// content sections in the current process. +private func _testContentSectionBounds() -> [SectionBounds] { + HMODULE.all.compactMap { _findSection(named: ".sw5test", in: $0) } +} +#else +/// The fallback implementation of ``SectionBounds/all`` for platforms that +/// support dynamic linking. +/// +/// - Returns: The empty array. +private func _testContentSectionBounds() -> [SectionBounds] { + #warning("Platform-specific implementation missing: Runtime test discovery unavailable (dynamic)") + return [] +} +#endif +#else +// MARK: - Statically-linked implementation + +/// The common implementation of ``SectionBounds/all`` for platforms that do not +/// support dynamic linking. +/// +/// - Returns: A structure describing the bounds of the test content section +/// contained in the same image as the testing library itself. +private func _testContentSectionBounds() -> CollectionOfOne { + let (sectionBegin, sectionEnd) = SWTTestContentSectionBounds + let buffer = UnsafeRawBufferPointer(start: n, count: max(0, sectionEnd - sectionBegin)) + let sb = SectionBounds(imageAddress: nil, buffer: buffer) + return CollectionOfOne(sb) +} +#endif diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift new file mode 100644 index 000000000..b2fc7825c --- /dev/null +++ b/Sources/Testing/Discovery.swift @@ -0,0 +1,173 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// The content of a test content record. +/// +/// - Parameters: +/// - kind: The kind of this record. +/// - reserved1: Reserved for future use. +/// - accessor: A function which, when called, produces the test content. +/// - context: Kind-specific context for this record. +/// - reserved2: Reserved for future use. +/// +/// - Warning: This type is used to implement the `@Test` macro. Do not use it +/// directly. +public typealias __TestContentRecord = ( + kind: UInt32, + reserved1: UInt32, + accessor: (@convention(c) (_ outValue: UnsafeMutableRawPointer, _ hint: UnsafeRawPointer?) -> CBool)?, + context: UInt, + reserved2: UInt +) + +/// Resign any pointers in a test content record. +/// +/// - Parameters: +/// - record: The test content record to resign. +/// +/// - Returns: A copy of `record` with its pointers resigned. +/// +/// On platforms/architectures without pointer authentication, this function has +/// no effect. +private func _resign(_ record: __TestContentRecord) -> __TestContentRecord { + var record = record + record.accessor = record.accessor.map(swt_resign) + return record +} + +// MARK: - + +/// A protocol describing a type that can be stored as test content at compile +/// time and later discovered at runtime. +/// +/// This protocol is used to bring some Swift type safety to the ABI described +/// in `ABI/TestContent.md`. Refer to that document for more information about +/// this protocol's requirements. +/// +/// This protocol is not part of the public interface of the testing library. In +/// the future, we could make it public if we want to support runtime discovery +/// of test content by second- or third-party code. +protocol TestContent: ~Copyable { + /// The unique "kind" value associated with this type. + /// + /// The value of this property is reserved for each test content type. See + /// `ABI/TestContent.md` for a list of values and corresponding types. + static var testContentKind: UInt32 { get } + + /// The type of value returned by the test content accessor for this type. + /// + /// This type may or may not equal `Self` depending on the type's compile-time + /// and runtime requirements. If it does not equal `Self`, it should equal a + /// type whose instances can be converted to instances of `Self` (e.g. by + /// calling them if they are functions.) + associatedtype TestContentAccessorResult: ~Copyable + + /// A type of "hint" passed to ``discover(withHint:)`` to help the testing + /// library find the correct result. + /// + /// By default, this type equals `Never`, indicating that this type of test + /// content does not support hinting during discovery. + associatedtype TestContentAccessorHint: Sendable = Never +} + +extension TestContent where Self: ~Copyable { + /// Enumerate all test content records found in the given test content section + /// in the current process that match this ``TestContent`` type. + /// + /// - Parameters: + /// - sectionBounds: The bounds of the section to inspect. + /// + /// - Returns: A sequence of tuples. Each tuple contains an instance of + /// `__TestContentRecord` and the base address of the image containing that + /// test content record. Only test content records matching this + /// ``TestContent`` type's requirements are included in the sequence. + private static func _testContentRecords(in sectionBounds: SectionBounds) -> some Sequence<(imageAddress: UnsafeRawPointer?, record: __TestContentRecord)> { + sectionBounds.buffer.withMemoryRebound(to: __TestContentRecord.self) { records in + records.lazy + .filter { $0.kind == testContentKind } + .map(_resign) + .map { (sectionBounds.imageAddress, $0) } + } + } + + /// Call the given accessor function. + /// + /// - Parameters: + /// - accessor: The C accessor function of a test content record matching + /// this type. + /// - hint: A pointer to a kind-specific hint value. If not `nil`, this + /// value is passed to `accessor`, allowing that function to determine if + /// its record matches before initializing its out-result. + /// + /// - Returns: An instance of this type's accessor result or `nil` if an + /// instance could not be created (or if `hint` did not match.) + /// + /// The caller is responsible for ensuring that `accessor` corresponds to a + /// test content record of this type. + private static func _callAccessor(_ accessor: SWTTestContentAccessor, withHint hint: TestContentAccessorHint?) -> TestContentAccessorResult? { + withUnsafeTemporaryAllocation(of: TestContentAccessorResult.self, capacity: 1) { buffer in + let initialized = if let hint { + withUnsafePointer(to: hint) { hint in + accessor(buffer.baseAddress!, hint) + } + } else { + accessor(buffer.baseAddress!, nil) + } + guard initialized else { + return nil + } + return buffer.baseAddress!.move() + } + } + + /// The type of callback called by ``enumerateTestContent(withHint:_:)``. + /// + /// - Parameters: + /// - imageAddress: A pointer to the start of the image. This value is _not_ + /// equal to the value returned from `dlopen()`. On platforms that do not + /// support dynamic loading (and so do not have loadable images), the + /// value of this argument is unspecified. + /// - content: The value produced by the test content record's accessor. + /// - context: Context associated with `content`. The value of this argument + /// is dependent on the type of test content being enumerated. + /// - stop: An `inout` boolean variable indicating whether test content + /// enumeration should stop after the function returns. Set `stop` to + /// `true` to stop test content enumeration. + typealias TestContentEnumerator = (_ imageAddress: UnsafeRawPointer?, _ content: borrowing TestContentAccessorResult, _ context: UInt, _ stop: inout Bool) -> Void + + /// Enumerate all test content of this type known to Swift and found in the + /// current process. + /// + /// - Parameters: + /// - hint: An optional hint value. If not `nil`, this value is passed to + /// the accessor function of each test content record whose `kind` field + /// matches this type's ``testContentKind`` property. + /// - body: A function to invoke, once per matching test content record. + /// + /// This function uses a callback instead of producing a sequence because it + /// is used with move-only types (specifically ``ExitTest``) and + /// `Sequence.Element` must be copyable. + static func enumerateTestContent(withHint hint: TestContentAccessorHint? = nil, _ body: TestContentEnumerator) { + let testContentRecords = SectionBounds.allTestContent.lazy.flatMap(_testContentRecords(in:)) + + var stop = false + for (imageAddress, record) in testContentRecords { + if let accessor = record.accessor, let result = _callAccessor(accessor, withHint: hint) { + // Call the callback. + body(imageAddress, result, record.context, &stop) + if stop { + break + } + } + } + } +} diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index af7981297..01810a7ca 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -25,17 +25,50 @@ private import _TestingInternals /// A type describing an exit test. /// /// Instances of this type describe an exit test defined by the test author and -/// discovered or called at runtime. +/// discovered or called at runtime. Tools that implement custom exit test +/// handling will encounter instances of this type in two contexts: +/// +/// - When the current configuration's exit test handler, set with +/// ``Configuration/exitTestHandler``, is called; and +/// - When, in a child process, they need to look up the exit test to call. +/// +/// If you are writing tests, you don't usually need to interact directly with +/// an instance of this type. To create an exit test, use the +/// ``expect(exitsWith:_:sourceLocation:performing:)`` or +/// ``require(exitsWith:_:sourceLocation:performing:)`` macro. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -public struct ExitTest: Sendable, ~Copyable { +public typealias ExitTest = __ExitTest + +/// A type describing an exit test. +/// +/// - Warning: This type is used to implement the `#expect(exitsWith:)` macro. +/// Do not use it directly. Tools can use the SPI ``ExitTest`` typealias if +/// needed. +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +public struct __ExitTest: Sendable, ~Copyable { /// The expected exit condition of the exit test. + @_spi(ForToolsIntegrationOnly) public var expectedExitCondition: ExitCondition + /// The source location of the exit test. + /// + /// The source location is unique to each exit test and is consistent between + /// processes, so it can be used to uniquely identify an exit test at runtime. + @_spi(ForToolsIntegrationOnly) + public var sourceLocation: SourceLocation + /// The body closure of the exit test. - fileprivate var body: @Sendable () async throws -> Void = {} + /// + /// Do not invoke this closure directly. Instead, invoke ``callAsFunction()`` + /// to run the exit test. Running the exit test will always terminate the + /// current process. + fileprivate var body: @Sendable () async throws -> Void /// Storage for ``observedValues``. /// @@ -72,16 +105,25 @@ public struct ExitTest: Sendable, ~Copyable { } } - /// The source location of the exit test. + /// Initialize an exit test at runtime. /// - /// The source location is unique to each exit test and is consistent between - /// processes, so it can be used to uniquely identify an exit test at runtime. - public var sourceLocation: SourceLocation + /// - Warning: This initializer is used to implement the `#expect(exitsWith:)` + /// macro. Do not use it directly. + public init( + __expectedExitCondition expectedExitCondition: ExitCondition, + sourceLocation: SourceLocation, + body: @escaping @Sendable () async throws -> Void = {} + ) { + self.expectedExitCondition = expectedExitCondition + self.sourceLocation = sourceLocation + self.body = body + } } #if !SWT_NO_EXIT_TESTS // MARK: - Invocation +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) extension ExitTest { /// Disable crash reporting, crash logging, or core dumps for the current /// process. @@ -174,28 +216,17 @@ extension ExitTest { // MARK: - Discovery -/// A protocol describing a type that contains an exit test. -/// -/// - Warning: This protocol is used to implement the `#expect(exitsWith:)` -/// macro. Do not use it directly. -@_alwaysEmitConformanceMetadata -@_spi(Experimental) -public protocol __ExitTestContainer { - /// The expected exit condition of the exit test. - static var __expectedExitCondition: ExitCondition { get } - - /// The source location of the exit test. - static var __sourceLocation: SourceLocation { get } +extension ExitTest: TestContent { + static var testContentKind: UInt32 { + 0x65786974 + } - /// The body function of the exit test. - static var __body: @Sendable () async throws -> Void { get } + typealias TestContentAccessorResult = Self + typealias TestContentAccessorHint = SourceLocation } +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) extension ExitTest { - /// A string that appears within all auto-generated types conforming to the - /// `__ExitTestContainer` protocol. - private static let _exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" - /// Find the exit test function at the given source location. /// /// - Parameters: @@ -206,17 +237,34 @@ extension ExitTest { public static func find(at sourceLocation: SourceLocation) -> Self? { var result: Self? - enumerateTypes(withNamesContaining: _exitTestContainerTypeNameMagic) { _, type, stop in - if let type = type as? any __ExitTestContainer.Type, type.__sourceLocation == sourceLocation { + enumerateTestContent(withHint: sourceLocation) { _, exitTest, _, stop in + if exitTest.sourceLocation == sourceLocation { result = ExitTest( - expectedExitCondition: type.__expectedExitCondition, - body: type.__body, - sourceLocation: type.__sourceLocation + __expectedExitCondition: exitTest.expectedExitCondition, + sourceLocation: exitTest.sourceLocation, + body: exitTest.body ) stop = true } } + if result == nil { + // Call the legacy lookup function that discovers tests embedded in types. + enumerateTypes(withNamesContaining: exitTestContainerTypeNameMagic) { _, type, stop in + guard let type = type as? any __ExitTestContainer.Type else { + return + } + if type.__sourceLocation == sourceLocation { + result = ExitTest( + __expectedExitCondition: type.__expectedExitCondition, + sourceLocation: type.__sourceLocation, + body: type.__body + ) + stop = true + } + } + } + return result } } @@ -259,7 +307,7 @@ func callExitTest( var result: ExitTestArtifacts do { - var exitTest = ExitTest(expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation) + var exitTest = ExitTest(__expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation) exitTest.observedValues = observedValues result = try await configuration.exitTestHandler(exitTest) @@ -312,6 +360,7 @@ func callExitTest( // MARK: - SwiftPM/tools integration +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) extension ExitTest { /// A handler that is invoked when an exit test starts. /// diff --git a/Sources/Testing/Support/Additions/WinSDKAdditions.swift b/Sources/Testing/Support/Additions/WinSDKAdditions.swift index 9b902c5d1..18d08bfcd 100644 --- a/Sources/Testing/Support/Additions/WinSDKAdditions.swift +++ b/Sources/Testing/Support/Additions/WinSDKAdditions.swift @@ -101,5 +101,32 @@ extension HMODULE { return nil } } + + /// Get the NT header corresponding to this module. + /// + /// - Parameters: + /// - body: The function to invoke. A pointer to the module's NT header is + /// passed to this function, or `nil` if it could not be found. + /// + /// - Returns: Whatever is returned by `body`. + /// + /// - Throws: Whatever is thrown by `body`. + func withNTHeader(_ body: (UnsafePointer?) throws -> R) rethrows -> R { + // Get the DOS header (to which the HMODULE directly points, conveniently!) + // and check it's sufficiently valid for us to walk. The DOS header then + // tells us where to find the NT header. + try withMemoryRebound(to: IMAGE_DOS_HEADER.self, capacity: 1) { dosHeader in + guard dosHeader.pointee.e_magic == IMAGE_DOS_SIGNATURE, + let e_lfanew = Int(exactly: dosHeader.pointee.e_lfanew), e_lfanew > 0 else { + return try body(nil) + } + + let ntHeader = (UnsafeRawPointer(dosHeader) + e_lfanew).assumingMemoryBound(to: IMAGE_NT_HEADERS.self) + guard ntHeader.pointee.Signature == IMAGE_NT_SIGNATURE else { + return try body(nil) + } + return try body(ntHeader) + } + } } #endif diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift new file mode 100644 index 000000000..746c4128f --- /dev/null +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -0,0 +1,81 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// A protocol describing a type that contains tests. +/// +/// - Warning: This protocol is used to implement the `@Test` macro. Do not use +/// it directly. +@_alwaysEmitConformanceMetadata +public protocol __TestContainer { + /// The set of tests contained by this type. + static var __tests: [Test] { get async } +} + +/// A string that appears within all auto-generated types conforming to the +/// `__TestContainer` protocol. +let testContainerTypeNameMagic = "__🟠$test_container__" + +/// A protocol describing a type that contains an exit test. +/// +/// - Warning: This protocol is used to implement the `#expect(exitsWith:)` +/// macro. Do not use it directly. +@_alwaysEmitConformanceMetadata +@_spi(Experimental) +public protocol __ExitTestContainer { + /// The expected exit condition of the exit test. + static var __expectedExitCondition: ExitCondition { get } + + /// The source location of the exit test. + static var __sourceLocation: SourceLocation { get } + + /// The body function of the exit test. + static var __body: @Sendable () async throws -> Void { get } +} + +/// A string that appears within all auto-generated types conforming to the +/// `__ExitTestContainer` protocol. +let exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" + +// MARK: - + +/// The type of callback called by ``enumerateTypes(withNamesContaining:_:)``. +/// +/// - Parameters: +/// - imageAddress: A pointer to the start of the image. This value is _not_ +/// equal to the value returned from `dlopen()`. On platforms that do not +/// support dynamic loading (and so do not have loadable images), this +/// argument is unspecified. +/// - type: A Swift type. +/// - stop: An `inout` boolean variable indicating whether type enumeration +/// should stop after the function returns. Set `stop` to `true` to stop +/// type enumeration. +typealias TypeEnumerator = (_ imageAddress: UnsafeRawPointer?, _ type: Any.Type, _ stop: inout Bool) -> Void + +/// Enumerate all types known to Swift found in the current process whose names +/// contain a given substring. +/// +/// - Parameters: +/// - nameSubstring: A string which the names of matching classes all contain. +/// - body: A function to invoke, once per matching type. +func enumerateTypes(withNamesContaining nameSubstring: String, _ typeEnumerator: TypeEnumerator) { + withoutActuallyEscaping(typeEnumerator) { typeEnumerator in + withUnsafePointer(to: typeEnumerator) { context in + swt_enumerateTypes(withNamesContaining: nameSubstring, .init(mutating: context)) { imageAddress, type, stop, context in + let typeEnumerator = context!.load(as: TypeEnumerator.self) + let type = unsafeBitCast(type, to: Any.Type.self) + var stop2 = false + typeEnumerator(imageAddress, type, &stop2) + stop.pointee = stop2 + } + } + } +} diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 389d4cc92..9a187c917 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,71 +10,73 @@ private import _TestingInternals -/// A protocol describing a type that contains tests. -/// -/// - Warning: This protocol is used to implement the `@Test` macro. Do not use -/// it directly. -@_alwaysEmitConformanceMetadata -public protocol __TestContainer { - /// The set of tests contained by this type. - static var __tests: [Test] { get async } -} +extension Test: TestContent { + static var testContentKind: UInt32 { + 0x74657374 + } -extension Test { - /// A string that appears within all auto-generated types conforming to the - /// `__TestContainer` protocol. - private static let _testContainerTypeNameMagic = "__🟠$test_container__" + typealias TestContentAccessorResult = @Sendable () async -> Self /// All available ``Test`` instances in the process, according to the runtime. /// /// The order of values in this sequence is unspecified. static var all: some Sequence { get async { - await withTaskGroup(of: [Self].self) { taskGroup in - enumerateTypes(withNamesContaining: _testContainerTypeNameMagic) { _, type, _ in - if let type = type as? any __TestContainer.Type { - taskGroup.addTask { - await type.__tests - } - } - } + var generators = [@Sendable () async -> [Self]]() - return await taskGroup.reduce(into: [], +=) + // Figure out which discovery mechanism to use. By default, we'll use both + // the legacy and new mechanisms, but we can set an environment variable + // to explicitly select one or the other. When we remove legacy support, + // we can also remove this enumeration and environment variable check. + enum DiscoveryMode { + case tryBoth + case newOnly + case legacyOnly + } + let discoveryMode: DiscoveryMode = switch Environment.flag(named: "SWT_USE_LEGACY_TEST_DISCOVERY") { + case .none: + .tryBoth + case .some(true): + .legacyOnly + case .some(false): + .newOnly } - } - } -} -// MARK: - + // Walk all test content and gather generator functions. Note we don't + // actually call the generators yet because enumerating test content may + // involve holding some internal lock such as the ones in libobjc or + // dl_iterate_phdr(), and we don't want to accidentally deadlock if the + // user code we call ends up loading another image. + if discoveryMode != .legacyOnly { + enumerateTestContent { imageAddress, generator, _, _ in + generators.append { @Sendable in + await [generator()] + } + } + } -/// The type of callback called by ``enumerateTypes(withNamesContaining:_:)``. -/// -/// - Parameters: -/// - imageAddress: A pointer to the start of the image. This value is _not_ -/// equal to the value returned from `dlopen()`. On platforms that do not -/// support dynamic loading (and so do not have loadable images), this -/// argument is unspecified. -/// - type: A Swift type. -/// - stop: An `inout` boolean variable indicating whether type enumeration -/// should stop after the function returns. Set `stop` to `true` to stop -/// type enumeration. -typealias TypeEnumerator = (_ imageAddress: UnsafeRawPointer?, _ type: Any.Type, _ stop: inout Bool) -> Void + if discoveryMode != .newOnly && generators.isEmpty { + enumerateTypes(withNamesContaining: testContainerTypeNameMagic) { imageAddress, type, _ in + guard let type = type as? any __TestContainer.Type else { + return + } + generators.append { @Sendable in + await type.__tests + } + } + } -/// Enumerate all types known to Swift found in the current process whose names -/// contain a given substring. -/// -/// - Parameters: -/// - nameSubstring: A string which the names of matching classes all contain. -/// - body: A function to invoke, once per matching type. -func enumerateTypes(withNamesContaining nameSubstring: String, _ typeEnumerator: TypeEnumerator) { - withoutActuallyEscaping(typeEnumerator) { typeEnumerator in - withUnsafePointer(to: typeEnumerator) { context in - swt_enumerateTypes(withNamesContaining: nameSubstring, .init(mutating: context)) { imageAddress, type, stop, context in - let typeEnumerator = context!.load(as: TypeEnumerator.self) - let type = unsafeBitCast(type, to: Any.Type.self) - var stop2 = false - typeEnumerator(imageAddress, type, &stop2) - stop.pointee = stop2 + // *Now* we call all the generators and return their results. + // Reduce into a set rather than an array to deduplicate tests that were + // generated multiple times (e.g. from multiple discovery modes or from + // defective test records.) + return await withTaskGroup(of: [Self].self) { taskGroup in + for generator in generators { + taskGroup.addTask { + await generator() + } + } + return await taskGroup.reduce(into: Set()) { $0.formUnion($1) } } } } diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index baf4ebd90..8af5e1690 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -10,6 +10,31 @@ #include "Discovery.h" +#if defined(SWT_NO_DYNAMIC_LINKING) +#pragma mark - Statically-linked section bounds + +#if defined(__APPLE__) +extern "C" const char testContentSectionBegin __asm("section$start$__DATA_CONST$__swift5_tests"); +extern "C" const char testContentSectionEnd __asm("section$end$__DATA_CONST$__swift5_tests"); +#elif defined(__wasi__) +extern "C" const char testContentSectionBegin __asm__("__start_swift5_tests"); +extern "C" const char testContentSectionEnd __asm__("__stop_swift5_tests"); +#else +#warning Platform-specific implementation missing: Runtime test discovery unavailable (static) +static const char testContentSectionBegin = 0; +static const char& testContentSectionEnd = testContentSectionBegin; +#endif + +/// The bounds of the test content section statically linked into the image +/// containing Swift Testing. +const void *_Nonnull const SWTTestContentSectionBounds[2] = { + &testContentSectionBegin, + &testContentSectionEnd +}; +#endif + +#pragma mark - Legacy test discovery + #include #include #include diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index d12f623ee..9d7a5a6e9 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -16,6 +16,62 @@ SWT_ASSUME_NONNULL_BEGIN +#pragma mark - Test content records + +/// The type of a test content accessor. +/// +/// - Parameters: +/// - outValue: On successful return, initialized to the value of the +/// represented test content record. +/// - hint: A hint value whose type and meaning depend on the type of test +/// record being accessed. +/// +/// - Returns: Whether or not the test record was initialized at `outValue`. If +/// this function returns `true`, the caller is responsible for deinitializing +/// the memory at `outValue` when done. +typedef bool (* SWTTestContentAccessor)(void *outValue, const void *_Null_unspecified hint); + +/// Resign an accessor function from a test content record. +/// +/// - Parameters: +/// - accessor: The accessor function to resign. +/// +/// - Returns: A resigned copy of `accessor` on platforms that use pointer +/// authentication, and an exact copy of `accessor` elsewhere. +/// +/// - Bug: This C function is needed because Apple's pointer authentication +/// intrinsics are not available in Swift. ([141465242](rdar://141465242)) +SWT_SWIFT_NAME(swt_resign(_:)) +static SWTTestContentAccessor swt_resignTestContentAccessor(SWTTestContentAccessor accessor) { +#if defined(__APPLE__) && __has_include() + accessor = ptrauth_strip(accessor, ptrauth_key_function_pointer); + accessor = ptrauth_sign_unauthenticated(accessor, ptrauth_key_function_pointer, 0); +#endif + return accessor; +} + +#if defined(__ELF__) && defined(__swift__) +/// A function exported by the Swift runtime that enumerates all metadata +/// sections loaded into the current process. +/// +/// This function is needed on ELF-based platforms because they do not preserve +/// section information that we can discover at runtime. +SWT_IMPORT_FROM_STDLIB void swift_enumerateAllMetadataSections( + bool (* body)(const void *sections, void *context), + void *context +); +#endif + +#if defined(SWT_NO_DYNAMIC_LINKING) +#pragma mark - Statically-linked section bounds + +/// The bounds of the test content section statically linked into the image +/// containing Swift Testing. +SWT_EXTERN const void *_Nonnull const SWTTestContentSectionBounds[2]; +#endif + +#pragma mark - Legacy test discovery + /// The type of callback called by `swt_enumerateTypes()`. /// /// - Parameters: diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index b1f4c7973..dfcbf50f0 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -127,6 +127,10 @@ #if !SWT_NO_LIBDISPATCH #include #endif + +#if __has_include() +#include +#endif #endif #if defined(__FreeBSD__) diff --git a/Sources/_TestingInternals/include/Stubs.h b/Sources/_TestingInternals/include/Stubs.h index caeb7c493..303cf0c46 100644 --- a/Sources/_TestingInternals/include/Stubs.h +++ b/Sources/_TestingInternals/include/Stubs.h @@ -91,6 +91,14 @@ static LANGID swt_MAKELANGID(int p, int s) { static DWORD_PTR swt_PROC_THREAD_ATTRIBUTE_HANDLE_LIST(void) { return PROC_THREAD_ATTRIBUTE_HANDLE_LIST; } + +/// Get the first section in an NT image. +/// +/// This function is provided because `IMAGE_FIRST_SECTION()` is a complex macro +/// and cannot be imported directly into Swift. +static const IMAGE_SECTION_HEADER *_Null_unspecified swt_IMAGE_FIRST_SECTION(const IMAGE_NT_HEADERS *ntHeader) { + return IMAGE_FIRST_SECTION(ntHeader); +} #endif #if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__ANDROID__) diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index f259fc8cf..86ede749e 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -52,6 +52,7 @@ struct ABIEntryPointTests { passing arguments: __CommandLineArguments_v0, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void = { _ in } ) async throws -> CInt { +#if !SWT_NO_DYNAMIC_LINKING // Get the ABI entry point by dynamically looking it up at runtime. let copyABIEntryPoint_v0 = try withTestingLibraryImageAddress { testingLibrary in try #require( @@ -60,6 +61,7 @@ struct ABIEntryPointTests { } ) } +#endif let abiEntryPoint = copyABIEntryPoint_v0().assumingMemoryBound(to: ABIEntryPoint_v0.self) defer { abiEntryPoint.deinitialize(count: 1) diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 02f2cc768..3c987a9ad 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -9,6 +9,7 @@ // @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +private import _TestingInternals @Test(/* name unspecified */ .hidden) @Sendable func freeSyncFunction() {} @@ -569,4 +570,77 @@ struct MiscellaneousTests { } #expect(duration < .seconds(1)) } + +#if !SWT_NO_DYNAMIC_LINKING && hasFeature(SymbolLinkageMarkers) + struct DiscoverableTestContent: TestContent { + typealias TestContentAccessorHint = UInt32 + typealias TestContentAccessorResult = UInt32 + + static var testContentKind: UInt32 { + record.kind + } + + static var expectedHint: TestContentAccessorHint { + 0x01020304 + } + + static var expectedResult: TestContentAccessorResult { + 0xCAFEF00D + } + + static var expectedContext: UInt { + record.context + } + +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) + @_section("__DATA_CONST,__swift5_tests") +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) + @_section("swift5_tests") +#elseif os(Windows) + @_section(".sw5test$B") +#endif + @_used + private static let record: __TestContentRecord = ( + 0xABCD1234, + 0, + { outValue, hint in + if let hint, hint.loadUnaligned(as: TestContentAccessorHint.self) != expectedHint { + return false + } + _ = outValue.initializeMemory(as: TestContentAccessorResult.self, to: expectedResult) + return true + }, + UInt(UInt64(0x0204060801030507) & UInt64(UInt.max)), + 0 + ) + } + + @Test func testDiscovery() async { + await confirmation("Can find a single test record") { found in + DiscoverableTestContent.enumerateTestContent { _, value, context, _ in + if value == DiscoverableTestContent.expectedResult && context == DiscoverableTestContent.expectedContext { + found() + } + } + } + + await confirmation("Can find a test record with matching hint") { found in + let hint = DiscoverableTestContent.expectedHint + DiscoverableTestContent.enumerateTestContent(withHint: hint) { _, value, context, _ in + if value == DiscoverableTestContent.expectedResult && context == DiscoverableTestContent.expectedContext { + found() + } + } + } + + await confirmation("Doesn't find a test record with a mismatched hint", expectedCount: 0) { found in + let hint = ~DiscoverableTestContent.expectedHint + DiscoverableTestContent.enumerateTestContent(withHint: hint) { _, value, context, _ in + if value == DiscoverableTestContent.expectedResult && context == DiscoverableTestContent.expectedContext { + found() + } + } + } + } +#endif } From 6f7688aec3afb935583565e4c0b4c80c6ca88950 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 7 Jan 2025 14:05:42 -0500 Subject: [PATCH 037/234] Add an attribute that emits a warning. (#895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds an attribute (an overload of `@__testing()`) that emits a warning. This allows us to emit compile-time warnings from contexts where only attributes are semantically valid. We need this in particular for test content records because their section info is comprised of a big `IfConfigDecl` node and platforms that aren't covered (i.e. the `#else` clause) don't have a way to emit a diagnostic that says "we need to fix this platform's Swift Testing port!" like we can do in other contexts with `#warning()`: ```swift #if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) @_section("__DATA_CONST,__swift5_tests") #elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) @_section("swift5_tests") #elseif os(Windows) @_section(".sw5test$B") #else // ⚠️ #warning isn't valid here, what do we do!? #endif @_used private static let record: __TestContentRecord = (...) ``` ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Test+Macro.swift | 16 ++++++++++++++-- Sources/TestingMacros/PragmaMacro.swift | 15 ++++++++++++++- .../TestingMacrosTests/PragmaMacroTests.swift | 19 ++++++++++++++++--- .../TestSupport/Parse.swift | 1 + Tests/TestingTests/MiscellaneousTests.swift | 2 ++ 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index 0fb29562d..86fb42c14 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -499,12 +499,24 @@ extension Test { /// - Note: This macro has compile-time effects _only_ and should not affect a /// compiled test target. /// -/// - Warning: This macro is used to implement other macros declared by the testing -/// library. Do not use it directly. +/// - Warning: This macro is used to implement other macros declared by the +/// testing library. Do not use it directly. @attached(peer) public macro __testing( semantics arguments: _const String... ) = #externalMacro(module: "TestingMacros", type: "PragmaMacro") +/// A macro used similarly to `#warning()` but in a position where only an +/// attribute is valid. +/// +/// - Parameters: +/// - message: A string to emit as a warning. +/// +/// - Warning: This macro is used to implement other macros declared by the +/// testing library. Do not use it directly. +@attached(peer) public macro __testing( + warning message: _const String +) = #externalMacro(module: "TestingMacros", type: "PragmaMacro") + // MARK: - Helper functions /// A function that abstracts away whether or not the `try` keyword is needed on diff --git a/Sources/TestingMacros/PragmaMacro.swift b/Sources/TestingMacros/PragmaMacro.swift index 48027b213..783440764 100644 --- a/Sources/TestingMacros/PragmaMacro.swift +++ b/Sources/TestingMacros/PragmaMacro.swift @@ -17,7 +17,9 @@ public import SwiftSyntaxMacros /// /// - `@__testing(semantics: "nomacrowarnings")`: suppress warning diagnostics /// generated by macros. (The implementation of this use case is held in trust -/// at ``MacroExpansionContext/areWarningsSuppressed``. +/// at ``MacroExpansionContext/areWarningsSuppressed``.) +/// - `@__testing(warning: "...")`: emits `"..."` as a diagnostic message +/// attributed to the node to which the attribute is attached. /// /// This type is used to implement the `@__testing` attribute macro. Do not use /// it directly. @@ -27,6 +29,17 @@ public struct PragmaMacro: PeerMacro, Sendable { providingPeersOf declaration: some DeclSyntaxProtocol, in context: some MacroExpansionContext ) throws -> [DeclSyntax] { + if case let .argumentList(arguments) = node.arguments, + arguments.first?.label?.textWithoutBackticks == "warning" { + let targetNode = Syntax(declaration) + let messages = arguments + .map(\.expression) + .compactMap { $0.as(StringLiteralExprSyntax.self) } + .compactMap(\.representedLiteralValue) + .map { DiagnosticMessage(syntax: targetNode, message: $0, severity: .warning) } + context.diagnose(messages) + } + return [] } diff --git a/Tests/TestingMacrosTests/PragmaMacroTests.swift b/Tests/TestingMacrosTests/PragmaMacroTests.swift index 9e85419da..0d430c036 100644 --- a/Tests/TestingMacrosTests/PragmaMacroTests.swift +++ b/Tests/TestingMacrosTests/PragmaMacroTests.swift @@ -18,12 +18,25 @@ import SwiftSyntax struct PragmaMacroTests { @Test func findSemantics() throws { let node = """ - @Testing.__testing(semantics: "abc123") - @__testing(semantics: "def456") - let x = 0 + @Testing.__testing(semantics: "abc123") + @__testing(semantics: "def456") + let x = 0 """ as DeclSyntax let nodeWithAttributes = try #require(node.asProtocol((any WithAttributesSyntax).self)) let semantics = semantics(of: nodeWithAttributes) #expect(semantics == ["abc123", "def456"]) } + + @Test func warningGenerated() throws { + let sourceCode = """ + @__testing(warning: "abc123") + let x = 0 + """ + + let (_, diagnostics) = try parse(sourceCode) + #expect(diagnostics.count == 1) + #expect(diagnostics[0].message == "abc123") + #expect(diagnostics[0].diagMessage.severity == .warning) + #expect(diagnostics[0].node.is(VariableDeclSyntax.self)) + } } diff --git a/Tests/TestingMacrosTests/TestSupport/Parse.swift b/Tests/TestingMacrosTests/TestSupport/Parse.swift index e6b36e3b2..ecff8de58 100644 --- a/Tests/TestingMacrosTests/TestSupport/Parse.swift +++ b/Tests/TestingMacrosTests/TestSupport/Parse.swift @@ -30,6 +30,7 @@ fileprivate let allMacros: [String: any Macro.Type] = [ "Suite": SuiteDeclarationMacro.self, "Test": TestDeclarationMacro.self, "Tag": TagMacro.self, + "__testing": PragmaMacro.self, ] func parse(_ sourceCode: String, activeMacros activeMacroNames: [String] = [], removeWhitespace: Bool = false) throws -> (sourceCode: String, diagnostics: [Diagnostic]) { diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 3c987a9ad..a8cd56a7b 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -598,6 +598,8 @@ struct MiscellaneousTests { @_section("swift5_tests") #elseif os(Windows) @_section(".sw5test$B") +#else + @__testing(warning: "Platform-specific implementation missing: test content section name unavailable") #endif @_used private static let record: __TestContentRecord = ( From 200fdd7e163afc05d284bbf2d8553eaeab52969a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 7 Jan 2025 15:24:24 -0500 Subject: [PATCH 038/234] Add a missing `#if !SWT_NO_EXIT_TESTS`. (#896) This PR adds a missing `#if !SWT_NO_EXIT_TESTS` statement in order to resolve a build failure on platforms that do not support exit tests (such as WASI.) ### Checklist: - [ ] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [ ] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Test+Discovery+Legacy.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index 746c4128f..b65d72b39 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -24,6 +24,7 @@ public protocol __TestContainer { /// `__TestContainer` protocol. let testContainerTypeNameMagic = "__🟠$test_container__" +#if !SWT_NO_EXIT_TESTS /// A protocol describing a type that contains an exit test. /// /// - Warning: This protocol is used to implement the `#expect(exitsWith:)` @@ -44,6 +45,7 @@ public protocol __ExitTestContainer { /// A string that appears within all auto-generated types conforming to the /// `__ExitTestContainer` protocol. let exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" +#endif // MARK: - From de09c23167c018ef89661241ef0d37a9ef2f89a5 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 7 Jan 2025 15:18:05 -0600 Subject: [PATCH 039/234] [SWT-NNNN] Introduce API allowing traits to customize test execution (#733) This includes an API proposal and code changes to introduce new API for custom traits to customize test execution. View the [API proposal](https://github.com/stmontgomery/swift-testing/blob/publicize-CustomExecutionTrait/Documentation/Proposals/NNNN-custom-test-execution-traits.md) for more details. ### Motivation: One of the primary motivations for the trait system in Swift Testing, as [described in the vision document](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md#trait-extensibility), is to provide a way to customize the behavior of tests which have things in common. If all the tests in a given suite type need the same custom behavior, `init` and/or `deinit` (if applicable) can be used today. But if only _some_ of the tests in a suite need custom behavior, or tests across different levels of the suite hierarchy need it, traits would be a good place to encapsulate common logic since they can be applied granularly per-test or per-suite. This aspect of the vision for traits hasn't been realized yet, though: the `Trait` protocol does not offer a way for a trait to customize the execution of the tests or suites it's applied to. Customizing a test's behavior typically means running code either before or after it runs, or both. Consolidating common set-up and tear-down logic allows each test function to be more succinct with less repetitive boilerplate so it can focus on what makes it unique. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../NNNN-custom-test-execution-traits.md | 510 ++++++++++++++++++ Sources/Testing/Running/Runner.swift | 41 +- Sources/Testing/Testing.docc/Traits.md | 1 + Sources/Testing/Testing.docc/Traits/Trait.md | 7 + Sources/Testing/Traits/Trait.swift | 197 +++++-- .../Traits/CustomExecutionTraitTests.swift | 104 ---- .../Traits/TestScopingTraitTests.swift | 184 +++++++ 7 files changed, 883 insertions(+), 161 deletions(-) create mode 100644 Documentation/Proposals/NNNN-custom-test-execution-traits.md delete mode 100644 Tests/TestingTests/Traits/CustomExecutionTraitTests.swift create mode 100644 Tests/TestingTests/Traits/TestScopingTraitTests.swift diff --git a/Documentation/Proposals/NNNN-custom-test-execution-traits.md b/Documentation/Proposals/NNNN-custom-test-execution-traits.md new file mode 100644 index 000000000..731387b24 --- /dev/null +++ b/Documentation/Proposals/NNNN-custom-test-execution-traits.md @@ -0,0 +1,510 @@ +# Test Scoping Traits + +* Proposal: [SWT-NNNN](NNNN-filename.md) +* Authors: [Stuart Montgomery](https://github.com/stmontgomery) +* Status: **Awaiting review** +* Implementation: [swiftlang/swift-testing#733](https://github.com/swiftlang/swift-testing/pull/733), [swiftlang/swift-testing#86](https://github.com/swiftlang/swift-testing/pull/86) +* Review: ([pitch](https://forums.swift.org/t/pitch-custom-test-execution-traits/75055)) + +### Revision history + +* **v1**: Initial pitch. +* **v2**: Dropped 'Custom' prefix from the proposed API names (although kept the + word in certain documentation passages where it clarified behavior). +* **v3**: Changed the `Trait` requirement from a property to a method which + accepts the test and/or test case, and modify its default implementations such + that custom behavior is either performed per-suite or per-test case by default. +* **v4**: Renamed the APIs to use "scope" as the base verb instead of "execute". + +## Introduction + +This introduces API which enables a `Trait`-conforming type to provide a custom +execution scope for test functions and suites, including running code before or +after them. + +## Motivation + +One of the primary motivations for the trait system in Swift Testing, as +[described in the vision document](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md#trait-extensibility), +is to provide a way to customize the behavior of tests which have things in +common. If all the tests in a given suite type need the same custom behavior, +`init` and/or `deinit` (if applicable) can be used today. But if only _some_ of +the tests in a suite need custom behavior, or tests across different levels of +the suite hierarchy need it, traits would be a good place to encapsulate common +logic since they can be applied granularly per-test or per-suite. This aspect of +the vision for traits hasn't been realized yet, though: the `Trait` protocol +does not offer a way for a trait to customize the execution of the tests or +suites it's applied to. + +Customizing a test's behavior typically means running code either before or +after it runs, or both. Consolidating common set-up and tear-down logic allows +each test function to be more succinct with less repetitive boilerplate so it +can focus on what makes it unique. + +## Proposed solution + +At a high level, this proposal entails adding API to the `Trait` protocol +allowing a conforming type to opt-in to providing a custom execution scope for a +test. We discuss how that capability should be exposed to trait types below. + +### Supporting scoped access + +There are different approaches one could take to expose hooks for a trait to +customize test behavior. To illustrate one of them, consider the following +example of a `@Test` function with a custom trait whose purpose is to set mock +API credentials for the duration of each test it's applied to: + +```swift +@Test(.mockAPICredentials) +func example() { + // ... +} + +struct MockAPICredentialsTrait: TestTrait { ... } + +extension Trait where Self == MockAPICredentialsTrait { + static var mockAPICredentials: Self { ... } +} +``` + +In this hypothetical example, the current API credentials are stored via a +static property on an `APICredentials` type which is part of the module being +tested: + +```swift +struct APICredentials { + var apiKey: String + + static var shared: Self? +} +``` + +One way that this custom trait could customize the API credentials during each +test is if the `Trait` protocol were to expose a pair of method requirements +which were then called before and after the test, respectively: + +```swift +public protocol Trait: Sendable { + // ... + func setUp() async throws + func tearDown() async throws +} + +extension Trait { + // ... + public func setUp() async throws { /* No-op */ } + public func tearDown() async throws { /* No-op */ } +} +``` + +The custom trait type could adopt these using code such as the following: + +```swift +extension MockAPICredentialsTrait { + func setUp() { + APICredentials.shared = .init(apiKey: "...") + } + + func tearDown() { + APICredentials.shared = nil + } +} +``` + +Many testing systems use this pattern, including XCTest. However, this approach +encourages the use of global mutable state such as the `APICredentials.shared` +variable, and this limits the testing library's ability to parallelize test +execution, which is +[another part of the Swift Testing vision](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md#parallelization-and-concurrency). + +The use of nonisolated static variables is generally discouraged now, and in +Swift 6 the above `APICredentials.shared` property produces an error. One way +to resolve that is to change it to a `@TaskLocal` variable, as this would be +concurrency-safe and still allow tests accessing this state to run in parallel: + +```swift +extension APICredentials { + @TaskLocal static var current: Self? +} +``` + +Binding task local values requires using the scoped access +[`TaskLocal.withValue()`](https://developer.apple.com/documentation/swift/tasklocal/withvalue(_:operation:isolation:file:line:)) +API though, and that would not be possible if `Trait` exposed separate methods +like `setUp()` and `tearDown()`. + +For these reasons, I believe it's important to expose this trait capability +using a single, scoped access-style API which accepts a closure. A simplified +version of that idea might look like this: + +```swift +public protocol Trait: Sendable { + // ... + + // Simplified example, not the actual proposal + func executeTest(_ body: @Sendable () async throws -> Void) async throws +} + +extension MockAPICredentialsTrait { + func executeTest(_ body: @Sendable () async throws -> Void) async throws { + let mockCredentials = APICredentials(apiKey: "...") + try await APICredentials.$current.withValue(mockCredentials) { + try await body() + } + } +} +``` + +### Avoiding unnecessarily lengthy backtraces + +A scoped access-style API has some potential downsides. To apply this approach +to a test function, the scoped call of a trait must wrap the invocation of that +test function, and every _other_ trait applied to that same test which offers +custom behavior _also_ must wrap the other traits' calls in a nesting fashion. +To visualize this, imagine a test function with multiple traits: + +```swift +@Test(.traitA, .traitB, .traitC) +func exampleTest() { + // ... +} +``` + +If all three of those traits provide a custom scope for tests, then each of them +needs to wrap the call to the next one, and the last trait needs to wrap the +invocation of the test, illustrated by the following: + +``` +TraitA.executeTest { + TraitB.executeTest { + TraitC.executeTest { + exampleTest() + } + } +} +``` + +Tests may have an arbitrary number of traits applied to them, including those +inherited from containing suite types. A naïve implementation in which _every_ +trait is given the opportunity to customize test behavior by calling its scoped +access API might cause unnecessarily lengthy backtraces that make debugging the +body of tests more difficult. Or worse: if the number of traits is great enough, +it could cause a stack overflow. + +In practice, most traits probably do _not_ need to provide a custom scope for +the tests they're applied to, so to mitigate these downsides it's important that +there be some way to distinguish traits which customize test behavior. That way, +the testing library can limit these scoped access calls to only traits which +need it. + +### Avoiding unnecessary (re-)execution + +Traits can be applied to either test functions or suites, and traits applied to +suites can optionally support inheritance by implementing the `isRecursive` +property of the `SuiteTrait` protocol. When a trait is directly applied to a +test function, if the trait customizes the behavior of tests it's applied to, it +should be given the opportunity to perform its custom behavior once for every +invocation of that test function. In particular, if the test function is +parameterized and runs multiple times, then the trait applied to it should +perform its custom behavior once for every invocation. This should not be +surprising to users, since it's consistent with the behavior of `init` and +`deinit` for an instance `@Test` method. + +It may be useful for certain kinds of traits to perform custom logic once for +_all_ the invocations of a parameterized test. Although this should be possible, +we believe it shouldn't be the default since it could lead to work being +repeated multiple times needlessly, or unintentional state sharing across tests, +unless the trait is implemented carefully to avoid those problems. + +When a trait conforms to `SuiteTrait` and is applied to a suite, the question of +when its custom scope (if any) should be applied is less obvious. Some suite +traits support inheritance and are recursively applied to all the test functions +they contain (including transitively, via sub-suites). Other suite traits don't +support inheritance, and only affect the specific suite they're applied to. +(It's also worth noting that a sub-suite _can_ have the same non-recursive suite +trait one of its ancestors has, as long as it's applied explicitly.) + +As a general rule of thumb, we believe most traits will either want to perform +custom logic once for _all_ children or once for _each_ child, not both. +Therefore, when it comes to suite traits, the default behavior should depend on +whether it supports inheritance: a recursive suite trait should by default +perform custom logic before each test, and a non-recursive one per-suite. But +the APIs should be flexible enough to support both, for advanced traits which +need it. + +## Detailed design + +I propose the following new APIs: + +- A new protocol `TestScoping` with a single required `provideScope(...)` method. + This will be called to provide scope for a test, and allows the conforming + type to perform custom logic before or after. +- A new method `scopeProvider(for:testCase:)` on the `Trait` protocol whose + result type is an `Optional` value of a type conforming to `TestScoping`. A + `nil` value returned by this method will skip calling the `provideScope(...)` + method. +- A default implementation of `Trait.scopeProvider(...)` which returns `nil`. +- A conditional implementation of `Trait.scopeProvider(...)` which returns `self` + in the common case where the trait type conforms to `TestScoping` itself. + +Since the `scopeProvider(...)` method's return type is optional and returns `nil` +by default, the testing library cannot invoke the `provideScope(...)` method +unless a trait customizes test behavior. This avoids the "unnecessarily lengthy +backtraces" problem above. + +Below are the proposed interfaces: + +```swift +/// A protocol that allows providing a custom execution scope for a test +/// function (and each of its cases) or a test suite by performing custom code +/// before or after it runs. +/// +/// Types conforming to this protocol may be used in conjunction with a +/// ``Trait``-conforming type by implementing the +/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, allowing custom traits +/// to provide custom scope for tests. Consolidating common set-up and tear-down +/// logic for tests which have similar needs allows each test function to be +/// more succinct with less repetitive boilerplate so it can focus on what makes +/// it unique. +public protocol TestScoping: Sendable { + /// Provide custom execution scope for a function call which is related to the + /// specified test and/or test case. + /// + /// - Parameters: + /// - test: The test under which `function` is being performed. + /// - testCase: The test case, if any, under which `function` is being + /// performed. When invoked on a suite, the value of this argument is + /// `nil`. + /// - function: The function to perform. If `test` represents a test suite, + /// this function encapsulates running all the tests in that suite. If + /// `test` represents a test function, this function is the body of that + /// test function (including all cases if it is parameterized.) + /// + /// - Throws: Whatever is thrown by `function`, or an error preventing this + /// type from providing a custom scope correctly. An error thrown from this + /// method is recorded as an issue associated with `test`. If an error is + /// thrown before `function` is called, the corresponding test will not run. + /// + /// When the testing library is preparing to run a test, it starts by finding + /// all traits applied to that test, including those inherited from containing + /// suites. It begins with inherited suite traits, sorting them + /// outermost-to-innermost, and if the test is a function, it then adds all + /// traits applied directly to that functions in the order they were applied + /// (left-to-right). It then asks each trait for its scope provider (if any) + /// by calling ``Trait/scopeProvider(for:testCase:)-cjmg``. Finally, it calls + /// this method on all non-`nil` scope providers, giving each an opportunity + /// to perform arbitrary work before or after invoking `function`. + /// + /// This method should either invoke `function` once before returning or throw + /// an error if it is unable to provide a custom scope. + /// + /// Issues recorded by this method are associated with `test`. + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws +} + +public protocol Trait: Sendable { + // ... + + /// The type of the test scope provider for this trait. + /// + /// The default type is `Never`, which cannot be instantiated. The + /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with this + /// default type must return `nil`, meaning that trait will not provide a + /// custom scope for the tests it's applied to. + associatedtype TestScopeProvider: TestScoping = Never + + /// Get this trait's scope provider for the specified test and/or test case, + /// if any. + /// + /// - Parameters: + /// - test: The test for which a scope provider is being requested. + /// - testCase: The test case for which a scope provider is being requested, + /// if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// - Returns: A value conforming to ``Trait/TestScopeProvider`` which may be + /// used to provide custom scoping for `test` and/or `testCase`, or `nil` if + /// they should not have any custom scope. + /// + /// If this trait's type conforms to ``TestScoping``, the default value + /// returned by this method depends on `test` and/or `testCase`: + /// + /// - If `test` represents a suite, this trait must conform to ``SuiteTrait``. + /// If the value of this suite trait's ``SuiteTrait/isRecursive`` property + /// is `true`, then this method returns `nil`; otherwise, it returns `self`. + /// This means that by default, a suite trait will _either_ provide its + /// custom scope once for the entire suite, or once per-test function it + /// contains. + /// - Otherwise `test` represents a test function. If `testCase` is `nil`, + /// this method returns `nil`; otherwise, it returns `self`. This means that + /// by default, a trait which is applied to or inherited by a test function + /// will provide its custom scope once for each of that function's cases. + /// + /// A trait may explicitly implement this method to further customize the + /// default behaviors above. For example, if a trait should provide custom + /// test scope both once per-suite and once per-test function in that suite, + /// it may implement the method and return a non-`nil` scope provider under + /// those conditions. + /// + /// A trait may also implement this method and return `nil` if it determines + /// that it does not need to provide a custom scope for a particular test at + /// runtime, even if the test has the trait applied. This can improve + /// performance and make diagnostics clearer by avoiding an unnecessary call + /// to ``TestScoping/provideScope(for:testCase:performing:)``. + /// + /// If this trait's type does not conform to ``TestScoping`` and its + /// associated ``Trait/TestScopeProvider`` type is the default `Never`, then + /// this method returns `nil` by default. This means that instances of this + /// trait will not provide a custom scope for tests to which they're applied. + func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider? +} + +extension Trait where Self: TestScoping { + // Returns `nil` if `testCase` is `nil`, else `self`. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? +} + +extension SuiteTrait where Self: TestScoping { + // If `test` is a suite, returns `nil` if `isRecursive` is `true`, else `self`. + // Otherwise, `test` is a function and this returns `nil` if `testCase` is + // `nil`, else `self`. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? +} + +extension Trait where TestScopeProvider == Never { + // Returns `nil`. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? +} + +extension Never: TestScoping {} +``` + +Here is a complete example of the usage scenario described earlier, showcasing +the proposed APIs: + +```swift +@Test(.mockAPICredentials) +func example() { + // ...validate API usage, referencing `APICredentials.current`... +} + +struct MockAPICredentialsTrait: TestTrait, TestScoping { + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + let mockCredentials = APICredentials(apiKey: "...") + try await APICredentials.$current.withValue(mockCredentials) { + try await function() + } + } +} + +extension Trait where Self == MockAPICredentialsTrait { + static var mockAPICredentials: Self { + Self() + } +} +``` + +## Source compatibility + +The proposed APIs are purely additive. + +This proposal will replace the existing `CustomExecutionTrait` SPI, and after +further refactoring we anticipate it will obsolete the need for the +`SPIAwareTrait` SPI as well. + +## Integration with supporting tools + +Although some built-in traits are relevant to supporting tools (such as +SourceKit-LSP statically discovering `.tags` traits), custom test behaviors are +only relevant within the test executable process while tests are running. We +don't anticipate any particular need for this feature to integrate with +supporting tools. + +## Future directions + +### Access to suite type instances + +Some test authors have expressed interest in allowing custom traits to access +the instance of a suite type for `@Test` instance methods, so the trait could +inspect or mutate the instance. Currently, only instance-level members of a +suite type (including `init`, `deinit`, and the test function itself) can access +`self`, so this would grant traits applied to an instance test method access to +the instance as well. This is certainly interesting, but poses several technical +challenges that puts it out of scope of this proposal. + +### Convenience trait for setting task locals + +Some reviewers of this proposal pointed out that the hypothetical usage example +shown earlier involving setting a task local value while a test is executing +will likely become a common use of these APIs. To streamline that pattern, it +would be very natural to add a built-in trait type which facilitates this. I +have prototyped this idea and plan to add it once this new trait functionality +lands. + +## Alternatives considered + +### Separate set up & tear down methods on `Trait` + +This idea was discussed in [Supporting scoped access](#supporting-scoped-access) +above, and as mentioned there, the primary problem with this approach is that it +cannot be used with scoped access-style APIs, including (importantly) +`TaskLocal.withValue()`. For that reason, it prevents using that common Swift +concurrency technique and reduces the potential for test parallelization. + +### Add `provideScope(...)` directly to the `Trait` protocol + +The proposed `provideScope(...)` method could be added as a requirement of the +`Trait` protocol instead of being part of a separate `TestScoping` protocol, and +it could have a default implementation which directly invokes the passed-in +closure. But this approach would suffer from the lengthy backtrace problem +described above. + +### Extend the `Trait` protocol + +The original, experimental implementation of this feature included a protocol +named`CustomExecutionTrait` which extended `Trait` and had roughly the same +method requirement as the `TestScoping` protocol proposed above. This design +worked, provided scoped access, and avoided the lengthy backtrace problem. + +After evaluating the design and usage of this SPI though, it seemed unfortunate +to structure it as a sub-protocol of `Trait` because it means that the full +capabilities of the trait system are spread across multiple protocols. In the +proposed design, the ability to return a test scoping provider is exposed via +the main `Trait` protocol, and it relies on an associated type to conditionally +opt-in to custom test behavior. In other words, the proposed design expresses +custom test behavior as just a _capability_ that a trait may have, rather than a +distinct sub-type of trait. + +Also, the implementation of this approach within the testing library was not +ideal as it required a conditional `trait as? CustomExecutionTrait` downcast at +runtime, in contrast to the simpler and more performant Optional property of the +proposed API. + +### API names + +We first considered "execute" as the base verb for the proposed new concept, but +felt this wasn't appropriate since these trait types are not "the executor" of +tests, they merely customize behavior and provide scope(s) for tests to run +within. Also, the term "executor" has prior art in Swift Concurrency, and +although that word is used in other contexts too, it may be helpful to avoid +potential confusion with concurrency executors. + +We also considered "run" as the base verb for the proposed new concept instead +of "execute", which would imply the names `TestRunning`, `TestRunner`, +`runner(for:testCase)`, and `run(_:for:testCase:)`. The word "run" is used in +many other contexts related to testing though, such as the `Runner` SPI type and +more casually to refer to a run which occurred of a test, in the past tense, so +overloading this term again may cause confusion. + +## Acknowledgments + +Thanks to [Dennis Weissmann](https://github.com/dennisweissmann) for originally +implementing this as SPI, and for helping promote its usefulness. + +Thanks to [Jonathan Grynspan](https://github.com/grynspan) for exploring ideas +to refine the API, and considering alternatives to avoid unnecessarily long +backtraces. + +Thanks to [Brandon Williams](https://github.com/mbrandonw) for feedback on the +Forum pitch thread which ultimately led to the refinements described in the +"Avoiding unnecessary (re-)execution" section. diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 954485339..4e20b4b4e 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -56,31 +56,28 @@ public struct Runner: Sendable { // MARK: - Running tests extension Runner { - /// Execute the ``CustomExecutionTrait/execute(_:for:testCase:)`` functions - /// associated with the test in a plan step. + /// Apply the custom scope for any test scope providers of the traits + /// associated with a specified test by calling their + /// ``TestScoping/provideScope(for:testCase:performing:)`` function. /// /// - Parameters: - /// - step: The step being performed. - /// - testCase: The test case, if applicable, for which to execute the - /// custom trait. + /// - test: The test being run, for which to provide custom scope. + /// - testCase: The test case, if applicable, for which to provide custom + /// scope. /// - body: A function to execute from within the - /// ``CustomExecutionTrait/execute(_:for:testCase:)`` functions of each - /// trait applied to `step.test`. + /// ``TestScoping/provideScope(for:testCase:performing:)`` function of + /// each non-`nil` scope provider of the traits applied to `test`. /// /// - Throws: Whatever is thrown by `body` or by any of the - /// ``CustomExecutionTrait/execute(_:for:testCase:)`` functions. - private func _executeTraits( - for step: Plan.Step, + /// ``TestScoping/provideScope(for:testCase:performing:)`` function calls. + private func _applyScopingTraits( + for test: Test, testCase: Test.Case?, _ body: @escaping @Sendable () async throws -> Void ) async throws { // If the test does not have any traits, exit early to avoid unnecessary // heap allocations below. - if step.test.traits.isEmpty { - return try await body() - } - - if case .skip = step.action { + if test.traits.isEmpty { return try await body() } @@ -88,13 +85,13 @@ extension Runner { // function. The order of the sequence is reversed so that the last trait is // the one that invokes body, then the second-to-last invokes the last, etc. // and ultimately the first trait is the first one to be invoked. - let executeAllTraits = step.test.traits.lazy + let executeAllTraits = test.traits.lazy .reversed() - .compactMap { $0 as? any CustomExecutionTrait } - .compactMap { $0.execute(_:for:testCase:) } - .reduce(body) { executeAllTraits, traitExecutor in + .compactMap { $0.scopeProvider(for: test, testCase: testCase) } + .map { $0.provideScope(for:testCase:performing:) } + .reduce(body) { executeAllTraits, provideScope in { - try await traitExecutor(executeAllTraits, step.test, testCase) + try await provideScope(test, testCase, executeAllTraits) } } @@ -200,7 +197,7 @@ extension Runner { if let step = stepGraph.value, case .run = step.action { await Test.withCurrent(step.test) { _ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) { - try await _executeTraits(for: step, testCase: nil) { + try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { try await _runTestCases(testCases, within: step) @@ -336,7 +333,7 @@ extension Runner { let sourceLocation = step.test.sourceLocation await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) { try await withTimeLimit(for: step.test, configuration: configuration) { - try await _executeTraits(for: step, testCase: testCase) { + try await _applyScopingTraits(for: step.test, testCase: testCase) { try await testCase.body() } } timeoutHandler: { timeLimit in diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index 3fe181bb9..5fadb2bdc 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -53,6 +53,7 @@ behavior of test functions. - ``Trait`` - ``TestTrait`` - ``SuiteTrait`` +- ``TestScoping`` ### Supporting types diff --git a/Sources/Testing/Testing.docc/Traits/Trait.md b/Sources/Testing/Testing.docc/Traits/Trait.md index d5a110602..f0e84aaeb 100644 --- a/Sources/Testing/Testing.docc/Traits/Trait.md +++ b/Sources/Testing/Testing.docc/Traits/Trait.md @@ -39,8 +39,15 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors - ``Trait/bug(_:id:_:)-3vtpl`` ### Adding information to tests + - ``Trait/comments`` ### Preparing internal state - ``Trait/prepare(for:)-3s3zo`` + +### Providing custom execution scope for tests + +- ``TestScoping`` +- ``Trait/scopeProvider(for:testCase:)-cjmg`` +- ``Trait/TestScopeProvider`` diff --git a/Sources/Testing/Traits/Trait.swift b/Sources/Testing/Traits/Trait.swift index e6a42b4d5..4c942f52a 100644 --- a/Sources/Testing/Traits/Trait.swift +++ b/Sources/Testing/Traits/Trait.swift @@ -41,6 +41,150 @@ public protocol Trait: Sendable { /// /// By default, the value of this property is an empty array. var comments: [Comment] { get } + + /// The type of the test scope provider for this trait. + /// + /// The default type is `Never`, which cannot be instantiated. The + /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with this + /// default type must return `nil`, meaning that trait will not provide a + /// custom scope for the tests it's applied to. + associatedtype TestScopeProvider: TestScoping = Never + + /// Get this trait's scope provider for the specified test and/or test case, + /// if any. + /// + /// - Parameters: + /// - test: The test for which a scope provider is being requested. + /// - testCase: The test case for which a scope provider is being requested, + /// if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// - Returns: A value conforming to ``Trait/TestScopeProvider`` which may be + /// used to provide custom scoping for `test` and/or `testCase`, or `nil` if + /// they should not have any custom scope. + /// + /// If this trait's type conforms to ``TestScoping``, the default value + /// returned by this method depends on `test` and/or `testCase`: + /// + /// - If `test` represents a suite, this trait must conform to ``SuiteTrait``. + /// If the value of this suite trait's ``SuiteTrait/isRecursive`` property + /// is `true`, then this method returns `nil`; otherwise, it returns `self`. + /// This means that by default, a suite trait will _either_ provide its + /// custom scope once for the entire suite, or once per-test function it + /// contains. + /// - Otherwise `test` represents a test function. If `testCase` is `nil`, + /// this method returns `nil`; otherwise, it returns `self`. This means that + /// by default, a trait which is applied to or inherited by a test function + /// will provide its custom scope once for each of that function's cases. + /// + /// A trait may explicitly implement this method to further customize the + /// default behaviors above. For example, if a trait should provide custom + /// test scope both once per-suite and once per-test function in that suite, + /// it may implement the method and return a non-`nil` scope provider under + /// those conditions. + /// + /// A trait may also implement this method and return `nil` if it determines + /// that it does not need to provide a custom scope for a particular test at + /// runtime, even if the test has the trait applied. This can improve + /// performance and make diagnostics clearer by avoiding an unnecessary call + /// to ``TestScoping/provideScope(for:testCase:performing:)``. + /// + /// If this trait's type does not conform to ``TestScoping`` and its + /// associated ``Trait/TestScopeProvider`` type is the default `Never`, then + /// this method returns `nil` by default. This means that instances of this + /// trait will not provide a custom scope for tests to which they're applied. + func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider? +} + +/// A protocol that allows providing a custom execution scope for a test +/// function (and each of its cases) or a test suite by performing custom code +/// before or after it runs. +/// +/// Types conforming to this protocol may be used in conjunction with a +/// ``Trait``-conforming type by implementing the +/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, allowing custom traits +/// to provide custom scope for tests. Consolidating common set-up and tear-down +/// logic for tests which have similar needs allows each test function to be +/// more succinct with less repetitive boilerplate so it can focus on what makes +/// it unique. +public protocol TestScoping: Sendable { + /// Provide custom execution scope for a function call which is related to the + /// specified test and/or test case. + /// + /// - Parameters: + /// - test: The test under which `function` is being performed. + /// - testCase: The test case, if any, under which `function` is being + /// performed. When invoked on a suite, the value of this argument is + /// `nil`. + /// - function: The function to perform. If `test` represents a test suite, + /// this function encapsulates running all the tests in that suite. If + /// `test` represents a test function, this function is the body of that + /// test function (including all cases if it is parameterized.) + /// + /// - Throws: Whatever is thrown by `function`, or an error preventing this + /// type from providing a custom scope correctly. An error thrown from this + /// method is recorded as an issue associated with `test`. If an error is + /// thrown before `function` is called, the corresponding test will not run. + /// + /// When the testing library is preparing to run a test, it starts by finding + /// all traits applied to that test, including those inherited from containing + /// suites. It begins with inherited suite traits, sorting them + /// outermost-to-innermost, and if the test is a function, it then adds all + /// traits applied directly to that functions in the order they were applied + /// (left-to-right). It then asks each trait for its scope provider (if any) + /// by calling ``Trait/scopeProvider(for:testCase:)-cjmg``. Finally, it calls + /// this method on all non-`nil` scope providers, giving each an opportunity + /// to perform arbitrary work before or after invoking `function`. + /// + /// This method should either invoke `function` once before returning or throw + /// an error if it is unable to provide a custom scope. + /// + /// Issues recorded by this method are associated with `test`. + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws +} + +extension Trait where Self: TestScoping { + /// Get this trait's scope provider for the specified test and/or test case, + /// if any. + /// + /// - Parameters: + /// - test: The test for which a scope provider is being requested. + /// - testCase: The test case for which a scope provider is being requested, + /// if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// This default implementation is used when this trait type conforms to + /// ``TestScoping`` and its return value is discussed in + /// ``Trait/scopeProvider(for:testCase:)-cjmg``. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + testCase == nil ? nil : self + } +} + +extension SuiteTrait where Self: TestScoping { + /// Get this trait's scope provider for the specified test and/or test case, + /// if any. + /// + /// - Parameters: + /// - test: The test for which a scope provider is being requested. + /// - testCase: The test case for which a scope provider is being requested, + /// if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// This default implementation is used when this trait type conforms to + /// ``TestScoping`` and its return value is discussed in + /// ``Trait/scopeProvider(for:testCase:)-cjmg``. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + if test.isSuite { + isRecursive ? nil : self + } else { + testCase == nil ? nil : self + } + } +} + +extension Never: TestScoping { + public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws {} } /// A protocol describing traits that can be added to a test function. @@ -72,43 +216,26 @@ extension Trait { } } +extension Trait where TestScopeProvider == Never { + /// Get this trait's scope provider for the specified test and/or test case, + /// if any. + /// + /// - Parameters: + /// - test: The test for which a scope provider is being requested. + /// - testCase: The test case for which a scope provider is being requested, + /// if any. When `test` represents a suite, the value of this argument is + /// `nil`. + /// + /// This default implementation is used when this trait type's associated + /// ``Trait/TestScopeProvider`` type is the default value of `Never`, and its + /// return value is discussed in ``Trait/scopeProvider(for:testCase:)-cjmg``. + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? { + nil + } +} + extension SuiteTrait { public var isRecursive: Bool { false } } - -/// A protocol extending ``Trait`` that offers an additional customization point -/// for trait authors to execute code before and after each test function (if -/// added to the traits of a test function), or before and after each test suite -/// (if added to the traits of a test suite). -@_spi(Experimental) -public protocol CustomExecutionTrait: Trait { - - /// Execute a function with the effects of this trait applied. - /// - /// - Parameters: - /// - function: The function to perform. If `test` represents a test suite, - /// this function encapsulates running all the tests in that suite. If - /// `test` represents a test function, this function is the body of that - /// test function (including all cases if it is parameterized.) - /// - test: The test under which `function` is being performed. - /// - testCase: The test case, if any, under which `function` is being - /// performed. This is `nil` when invoked on a suite. - /// - /// - Throws: Whatever is thrown by `function`, or an error preventing the - /// trait from running correctly. - /// - /// This function is called for each ``CustomExecutionTrait`` on a test suite - /// or test function and allows additional work to be performed before and - /// after the test runs. - /// - /// This function is invoked once for the test it is applied to, and then once - /// for each test case in that test, if applicable. - /// - /// Issues recorded by this function are recorded against `test`. - /// - /// - Note: If a test function or test suite is skipped, this function does - /// not get invoked by the runner. - func execute(_ function: @Sendable () async throws -> Void, for test: Test, testCase: Test.Case?) async throws -} diff --git a/Tests/TestingTests/Traits/CustomExecutionTraitTests.swift b/Tests/TestingTests/Traits/CustomExecutionTraitTests.swift deleted file mode 100644 index aedc06de3..000000000 --- a/Tests/TestingTests/Traits/CustomExecutionTraitTests.swift +++ /dev/null @@ -1,104 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing - -@Suite("CustomExecutionTrait Tests") -struct CustomExecutionTraitTests { - @Test("Execute code before and after a non-parameterized test.") - func executeCodeBeforeAndAfterNonParameterizedTest() async { - // `expectedCount` is 2 because we run it both for the test and the test case - await confirmation("Code was run before the test", expectedCount: 2) { before in - await confirmation("Code was run after the test", expectedCount: 2) { after in - await Test(CustomTrait(before: before, after: after)) { - // do nothing - }.run() - } - } - } - - @Test("Execute code before and after a parameterized test.") - func executeCodeBeforeAndAfterParameterizedTest() async { - // `expectedCount` is 3 because we run it both for the test and each test case - await confirmation("Code was run before the test", expectedCount: 3) { before in - await confirmation("Code was run after the test", expectedCount: 3) { after in - await Test(CustomTrait(before: before, after: after), arguments: ["Hello", "World"]) { _ in - // do nothing - }.run() - } - } - } - - @Test("Custom execution trait throws an error") - func customExecutionTraitThrowsAnError() async throws { - var configuration = Configuration() - await confirmation("Error thrown", expectedCount: 1) { errorThrownConfirmation in - configuration.eventHandler = { event, _ in - guard case let .issueRecorded(issue) = event.kind, - case let .errorCaught(error) = issue.kind else { - return - } - - #expect(error is CustomThrowingErrorTrait.CustomTraitError) - errorThrownConfirmation() - } - - await Test(CustomThrowingErrorTrait()) { - // Make sure this does not get reached - Issue.record("Expected trait to fail the test. Should not have reached test body.") - }.run(configuration: configuration) - } - } - - @Test("Teardown occurs after child tests run") - func teardownOccursAtEnd() async throws { - await runTest(for: TestsWithCustomTraitWithStrongOrdering.self, configuration: .init()) - } -} - -// MARK: - Fixtures - -private struct CustomTrait: CustomExecutionTrait, TestTrait { - var before: Confirmation - var after: Confirmation - func execute(_ function: @Sendable () async throws -> Void, for test: Test, testCase: Test.Case?) async throws { - before() - defer { - after() - } - try await function() - } -} - -private struct CustomThrowingErrorTrait: CustomExecutionTrait, TestTrait { - fileprivate struct CustomTraitError: Error {} - - func execute(_ function: @Sendable () async throws -> Void, for test: Test, testCase: Test.Case?) async throws { - throw CustomTraitError() - } -} - -struct DoSomethingBeforeAndAfterTrait: CustomExecutionTrait, SuiteTrait, TestTrait { - static let state = Locked(rawValue: 0) - - func execute(_ function: @Sendable () async throws -> Void, for test: Testing.Test, testCase: Testing.Test.Case?) async throws { - #expect(Self.state.increment() == 1) - - try await function() - #expect(Self.state.increment() == 3) - } -} - -@Suite(.hidden, DoSomethingBeforeAndAfterTrait()) -struct TestsWithCustomTraitWithStrongOrdering { - @Test(.hidden) func f() async { - #expect(DoSomethingBeforeAndAfterTrait.state.increment() == 2) - } -} diff --git a/Tests/TestingTests/Traits/TestScopingTraitTests.swift b/Tests/TestingTests/Traits/TestScopingTraitTests.swift new file mode 100644 index 000000000..af63deb5e --- /dev/null +++ b/Tests/TestingTests/Traits/TestScopingTraitTests.swift @@ -0,0 +1,184 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("TestScoping-conforming Trait Tests") +struct TestScopingTraitTests { + @Test("Execute code before and after a non-parameterized test.") + func executeCodeBeforeAndAfterNonParameterizedTest() async { + await confirmation("Code was run before the test") { before in + await confirmation("Code was run after the test") { after in + await Test(CustomTrait(before: before, after: after)) { + // do nothing + }.run() + } + } + } + + @Test("Execute code before and after a parameterized test.") + func executeCodeBeforeAndAfterParameterizedTest() async { + // `expectedCount` is 2 because we run it for each test case + await confirmation("Code was run before the test", expectedCount: 2) { before in + await confirmation("Code was run after the test", expectedCount: 2) { after in + await Test(CustomTrait(before: before, after: after), arguments: ["Hello", "World"]) { _ in + // do nothing + }.run() + } + } + } + + @Test("Custom execution trait throws an error") + func customExecutionTraitThrowsAnError() async throws { + var configuration = Configuration() + await confirmation("Error thrown", expectedCount: 1) { errorThrownConfirmation in + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind, + case let .errorCaught(error) = issue.kind else { + return + } + + #expect(error is CustomThrowingErrorTrait.CustomTraitError) + errorThrownConfirmation() + } + + await Test(CustomThrowingErrorTrait()) { + // Make sure this does not get reached + Issue.record("Expected trait to fail the test. Should not have reached test body.") + }.run(configuration: configuration) + } + } + + @Test("Teardown occurs after child tests run") + func teardownOccursAtEnd() async throws { + await runTest(for: TestsWithCustomTraitWithStrongOrdering.self, configuration: .init()) + } + + struct ExecutionControl { + @Test("Trait applied directly to function is executed once") + func traitAppliedToFunction() async { + let counter = Locked(rawValue: 0) + await DefaultExecutionTrait.$counter.withValue(counter) { + await Test(DefaultExecutionTrait()) {}.run() + } + #expect(counter.rawValue == 1) + } + + @Test("Non-recursive suite trait with default scope provider implementation") + func nonRecursiveSuiteTrait() async { + let counter = Locked(rawValue: 0) + await DefaultExecutionTrait.$counter.withValue(counter) { + await runTest(for: SuiteWithNonRecursiveDefaultExecutionTrait.self) + } + #expect(counter.rawValue == 1) + } + + @Test("Recursive suite trait with default scope provider implementation") + func recursiveSuiteTrait() async { + let counter = Locked(rawValue: 0) + await DefaultExecutionTrait.$counter.withValue(counter) { + await runTest(for: SuiteWithRecursiveDefaultExecutionTrait.self) + } + #expect(counter.rawValue == 1) + } + + @Test("Recursive, all-inclusive suite trait") + func recursiveAllInclusiveSuiteTrait() async { + let counter = Locked(rawValue: 0) + await AllInclusiveExecutionTrait.$counter.withValue(counter) { + await runTest(for: SuiteWithAllInclusiveExecutionTrait.self) + } + #expect(counter.rawValue == 3) + } + } +} + +// MARK: - Fixtures + +private struct CustomTrait: TestTrait, TestScoping { + var before: Confirmation + var after: Confirmation + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + before() + defer { + after() + } + try await function() + } +} + +private struct CustomThrowingErrorTrait: TestTrait, TestScoping { + fileprivate struct CustomTraitError: Error {} + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + throw CustomTraitError() + } +} + +struct DoSomethingBeforeAndAfterTrait: SuiteTrait, TestTrait, TestScoping { + static let state = Locked(rawValue: 0) + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + #expect(Self.state.increment() == 1) + + try await function() + #expect(Self.state.increment() == 3) + } +} + +@Suite(.hidden, DoSomethingBeforeAndAfterTrait()) +struct TestsWithCustomTraitWithStrongOrdering { + @Test(.hidden) func f() async { + #expect(DoSomethingBeforeAndAfterTrait.state.increment() == 2) + } +} + +private struct DefaultExecutionTrait: SuiteTrait, TestTrait, TestScoping { + @TaskLocal static var counter: Locked? + var isRecursive: Bool = false + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + Self.counter!.increment() + try await function() + } +} + +@Suite(.hidden, DefaultExecutionTrait()) +private struct SuiteWithNonRecursiveDefaultExecutionTrait { + @Test func f() {} +} + +@Suite(.hidden, DefaultExecutionTrait(isRecursive: true)) +private struct SuiteWithRecursiveDefaultExecutionTrait { + @Test func f() {} +} + +private struct AllInclusiveExecutionTrait: SuiteTrait, TestTrait, TestScoping { + @TaskLocal static var counter: Locked? + + var isRecursive: Bool { + true + } + + func scopeProvider(for test: Test, testCase: Test.Case?) -> AllInclusiveExecutionTrait? { + // Unconditionally returning self makes this trait "all inclusive". + self + } + + func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + Self.counter!.increment() + try await function() + } +} + +@Suite(.hidden, AllInclusiveExecutionTrait()) +private struct SuiteWithAllInclusiveExecutionTrait { + @Test func f() {} +} From aee08217f0daa50e662373fe7b49f0b767354986 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 7 Jan 2025 18:23:25 -0500 Subject: [PATCH 040/234] Fix statically-linked platforms building new test content discovery code. (#897) This PR fixes an undefined symbol error for `SWTTestContentSectionBounds` when building for statically-linked platforms. This issue appears to only occur with CMake and is (I think) a quirk of that build system. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery+Platform.swift | 2 +- Sources/_TestingInternals/include/Discovery.h | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index db5594703..d85554498 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -210,7 +210,7 @@ private func _testContentSectionBounds() -> [SectionBounds] { /// contained in the same image as the testing library itself. private func _testContentSectionBounds() -> CollectionOfOne { let (sectionBegin, sectionEnd) = SWTTestContentSectionBounds - let buffer = UnsafeRawBufferPointer(start: n, count: max(0, sectionEnd - sectionBegin)) + let buffer = UnsafeRawBufferPointer(start: sectionBegin, count: max(0, sectionEnd - sectionBegin)) let sb = SectionBounds(imageAddress: nil, buffer: buffer) return CollectionOfOne(sb) } diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index 9d7a5a6e9..6432a798c 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -62,13 +62,16 @@ SWT_IMPORT_FROM_STDLIB void swift_enumerateAllMetadataSections( ); #endif -#if defined(SWT_NO_DYNAMIC_LINKING) #pragma mark - Statically-linked section bounds /// The bounds of the test content section statically linked into the image /// containing Swift Testing. +/// +/// - Note: This symbol is _declared_, but not _defined_, on platforms with +/// dynamic linking because the `SWT_NO_DYNAMIC_LINKING` C++ macro (not the +/// Swift compiler conditional of the same name) is not consistently declared +/// when Swift files import the `_TestingInternals` C++ module. SWT_EXTERN const void *_Nonnull const SWTTestContentSectionBounds[2]; -#endif #pragma mark - Legacy test discovery From 6783a1afbf2a39977e7af381d869c793d66a55c9 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 8 Jan 2025 12:00:26 -0500 Subject: [PATCH 041/234] Use `Never` instead of `Void` as the fallback lock/mutex type. (#898) When using WASI without multithreading, or when using a platform to which Swift Testing has been incompletely ported, we don't know what type to use as a lock/mutex, so we use `Void`. However, turns out that produces a diagnostic: > warning: UnsafeMutablePointer has been replaced by UnsafeMutableRawPointer So use `Never` instead. (Yes, this will produce a pointer to an uninitialized instance of `Never`. If this affects you, please read Porting.md!) ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Support/Locked.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index d0edbc801..1294da195 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -44,10 +44,10 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { typealias PlatformLock = SRWLOCK #elseif os(WASI) // No locks on WASI without multithreaded runtime. - typealias PlatformLock = Void + typealias PlatformLock = Never #else #warning("Platform-specific implementation missing: locking unavailable") - typealias PlatformLock = Void + typealias PlatformLock = Never #endif /// A type providing heap-allocated storage for an instance of ``Locked``. From 4fa7afbae75656f0c74791691bbf8bde5989b31a Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 8 Jan 2025 12:10:40 -0600 Subject: [PATCH 042/234] Assign "Test Scoping Traits" proposal a number (#900) Assign the "Test Scoping Traits" proposal, which was recently approved and merged in #733, a number ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- ...tom-test-execution-traits.md => 0007-test-scoping-traits.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Documentation/Proposals/{NNNN-custom-test-execution-traits.md => 0007-test-scoping-traits.md} (99%) diff --git a/Documentation/Proposals/NNNN-custom-test-execution-traits.md b/Documentation/Proposals/0007-test-scoping-traits.md similarity index 99% rename from Documentation/Proposals/NNNN-custom-test-execution-traits.md rename to Documentation/Proposals/0007-test-scoping-traits.md index 731387b24..3b0471550 100644 --- a/Documentation/Proposals/NNNN-custom-test-execution-traits.md +++ b/Documentation/Proposals/0007-test-scoping-traits.md @@ -1,6 +1,6 @@ # Test Scoping Traits -* Proposal: [SWT-NNNN](NNNN-filename.md) +* Proposal: [SWT-0007](0007-test-scoping-traits.md) * Authors: [Stuart Montgomery](https://github.com/stmontgomery) * Status: **Awaiting review** * Implementation: [swiftlang/swift-testing#733](https://github.com/swiftlang/swift-testing/pull/733), [swiftlang/swift-testing#86](https://github.com/swiftlang/swift-testing/pull/86) From 89ff719e6ed351a13471b4a266b201fd7c4aa7c2 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 8 Jan 2025 13:32:14 -0500 Subject: [PATCH 043/234] Fix several compile-time warnings that have snuck in. (#899) This PR resolves some compile-time warnings that occur when building the package from source. None of these warnings are critical issues. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 3 ++- Tests/TestingTests/ExitTestTests.swift | 4 ++-- Tests/TestingTests/Traits/TimeLimitTraitTests.swift | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Package.swift b/Package.swift index ab06f693d..4a22b98e2 100644 --- a/Package.swift +++ b/Package.swift @@ -53,7 +53,7 @@ let package = Package( "_TestingInternals", "TestingMacros", ], - exclude: ["CMakeLists.txt"], + exclude: ["CMakeLists.txt", "Testing.swiftcrossimport"], cxxSettings: .packageSettings, swiftSettings: .packageSettings, linkerSettings: [ @@ -122,6 +122,7 @@ let package = Package( "Testing", ], path: "Sources/Overlays/_Testing_Foundation", + exclude: ["CMakeLists.txt"], swiftSettings: .packageSettings ), ], diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index c82c4f73f..81a21aaec 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -243,8 +243,8 @@ private import _TestingInternals func exitConditionMatching() { #expect(Optional.none == Optional.none) #expect(Optional.none === Optional.none) - #expect(Optional.none !== .success) - #expect(Optional.none !== .failure) + #expect(Optional.none !== .some(.success)) + #expect(Optional.none !== .some(.failure)) #expect(ExitCondition.success == .success) #expect(ExitCondition.success === .success) diff --git a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift index 8978e86fd..00c9cbb44 100644 --- a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift +++ b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift @@ -203,13 +203,12 @@ struct TimeLimitTraitTests { guard case let .issueRecorded(issue) = event.kind, case .timeLimitExceeded = issue.kind, let test = context.test, - let testCase = context.testCase + context.testCase != nil else { return } issueRecorded() #expect(test.timeLimit == .milliseconds(10)) - #expect(testCase != nil) } await Test(.timeLimit(.milliseconds(10))) { From 40056767456c31d050245e9540ec5e79ff9c6b4d Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 8 Jan 2025 17:54:48 -0500 Subject: [PATCH 044/234] Add support for raw identifiers. (#887) This PR adds support for the raw identifiers feature introduced with [SE-0451](https://forums.swift.org/t/accepted-with-revision-se-0451-raw-identifiers/76387). Resolves #842. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Testing/Parameterization/TypeInfo.swift | 94 +++++++++++++++---- .../SourceAttribution/SourceLocation.swift | 5 +- .../FunctionDeclSyntaxAdditions.swift | 25 +++-- .../Additions/TokenSyntaxAdditions.swift | 30 +++++- .../Support/AttributeDiscovery.swift | 13 +++ .../Support/DiagnosticMessage.swift | 35 +++++++ .../TestingMacros/TestDeclarationMacro.swift | 4 +- .../TestDeclarationMacroTests.swift | 39 ++++++++ Tests/TestingTests/MiscellaneousTests.swift | 9 ++ Tests/TestingTests/SourceLocationTests.swift | 14 +++ Tests/TestingTests/TypeInfoTests.swift | 54 +++++++++++ 11 files changed, 291 insertions(+), 31 deletions(-) diff --git a/Sources/Testing/Parameterization/TypeInfo.swift b/Sources/Testing/Parameterization/TypeInfo.swift index ba471b7c8..eeda19418 100644 --- a/Sources/Testing/Parameterization/TypeInfo.swift +++ b/Sources/Testing/Parameterization/TypeInfo.swift @@ -69,7 +69,7 @@ public struct TypeInfo: Sendable { /// - mangled: The mangled name of the type, if available. init(fullyQualifiedName: String, unqualifiedName: String, mangledName: String?) { self.init( - fullyQualifiedNameComponents: fullyQualifiedName.split(separator: ".").map(String.init), + fullyQualifiedNameComponents: Self.fullyQualifiedNameComponents(ofTypeWithName: fullyQualifiedName), unqualifiedName: unqualifiedName, mangledName: mangledName ) @@ -95,10 +95,85 @@ public struct TypeInfo: Sendable { // MARK: - Name +/// Split a string with a separator while respecting raw identifiers and their +/// enclosing backtick characters. +/// +/// - Parameters: +/// - string: The string to split. +/// - separator: The character that separates components of `string`. +/// - maxSplits: The maximum number of splits to perform on `string`. The +/// resulting array contains up to `maxSplits + 1` elements. +/// +/// - Returns: An array of substrings of `string`. +/// +/// Unlike `String.split(separator:maxSplits:omittingEmptySubsequences:)`, this +/// function does not split the string on separator characters that occur +/// between pairs of backtick characters. This is useful when splitting strings +/// containing raw identifiers. +/// +/// - Complexity: O(_n_), where _n_ is the length of `string`. +func rawIdentifierAwareSplit(_ string: S, separator: Character, maxSplits: Int = .max) -> [S.SubSequence] where S: StringProtocol { + var result = [S.SubSequence]() + + var inRawIdentifier = false + var componentStartIndex = string.startIndex + for i in string.indices { + let c = string[i] + if c == "`" { + // We are either entering or exiting a raw identifier. While inside a raw + // identifier, separator characters are ignored. + inRawIdentifier.toggle() + } else if c == separator && !inRawIdentifier { + // Add everything up to this separator as the next component, then start + // a new component after the separator. + result.append(string[componentStartIndex ..< i]) + componentStartIndex = string.index(after: i) + + if result.count == maxSplits { + // We don't need to find more separators. We'll add the remainder of the + // string outside the loop as the last component, then return. + break + } + } + } + result.append(string[componentStartIndex...]) + + return result +} + extension TypeInfo { /// An in-memory cache of fully-qualified type name components. private static let _fullyQualifiedNameComponentsCache = Locked<[ObjectIdentifier: [String]]>() + /// Split the given fully-qualified type name into its components. + /// + /// - Parameters: + /// - fullyQualifiedName: The string to split. + /// + /// - Returns: The components of `fullyQualifiedName` as substrings thereof. + static func fullyQualifiedNameComponents(ofTypeWithName fullyQualifiedName: String) -> [String] { + var components = rawIdentifierAwareSplit(fullyQualifiedName, separator: ".") + + // If a type is extended in another module and then referenced by name, + // its name according to the String(reflecting:) API will be prefixed with + // "(extension in MODULE_NAME):". For our purposes, we never want to + // preserve that prefix. + if let firstComponent = components.first, firstComponent.starts(with: "(extension in "), + let moduleName = rawIdentifierAwareSplit(firstComponent, separator: ":", maxSplits: 1).last { + // NOTE: even if the module name is a raw identifier, it comprises a + // single identifier (no splitting required) so we don't need to process + // it any further. + components[0] = moduleName + } + + // If a type is private or embedded in a function, its fully qualified + // name may include "(unknown context at $xxxxxxxx)" as a component. Strip + // those out as they're uninteresting to us. + components = components.filter { !$0.starts(with: "(unknown context at") } + + return components.map(String.init) + } + /// The complete name of this type, with the names of all referenced types /// fully-qualified by their module names when possible. /// @@ -121,22 +196,7 @@ extension TypeInfo { return cachedResult } - var result = String(reflecting: type) - .split(separator: ".") - .map(String.init) - - // If a type is extended in another module and then referenced by name, - // its name according to the String(reflecting:) API will be prefixed with - // "(extension in MODULE_NAME):". For our purposes, we never want to - // preserve that prefix. - if let firstComponent = result.first, firstComponent.starts(with: "(extension in ") { - result[0] = String(firstComponent.split(separator: ":", maxSplits: 1).last!) - } - - // If a type is private or embedded in a function, its fully qualified - // name may include "(unknown context at $xxxxxxxx)" as a component. Strip - // those out as they're uninteresting to us. - result = result.filter { !$0.starts(with: "(unknown context at") } + let result = Self.fullyQualifiedNameComponents(ofTypeWithName: String(reflecting: type)) Self._fullyQualifiedNameComponentsCache.withLock { fullyQualifiedNameComponentsCache in fullyQualifiedNameComponentsCache[ObjectIdentifier(type)] = result diff --git a/Sources/Testing/SourceAttribution/SourceLocation.swift b/Sources/Testing/SourceAttribution/SourceLocation.swift index bbf3cf3a6..3aca54d2f 100644 --- a/Sources/Testing/SourceAttribution/SourceLocation.swift +++ b/Sources/Testing/SourceAttribution/SourceLocation.swift @@ -46,7 +46,7 @@ public struct SourceLocation: Sendable { /// - ``moduleName`` public var fileName: String { let lastSlash = fileID.lastIndex(of: "/")! - return String(fileID[fileID.index(after: lastSlash)...]) + return String(fileID[lastSlash...].dropFirst()) } /// The name of the module containing the source file. @@ -67,8 +67,7 @@ public struct SourceLocation: Sendable { /// - ``fileName`` /// - [`#fileID`](https://developer.apple.com/documentation/swift/fileID()) public var moduleName: String { - let firstSlash = fileID.firstIndex(of: "/")! - return String(fileID[.. TokenSyntax { + if let rawIdentifier = token.rawIdentifier { + return .identifier("`\(rawIdentifier)`") + } + return .identifier(token.textWithoutBackticks) } - result.append(")") - return result.joined() + return DeclReferenceExprSyntax( + baseName: possiblyRaw(name), + argumentNames: DeclNameArgumentsSyntax( + arguments: DeclNameArgumentListSyntax { + for parameter in signature.parameterClause.parameters { + DeclNameArgumentSyntax(name: possiblyRaw(parameter.firstName)) + } + } + ) + ) } /// An array of tuples representing this function's parameters. diff --git a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift index 2281f9f5a..12e6abb24 100644 --- a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift @@ -11,11 +11,39 @@ import SwiftSyntax extension TokenSyntax { + /// A tuple containing the text of this instance with enclosing backticks + /// removed and whether or not they were removed. + private var _textWithoutBackticks: (String, backticksRemoved: Bool) { + let text = text + if case .identifier = tokenKind, text.first == "`" && text.last == "`" && text.count >= 2 { + return (String(text.dropFirst().dropLast()), true) + } + + return (text, false) + } + /// The text of this instance with all backticks removed. /// /// - Bug: This property works around the presence of backticks in `text.` /// ([swift-syntax-#1936](https://github.com/swiftlang/swift-syntax/issues/1936)) var textWithoutBackticks: String { - text.filter { $0 != "`" } + _textWithoutBackticks.0 + } + + /// The raw identifier, not including enclosing backticks, represented by this + /// token, or `nil` if it does not represent one. + var rawIdentifier: String? { + let (textWithoutBackticks, backticksRemoved) = _textWithoutBackticks + if backticksRemoved, !textWithoutBackticks.isValidSwiftIdentifier(for: .memberAccess) { + return textWithoutBackticks + } + + // TODO: remove this mock path once the toolchain fully supports raw IDs. + let mockPrefix = "__raw__$" + if backticksRemoved, textWithoutBackticks.starts(with: mockPrefix) { + return String(textWithoutBackticks.dropFirst(mockPrefix.count)) + } + + return nil } } diff --git a/Sources/TestingMacros/Support/AttributeDiscovery.swift b/Sources/TestingMacros/Support/AttributeDiscovery.swift index 77b2b174e..dce4bddd3 100644 --- a/Sources/TestingMacros/Support/AttributeDiscovery.swift +++ b/Sources/TestingMacros/Support/AttributeDiscovery.swift @@ -100,6 +100,7 @@ struct AttributeInfo { init(byParsing attribute: AttributeSyntax, on declaration: some SyntaxProtocol, in context: some MacroExpansionContext) { self.attribute = attribute + var displayNameArgument: LabeledExprListSyntax.Element? var nonDisplayNameArguments: [Argument] = [] if let arguments = attribute.arguments, case let .argumentList(argumentList) = arguments { // If the first argument is an unlabelled string literal, it's the display @@ -109,8 +110,10 @@ struct AttributeInfo { let firstArgumentHasLabel = (firstArgument.label != nil) if !firstArgumentHasLabel, let stringLiteral = firstArgument.expression.as(StringLiteralExprSyntax.self) { displayName = stringLiteral + displayNameArgument = firstArgument nonDisplayNameArguments = argumentList.dropFirst().map(Argument.init) } else if !firstArgumentHasLabel, firstArgument.expression.is(NilLiteralExprSyntax.self) { + displayNameArgument = firstArgument nonDisplayNameArguments = argumentList.dropFirst().map(Argument.init) } else { nonDisplayNameArguments = argumentList.map(Argument.init) @@ -118,6 +121,16 @@ struct AttributeInfo { } } + // Disallow an explicit display name for tests and suites with raw + // identifier names as it's redundant and potentially confusing. + if let namedDecl = declaration.asProtocol((any NamedDeclSyntax).self), + let rawIdentifier = namedDecl.name.rawIdentifier { + if let displayName, let displayNameArgument { + context.diagnose(.declaration(namedDecl, hasExtraneousDisplayName: displayName, fromArgument: displayNameArgument, using: attribute)) + } + displayName = StringLiteralExprSyntax(content: rawIdentifier) + } + // Remove leading "Self." expressions from the arguments of the attribute. // See _SelfRemover for more information. Rewriting a syntax tree discards // location information from the copy, so only invoke the rewriter if the diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index 7d8a83c20..aa26b9dc7 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -645,6 +645,41 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage { ) } + /// Create a diagnostic message stating that a declaration has two display + /// names. + /// + /// - Parameters: + /// - decl: The declaration that has two display names. + /// - displayNameFromAttribute: The display name provided by the `@Test` or + /// `@Suite` attribute. + /// - argumentContainingDisplayName: The argument node containing the node + /// `displayNameFromAttribute`. + /// - attribute: The `@Test` or `@Suite` attribute. + /// + /// - Returns: A diagnostic message. + static func declaration( + _ decl: some NamedDeclSyntax, + hasExtraneousDisplayName displayNameFromAttribute: StringLiteralExprSyntax, + fromArgument argumentContainingDisplayName: LabeledExprListSyntax.Element, + using attribute: AttributeSyntax + ) -> Self { + Self( + syntax: Syntax(decl), + message: "Attribute \(_macroName(attribute)) specifies display name '\(displayNameFromAttribute.representedLiteralValue!)' for \(_kindString(for: decl)) with implicit display name '\(decl.name.rawIdentifier!)'", + severity: .error, + fixIts: [ + FixIt( + message: MacroExpansionFixItMessage("Remove '\(displayNameFromAttribute.representedLiteralValue!)'"), + changes: [.replace(oldNode: Syntax(argumentContainingDisplayName), newNode: Syntax("" as ExprSyntax))] + ), + FixIt( + message: MacroExpansionFixItMessage("Rename '\(decl.name.textWithoutBackticks)'"), + changes: [.replace(oldNode: Syntax(decl.name), newNode: Syntax(EditorPlaceholderExprSyntax("name")))] + ), + ] + ) + } + /// Create a diagnostic messages stating that the expression passed to /// `#require()` is ambiguous. /// diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 463412d2a..1a3f2c448 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -407,7 +407,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { var testsBody: CodeBlockItemListSyntax = """ return [ .__function( - named: \(literal: functionDecl.completeName), + named: \(literal: functionDecl.completeName.trimmedDescription), in: \(typeNameExpr), xcTestCompatibleSelector: \(selectorExpr ?? "nil"), \(raw: attributeInfo.functionArgumentList(in: context)), @@ -433,7 +433,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { private \(_staticKeyword(for: typeName)) nonisolated func \(unavailableTestName)() async -> [Testing.Test] { [ .__function( - named: \(literal: functionDecl.completeName), + named: \(literal: functionDecl.completeName.trimmedDescription), in: \(typeNameExpr), xcTestCompatibleSelector: \(selectorExpr ?? "nil"), \(raw: attributeInfo.functionArgumentList(in: context)), diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index c75166c66..a77acfea1 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -209,6 +209,21 @@ struct TestDeclarationMacroTests { ), ] ), + + #"@Test("Goodbye world") func `__raw__$helloWorld`()"#: + ( + message: "Attribute 'Test' specifies display name 'Goodbye world' for function with implicit display name 'helloWorld'", + fixIts: [ + ExpectedFixIt( + message: "Remove 'Goodbye world'", + changes: [.replace(oldSourceCode: #""Goodbye world""#, newSourceCode: "")] + ), + ExpectedFixIt( + message: "Rename '__raw__$helloWorld'", + changes: [.replace(oldSourceCode: "`__raw__$helloWorld`", newSourceCode: "\(EditorPlaceholderExprSyntax("name"))")] + ), + ] + ), ] } @@ -241,6 +256,30 @@ struct TestDeclarationMacroTests { } } + @Test("Raw identifier is detected") + func rawIdentifier() { + #expect(TokenSyntax.identifier("`hello`").rawIdentifier == nil) + #expect(TokenSyntax.identifier("`helloworld`").rawIdentifier == nil) + #expect(TokenSyntax.identifier("`hélloworld`").rawIdentifier == nil) + #expect(TokenSyntax.identifier("`hello_world`").rawIdentifier == nil) + #expect(TokenSyntax.identifier("`hello world`").rawIdentifier != nil) + #expect(TokenSyntax.identifier("`hello/world`").rawIdentifier != nil) + #expect(TokenSyntax.identifier("`hello\tworld`").rawIdentifier != nil) + + #expect(TokenSyntax.identifier("`class`").rawIdentifier == nil) + #expect(TokenSyntax.identifier("`struct`").rawIdentifier == nil) + #expect(TokenSyntax.identifier("`class struct`").rawIdentifier != nil) + } + + @Test("Raw function name components") + func rawFunctionNameComponents() throws { + let decl = """ + func `__raw__$hello`(`__raw__$world`: T, etc: U, `blah`: V) {} + """ as DeclSyntax + let functionDecl = try #require(decl.as(FunctionDeclSyntax.self)) + #expect(functionDecl.completeName.trimmedDescription == "`hello`(`world`:etc:blah:)") + } + @Test("Warning diagnostics emitted on API misuse", arguments: [ // return types diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index a8cd56a7b..49df6cc6e 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -287,6 +287,15 @@ struct MiscellaneousTests { #expect(testType.displayName == "Named Sendable test type") } + @Test func `__raw__$raw_identifier_provides_a_display_name`() throws { + let test = try #require(Test.current) + #expect(test.displayName == "raw_identifier_provides_a_display_name") + #expect(test.name == "`raw_identifier_provides_a_display_name`()") + let id = test.id + #expect(id.moduleName == "TestingTests") + #expect(id.nameComponents == ["MiscellaneousTests", "`raw_identifier_provides_a_display_name`()"]) + } + @Test("Free functions are runnable") func freeFunction() async throws { await Test(testFunction: freeSyncFunction).run() diff --git a/Tests/TestingTests/SourceLocationTests.swift b/Tests/TestingTests/SourceLocationTests.swift index a6a22c3b4..75a8791db 100644 --- a/Tests/TestingTests/SourceLocationTests.swift +++ b/Tests/TestingTests/SourceLocationTests.swift @@ -44,6 +44,20 @@ struct SourceLocationTests { #expect(sourceLocation.moduleName == "FakeModule") } + @Test("SourceLocation.moduleName property with raw identifier", + arguments: [ + ("Foo/Bar.swift", "Foo", "Bar.swift"), + ("`Foo`/Bar.swift", "`Foo`", "Bar.swift"), + ("`Foo.Bar`/Quux.swift", "`Foo.Bar`", "Quux.swift"), + ("`Foo./.Bar`/Quux.swift", "`Foo./.Bar`", "Quux.swift"), + ] + ) + func sourceLocationModuleNameWithRawIdentifier(fileID: String, expectedModuleName: String, expectedFileName: String) throws { + let sourceLocation = SourceLocation(fileID: fileID, filePath: "", line: 1, column: 1) + #expect(sourceLocation.moduleName == expectedModuleName) + #expect(sourceLocation.fileName == expectedFileName) + } + @Test("SourceLocation.fileID property ignores middle components") func sourceLocationFileIDMiddleIgnored() { let sourceLocation = SourceLocation(fileID: "A/B/C/D.swift", filePath: "", line: 1, column: 1) diff --git a/Tests/TestingTests/TypeInfoTests.swift b/Tests/TestingTests/TypeInfoTests.swift index a8d8327b2..b2a79f1ab 100644 --- a/Tests/TestingTests/TypeInfoTests.swift +++ b/Tests/TestingTests/TypeInfoTests.swift @@ -50,6 +50,60 @@ struct TypeInfoTests { #expect(TypeInfo(describing: T.self).fullyQualifiedName == "(Swift.Int, Swift.String) -> Swift.Bool") } + @Test("Splitting raw identifiers", + arguments: [ + ("Foo.Bar", ["Foo", "Bar"]), + ("`Foo`.Bar", ["`Foo`", "Bar"]), + ("`Foo`.`Bar`", ["`Foo`", "`Bar`"]), + ("Foo.`Bar`", ["Foo", "`Bar`"]), + ("Foo.`Bar`.Quux", ["Foo", "`Bar`", "Quux"]), + ("Foo.`B.ar`.Quux", ["Foo", "`B.ar`", "Quux"]), + + // These have substrings we intentionally strip out. + ("Foo.`B.ar`.(unknown context at $0).Quux", ["Foo", "`B.ar`", "Quux"]), + ("(extension in Module):Foo.`B.ar`.(unknown context at $0).Quux", ["Foo", "`B.ar`", "Quux"]), + ("(extension in `Module`):Foo.`B.ar`.(unknown context at $0).Quux", ["Foo", "`B.ar`", "Quux"]), + ("(extension in `Module`):`Foo`.`B.ar`.(unknown context at $0).Quux", ["`Foo`", "`B.ar`", "Quux"]), + ("(extension in `Mo:dule`):`Foo`.`B.ar`.(unknown context at $0).Quux", ["`Foo`", "`B.ar`", "Quux"]), + ("(extension in `Module`):`F:oo`.`B.ar`.(unknown context at $0).Quux", ["`F:oo`", "`B.ar`", "Quux"]), + ("`(extension in Foo):Bar`.Baz", ["`(extension in Foo):Bar`", "Baz"]), + ("(extension in `(extension in Foo2):Bar2`):`(extension in Foo):Bar`.Baz", ["`(extension in Foo):Bar`", "Baz"]), + + // These aren't syntactically valid, but we should at least not crash. + ("Foo.`B.ar`.Quux.`Alpha`..Beta", ["Foo", "`B.ar`", "Quux", "`Alpha`", "", "Beta"]), + ("Foo.`B.ar`.Quux.`Alpha", ["Foo", "`B.ar`", "Quux", "`Alpha"]), + ("Foo.`B.ar`.Quux.`Alpha``", ["Foo", "`B.ar`", "Quux", "`Alpha``"]), + ("Foo.`B.ar`.Quux.`Alpha...", ["Foo", "`B.ar`", "Quux", "`Alpha..."]), + ] + ) + func rawIdentifiers(fqn: String, expectedComponents: [String]) throws { + let actualComponents = TypeInfo.fullyQualifiedNameComponents(ofTypeWithName: fqn) + #expect(expectedComponents == actualComponents) + } + + // As above, but round-tripping through .fullyQualifiedName. + @Test("Round-tripping raw identifiers", + arguments: [ + ("Foo.Bar", ["Foo", "Bar"]), + ("`Foo`.Bar", ["`Foo`", "Bar"]), + ("`Foo`.`Bar`", ["`Foo`", "`Bar`"]), + ("Foo.`Bar`", ["Foo", "`Bar`"]), + ("Foo.`Bar`.Quux", ["Foo", "`Bar`", "Quux"]), + ("Foo.`B.ar`.Quux", ["Foo", "`B.ar`", "Quux"]), + + // These aren't syntactically valid, but we should at least not crash. + ("Foo.`B.ar`.Quux.`Alpha`..Beta", ["Foo", "`B.ar`", "Quux", "`Alpha`", "", "Beta"]), + ("Foo.`B.ar`.Quux.`Alpha", ["Foo", "`B.ar`", "Quux", "`Alpha"]), + ("Foo.`B.ar`.Quux.`Alpha``", ["Foo", "`B.ar`", "Quux", "`Alpha``"]), + ("Foo.`B.ar`.Quux.`Alpha...", ["Foo", "`B.ar`", "Quux", "`Alpha..."]), + ] + ) + func roundTrippedRawIdentifiers(fqn: String, expectedComponents: [String]) throws { + let typeInfo = TypeInfo(fullyQualifiedName: fqn, unqualifiedName: "", mangledName: "") + #expect(typeInfo.fullyQualifiedName == fqn) + #expect(typeInfo.fullyQualifiedNameComponents == expectedComponents) + } + @available(_mangledTypeNameAPI, *) @Test func mangledTypeName() { #expect(_mangledTypeName(String.self) == TypeInfo(describing: String.self).mangledName) From 801d5ea593cff3e1f038720aa5dc8a2700d9f5d9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 9 Jan 2025 14:50:22 -0600 Subject: [PATCH 045/234] Remove SPIAwareTrait and adopt TestScoping in its place (#901) This is a follow-on from #733 which introduced the Test Scoping Traits feature. As that proposal indicated, the `SPIAwareTrait` SPI protocol is no longer necessary, so this PR removes it and adopts `TestScoping` in its place. ### Motivation: Adopting Test Scoping Traits for the purpose which `SPIAwareTrait` previously served simplifies a lot of runner and planning logic. ### Modifications: - Delete `SPIAwareTrait` and related SPI. - Adopt `TestScoping` in `ParallelizationTrait` instead. - Changed `ParallelizationTrait` to _not_ always have the value `true` for `isRecursive`. Test Scoping Traits makes this no longer necessary: the invocation of children of any suites which have `.serialized` is scoped to have parallelization disabled. - Removed tracking of `isParallelizationEnabled` on the `Runner.Plan.Action.RunOptions` type. For now, I kept the `RunOptions` struct, since it could be useful in the future and it's SPI. - Removed obsolete logic in `Runner` for propagating ancestor steps, and refactored slightly to make many things `static`. This is to help ensure certain functions don't access the `self.configuration` property when instead they should be accessing `Configuration.current`. ### Result: No functional change, but implementation is simplified. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/CMakeLists.txt | 1 - Sources/Testing/Running/Runner.Plan.swift | 29 +++---- Sources/Testing/Running/Runner.swift | 81 ++++++++----------- .../Testing/Traits/ParallelizationTrait.swift | 34 ++++---- Sources/Testing/Traits/SPIAwareTrait.swift | 38 --------- .../Traits/ParallelizationTraitTests.swift | 10 --- 6 files changed, 57 insertions(+), 136 deletions(-) delete mode 100644 Sources/Testing/Traits/SPIAwareTrait.swift diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index f205561a8..16a173b4b 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -96,7 +96,6 @@ add_library(Testing Traits/ConditionTrait+Macro.swift Traits/HiddenTrait.swift Traits/ParallelizationTrait.swift - Traits/SPIAwareTrait.swift Traits/Tags/Tag.Color.swift Traits/Tags/Tag.Color+Loading.swift Traits/Tags/Tag.List.swift diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index 7553acf6e..26cb00d14 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -28,7 +28,13 @@ extension Runner { /// ## See Also /// /// - ``ParallelizationTrait`` - public var isParallelizationEnabled: Bool + @available(*, deprecated, message: "The 'isParallelizationEnabled' property is deprecated and no longer used. Its value is always false.") + public var isParallelizationEnabled: Bool { + get { + false + } + set {} + } } /// The test should be run. @@ -65,19 +71,6 @@ extension Runner { return true } } - - /// Whether or not this action enables parallelization. - /// - /// If this action is of case ``run(options:)``, the value of this - /// property equals the value of its associated - /// ``RunOptions/isParallelizationEnabled`` property. Otherwise, the value - /// of this property is `nil`. - var isParallelizationEnabled: Bool? { - if case let .run(options) = self { - return options.isParallelizationEnabled - } - return nil - } } /// A type describing a step in a runner plan. @@ -218,7 +211,7 @@ extension Runner.Plan { // Convert the list of test into a graph of steps. The actions for these // steps will all be .run() *unless* an error was thrown while examining // them, in which case it will be .recordIssue(). - let runAction = Action.run(options: .init(isParallelizationEnabled: configuration.isParallelizationEnabled)) + let runAction = Action.run(options: .init()) var testGraph = Graph() var actionGraph = Graph(value: runAction) for test in tests { @@ -278,11 +271,7 @@ extension Runner.Plan { // `SkipInfo`, the error should not be recorded. for trait in test.traits { do { - if let trait = trait as? any SPIAwareTrait { - try await trait.prepare(for: test, action: &action) - } else { - try await trait.prepare(for: test) - } + try await trait.prepare(for: test) } catch let error as SkipInfo { action = .skip(error) break diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 4e20b4b4e..16eff103f 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -56,6 +56,16 @@ public struct Runner: Sendable { // MARK: - Running tests extension Runner { + /// The current configuration _while_ running. + /// + /// This should be used from the functions in this extension which access the + /// current configuration. This is important since individual tests or suites + /// may have traits which customize the execution scope of their children, + /// including potentially modifying the current configuration. + private static var _configuration: Configuration { + .current ?? .init() + } + /// Apply the custom scope for any test scope providers of the traits /// associated with a specified test by calling their /// ``TestScoping/provideScope(for:testCase:performing:)`` function. @@ -70,7 +80,7 @@ extension Runner { /// /// - Throws: Whatever is thrown by `body` or by any of the /// ``TestScoping/provideScope(for:testCase:performing:)`` function calls. - private func _applyScopingTraits( + private static func _applyScopingTraits( for test: Test, testCase: Test.Case?, _ body: @escaping @Sendable () async throws -> Void @@ -103,21 +113,13 @@ extension Runner { /// /// - Parameters: /// - sequence: The sequence to enumerate. - /// - step: The plan step that controls parallelization. If `nil`, or if its - /// ``Runner/Plan/Step/action`` property is not of case - /// ``Runner/Plan/Action/run(options:)``, the - /// ``Configuration/isParallelizationEnabled`` property of this runner's - /// ``configuration`` property is used instead to determine if - /// parallelization is enabled. /// - body: The function to invoke. /// /// - Throws: Whatever is thrown by `body`. - private func _forEach( + private static func _forEach( in sequence: some Sequence, - for step: Plan.Step?, _ body: @Sendable @escaping (E) async throws -> Void ) async throws where E: Sendable { - let isParallelizationEnabled = step?.action.isParallelizationEnabled ?? configuration.isParallelizationEnabled try await withThrowingTaskGroup(of: Void.self) { taskGroup in for element in sequence { // Each element gets its own subtask to run in. @@ -126,7 +128,7 @@ extension Runner { } // If not parallelizing, wait after each task. - if !isParallelizationEnabled { + if !_configuration.isParallelizationEnabled { try await taskGroup.waitForAll() } } @@ -137,12 +139,6 @@ extension Runner { /// /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, is to be run. - /// - depth: How deep into the step graph this call is. The first call has a - /// depth of `0`. - /// - lastAncestorStep: The last-known ancestral step, if any, of the step - /// at the root of `stepGraph`. The options in this step (if its action is - /// of case ``Runner/Plan/Action/run(options:)``) inform the execution of - /// `stepGraph`. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. @@ -157,7 +153,7 @@ extension Runner { /// ## See Also /// /// - ``Runner/run()`` - private func _runStep(atRootOf stepGraph: Graph, depth: Int, lastAncestorStep: Plan.Step?) async throws { + private static func _runStep(atRootOf stepGraph: Graph) async throws { // Exit early if the task has already been cancelled. try Task.checkCancellation() @@ -166,6 +162,8 @@ extension Runner { // example, a skip event only sends `.testSkipped`. let shouldSendTestEnded: Bool + let configuration = _configuration + // Determine what action to take for this step. if let step = stepGraph.value { Event.post(.planStepStarted(step), for: (step.test, nil), configuration: configuration) @@ -204,14 +202,14 @@ extension Runner { } // Run the children of this test (i.e. the tests in this suite.) - try await _runChildren(of: stepGraph, depth: depth, lastAncestorStep: lastAncestorStep) + try await _runChildren(of: stepGraph) } } } } else { // There is no test at this node in the graph, so just skip down to the // child nodes. - try await _runChildren(of: stepGraph, depth: depth, lastAncestorStep: lastAncestorStep) + try await _runChildren(of: stepGraph) } } @@ -222,7 +220,7 @@ extension Runner { /// /// - Returns: The source location of the root node of `stepGraph`, or of the /// first descendant node thereof (sorted by source location.) - private func _sourceLocation(of stepGraph: Graph) -> SourceLocation? { + private static func _sourceLocation(of stepGraph: Graph) -> SourceLocation? { if let result = stepGraph.value?.test.sourceLocation { return result } @@ -234,26 +232,13 @@ extension Runner { /// Recursively run the tests that are children of a given plan step. /// /// - Parameters: - /// - stepGraph: The subgraph whose root value, a step, is to be run. - /// - depth: How deep into the step graph this call is. The first call has a - /// depth of `0`. - /// - lastAncestorStep: The last-known ancestral step, if any, of the step - /// at the root of `stepGraph`. The options in this step (if its action is - /// of case ``Runner/Plan/Action/run(options:)``) inform the execution of - /// `stepGraph`. + /// - stepGraph: The subgraph whose root value, a step, will be used to + /// find children to run. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. - private func _runChildren(of stepGraph: Graph, depth: Int, lastAncestorStep: Plan.Step?) async throws { - // Figure out the last-good step, either the one at the root of `stepGraph` - // or, if it is nil, the one passed into this function. We need to track - // this value in case we run into sparse sections of the graph so we don't - // lose track of the recursive `isParallelizationEnabled` property in the - // runnable steps' options. - let stepOrAncestor = stepGraph.value ?? lastAncestorStep - - let isParallelizationEnabled = stepOrAncestor?.action.isParallelizationEnabled ?? configuration.isParallelizationEnabled - let childGraphs = if isParallelizationEnabled { + private static func _runChildren(of stepGraph: Graph) async throws { + let childGraphs = if _configuration.isParallelizationEnabled { // Explicitly shuffle the steps to help detect accidental dependencies // between tests due to their ordering. Array(stepGraph.children) @@ -282,8 +267,8 @@ extension Runner { } // Run the child nodes. - try await _forEach(in: childGraphs, for: stepOrAncestor) { _, childGraph in - try await _runStep(atRootOf: childGraph, depth: depth + 1, lastAncestorStep: stepOrAncestor) + try await _forEach(in: childGraphs) { _, childGraph in + try await _runStep(atRootOf: childGraph) } } @@ -298,13 +283,14 @@ extension Runner { /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async throws { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async throws { // Apply the configuration's test case filter. + let testCaseFilter = _configuration.testCaseFilter let testCases = testCases.lazy.filter { testCase in - configuration.testCaseFilter(testCase, step.test) + testCaseFilter(testCase, step.test) } - try await _forEach(in: testCases, for: step) { testCase in + try await _forEach(in: testCases) { testCase in try await _runTestCase(testCase, within: step) } } @@ -320,10 +306,12 @@ extension Runner { /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async throws { + private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async throws { // Exit early if the task has already been cancelled. try Task.checkCancellation() + let configuration = _configuration + Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) defer { Event.post(.testCaseEnded, for: (step.test, testCase), configuration: configuration) @@ -357,9 +345,6 @@ extension Runner { /// /// - Parameters: /// - runner: The runner to run. - /// - configuration: The configuration to use for running. The value of this - /// argument temporarily replaces the value of `runner`'s - /// ``Runner/configuration`` property. /// /// This function is `static` so that it cannot accidentally reference `self` /// or `self.configuration` when it should use a modified copy of either. @@ -399,7 +384,7 @@ extension Runner { await withTaskGroup(of: Void.self) { [runner] taskGroup in _ = taskGroup.addTaskUnlessCancelled { - try? await runner._runStep(atRootOf: runner.plan.stepGraph, depth: 0, lastAncestorStep: nil) + try? await _runStep(atRootOf: runner.plan.stepGraph) } await taskGroup.waitForAll() } diff --git a/Sources/Testing/Traits/ParallelizationTrait.swift b/Sources/Testing/Traits/ParallelizationTrait.swift index d41052e25..df34f4d63 100644 --- a/Sources/Testing/Traits/ParallelizationTrait.swift +++ b/Sources/Testing/Traits/ParallelizationTrait.swift @@ -11,14 +11,13 @@ /// A type that affects whether or not a test or suite is parallelized. /// /// When added to a parameterized test function, this trait causes that test to -/// run its cases serially instead of in parallel. When applied to a -/// non-parameterized test function, this trait has no effect. When applied to a -/// test suite, this trait causes that suite to run its contained test functions -/// and sub-suites serially instead of in parallel. +/// run its cases serially instead of in parallel. When added to a +/// non-parameterized test function, this trait has no effect. /// -/// This trait is recursively applied: if it is applied to a suite, any -/// parameterized tests or test suites contained in that suite are also -/// serialized (as are any tests contained in those suites, and so on.) +/// When added to a test suite, this trait causes that suite to run its +/// contained test functions (including their cases, when parameterized) and +/// sub-suites serially instead of in parallel. Any children of sub-suites are +/// also run serially. /// /// This trait does not affect the execution of a test relative to its peers or /// to unrelated tests. This trait has no effect if test parallelization is @@ -26,21 +25,18 @@ /// `swift test` command.) /// /// To add this trait to a test, use ``Trait/serialized``. -public struct ParallelizationTrait: TestTrait, SuiteTrait { - public var isRecursive: Bool { - true - } -} +public struct ParallelizationTrait: TestTrait, SuiteTrait {} -// MARK: - SPIAwareTrait +// MARK: - TestScoping -@_spi(ForToolsIntegrationOnly) -extension ParallelizationTrait: SPIAwareTrait { - public func prepare(for test: Test, action: inout Runner.Plan.Action) async throws { - if case var .run(options) = action { - options.isParallelizationEnabled = false - action = .run(options: options) +extension ParallelizationTrait: TestScoping { + public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + guard var configuration = Configuration.current else { + throw SystemError(description: "There is no current Configuration when attempting to provide scope for test '\(test.name)'. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } + + configuration.isParallelizationEnabled = false + try await Configuration.withCurrent(configuration, perform: function) } } diff --git a/Sources/Testing/Traits/SPIAwareTrait.swift b/Sources/Testing/Traits/SPIAwareTrait.swift deleted file mode 100644 index 780dba2d6..000000000 --- a/Sources/Testing/Traits/SPIAwareTrait.swift +++ /dev/null @@ -1,38 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// A protocol describing traits that can be added to a test function or to a -/// test suite and that make use of SPI symbols in the testing library. -/// -/// This protocol refines ``Trait`` in various ways that require the use of SPI. -/// Ideally, such requirements will be promoted to API when their design -/// stabilizes. -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) -public protocol SPIAwareTrait: Trait { - /// Prepare to run the test to which this trait was added. - /// - /// - Parameters: - /// - test: The test to which this trait was added. - /// - action: The test plan action to use with `test`. The implementation - /// may modify this value. - /// - /// - Throws: Any error that would prevent the test from running. If an error - /// is thrown from this method, the test will be skipped and the error will - /// be recorded as an ``Issue``. - /// - /// This method is called after all tests and their traits have been - /// discovered by the testing library, but before any test has begun running. - /// It may be used to prepare necessary internal state, or to influence - /// whether the test should run. - /// - /// For types that conform to this protocol, ``Runner/Plan`` calls this method - /// instead of ``Trait/prepare(for:)``. - func prepare(for test: Test, action: inout Runner.Plan.Action) async throws -} diff --git a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift index 85f69c06c..e43ca50b7 100644 --- a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift +++ b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift @@ -12,16 +12,6 @@ @Suite("Parallelization Trait Tests", .tags(.traitRelated)) struct ParallelizationTraitTests { - @Test(".serialized trait is recursively applied") - func serializedTrait() async { - var configuration = Configuration() - configuration.isParallelizationEnabled = true - let plan = await Runner.Plan(selecting: OuterSuite.self, configuration: configuration) - for step in plan.steps { - #expect(step.action.isParallelizationEnabled == false, "Step \(step) should have had parallelization disabled") - } - } - @Test(".serialized trait serializes parameterized test") func serializesParameterizedTestFunction() async { var configuration = Configuration() From 52aa355f08587fc60b9a20174972a6abb9ca5e2d Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 10 Jan 2025 16:19:51 -0500 Subject: [PATCH 046/234] Remove C++-based section discovery logic. (#902) This PR removes the C++ _section_ discovery logic that's used to look up Swift's type metadata sections. Instead, we reuse the new logic used for test content sections (written in Swift.) I did not attempt to translate to Swift the logic to extract Swift types from Swift's metadata records. This logic relies pretty heavily on precise C++ data structure layouts and uses pointer authentication as well as relative pointers. Swift does not guarantee the same structure layout as C++, nor does it provide API for pointer authentication, nor do value types in Swift have stable addresses that can be used to compute addresses from relative offsets. Resolves #764. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/Porting.md | 146 ++++---- Sources/Testing/Discovery+Platform.swift | 146 ++++++-- Sources/Testing/Discovery.swift | 2 +- Sources/Testing/ExitTests/ExitTest.swift | 13 +- Sources/Testing/Test+Discovery+Legacy.swift | 40 +-- Sources/Testing/Test+Discovery.swift | 11 +- Sources/_TestingInternals/Discovery.cpp | 331 +++--------------- Sources/_TestingInternals/include/Discovery.h | 45 +-- 8 files changed, 281 insertions(+), 453 deletions(-) diff --git a/Documentation/Porting.md b/Documentation/Porting.md index 39d1d8e88..ce179d53d 100644 --- a/Documentation/Porting.md +++ b/Documentation/Porting.md @@ -66,7 +66,7 @@ platform-specific attention. > These errors are produced when the configuration you're trying to build has > conflicting requirements (for example, attempting to enable support for pipes > without also enabling support for file I/O.) You should be able to resolve -> these issues by updating Package.swift and/or CompilerSettings.cmake. +> these issues by updating `Package.swift` and/or `CompilerSettings.cmake`. Most platform dependencies can be resolved through the use of platform-specific API. For example, Swift Testing uses the C11 standard [`timespec`](https://en.cppreference.com/w/c/chrono/timespec) @@ -123,69 +123,110 @@ Once the header is included, we can call `GetDateTime()` from `Clock.swift`: ## Runtime test discovery When porting to a new platform, you may need to provide a new implementation for -`enumerateTypeMetadataSections()` in `Discovery.cpp`. Test discovery is -dependent on Swift metadata discovery which is an inherently platform-specific -operation. - -_Most_ platforms will be able to reuse the implementation used by Linux and -Windows that calls an internal Swift runtime function to enumerate available -metadata. If you are porting Swift Testing to Classic, this function won't be -available, so you'll need to write a custom implementation instead. Assuming -that the Swift compiler emits section information into the resource fork on -Classic, you could use the [Resource Manager](https://developer.apple.com/library/archive/documentation/mac/pdf/MoreMacintoshToolbox.pdf) +`_sectionBounds(_:)` in `Discovery+Platform.swift`. Test discovery is dependent +on Swift metadata discovery which is an inherently platform-specific operation. + +_Most_ platforms in use today use the ELF image format and will be able to reuse +the implementation used by Linux. + +Classic does not use the ELF image format, so you'll need to write a custom +implementation of `_sectionBounds(_:)` instead. Assuming that the Swift compiler +emits section information into the resource fork on Classic, you would use the +[Resource Manager](https://developer.apple.com/library/archive/documentation/mac/pdf/MoreMacintoshToolbox.pdf) to load that information: ```diff ---- a/Sources/_TestingInternals/Discovery.cpp -+++ b/Sources/_TestingInternals/Discovery.cpp +--- a/Sources/Testing/Discovery+Platform.swift ++++ b/Sources/Testing/Discovery+Platform.swift // ... -+#elif defined(macintosh) -+template -+static void enumerateTypeMetadataSections(const SectionEnumerator& body) { -+ ResFileRefNum refNum; -+ if (noErr == GetTopResourceFile(&refNum)) { -+ ResFileRefNum oldRefNum = refNum; -+ do { -+ UseResFile(refNum); -+ Handle handle = Get1NamedResource('swft', "\p__swift5_types"); -+ if (handle && *handle) { -+ auto imageAddress = reinterpret_cast(static_cast(refNum)); -+ SWTSectionBounds sb = { imageAddress, *handle, GetHandleSize(handle) }; -+ bool stop = false; -+ body(sb, &stop); -+ if (stop) { -+ break; -+ } -+ } -+ } while (noErr == GetNextResourceFile(refNum, &refNum)); -+ UseResFile(oldRefNum); ++#elseif os(Classic) ++private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { ++ let resourceName: Str255 = switch kind { ++ case .testContent: ++ "__swift5_tests" ++ case .typeMetadata: ++ "__swift5_types" ++ } ++ ++ let oldRefNum = CurResFile() ++ defer { ++ UseResFile(oldRefNum) ++ } ++ ++ var refNum = ResFileRefNum(0) ++ guard noErr == GetTopResourceFile(&refNum) else { ++ return [] + } ++ ++ var result = [SectionBounds]() ++ repeat { ++ UseResFile(refNum) ++ guard let handle = Get1NamedResource(ResType("swft"), resourceName) else { ++ continue ++ } ++ let sb = SectionBounds( ++ imageAddress: UnsafeRawPointer(bitPattern: UInt(refNum)), ++ start: handle.pointee!, ++ size: GetHandleSize(handle) ++ ) ++ result.append(sb) ++ } while noErr == GetNextResourceFile(refNum, &refNum)) ++ return result +} #else - #warning Platform-specific implementation missing: Runtime test discovery unavailable (dynamic) - template - static void enumerateTypeMetadataSections(const SectionEnumerator& body) {} + private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { + #warning("Platform-specific implementation missing: Runtime test discovery unavailable (dynamic)") + return [] + } #endif ``` +You will also need to update the `makeTestContentRecordDecl()` function in the +`TestingMacros` target to emit the correct `@_section` attribute for your +platform. If your platform uses the ELF image format and supports the +`dl_iterate_phdr()` function, add it to the existing `#elseif os(Linux) || ...` +case. Otherwise, add a new case for your platform: + +```diff +--- a/Sources/TestingMacros/Support/TestContentGeneration.swift ++++ b/Sources/TestingMacros/Support/TestContentGeneration.swift + // ... ++ #elseif os(Classic) ++ @_section(".rsrc,swft,__swift5_tests") + #else + @__testing(warning: "Platform-specific implementation missing: test content section name unavailable") + #endif +``` + +Keep in mind that this code is emitted by the `@Test` and `@Suite` macros +directly into test authors' test targets, so you will not be able to use +compiler conditionals defined in the Swift Testing package (including those that +start with `"SWT_"`). + ## Runtime test discovery with static linkage If your platform does not support dynamic linking and loading, you will need to use static linkage instead. Define the `"SWT_NO_DYNAMIC_LINKING"` compiler -conditional for your platform in both Package.swift and CompilerSettings.cmake, -then define the `sectionBegin` and `sectionEnd` symbols in Discovery.cpp: +conditional for your platform in both `Package.swift` and +`CompilerSettings.cmake`, then define the symbols `testContentSectionBegin`, +`testContentSectionEnd`, `typeMetadataSectionBegin`, and +`typeMetadataSectionEnd` in `Discovery.cpp`. ```diff diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp // ... +#elif defined(macintosh) -+extern "C" const char sectionBegin __asm__("..."); -+extern "C" const char sectionEnd __asm__("..."); ++extern "C" const char testContentSectionBegin __asm__("..."); ++extern "C" const char testContentSectionEnd __asm__("..."); ++extern "C" const char typeMetadataSectionBegin __asm__("..."); ++extern "C" const char typeMetadataSectionEnd __asm__("..."); #else #warning Platform-specific implementation missing: Runtime test discovery unavailable (static) - static const char sectionBegin = 0; - static const char& sectionEnd = sectionBegin; + static const char testContentSectionBegin = 0; + static const char& testContentSectionEnd = testContentSectionBegin; + static const char typeMetadataSectionBegin = 0; + static const char& typeMetadataSectionEnd = testContentSectionBegin; #endif ``` @@ -195,27 +236,6 @@ respectively. Their linker-level names will be platform-dependent: refer to the linker documentation for your platform to determine what names to place in the `__asm__` attribute applied to each. -If you can't use `__asm__` on your platform, you can declare these symbols as -C++ references to linker-defined symbols: - -```diff -diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp - // ... -+#elif defined(macintosh) -+extern "C" const char __linker_defined_begin_symbol; -+extern "C" const char __linker_defined_end_symbol; -+static const auto& sectionBegin = __linker_defined_begin_symbol; -+static const auto& sectionEnd = __linker_defined_end_symbol; - #else - #warning Platform-specific implementation missing: Runtime test discovery unavailable (static) - static const char sectionBegin = 0; - static const char& sectionEnd = sectionBegin; - #endif -``` - -The names of `__linker_defined_begin_symbol` and `__linker_defined_end_symbol` -in this example are, as with the shorter implementation, platform-dependent. - ## C++ stub implementations Some symbols defined in C and C++ headers, especially "complex" macros, cannot diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index d85554498..7fcb94fe5 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -18,9 +18,25 @@ struct SectionBounds: Sendable { /// The in-memory representation of the section. nonisolated(unsafe) var buffer: UnsafeRawBufferPointer - /// All test content section bounds found in the current process. - static var allTestContent: some RandomAccessCollection { - _testContentSectionBounds() + /// An enumeration describing the different sections discoverable by the + /// testing library. + enum Kind: Equatable, Hashable, CaseIterable { + /// The test content metadata section. + case testContent + + /// The type metadata section. + case typeMetadata + } + + /// All section bounds of the given kind found in the current process. + /// + /// - Parameters: + /// - kind: Which kind of metadata section to return. + /// + /// - Returns: A sequence of structures describing the bounds of metadata + /// sections of the given kind found in the current process. + static func all(_ kind: Kind) -> some RandomAccessCollection { + _sectionBounds(kind) } } @@ -30,14 +46,17 @@ struct SectionBounds: Sendable { /// An array containing all of the test content section bounds known to the /// testing library. -private let _sectionBounds = Locked<[SectionBounds]>(rawValue: []) +private let _sectionBounds = Locked<[SectionBounds.Kind: [SectionBounds]]>() /// A call-once function that initializes `_sectionBounds` and starts listening /// for loaded Mach headers. private let _startCollectingSectionBounds: Void = { // Ensure _sectionBounds is initialized before we touch libobjc or dyld. _sectionBounds.withLock { sectionBounds in - sectionBounds.reserveCapacity(Int(_dyld_image_count())) + let imageCount = Int(clamping: _dyld_image_count()) + for kind in SectionBounds.Kind.allCases { + sectionBounds[kind, default: []].reserveCapacity(imageCount) + } } func addSectionBounds(from mh: UnsafePointer) { @@ -55,12 +74,32 @@ private let _startCollectingSectionBounds: Void = { // If this image contains the Swift section we need, acquire the lock and // store the section's bounds. - var size = CUnsignedLong(0) - if let start = getsectiondata(mh, "__DATA_CONST", "__swift5_tests", &size), size > 0 { - _sectionBounds.withLock { sectionBounds in + let testContentSectionBounds: SectionBounds? = { + var size = CUnsignedLong(0) + if let start = getsectiondata(mh, "__DATA_CONST", "__swift5_tests", &size), size > 0 { + let buffer = UnsafeRawBufferPointer(start: start, count: Int(clamping: size)) + return SectionBounds(imageAddress: mh, buffer: buffer) + } + return nil + }() + + let typeMetadataSectionBounds: SectionBounds? = { + var size = CUnsignedLong(0) + if let start = getsectiondata(mh, "__TEXT", "__swift5_types", &size), size > 0 { let buffer = UnsafeRawBufferPointer(start: start, count: Int(clamping: size)) - let sb = SectionBounds(imageAddress: mh, buffer: buffer) - sectionBounds.append(sb) + return SectionBounds(imageAddress: mh, buffer: buffer) + } + return nil + }() + + if testContentSectionBounds != nil || typeMetadataSectionBounds != nil { + _sectionBounds.withLock { sectionBounds in + if let testContentSectionBounds { + sectionBounds[.testContent]!.append(testContentSectionBounds) + } + if let typeMetadataSectionBounds { + sectionBounds[.typeMetadata]!.append(typeMetadataSectionBounds) + } } } } @@ -76,13 +115,16 @@ private let _startCollectingSectionBounds: Void = { #endif }() -/// The Apple-specific implementation of ``SectionBounds/all``. +/// The Apple-specific implementation of ``SectionBounds/all(_:)``. +/// +/// - Parameters: +/// - kind: Which kind of metadata section to return. /// /// - Returns: An array of structures describing the bounds of all known test /// content sections in the current process. -private func _testContentSectionBounds() -> [SectionBounds] { +private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { _startCollectingSectionBounds - return _sectionBounds.rawValue + return _sectionBounds.rawValue[kind]! } #elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) @@ -90,37 +132,51 @@ private func _testContentSectionBounds() -> [SectionBounds] { private import SwiftShims // For MetadataSections -/// The ELF-specific implementation of ``SectionBounds/all``. +/// The ELF-specific implementation of ``SectionBounds/all(_:)``. +/// +/// - Parameters: +/// - kind: Which kind of metadata section to return. /// /// - Returns: An array of structures describing the bounds of all known test /// content sections in the current process. -private func _testContentSectionBounds() -> [SectionBounds] { - var result = [SectionBounds]() +private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { + struct Context { + var kind: SectionBounds.Kind + var result = [SectionBounds]() + } + var context = Context(kind: kind) - withUnsafeMutablePointer(to: &result) { result in + withUnsafeMutablePointer(to: &context) { context in swift_enumerateAllMetadataSections({ sections, context in + let context = context.assumingMemoryBound(to: Context.self) + let version = sections.load(as: UInt.self) - guard version >= 4 else { + guard context.pointee.kind != .testContent || version >= 4 else { // This structure is too old to contain the swift5_tests field. return true } - let sections = sections.load(as: MetadataSections.self) - let result = context.assumingMemoryBound(to: [SectionBounds].self) - let start = UnsafeRawPointer(bitPattern: sections.swift5_tests.start) - let size = Int(clamping: sections.swift5_tests.length) + let range = switch context.pointee.kind { + case .testContent: + sections.swift5_tests + case .typeMetadata: + sections.swift5_type_metadata + } + let start = UnsafeRawPointer(bitPattern: range.start) + let size = Int(clamping: range.length) if let start, size > 0 { let buffer = UnsafeRawBufferPointer(start: start, count: size) let sb = SectionBounds(imageAddress: sections.baseAddress, buffer: buffer) - result.pointee.append(sb) + + context.pointee.result.append(sb) } return true - }, result) + }, context) } - return result + return context.result } #elseif os(Windows) @@ -183,19 +239,31 @@ private func _findSection(named sectionName: String, in hModule: HMODULE) -> Sec } } -/// The Windows-specific implementation of ``SectionBounds/all``. +/// The Windows-specific implementation of ``SectionBounds/all(_:)``. +/// +/// - Parameters: +/// - kind: Which kind of metadata section to return. /// /// - Returns: An array of structures describing the bounds of all known test /// content sections in the current process. -private func _testContentSectionBounds() -> [SectionBounds] { - HMODULE.all.compactMap { _findSection(named: ".sw5test", in: $0) } +private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { + let sectionName = switch kind { + case .testContent: + ".sw5test" + case .typeMetadata: + ".sw5tymd" + } + return HMODULE.all.compactMap { _findSection(named: sectionName, in: $0) } } #else -/// The fallback implementation of ``SectionBounds/all`` for platforms that +/// The fallback implementation of ``SectionBounds/all(_:)`` for platforms that /// support dynamic linking. /// +/// - Parameters: +/// - kind: Ignored. +/// /// - Returns: The empty array. -private func _testContentSectionBounds() -> [SectionBounds] { +private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { #warning("Platform-specific implementation missing: Runtime test discovery unavailable (dynamic)") return [] } @@ -203,13 +271,21 @@ private func _testContentSectionBounds() -> [SectionBounds] { #else // MARK: - Statically-linked implementation -/// The common implementation of ``SectionBounds/all`` for platforms that do not -/// support dynamic linking. +/// The common implementation of ``SectionBounds/all(_:)`` for platforms that do +/// not support dynamic linking. +/// +/// - Parameters: +/// - kind: Which kind of metadata section to return. /// -/// - Returns: A structure describing the bounds of the test content section +/// - Returns: A structure describing the bounds of the type metadata section /// contained in the same image as the testing library itself. -private func _testContentSectionBounds() -> CollectionOfOne { - let (sectionBegin, sectionEnd) = SWTTestContentSectionBounds +private func _sectionBounds(_ kind: SectionBounds.Kind) -> CollectionOfOne { + let (sectionBegin, sectionEnd) = switch kind { + case .testContent: + SWTTestContentSectionBounds + case .typeMetadata: + SWTTypeMetadataSectionBounds + } let buffer = UnsafeRawBufferPointer(start: sectionBegin, count: max(0, sectionEnd - sectionBegin)) let sb = SectionBounds(imageAddress: nil, buffer: buffer) return CollectionOfOne(sb) diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index b2fc7825c..6343b92f4 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -157,7 +157,7 @@ extension TestContent where Self: ~Copyable { /// is used with move-only types (specifically ``ExitTest``) and /// `Sequence.Element` must be copyable. static func enumerateTestContent(withHint hint: TestContentAccessorHint? = nil, _ body: TestContentEnumerator) { - let testContentRecords = SectionBounds.allTestContent.lazy.flatMap(_testContentRecords(in:)) + let testContentRecords = SectionBounds.all(.testContent).lazy.flatMap(_testContentRecords(in:)) var stop = false for (imageAddress, record) in testContentRecords { diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 01810a7ca..96d636b11 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -250,19 +250,16 @@ extension ExitTest { if result == nil { // Call the legacy lookup function that discovers tests embedded in types. - enumerateTypes(withNamesContaining: exitTestContainerTypeNameMagic) { _, type, stop in - guard let type = type as? any __ExitTestContainer.Type else { - return - } - if type.__sourceLocation == sourceLocation { - result = ExitTest( + result = types(withNamesContaining: exitTestContainerTypeNameMagic).lazy + .compactMap { $0 as? any __ExitTestContainer.Type } + .first { $0.__sourceLocation == sourceLocation } + .map { type in + ExitTest( __expectedExitCondition: type.__expectedExitCondition, sourceLocation: type.__sourceLocation, body: type.__body ) - stop = true } - } } return result diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index b65d72b39..d330104ef 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -49,35 +49,23 @@ let exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" // MARK: - -/// The type of callback called by ``enumerateTypes(withNamesContaining:_:)``. -/// -/// - Parameters: -/// - imageAddress: A pointer to the start of the image. This value is _not_ -/// equal to the value returned from `dlopen()`. On platforms that do not -/// support dynamic loading (and so do not have loadable images), this -/// argument is unspecified. -/// - type: A Swift type. -/// - stop: An `inout` boolean variable indicating whether type enumeration -/// should stop after the function returns. Set `stop` to `true` to stop -/// type enumeration. -typealias TypeEnumerator = (_ imageAddress: UnsafeRawPointer?, _ type: Any.Type, _ stop: inout Bool) -> Void - -/// Enumerate all types known to Swift found in the current process whose names +/// Get all types known to Swift found in the current process whose names /// contain a given substring. /// /// - Parameters: /// - nameSubstring: A string which the names of matching classes all contain. -/// - body: A function to invoke, once per matching type. -func enumerateTypes(withNamesContaining nameSubstring: String, _ typeEnumerator: TypeEnumerator) { - withoutActuallyEscaping(typeEnumerator) { typeEnumerator in - withUnsafePointer(to: typeEnumerator) { context in - swt_enumerateTypes(withNamesContaining: nameSubstring, .init(mutating: context)) { imageAddress, type, stop, context in - let typeEnumerator = context!.load(as: TypeEnumerator.self) - let type = unsafeBitCast(type, to: Any.Type.self) - var stop2 = false - typeEnumerator(imageAddress, type, &stop2) - stop.pointee = stop2 +/// +/// - Returns: A sequence of Swift types whose names contain `nameSubstring`. +func types(withNamesContaining nameSubstring: String) -> some Sequence { + SectionBounds.all(.typeMetadata).lazy + .map { sb in + var count = 0 + let start = swt_copyTypes(in: sb.buffer.baseAddress!, sb.buffer.count, withNamesContaining: nameSubstring, count: &count) + defer { + free(start) + } + return start.withMemoryRebound(to: Any.Type.self, capacity: count) { start in + Array(UnsafeBufferPointer(start: start, count: count)) } - } - } + }.joined() } diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 9a187c917..751a7de85 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -56,14 +56,11 @@ extension Test: TestContent { } if discoveryMode != .newOnly && generators.isEmpty { - enumerateTypes(withNamesContaining: testContainerTypeNameMagic) { imageAddress, type, _ in - guard let type = type as? any __TestContainer.Type else { - return + generators += types(withNamesContaining: testContainerTypeNameMagic).lazy + .compactMap { $0 as? any __TestContainer.Type } + .map { type in + { @Sendable in await type.__tests } } - generators.append { @Sendable in - await type.__tests - } - } } // *Now* we call all the generators and return their results. diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index 8af5e1690..08ffc5f47 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -16,13 +16,19 @@ #if defined(__APPLE__) extern "C" const char testContentSectionBegin __asm("section$start$__DATA_CONST$__swift5_tests"); extern "C" const char testContentSectionEnd __asm("section$end$__DATA_CONST$__swift5_tests"); +extern "C" const char typeMetadataSectionBegin __asm__("section$start$__TEXT$__swift5_types"); +extern "C" const char typeMetadataSectionEnd __asm__("section$end$__TEXT$__swift5_types"); #elif defined(__wasi__) extern "C" const char testContentSectionBegin __asm__("__start_swift5_tests"); extern "C" const char testContentSectionEnd __asm__("__stop_swift5_tests"); +extern "C" const char typeMetadataSectionBegin __asm__("__start_swift5_type_metadata"); +extern "C" const char typeMetadataSectionEnd __asm__("__stop_swift5_type_metadata"); #else #warning Platform-specific implementation missing: Runtime test discovery unavailable (static) static const char testContentSectionBegin = 0; static const char& testContentSectionEnd = testContentSectionBegin; +static const char typeMetadataSectionBegin = 0; +static const char& typeMetadataSectionEnd = typeMetadataSectionBegin; #endif /// The bounds of the test content section statically linked into the image @@ -31,6 +37,13 @@ const void *_Nonnull const SWTTestContentSectionBounds[2] = { &testContentSectionBegin, &testContentSectionEnd }; + +/// The bounds of the type metadata section statically linked into the image +/// containing Swift Testing. +const void *_Nonnull const SWTTypeMetadataSectionBounds[2] = { + &typeMetadataSectionBegin, + &typeMetadataSectionEnd +}; #endif #pragma mark - Legacy test discovery @@ -40,19 +53,12 @@ const void *_Nonnull const SWTTestContentSectionBounds[2] = { #include #include #include +#include #include #include #include #include -#if defined(__APPLE__) && !defined(SWT_NO_DYNAMIC_LINKING) -#include -#include -#include -#include -#include -#endif - /// Enumerate over all Swift type metadata sections in the current process. /// /// - Parameters: @@ -264,298 +270,41 @@ struct SWTTypeMetadataRecord { } }; -#if !defined(SWT_NO_DYNAMIC_LINKING) -#if defined(__APPLE__) -#pragma mark - Apple implementation - -/// Get a copy of the currently-loaded type metadata sections list. -/// -/// - Returns: A list of type metadata sections in images loaded into the -/// current process. The order of the resulting list is unspecified. -/// -/// On ELF-based platforms, the `swift_enumerateAllMetadataSections()` function -/// exported by the runtime serves the same purpose as this function. -static SWTSectionBoundsList getSectionBounds(void) { - /// This list is necessarily mutated while a global libobjc- or dyld-owned - /// lock is held. Hence, code using this list must avoid potentially - /// re-entering either library (otherwise it could potentially deadlock.) - /// - /// To see how the Swift runtime accomplishes the above goal, see - /// `ConcurrentReadableArray` in that project's Concurrent.h header. Since the - /// testing library is not tasked with the same performance constraints as - /// Swift's runtime library, we just use a `std::vector` guarded by an unfair - /// lock. - static constinit SWTSectionBoundsList *sectionBounds = nullptr; - static constinit os_unfair_lock lock = OS_UNFAIR_LOCK_INIT; - - static constinit dispatch_once_t once = 0; - dispatch_once_f(&once, nullptr, [] (void *) { - sectionBounds = reinterpret_cast *>(std::malloc(sizeof(SWTSectionBoundsList))); - ::new (sectionBounds) SWTSectionBoundsList(); - sectionBounds->reserve(_dyld_image_count()); - - objc_addLoadImageFunc([] (const mach_header *mh) { -#if __LP64__ - auto mhn = reinterpret_cast(mh); -#else - auto mhn = mh; -#endif - - // Ignore this Mach header if it is in the shared cache. On platforms that - // support it (Darwin), most system images are contained in this range. - // System images can be expected not to contain test declarations, so we - // don't need to walk them. - if (mhn->flags & MH_DYLIB_IN_CACHE) { - return; - } +#pragma mark - - // If this image contains the Swift section we need, acquire the lock and - // store the section's bounds. - unsigned long size = 0; - auto start = getsectiondata(mhn, SEG_TEXT, "__swift5_types", &size); - if (start && size > 0) { - os_unfair_lock_lock(&lock); { - sectionBounds->emplace_back(mhn, start, size); - } os_unfair_lock_unlock(&lock); - } - }); - }); - - // After the first call sets up the loader hook, all calls take the lock and - // make a copy of whatever has been loaded so far. - SWTSectionBoundsList result; - result.reserve(_dyld_image_count()); - os_unfair_lock_lock(&lock); { - result = *sectionBounds; - } os_unfair_lock_unlock(&lock); - result.shrink_to_fit(); - return result; -} +void **swt_copyTypesWithNamesContaining(const void *sectionBegin, size_t sectionSize, const char *nameSubstring, size_t *outCount) { + SWTSectionBounds sb = { nullptr, sectionBegin, sectionSize }; + std::vector> result; -template -static void enumerateTypeMetadataSections(const SectionEnumerator& body) { - bool stop = false; - for (const auto& sb : getSectionBounds()) { - body(sb, &stop); - if (stop) { - break; + for (const auto& record : sb) { + auto contextDescriptor = record.getContextDescriptor(); + if (!contextDescriptor) { + // This type metadata record is invalid (or we don't understand how to + // get its context descriptor), so skip it. + continue; + } else if (contextDescriptor->isGeneric()) { + // Generic types cannot be fully instantiated without generic + // parameters, which is not something we can know abstractly. + continue; } - } -} - -#elif defined(_WIN32) -#pragma mark - Windows implementation - -/// Find the section with the given name in the given module. -/// -/// - Parameters: -/// - hModule: The module to inspect. -/// - sectionName: The name of the section to look for. Long section names are -/// not supported. -/// -/// - Returns: A pointer to the start of the given section along with its size -/// in bytes, or `std::nullopt` if the section could not be found. If the -/// section was emitted by the Swift toolchain, be aware it will have leading -/// and trailing bytes (`sizeof(uintptr_t)` each.) -static std::optional> findSection(HMODULE hModule, const char *sectionName) { - if (!hModule) { - return std::nullopt; - } - // Get the DOS header (to which the HMODULE directly points, conveniently!) - // and check it's sufficiently valid for us to walk. - auto dosHeader = reinterpret_cast(hModule); - if (dosHeader->e_magic != IMAGE_DOS_SIGNATURE || dosHeader->e_lfanew <= 0) { - return std::nullopt; - } - - // Check the NT header. Since we don't use the optional header, skip it. - auto ntHeader = reinterpret_cast(reinterpret_cast(dosHeader) + dosHeader->e_lfanew); - if (!ntHeader || ntHeader->Signature != IMAGE_NT_SIGNATURE) { - return std::nullopt; - } - - auto sectionCount = ntHeader->FileHeader.NumberOfSections; - auto section = IMAGE_FIRST_SECTION(ntHeader); - for (size_t i = 0; i < sectionCount; i++, section += 1) { - if (section->VirtualAddress == 0) { + // Check that the type's name passes. This will be more expensive than the + // checks above, but should be cheaper than realizing the metadata. + const char *typeName = contextDescriptor->getName(); + bool nameOK = typeName && nullptr != std::strstr(typeName, nameSubstring); + if (!nameOK) { continue; } - auto start = reinterpret_cast(reinterpret_cast(dosHeader) + section->VirtualAddress); - size_t size = std::min(section->Misc.VirtualSize, section->SizeOfRawData); - if (start && size > 0) { - // FIXME: Handle longer names ("/%u") from string table - auto thisSectionName = reinterpret_cast(section->Name); - if (0 == std::strncmp(sectionName, thisSectionName, IMAGE_SIZEOF_SHORT_NAME)) { - return SWTSectionBounds { hModule, start, size }; - } + if (void *typeMetadata = contextDescriptor->getMetadata()) { + result.push_back(typeMetadata); } } - return std::nullopt; -} - -template -static void enumerateTypeMetadataSections(const SectionEnumerator& body) { - // Find all the modules loaded in the current process. We assume there aren't - // more than 1024 loaded modules (as does Microsoft sample code.) - std::array hModules; - DWORD byteCountNeeded = 0; - if (!EnumProcessModules(GetCurrentProcess(), &hModules[0], hModules.size() * sizeof(HMODULE), &byteCountNeeded)) { - return; + auto resultCopy = reinterpret_cast(std::calloc(sizeof(void *), result.size())); + if (resultCopy) { + std::uninitialized_move(result.begin(), result.end(), resultCopy); + *outCount = result.size(); } - size_t hModuleCount = std::min(hModules.size(), static_cast(byteCountNeeded) / sizeof(HMODULE)); - - // Look in all the loaded modules for Swift type metadata sections and store - // them in a side table. - // - // This two-step process is more complicated to read than a single loop would - // be but it is safer: the callback will eventually invoke developer code that - // could theoretically unload a module from the list we're enumerating. (Swift - // modules do not support unloading, so we'll just not worry about them.) - SWTSectionBoundsList sectionBounds; - sectionBounds.reserve(hModuleCount); - for (size_t i = 0; i < hModuleCount; i++) { - if (auto sb = findSection(hModules[i], ".sw5tymd")) { - sectionBounds.push_back(*sb); - } - } - - // Pass each discovered section back to the body callback. - // - // NOTE: we ignore the leading and trailing uintptr_t values: they're both - // always set to zero so we'll skip them in the callback, and in the future - // the toolchain might not emit them at all in which case we don't want to - // skip over real section data. - bool stop = false; - for (const auto& sb : sectionBounds) { - body(sb, &stop); - if (stop) { - break; - } - } -} - -#elif defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__ANDROID__) -#pragma mark - ELF implementation - -/// Specifies the address range corresponding to a section. -struct MetadataSectionRange { - uintptr_t start; - size_t length; -}; - -/// Identifies the address space ranges for the Swift metadata required by the -/// Swift runtime. -struct MetadataSections { - uintptr_t version; - std::atomic baseAddress; - - void *unused0; - void *unused1; - - MetadataSectionRange swift5_protocols; - MetadataSectionRange swift5_protocol_conformances; - MetadataSectionRange swift5_type_metadata; - MetadataSectionRange swift5_typeref; - MetadataSectionRange swift5_reflstr; - MetadataSectionRange swift5_fieldmd; - MetadataSectionRange swift5_assocty; - MetadataSectionRange swift5_replace; - MetadataSectionRange swift5_replac2; - MetadataSectionRange swift5_builtin; - MetadataSectionRange swift5_capture; - MetadataSectionRange swift5_mpenum; - MetadataSectionRange swift5_accessible_functions; -}; - -/// A function exported by the Swift runtime that enumerates all metadata -/// sections loaded into the current process. -SWT_IMPORT_FROM_STDLIB void swift_enumerateAllMetadataSections( - bool (* body)(const MetadataSections *sections, void *context), - void *context -); - -template -static void enumerateTypeMetadataSections(const SectionEnumerator& body) { - swift_enumerateAllMetadataSections([] (const MetadataSections *sections, void *context) { - bool stop = false; - - const auto& body = *reinterpret_cast(context); - MetadataSectionRange section = sections->swift5_type_metadata; - if (section.start && section.length > 0) { - SWTSectionBounds sb = { - sections->baseAddress.load(), - reinterpret_cast(section.start), - section.length - }; - body(sb, &stop); - } - - return !stop; - }, const_cast(&body)); -} -#else -#warning Platform-specific implementation missing: Runtime test discovery unavailable (dynamic) -template -static void enumerateTypeMetadataSections(const SectionEnumerator& body) {} -#endif - -#else -#pragma mark - Statically-linked implementation - -#if defined(__APPLE__) -extern "C" const char sectionBegin __asm__("section$start$__TEXT$__swift5_types"); -extern "C" const char sectionEnd __asm__("section$end$__TEXT$__swift5_types"); -#elif defined(__wasi__) -extern "C" const char sectionBegin __asm__("__start_swift5_type_metadata"); -extern "C" const char sectionEnd __asm__("__stop_swift5_type_metadata"); -#else -#warning Platform-specific implementation missing: Runtime test discovery unavailable (static) -static const char sectionBegin = 0; -static const char& sectionEnd = sectionBegin; -#endif - -template -static void enumerateTypeMetadataSections(const SectionEnumerator& body) { - SWTSectionBounds sb = { - nullptr, - §ionBegin, - static_cast(std::distance(§ionBegin, §ionEnd)) - }; - bool stop = false; - body(sb, &stop); -} -#endif - -#pragma mark - - -void swt_enumerateTypesWithNamesContaining(const char *nameSubstring, void *context, SWTTypeEnumerator body) { - enumerateTypeMetadataSections([=] (const SWTSectionBounds& sectionBounds, bool *stop) { - for (const auto& record : sectionBounds) { - auto contextDescriptor = record.getContextDescriptor(); - if (!contextDescriptor) { - // This type metadata record is invalid (or we don't understand how to - // get its context descriptor), so skip it. - continue; - } else if (contextDescriptor->isGeneric()) { - // Generic types cannot be fully instantiated without generic - // parameters, which is not something we can know abstractly. - continue; - } - - // Check that the type's name passes. This will be more expensive than the - // checks above, but should be cheaper than realizing the metadata. - const char *typeName = contextDescriptor->getName(); - bool nameOK = typeName && nullptr != std::strstr(typeName, nameSubstring); - if (!nameOK) { - continue; - } - - if (void *typeMetadata = contextDescriptor->getMetadata()) { - body(sectionBounds.imageAddress, typeMetadata, stop, context); - } - } - }); + return resultCopy; } diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index 6432a798c..2cfd63339 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -73,34 +73,35 @@ SWT_IMPORT_FROM_STDLIB void swift_enumerateAllMetadataSections( /// when Swift files import the `_TestingInternals` C++ module. SWT_EXTERN const void *_Nonnull const SWTTestContentSectionBounds[2]; -#pragma mark - Legacy test discovery - -/// The type of callback called by `swt_enumerateTypes()`. +/// The bounds of the type metadata section statically linked into the image +/// containing Swift Testing. /// -/// - Parameters: -/// - imageAddress: A pointer to the start of the image. This value is _not_ -/// equal to the value returned from `dlopen()`. On platforms that do not -/// support dynamic loading (and so do not have loadable images), this -/// argument is unspecified. -/// - typeMetadata: A type metadata pointer that can be bitcast to `Any.Type`. -/// - stop: A pointer to a boolean variable indicating whether type -/// enumeration should stop after the function returns. Set `*stop` to -/// `true` to stop type enumeration. -/// - context: An arbitrary pointer passed by the caller to -/// `swt_enumerateTypes()`. -typedef void (* SWTTypeEnumerator)(const void *_Null_unspecified imageAddress, void *typeMetadata, bool *stop, void *_Null_unspecified context); +/// - Note: This symbol is _declared_, but not _defined_, on platforms with +/// dynamic linking because the `SWT_NO_DYNAMIC_LINKING` C++ macro (not the +/// Swift compiler conditional of the same name) is not consistently declared +/// when Swift files import the `_TestingInternals` C++ module. +SWT_EXTERN const void *_Nonnull const SWTTypeMetadataSectionBounds[2]; -/// Enumerate all types known to Swift found in the current process. +#pragma mark - Legacy test discovery + +/// Copy all types known to Swift found in the given type metadata section with +/// a name containing the given substring. /// /// - Parameters: +/// - sectionBegin: The address of the first byte of the Swift type metadata +/// section. +/// - sectionSize: The size, in bytes, of the Swift type metadata section. /// - nameSubstring: A string which the names of matching classes all contain. -/// - context: An arbitrary pointer to pass to `body`. -/// - body: A function to invoke, once per matching type. -SWT_EXTERN void swt_enumerateTypesWithNamesContaining( +/// - outCount: On return, the number of type metadata pointers returned. +/// +/// - Returns: A pointer to an array of type metadata pointers. The caller is +/// responsible for freeing this memory with `free()` when done. +SWT_EXTERN void *_Nonnull *_Nonnull swt_copyTypesWithNamesContaining( + const void *sectionBegin, + size_t sectionSize, const char *nameSubstring, - void *_Null_unspecified context, - SWTTypeEnumerator body -) SWT_SWIFT_NAME(swt_enumerateTypes(withNamesContaining:_:_:)); + size_t *outCount +) SWT_SWIFT_NAME(swt_copyTypes(in:_:withNamesContaining:count:)); SWT_ASSUME_NONNULL_END From c488632e1dbe26d17686fc0b175448f1ce696bb9 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 13 Jan 2025 14:52:42 -0500 Subject: [PATCH 047/234] Remove some remnants of the C++ section discovery code that we no longer need. (#905) This PR removes additional C++ section discovery code that is no longer needed. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery+Platform.swift | 32 ++--- Sources/Testing/Test+Discovery+Legacy.swift | 20 ++- Sources/_TestingInternals/Discovery.cpp | 134 ++++-------------- Sources/_TestingInternals/Versions.cpp | 6 - Sources/_TestingInternals/include/Discovery.h | 7 +- 5 files changed, 50 insertions(+), 149 deletions(-) diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index 7fcb94fe5..1615c2384 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -72,36 +72,20 @@ private let _startCollectingSectionBounds: Void = { return } - // If this image contains the Swift section we need, acquire the lock and + // If this image contains the Swift section(s) we need, acquire the lock and // store the section's bounds. - let testContentSectionBounds: SectionBounds? = { + func findSectionBounds(forSectionNamed segmentName: String, _ sectionName: String, ofKind kind: SectionBounds.Kind) { var size = CUnsignedLong(0) - if let start = getsectiondata(mh, "__DATA_CONST", "__swift5_tests", &size), size > 0 { + if let start = getsectiondata(mh, segmentName, sectionName, &size), size > 0 { let buffer = UnsafeRawBufferPointer(start: start, count: Int(clamping: size)) - return SectionBounds(imageAddress: mh, buffer: buffer) - } - return nil - }() - - let typeMetadataSectionBounds: SectionBounds? = { - var size = CUnsignedLong(0) - if let start = getsectiondata(mh, "__TEXT", "__swift5_types", &size), size > 0 { - let buffer = UnsafeRawBufferPointer(start: start, count: Int(clamping: size)) - return SectionBounds(imageAddress: mh, buffer: buffer) - } - return nil - }() - - if testContentSectionBounds != nil || typeMetadataSectionBounds != nil { - _sectionBounds.withLock { sectionBounds in - if let testContentSectionBounds { - sectionBounds[.testContent]!.append(testContentSectionBounds) - } - if let typeMetadataSectionBounds { - sectionBounds[.typeMetadata]!.append(typeMetadataSectionBounds) + let sb = SectionBounds(imageAddress: mh, buffer: buffer) + _sectionBounds.withLock { sectionBounds in + sectionBounds[kind]!.append(sb) } } } + findSectionBounds(forSectionNamed: "__DATA_CONST", "__swift5_tests", ofKind: .testContent) + findSectionBounds(forSectionNamed: "__TEXT", "__swift5_types", ofKind: .typeMetadata) } #if _runtime(_ObjC) diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index d330104ef..e24fca55d 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -57,15 +57,13 @@ let exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" /// /// - Returns: A sequence of Swift types whose names contain `nameSubstring`. func types(withNamesContaining nameSubstring: String) -> some Sequence { - SectionBounds.all(.typeMetadata).lazy - .map { sb in - var count = 0 - let start = swt_copyTypes(in: sb.buffer.baseAddress!, sb.buffer.count, withNamesContaining: nameSubstring, count: &count) - defer { - free(start) - } - return start.withMemoryRebound(to: Any.Type.self, capacity: count) { start in - Array(UnsafeBufferPointer(start: start, count: count)) - } - }.joined() + SectionBounds.all(.typeMetadata).lazy.flatMap { sb in + var count = 0 + let start = swt_copyTypes(in: sb.buffer.baseAddress!, sb.buffer.count, withNamesContaining: nameSubstring, count: &count) + defer { + free(start) + } + return UnsafeBufferPointer(start: start, count: count) + .withMemoryRebound(to: Any.Type.self) { Array($0) } + } } diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index 08ffc5f47..f66eaf874 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -10,6 +10,12 @@ #include "Discovery.h" +#include +#include +#include +#include +#include + #if defined(SWT_NO_DYNAMIC_LINKING) #pragma mark - Statically-linked section bounds @@ -31,103 +37,22 @@ static const char typeMetadataSectionBegin = 0; static const char& typeMetadataSectionEnd = typeMetadataSectionBegin; #endif -/// The bounds of the test content section statically linked into the image -/// containing Swift Testing. const void *_Nonnull const SWTTestContentSectionBounds[2] = { - &testContentSectionBegin, - &testContentSectionEnd + &testContentSectionBegin, &testContentSectionEnd }; -/// The bounds of the type metadata section statically linked into the image -/// containing Swift Testing. const void *_Nonnull const SWTTypeMetadataSectionBounds[2] = { - &typeMetadataSectionBegin, - &typeMetadataSectionEnd + &typeMetadataSectionBegin, &typeMetadataSectionEnd }; #endif -#pragma mark - Legacy test discovery - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/// Enumerate over all Swift type metadata sections in the current process. -/// -/// - Parameters: -/// - body: A function to call once for every section in the current process. -/// A pointer to the first type metadata record and the number of records -/// are passed to this function. -template -static void enumerateTypeMetadataSections(const SectionEnumerator& body); - -/// A type that acts as a C++ [Allocator](https://en.cppreference.com/w/cpp/named_req/Allocator) -/// without using global `operator new` or `operator delete`. -/// -/// This type is necessary because global `operator new` and `operator delete` -/// can be overridden in developer-supplied code and cause deadlocks or crashes -/// when subsequently used while holding a dyld- or libobjc-owned lock. Using -/// `std::malloc()` and `std::free()` allows the use of C++ container types -/// without this risk. -template -struct SWTHeapAllocator { - using value_type = T; - - T *allocate(size_t count) { - return reinterpret_cast(std::calloc(count, sizeof(T))); - } - - void deallocate(T *ptr, size_t count) { - std::free(ptr); - } -}; - -/// A structure describing the bounds of a Swift metadata section. -/// -/// The template argument `T` is the element type of the metadata section. -/// Instances of this type can be used with a range-based `for`-loop to iterate -/// the contents of the section. -template -struct SWTSectionBounds { - /// The base address of the image containing the section, if known. - const void *imageAddress; - - /// The base address of the section. - const void *start; - - /// The size of the section in bytes. - size_t size; - - const struct SWTTypeMetadataRecord *begin(void) const { - return reinterpret_cast(start); - } - - const struct SWTTypeMetadataRecord *end(void) const { - return reinterpret_cast(reinterpret_cast(start) + size); - } -}; - -/// A type that acts as a C++ [Container](https://en.cppreference.com/w/cpp/named_req/Container) -/// and which contains a sequence of instances of `SWTSectionBounds`. -template -using SWTSectionBoundsList = std::vector, SWTHeapAllocator>>; - #pragma mark - Swift ABI #if defined(__PTRAUTH_INTRINSICS__) -#include -#define SWT_PTRAUTH __ptrauth +#define SWT_PTRAUTH_SWIFT_TYPE_DESCRIPTOR __ptrauth(ptrauth_key_process_independent_data, 1, 0xae86) #else -#define SWT_PTRAUTH(...) +#define SWT_PTRAUTH_SWIFT_TYPE_DESCRIPTOR #endif -#define SWT_PTRAUTH_SWIFT_TYPE_DESCRIPTOR SWT_PTRAUTH(ptrauth_key_process_independent_data, 1, 0xae86) /// A type representing a pointer relative to itself. /// @@ -164,10 +89,6 @@ struct SWTRelativePointer { #endif return reinterpret_cast(result); } - - const T *_Nullable operator ->(void) const& { - return get(); - } }; /// A type representing a 32-bit absolute function pointer, usually used on platforms @@ -184,10 +105,6 @@ struct SWTAbsoluteFunctionPointer { const T *_Nullable get(void) const & { return _pointer; } - - const T *_Nullable operator ->(void) const & { - return get(); - } }; /// A type representing a pointer relative to itself with low bits reserved for @@ -270,14 +187,16 @@ struct SWTTypeMetadataRecord { } }; -#pragma mark - +#pragma mark - Legacy test discovery void **swt_copyTypesWithNamesContaining(const void *sectionBegin, size_t sectionSize, const char *nameSubstring, size_t *outCount) { - SWTSectionBounds sb = { nullptr, sectionBegin, sectionSize }; - std::vector> result; + void **result = nullptr; + size_t resultCount = 0; - for (const auto& record : sb) { - auto contextDescriptor = record.getContextDescriptor(); + auto records = reinterpret_cast(sectionBegin); + size_t recordCount = sectionSize / sizeof(SWTTypeMetadataRecord); + for (size_t i = 0; i < recordCount; i++) { + auto contextDescriptor = records[i].getContextDescriptor(); if (!contextDescriptor) { // This type metadata record is invalid (or we don't understand how to // get its context descriptor), so skip it. @@ -297,14 +216,19 @@ void **swt_copyTypesWithNamesContaining(const void *sectionBegin, size_t section } if (void *typeMetadata = contextDescriptor->getMetadata()) { - result.push_back(typeMetadata); + if (!result) { + // This is the first matching type we've found. That presumably means + // we'll find more, so allocate enough space for all remaining types in + // the section. Is this necessarily space-efficient? No, but this + // allocation is short-lived and is immediately copied and freed in the + // Swift caller. + result = reinterpret_cast(std::calloc(recordCount - i, sizeof(void *))); + } + result[resultCount] = typeMetadata; + resultCount += 1; } } - auto resultCopy = reinterpret_cast(std::calloc(sizeof(void *), result.size())); - if (resultCopy) { - std::uninitialized_move(result.begin(), result.end(), resultCopy); - *outCount = result.size(); - } - return resultCopy; + *outCount = resultCount; + return result; } diff --git a/Sources/_TestingInternals/Versions.cpp b/Sources/_TestingInternals/Versions.cpp index 9af19c50c..97eace99e 100644 --- a/Sources/_TestingInternals/Versions.cpp +++ b/Sources/_TestingInternals/Versions.cpp @@ -10,12 +10,6 @@ #include "Versions.h" -#if defined(_SWT_TESTING_LIBRARY_VERSION) && !defined(SWT_TESTING_LIBRARY_VERSION) -#warning _SWT_TESTING_LIBRARY_VERSION is deprecated -#warning Define SWT_TESTING_LIBRARY_VERSION and optionally SWT_TARGET_TRIPLE instead -#define SWT_TESTING_LIBRARY_VERSION _SWT_TESTING_LIBRARY_VERSION -#endif - const char *swt_getTestingLibraryVersion(void) { #if defined(SWT_TESTING_LIBRARY_VERSION) return SWT_TESTING_LIBRARY_VERSION; diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index 2cfd63339..cbcfc2d79 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -94,9 +94,10 @@ SWT_EXTERN const void *_Nonnull const SWTTypeMetadataSectionBounds[2]; /// - nameSubstring: A string which the names of matching classes all contain. /// - outCount: On return, the number of type metadata pointers returned. /// -/// - Returns: A pointer to an array of type metadata pointers. The caller is -/// responsible for freeing this memory with `free()` when done. -SWT_EXTERN void *_Nonnull *_Nonnull swt_copyTypesWithNamesContaining( +/// - Returns: A pointer to an array of type metadata pointers, or `nil` if no +/// matching types were found. The caller is responsible for freeing this +/// memory with `free()` when done. +SWT_EXTERN void *_Nonnull *_Nullable swt_copyTypesWithNamesContaining( const void *sectionBegin, size_t sectionSize, const char *nameSubstring, From a580413a900ee673fb81b4ce59bf468366ff519b Mon Sep 17 00:00:00 2001 From: "Sven A. Schmidt" Date: Tue, 14 Jan 2025 18:02:31 +0100 Subject: [PATCH 048/234] Update .spi.yml (#908) Swift 6 is now the default doc build version on SPI. --- .spi.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.spi.yml b/.spi.yml index a85a01a68..d4e571475 100644 --- a/.spi.yml +++ b/.spi.yml @@ -3,6 +3,5 @@ metadata: authors: Apple Inc. builder: configs: - - swift_version: 6.0 - documentation_targets: [Testing] + - documentation_targets: [Testing] scheme: Testing From 26ac6a34d74e587f43eb3165972e856643bd2188 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 14 Jan 2025 21:26:14 -0600 Subject: [PATCH 049/234] Remove @dennisweissmann from CODEOWNERS (#910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove @dennisweissmann from `CODEOWNERS` to reflect his current role 🫡 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index c72baa0ea..546238eb3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,4 +8,4 @@ # See https://swift.org/CONTRIBUTORS.txt for Swift project authors # -* @stmontgomery @grynspan @dennisweissmann @briancroom @SeanROlszewski @suzannaratcliff +* @stmontgomery @grynspan @briancroom @SeanROlszewski @suzannaratcliff From 54949828879fee1ffa965b0f5ad7cf4e232a6a47 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 15 Jan 2025 15:17:38 -0500 Subject: [PATCH 050/234] Update README.md's note about development toolchain requirements. (#911) We are well past Swift 6.0 development toolchains. Update the readme to reflect that. Resolves #846. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 153129e74..2e7381a31 100644 --- a/README.md +++ b/README.md @@ -112,18 +112,17 @@ incrementally, at your own pace. ## Documentation -> [!IMPORTANT] -> This package is under active, ongoing development and requires a recent -> **6.0 development snapshot** toolchain. Its contents and interfaces are still -> considered experimental at this time and may change. See this -> [Forum post](https://forums.swift.org/t/an-update-on-swift-testing-progress-and-stable-release-plans/71455) -> for details about stable release plans. - Detailed documentation for Swift Testing can be found on the [Swift Package Index](https://swiftpackageindex.com/swiftlang/swift-testing/main/documentation/testing). There, you can delve into comprehensive guides, tutorials, and API references to make the most out of this package. Swift Testing is included with the Swift 6 -toolchain and Xcode 16. +toolchain and Xcode 16. You do not need to add it as a package dependency to +your Swift package or Xcode project. + +> [!IMPORTANT] +> Swift Testing depends on upcoming language and compiler features. If you are +> building Swift Testing from source, be aware that the main branch of this +> repository requires a recent **main-branch development snapshot** toolchain. Other documentation resources for this project can be found in the [README](https://github.com/swiftlang/swift-testing/blob/main/Documentation/README.md) From 46afc1512ff7626adf3d399185c5e1d38bc2edf8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 15 Jan 2025 15:33:34 -0500 Subject: [PATCH 051/234] Don't capture arguments to `#expect(exitsWith:)` during macro expansion. (#912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Other than the body closure, the arguments to `#expect(exitsWith:)` are not guaranteed to be literals or otherwise context-free, but we capture them during macro expansion. This can result in obscure errors if the developer writes an exit test with an argument that is computed: ```swift let foo = ... await #expect(exitsWith: self.exitCondition(for: foo)) { // 🛑 error: closure captures 'foo' before it is declared // 🛑 error: enum declaration cannot close over value 'self' defined in outer scope // (and other possible errors) ... } ``` This PR removes the macro's compile-time dependency on its `exitCondition` and `sourceLocation` arguments and introduces an `ID` type to identify exit tests (instead of using their source locations as unique IDs.) The `ID` type is meant to be a UUID, but for the moment I've just strung together two 64-bit integers instead to avoid a dependency on Foundation or platform API. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 14 +++- Sources/Testing/ExitTests/ExitTest.swift | 83 +++++++++---------- .../Expectations/Expectation+Macro.swift | 4 +- .../ExpectationChecking+Macro.swift | 2 + Sources/Testing/Test+Discovery+Legacy.swift | 7 +- Sources/TestingMacros/ConditionMacro.swift | 21 +++-- Tests/TestingTests/ExitTestTests.swift | 11 +++ 7 files changed, 84 insertions(+), 58 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index fd7b9f893..c1a90aff1 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -118,9 +118,17 @@ to by `hint` depend on the kind of record: - For exit test declarations (kind `0x65786974`), the accessor produces a structure describing the exit test (of type `__ExitTest`.) - Test content records of this kind accept a `hint` of type `SourceLocation`. - They only produce a result if they represent an exit test declared at the same - source location (or if the hint is `nil`.) + Test content records of this kind accept a `hint` of type `__ExitTest.ID`. + They only produce a result if they represent an exit test declared with the + same ID (or if `hint` is `nil`.) + +> [!WARNING] +> Calling code should use [`withUnsafeTemporaryAllocation(of:capacity:_:)`](https://developer.apple.com/documentation/swift/withunsafetemporaryallocation(of:capacity:_:)) +> and [`withUnsafePointer(to:_:)`](https://developer.apple.com/documentation/swift/withunsafepointer(to:_:)-35wrn), +> respectively, to ensure the pointers passed to `accessor` are large enough and +> are well-aligned. If they are not large enough to contain values of the +> appropriate types (per above), or if `hint` points to uninitialized or +> incorrectly-typed memory, the result is undefined. #### The context field diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 96d636b11..b453a6b2d 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -52,16 +52,25 @@ public typealias ExitTest = __ExitTest @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif public struct __ExitTest: Sendable, ~Copyable { - /// The expected exit condition of the exit test. - @_spi(ForToolsIntegrationOnly) - public var expectedExitCondition: ExitCondition + /// A type whose instances uniquely identify instances of `__ExitTest`. + public struct ID: Sendable, Equatable, Codable { + /// An underlying UUID (stored as two `UInt64` values to avoid relying on + /// `UUID` from Foundation or any platform-specific interfaces.) + private var _lo: UInt64 + private var _hi: UInt64 + + /// Initialize an instance of this type. + /// + /// - Warning: This member is used to implement the `#expect(exitsWith:)` + /// macro. Do not use it directly. + public init(__uuid uuid: (UInt64, UInt64)) { + self._lo = uuid.0 + self._hi = uuid.1 + } + } - /// The source location of the exit test. - /// - /// The source location is unique to each exit test and is consistent between - /// processes, so it can be used to uniquely identify an exit test at runtime. - @_spi(ForToolsIntegrationOnly) - public var sourceLocation: SourceLocation + /// A value that uniquely identifies this instance. + public var id: ID /// The body closure of the exit test. /// @@ -110,12 +119,10 @@ public struct __ExitTest: Sendable, ~Copyable { /// - Warning: This initializer is used to implement the `#expect(exitsWith:)` /// macro. Do not use it directly. public init( - __expectedExitCondition expectedExitCondition: ExitCondition, - sourceLocation: SourceLocation, + __identifiedBy id: ID, body: @escaping @Sendable () async throws -> Void = {} ) { - self.expectedExitCondition = expectedExitCondition - self.sourceLocation = sourceLocation + self.id = id self.body = body } } @@ -222,7 +229,7 @@ extension ExitTest: TestContent { } typealias TestContentAccessorResult = Self - typealias TestContentAccessorHint = SourceLocation + typealias TestContentAccessorHint = ID } @_spi(Experimental) @_spi(ForToolsIntegrationOnly) @@ -230,20 +237,16 @@ extension ExitTest { /// Find the exit test function at the given source location. /// /// - Parameters: - /// - sourceLocation: The source location of the exit test to find. + /// - id: The unique identifier of the exit test to find. /// /// - Returns: The specified exit test function, or `nil` if no such exit test /// could be found. - public static func find(at sourceLocation: SourceLocation) -> Self? { + public static func find(identifiedBy id: ExitTest.ID) -> Self? { var result: Self? - enumerateTestContent(withHint: sourceLocation) { _, exitTest, _, stop in - if exitTest.sourceLocation == sourceLocation { - result = ExitTest( - __expectedExitCondition: exitTest.expectedExitCondition, - sourceLocation: exitTest.sourceLocation, - body: exitTest.body - ) + enumerateTestContent(withHint: id) { _, exitTest, _, stop in + if exitTest.id == id { + result = ExitTest(__identifiedBy: id, body: exitTest.body) stop = true } } @@ -252,14 +255,8 @@ extension ExitTest { // Call the legacy lookup function that discovers tests embedded in types. result = types(withNamesContaining: exitTestContainerTypeNameMagic).lazy .compactMap { $0 as? any __ExitTestContainer.Type } - .first { $0.__sourceLocation == sourceLocation } - .map { type in - ExitTest( - __expectedExitCondition: type.__expectedExitCondition, - sourceLocation: type.__sourceLocation, - body: type.__body - ) - } + .first { $0.__id == id } + .map { ExitTest(__identifiedBy: $0.__id, body: $0.__body) } } return result @@ -272,6 +269,7 @@ extension ExitTest { /// a given status. /// /// - Parameters: +/// - exitTestID: The unique identifier of the exit test. /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The @@ -290,6 +288,7 @@ extension ExitTest { /// `await #expect(exitsWith:) { }` invocations regardless of calling /// convention. func callExitTest( + identifiedBy exitTestID: ExitTest.ID, exitsWith expectedExitCondition: ExitCondition, observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, @@ -304,7 +303,7 @@ func callExitTest( var result: ExitTestArtifacts do { - var exitTest = ExitTest(__expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation) + var exitTest = ExitTest(__identifiedBy: exitTestID) exitTest.observedValues = observedValues result = try await configuration.exitTestHandler(exitTest) @@ -424,23 +423,21 @@ extension ExitTest { /// `__swiftPMEntryPoint()` function. The effect of using it under other /// configurations is undefined. static func findInEnvironmentForEntryPoint() -> Self? { - // Find the source location of the exit test to run, if any, in the - // environment block. - var sourceLocation: SourceLocation? - if var sourceLocationString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_SOURCE_LOCATION") { - sourceLocation = try? sourceLocationString.withUTF8 { sourceLocationBuffer in - let sourceLocationBuffer = UnsafeRawBufferPointer(sourceLocationBuffer) - return try JSON.decode(SourceLocation.self, from: sourceLocationBuffer) + // Find the ID of the exit test to run, if any, in the environment block. + var id: __ExitTest.ID? + if var idString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") { + id = try? idString.withUTF8 { idBuffer in + try JSON.decode(__ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) } } - guard let sourceLocation else { + guard let id else { return nil } // If an exit test was found, inject back channel handling into its body. // External tools authors should set up their own back channel mechanisms // and ensure they're installed before calling ExitTest.callAsFunction(). - guard var result = find(at: sourceLocation) else { + guard var result = find(identifiedBy: id) else { return nil } @@ -560,8 +557,8 @@ extension ExitTest { // Insert a specific variable that tells the child process which exit test // to run. - try JSON.withEncoding(of: exitTest.sourceLocation) { json in - childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_SOURCE_LOCATION"] = String(decoding: json, as: UTF8.self) + try JSON.withEncoding(of: exitTest.id) { json in + childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) } typealias ResultUpdater = @Sendable (inout ExitTestArtifacts) -> Void diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 7df075475..7e82b2144 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -546,7 +546,7 @@ public macro require( observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, - performing expression: @convention(thin) () async throws -> Void + performing expression: @escaping @Sendable @convention(thin) () async throws -> Void ) -> ExitTestArtifacts? = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") /// Check that an expression causes the process to terminate in a given fashion @@ -658,5 +658,5 @@ public macro require( observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, - performing expression: @convention(thin) () async throws -> Void + performing expression: @escaping @Sendable @convention(thin) () async throws -> Void ) -> ExitTestArtifacts = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index ca452e2f8..267fddba4 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1147,6 +1147,7 @@ public func __checkClosureCall( /// `#require()` macros. Do not call it directly. @_spi(Experimental) public func __checkClosureCall( + identifiedBy exitTestID: __ExitTest.ID, exitsWith expectedExitCondition: ExitCondition, observing observedValues: [any PartialKeyPath & Sendable], performing body: @convention(thin) () -> Void, @@ -1157,6 +1158,7 @@ public func __checkClosureCall( sourceLocation: SourceLocation ) async -> Result { await callExitTest( + identifiedBy: exitTestID, exitsWith: expectedExitCondition, observing: observedValues, expression: expression, diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index e24fca55d..c90e8c590 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -32,11 +32,8 @@ let testContainerTypeNameMagic = "__🟠$test_container__" @_alwaysEmitConformanceMetadata @_spi(Experimental) public protocol __ExitTestContainer { - /// The expected exit condition of the exit test. - static var __expectedExitCondition: ExitCondition { get } - - /// The source location of the exit test. - static var __sourceLocation: SourceLocation { get } + /// The unique identifier of the exit test. + static var __id: __ExitTest.ID { get } /// The body function of the exit test. static var __body: @Sendable () async throws -> Void { get } diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 3d2013c69..346cb68bf 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -416,6 +416,10 @@ extension ExitTestConditionMacro { let bodyArgumentExpr = arguments[trailingClosureIndex].expression + // TODO: use UUID() here if we can link to Foundation + let exitTestID = (UInt64.random(in: 0 ... .max), UInt64.random(in: 0 ... .max)) + let exitTestIDExpr: ExprSyntax = "Testing.__ExitTest.ID(__uuid: (\(literal: exitTestID.0), \(literal: exitTestID.1)))" + var decls = [DeclSyntax]() // Implement the body of the exit test outside the enum we're declaring so @@ -436,15 +440,12 @@ extension ExitTestConditionMacro { """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") enum \(enumName): Testing.__ExitTestContainer, Sendable { - static var __sourceLocation: Testing.SourceLocation { - \(createSourceLocationExpr(of: macro, context: context)) + static var __id: Testing.__ExitTest.ID { + \(exitTestIDExpr) } static var __body: @Sendable () async throws -> Void { \(bodyThunkName) } - static var __expectedExitCondition: Testing.ExitCondition { - \(arguments[expectedExitConditionIndex].expression.trimmed) - } } """ ) @@ -458,6 +459,16 @@ extension ExitTestConditionMacro { } ) + // Insert the exit test's ID as the first argument. Note that this will + // invalidate all indices into `arguments`! + arguments.insert( + Argument( + label: "identifiedBy", + expression: exitTestIDExpr + ), + at: arguments.startIndex + ) + // Replace the exit test body (as an argument to the macro) with a stub // closure that hosts the type we created above. var macro = macro diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 81a21aaec..0c5a9dcab 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -405,6 +405,17 @@ private import _TestingInternals #expect(result.standardOutputContent.isEmpty) #expect(result.standardErrorContent.contains("STANDARD ERROR".utf8.reversed())) } + + @Test("Arguments to the macro are not captured during expansion (do not need to be literals/const)") + func argumentsAreNotCapturedDuringMacroExpansion() async throws { + let unrelatedSourceLocation = #_sourceLocation + func nonConstExitCondition() async throws -> ExitCondition { + .failure + } + await #expect(exitsWith: try await nonConstExitCondition(), sourceLocation: unrelatedSourceLocation) { + fatalError() + } + } } // MARK: - Fixtures From dfa0d2460e4e6d27b18e50b34b942ff5f81c12ae Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 16 Jan 2025 18:19:27 -0500 Subject: [PATCH 052/234] Refactor test content discovery to produce a sequence. (#914) This PR does away with `enumerateTestContent {}` and replaces it with `discover()`, a function that returns a sequence of `TestContentRecord` instances that the caller can then inspect. This simplifies the logic in callers by letting them use sequence operations/monads like `map()`, and simplifies the logic in the implementation by doing away with intermediate tuples and allowing it to just map section bounds to sequences of `TestContentRecord` values. Evaluation of the accessor functions for test content records is now lazier, allowing us to potentially leverage the `context` field of a test content record to avoid calling accessors for records we know won't match due to the content of that field. (This functionality isn't currently needed, but could be useful for future test content types.) ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery+Platform.swift | 4 +- Sources/Testing/Discovery.swift | 149 ++++++++++---------- Sources/Testing/ExitTests/ExitTest.swift | 23 ++- Sources/Testing/Test+Discovery.swift | 62 ++++---- Tests/TestingTests/MiscellaneousTests.swift | 50 ++++--- 5 files changed, 135 insertions(+), 153 deletions(-) diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index 1615c2384..9c352bb41 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -35,7 +35,7 @@ struct SectionBounds: Sendable { /// /// - Returns: A sequence of structures describing the bounds of metadata /// sections of the given kind found in the current process. - static func all(_ kind: Kind) -> some RandomAccessCollection { + static func all(_ kind: Kind) -> some Sequence { _sectionBounds(kind) } } @@ -237,7 +237,7 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { case .typeMetadata: ".sw5tymd" } - return HMODULE.all.compactMap { _findSection(named: sectionName, in: $0) } + return HMODULE.all.lazy.compactMap { _findSection(named: sectionName, in: $0) } } #else /// The fallback implementation of ``SectionBounds/all(_:)`` for platforms that diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index 6343b92f4..0a566c36c 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -29,21 +29,6 @@ public typealias __TestContentRecord = ( reserved2: UInt ) -/// Resign any pointers in a test content record. -/// -/// - Parameters: -/// - record: The test content record to resign. -/// -/// - Returns: A copy of `record` with its pointers resigned. -/// -/// On platforms/architectures without pointer authentication, this function has -/// no effect. -private func _resign(_ record: __TestContentRecord) -> __TestContentRecord { - var record = record - record.accessor = record.accessor.map(swt_resign) - return record -} - // MARK: - /// A protocol describing a type that can be stored as test content at compile @@ -79,42 +64,68 @@ protocol TestContent: ~Copyable { associatedtype TestContentAccessorHint: Sendable = Never } -extension TestContent where Self: ~Copyable { - /// Enumerate all test content records found in the given test content section - /// in the current process that match this ``TestContent`` type. +// MARK: - Individual test content records + +/// A type describing a test content record of a particular (known) type. +/// +/// Instances of this type can be created by calling +/// ``TestContent/allTestContentRecords()`` on a type that conforms to +/// ``TestContent``. +/// +/// This type is not part of the public interface of the testing library. In the +/// future, we could make it public if we want to support runtime discovery of +/// test content by second- or third-party code. +struct TestContentRecord: Sendable where T: ~Copyable { + /// The base address of the image containing this instance, if known. /// - /// - Parameters: - /// - sectionBounds: The bounds of the section to inspect. + /// This property is not available on platforms such as WASI that statically + /// link to the testing library. /// - /// - Returns: A sequence of tuples. Each tuple contains an instance of - /// `__TestContentRecord` and the base address of the image containing that - /// test content record. Only test content records matching this - /// ``TestContent`` type's requirements are included in the sequence. - private static func _testContentRecords(in sectionBounds: SectionBounds) -> some Sequence<(imageAddress: UnsafeRawPointer?, record: __TestContentRecord)> { - sectionBounds.buffer.withMemoryRebound(to: __TestContentRecord.self) { records in - records.lazy - .filter { $0.kind == testContentKind } - .map(_resign) - .map { (sectionBounds.imageAddress, $0) } - } + /// - Note: The value of this property is distinct from the pointer returned + /// by `dlopen()` (on platforms that have that function) and cannot be used + /// with interfaces such as `dlsym()` that expect such a pointer. +#if SWT_NO_DYNAMIC_LINKING + @available(*, unavailable, message: "Image addresses are not available on this platform.") +#endif + nonisolated(unsafe) var imageAddress: UnsafeRawPointer? + + /// The underlying test content record loaded from a metadata section. + private var _record: __TestContentRecord + + fileprivate init(imageAddress: UnsafeRawPointer?, record: __TestContentRecord) { +#if !SWT_NO_DYNAMIC_LINKING + self.imageAddress = imageAddress +#endif + self._record = record + } +} + +// This `T: TestContent` constraint is in an extension in order to work around a +// compiler crash. SEE: rdar://143049814 +extension TestContentRecord where T: TestContent & ~Copyable { + /// The context value for this test content record. + var context: UInt { + _record.context } - /// Call the given accessor function. + /// Load the value represented by this record. /// /// - Parameters: - /// - accessor: The C accessor function of a test content record matching - /// this type. - /// - hint: A pointer to a kind-specific hint value. If not `nil`, this - /// value is passed to `accessor`, allowing that function to determine if - /// its record matches before initializing its out-result. + /// - hint: An optional hint value. If not `nil`, this value is passed to + /// the accessor function of the underlying test content record. /// - /// - Returns: An instance of this type's accessor result or `nil` if an - /// instance could not be created (or if `hint` did not match.) + /// - Returns: An instance of the associated ``TestContentAccessorResult`` + /// type, or `nil` if the underlying test content record did not match + /// `hint` or otherwise did not produce a value. /// - /// The caller is responsible for ensuring that `accessor` corresponds to a - /// test content record of this type. - private static func _callAccessor(_ accessor: SWTTestContentAccessor, withHint hint: TestContentAccessorHint?) -> TestContentAccessorResult? { - withUnsafeTemporaryAllocation(of: TestContentAccessorResult.self, capacity: 1) { buffer in + /// If this function is called more than once on the same instance, a new + /// value is created on each call. + func load(withHint hint: T.TestContentAccessorHint? = nil) -> T.TestContentAccessorResult? { + guard let accessor = _record.accessor.map(swt_resign) else { + return nil + } + + return withUnsafeTemporaryAllocation(of: T.TestContentAccessorResult.self, capacity: 1) { buffer in let initialized = if let hint { withUnsafePointer(to: hint) { hint in accessor(buffer.baseAddress!, hint) @@ -128,46 +139,28 @@ extension TestContent where Self: ~Copyable { return buffer.baseAddress!.move() } } +} - /// The type of callback called by ``enumerateTestContent(withHint:_:)``. - /// - /// - Parameters: - /// - imageAddress: A pointer to the start of the image. This value is _not_ - /// equal to the value returned from `dlopen()`. On platforms that do not - /// support dynamic loading (and so do not have loadable images), the - /// value of this argument is unspecified. - /// - content: The value produced by the test content record's accessor. - /// - context: Context associated with `content`. The value of this argument - /// is dependent on the type of test content being enumerated. - /// - stop: An `inout` boolean variable indicating whether test content - /// enumeration should stop after the function returns. Set `stop` to - /// `true` to stop test content enumeration. - typealias TestContentEnumerator = (_ imageAddress: UnsafeRawPointer?, _ content: borrowing TestContentAccessorResult, _ context: UInt, _ stop: inout Bool) -> Void +// MARK: - Enumeration of test content records - /// Enumerate all test content of this type known to Swift and found in the - /// current process. +extension TestContent where Self: ~Copyable { + /// Get all test content of this type known to Swift and found in the current + /// process. /// - /// - Parameters: - /// - hint: An optional hint value. If not `nil`, this value is passed to - /// the accessor function of each test content record whose `kind` field - /// matches this type's ``testContentKind`` property. - /// - body: A function to invoke, once per matching test content record. + /// - Returns: A sequence of instances of ``TestContentRecord``. Only test + /// content records matching this ``TestContent`` type's requirements are + /// included in the sequence. /// - /// This function uses a callback instead of producing a sequence because it - /// is used with move-only types (specifically ``ExitTest``) and - /// `Sequence.Element` must be copyable. - static func enumerateTestContent(withHint hint: TestContentAccessorHint? = nil, _ body: TestContentEnumerator) { - let testContentRecords = SectionBounds.all(.testContent).lazy.flatMap(_testContentRecords(in:)) - - var stop = false - for (imageAddress, record) in testContentRecords { - if let accessor = record.accessor, let result = _callAccessor(accessor, withHint: hint) { - // Call the callback. - body(imageAddress, result, record.context, &stop) - if stop { - break - } + /// - Bug: This function returns an instance of `AnySequence` instead of an + /// opaque type due to a compiler crash. ([143080508](rdar://143080508)) + static func allTestContentRecords() -> AnySequence> { + let result = SectionBounds.all(.testContent).lazy.flatMap { sb in + sb.buffer.withMemoryRebound(to: __TestContentRecord.self) { records in + records.lazy + .filter { $0.kind == testContentKind } + .map { TestContentRecord(imageAddress: sb.imageAddress, record: $0) } } } + return AnySequence(result) } } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index b453a6b2d..a3a92c036 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -242,24 +242,17 @@ extension ExitTest { /// - Returns: The specified exit test function, or `nil` if no such exit test /// could be found. public static func find(identifiedBy id: ExitTest.ID) -> Self? { - var result: Self? - - enumerateTestContent(withHint: id) { _, exitTest, _, stop in - if exitTest.id == id { - result = ExitTest(__identifiedBy: id, body: exitTest.body) - stop = true + for record in Self.allTestContentRecords() { + if let exitTest = record.load(withHint: id) { + return exitTest } } - if result == nil { - // Call the legacy lookup function that discovers tests embedded in types. - result = types(withNamesContaining: exitTestContainerTypeNameMagic).lazy - .compactMap { $0 as? any __ExitTestContainer.Type } - .first { $0.__id == id } - .map { ExitTest(__identifiedBy: $0.__id, body: $0.__body) } - } - - return result + // Call the legacy lookup function that discovers tests embedded in types. + return types(withNamesContaining: exitTestContainerTypeNameMagic).lazy + .compactMap { $0 as? any __ExitTestContainer.Type } + .first { $0.__id == id } + .map { ExitTest(__identifiedBy: $0.__id, body: $0.__body) } } } diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 751a7de85..939e85947 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -22,59 +22,51 @@ extension Test: TestContent { /// The order of values in this sequence is unspecified. static var all: some Sequence { get async { - var generators = [@Sendable () async -> [Self]]() + // The result is a set rather than an array to deduplicate tests that were + // generated multiple times (e.g. from multiple discovery modes or from + // defective test records.) + var result = Set() // Figure out which discovery mechanism to use. By default, we'll use both // the legacy and new mechanisms, but we can set an environment variable // to explicitly select one or the other. When we remove legacy support, // we can also remove this enumeration and environment variable check. - enum DiscoveryMode { - case tryBoth - case newOnly - case legacyOnly - } - let discoveryMode: DiscoveryMode = switch Environment.flag(named: "SWT_USE_LEGACY_TEST_DISCOVERY") { + let (useNewMode, useLegacyMode) = switch Environment.flag(named: "SWT_USE_LEGACY_TEST_DISCOVERY") { case .none: - .tryBoth + (true, true) case .some(true): - .legacyOnly + (false, true) case .some(false): - .newOnly + (true, false) } - // Walk all test content and gather generator functions. Note we don't - // actually call the generators yet because enumerating test content may - // involve holding some internal lock such as the ones in libobjc or - // dl_iterate_phdr(), and we don't want to accidentally deadlock if the - // user code we call ends up loading another image. - if discoveryMode != .legacyOnly { - enumerateTestContent { imageAddress, generator, _, _ in - generators.append { @Sendable in - await [generator()] + // Walk all test content and gather generator functions, then call them in + // a task group and collate their results. + if useNewMode { + let generators = Self.allTestContentRecords().lazy.compactMap { $0.load() } + await withTaskGroup(of: Self.self) { taskGroup in + for generator in generators { + taskGroup.addTask(operation: generator) } + result = await taskGroup.reduce(into: result) { $0.insert($1) } } } - if discoveryMode != .newOnly && generators.isEmpty { - generators += types(withNamesContaining: testContainerTypeNameMagic).lazy + // Perform legacy test discovery if needed. + if useLegacyMode && result.isEmpty { + let types = types(withNamesContaining: testContainerTypeNameMagic).lazy .compactMap { $0 as? any __TestContainer.Type } - .map { type in - { @Sendable in await type.__tests } - } - } - - // *Now* we call all the generators and return their results. - // Reduce into a set rather than an array to deduplicate tests that were - // generated multiple times (e.g. from multiple discovery modes or from - // defective test records.) - return await withTaskGroup(of: [Self].self) { taskGroup in - for generator in generators { - taskGroup.addTask { - await generator() + await withTaskGroup(of: [Self].self) { taskGroup in + for type in types { + taskGroup.addTask { + await type.__tests + } } + result = await taskGroup.reduce(into: result) { $0.formUnion($1) } } - return await taskGroup.reduce(into: Set()) { $0.formUnion($1) } } + + return result } } } diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 49df6cc6e..9cc3b7910 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -615,43 +615,47 @@ struct MiscellaneousTests { 0xABCD1234, 0, { outValue, hint in - if let hint, hint.loadUnaligned(as: TestContentAccessorHint.self) != expectedHint { + if let hint, hint.load(as: TestContentAccessorHint.self) != expectedHint { return false } _ = outValue.initializeMemory(as: TestContentAccessorResult.self, to: expectedResult) return true }, - UInt(UInt64(0x0204060801030507) & UInt64(UInt.max)), + UInt(truncatingIfNeeded: UInt64(0x0204060801030507)), 0 ) } @Test func testDiscovery() async { - await confirmation("Can find a single test record") { found in - DiscoverableTestContent.enumerateTestContent { _, value, context, _ in - if value == DiscoverableTestContent.expectedResult && context == DiscoverableTestContent.expectedContext { - found() - } - } - } + // Check the type of the test record sequence (it should be lazy.) + let allRecords = DiscoverableTestContent.allTestContentRecords() +#if SWT_FIXED_143080508 + #expect(allRecords is any LazySequenceProtocol) + #expect(!(allRecords is [TestContentRecord])) +#endif + + // It should have exactly one matching record (because we only emitted one.) + #expect(Array(allRecords).count == 1) + + // Can find a single test record + #expect(allRecords.contains { record in + record.load() == DiscoverableTestContent.expectedResult + && record.context == DiscoverableTestContent.expectedContext + }) - await confirmation("Can find a test record with matching hint") { found in + // Can find a test record with matching hint + #expect(allRecords.contains { record in let hint = DiscoverableTestContent.expectedHint - DiscoverableTestContent.enumerateTestContent(withHint: hint) { _, value, context, _ in - if value == DiscoverableTestContent.expectedResult && context == DiscoverableTestContent.expectedContext { - found() - } - } - } + return record.load(withHint: hint) == DiscoverableTestContent.expectedResult + && record.context == DiscoverableTestContent.expectedContext + }) - await confirmation("Doesn't find a test record with a mismatched hint", expectedCount: 0) { found in + // Doesn't find a test record with a mismatched hint + #expect(!allRecords.contains { record in let hint = ~DiscoverableTestContent.expectedHint - DiscoverableTestContent.enumerateTestContent(withHint: hint) { _, value, context, _ in - if value == DiscoverableTestContent.expectedResult && context == DiscoverableTestContent.expectedContext { - found() - } - } - } + return record.load(withHint: hint) == DiscoverableTestContent.expectedResult + && record.context == DiscoverableTestContent.expectedContext + }) } #endif } From 893389fb6b4ae8c064801d6d1665d6247330bcf0 Mon Sep 17 00:00:00 2001 From: Allan Shortlidge Date: Thu, 16 Jan 2025 18:17:53 -0800 Subject: [PATCH 053/234] Fix the build for WASI. --- Sources/Testing/Discovery.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index 0a566c36c..d9acdca19 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -86,8 +86,13 @@ struct TestContentRecord: Sendable where T: ~Copyable { /// with interfaces such as `dlsym()` that expect such a pointer. #if SWT_NO_DYNAMIC_LINKING @available(*, unavailable, message: "Image addresses are not available on this platform.") -#endif + nonisolated(unsafe) var imageAddress: UnsafeRawPointer? { + get { fatalError() } + set { fatalError() } + } +#else nonisolated(unsafe) var imageAddress: UnsafeRawPointer? +#endif /// The underlying test content record loaded from a metadata section. private var _record: __TestContentRecord From 0538f22705df83d33a6c08a915e9c408c5e2f2f3 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 17 Jan 2025 12:22:49 -0500 Subject: [PATCH 054/234] Revert #916 and apply a slightly different change. (#917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you to @tshortli for the patch! We don't actually _need_ this property to be unavailable on WASI—it was just meant as a convenience. But since we're not using the property for anything at this time (a future PR might try to productize `TestContentRecord`), it's overkill. So instead I'll just leave the property available, and document it's always `nil` on statically-linked platforms. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery.swift | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index d9acdca19..5ce454d1c 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -78,29 +78,19 @@ protocol TestContent: ~Copyable { struct TestContentRecord: Sendable where T: ~Copyable { /// The base address of the image containing this instance, if known. /// - /// This property is not available on platforms such as WASI that statically - /// link to the testing library. + /// On platforms such as WASI that statically link to the testing library, the + /// value of this property is always `nil`. /// /// - Note: The value of this property is distinct from the pointer returned /// by `dlopen()` (on platforms that have that function) and cannot be used /// with interfaces such as `dlsym()` that expect such a pointer. -#if SWT_NO_DYNAMIC_LINKING - @available(*, unavailable, message: "Image addresses are not available on this platform.") - nonisolated(unsafe) var imageAddress: UnsafeRawPointer? { - get { fatalError() } - set { fatalError() } - } -#else nonisolated(unsafe) var imageAddress: UnsafeRawPointer? -#endif /// The underlying test content record loaded from a metadata section. private var _record: __TestContentRecord fileprivate init(imageAddress: UnsafeRawPointer?, record: __TestContentRecord) { -#if !SWT_NO_DYNAMIC_LINKING self.imageAddress = imageAddress -#endif self._record = record } } From 6d2020066841239855a0e8151fce238d58453240 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 17 Jan 2025 12:31:24 -0600 Subject: [PATCH 055/234] Limit the amount of value reflection data expectation checking collects by default (#915) This modifies the internal logic which recursively collects reflection data about values passed to `#expect(...)` and similar expectation APIs so that it imposes an artificial limit on the amount of data collected. ### Motivation: Some users have attempted to use `#expect` with instances of values with large collections or with deep value hierarchies involving many sub-properties, etc. In these situations they have noticed that the automatic value reflection is adding a noticeable amount of overhead to the overall test execution time, and would like to reduce that by default (near term), as well as have some ability to control that so the behavior could be tweaked for particular tests (in the future). ### Modifications: - Add some new `Configuration` properties controlling data collection behaviors and setting default limits. - Consult these new configuration properties in the relevant places in `Expression.Value`. - Make `Expression.Value.init(reflecting:)` optional and return `nil` when value reflection is disabled in the configuration. - Add a new initializer `Expression.Value.init(describing:)` as a lighter-weight alternative to `init(reflecting:)` which only forms a string description of its subject, and adopt this as a fallback in places affected by the new optional in the previous bullet. - Add representative new unit tests. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes rdar://138208832 --- .../Testing/Parameterization/Test.Case.swift | 2 +- Sources/Testing/Running/Configuration.swift | 47 ++++++++ .../Testing/Running/Runner.RuntimeState.swift | 21 ++++ .../SourceAttribution/Expression.swift | 111 ++++++++++++++---- .../TestingTests/Expression.ValueTests.swift | 96 +++++++++++++-- 5 files changed, 247 insertions(+), 30 deletions(-) diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index db35c3aa1..80ff101da 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -214,7 +214,7 @@ extension Test.Case.Argument { /// - argument: The original test case argument to snapshot. public init(snapshotting argument: Test.Case.Argument) { id = argument.id - value = Expression.Value(reflecting: argument.value) + value = Expression.Value(reflecting: argument.value) ?? .init(describing: argument.value) parameter = argument.parameter } } diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 3f9ed67f4..bdb9cbbef 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -258,6 +258,53 @@ public struct Configuration: Sendable { /// The test case filter to which test cases should be filtered when run. public var testCaseFilter: TestCaseFilter = { _, _ in true } + + // MARK: - Expectation value reflection + + /// The options to use when reflecting values in expressions checked by + /// expectations, or `nil` if reflection is disabled. + /// + /// When the value of this property is a non-`nil` instance, values checked by + /// expressions will be reflected using `Mirror` and the specified options + /// will influence how that reflection is formed. Otherwise, when its value is + /// `nil`, value reflection will not use `Mirror` and instead will use + /// `String(describing:)`. + /// + /// The default value of this property is an instance of ``ValueReflectionOptions-swift.struct`` + /// with its properties initialized to their default values. + public var valueReflectionOptions: ValueReflectionOptions? = .init() + + /// A type describing options to use when forming a reflection of a value + /// checked by an expectation. + public struct ValueReflectionOptions: Sendable { + /// The maximum number of elements that can included in a single child + /// collection when reflecting a value checked by an expectation. + /// + /// When ``Expression/Value/init(reflecting:)`` is reflecting a value and it + /// encounters a child value which is a collection, it consults the value of + /// this property and only includes the children of that collection up to + /// this maximum count. After this maximum is reached, all subsequent + /// elements are omitted and a single placeholder child is added indicating + /// the number of elements which have been truncated. + public var maximumCollectionCount = 10 + + /// The maximum depth of children that can be included in the reflection of + /// a checked expectation value. + /// + /// When ``Expression/Value/init(reflecting:)`` is reflecting a value, it + /// recursively reflects that value's children. Before doing so, it consults + /// the value of this property to determine the maximum depth of the + /// children to include. After this maximum depth is reached, all children + /// at deeper levels are omitted and the ``Expression/Value/isTruncated`` + /// property is set to `true` to reflect that the reflection is incomplete. + /// + /// - Note: `Optional` values contribute twice towards this maximum, since + /// their mirror represents the wrapped value as a child of the optional. + /// Since optionals are common, the default value of this property is + /// somewhat larger than it otherwise would be in an attempt to make the + /// defaults useful for real-world tests. + public var maximumChildDepth = 10 + } } // MARK: - Deprecated diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index a128d9ce9..f69e13cd6 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -72,6 +72,27 @@ extension Configuration { /// - Returns: Whatever is returned by `body`. /// /// - Throws: Whatever is thrown by `body`. + static func withCurrent(_ configuration: Self, perform body: () throws -> R) rethrows -> R { + let id = configuration._addToAll() + defer { + configuration._removeFromAll(identifiedBy: id) + } + + var runtimeState = Runner.RuntimeState.current ?? .init() + runtimeState.configuration = configuration + return try Runner.RuntimeState.$current.withValue(runtimeState, operation: body) + } + + /// Call an asynchronous function while the value of ``Configuration/current`` + /// is set. + /// + /// - Parameters: + /// - configuration: The new value to set for ``Configuration/current``. + /// - body: A function to call. + /// + /// - Returns: Whatever is returned by `body`. + /// + /// - Throws: Whatever is thrown by `body`. static func withCurrent(_ configuration: Self, perform body: () async throws -> R) async rethrows -> R { let id = configuration._addToAll() defer { diff --git a/Sources/Testing/SourceAttribution/Expression.swift b/Sources/Testing/SourceAttribution/Expression.swift index 3f6a7e716..dce4ed2a2 100644 --- a/Sources/Testing/SourceAttribution/Expression.swift +++ b/Sources/Testing/SourceAttribution/Expression.swift @@ -156,6 +156,16 @@ public struct __Expression: Sendable { /// property is `nil`. public var label: String? + /// Whether or not the values of certain properties of this instance have + /// been truncated for brevity. + /// + /// If the value of this property is `true`, this instance does not + /// represent its original value completely because doing so would exceed + /// the maximum allowed data collection settings of the ``Configuration`` in + /// effect. When this occurs, the value ``children`` is not guaranteed to be + /// accurate or complete. + public var isTruncated: Bool = false + /// Whether or not this value represents a collection of values. public var isCollection: Bool @@ -167,14 +177,47 @@ public struct __Expression: Sendable { /// the value it represents contains substructural values. public var children: [Self]? + /// Initialize an instance of this type describing the specified subject. + /// + /// - Parameters: + /// - subject: The subject this instance should describe. + init(describing subject: Any) { + description = String(describingForTest: subject) + debugDescription = String(reflecting: subject) + typeInfo = TypeInfo(describingTypeOf: subject) + + let mirror = Mirror(reflecting: subject) + isCollection = mirror.displayStyle?.isCollection ?? false + } + + /// Initialize an instance of this type with the specified description. + /// + /// - Parameters: + /// - description: The value to use for this instance's `description` + /// property. + /// + /// Unlike ``init(describing:)``, this initializer does not use + /// ``String/init(describingForTest:)`` to form a description. + private init(_description description: String) { + self.description = description + self.debugDescription = description + typeInfo = TypeInfo(describing: String.self) + isCollection = false + } + /// Initialize an instance of this type describing the specified subject and /// its children (if any). /// /// - Parameters: - /// - subject: The subject this instance should describe. - init(reflecting subject: Any) { + /// - subject: The subject this instance should reflect. + init?(reflecting subject: Any) { + let configuration = Configuration.current ?? .init() + guard let options = configuration.valueReflectionOptions else { + return nil + } + var seenObjects: [ObjectIdentifier: AnyObject] = [:] - self.init(_reflecting: subject, label: nil, seenObjects: &seenObjects) + self.init(_reflecting: subject, label: nil, seenObjects: &seenObjects, depth: 0, options: options) } /// Initialize an instance of this type describing the specified subject and @@ -189,11 +232,28 @@ public struct __Expression: Sendable { /// this initializer recursively, keyed by their object identifiers. /// This is used to halt further recursion if a previously-seen object /// is encountered again. + /// - depth: The depth of this recursive call. + /// - options: The configuration options to use when deciding how to + /// reflect `subject`. private init( _reflecting subject: Any, label: String?, - seenObjects: inout [ObjectIdentifier: AnyObject] + seenObjects: inout [ObjectIdentifier: AnyObject], + depth: Int, + options: Configuration.ValueReflectionOptions ) { + // Stop recursing if we've reached the maximum allowed depth for + // reflection. Instead, return a node describing this value instead and + // set `isTruncated` to `true`. + if depth >= options.maximumChildDepth { + self = Self(describing: subject) + isTruncated = true + return + } + + self.init(describing: subject) + self.label = label + let mirror = Mirror(reflecting: subject) // If the subject being reflected is an instance of a reference type (e.g. @@ -236,24 +296,19 @@ public struct __Expression: Sendable { } } - description = String(describingForTest: subject) - debugDescription = String(reflecting: subject) - typeInfo = TypeInfo(describingTypeOf: subject) - self.label = label - - isCollection = switch mirror.displayStyle { - case .some(.collection), - .some(.dictionary), - .some(.set): - true - default: - false - } - if shouldIncludeChildren && (!mirror.children.isEmpty || isCollection) { - self.children = mirror.children.map { child in - Self(_reflecting: child.value, label: child.label, seenObjects: &seenObjects) + var children: [Self] = [] + for (index, child) in mirror.children.enumerated() { + if isCollection && index >= options.maximumCollectionCount { + isTruncated = true + let message = "(\(mirror.children.count - index) out of \(mirror.children.count) elements omitted for brevity)" + children.append(Self(_description: message)) + break + } + + children.append(Self(_reflecting: child.value, label: child.label, seenObjects: &seenObjects, depth: depth + 1, options: options)) } + self.children = children } } } @@ -274,7 +329,7 @@ public struct __Expression: Sendable { /// value captured for future use. func capturingRuntimeValue(_ value: (some Any)?) -> Self { var result = self - result.runtimeValue = value.map { Value(reflecting: $0) } + result.runtimeValue = value.flatMap(Value.init(reflecting:)) if case let .negation(subexpression, isParenthetical) = kind, let value = value as? Bool { result.kind = .negation(subexpression.capturingRuntimeValue(!value), isParenthetical: isParenthetical) } @@ -547,3 +602,17 @@ extension __Expression.Value: CustomStringConvertible, CustomDebugStringConverti /// ``` @_spi(ForToolsIntegrationOnly) public typealias Expression = __Expression + +extension Mirror.DisplayStyle { + /// Whether or not this display style represents a collection of values. + fileprivate var isCollection: Bool { + switch self { + case .collection, + .dictionary, + .set: + true + default: + false + } + } +} diff --git a/Tests/TestingTests/Expression.ValueTests.swift b/Tests/TestingTests/Expression.ValueTests.swift index c9929f59c..e1e18d4ea 100644 --- a/Tests/TestingTests/Expression.ValueTests.swift +++ b/Tests/TestingTests/Expression.ValueTests.swift @@ -21,7 +21,7 @@ struct Expression_ValueTests { let foo = Foo() - let value = Expression.Value(reflecting: foo) + let value = try #require(Expression.Value(reflecting: foo)) let children = try #require(value.children) try #require(children.count == 1) @@ -49,7 +49,7 @@ struct Expression_ValueTests { x.one = y x.two = y - let value = Expression.Value(reflecting: x) + let value = try #require(Expression.Value(reflecting: x)) let children = try #require(value.children) try #require(children.count == 3) @@ -78,7 +78,7 @@ struct Expression_ValueTests { x.two = y y.two = x - let value = Expression.Value(reflecting: x) + let value = try #require(Expression.Value(reflecting: x)) let children = try #require(value.children) try #require(children.count == 3) @@ -116,7 +116,7 @@ struct Expression_ValueTests { let recursiveItem = RecursiveItem() recursiveItem.anotherItem = recursiveItem - let value = Expression.Value(reflecting: recursiveItem) + let value = try #require(Expression.Value(reflecting: recursiveItem)) let children = try #require(value.children) try #require(children.count == 2) @@ -142,7 +142,7 @@ struct Expression_ValueTests { one.two = two two.one = one - let value = Expression.Value(reflecting: one) + let value = try #require(Expression.Value(reflecting: one)) let children = try #require(value.children) try #require(children.count == 1) @@ -168,7 +168,7 @@ struct Expression_ValueTests { @Test("Value reflecting an object with two back-references to itself", .bug("https://github.com/swiftlang/swift-testing/issues/785#issuecomment-2440222995")) - func multipleSelfReferences() { + func multipleSelfReferences() throws { class A { weak var one: A? weak var two: A? @@ -178,7 +178,7 @@ struct Expression_ValueTests { a.one = a a.two = a - let value = Expression.Value(reflecting: a) + let value = try #require(Expression.Value(reflecting: a)) #expect(value.children?.count == 2) } @@ -208,8 +208,88 @@ struct Expression_ValueTests { b.c = c c.a = a - let value = Expression.Value(reflecting: a) + let value = try #require(Expression.Value(reflecting: a)) #expect(value.children?.count == 3) } + @Test("Value reflection can be disabled via Configuration") + func valueReflectionDisabled() { + var configuration = Configuration.current ?? .init() + configuration.valueReflectionOptions = nil + Configuration.withCurrent(configuration) { + #expect(Expression.Value(reflecting: "hello") == nil) + } + } + + @Test("Value reflection truncates large values") + func reflectionOfLargeValues() throws { + struct Large { + var foo: Int? + var bar: [Int] + } + + var configuration = Configuration.current ?? .init() + var options = configuration.valueReflectionOptions ?? .init() + options.maximumCollectionCount = 2 + options.maximumChildDepth = 2 + configuration.valueReflectionOptions = options + + try Configuration.withCurrent(configuration) { + let large = Large(foo: 123, bar: [4, 5, 6, 7]) + let value = try #require(Expression.Value(reflecting: large)) + + #expect(!value.isTruncated) + do { + let fooValue = try #require(value.children?.first) + #expect(!fooValue.isTruncated) + let fooChildren = try #require(fooValue.children) + try #require(fooChildren.count == 1) + let fooChild = try #require(fooChildren.first) + #expect(fooChild.isTruncated) + #expect(fooChild.children == nil) + } + do { + let barValue = try #require(value.children?.last) + #expect(barValue.isTruncated) + #expect(barValue.children?.count == 3) + let lastBarChild = try #require(barValue.children?.last) + #expect(String(describing: lastBarChild) == "(2 out of 4 elements omitted for brevity)") + } + } + } + + @Test("Value reflection max collection count only applies to collections") + func reflectionMaximumCollectionCount() throws { + struct X { + var a = 1 + var b = 2 + var c = 3 + var d = 4 + } + + var configuration = Configuration.current ?? .init() + var options = configuration.valueReflectionOptions ?? .init() + options.maximumCollectionCount = 2 + configuration.valueReflectionOptions = options + + try Configuration.withCurrent(configuration) { + let x = X() + let value = try #require(Expression.Value(reflecting: x)) + #expect(!value.isTruncated) + #expect(value.children?.count == 4) + } + } + + @Test("Value describing a simple struct") + func describeSimpleStruct() { + struct Foo { + var x: Int = 123 + } + + let foo = Foo() + let value = Expression.Value(describing: foo) + #expect(String(describing: value) == "Foo(x: 123)") + #expect(value.children == nil) + } + } From a5921e01143bfcc831cf3a36dcaca891a3803dd1 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 17 Jan 2025 16:22:37 -0500 Subject: [PATCH 056/234] Don't allocate a buffer on the heap when enumerating type metadata. (#918) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR eliminates the need for us to allocate a large buffer to contain type metadata pointers we discover in type metadata sections using the legacy test discovery mechanism. Why do I keep opening these PRs? Because for each line of C++ I remove from the Swift toolchain, I get a cookie! 🍪 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Test+Discovery+Legacy.swift | 10 +-- Sources/_TestingInternals/Discovery.cpp | 64 +++++++------------ Sources/_TestingInternals/include/Discovery.h | 29 ++++----- 3 files changed, 40 insertions(+), 63 deletions(-) diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index c90e8c590..6ee080051 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -55,12 +55,8 @@ let exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" /// - Returns: A sequence of Swift types whose names contain `nameSubstring`. func types(withNamesContaining nameSubstring: String) -> some Sequence { SectionBounds.all(.typeMetadata).lazy.flatMap { sb in - var count = 0 - let start = swt_copyTypes(in: sb.buffer.baseAddress!, sb.buffer.count, withNamesContaining: nameSubstring, count: &count) - defer { - free(start) - } - return UnsafeBufferPointer(start: start, count: count) - .withMemoryRebound(to: Any.Type.self) { Array($0) } + stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy + .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: nameSubstring) } + .map { unsafeBitCast($0, to: Any.Type.self) } } } diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index f66eaf874..6173126c1 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -10,11 +10,9 @@ #include "Discovery.h" -#include #include #include #include -#include #if defined(SWT_NO_DYNAMIC_LINKING) #pragma mark - Statically-linked section bounds @@ -189,46 +187,32 @@ struct SWTTypeMetadataRecord { #pragma mark - Legacy test discovery -void **swt_copyTypesWithNamesContaining(const void *sectionBegin, size_t sectionSize, const char *nameSubstring, size_t *outCount) { - void **result = nullptr; - size_t resultCount = 0; - - auto records = reinterpret_cast(sectionBegin); - size_t recordCount = sectionSize / sizeof(SWTTypeMetadataRecord); - for (size_t i = 0; i < recordCount; i++) { - auto contextDescriptor = records[i].getContextDescriptor(); - if (!contextDescriptor) { - // This type metadata record is invalid (or we don't understand how to - // get its context descriptor), so skip it. - continue; - } else if (contextDescriptor->isGeneric()) { - // Generic types cannot be fully instantiated without generic - // parameters, which is not something we can know abstractly. - continue; - } +const size_t SWTTypeMetadataRecordByteCount = sizeof(SWTTypeMetadataRecord); - // Check that the type's name passes. This will be more expensive than the - // checks above, but should be cheaper than realizing the metadata. - const char *typeName = contextDescriptor->getName(); - bool nameOK = typeName && nullptr != std::strstr(typeName, nameSubstring); - if (!nameOK) { - continue; - } +const void *swt_getTypeFromTypeMetadataRecord(const void *recordAddress, const char *nameSubstring) { + auto record = reinterpret_cast(recordAddress); + auto contextDescriptor = record->getContextDescriptor(); + if (!contextDescriptor) { + // This type metadata record is invalid (or we don't understand how to + // get its context descriptor), so skip it. + return nullptr; + } else if (contextDescriptor->isGeneric()) { + // Generic types cannot be fully instantiated without generic + // parameters, which is not something we can know abstractly. + return nullptr; + } - if (void *typeMetadata = contextDescriptor->getMetadata()) { - if (!result) { - // This is the first matching type we've found. That presumably means - // we'll find more, so allocate enough space for all remaining types in - // the section. Is this necessarily space-efficient? No, but this - // allocation is short-lived and is immediately copied and freed in the - // Swift caller. - result = reinterpret_cast(std::calloc(recordCount - i, sizeof(void *))); - } - result[resultCount] = typeMetadata; - resultCount += 1; - } + // Check that the type's name passes. This will be more expensive than the + // checks above, but should be cheaper than realizing the metadata. + const char *typeName = contextDescriptor->getName(); + bool nameOK = typeName && nullptr != std::strstr(typeName, nameSubstring); + if (!nameOK) { + return nullptr; + } + + if (void *typeMetadata = contextDescriptor->getMetadata()) { + return typeMetadata; } - *outCount = resultCount; - return result; + return nullptr; } diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index cbcfc2d79..9d84b9f8c 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -84,25 +84,22 @@ SWT_EXTERN const void *_Nonnull const SWTTypeMetadataSectionBounds[2]; #pragma mark - Legacy test discovery -/// Copy all types known to Swift found in the given type metadata section with -/// a name containing the given substring. +/// The size, in bytes, of a Swift type metadata record. +SWT_EXTERN const size_t SWTTypeMetadataRecordByteCount; + +/// Get the type represented by the type metadata record at the given address if +/// its name contains the given string. /// /// - Parameters: -/// - sectionBegin: The address of the first byte of the Swift type metadata -/// section. -/// - sectionSize: The size, in bytes, of the Swift type metadata section. -/// - nameSubstring: A string which the names of matching classes all contain. -/// - outCount: On return, the number of type metadata pointers returned. +/// - recordAddress: The address of the Swift type metadata record. +/// - nameSubstring: A string which the names of matching types contain. /// -/// - Returns: A pointer to an array of type metadata pointers, or `nil` if no -/// matching types were found. The caller is responsible for freeing this -/// memory with `free()` when done. -SWT_EXTERN void *_Nonnull *_Nullable swt_copyTypesWithNamesContaining( - const void *sectionBegin, - size_t sectionSize, - const char *nameSubstring, - size_t *outCount -) SWT_SWIFT_NAME(swt_copyTypes(in:_:withNamesContaining:count:)); +/// - Returns: A Swift metatype (as `const void *`) or `nullptr` if it wasn't a +/// usable type metadata record or its name did not contain `nameSubstring`. +SWT_EXTERN const void *_Nullable swt_getTypeFromTypeMetadataRecord( + const void *recordAddress, + const char *nameSubstring +) SWT_SWIFT_NAME(swt_getType(fromTypeMetadataRecord:ifNameContains:)); SWT_ASSUME_NONNULL_END From 537a0c4c3c63f61251c40b2ce54f01708c3993d7 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Tue, 21 Jan 2025 12:13:32 +0000 Subject: [PATCH 057/234] [TestingMacros] Remove reference to implicit backtracing import. The _Backtracing module was never implicitly imported by default, so this wasn't necessary anyway, but adding it is a blocker for SE-0419. rdar://143310300 --- Sources/TestingMacros/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index acf09f339..4fc8b3b58 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -108,8 +108,7 @@ target_sources(TestingMacros PRIVATE TestingMacrosMain.swift) target_compile_options(TestingMacros PRIVATE - "SHELL:-Xfrontend -disable-implicit-string-processing-module-import" - "SHELL:-Xfrontend -disable-implicit-backtracing-module-import") + "SHELL:-Xfrontend -disable-implicit-string-processing-module-import") target_link_libraries(TestingMacros PRIVATE SwiftSyntax::SwiftSyntax From 1747fb0c795ca490463aad8c3b2d8815f7c3222f Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 21 Jan 2025 14:22:36 -0500 Subject: [PATCH 058/234] Remove arm64e ptrauth resigning from test content discovery. (#919) Finally got around to double-checking and we don't need to do ptrauth resigning on the `accessor` function pointers in the new test content section as dyld already does it for us at load time. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery.swift | 2 +- Sources/_TestingInternals/Discovery.cpp | 1 + Sources/_TestingInternals/include/Discovery.h | 36 ++----------------- Sources/_TestingInternals/include/Includes.h | 4 --- 4 files changed, 4 insertions(+), 39 deletions(-) diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index 5ce454d1c..2ad44b324 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -116,7 +116,7 @@ extension TestContentRecord where T: TestContent & ~Copyable { /// If this function is called more than once on the same instance, a new /// value is created on each call. func load(withHint hint: T.TestContentAccessorHint? = nil) -> T.TestContentAccessorResult? { - guard let accessor = _record.accessor.map(swt_resign) else { + guard let accessor = _record.accessor else { return nil } diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index 6173126c1..314b7794d 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -47,6 +47,7 @@ const void *_Nonnull const SWTTypeMetadataSectionBounds[2] = { #pragma mark - Swift ABI #if defined(__PTRAUTH_INTRINSICS__) +#include #define SWT_PTRAUTH_SWIFT_TYPE_DESCRIPTOR __ptrauth(ptrauth_key_process_independent_data, 1, 0xae86) #else #define SWT_PTRAUTH_SWIFT_TYPE_DESCRIPTOR diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index 9d84b9f8c..8fc36d8c7 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -16,41 +16,9 @@ SWT_ASSUME_NONNULL_BEGIN -#pragma mark - Test content records - -/// The type of a test content accessor. -/// -/// - Parameters: -/// - outValue: On successful return, initialized to the value of the -/// represented test content record. -/// - hint: A hint value whose type and meaning depend on the type of test -/// record being accessed. -/// -/// - Returns: Whether or not the test record was initialized at `outValue`. If -/// this function returns `true`, the caller is responsible for deinitializing -/// the memory at `outValue` when done. -typedef bool (* SWTTestContentAccessor)(void *outValue, const void *_Null_unspecified hint); - -/// Resign an accessor function from a test content record. -/// -/// - Parameters: -/// - accessor: The accessor function to resign. -/// -/// - Returns: A resigned copy of `accessor` on platforms that use pointer -/// authentication, and an exact copy of `accessor` elsewhere. -/// -/// - Bug: This C function is needed because Apple's pointer authentication -/// intrinsics are not available in Swift. ([141465242](rdar://141465242)) -SWT_SWIFT_NAME(swt_resign(_:)) -static SWTTestContentAccessor swt_resignTestContentAccessor(SWTTestContentAccessor accessor) { -#if defined(__APPLE__) && __has_include() - accessor = ptrauth_strip(accessor, ptrauth_key_function_pointer); - accessor = ptrauth_sign_unauthenticated(accessor, ptrauth_key_function_pointer, 0); -#endif - return accessor; -} - #if defined(__ELF__) && defined(__swift__) +#pragma mark - ELF image enumeration + /// A function exported by the Swift runtime that enumerates all metadata /// sections loaded into the current process. /// diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index dfcbf50f0..b1f4c7973 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -127,10 +127,6 @@ #if !SWT_NO_LIBDISPATCH #include #endif - -#if __has_include() -#include -#endif #endif #if defined(__FreeBSD__) From 8836b38af93b3690c86d7feebe39c63f154fcf6a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 21 Jan 2025 19:39:38 -0500 Subject: [PATCH 059/234] Make all test content types directly conform to `TestContent`. (#920) This PR eliminates the `TestContentAccessorResult` associated type from the (currently internal, potentially eventually API) `TestContent` protocol. This associated type needed to be `~Copyable` so `ExitTest` could be used with it, but that appears to pose some _problems_ for the compiler (rdar://143049814&143080508). Instead, we remove the associated type and just say "the test content record is the type that conforms to `TestContent`". `ExitTest` is happy with this, but `Test`'s produced type is a non-nominal function type, so we wrap that function in a small private type with identical layout and have that type conform. The ultimate purpose of this PR is to get us a bit closer to turning `TestContent` into a public or tools-SPI protocol that other components can use for test discovery. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 14 +++++++++ Sources/Testing/Discovery+Platform.swift | 2 +- Sources/Testing/Discovery.swift | 26 +++++------------ Sources/Testing/ExitTests/ExitTest.swift | 1 - Sources/Testing/Test+Discovery.swift | 32 +++++++++++++++++---- Tests/TestingTests/MiscellaneousTests.swift | 22 +++++++------- 6 files changed, 60 insertions(+), 37 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index c1a90aff1..4f1346b95 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -63,6 +63,17 @@ struct SWTTestContentRecord { }; ``` +Do not use the `__TestContentRecord` typealias defined in the testing library. +This type exists to support the testing library's macros and may change in the +future (e.g. to accomodate a generic argument or to make use of one of the +reserved fields.) + +Instead, define your own copy of this type where needed—you can copy the +definition above _verbatim_. If your test record type's `context` field (as +described below) is a pointer type, make sure to change its type in your version +of `TestContentRecord` accordingly so that, on systems with pointer +authentication enabled, the pointer is correctly resigned at load time. + ### Record content #### The kind field @@ -79,6 +90,9 @@ record's kind is a 32-bit unsigned value. The following kinds are defined: +If a test content record's `kind` field equals `0x00000000`, the values of all +other fields in that record are undefined. + #### The accessor field The function `accessor` is a C function. When called, it initializes the memory diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index 9c352bb41..a42800577 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -230,7 +230,7 @@ private func _findSection(named sectionName: String, in hModule: HMODULE) -> Sec /// /// - Returns: An array of structures describing the bounds of all known test /// content sections in the current process. -private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { +private func _sectionBounds(_ kind: SectionBounds.Kind) -> some Sequence { let sectionName = switch kind { case .testContent: ".sw5test" diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index 2ad44b324..647da522f 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -48,15 +48,7 @@ protocol TestContent: ~Copyable { /// `ABI/TestContent.md` for a list of values and corresponding types. static var testContentKind: UInt32 { get } - /// The type of value returned by the test content accessor for this type. - /// - /// This type may or may not equal `Self` depending on the type's compile-time - /// and runtime requirements. If it does not equal `Self`, it should equal a - /// type whose instances can be converted to instances of `Self` (e.g. by - /// calling them if they are functions.) - associatedtype TestContentAccessorResult: ~Copyable - - /// A type of "hint" passed to ``discover(withHint:)`` to help the testing + /// A type of "hint" passed to ``allTestContentRecords()`` to help the testing /// library find the correct result. /// /// By default, this type equals `Never`, indicating that this type of test @@ -75,7 +67,7 @@ protocol TestContent: ~Copyable { /// This type is not part of the public interface of the testing library. In the /// future, we could make it public if we want to support runtime discovery of /// test content by second- or third-party code. -struct TestContentRecord: Sendable where T: ~Copyable { +struct TestContentRecord: Sendable where T: TestContent & ~Copyable { /// The base address of the image containing this instance, if known. /// /// On platforms such as WASI that statically link to the testing library, the @@ -93,11 +85,7 @@ struct TestContentRecord: Sendable where T: ~Copyable { self.imageAddress = imageAddress self._record = record } -} -// This `T: TestContent` constraint is in an extension in order to work around a -// compiler crash. SEE: rdar://143049814 -extension TestContentRecord where T: TestContent & ~Copyable { /// The context value for this test content record. var context: UInt { _record.context @@ -109,18 +97,18 @@ extension TestContentRecord where T: TestContent & ~Copyable { /// - hint: An optional hint value. If not `nil`, this value is passed to /// the accessor function of the underlying test content record. /// - /// - Returns: An instance of the associated ``TestContentAccessorResult`` - /// type, or `nil` if the underlying test content record did not match - /// `hint` or otherwise did not produce a value. + /// - Returns: An instance of the test content type `T`, or `nil` if the + /// underlying test content record did not match `hint` or otherwise did not + /// produce a value. /// /// If this function is called more than once on the same instance, a new /// value is created on each call. - func load(withHint hint: T.TestContentAccessorHint? = nil) -> T.TestContentAccessorResult? { + func load(withHint hint: T.TestContentAccessorHint? = nil) -> T? { guard let accessor = _record.accessor else { return nil } - return withUnsafeTemporaryAllocation(of: T.TestContentAccessorResult.self, capacity: 1) { buffer in + return withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in let initialized = if let hint { withUnsafePointer(to: hint) { hint in accessor(buffer.baseAddress!, hint) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index a3a92c036..f6ea88d22 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -228,7 +228,6 @@ extension ExitTest: TestContent { 0x65786974 } - typealias TestContentAccessorResult = Self typealias TestContentAccessorHint = ID } diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 939e85947..015a0b1c8 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -10,12 +10,27 @@ private import _TestingInternals -extension Test: TestContent { - static var testContentKind: UInt32 { - 0x74657374 - } +extension Test { + /// A type that encapsulates test content records that produce instances of + /// ``Test``. + /// + /// This type is necessary because such test content records produce an + /// indirect `async` accessor function rather than directly producing + /// instances of ``Test``, but functions are non-nominal types and cannot + /// directly conform to protocols. + /// + /// - Note: This helper type must have the exact in-memory layout of the + /// `async` accessor function. Do not add any additional cases or associated + /// values. The layout of this type is [guaranteed](https://github.com/swiftlang/swift/blob/main/docs/ABI/TypeLayout.rst#fragile-enum-layout) + /// by the Swift ABI. + /* @frozen */ private enum _Record: TestContent { + static var testContentKind: UInt32 { + 0x74657374 + } - typealias TestContentAccessorResult = @Sendable () async -> Self + /// The actual (asynchronous) accessor function. + case generator(@Sendable () async -> Test) + } /// All available ``Test`` instances in the process, according to the runtime. /// @@ -43,7 +58,12 @@ extension Test: TestContent { // Walk all test content and gather generator functions, then call them in // a task group and collate their results. if useNewMode { - let generators = Self.allTestContentRecords().lazy.compactMap { $0.load() } + let generators = _Record.allTestContentRecords().lazy.compactMap { record in + if case let .generator(generator) = record.load() { + return generator + } + return nil // currently unreachable, but not provably so + } await withTaskGroup(of: Self.self) { taskGroup in for generator in generators { taskGroup.addTask(operation: generator) diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 9cc3b7910..a172f7d5a 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -583,7 +583,8 @@ struct MiscellaneousTests { #if !SWT_NO_DYNAMIC_LINKING && hasFeature(SymbolLinkageMarkers) struct DiscoverableTestContent: TestContent { typealias TestContentAccessorHint = UInt32 - typealias TestContentAccessorResult = UInt32 + + var value: UInt32 static var testContentKind: UInt32 { record.kind @@ -593,7 +594,7 @@ struct MiscellaneousTests { 0x01020304 } - static var expectedResult: TestContentAccessorResult { + static var expectedValue: UInt32 { 0xCAFEF00D } @@ -618,7 +619,7 @@ struct MiscellaneousTests { if let hint, hint.load(as: TestContentAccessorHint.self) != expectedHint { return false } - _ = outValue.initializeMemory(as: TestContentAccessorResult.self, to: expectedResult) + _ = outValue.initializeMemory(as: Self.self, to: .init(value: expectedValue)) return true }, UInt(truncatingIfNeeded: UInt64(0x0204060801030507)), @@ -628,32 +629,33 @@ struct MiscellaneousTests { @Test func testDiscovery() async { // Check the type of the test record sequence (it should be lazy.) - let allRecords = DiscoverableTestContent.allTestContentRecords() + let allRecordsSeq = DiscoverableTestContent.allTestContentRecords() #if SWT_FIXED_143080508 - #expect(allRecords is any LazySequenceProtocol) - #expect(!(allRecords is [TestContentRecord])) + #expect(allRecordsSeq is any LazySequenceProtocol) + #expect(!(allRecordsSeq is [TestContentRecord])) #endif // It should have exactly one matching record (because we only emitted one.) - #expect(Array(allRecords).count == 1) + let allRecords = Array(allRecordsSeq) + #expect(allRecords.count == 1) // Can find a single test record #expect(allRecords.contains { record in - record.load() == DiscoverableTestContent.expectedResult + record.load()?.value == DiscoverableTestContent.expectedValue && record.context == DiscoverableTestContent.expectedContext }) // Can find a test record with matching hint #expect(allRecords.contains { record in let hint = DiscoverableTestContent.expectedHint - return record.load(withHint: hint) == DiscoverableTestContent.expectedResult + return record.load(withHint: hint)?.value == DiscoverableTestContent.expectedValue && record.context == DiscoverableTestContent.expectedContext }) // Doesn't find a test record with a mismatched hint #expect(!allRecords.contains { record in let hint = ~DiscoverableTestContent.expectedHint - return record.load(withHint: hint) == DiscoverableTestContent.expectedResult + return record.load(withHint: hint)?.value == DiscoverableTestContent.expectedValue && record.context == DiscoverableTestContent.expectedContext }) } From faaabba3a977d670bb34dbba965a74ce500cb07f Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 21 Jan 2025 19:39:51 -0500 Subject: [PATCH 060/234] Use `swift_getErrorValue()` unless Objective-C is available. (#922) Tweak the logic for when we use the runtime-internal `swift_getErrorValue()` function in our `_swift_willThrow` handler so that we use it if the runtime does not support Objective-C interop. Currently, we toggle based on whether we're on an Apple platform, but this isn't quite right. Resolves rdar://143328537. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/SourceAttribution/Backtrace.swift | 4 ++-- Sources/_TestingInternals/include/WillThrow.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/SourceAttribution/Backtrace.swift b/Sources/Testing/SourceAttribution/Backtrace.swift index a6019860c..78227e3da 100644 --- a/Sources/Testing/SourceAttribution/Backtrace.swift +++ b/Sources/Testing/SourceAttribution/Backtrace.swift @@ -133,7 +133,7 @@ extension Backtrace { /// - errorAddress: The address of the error existential box. init(_ errorAddress: UnsafeMutableRawPointer) { _rawValue = errorAddress -#if SWT_TARGET_OS_APPLE +#if _runtime(_ObjC) let error = Unmanaged.fromOpaque(errorAddress).takeUnretainedValue() as! any Error if type(of: error) is AnyObject.Type { _rawValue = Unmanaged.passUnretained(error as AnyObject).toOpaque() @@ -336,7 +336,7 @@ extension Backtrace { /// - Note: The underlying Foundation function is called (if present) the /// first time the value of this property is read. static let isFoundationCaptureEnabled = { -#if SWT_TARGET_OS_APPLE && !SWT_NO_DYNAMIC_LINKING +#if _runtime(_ObjC) && !SWT_NO_DYNAMIC_LINKING if Environment.flag(named: "SWT_FOUNDATION_ERROR_BACKTRACING_ENABLED") == true { let _CFErrorSetCallStackCaptureEnabled = symbol(named: "_CFErrorSetCallStackCaptureEnabled").map { unsafeBitCast($0, to: (@convention(c) (DarwinBoolean) -> DarwinBoolean).self) diff --git a/Sources/_TestingInternals/include/WillThrow.h b/Sources/_TestingInternals/include/WillThrow.h index 3d5a7c319..13331773f 100644 --- a/Sources/_TestingInternals/include/WillThrow.h +++ b/Sources/_TestingInternals/include/WillThrow.h @@ -71,8 +71,8 @@ typedef void (* SWT_SENDABLE SWTWillThrowTypedHandler)(void *error, const void * /// ``SWTWillThrowTypedHandler`` SWT_EXTERN SWTWillThrowTypedHandler SWT_SENDABLE _Nullable swt_setWillThrowTypedHandler(SWTWillThrowTypedHandler SWT_SENDABLE _Nullable handler); -#if !defined(__APPLE__) -/// The result of `swift__getErrorValue()`. +#if defined(__swift__) && !defined(__OBJC__) +/// The result of `swift_getErrorValue()`. /// /// For more information, see this type's declaration /// [in the Swift repository](https://github.com/swiftlang/swift/blob/main/include/swift/Runtime/Error.h). From 95b97a624d1a621963c084e86f3253a8faf9be67 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 27 Jan 2025 12:49:12 -0500 Subject: [PATCH 061/234] Remove Stubs.cpp and reimplement what it has in Swift. (#929) This PR removes Stubs.cpp, which currently houses some thunks for functions that are conditionally unavailable in glibc, and replaces it with runtime function lookups in Swift. Is there potentially a one-time non-zero performance cost? Yes. Is that performance cost prohibitive given that the functions are only looked up once and then cached? No. These functions won't get called on Linux if `SWT_NO_DYNAMIC_LINKING` is defined but we don't currently support that combination anyway. Even if you're using a statically-linked Swift standard library, we'd expect Linux to still support calling `dlsym()`. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/SpawnProcess.swift | 21 +++++++-- Sources/Testing/ExitTests/WaitFor.swift | 15 ++++++- Sources/_TestingInternals/CMakeLists.txt | 1 - Sources/_TestingInternals/Stubs.cpp | 45 -------------------- Sources/_TestingInternals/include/Stubs.h | 19 --------- 5 files changed, 31 insertions(+), 70 deletions(-) delete mode 100644 Sources/_TestingInternals/Stubs.cpp diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index fd18aad8a..2d4a67442 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -26,6 +26,18 @@ typealias ProcessID = HANDLE typealias ProcessID = Never #endif +#if os(Linux) && !SWT_NO_DYNAMIC_LINKING +/// Close file descriptors above a given value when spawing a new process. +/// +/// This symbol is provided because the underlying function was added to glibc +/// relatively recently and may not be available on all targets. Checking +/// `__GLIBC_PREREQ()` is insufficient because `_DEFAULT_SOURCE` may not be +/// defined at the point spawn.h is first included. +private let _posix_spawn_file_actions_addclosefrom_np = symbol(named: "posix_spawn_file_actions_addclosefrom_np").map { + unsafeBitCast($0, to: (@convention(c) (UnsafeMutablePointer, CInt) -> CInt).self) +} +#endif + /// Spawn a process and wait for it to terminate. /// /// - Parameters: @@ -141,17 +153,18 @@ func spawnExecutable( // Close all other file descriptors open in the parent. flags |= CShort(POSIX_SPAWN_CLOEXEC_DEFAULT) #elseif os(Linux) +#if !SWT_NO_DYNAMIC_LINKING // This platform doesn't have POSIX_SPAWN_CLOEXEC_DEFAULT, but we can at // least close all file descriptors higher than the highest inherited one. // We are assuming here that the caller didn't set FD_CLOEXEC on any of // these file descriptors. - _ = swt_posix_spawn_file_actions_addclosefrom_np(fileActions, highestFD + 1) + _ = _posix_spawn_file_actions_addclosefrom_np?(fileActions, highestFD + 1) +#endif #elseif os(FreeBSD) // Like Linux, this platform doesn't have POSIX_SPAWN_CLOEXEC_DEFAULT. // Unlike Linux, all non-EOL FreeBSD versions (≥13.1) support - // `posix_spawn_file_actions_addclosefrom_np`. Therefore, we don't need - // `swt_posix_spawn_file_actions_addclosefrom_np` to guard the - // availability of this function. + // `posix_spawn_file_actions_addclosefrom_np`, and FreeBSD does not use + // glibc nor guard symbols behind `_DEFAULT_SOURCE`. _ = posix_spawn_file_actions_addclosefrom_np(fileActions, highestFD + 1) #elseif os(OpenBSD) // OpenBSD does not have posix_spawn_file_actions_addclosefrom_np(). diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index 239b4a4ba..fac3e1496 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -94,6 +94,17 @@ private nonisolated(unsafe) let _waitThreadNoChildrenCondition = { return result }() +#if os(Linux) && !SWT_NO_DYNAMIC_LINKING +/// Set the name of the current thread. +/// +/// This function declaration is provided because `pthread_setname_np()` is +/// only declared if `_GNU_SOURCE` is set, but setting it causes build errors +/// due to conflicts with Swift's Glibc module. +private let _pthread_setname_np = symbol(named: "pthread_setname_np").map { + unsafeBitCast($0, to: (@convention(c) (pthread_t, UnsafePointer) -> CInt).self) +} +#endif + /// Create a waiter thread that is responsible for waiting for child processes /// to exit. private let _createWaitThread: Void = { @@ -152,7 +163,9 @@ private let _createWaitThread: Void = { #if SWT_TARGET_OS_APPLE _ = pthread_setname_np("Swift Testing exit test monitor") #elseif os(Linux) - _ = swt_pthread_setname_np(pthread_self(), "SWT ExT monitor") +#if !SWT_NO_DYNAMIC_LINKING + _ = _pthread_setname_np?(pthread_self(), "SWT ExT monitor") +#endif #elseif os(FreeBSD) _ = pthread_set_name_np(pthread_self(), "SWT ex test monitor") #elseif os(OpenBSD) diff --git a/Sources/_TestingInternals/CMakeLists.txt b/Sources/_TestingInternals/CMakeLists.txt index e72143e63..ed707cd78 100644 --- a/Sources/_TestingInternals/CMakeLists.txt +++ b/Sources/_TestingInternals/CMakeLists.txt @@ -12,7 +12,6 @@ include(LibraryVersion) include(TargetTriple) add_library(_TestingInternals STATIC Discovery.cpp - Stubs.cpp Versions.cpp WillThrow.cpp) target_include_directories(_TestingInternals PUBLIC diff --git a/Sources/_TestingInternals/Stubs.cpp b/Sources/_TestingInternals/Stubs.cpp deleted file mode 100644 index 5fb8b4ff4..000000000 --- a/Sources/_TestingInternals/Stubs.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// This source file includes implementations of functions that _should_ simply -/// be `static` stubs in Stubs.h, but which for technical reasons cannot be -/// imported into Swift when defined in a header. -/// -/// Do not, as a rule, add function implementations in this file. Prefer to add -/// them to Stubs.h so that they can be inlined at compile- or link-time. Only -/// include functions here if Swift cannot successfully import and call them -/// otherwise. - -#undef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE 1 -#undef _GNU_SOURCE -#define _GNU_SOURCE 1 - -#include "Stubs.h" - -#if defined(__linux__) -int swt_pthread_setname_np(pthread_t thread, const char *name) { - return pthread_setname_np(thread, name); -} -#endif - -#if defined(__GLIBC__) -int swt_posix_spawn_file_actions_addclosefrom_np(posix_spawn_file_actions_t *fileActions, int from) { - int result = 0; - -#if defined(__GLIBC_PREREQ) -#if __GLIBC_PREREQ(2, 34) - result = posix_spawn_file_actions_addclosefrom_np(fileActions, from); -#endif -#endif - - return result; -} -#endif diff --git a/Sources/_TestingInternals/include/Stubs.h b/Sources/_TestingInternals/include/Stubs.h index 303cf0c46..8093a3722 100644 --- a/Sources/_TestingInternals/include/Stubs.h +++ b/Sources/_TestingInternals/include/Stubs.h @@ -117,25 +117,6 @@ static char *_Nullable *_Null_unspecified swt_environ(void) { } #endif -#if defined(__linux__) -/// Set the name of the current thread. -/// -/// This function declaration is provided because `pthread_setname_np()` is -/// only declared if `_GNU_SOURCE` is set, but setting it causes build errors -/// due to conflicts with Swift's Glibc module. -SWT_EXTERN int swt_pthread_setname_np(pthread_t thread, const char *name); -#endif - -#if defined(__GLIBC__) -/// Close file descriptors above a given value when spawing a new process. -/// -/// This symbol is provided because the underlying function was added to glibc -/// relatively recently and may not be available on all targets. Checking -/// `__GLIBC_PREREQ()` is insufficient because `_DEFAULT_SOURCE` may not be -/// defined at the point spawn.h is first included. -SWT_EXTERN int swt_posix_spawn_file_actions_addclosefrom_np(posix_spawn_file_actions_t *fileActions, int from); -#endif - #if !defined(__ANDROID__) #if __has_include() && defined(si_pid) /// Get the value of the `si_pid` field of a `siginfo_t` structure. From 33ea4dd1031ae9546095a4d83fa6960d1adb25bc Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 27 Jan 2025 13:57:49 -0600 Subject: [PATCH 062/234] Declare Swift 6.1 availability for TestScoping-related APIs (#927) The `TestScoping` protocol and its related APIs have been introduced in Swift 6.1, so this annotates their availability accordingly. Once this PR lands, I will cherry-pick it to the `release/6.1` branch as well. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- ...t-scopeProvider-default-implementation-Self.md | 15 +++++++++++++++ .../AvailabilityStubs/Traits/TestScopeProvider.md | 15 +++++++++++++++ .../Traits/TestScoping-provideScope.md | 15 +++++++++++++++ .../AvailabilityStubs/Traits/TestScoping.md | 15 +++++++++++++++ ...-scopeProvider-default-implementation-Never.md | 15 +++++++++++++++ ...t-scopeProvider-default-implementation-Self.md | 15 +++++++++++++++ .../Trait-scopeProvider-protocol-requirement.md | 15 +++++++++++++++ 7 files changed, 105 insertions(+) create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md create mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md new file mode 100644 index 000000000..6136ee8cb --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)-1z8kh`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md new file mode 100644 index 000000000..25281bfe6 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md @@ -0,0 +1,15 @@ +# ``Trait/TestScopeProvider`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md new file mode 100644 index 000000000..809fb833e --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md @@ -0,0 +1,15 @@ +# ``TestScoping/provideScope(for:testCase:performing:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md new file mode 100644 index 000000000..a0ab00e1f --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md @@ -0,0 +1,15 @@ +# ``TestScoping`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md new file mode 100644 index 000000000..8e903eb3b --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)-9fxg4`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md new file mode 100644 index 000000000..0ff33a204 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)-inmj`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md new file mode 100644 index 000000000..5fcff2667 --- /dev/null +++ b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md @@ -0,0 +1,15 @@ +# ``Trait/scopeProvider(for:testCase:)`` + + + +@Metadata { + @Available(Swift, introduced: 6.1) +} From 65cb40a4e42d21e811d40948f77479bc1c90eaab Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 28 Jan 2025 11:56:39 -0500 Subject: [PATCH 063/234] Update Documentation/README.md to cover more files. (#928) Update Documentation/README.md to cover the Proposals and ABI directories. I have also copied my forum post regarding #162 and #840 into a Markdown document, but haven't added it to README.md as it's prospective only and hasn't been accepted as either a vision document nor formal proposal. I have _also_ deleted Releases.md as obsolete now that our releases are tied to the Swift project's releases. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ExpectationCapture.md | 405 ++++++++++++++++++++++++++++ Documentation/README.md | 21 +- Documentation/Releases.md | 131 --------- 3 files changed, 421 insertions(+), 136 deletions(-) create mode 100644 Documentation/ExpectationCapture.md delete mode 100644 Documentation/Releases.md diff --git a/Documentation/ExpectationCapture.md b/Documentation/ExpectationCapture.md new file mode 100644 index 000000000..7bbce174f --- /dev/null +++ b/Documentation/ExpectationCapture.md @@ -0,0 +1,405 @@ +# Rethinking expectation capture in Swift Testing + + + + + +Since we announced Swift Testing and shipped it to developers with the Swift 6 +toolchain, we've been looking at how we could improve the library. Some changes +we hope to introduce, like exit tests and attachments, are major new features. +Others are smaller quality-of-life changes (bug fixes, performance improvements, +and so on.) + +### ℹ️ Be advised +This post assumes some knowledge and understanding of Swift macros and how they +work. In particular, you should know that when the Swift compiler encounters an +_expression_ of the form `#m(a, b, c)`, it passes that expression's +_abstract syntax tree_ (AST) to a compiler plugin that then replaces it with an +equivalent AST known as its _expansion_. + +## Our expectations for expectations + +One area we knew we'd want to revisit was the `#expect()` and `#require()` +macros and how they work. `#expect()` and `#require()` are more than just +functions: they're _expression macros_. We wrote them as expression macros so +that we could give them some special powers. + +When you use one macro or the other, it examines the AST of its condition +argument and looks for one of several known kinds of expression (e.g. binary +operators, member function calls, or `is`/`as?` casts). If found, the macro +rewrites the expression in a form that Swift Testing can examine at runtime. +Then, if the expectation fails, Swift Testing can produce better diagnostics +than it could if it just treated the condition expression as a boolean value. + +For example, if you write this expectation: + +```swift +#expect(f() < g()) +``` + +… Swift Testing is able to tell you the results of `f()` and `g()` in addition +to the overall expression `f() < g()`. + +### Effectively ineffective macros + +This works fairly well for simple expressions like that one, but it doesn't have +the flexibility needed to tell a test author what goes wrong when a more complex +expression is used. For example, this expression: + +```swift +#expect(x && y && !z) +``` + +… is subject to a transformation called "operator folding" that takes place +prior to macro expansion, and the AST represents it as a binary operator whose +left-hand operand is _another_ binary operator. Swift Testing doesn't know how +to recursively expand the nested binary operator, so it only produces values for +`x && y` and `!z`. + +We also run into trouble when effects (`try` and `await`) are in play. Swift +Testing's implementation can't readily handle arbitrary combinations of +subexpressions that may or may not need either keyword. As a result, it doesn't +try to expand expressions that contain effects. If `f()` or `g()` is a throwing +function: + +```swift +#expect(try f() < g()) +``` + +… Swift Testing will not attempt to do any further processing, and only the +outermost expression (`try f() < g()`) will be captured. + +## Looking forward + +Now that Swift 6.1 has branched for its upcoming release, we can start to look +at the _next_ Swift version and how we can improve Swift Testing to help make it +the _Awesomest Swift Release Ever_. And we'd like to start by revisiting how +we've implemented these macros. + +We've been working on [a branch](https://github.com/swiftlang/swift-testing/tree/jgrynspan/162-redesign-value-capture) +of Swift Testing (with a corresponding [draft PR](https://github.com/swiftlang/swift-testing/pull/840)) +that completely redesigns the implementation of the `#expect()` and `#require()` +macros. Instead of trying to sniff out an "interesting" expression to expand, +the code on this branch walks the AST of the condition expression and rewrites +_all_ interesting subexpressions. + +This expectation: + +```swift +#expect(x && y && !z) +``` + +Can now be fully expanded and will provide a full breakdown of the condition at +runtime if it fails: + +``` +◇ Test example() started. +✘ Test example() recorded an issue at Example.swift:1:2: Expectation failed: x && y && !z → false +↳ x && y && !z → false +↳ x && y → true +↳ x → true +↳ y → true +↳ !z → false +↳ z → true +✘ Test example() failed after 0.002 seconds with 1 issue. +``` + +## How you can help + +Before we merge this PR and enable these changes in prerelease Swift toolchains, +we’d love it if you could try it out! These changes are a major change for Swift +Testing and the more feedback we can get, the better. To try out the changes: + +1. _Temporarily_ add an explicit package dependency on Swift Testing and point + Swift Package Manager or Xcode to the branch. In your Package.swift file, add + this dependency: + + ```swift + dependencies: [ + /* ... */ + .package( + url: "https://github.com/swiftlang/swift-testing.git", + branch: "jgrynspan/162-redesign-value-capture" + ), + ], + ``` + + And update your test target: + + ```swift + .testTarget( + name: "MyTests", + dependencies: [ + /* ... */ + .productItem(name: "Testing", package: "swift-testing"), + ] + ) + ``` + + If you’re using an Xcode project, you can add a package dependency via the + **Package Dependencies** tab in your project’s configuration. Add the + `Testing` target as a dependency of your test target and click + **Trust & Enable** to use the locally-built `TestingMacros` target. + +1. Once you’ve added the package dependency, clean your package + (`swift package clean`) or project (**Product** → **Clean Build Folder…**), + then build and run your tests. + + Swift Testing will be built from source along with swift-syntax, which may + significantly increase your build times, so we don’t recommend doing this in + production—this is an at-desk experiment. Swift Testing will be built with + optimizations off by default, so runtime performance may be impacted, but + that’s okay: we’re mostly concerned about correctness rather than raw + performance measurements for the moment. + +1. Let us know how your experience goes, especially if you run into problems. + You can reach me by sending me [a forum DM](https://forums.swift.org/u/grynspan/summary) + or by commenting on [this PR](https://github.com/swiftlang/swift-testing/pull/840). + +## Here be caveats + +There are some expressions that Swift Testing could previously successfully +expand that will cause problems with this new implementation: + +- Expectations with effects where the effect keyword is to the left of the macro + name: + + ```swift + try #expect(h()) + ``` + + Macros cannot currently "see" effect keywords in this position. The old + implementation would often compile because the expansion didn't introduce a + nested closure scope (which necessarily must repeat these keywords in other + positions.) The new implementation does not know it needs to insert the `try` + keyword anywhere in this case, resulting in some confusing diagnostics: + + > ⚠️ No calls to throwing functions occur within 'try' expression + > + > 🛑 Call can throw, but it is not marked with 'try' and the error is not + > handled + + [Stuart Montgomery](https://github.com/stmontgomery) and I chatted with + [Doug Gregor](https://github.com/DougGregor) and [Holly Borla](https://github.com/hborla) + recently about this issue; they're looking at the problem and seeing what sort + of compiler-side support might be possible to help solve it. + + [Stuart Montgomery](https://github.com/stmontgomery) has opened [a PR](https://github.com/swiftlang/swift-syntax/pull/2724) + against swift-syntax that we hope will help resolve this issue. + + **To avoid this issue**, always place `try` and `await` _within_ the argument + list of `#expect()`: + + ```swift + #expect(try h()) + ``` + + For `#require()`, the implementation knows that `try` must be present to the + left of the macro. + +- Expectations with particularly complex conditions can, after expansion, + overwhelm the type checker and fail to compile: + + > 🛑 The compiler is unable to type-check this expression in reasonable time; + > try breaking up the expression into distinct sub-expressions + + Because macros have little-to-no type information, there aren't a lot of + opportunities for us to provide any in the macro expansion. I've reached out + to [Holly Borla](https://github.com/hborla) and her colleagues to see if + there's room for us to improve our implementation in ways that help the type + checker. + + **If your expectation fails with this error,** break up the expression as + recommended and only include part of the expression in the macro's argument + list: + + ```swift + let x = ... + let y = ... + #expect(x == y) + ``` + +- Type names are (syntactically speaking) indistinguishable from variable names. +That means there may be some expressions that we _could_ expand further, but +because we can't tell if a syntax node refers to a variable, a type, or a +module, we don't try: + + ```swift + #expect(a.b == c) // a may not be expressible in isolation + ``` + + Where we think we can expand such a syntax node, the macro expansion appends + `.self` to the node in case it refers to a type. There may be cases where the + macro expansion logic does not work as intended: please send us bug reports if + you find them! + +- The `==`, `!=`, `===`, and `!==` operators are special-cased so that we can + use [`difference(from:)`](https://developer.apple.com/documentation/swift/bidirectionalcollection/difference(from:)) + to compare operands where possible. The macro implementation assumes that + these operators eagerly evaluate their arguments (unlike operators like `&&` + that short-circuit their right-hand arguments using `@autoclosure`.) This + assumption is true of all implementations of these operators in the Swift + Standard Library, but we can't make any real guarantees about third-party code. + + We believe that this should not be a common issue in real-world code, but + please reach out if Swift Testing is expanding these operators incorrectly in + your code. + +### Disabling expression expansion + +If you have a condition expression that you don't want expanded at all (for +instance, because the macro is incorrectly expanding it, or because its +implementation might be affected by side effects from Swift Testing), you can +cast the expression with `as Bool` or `as T?`: + +```swift +let x = ... +let y = ... +#expect((x == y) as Bool) + +let z: String? +let w = try #require(z as String?) +``` + +## Example expansions + +Here (hidden behind disclosure triangles so as not to frighten children and +pets) are some before-and-after examples of how Swift Testing expands the +`#expect()` macro. I've cleaned up the whitespace to make it easier to read. + +
+#expect(f() < g()) + +#### Before +```swift +Testing.__checkBinaryOperation( + f(), + { $0 < $1() }, + g(), + expression: .__fromBinaryOperation( + .__fromSyntaxNode("f()"), + "<", + .__fromSyntaxNode("g()") + ), + comments: [], + isRequired: false, + sourceLocation: Testing.SourceLocation.__here() +).__expected() +``` + +#### After +```swift +Testing.__checkCondition( + { (__ec: inout Testing.__ExpectationContext) -> Swift.Bool in + __ec(__ec(f(), 0x2) < __ec(g(), 0x400), 0x0) + }, + sourceCode: [ + 0x0: "f() < g()", + 0x2: "f()", + 0x400: "g()" + ], + comments: [], + isRequired: false, + sourceLocation: Testing.SourceLocation.__here() +).__expected() +``` +
+ +
+#expect(x && y && !z) + +#### Before +```swift +Testing.__checkBinaryOperation( + x && y, + { $0 && $1() }, + !z, + expression: .__fromBinaryOperation( + .__fromSyntaxNode("x && y"), + "&&", + .__fromSyntaxNode("!z") + ), + comments: [], + isRequired: false, + sourceLocation: Testing.SourceLocation.__here() +).__expected() +``` + +#### After +```swift +Testing.__checkCondition( + { (__ec: inout Testing.__ExpectationContext) -> Swift.Bool in + __ec(__ec(__ec(x, 0x6) && __ec(y, 0x42), 0x2) && __ec(!__ec(z, 0x1400), 0x400), 0x0) + }, + sourceCode: [ + 0x0: "x && y && !z", + 0x2: "x && y", + 0x6: "x", + 0x42: "y", + 0x400: "!z", + 0x1400: "z" + ], + comments: [], + isRequired: false, + sourceLocation: Testing.SourceLocation.__here() +).__expected() +``` +
+ +
+#expect(try f() < g()) + +#### Before +```swift +Testing.__checkValue( + try f() < g(), + expression: .__fromSyntaxNode("try f() < g()"), + comments: [], + isRequired: false, + sourceLocation: Testing.SourceLocation.__here() +).__expected() +``` + +#### After +```swift +try Testing.__checkCondition( + { (__ec: inout Testing.__ExpectationContext) -> Swift.Bool in + try __ec(__ec(f(), 0xc) < __ec(g(), 0x1004), 0x4) + }, + sourceCode: [ + 0x4: "f() < g()", + 0xc: "f()", + 0x1004: "g()" + ], + comments: [], + isRequired: false, + sourceLocation: Testing.SourceLocation.__here() +).__expected() +``` +
+ +> [!NOTE] +> **What's with the hexadecimal?** +> +> You'll note that all calls to `__ec()` include an integer literal argument, +> and the `sourceCode` argument is a dictionary whose keys are integer literals +> too. These values represent the unique identifiers of each captured syntax +> node from the original AST. They uniquely encode the syntax nodes' positions +> in the tree so that we can reconstruct the (sparse) tree at runtime when a +> test fails. +> +> [I](http://github.com/grynspan) find this subtopic interesting enough to want +> to devote a whole forum thread to it, personally, but it's a bit arcane—for +> more information, see the implementation [here](https://github.com/swiftlang/swift-testing/blob/jgrynspan/162-redesign-value-capture/Sources/Testing/SourceAttribution/ExpressionID.swift) +> or feel free to reach out to me via forum DM. diff --git a/Documentation/README.md b/Documentation/README.md index ea8ff2f69..e41bc9568 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -23,12 +23,15 @@ as supplemental content located in the [`Sources/Testing/Testing.docc/`](https://github.com/swiftlang/swift-testing/tree/main/Sources/Testing/Testing.docc) directory. -## Vision document +## Vision document and API proposals The [Vision document](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md) for Swift Testing offers a comprehensive discussion of the project's design principles and goals. +The [`Proposals`](Proposals/) directory contains API proposals that have been +accepted and merged into Swift Testing. + ## Development and contribution - The top-level [`README`](https://github.com/swiftlang/swift-testing/blob/main/README.md) @@ -49,8 +52,16 @@ principles and goals. - Instructions are provided for running tests against a [WASI/WebAssembly target](https://github.com/swiftlang/swift-testing/blob/main/Documentation/WASI.md). -## Project maintenance +## Testing library ABI + +The [`ABI`](ABI/) directory contains documents related to Swift Testing's ABI: +that is, parts of its interface that are intended to be stable over time and can +be used without needing to write any code in Swift: -- The [Releases](https://github.com/swiftlang/swift-testing/blob/main/Documentation/Releases.md) - document describes the process of creating and publishing a new release of - Swift Testing — a task which may be performed by project administrators. +- [`ABI/JSON.md`](ABI/JSON.md) contains Swift Testing's JSON specification that + can be used by tools to interact with Swift Testing either directly or via the + `swift test` command-line tool. +- [`ABI/TestContent.md`](ABI/TestContent.md) documents the section emitted by + the Swift compiler into test products that contains test definitions and other + metadata used by Swift Testing (and extensible by third-party testing + libraries.) diff --git a/Documentation/Releases.md b/Documentation/Releases.md deleted file mode 100644 index 5f1a20cf4..000000000 --- a/Documentation/Releases.md +++ /dev/null @@ -1,131 +0,0 @@ -# How to create a release of Swift Testing - - - -This document describes how to create a new release of Swift Testing using Git -tags. - -> [!IMPORTANT] -> You must have administrator privileges to create a new release in this -> repository. - -## Version numbering - -Swift Testing uses [semantic versioning](https://semver.org) numbers for its -open source releases. We use Git _tags_ to publish new releases; we don't use -the GitHub [releases](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases) -feature. - -At this time, all Swift Testing releases are experimental, so the major version -is always `0`. We aren't using the patch component, so it's usually (if not -always) `0`. The minor component should be incremented by one for each release. - -For example, if the current release is version `0.1.0` and you are publishing -the next release, it should be `0.2.0`. - -> [!NOTE] -> Where you see `x.y.z` in this document, substitute the semantic version you -> are deploying. - -## Creating a branch for the release - -Before a release can be published, a branch must be created so that the -repository can be configured correctly. Ensure any local changes have been saved -and cleared from the repository (e.g. with `git stash` or `git reset --hard`), -then run the following commands from within the repository's root directory: - -```sh -git checkout main # or other branch as appropriate -git pull -git checkout -b release/x.y.z -``` - -## Preparing the repository's contents - -The package manifest files (Package.swift _and_ Package@swift-6.0.swift) must -be updated so that the release can be used as a package dependency: - -1. Delete any unsafe flags from `var packageSettings` as well as elsewhere in - the package manifest files. - -The repository's local state is now updated. To commit it to your branch, run -the typical commit command: - -```sh -git commit -a -m "Deploy x.y.z" -``` - -## Smoke-testing the branch - -Before deploying the tag publicly, test it by creating a simple package locally. -For example, you can initialize a new package in an empty directory with: - -```sh -swift package init --enable-experimental-swift-testing -``` - -Then modify the package's `Package.swift` file to point at your local clone of -the Swift Testing repository. Ensure that the package's test target builds and -runs successfully with: - -```sh -swift test -``` - -> [!NOTE] -> Be sure to test changes on both macOS and Linux using the most recent -> main-branch Swift toolchain. - -If changes to Swift Testing are necessary for the build to succeed, open -appropriate pull requests on GitHub, then rebase your tag branch after they're -merged. - -## Committing changes and pushing the release - -Run the following commands to push the release and make it publicly visible: - -```sh -git tag x.y.z -git push -u origin x.y.z -``` - -The release is now live and publicly visible [here](https://github.com/swiftlang/swift-testing/tags). -Developers using Swift Package Manager and listing Swift Testing as a dependency -will automatically update to it. - -## Oh no, I made a mistake… - -Don't panic. We all make mistakes. - -### … but I haven't pushed the release yet. - -If you've already created the release's tag locally, but haven't pushed it yet, -delete it with `git tag -d x.y.z`, resolve the issue, and recreate the tag by -following the steps above. - -### … but I can fix it. - -If the release is usable, but contains a bug that _cannot_ wait until the next -planned release to be fixed, a patch release can be deployed. First, fix the -issue locally. Then, follow the steps above to create a new release. Where you -would normally increment the _minor_ version component, increment the _patch_ -version component instead. For example, if the most recent release was `0.1.2`, -the fix should be released as `0.1.3`. - -### … and the release is completely unusable! - -If the release is broken and will not be usable by developers, delete the -release's tag from GitHub using `git push --delete origin x.y.z` so that -developers do not inadvertently download it. - -> [!IMPORTANT] -> Deleting a release or tag is often considered bad form, so only do so if the -> release is truly unusable. From 0da75fa9e47d4f5fcd163797680d1485710cb1f8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 28 Jan 2025 11:57:31 -0500 Subject: [PATCH 064/234] Missing an explicit include of `` on Apple platforms. (#932) On some Apple platforms (e.g. iOS), the set of includes we have in `_TestingInternals` doesn't transitively include dyld.h, causing a build failure when building for those platforms. So explicitly include it. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/_TestingInternals/include/Includes.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index b1f4c7973..9fde898f2 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -127,6 +127,10 @@ #if !SWT_NO_LIBDISPATCH #include #endif + +#if !SWT_NO_DYNAMIC_LINKING +#include +#endif #endif #if defined(__FreeBSD__) From 88cb1a402d1eb95aaada94d8bf26686ba3cda7c0 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 28 Jan 2025 13:13:09 -0500 Subject: [PATCH 065/234] Ensure the `ObjectiveC` module is visible to Discovery.swift (#933) With recent work to rewrite our C++ code in Swift, we've wound up not including any Objective-C headers in `_TestingInternals`, but they are included transitively on some Apple platforms including macOS. Ensure the `ObjectiveC` module is included in Discovery.swift when available so that platforms that don't transitively include the libobjc headers can see (in particular) the `objc_addLoadImageFunc()` function. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery+Platform.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index a42800577..3f66b62ad 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -9,6 +9,9 @@ // internal import _TestingInternals +#if _runtime(_ObjC) +private import ObjectiveC +#endif /// A structure describing the bounds of a Swift metadata section. struct SectionBounds: Sendable { From ff27a808980e6af4219e80032368b15b036cb318 Mon Sep 17 00:00:00 2001 From: michael-yuji Date: Tue, 28 Jan 2025 10:18:52 -0800 Subject: [PATCH 066/234] Fix building on FreeBSD via cmake (#925) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently building swift-testing via cmake (e.g. when building the toolchain) fail if not compile as PIC. Adding "-fPIC" fixed this problem. Moreover, we need to link against `libexecinfo` on FreeBSD where the symbol `backtrace` lives. ### Result: Swift testing can now successfully build via cmake thus installed to the toolchain 🥳 --- Sources/CMakeLists.txt | 2 +- Sources/Testing/CMakeLists.txt | 3 +++ Sources/_TestingInternals/CMakeLists.txt | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 56c28cba1..1f3cf3680 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -66,7 +66,7 @@ if(SwiftTesting_MACRO STREQUAL "") if(NOT SwiftTesting_BuildMacrosAsExecutables) if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(SwiftTesting_MACRO_PATH "${SwiftTesting_MACRO_INSTALL_PREFIX}/lib/swift/host/plugins/testing/libTestingMacros.dylib") - elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux" or CMAKE_HOST_SYSTEM_NAME STREQUAL "FreeBSD") set(SwiftTesting_MACRO_PATH "${SwiftTesting_MACRO_INSTALL_PREFIX}/lib/swift/host/plugins/libTestingMacros.so") elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") set(SwiftTesting_MACRO_PATH "${SwiftTesting_MACRO_INSTALL_PREFIX}/bin/TestingMacros.dll") diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 16a173b4b..5971a57b4 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -113,6 +113,9 @@ if(NOT APPLE) endif() target_link_libraries(Testing PUBLIC Foundation) + if (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + target_link_libraries(Testing PUBLIC execinfo) + endif() endif() if(NOT BUILD_SHARED_LIBS) # When building a static library, tell clients to autolink the internal diff --git a/Sources/_TestingInternals/CMakeLists.txt b/Sources/_TestingInternals/CMakeLists.txt index ed707cd78..972a0c56b 100644 --- a/Sources/_TestingInternals/CMakeLists.txt +++ b/Sources/_TestingInternals/CMakeLists.txt @@ -20,6 +20,9 @@ if("${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "MSVC" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") target_compile_options(_TestingInternals PRIVATE /EHa-c) +elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + target_compile_options(_TestingInternals PRIVATE + -fno-exceptions -fPIC) else() target_compile_options(_TestingInternals PRIVATE -fno-exceptions) From 00bb5ac6e2964ef066210e04cca7c22b0974692c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 28 Jan 2025 17:46:40 -0500 Subject: [PATCH 067/234] Implement `CommandLine.executablePath` on all Apple platforms. (#934) This change just makes sure `CommandLine.executablePath` is implemented consistently across Apple platforms and doesn't hit the unimplemented/warning path on e.g. iOS. (We're not using this property there, but the source file it lives in gets compiled regardless.) ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Support/Additions/CommandLineAdditions.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Support/Additions/CommandLineAdditions.swift b/Sources/Testing/Support/Additions/CommandLineAdditions.swift index 0fda59839..57d9851a8 100644 --- a/Sources/Testing/Support/Additions/CommandLineAdditions.swift +++ b/Sources/Testing/Support/Additions/CommandLineAdditions.swift @@ -14,7 +14,7 @@ extension CommandLine { /// The path to the current process' executable. static var executablePath: String { get throws { -#if os(macOS) +#if SWT_TARGET_OS_APPLE var result: String? #if DEBUG var bufferCount = UInt32(1) // force looping From 6df23a443543acc5f1c3f81a0f07a10368bf7d46 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 30 Jan 2025 14:37:33 -0600 Subject: [PATCH 068/234] [Code style] Require explicitly specified types for public properties (#926) This amends the code style guidelines for the project to require an explicit type for all properties whose access level is `public` or greater, and adjusts the code accordingly. ### Motivation: See the explanation in the code style document for rationale. This topic recently [came up](https://github.com/swiftlang/swift-testing/pull/915#discussion_r1919360835) in a PR discussion. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/StyleGuide.md | 8 ++++++++ .../Events/Recorder/Event.ConsoleOutputRecorder.swift | 4 ++-- Sources/Testing/ExitTests/ExitTestArtifacts.swift | 4 ++-- Sources/Testing/Issues/Issue.swift | 4 ++-- Sources/Testing/Running/Configuration.swift | 10 +++++----- Sources/Testing/Test.swift | 2 +- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Documentation/StyleGuide.md b/Documentation/StyleGuide.md index f20c4d7e1..006cf9c51 100644 --- a/Documentation/StyleGuide.md +++ b/Documentation/StyleGuide.md @@ -63,6 +63,14 @@ public var errorCount: Int { } ``` +Properties, variables, and constants in Swift whose access level is `public` or +greater, or which have the `@usableFromInline` attribute, must have an +explicitly specified type even if they have an initialization expression and the +compiler could infer their type. This is meant to protect against future changes +to the code called by the initialization expression causing the inferred type of +its property to change unknowingly, which could break clients. Properties with +lower access levels may have an inferred type. + Exported C and C++ symbols that are exported should be given the prefix `swt_` and should otherwise be named using the same lowerCamelCase naming rules as in Swift. Use the `SWT_EXTERN` macro to ensure that symbols are consistently diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index b1e90c535..cce3a732c 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -32,7 +32,7 @@ extension Event { /// On Windows, `GetFileType()` returns `FILE_TYPE_CHAR` for console file /// handles, and the [Console API](https://learn.microsoft.com/en-us/windows/console/) /// can be used to perform more complex console operations. - public var useANSIEscapeCodes = false + public var useANSIEscapeCodes: Bool = false /// The supported color bit depth when adding color to the output using /// [ANSI escape codes](https://en.wikipedia.org/wiki/ANSI_escape_code). @@ -58,7 +58,7 @@ extension Event { /// If the SF Symbols app is not installed on the system where the /// output is being rendered, the effect of setting the value of this /// property to `true` is unspecified. - public var useSFSymbols = false + public var useSFSymbols: Bool = false #endif /// Storage for ``tagColors``. diff --git a/Sources/Testing/ExitTests/ExitTestArtifacts.swift b/Sources/Testing/ExitTests/ExitTestArtifacts.swift index 6696c791b..6e1710e2a 100644 --- a/Sources/Testing/ExitTests/ExitTestArtifacts.swift +++ b/Sources/Testing/ExitTests/ExitTestArtifacts.swift @@ -55,7 +55,7 @@ public struct ExitTestArtifacts: Sendable { /// /// If you did not request standard output content when running an exit test, /// the value of this property is the empty array. - public var standardOutputContent = [UInt8]() + public var standardOutputContent: [UInt8] = [] /// All bytes written to the standard error stream of the exit test before /// it exited. @@ -81,7 +81,7 @@ public struct ExitTestArtifacts: Sendable { /// /// If you did not request standard error content when running an exit test, /// the value of this property is the empty array. - public var standardErrorContent = [UInt8]() + public var standardErrorContent: [UInt8] = [] @_spi(ForToolsIntegrationOnly) public init(exitCondition: ExitCondition) { diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 731a240d1..dae58400a 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -91,7 +91,7 @@ public struct Issue: Sendable { /// Whether or not this issue is known to occur. @_spi(ForToolsIntegrationOnly) - public var isKnown = false + public var isKnown: Bool = false /// Initialize an issue instance with the specified details. /// @@ -254,7 +254,7 @@ extension Issue { public var sourceContext: SourceContext /// Whether or not this issue is known to occur. - public var isKnown = false + public var isKnown: Bool = false /// Initialize an issue instance with the specified details. /// diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index bdb9cbbef..f4ae59813 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -18,7 +18,7 @@ public struct Configuration: Sendable { // MARK: - Parallelization /// Whether or not to parallelize the execution of tests and test cases. - public var isParallelizationEnabled = true + public var isParallelizationEnabled: Bool = true /// How to symbolicate backtraces captured during a test run. /// @@ -185,7 +185,7 @@ public struct Configuration: Sendable { /// By default, events of this kind are not delivered to event handlers /// because they occur frequently in a typical test run and can generate /// significant backpressure on the event handler. - public var deliverExpectationCheckedEvents = false + public var deliverExpectationCheckedEvents: Bool = false /// The event handler to which events should be passed when they occur. public var eventHandler: Event.Handler = { _, _ in } @@ -237,7 +237,7 @@ public struct Configuration: Sendable { /// is provided. When the value of this property is less than `0`, some /// output is suppressed. The exact effects of this property are determined by /// the instance's event handler. - public var verbosity = 0 + public var verbosity: Int = 0 // MARK: - Test selection @@ -286,7 +286,7 @@ public struct Configuration: Sendable { /// this maximum count. After this maximum is reached, all subsequent /// elements are omitted and a single placeholder child is added indicating /// the number of elements which have been truncated. - public var maximumCollectionCount = 10 + public var maximumCollectionCount: Int = 10 /// The maximum depth of children that can be included in the reflection of /// a checked expectation value. @@ -303,7 +303,7 @@ public struct Configuration: Sendable { /// Since optionals are common, the default value of this property is /// somewhat larger than it otherwise would be in an attempt to make the /// defaults useful for real-world tests. - public var maximumChildDepth = 10 + public var maximumChildDepth: Int = 10 } } diff --git a/Sources/Testing/Test.swift b/Sources/Testing/Test.swift index 0e4292078..738daf72d 100644 --- a/Sources/Testing/Test.swift +++ b/Sources/Testing/Test.swift @@ -199,7 +199,7 @@ public struct Test: Sendable { /// being added to the plan. For such suites, the value of this property is /// `true`. @_spi(ForToolsIntegrationOnly) - public var isSynthesized = false + public var isSynthesized: Bool = false /// Initialize an instance of this type representing a test suite type. init( From 10ab127c7aaa0e85842bd343334efef9ac743249 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Feb 2025 13:49:42 -0500 Subject: [PATCH 069/234] Avoid using `String` while holding an internal libobjc lock. (#938) `String` can inadvertently touch libobjc when we implicitly cast it to a C string (when calling `getsectiondata()`), which can result in a deadlock if that operation needs to acquire one of libobjc's internal locks. Switch to `StaticString` which we can guarantee never touches libobjc. Resolves rdar://144093524. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery+Platform.swift | 28 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index 3f66b62ad..2006a98a3 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -47,6 +47,25 @@ struct SectionBounds: Sendable { #if SWT_TARGET_OS_APPLE // MARK: - Apple implementation +extension SectionBounds.Kind { + /// The Mach-O segment and section name for this instance as a pair of + /// null-terminated UTF-8 C strings and pass them to a function. + /// + /// The values of this property within this function are instances of + /// `StaticString` rather than `String` because the latter's inner storage is + /// sometimes Objective-C-backed and touching it here can cause a recursive + /// access to an internal libobjc lock, whereas `StaticString`'s internal + /// storage is immediately available. + fileprivate var segmentAndSectionName: (segmentName: StaticString, sectionName: StaticString) { + switch self { + case .testContent: + ("__DATA_CONST", "__swift5_tests") + case .typeMetadata: + ("__TEXT", "__swift5_types") + } + } +} + /// An array containing all of the test content section bounds known to the /// testing library. private let _sectionBounds = Locked<[SectionBounds.Kind: [SectionBounds]]>() @@ -77,9 +96,10 @@ private let _startCollectingSectionBounds: Void = { // If this image contains the Swift section(s) we need, acquire the lock and // store the section's bounds. - func findSectionBounds(forSectionNamed segmentName: String, _ sectionName: String, ofKind kind: SectionBounds.Kind) { + for kind in SectionBounds.Kind.allCases { + let (segmentName, sectionName) = kind.segmentAndSectionName var size = CUnsignedLong(0) - if let start = getsectiondata(mh, segmentName, sectionName, &size), size > 0 { + if let start = getsectiondata(mh, segmentName.utf8Start, sectionName.utf8Start, &size), size > 0 { let buffer = UnsafeRawBufferPointer(start: start, count: Int(clamping: size)) let sb = SectionBounds(imageAddress: mh, buffer: buffer) _sectionBounds.withLock { sectionBounds in @@ -87,8 +107,6 @@ private let _startCollectingSectionBounds: Void = { } } } - findSectionBounds(forSectionNamed: "__DATA_CONST", "__swift5_tests", ofKind: .testContent) - findSectionBounds(forSectionNamed: "__TEXT", "__swift5_types", ofKind: .typeMetadata) } #if _runtime(_ObjC) @@ -97,7 +115,7 @@ private let _startCollectingSectionBounds: Void = { } #else _dyld_register_func_for_add_image { mh, _ in - addSectionBounds(from: mh) + addSectionBounds(from: mh!) } #endif }() From f4abd7a0ec3e0977d909e295ae546d0d45946a4e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Feb 2025 16:18:00 -0500 Subject: [PATCH 070/234] Update `Comment` to use `CustomTestStringConvertible` during string interpolation. (#936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR ensures that `Comment`, when constructed from a string literal with interpolations, stringifies interpolated values using `String.init(descriptionForTest:)` rather than the default behaviour (i.e. `String.init(description:)`.) For example: ```swift enum E: CustomTestStringConvertible { case a case b case c var testDescription: String { switch self { case .a: "Once I was the King of Spain" case .b: "Now I eat humble pie" case .c: "Now I vacuum the turf at SkyDome™" } } } #expect(1 == 2, "\(E.a) / \(E.b) / \(E.a) / \(E.c)") ``` Before: > 🛑 Expectation failed: 1 == 2 - .a / .b / .a / .c After: > 🛑 Expectation failed: 1 == 2 - Once I was the King of Spain / Now I eat > humble pie / Once I was the King of Spain / Now I vacuum the turf at SkyDome™ ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Traits/Comment.swift | 43 +++++++++++++++++++- Tests/TestingTests/Traits/CommentTests.swift | 25 ++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Traits/Comment.swift b/Sources/Testing/Traits/Comment.swift index 3497cf6e9..0b282e634 100644 --- a/Sources/Testing/Traits/Comment.swift +++ b/Sources/Testing/Traits/Comment.swift @@ -77,9 +77,9 @@ public struct Comment: RawRepresentable, Sendable { } } -// MARK: - ExpressibleByStringLiteral, ExpressibleByStringInterpolation, CustomStringConvertible +// MARK: - ExpressibleByStringLiteral, CustomStringConvertible -extension Comment: ExpressibleByStringLiteral, ExpressibleByStringInterpolation, CustomStringConvertible { +extension Comment: ExpressibleByStringLiteral, CustomStringConvertible { public init(stringLiteral: String) { self.init(rawValue: stringLiteral, kind: .stringLiteral) } @@ -89,6 +89,45 @@ extension Comment: ExpressibleByStringLiteral, ExpressibleByStringInterpolation, } } +// MARK: - ExpressibleByStringInterpolation + +extension Comment: ExpressibleByStringInterpolation { + public init(stringInterpolation: StringInterpolation) { + self.init(rawValue: stringInterpolation.rawValue) + } + + /// The string interpolation handler type for ``Comment``. + @_documentation(visibility: private) + public struct StringInterpolation: StringInterpolationProtocol, Sendable { + /// Storage for the string constructed by this instance. + /// + /// `DefaultStringInterpolation` in the Swift standard library also simply + /// accumulates its result in a string. + @usableFromInline var rawValue: String = "" + + public init(literalCapacity: Int, interpolationCount: Int) {} + + @inlinable public mutating func appendLiteral(_ literal: String) { + rawValue += literal + } + + @inlinable public mutating func appendInterpolation(_ value: (some Any)?) { + rawValue += String(describingForTest: value) + } + + @inlinable public mutating func appendInterpolation(_ value: (some StringProtocol)?) { + // Special-case strings to not include the quotation marks added by + // CustomTestStringConvertible (which in the context of interpolation + // probably violate the Principle of Least Surprise). + if let value { + rawValue += value + } else { + rawValue += String(describingForTest: value) + } + } + } +} + // MARK: - Equatable, Hashable, Comparable extension Comment: Equatable, Hashable {} diff --git a/Tests/TestingTests/Traits/CommentTests.swift b/Tests/TestingTests/Traits/CommentTests.swift index 679d3fcca..2d497e72f 100644 --- a/Tests/TestingTests/Traits/CommentTests.swift +++ b/Tests/TestingTests/Traits/CommentTests.swift @@ -57,6 +57,31 @@ struct CommentTests { func explicitlyNilComment() { #expect(true as Bool, nil as Comment?) } + + @Test("String interpolation") + func stringInterpolation() { + let value1: Int = 123 + let value2: Int? = nil + let value3: Any.Type = Int.self + let comment: Comment = "abc\(value1)def\(value2)ghi\(value3)" + #expect(comment.rawValue == "abc123defnilghiInt") + } + + @Test("String interpolation with a custom type") + func stringInterpolationWithCustomType() { + struct S: CustomStringConvertible, CustomTestStringConvertible { + var description: String { + "wrong!" + } + + var testDescription: String { + "right!" + } + } + + let comment: Comment = "abc\(S())def\(S() as S?)ghi\(S.self)jkl\("string")" + #expect(comment.rawValue == "abcright!defright!ghiSjklstring") + } } // MARK: - Fixtures From 900bf8c93d6d9d9812b16ad447589ccbf73e1b97 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 4 Feb 2025 13:04:26 -0500 Subject: [PATCH 071/234] Add a `type` argument to the ABI accessor function signature. (#939) We access the test content record section directly instead of using some (non-existent) type-safe Swift API. On some platforms (Darwin in particular), it is possible for two copies of the testing library to be loaded at runtime, and for them to have incompatible definitions of types like `Test`. That means one library's `@Test` macro could produce a test content record that is then read by the other library's discovery logic, resulting in a stack smash or worse. We can resolve this issue by adding a `type` argument to the accessor function we define for test content records; the body of the accessor can then compare the value of this argument against the expected Swift type of its result and, if they don't match, bail early. The new argument is defined as a pointer _to_ a Swift type rather than as a Swift type directly because `@convention(c)` functions cannot directly reference Swift types. The value of the new argument can be safely ignored if the type of the test content record's value is and always has been `@frozen`. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 59 +++++++++++++++------ Sources/Testing/Discovery.swift | 58 ++++++++++++++++---- Sources/Testing/Test+Discovery.swift | 9 +++- Tests/TestingTests/MiscellaneousTests.swift | 5 +- 4 files changed, 103 insertions(+), 28 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index 4f1346b95..d6eacf20f 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -41,10 +41,16 @@ Regardless of platform, all test content records created and discoverable by the testing library have the following layout: ```swift +typealias Accessor = @convention(c) ( + _ outValue: UnsafeMutableRawPointer, + _ type: UnsafeRawPointer, + _ hint: UnsafeRawPointer? +) -> CBool + typealias TestContentRecord = ( kind: UInt32, reserved1: UInt32, - accessor: (@convention(c) (_ outValue: UnsafeMutableRawPointer, _ hint: UnsafeRawPointer?) -> CBool)?, + accessor: Accessor?, context: UInt, reserved2: UInt ) @@ -54,10 +60,16 @@ This type has natural size, stride, and alignment. Its fields are native-endian. If needed, this type can be represented in C as a structure: ```c +typedef bool (* SWTAccessor)( + void *outValue, + const void *type, + const void *_Nullable hint +); + struct SWTTestContentRecord { uint32_t kind; uint32_t reserved1; - bool (* _Nullable accessor)(void *outValue, const void *_Null_unspecified hint); + SWTAccessor _Nullable accessor; uintptr_t context; uintptr_t reserved2; }; @@ -105,42 +117,59 @@ If `accessor` is `nil`, the test content record is ignored. The testing library may, in the future, define record kinds that do not provide an accessor function (that is, they represent pure compile-time information only.) -The second argument to this function, `hint`, is an optional input that can be +The second argument to this function, `type`, is a pointer to the type[^mightNotBeSwift] +(not a bitcast Swift type) of the value expected to be written to `outValue`. +This argument helps to prevent memory corruption if two copies of Swift Testing +or a third-party library are inadvertently loaded into the same process. If the +value at `type` does not match the test content record's expected type, the +accessor function must return `false` and must not modify `outValue`. + + + +[^mightNotBeSwift]: Although this document primarily deals with Swift, the test + content record section is generally language-agnostic. The use of languages + other than Swift is beyond the scope of this document. With that in mind, it + is _technically_ feasible for a test content accessor to be written in (for + example) C++, expect the `type` argument to point to a C++ value of type + `std::type_info`, and write a C++ class instance to `outValue`. + +The third argument to this function, `hint`, is an optional input that can be passed to help the accessor function determine if its corresponding test content record matches what the caller is looking for. If the caller passes `nil` as the `hint` argument, the accessor behaves as if it matched (that is, no additional filtering is performed.) -The concrete Swift type of the value written to `outValue` and the value pointed -to by `hint` depend on the kind of record: +The concrete Swift type of the value written to `outValue`, the type pointed to +by `type`, and the value pointed to by `hint` depend on the kind of record: - For test or suite declarations (kind `0x74657374`), the accessor produces an - asynchronous Swift function that returns an instance of `Test`: + asynchronous Swift function[^notAccessorSignature] that returns an instance of + `Testing.Test`: ```swift @Sendable () async -> Test ``` - This signature is not the signature of `accessor`, but of the Swift function - reference it writes to `outValue`. This level of indirection is necessary - because loading a test or suite declaration is an asynchronous operation, but - C functions cannot be `async`. + [^notAccessorSignature]: This signature is not the signature of `accessor`, + but of the Swift function reference it writes to `outValue`. This level of + indirection is necessary because loading a test or suite declaration is an + asynchronous operation, but C functions cannot be `async`. Test content records of this kind do not specify a type for `hint`. Always pass `nil`. - For exit test declarations (kind `0x65786974`), the accessor produces a - structure describing the exit test (of type `__ExitTest`.) + structure describing the exit test (of type `Testing.__ExitTest`.) - Test content records of this kind accept a `hint` of type `__ExitTest.ID`. + Test content records of this kind accept a `hint` of type `Testing.__ExitTest.ID`. They only produce a result if they represent an exit test declared with the same ID (or if `hint` is `nil`.) > [!WARNING] > Calling code should use [`withUnsafeTemporaryAllocation(of:capacity:_:)`](https://developer.apple.com/documentation/swift/withunsafetemporaryallocation(of:capacity:_:)) -> and [`withUnsafePointer(to:_:)`](https://developer.apple.com/documentation/swift/withunsafepointer(to:_:)-35wrn), -> respectively, to ensure the pointers passed to `accessor` are large enough and -> are well-aligned. If they are not large enough to contain values of the +> and/or [`withUnsafePointer(to:_:)`](https://developer.apple.com/documentation/swift/withunsafepointer(to:_:)-35wrn) +> to ensure the pointers passed to `accessor` are large enough and are +> well-aligned. If they are not large enough to contain values of the > appropriate types (per above), or if `hint` points to uninitialized or > incorrectly-typed memory, the result is undefined. diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index 647da522f..d80d33826 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -10,6 +10,26 @@ private import _TestingInternals +/// The type of the accessor function used to access a test content record. +/// +/// - Parameters: +/// - outValue: A pointer to uninitialized memory large enough to contain the +/// corresponding test content record's value. +/// - type: A pointer to the expected type of `outValue`. Use `load(as:)` to +/// get the Swift type, not `unsafeBitCast(_:to:)`. +/// - hint: An optional pointer to a hint value. +/// +/// - Returns: Whether or not `outValue` was initialized. The caller is +/// responsible for deinitializing `outValue` if it was initialized. +/// +/// - Warning: This type is used to implement the `@Test` macro. Do not use it +/// directly. +public typealias __TestContentRecordAccessor = @convention(c) ( + _ outValue: UnsafeMutableRawPointer, + _ type: UnsafeRawPointer, + _ hint: UnsafeRawPointer? +) -> CBool + /// The content of a test content record. /// /// - Parameters: @@ -24,7 +44,7 @@ private import _TestingInternals public typealias __TestContentRecord = ( kind: UInt32, reserved1: UInt32, - accessor: (@convention(c) (_ outValue: UnsafeMutableRawPointer, _ hint: UnsafeRawPointer?) -> CBool)?, + accessor: __TestContentRecordAccessor?, context: UInt, reserved2: UInt ) @@ -54,6 +74,20 @@ protocol TestContent: ~Copyable { /// By default, this type equals `Never`, indicating that this type of test /// content does not support hinting during discovery. associatedtype TestContentAccessorHint: Sendable = Never + + /// The type to pass (by address) as the accessor function's `type` argument. + /// + /// The default value of this property is `Self.self`. A conforming type can + /// override the default implementation to substitute another type (e.g. if + /// the conforming type is not public but records are created during macro + /// expansion and can only reference public types.) + static var testContentAccessorTypeArgument: any ~Copyable.Type { get } +} + +extension TestContent where Self: ~Copyable { + static var testContentAccessorTypeArgument: any ~Copyable.Type { + self + } } // MARK: - Individual test content records @@ -108,18 +142,20 @@ struct TestContentRecord: Sendable where T: TestContent & ~Copyable { return nil } - return withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in - let initialized = if let hint { - withUnsafePointer(to: hint) { hint in - accessor(buffer.baseAddress!, hint) + return withUnsafePointer(to: T.testContentAccessorTypeArgument) { type in + withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in + let initialized = if let hint { + withUnsafePointer(to: hint) { hint in + accessor(buffer.baseAddress!, type, hint) + } + } else { + accessor(buffer.baseAddress!, type, nil) } - } else { - accessor(buffer.baseAddress!, nil) - } - guard initialized else { - return nil + guard initialized else { + return nil + } + return buffer.baseAddress!.move() } - return buffer.baseAddress!.move() } } } diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 015a0b1c8..d515bc98f 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -28,8 +28,15 @@ extension Test { 0x74657374 } + static var testContentAccessorTypeArgument: any ~Copyable.Type { + Generator.self + } + + /// The type of the actual (asynchronous) generator function. + typealias Generator = @Sendable () async -> Test + /// The actual (asynchronous) accessor function. - case generator(@Sendable () async -> Test) + case generator(Generator) } /// All available ``Test`` instances in the process, according to the runtime. diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index a172f7d5a..79db275a7 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -615,7 +615,10 @@ struct MiscellaneousTests { private static let record: __TestContentRecord = ( 0xABCD1234, 0, - { outValue, hint in + { outValue, type, hint in + guard type.load(as: Any.Type.self) == DiscoverableTestContent.self else { + return false + } if let hint, hint.load(as: TestContentAccessorHint.self) != expectedHint { return false } From 57335d72b1ed5dda16db61c35dd4551976bda920 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 5 Feb 2025 16:14:04 -0500 Subject: [PATCH 072/234] Simplify statically-linked section discovery. (#940) This PR simplifies the code used to discover metadata sections when the testing library is statically linked into a test target. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Discovery+Platform.swift | 12 ++++-------- Sources/_TestingInternals/Discovery.cpp | 14 +++++++++----- Sources/_TestingInternals/include/Discovery.h | 17 ++++++----------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/Testing/Discovery+Platform.swift index 2006a98a3..3da974386 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/Testing/Discovery+Platform.swift @@ -23,7 +23,7 @@ struct SectionBounds: Sendable { /// An enumeration describing the different sections discoverable by the /// testing library. - enum Kind: Equatable, Hashable, CaseIterable { + enum Kind: Int, Equatable, Hashable, CaseIterable { /// The test content metadata section. case testContent @@ -285,13 +285,9 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { /// - Returns: A structure describing the bounds of the type metadata section /// contained in the same image as the testing library itself. private func _sectionBounds(_ kind: SectionBounds.Kind) -> CollectionOfOne { - let (sectionBegin, sectionEnd) = switch kind { - case .testContent: - SWTTestContentSectionBounds - case .typeMetadata: - SWTTypeMetadataSectionBounds - } - let buffer = UnsafeRawBufferPointer(start: sectionBegin, count: max(0, sectionEnd - sectionBegin)) + var (baseAddress, count): (UnsafeRawPointer?, Int) = (nil, 0) + swt_getStaticallyLinkedSectionBounds(kind.rawValue, &baseAddress, &count) + let buffer = UnsafeRawBufferPointer(start: baseAddress, count: count) let sb = SectionBounds(imageAddress: nil, buffer: buffer) return CollectionOfOne(sb) } diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index 314b7794d..4afc1d10c 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -10,6 +10,7 @@ #include "Discovery.h" +#include #include #include #include @@ -35,13 +36,16 @@ static const char typeMetadataSectionBegin = 0; static const char& typeMetadataSectionEnd = typeMetadataSectionBegin; #endif -const void *_Nonnull const SWTTestContentSectionBounds[2] = { - &testContentSectionBegin, &testContentSectionEnd +static constexpr const char *const staticallyLinkedSectionBounds[][2] = { + { &testContentSectionBegin, &testContentSectionEnd }, + { &typeMetadataSectionBegin, &typeMetadataSectionEnd }, }; -const void *_Nonnull const SWTTypeMetadataSectionBounds[2] = { - &typeMetadataSectionBegin, &typeMetadataSectionEnd -}; +void swt_getStaticallyLinkedSectionBounds(size_t kind, const void **outSectionBegin, size_t *outByteCount) { + auto [sectionBegin, sectionEnd] = staticallyLinkedSectionBounds[kind]; + *outSectionBegin = sectionBegin; + *outByteCount = std::distance(sectionBegin, sectionEnd); +} #endif #pragma mark - Swift ABI diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index 8fc36d8c7..9bda12b93 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -32,23 +32,18 @@ SWT_IMPORT_FROM_STDLIB void swift_enumerateAllMetadataSections( #pragma mark - Statically-linked section bounds -/// The bounds of the test content section statically linked into the image -/// containing Swift Testing. +/// Get the bounds of a statically linked section in this image. /// -/// - Note: This symbol is _declared_, but not _defined_, on platforms with -/// dynamic linking because the `SWT_NO_DYNAMIC_LINKING` C++ macro (not the -/// Swift compiler conditional of the same name) is not consistently declared -/// when Swift files import the `_TestingInternals` C++ module. -SWT_EXTERN const void *_Nonnull const SWTTestContentSectionBounds[2]; - -/// The bounds of the type metadata section statically linked into the image -/// containing Swift Testing. +/// - Parameters: +/// - kind: The value of `SectionBounds.Kind.rawValue` for the given section. +/// - outSectionBegin: On return, a pointer to the first byte of the section. +/// - outByteCount: On return, the number of bytes in the section. /// /// - Note: This symbol is _declared_, but not _defined_, on platforms with /// dynamic linking because the `SWT_NO_DYNAMIC_LINKING` C++ macro (not the /// Swift compiler conditional of the same name) is not consistently declared /// when Swift files import the `_TestingInternals` C++ module. -SWT_EXTERN const void *_Nonnull const SWTTypeMetadataSectionBounds[2]; +SWT_EXTERN void swt_getStaticallyLinkedSectionBounds(size_t kind, const void *_Nullable *_Nonnull outSectionBegin, size_t *outByteCount); #pragma mark - Legacy test discovery From 63eb1d97e305befeea9cee3482a9ce38507a2cf6 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 6 Feb 2025 09:42:59 -0500 Subject: [PATCH 073/234] Use `os_unfair_lock` on Darwin when available. (#942) This PR refactors the `Locked` type to be generic over the type of lock it uses, allowing us to adopt `os_unfair_lock` in most places, but also allowing us to use other lock types as needed (i.e. `pthread_mutex_t` in WaitFor.swift if libdispatch is unavailable.) Under `-O`, locking and unlocking is fully inlined, while initialization (which is relatively rare) is partially, but not completely, inlined. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/CMakeLists.txt | 1 + Sources/Testing/ExitTests/WaitFor.swift | 4 +- Sources/Testing/Support/Locked+Platform.swift | 96 ++++++++++++ Sources/Testing/Support/Locked.swift | 137 ++++++++---------- Sources/_TestingInternals/include/Includes.h | 4 + Tests/TestingTests/Support/LockTests.swift | 49 ++++++- .../Traits/ParallelizationTraitTests.swift | 2 +- 7 files changed, 210 insertions(+), 83 deletions(-) create mode 100644 Sources/Testing/Support/Locked+Platform.swift diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 5971a57b4..cd9983c07 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -79,6 +79,7 @@ add_library(Testing Support/Graph.swift Support/JSON.swift Support/Locked.swift + Support/Locked+Platform.swift Support/SystemError.swift Support/Versions.swift Discovery.swift diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index fac3e1496..c6841c580 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -80,7 +80,7 @@ func wait(for pid: consuming pid_t) async throws -> ExitCondition { } #elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) /// A mapping of awaited child PIDs to their corresponding Swift continuations. -private let _childProcessContinuations = Locked<[pid_t: CheckedContinuation]>() +private let _childProcessContinuations = LockedWith]>() /// A condition variable used to suspend the waiter thread created by /// `_createWaitThread()` when there are no child processes to await. @@ -137,7 +137,7 @@ private let _createWaitThread: Void = { // newly-scheduled waiter process. (If this condition is spuriously // woken, we'll just loop again, which is fine.) Note that we read errno // outside the lock in case acquiring the lock perturbs it. - _childProcessContinuations.withUnsafePlatformLock { lock, childProcessContinuations in + _childProcessContinuations.withUnsafeUnderlyingLock { lock, childProcessContinuations in if childProcessContinuations.isEmpty { _ = pthread_cond_wait(_waitThreadNoChildrenCondition, lock) } diff --git a/Sources/Testing/Support/Locked+Platform.swift b/Sources/Testing/Support/Locked+Platform.swift new file mode 100644 index 000000000..951e62da8 --- /dev/null +++ b/Sources/Testing/Support/Locked+Platform.swift @@ -0,0 +1,96 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +internal import _TestingInternals + +extension Never: Lockable { + static func initializeLock(at lock: UnsafeMutablePointer) {} + static func deinitializeLock(at lock: UnsafeMutablePointer) {} + static func unsafelyAcquireLock(at lock: UnsafeMutablePointer) {} + static func unsafelyRelinquishLock(at lock: UnsafeMutablePointer) {} +} + +#if SWT_TARGET_OS_APPLE && !SWT_NO_OS_UNFAIR_LOCK +extension os_unfair_lock_s: Lockable { + static func initializeLock(at lock: UnsafeMutablePointer) { + lock.initialize(to: .init()) + } + + static func deinitializeLock(at lock: UnsafeMutablePointer) { + // No deinitialization needed. + } + + static func unsafelyAcquireLock(at lock: UnsafeMutablePointer) { + os_unfair_lock_lock(lock) + } + + static func unsafelyRelinquishLock(at lock: UnsafeMutablePointer) { + os_unfair_lock_unlock(lock) + } +} +#endif + +#if os(FreeBSD) || os(OpenBSD) +typealias pthread_mutex_t = _TestingInternals.pthread_mutex_t? +#endif + +#if SWT_TARGET_OS_APPLE || os(Linux) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) || os(FreeBSD) || os(OpenBSD) +extension pthread_mutex_t: Lockable { + static func initializeLock(at lock: UnsafeMutablePointer) { + _ = pthread_mutex_init(lock, nil) + } + + static func deinitializeLock(at lock: UnsafeMutablePointer) { + _ = pthread_mutex_destroy(lock) + } + + static func unsafelyAcquireLock(at lock: UnsafeMutablePointer) { + _ = pthread_mutex_lock(lock) + } + + static func unsafelyRelinquishLock(at lock: UnsafeMutablePointer) { + _ = pthread_mutex_unlock(lock) + } +} +#endif + +#if os(Windows) +extension SRWLOCK: Lockable { + static func initializeLock(at lock: UnsafeMutablePointer) { + InitializeSRWLock(lock) + } + + static func deinitializeLock(at lock: UnsafeMutablePointer) { + // No deinitialization needed. + } + + static func unsafelyAcquireLock(at lock: UnsafeMutablePointer) { + AcquireSRWLockExclusive(lock) + } + + static func unsafelyRelinquishLock(at lock: UnsafeMutablePointer) { + ReleaseSRWLockExclusive(lock) + } +} +#endif + +#if SWT_TARGET_OS_APPLE && !SWT_NO_OS_UNFAIR_LOCK +typealias DefaultLock = os_unfair_lock +#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) || os(FreeBSD) || os(OpenBSD) +typealias DefaultLock = pthread_mutex_t +#elseif os(Windows) +typealias DefaultLock = SRWLOCK +#elseif os(WASI) +// No locks on WASI without multithreaded runtime. +typealias DefaultLock = Never +#else +#warning("Platform-specific implementation missing: locking unavailable") +typealias DefaultLock = Never +#endif diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index 1294da195..e8b17be7b 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,6 +10,37 @@ internal import _TestingInternals +/// A protocol defining a type, generally platform-specific, that satisfies the +/// requirements of a lock or mutex. +protocol Lockable { + /// Initialize the lock at the given address. + /// + /// - Parameters: + /// - lock: A pointer to uninitialized memory that should be initialized as + /// an instance of this type. + static func initializeLock(at lock: UnsafeMutablePointer) + + /// Deinitialize the lock at the given address. + /// + /// - Parameters: + /// - lock: A pointer to initialized memory that should be deinitialized. + static func deinitializeLock(at lock: UnsafeMutablePointer) + + /// Acquire the lock at the given address. + /// + /// - Parameters: + /// - lock: The address of the lock to acquire. + static func unsafelyAcquireLock(at lock: UnsafeMutablePointer) + + /// Relinquish the lock at the given address. + /// + /// - Parameters: + /// - lock: The address of the lock to relinquish. + static func unsafelyRelinquishLock(at lock: UnsafeMutablePointer) +} + +// MARK: - + /// A type that wraps a value requiring access from a synchronous caller during /// concurrent execution. /// @@ -21,67 +52,23 @@ internal import _TestingInternals /// concurrency tools. /// /// This type is not part of the public interface of the testing library. -/// -/// - Bug: The state protected by this type should instead be protected using -/// actor isolation, but actor-isolated functions cannot be called from -/// synchronous functions. ([83888717](rdar://83888717)) -struct Locked: RawRepresentable, Sendable where T: Sendable { - /// The platform-specific type to use for locking. - /// - /// It would be preferable to implement this lock in Swift, however there is - /// no standard lock or mutex type available across all platforms that is - /// visible in Swift. C11 has a standard `mtx_t` type, but it is not widely - /// supported and so cannot be relied upon. - /// - /// To keep the implementation of this type as simple as possible, - /// `pthread_mutex_t` is used on Apple platforms instead of `os_unfair_lock` - /// or `OSAllocatedUnfairLock`. -#if SWT_TARGET_OS_APPLE || os(Linux) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) - typealias PlatformLock = pthread_mutex_t -#elseif os(FreeBSD) || os(OpenBSD) - typealias PlatformLock = pthread_mutex_t? -#elseif os(Windows) - typealias PlatformLock = SRWLOCK -#elseif os(WASI) - // No locks on WASI without multithreaded runtime. - typealias PlatformLock = Never -#else -#warning("Platform-specific implementation missing: locking unavailable") - typealias PlatformLock = Never -#endif - +struct LockedWith: RawRepresentable where L: Lockable { /// A type providing heap-allocated storage for an instance of ``Locked``. - private final class _Storage: ManagedBuffer { + private final class _Storage: ManagedBuffer { deinit { withUnsafeMutablePointerToElements { lock in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) - _ = pthread_mutex_destroy(lock) -#elseif os(Windows) - // No deinitialization needed. -#elseif os(WASI) - // No locks on WASI without multithreaded runtime. -#else -#warning("Platform-specific implementation missing: locking unavailable") -#endif + L.deinitializeLock(at: lock) } } } /// Storage for the underlying lock and wrapped value. - private nonisolated(unsafe) var _storage: ManagedBuffer + private nonisolated(unsafe) var _storage: ManagedBuffer init(rawValue: T) { _storage = _Storage.create(minimumCapacity: 1, makingHeaderWith: { _ in rawValue }) _storage.withUnsafeMutablePointerToElements { lock in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) - _ = pthread_mutex_init(lock, nil) -#elseif os(Windows) - InitializeSRWLock(lock) -#elseif os(WASI) - // No locks on WASI without multithreaded runtime. -#else -#warning("Platform-specific implementation missing: locking unavailable") -#endif + L.initializeLock(at: lock) } } @@ -103,28 +90,16 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { /// concurrency tools. nonmutating func withLock(_ body: (inout T) throws -> R) rethrows -> R { try _storage.withUnsafeMutablePointers { rawValue, lock in -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || (os(WASI) && compiler(>=6.1) && _runtime(_multithreaded)) - _ = pthread_mutex_lock(lock) + L.unsafelyAcquireLock(at: lock) defer { - _ = pthread_mutex_unlock(lock) + L.unsafelyRelinquishLock(at: lock) } -#elseif os(Windows) - AcquireSRWLockExclusive(lock) - defer { - ReleaseSRWLockExclusive(lock) - } -#elseif os(WASI) - // No locks on WASI without multithreaded runtime. -#else -#warning("Platform-specific implementation missing: locking unavailable") -#endif - return try body(&rawValue.pointee) } } /// Acquire the lock and invoke a function while it is held, yielding both the - /// protected value and a reference to the lock itself. + /// protected value and a reference to the underlying lock guarding it. /// /// - Parameters: /// - body: A closure to invoke while the lock is held. @@ -134,16 +109,16 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { /// - Throws: Whatever is thrown by `body`. /// /// This function is equivalent to ``withLock(_:)`` except that the closure - /// passed to it also takes a reference to the underlying platform lock. This - /// function can be used when platform-specific functionality such as a - /// `pthread_cond_t` is needed. Because the caller has direct access to the - /// lock and is able to unlock and re-lock it, it is unsafe to modify the - /// protected value. + /// passed to it also takes a reference to the underlying lock guarding this + /// instance's wrapped value. This function can be used when platform-specific + /// functionality such as a `pthread_cond_t` is needed. Because the caller has + /// direct access to the lock and is able to unlock and re-lock it, it is + /// unsafe to modify the protected value. /// /// - Warning: Callers that unlock the lock _must_ lock it again before the /// closure returns. If the lock is not acquired when `body` returns, the /// effect is undefined. - nonmutating func withUnsafePlatformLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { + nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { try withLock { value in try _storage.withUnsafeMutablePointerToElements { lock in try body(lock, value) @@ -152,7 +127,16 @@ struct Locked: RawRepresentable, Sendable where T: Sendable { } } -extension Locked where T: AdditiveArithmetic { +extension LockedWith: Sendable where T: Sendable {} + +/// A type that wraps a value requiring access from a synchronous caller during +/// concurrent execution and which uses the default platform-specific lock type +/// for the current platform. +typealias Locked = LockedWith + +// MARK: - Additions + +extension LockedWith where T: AdditiveArithmetic { /// Add something to the current wrapped value of this instance. /// /// - Parameters: @@ -168,7 +152,7 @@ extension Locked where T: AdditiveArithmetic { } } -extension Locked where T: Numeric { +extension LockedWith where T: Numeric { /// Increment the current wrapped value of this instance. /// /// - Returns: The sum of ``rawValue`` and `1`. @@ -188,7 +172,7 @@ extension Locked where T: Numeric { } } -extension Locked { +extension LockedWith { /// Initialize an instance of this type with a raw value of `nil`. init() where T == V? { self.init(rawValue: nil) @@ -198,4 +182,9 @@ extension Locked { init() where T == Dictionary { self.init(rawValue: [:]) } + + /// Initialize an instance of this type with a raw value of `[]`. + init() where T == [V] { + self.init(rawValue: []) + } } diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index 9fde898f2..5ba496ee9 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -131,6 +131,10 @@ #if !SWT_NO_DYNAMIC_LINKING #include #endif + +#if !SWT_NO_OS_UNFAIR_LOCK +#include +#endif #endif #if defined(__FreeBSD__) diff --git a/Tests/TestingTests/Support/LockTests.swift b/Tests/TestingTests/Support/LockTests.swift index f41e66349..0113745e9 100644 --- a/Tests/TestingTests/Support/LockTests.swift +++ b/Tests/TestingTests/Support/LockTests.swift @@ -9,17 +9,54 @@ // @testable import Testing +private import _TestingInternals @Suite("Locked Tests") struct LockTests { - @Test("Mutating a value within withLock(_:)") + func testLock(_ lock: LockedWith) { + #expect(lock.rawValue == 0) + lock.withLock { value in + value = 1 + } + #expect(lock.rawValue == 1) + } + + @Test("Platform-default lock") func locking() { - let value = Locked(rawValue: 0) + testLock(Locked(rawValue: 0)) + } - #expect(value.rawValue == 0) - value.withLock { value in - value = 1 +#if SWT_TARGET_OS_APPLE && !SWT_NO_OS_UNFAIR_LOCK + @Test("pthread_mutex_t (Darwin alternate)") + func lockingWith_pthread_mutex_t() { + testLock(LockedWith(rawValue: 0)) + } +#endif + + @Test("No lock") + func noLock() async { + let lock = LockedWith(rawValue: 0) + await withTaskGroup(of: Void.self) { taskGroup in + for _ in 0 ..< 100_000 { + taskGroup.addTask { + lock.increment() + } + } + } + #expect(lock.rawValue != 100_000) + } + + @Test("Get the underlying lock") + func underlyingLock() { + let lock = Locked(rawValue: 0) + testLock(lock) + lock.withUnsafeUnderlyingLock { underlyingLock, _ in + DefaultLock.unsafelyRelinquishLock(at: underlyingLock) + lock.withLock { value in + value += 1000 + } + DefaultLock.unsafelyAcquireLock(at: underlyingLock) } - #expect(value.rawValue == 1) + #expect(lock.rawValue == 1001) } } diff --git a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift index e43ca50b7..776e5c320 100644 --- a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift +++ b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift @@ -17,7 +17,7 @@ struct ParallelizationTraitTests { var configuration = Configuration() configuration.isParallelizationEnabled = true - let indicesRecorded = Locked<[Int]>(rawValue: []) + let indicesRecorded = Locked<[Int]>() configuration.eventHandler = { event, _ in if case let .issueRecorded(issue) = event.kind, let comment = issue.comments.first, From 6a49142b223f214eff5da6c1224935a80e62e512 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 11 Feb 2025 11:04:47 -0600 Subject: [PATCH 074/234] Introduce a severity level for issues, and a 'warning' severity (#931) This introduces the concept of severity to the `Issue` type, represented by a new enum `Issue.Severity` with two cases: `.warning` and `.error`. Error is the default severity for all issues, matching current behavior, but warning is provided as a new option which does not cause the test the issue is associated with to be marked as a failure. In this PR, these are [SPI](https://github.com/swiftlang/swift-testing/blob/main/Documentation/SPI.md) but they could be considered for promotion to public API eventually. Additional work would be needed to permit test authors to record issues with severity < `.error`, since APIs like `Issue.record()` are not being modified at this time to allow customizing the severity. ### Motivation: There are certain situations where a problem may arise during a test that doesn't necessarily affect its outcome or signal an important problem, but is worth calling attention to. A specific example use case I have in mind is to allow the testing library to record a warning issue about problems with the arguments passed to a parameterized test, such as having duplicate arguments. ### Modifications: - Introduce `Issue.Severity` as an SPI enum. - Introduce an SPI property `severity` to `Issue` with default value `.error`. - Modify entry point logic to exit with `EXIT_SUCCESS` if all issues recorded had severity < `.error`. - Modify console output formatting logic and data structures to represent warning issues sensibly. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. - [x] Add new tests --- .../ABI/EntryPoints/ABIEntryPoint.swift | 6 +- .../Testing/ABI/EntryPoints/EntryPoint.swift | 37 +++++- .../ABI/v0/Encoded/ABIv0.EncodedIssue.swift | 19 +++ .../ABI/v0/Encoded/ABIv0.EncodedMessage.swift | 3 + Sources/Testing/Events/Event.swift | 5 +- .../Event.ConsoleOutputRecorder.swift | 2 + .../Event.HumanReadableOutputRecorder.swift | 61 +++++---- .../Events/Recorder/Event.Symbol.swift | 16 ++- Sources/Testing/Issues/Issue.swift | 113 +++++++++++++---- .../Running/Configuration+EventHandling.swift | 20 +++ Sources/Testing/Running/Configuration.swift | 45 +++++-- .../Testing/Running/Runner.RuntimeState.swift | 13 +- Tests/TestingTests/ConfigurationTests.swift | 25 ++++ Tests/TestingTests/EntryPointTests.swift | 81 ++++++++++++ Tests/TestingTests/EventRecorderTests.swift | 117 +++++++++++++----- Tests/TestingTests/IssueTests.swift | 8 +- .../Runner.RuntimeStateTests.swift | 2 +- Tests/TestingTests/RunnerTests.swift | 4 +- 18 files changed, 468 insertions(+), 109 deletions(-) create mode 100644 Tests/TestingTests/ConfigurationTests.swift create mode 100644 Tests/TestingTests/EntryPointTests.swift diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index cc150740e..143f7b549 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -47,7 +47,7 @@ extension ABIv0 { /// callback. public static var entryPoint: EntryPoint { return { configurationJSON, recordHandler in - try await Testing.entryPoint( + try await _entryPoint( configurationJSON: configurationJSON, recordHandler: recordHandler ) == EXIT_SUCCESS @@ -87,7 +87,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in - try await entryPoint( + try await _entryPoint( configurationJSON: configurationJSON, eventStreamVersionIfNil: -1, recordHandler: recordHandler @@ -104,7 +104,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( /// /// This function will be removed (with its logic incorporated into /// ``ABIv0/entryPoint-swift.type.property``) in a future update. -private func entryPoint( +private func _entryPoint( configurationJSON: UnsafeRawBufferPointer?, eventStreamVersionIfNil: Int? = nil, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index a0a5df2a0..89094c88f 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -20,6 +20,8 @@ private import _TestingInternals /// writes events to the standard error stream in addition to passing them /// to this function. /// +/// - Returns: An exit code representing the result of running tests. +/// /// External callers cannot call this function directly. The can use /// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable /// version of this function. @@ -40,7 +42,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // Set up the event handler. configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown { + if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error { exitCode.withLock { exitCode in exitCode = EXIT_FAILURE } @@ -270,6 +272,13 @@ public struct __CommandLineArguments_v0: Sendable { /// The value(s) of the `--skip` argument. public var skip: [String]? + /// Whether or not to include tests with the `.hidden` trait when constructing + /// a test filter based on these arguments. + /// + /// This property is intended for use in testing the testing library itself. + /// It is not parsed as a command-line argument. + var includeHiddenTests: Bool? + /// The value of the `--repetitions` argument. public var repetitions: Int? @@ -278,6 +287,13 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--experimental-attachments-path` argument. public var experimentalAttachmentsPath: String? + + /// Whether or not the experimental warning issue severity feature should be + /// enabled. + /// + /// This property is intended for use in testing the testing library itself. + /// It is not parsed as a command-line argument. + var isWarningIssueRecordedEventEnabled: Bool? } extension __CommandLineArguments_v0: Codable { @@ -517,6 +533,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding)) configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) } + if args.includeHiddenTests == true { + configuration.testFilter.includeHiddenTests = true + } // Set up the iteration policy for the test run. var repetitionPolicy: Configuration.RepetitionPolicy = .once @@ -547,6 +566,22 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr configuration.exitTestHandler = ExitTest.handlerForEntryPoint() #endif + // Warning issues (experimental). + if args.isWarningIssueRecordedEventEnabled == true { + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true + } else { + switch args.eventStreamVersion { + case .some(...0): + // If the event stream version was explicitly specified to a value < 1, + // disable the warning issue event to maintain legacy behavior. + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false + default: + // Otherwise the requested event stream version is ≥ 1, so don't change + // the warning issue event setting. + break + } + } + return configuration } diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift index 2bf1c8462..97c051d28 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift @@ -16,6 +16,19 @@ extension ABIv0 { /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. struct EncodedIssue: Sendable { + /// An enumeration representing the level of severity of a recorded issue. + /// + /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``. + enum Severity: String, Sendable { + case warning + case error + } + + /// The severity of this issue. + /// + /// - Warning: Severity is not yet part of the JSON schema. + var _severity: Severity + /// Whether or not this issue is known to occur. var isKnown: Bool @@ -33,6 +46,11 @@ extension ABIv0 { var _error: EncodedError? init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { + _severity = switch issue.severity { + case .warning: .warning + case .error: .error + } + isKnown = issue.isKnown sourceLocation = issue.sourceLocation if let backtrace = issue.sourceContext.backtrace { @@ -48,3 +66,4 @@ extension ABIv0 { // MARK: - Codable extension ABIv0.EncodedIssue: Codable {} +extension ABIv0.EncodedIssue.Severity: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift index 5cfbf647c..cf44f0af0 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift @@ -25,6 +25,7 @@ extension ABIv0 { case `default` case skip case pass + case passWithWarnings = "_passWithWarnings" case passWithKnownIssue case fail case difference @@ -44,6 +45,8 @@ extension ABIv0 { } else { .pass } + case .passWithWarnings: + .passWithWarnings case .fail: .fail case .difference: diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 60e564d5a..b81f1c2c7 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -290,10 +290,7 @@ extension Event { if let configuration = configuration ?? Configuration.current { // The caller specified a configuration, or the current task has an // associated configuration. Post to either configuration's event handler. - switch kind { - case .expectationChecked where !configuration.deliverExpectationCheckedEvents: - break - default: + if configuration.eventHandlingOptions.shouldHandleEvent(self) { configuration.handleEvent(self, in: context) } } else { diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index cce3a732c..b375b2da1 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -162,6 +162,8 @@ extension Event.Symbol { return "\(_ansiEscapeCodePrefix)90m\(symbolCharacter)\(_resetANSIEscapeCode)" } return "\(_ansiEscapeCodePrefix)92m\(symbolCharacter)\(_resetANSIEscapeCode)" + case .passWithWarnings: + return "\(_ansiEscapeCodePrefix)93m\(symbolCharacter)\(_resetANSIEscapeCode)" case .fail: return "\(_ansiEscapeCodePrefix)91m\(symbolCharacter)\(_resetANSIEscapeCode)" case .warning: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 98303f11c..2e40b2789 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -56,8 +56,9 @@ extension Event { /// The instant at which the test started. var startInstant: Test.Clock.Instant - /// The number of issues recorded for the test. - var issueCount = 0 + /// The number of issues recorded for the test, grouped by their + /// level of severity. + var issueCount: [Issue.Severity: Int] = [:] /// The number of known issues recorded for the test. var knownIssueCount = 0 @@ -114,27 +115,36 @@ extension Event.HumanReadableOutputRecorder { /// - graph: The graph to walk while counting issues. /// /// - Returns: A tuple containing the number of issues recorded in `graph`. - private func _issueCounts(in graph: Graph?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { + private func _issueCounts(in graph: Graph?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { guard let graph else { - return (0, 0, 0, "") + return (0, 0, 0, 0, "") } - let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=) + let errorIssueCount = graph.compactMap(\.value?.issueCount[.error]).reduce(into: 0, +=) + let warningIssueCount = graph.compactMap(\.value?.issueCount[.warning]).reduce(into: 0, +=) let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=) - let totalIssueCount = issueCount + knownIssueCount + let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount // Construct a string describing the issue counts. - let description = switch (issueCount > 0, knownIssueCount > 0) { - case (true, true): + let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) { + case (true, true, true): + " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))" + case (true, false, true): " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))" - case (false, true): + case (false, true, true): + " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))" + case (false, false, true): " with \(knownIssueCount.counting("known issue"))" - case (true, false): + case (true, true, false): + " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))" + case (true, false, false): " with \(totalIssueCount.counting("issue"))" - case(false, false): + case(false, true, false): + " with \(warningIssueCount.counting("warning"))" + case(false, false, false): "" } - return (issueCount, knownIssueCount, totalIssueCount, description) + return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } } @@ -267,7 +277,8 @@ extension Event.HumanReadableOutputRecorder { if issue.isKnown { testData.knownIssueCount += 1 } else { - testData.issueCount += 1 + let issueCount = testData.issueCount[issue.severity] ?? 0 + testData.issueCount[issue.severity] = issueCount + 1 } context.testData[id] = testData @@ -355,7 +366,7 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - return if issues.issueCount > 0 { + return if issues.errorIssueCount > 0 { CollectionOfOne( Message( symbol: .fail, @@ -363,7 +374,7 @@ extension Event.HumanReadableOutputRecorder { ) ) + _formattedComments(for: test) } else { - [ + [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)." @@ -400,13 +411,19 @@ extension Event.HumanReadableOutputRecorder { "" } let symbol: Event.Symbol - let known: String + let subject: String if issue.isKnown { symbol = .pass(knownIssueCount: 1) - known = " known" + subject = "a known issue" } else { - symbol = .fail - known = "n" + switch issue.severity { + case .warning: + symbol = .passWithWarnings + subject = "a warning" + case .error: + symbol = .fail + subject = "an issue" + } } var additionalMessages = [Message]() @@ -435,13 +452,13 @@ extension Event.HumanReadableOutputRecorder { let primaryMessage: Message = if parameterCount == 0 { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } else { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } @@ -498,7 +515,7 @@ extension Event.HumanReadableOutputRecorder { let runStartInstant = context.runStartInstant ?? instant let duration = runStartInstant.descriptionOfDuration(to: instant) - return if issues.issueCount > 0 { + return if issues.errorIssueCount > 0 { [ Message( symbol: .fail, diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index 0f50ed95c..3a3f6df8e 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -22,10 +22,14 @@ extension Event { /// The symbol to use when a test passes. /// /// - Parameters: - /// - knownIssueCount: The number of known issues encountered by the end - /// of the test. + /// - knownIssueCount: The number of known issues recorded for the test. + /// The default value is `0`. case pass(knownIssueCount: Int = 0) + /// The symbol to use when a test passes with one or more warnings. + @_spi(Experimental) + case passWithWarnings + /// The symbol to use when a test fails. case fail @@ -62,6 +66,8 @@ extension Event.Symbol { } else { ("\u{10105B}", "checkmark.diamond.fill") } + case .passWithWarnings: + ("\u{100123}", "questionmark.diamond.fill") case .fail: ("\u{100884}", "xmark.diamond.fill") case .difference: @@ -122,6 +128,9 @@ extension Event.Symbol { // Unicode: HEAVY CHECK MARK return "\u{2714}" } + case .passWithWarnings: + // Unicode: QUESTION MARK + return "\u{003F}" case .fail: // Unicode: HEAVY BALLOT X return "\u{2718}" @@ -157,6 +166,9 @@ extension Event.Symbol { // Unicode: SQUARE ROOT return "\u{221A}" } + case .passWithWarnings: + // Unicode: QUESTION MARK + return "\u{003F}" case .fail: // Unicode: MULTIPLICATION SIGN return "\u{00D7}" diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index dae58400a..5d7449b7b 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -79,6 +79,32 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind + /// An enumeration representing the level of severity of a recorded issue. + /// + /// The supported levels, in increasing order of severity, are: + /// + /// - ``warning`` + /// - ``error`` + @_spi(Experimental) + public enum Severity: Sendable { + /// The severity level for an issue which should be noted but is not + /// necessarily an error. + /// + /// An issue with warning severity does not cause the test it's associated + /// with to be marked as a failure, but is noted in the results. + case warning + + /// The severity level for an issue which represents an error in a test. + /// + /// An issue with error severity causes the test it's associated with to be + /// marked as a failure. + case error + } + + /// The severity of this issue. + @_spi(Experimental) + public var severity: Severity + /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -97,16 +123,20 @@ public struct Issue: Sendable { /// /// - Parameters: /// - kind: The kind of issue this value represents. + /// - severity: The severity of this issue. The default value is + /// ``Severity-swift.enum/error``. /// - comments: An array of comments describing the issue. This array may be /// empty. /// - sourceContext: A ``SourceContext`` indicating where and how this issue /// occurred. init( kind: Kind, + severity: Severity = .error, comments: [Comment], sourceContext: SourceContext ) { self.kind = kind + self.severity = severity self.comments = comments self.sourceContext = sourceContext } @@ -154,27 +184,31 @@ public struct Issue: Sendable { } } +extension Issue.Severity: Comparable {} + // MARK: - CustomStringConvertible, CustomDebugStringConvertible extension Issue: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - if comments.isEmpty { - return String(describing: kind) + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind): \(joinedComments)" + return "\(kind) (\(severity))\(joinedComments)" } public var debugDescription: String { - if comments.isEmpty { - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments: String = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" } } @@ -234,6 +268,17 @@ extension Issue.Kind: CustomStringConvertible { } } +extension Issue.Severity: CustomStringConvertible { + public var description: String { + switch self { + case .warning: + "warning" + case .error: + "error" + } + } +} + #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting @@ -244,6 +289,10 @@ extension Issue { /// The kind of issue this value represents. public var kind: Kind.Snapshot + /// The severity of this issue. + @_spi(Experimental) + public var severity: Severity + /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -268,10 +317,22 @@ extension Issue { self.kind = Issue.Kind.Snapshot(snapshotting: issue.kind) self.comments = issue.comments } + self.severity = issue.severity self.sourceContext = issue.sourceContext self.isKnown = issue.isKnown } + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.kind = try container.decode(Issue.Kind.Snapshot.self, forKey: .kind) + self.comments = try container.decode([Comment].self, forKey: .comments) + self.sourceContext = try container.decode(SourceContext.self, forKey: .sourceContext) + self.isKnown = try container.decode(Bool.self, forKey: .isKnown) + + // Severity is a new field, so fall back to .error if it's not present. + self.severity = try container.decodeIfPresent(Issue.Severity.self, forKey: .severity) ?? .error + } + /// The error which was associated with this issue, if any. /// /// The value of this property is non-`nil` when ``kind-swift.property`` is @@ -295,6 +356,8 @@ extension Issue { } } +extension Issue.Severity: Codable {} + extension Issue.Kind { /// Serializable kinds of issues which may be recorded. @_spi(ForToolsIntegrationOnly) @@ -478,23 +541,25 @@ extension Issue.Kind { extension Issue.Snapshot: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - if comments.isEmpty { - return String(describing: kind) + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind): \(joinedComments)" + return "\(kind) (\(severity))\(joinedComments)" } public var debugDescription: String { - if comments.isEmpty { - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments: String = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" } } diff --git a/Sources/Testing/Running/Configuration+EventHandling.swift b/Sources/Testing/Running/Configuration+EventHandling.swift index 025f07d2c..e3c189f8b 100644 --- a/Sources/Testing/Running/Configuration+EventHandling.swift +++ b/Sources/Testing/Running/Configuration+EventHandling.swift @@ -38,3 +38,23 @@ extension Configuration { return eventHandler(event, contextCopy) } } + +extension Configuration.EventHandlingOptions { + /// Determine whether the specified event should be handled according to the + /// options in this instance. + /// + /// - Parameters: + /// - event: The event to consider handling. + /// + /// - Returns: Whether or not the event should be handled or suppressed. + func shouldHandleEvent(_ event: borrowing Event) -> Bool { + switch event.kind { + case let .issueRecorded(issue): + issue.severity > .warning || isWarningIssueRecordedEventEnabled + case .expectationChecked: + isExpectationCheckedEventEnabled + default: + true + } + } +} diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index f4ae59813..a917c2f5b 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -178,14 +178,33 @@ public struct Configuration: Sendable { // MARK: - Event handling - /// Whether or not events of the kind - /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to - /// this configuration's ``eventHandler`` closure. - /// - /// By default, events of this kind are not delivered to event handlers - /// because they occur frequently in a typical test run and can generate - /// significant backpressure on the event handler. - public var deliverExpectationCheckedEvents: Bool = false + /// A type describing options to use when delivering events to this + /// configuration's event handler + public struct EventHandlingOptions: Sendable { + /// Whether or not events of the kind ``Event/Kind-swift.enum/issueRecorded(_:)`` + /// containing issues with warning (or lower) severity should be delivered + /// to the event handler of the configuration these options are applied to. + /// + /// By default, events matching this criteria are not delivered to event + /// handlers since this is an experimental feature. + /// + /// - Warning: Warning issues are not yet an approved feature. + @_spi(Experimental) + public var isWarningIssueRecordedEventEnabled: Bool = false + + /// Whether or not events of the kind + /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to + /// the event handler of the configuration these options are applied to. + /// + /// By default, events of this kind are not delivered to event handlers + /// because they occur frequently in a typical test run and can generate + /// significant back-pressure on the event handler. + public var isExpectationCheckedEventEnabled: Bool = false + } + + /// The options to use when delivering events to this configuration's event + /// handler. + public var eventHandlingOptions: EventHandlingOptions = .init() /// The event handler to which events should be passed when they occur. public var eventHandler: Event.Handler = { _, _ in } @@ -325,4 +344,14 @@ extension Configuration { } } #endif + + @available(*, deprecated, message: "Set eventHandlingOptions.isExpectationCheckedEventEnabled instead.") + public var deliverExpectationCheckedEvents: Bool { + get { + eventHandlingOptions.isExpectationCheckedEventEnabled + } + set { + eventHandlingOptions.isExpectationCheckedEventEnabled = newValue + } + } } diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index f69e13cd6..9ae299412 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -132,7 +132,7 @@ extension Configuration { /// - Returns: A unique number identifying `self` that can be /// passed to `_removeFromAll(identifiedBy:)`` to unregister it. private func _addToAll() -> UInt64 { - if deliverExpectationCheckedEvents { + if eventHandlingOptions.isExpectationCheckedEventEnabled { Self._deliverExpectationCheckedEventsCount.increment() } return Self._all.withLock { all in @@ -152,16 +152,14 @@ extension Configuration { let configuration = Self._all.withLock { all in all.instances.removeValue(forKey: id) } - if let configuration, configuration.deliverExpectationCheckedEvents { + if let configuration, configuration.eventHandlingOptions.isExpectationCheckedEventEnabled { Self._deliverExpectationCheckedEventsCount.decrement() } } /// An atomic counter that tracks the number of "current" configurations that - /// have set ``deliverExpectationCheckedEvents`` to `true`. - /// - /// On older Apple platforms, this property is not available and ``all`` is - /// directly consulted instead (which is less efficient.) + /// have set ``EventHandlingOptions/isExpectationCheckedEventEnabled`` to + /// `true`. private static let _deliverExpectationCheckedEventsCount = Locked(rawValue: 0) /// Whether or not events of the kind @@ -171,7 +169,8 @@ extension Configuration { /// /// To determine if an individual instance of ``Configuration`` is listening /// for these events, consult the per-instance - /// ``Configuration/deliverExpectationCheckedEvents`` property. + /// ``Configuration/EventHandlingOptions/isExpectationCheckedEventEnabled`` + /// property. static var deliverExpectationCheckedEvents: Bool { _deliverExpectationCheckedEventsCount.rawValue > 0 } diff --git a/Tests/TestingTests/ConfigurationTests.swift b/Tests/TestingTests/ConfigurationTests.swift new file mode 100644 index 000000000..a735d8ac5 --- /dev/null +++ b/Tests/TestingTests/ConfigurationTests.swift @@ -0,0 +1,25 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Configuration Tests") +struct ConfigurationTests { + @Test + @available(*, deprecated, message: "Testing a deprecated SPI.") + func deliverExpectationCheckedEventsProperty() throws { + var configuration = Configuration() + #expect(!configuration.deliverExpectationCheckedEvents) + #expect(!configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) + + configuration.deliverExpectationCheckedEvents = true + #expect(configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) + } +} diff --git a/Tests/TestingTests/EntryPointTests.swift b/Tests/TestingTests/EntryPointTests.swift new file mode 100644 index 000000000..eae7d4b7e --- /dev/null +++ b/Tests/TestingTests/EntryPointTests.swift @@ -0,0 +1,81 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +private import _TestingInternals + +@Suite("Entry point tests") +struct EntryPointTests { + @Test("Entry point filter with filtering of hidden tests enabled") + func hiddenTests() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_someHiddenTest"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.verbosity = .min + + await confirmation("Test event started", expectedCount: 1) { testMatched in + _ = await entryPoint(passing: arguments) { event, context in + if case .testStarted = event.kind { + testMatched() + } + } + } + } + + @Test("Entry point with WarningIssues feature enabled exits with success if all issues have severity < .error") + func warningIssues() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_recordWarningIssue"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.verbosity = .min + + let exitCode = await confirmation("Test matched", expectedCount: 1) { testMatched in + await entryPoint(passing: arguments) { event, context in + if case .testStarted = event.kind { + testMatched() + } else if case let .issueRecorded(issue) = event.kind { + Issue.record("Unexpected issue \(issue) was recorded.") + } + } + } + #expect(exitCode == EXIT_SUCCESS) + } + + @Test("Entry point with WarningIssues feature enabled propagates warning issues and exits with success if all issues have severity < .error") + func warningIssuesEnabled() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_recordWarningIssue"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.isWarningIssueRecordedEventEnabled = true + arguments.verbosity = .min + + let exitCode = await confirmation("Warning issue recorded", expectedCount: 1) { issueRecorded in + await entryPoint(passing: arguments) { event, context in + if case let .issueRecorded(issue) = event.kind { + #expect(issue.severity == .warning) + issueRecorded() + } + } + } + #expect(exitCode == EXIT_SUCCESS) + } +} + +// MARK: - Fixtures + +@Test(.hidden) private func _someHiddenTest() {} + +@Test(.hidden) private func _recordWarningIssue() { + // Intentionally _only_ record issues with warning (or lower) severity. + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() +} diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 97619b755..6b5b9bd81 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -59,7 +59,7 @@ struct EventRecorderTests { } var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(options: options, writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -98,7 +98,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -123,7 +123,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -183,15 +183,20 @@ struct EventRecorderTests { @Test( "Issue counts are summed correctly on test end", arguments: [ - ("f()", false, (total: 5, expected: 3)), - ("g()", false, (total: 2, expected: 1)), - ("PredictablyFailingTests", true, (total: 7, expected: 4)), + ("f()", #".* Test f\(\) failed after .+ seconds with 5 issues \(including 3 known issues\)\."#), + ("g()", #".* Test g\(\) failed after .+ seconds with 2 issues \(including 1 known issue\)\."#), + ("h()", #".* Test h\(\) passed after .+ seconds with 1 warning\."#), + ("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#), + ("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#), + ("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#), ] ) - func issueCountSummingAtTestEnd(testName: String, isSuite: Bool, issueCount: (total: Int, expected: Int)) async throws { + func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws { let stream = Stream() var configuration = Configuration() + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -204,28 +209,13 @@ struct EventRecorderTests { print(buffer, terminator: "") } - let testFailureRegex = Regex { - One(.anyGraphemeCluster) - " \(isSuite ? "Suite" : "Test") \(testName) failed " - ZeroOrMore(.any) - " with " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " issue" - Optionally("s") - " (including " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " known issue" - Optionally("s") - ")." - } - let match = try #require( - buffer - .split(whereSeparator: \.isNewline) - .compactMap(testFailureRegex.wholeMatch(in:)) - .first + let expectedSuffixRegex = try Regex(expectedPattern) + #expect(try buffer + .split(whereSeparator: \.isNewline) + .compactMap(expectedSuffixRegex.wholeMatch(in:)) + .first != nil, + "buffer: \(buffer)" ) - #expect(issueCount.total == match.output.1) - #expect(issueCount.expected == match.output.2) } #endif @@ -294,8 +284,51 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 7) - #expect(match.output.2 == 4) + #expect(match.output.1 == 9) + #expect(match.output.2 == 5) + } + + @Test("Issue counts are summed correctly on run end for a test with only warning issues") + @available(_regexAPI, *) + func warningIssueCountSummingAtRunEnd() async throws { + let stream = Stream() + + var configuration = Configuration() + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true + let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) + configuration.eventHandler = { event, context in + eventRecorder.record(event, in: context) + } + + await runTestFunction(named: "h()", in: PredictablyFailingTests.self, configuration: configuration) + + let buffer = stream.buffer.rawValue + if testsWithSignificantIOAreEnabled { + print(buffer, terminator: "") + } + + let runFailureRegex = Regex { + One(.anyGraphemeCluster) + " Test run with " + OneOrMore(.digit) + " test" + Optionally("s") + " passed " + ZeroOrMore(.any) + " with " + Capture { OneOrMore(.digit) } transform: { Int($0) } + " warning" + Optionally("s") + "." + } + let match = try #require( + buffer + .split(whereSeparator: \.isNewline) + .compactMap(runFailureRegex.wholeMatch(in:)) + .first, + "buffer: \(buffer)" + ) + #expect(match.output.1 == 1) } #endif @@ -308,7 +341,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.JUnitXMLRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -510,4 +543,26 @@ struct EventRecorderTests { #expect(Bool(false)) } } + + @Test(.hidden) func h() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + } + + @Test(.hidden) func i() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + #expect(Bool(false)) + } + + @Test(.hidden) func j() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + withKnownIssue { + #expect(Bool(false)) + } + } + + @Test(.hidden) func k() { + withKnownIssue { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + } + } } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 4a4fda631..8e1e90b85 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -301,7 +301,7 @@ final class IssueTests: XCTestCase { let expectationChecked = expectation(description: "expectation checked") var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return @@ -1124,12 +1124,12 @@ final class IssueTests: XCTestCase { do { let sourceLocation = SourceLocation.init(fileID: "FakeModule/FakeFile.swift", filePath: "", line: 9999, column: 1) let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: sourceLocation)) - XCTAssertEqual(issue.description, "A system failure occurred: Some issue") - XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1: Some issue") + XCTAssertEqual(issue.description, "A system failure occurred (error): Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1 (error): Some issue") } do { let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: nil)) - XCTAssertEqual(issue.debugDescription, "A system failure occurred: Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred (error): Some issue") } } diff --git a/Tests/TestingTests/Runner.RuntimeStateTests.swift b/Tests/TestingTests/Runner.RuntimeStateTests.swift index e4ee33079..1576c49e7 100644 --- a/Tests/TestingTests/Runner.RuntimeStateTests.swift +++ b/Tests/TestingTests/Runner.RuntimeStateTests.swift @@ -34,7 +34,7 @@ struct Runner_RuntimeStateTests { // an event to be posted during the test below without causing any real // issues to be recorded or otherwise confuse the testing harness. var configuration = Configuration.current ?? .init() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true await Configuration.withCurrent(configuration) { await withTaskGroup(of: Void.self) { group in diff --git a/Tests/TestingTests/RunnerTests.swift b/Tests/TestingTests/RunnerTests.swift index 857cfdd81..335f8be37 100644 --- a/Tests/TestingTests/RunnerTests.swift +++ b/Tests/TestingTests/RunnerTests.swift @@ -426,7 +426,7 @@ final class RunnerTests: XCTestCase { func testExpectationCheckedEventHandlingWhenDisabled() async { var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = false + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = false configuration.eventHandler = { event, _ in if case .expectationChecked = event.kind { XCTFail("Expectation checked event was posted unexpectedly") @@ -459,7 +459,7 @@ final class RunnerTests: XCTestCase { #endif var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return From 5a9522811a619ad3ca651a94da497e0aaa513df8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 11 Feb 2025 13:00:29 -0500 Subject: [PATCH 075/234] Suppress warning about `#require(nonOptional)` in some cases. (#947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using `try #require()` to unwrap an optional value, we emit a compile-time warning if the value is not actually optional. However, there is a bug in the type checker (https://github.com/swiftlang/swift/issues/79202) that triggers a false positive when downcasting an object (of `class` type), e.g.: ```swift class Animal {} class Duck: Animal {} let beast: Animal = Duck() let definitelyADuck = try #require(beast as? Duck) // ⚠️ '#require(_:_:)' is redundant because 'beast as? Duck' never equals 'nil' ``` This change suppresses the warning we emit if the expression contains certain syntax tokens (namely `?`, `nil`, or `Optional`) on the assumption that their presence means the test author is expecting an optional value and we've hit a false positive. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/TestingMacros/ConditionMacro.swift | 17 +++++++++++++++++ .../ConditionMacroTests.swift | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 346cb68bf..0b39d2f78 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -324,7 +324,24 @@ public struct NonOptionalRequireMacro: RefinedConditionMacro { in context: some MacroExpansionContext ) throws -> ExprSyntax { if let argument = macro.arguments.first { +#if !SWT_FIXED_137943258 + // Silence this warning if we see a token (`?`, `nil`, or "Optional") that + // might indicate the test author expects the expression is optional. + let tokenKindsIndicatingOptionality: [TokenKind] = [ + .infixQuestionMark, + .postfixQuestionMark, + .keyword(.nil), + .identifier("Optional") + ] + let looksOptional = argument.tokens(viewMode: .sourceAccurate).lazy + .map(\.tokenKind) + .contains(where: tokenKindsIndicatingOptionality.contains) + if !looksOptional { + context.diagnose(.nonOptionalRequireIsRedundant(argument.expression, in: macro)) + } +#else context.diagnose(.nonOptionalRequireIsRedundant(argument.expression, in: macro)) +#endif } // Perform the normal macro expansion for #require(). diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index 9f1201367..e7307476d 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -352,6 +352,22 @@ struct ConditionMacroTests { #expect(diagnostic.message.contains("is redundant")) } +#if !SWT_FIXED_137943258 + @Test( + "#require(optional value mistyped as non-optional) diagnostic is suppressed", + .bug("https://github.com/swiftlang/swift/issues/79202"), + arguments: [ + "#requireNonOptional(expression as? T)", + "#requireNonOptional(expression as Optional)", + "#requireNonOptional(expression ?? nil)", + ] + ) + func requireNonOptionalDiagnosticSuppressed(input: String) throws { + let (_, diagnostics) = try parse(input) + #expect(diagnostics.isEmpty) + } +#endif + @Test("#require(throws: Never.self) produces a diagnostic", arguments: [ "#requireThrows(throws: Swift.Never.self)", From 55d0023fa2d44180e611650d225184292f438d66 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 11 Feb 2025 15:28:11 -0500 Subject: [PATCH 076/234] Add a (internal-only) `CustomIssueRepresentable` protocol. (#945) This PR adds a (for now internal-only) protocol that lets us transform arbitrary errors to arbitrary issues (rather than always emitting them as `.errorCaught`). This protocol replaces the special-casing we were doing for `SystemError` and `APIMisuseError`. This change also causes us to emit an API misuse issue if a developer passes an error of type `ExpectationFailedError` to `Issue.record()` as we can reasonably assume that such an error was already recorded correctly as `.expectationFailed`. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --------- Co-authored-by: Stuart Montgomery --- Sources/Testing/CMakeLists.txt | 2 +- Sources/Testing/Issues/Issue+Recording.swift | 16 +--- .../Support/CustomIssueRepresentable.swift | 86 +++++++++++++++++++ Sources/Testing/Support/SystemError.swift | 36 -------- Tests/TestingTests/IssueTests.swift | 78 +++++++++++++++++ 5 files changed, 169 insertions(+), 49 deletions(-) create mode 100644 Sources/Testing/Support/CustomIssueRepresentable.swift delete mode 100644 Sources/Testing/Support/SystemError.swift diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index cd9983c07..16efd0ee6 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -73,6 +73,7 @@ add_library(Testing Support/Additions/WinSDKAdditions.swift Support/CartesianProduct.swift Support/CError.swift + Support/CustomIssueRepresentable.swift Support/Environment.swift Support/FileHandle.swift Support/GetSymbol.swift @@ -80,7 +81,6 @@ add_library(Testing Support/JSON.swift Support/Locked.swift Support/Locked+Platform.swift - Support/SystemError.swift Support/Versions.swift Discovery.swift Discovery+Platform.swift diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 8a80e4467..aaf721c6a 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -27,19 +27,11 @@ extension Issue { /// - Returns: The issue that was recorded (`self` or a modified copy of it.) @discardableResult func record(configuration: Configuration? = nil) -> Self { - // If this issue is a caught error of kind SystemError, reinterpret it as a - // testing system issue instead (per the documentation for SystemError.) + // If this issue is a caught error that has a custom issue representation, + // perform that customization now. if case let .errorCaught(error) = kind { - // TODO: consider factoring this logic out into a protocol - if let error = error as? SystemError { - var selfCopy = self - selfCopy.kind = .system - selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) - return selfCopy.record(configuration: configuration) - } else if let error = error as? APIMisuseError { - var selfCopy = self - selfCopy.kind = .apiMisused - selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) + if let error = error as? any CustomIssueRepresentable { + let selfCopy = error.customize(self) return selfCopy.record(configuration: configuration) } } diff --git a/Sources/Testing/Support/CustomIssueRepresentable.swift b/Sources/Testing/Support/CustomIssueRepresentable.swift new file mode 100644 index 000000000..d76739d1f --- /dev/null +++ b/Sources/Testing/Support/CustomIssueRepresentable.swift @@ -0,0 +1,86 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A protocol that provides instances of conforming types with the ability to +/// record themselves as test issues. +/// +/// When a type conforms to this protocol, values of that type can be passed to +/// ``Issue/record(_:_:)``. The testing library then calls the +/// ``customize(_:)`` function and passes it an instance of ``Issue`` that will +/// be used to represent the value. The function can then reconfigure or replace +/// the issue as needed. +/// +/// This protocol may become part of the testing library's public interface in +/// the future. There's not really anything _requiring_ it to conform to `Error` +/// but all our current use cases are error types. If we want to allow other +/// types to be represented as issues, we will need to add an overload of +/// `Issue.record()` that takes `some CustomIssueRepresentable`. +protocol CustomIssueRepresentable: Error { + /// Customize the issue that will represent this value. + /// + /// - Parameters: + /// - issue: The issue to customize. The function consumes this value. + /// + /// - Returns: A customized copy of `issue`. + func customize(_ issue: consuming Issue) -> Issue +} + +// MARK: - Internal error types + +/// A type representing an error in the testing library or its underlying +/// infrastructure. +/// +/// When an error of this type is thrown and caught by the testing library, it +/// is recorded as an issue of kind ``Issue/Kind/system`` rather than +/// ``Issue/Kind/errorCaught(_:)``. +/// +/// This type is not part of the public interface of the testing library. +/// External callers should generally record issues by throwing their own errors +/// or by calling ``Issue/record(_:sourceLocation:)``. +struct SystemError: Error, CustomStringConvertible, CustomIssueRepresentable { + var description: String + + func customize(_ issue: consuming Issue) -> Issue { + issue.kind = .system + issue.comments.append("\(self)") + return issue + } +} + +/// A type representing misuse of testing library API. +/// +/// When an error of this type is thrown and caught by the testing library, it +/// is recorded as an issue of kind ``Issue/Kind/apiMisused`` rather than +/// ``Issue/Kind/errorCaught(_:)``. +/// +/// This type is not part of the public interface of the testing library. +/// External callers should generally record issues by throwing their own errors +/// or by calling ``Issue/record(_:sourceLocation:)``. +struct APIMisuseError: Error, CustomStringConvertible, CustomIssueRepresentable { + var description: String + + func customize(_ issue: consuming Issue) -> Issue { + issue.kind = .apiMisused + issue.comments.append("\(self)") + return issue + } +} + +extension ExpectationFailedError: CustomIssueRepresentable { + func customize(_ issue: consuming Issue) -> Issue { + // Instances of this error type are thrown by `#require()` after an issue of + // kind `.expectationFailed` has already been recorded. Code that rethrows + // this error does not generate a new issue, but code that passes this error + // to Issue.record() is misbehaving. + issue.kind = .apiMisused + issue.comments.append("Recorded an error of type \(Self.self) representing an expectation that failed and was already recorded: \(expectation)") + return issue + } +} diff --git a/Sources/Testing/Support/SystemError.swift b/Sources/Testing/Support/SystemError.swift deleted file mode 100644 index d2d4809e3..000000000 --- a/Sources/Testing/Support/SystemError.swift +++ /dev/null @@ -1,36 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// A type representing an error in the testing library or its underlying -/// infrastructure. -/// -/// When an error of this type is thrown and caught by the testing library, it -/// is recorded as an issue of kind ``Issue/Kind/system`` rather than -/// ``Issue/Kind/errorCaught(_:)``. -/// -/// This type is not part of the public interface of the testing library. -/// External callers should generally record issues by throwing their own errors -/// or by calling ``Issue/record(_:sourceLocation:)``. -struct SystemError: Error, CustomStringConvertible { - var description: String -} - -/// A type representing misuse of testing library API. -/// -/// When an error of this type is thrown and caught by the testing library, it -/// is recorded as an issue of kind ``Issue/Kind/apiMisused`` rather than -/// ``Issue/Kind/errorCaught(_:)``. -/// -/// This type is not part of the public interface of the testing library. -/// External callers should generally record issues by throwing their own errors -/// or by calling ``Issue/record(_:sourceLocation:)``. -struct APIMisuseError: Error, CustomStringConvertible { - var description: String -} diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 8e1e90b85..f73676d42 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -1515,6 +1515,84 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } + + func testThrowing(_ error: some Error, producesIssueMatching issueMatcher: @escaping @Sendable (Issue) -> Bool) async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + if issueMatcher(issue) { + issueRecorded.fulfill() + let description = String(describing: error) + #expect(issue.comments.map(String.init(describing:)).contains(description)) + } else { + Issue.record("Unexpected issue \(issue)") + } + } + + await Test { + throw error + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + + func testThrowingSystemErrorProducesSystemIssue() async { + await testThrowing( + SystemError(description: "flinging excuses"), + producesIssueMatching: { issue in + if case .system = issue.kind { + return true + } + return false + } + ) + } + + func testThrowingAPIMisuseErrorProducesAPIMisuseIssue() async { + await testThrowing( + APIMisuseError(description: "you did it wrong"), + producesIssueMatching: { issue in + if case .apiMisused = issue.kind { + return true + } + return false + } + ) + } + + func testRethrowingExpectationFailedErrorCausesAPIMisuseError() async { + let expectationFailed = expectation(description: "Expectation failed (issue recorded)") + let apiMisused = expectation(description: "API misused (issue recorded)") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + switch issue.kind { + case .expectationFailed: + expectationFailed.fulfill() + case .apiMisused: + apiMisused.fulfill() + default: + Issue.record("Unexpected issue \(issue)") + } + } + + await Test { + do { + try #require(Bool(false)) + } catch { + Issue.record(error) + } + }.run(configuration: configuration) + + await fulfillment(of: [expectationFailed, apiMisused], timeout: 0.0) + } } #endif From afe2505539181edbf0a26fdcd9a9a600ccfd457e Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 11 Feb 2025 18:04:34 -0600 Subject: [PATCH 077/234] Revert "Introduce a severity level for issues, and a 'warning' severity (#931)" This reverts commit 6a49142b223f214eff5da6c1224935a80e62e512. --- .../ABI/EntryPoints/ABIEntryPoint.swift | 6 +- .../Testing/ABI/EntryPoints/EntryPoint.swift | 37 +----- .../ABI/v0/Encoded/ABIv0.EncodedIssue.swift | 19 --- .../ABI/v0/Encoded/ABIv0.EncodedMessage.swift | 3 - Sources/Testing/Events/Event.swift | 5 +- .../Event.ConsoleOutputRecorder.swift | 2 - .../Event.HumanReadableOutputRecorder.swift | 61 ++++----- .../Events/Recorder/Event.Symbol.swift | 16 +-- Sources/Testing/Issues/Issue.swift | 113 ++++------------- .../Running/Configuration+EventHandling.swift | 20 --- Sources/Testing/Running/Configuration.swift | 45 ++----- .../Testing/Running/Runner.RuntimeState.swift | 13 +- Tests/TestingTests/ConfigurationTests.swift | 25 ---- Tests/TestingTests/EntryPointTests.swift | 81 ------------ Tests/TestingTests/EventRecorderTests.swift | 117 +++++------------- Tests/TestingTests/IssueTests.swift | 8 +- .../Runner.RuntimeStateTests.swift | 2 +- Tests/TestingTests/RunnerTests.swift | 4 +- 18 files changed, 109 insertions(+), 468 deletions(-) delete mode 100644 Tests/TestingTests/ConfigurationTests.swift delete mode 100644 Tests/TestingTests/EntryPointTests.swift diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index 143f7b549..cc150740e 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -47,7 +47,7 @@ extension ABIv0 { /// callback. public static var entryPoint: EntryPoint { return { configurationJSON, recordHandler in - try await _entryPoint( + try await Testing.entryPoint( configurationJSON: configurationJSON, recordHandler: recordHandler ) == EXIT_SUCCESS @@ -87,7 +87,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in - try await _entryPoint( + try await entryPoint( configurationJSON: configurationJSON, eventStreamVersionIfNil: -1, recordHandler: recordHandler @@ -104,7 +104,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( /// /// This function will be removed (with its logic incorporated into /// ``ABIv0/entryPoint-swift.type.property``) in a future update. -private func _entryPoint( +private func entryPoint( configurationJSON: UnsafeRawBufferPointer?, eventStreamVersionIfNil: Int? = nil, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 89094c88f..a0a5df2a0 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -20,8 +20,6 @@ private import _TestingInternals /// writes events to the standard error stream in addition to passing them /// to this function. /// -/// - Returns: An exit code representing the result of running tests. -/// /// External callers cannot call this function directly. The can use /// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable /// version of this function. @@ -42,7 +40,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // Set up the event handler. configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error { + if case let .issueRecorded(issue) = event.kind, !issue.isKnown { exitCode.withLock { exitCode in exitCode = EXIT_FAILURE } @@ -272,13 +270,6 @@ public struct __CommandLineArguments_v0: Sendable { /// The value(s) of the `--skip` argument. public var skip: [String]? - /// Whether or not to include tests with the `.hidden` trait when constructing - /// a test filter based on these arguments. - /// - /// This property is intended for use in testing the testing library itself. - /// It is not parsed as a command-line argument. - var includeHiddenTests: Bool? - /// The value of the `--repetitions` argument. public var repetitions: Int? @@ -287,13 +278,6 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--experimental-attachments-path` argument. public var experimentalAttachmentsPath: String? - - /// Whether or not the experimental warning issue severity feature should be - /// enabled. - /// - /// This property is intended for use in testing the testing library itself. - /// It is not parsed as a command-line argument. - var isWarningIssueRecordedEventEnabled: Bool? } extension __CommandLineArguments_v0: Codable { @@ -533,9 +517,6 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding)) configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) } - if args.includeHiddenTests == true { - configuration.testFilter.includeHiddenTests = true - } // Set up the iteration policy for the test run. var repetitionPolicy: Configuration.RepetitionPolicy = .once @@ -566,22 +547,6 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr configuration.exitTestHandler = ExitTest.handlerForEntryPoint() #endif - // Warning issues (experimental). - if args.isWarningIssueRecordedEventEnabled == true { - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true - } else { - switch args.eventStreamVersion { - case .some(...0): - // If the event stream version was explicitly specified to a value < 1, - // disable the warning issue event to maintain legacy behavior. - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false - default: - // Otherwise the requested event stream version is ≥ 1, so don't change - // the warning issue event setting. - break - } - } - return configuration } diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift index 97c051d28..2bf1c8462 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift @@ -16,19 +16,6 @@ extension ABIv0 { /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. struct EncodedIssue: Sendable { - /// An enumeration representing the level of severity of a recorded issue. - /// - /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``. - enum Severity: String, Sendable { - case warning - case error - } - - /// The severity of this issue. - /// - /// - Warning: Severity is not yet part of the JSON schema. - var _severity: Severity - /// Whether or not this issue is known to occur. var isKnown: Bool @@ -46,11 +33,6 @@ extension ABIv0 { var _error: EncodedError? init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { - _severity = switch issue.severity { - case .warning: .warning - case .error: .error - } - isKnown = issue.isKnown sourceLocation = issue.sourceLocation if let backtrace = issue.sourceContext.backtrace { @@ -66,4 +48,3 @@ extension ABIv0 { // MARK: - Codable extension ABIv0.EncodedIssue: Codable {} -extension ABIv0.EncodedIssue.Severity: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift index cf44f0af0..5cfbf647c 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift @@ -25,7 +25,6 @@ extension ABIv0 { case `default` case skip case pass - case passWithWarnings = "_passWithWarnings" case passWithKnownIssue case fail case difference @@ -45,8 +44,6 @@ extension ABIv0 { } else { .pass } - case .passWithWarnings: - .passWithWarnings case .fail: .fail case .difference: diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index b81f1c2c7..60e564d5a 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -290,7 +290,10 @@ extension Event { if let configuration = configuration ?? Configuration.current { // The caller specified a configuration, or the current task has an // associated configuration. Post to either configuration's event handler. - if configuration.eventHandlingOptions.shouldHandleEvent(self) { + switch kind { + case .expectationChecked where !configuration.deliverExpectationCheckedEvents: + break + default: configuration.handleEvent(self, in: context) } } else { diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index b375b2da1..cce3a732c 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -162,8 +162,6 @@ extension Event.Symbol { return "\(_ansiEscapeCodePrefix)90m\(symbolCharacter)\(_resetANSIEscapeCode)" } return "\(_ansiEscapeCodePrefix)92m\(symbolCharacter)\(_resetANSIEscapeCode)" - case .passWithWarnings: - return "\(_ansiEscapeCodePrefix)93m\(symbolCharacter)\(_resetANSIEscapeCode)" case .fail: return "\(_ansiEscapeCodePrefix)91m\(symbolCharacter)\(_resetANSIEscapeCode)" case .warning: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 2e40b2789..98303f11c 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -56,9 +56,8 @@ extension Event { /// The instant at which the test started. var startInstant: Test.Clock.Instant - /// The number of issues recorded for the test, grouped by their - /// level of severity. - var issueCount: [Issue.Severity: Int] = [:] + /// The number of issues recorded for the test. + var issueCount = 0 /// The number of known issues recorded for the test. var knownIssueCount = 0 @@ -115,36 +114,27 @@ extension Event.HumanReadableOutputRecorder { /// - graph: The graph to walk while counting issues. /// /// - Returns: A tuple containing the number of issues recorded in `graph`. - private func _issueCounts(in graph: Graph?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { + private func _issueCounts(in graph: Graph?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { guard let graph else { - return (0, 0, 0, 0, "") + return (0, 0, 0, "") } - let errorIssueCount = graph.compactMap(\.value?.issueCount[.error]).reduce(into: 0, +=) - let warningIssueCount = graph.compactMap(\.value?.issueCount[.warning]).reduce(into: 0, +=) + let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=) let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=) - let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount + let totalIssueCount = issueCount + knownIssueCount // Construct a string describing the issue counts. - let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) { - case (true, true, true): - " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))" - case (true, false, true): + let description = switch (issueCount > 0, knownIssueCount > 0) { + case (true, true): " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))" - case (false, true, true): - " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))" - case (false, false, true): + case (false, true): " with \(knownIssueCount.counting("known issue"))" - case (true, true, false): - " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))" - case (true, false, false): + case (true, false): " with \(totalIssueCount.counting("issue"))" - case(false, true, false): - " with \(warningIssueCount.counting("warning"))" - case(false, false, false): + case(false, false): "" } - return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) + return (issueCount, knownIssueCount, totalIssueCount, description) } } @@ -277,8 +267,7 @@ extension Event.HumanReadableOutputRecorder { if issue.isKnown { testData.knownIssueCount += 1 } else { - let issueCount = testData.issueCount[issue.severity] ?? 0 - testData.issueCount[issue.severity] = issueCount + 1 + testData.issueCount += 1 } context.testData[id] = testData @@ -366,7 +355,7 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - return if issues.errorIssueCount > 0 { + return if issues.issueCount > 0 { CollectionOfOne( Message( symbol: .fail, @@ -374,7 +363,7 @@ extension Event.HumanReadableOutputRecorder { ) ) + _formattedComments(for: test) } else { - [ + [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)." @@ -411,19 +400,13 @@ extension Event.HumanReadableOutputRecorder { "" } let symbol: Event.Symbol - let subject: String + let known: String if issue.isKnown { symbol = .pass(knownIssueCount: 1) - subject = "a known issue" + known = " known" } else { - switch issue.severity { - case .warning: - symbol = .passWithWarnings - subject = "a warning" - case .error: - symbol = .fail - subject = "an issue" - } + symbol = .fail + known = "n" } var additionalMessages = [Message]() @@ -452,13 +435,13 @@ extension Event.HumanReadableOutputRecorder { let primaryMessage: Message = if parameterCount == 0 { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } else { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } @@ -515,7 +498,7 @@ extension Event.HumanReadableOutputRecorder { let runStartInstant = context.runStartInstant ?? instant let duration = runStartInstant.descriptionOfDuration(to: instant) - return if issues.errorIssueCount > 0 { + return if issues.issueCount > 0 { [ Message( symbol: .fail, diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index 3a3f6df8e..0f50ed95c 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -22,14 +22,10 @@ extension Event { /// The symbol to use when a test passes. /// /// - Parameters: - /// - knownIssueCount: The number of known issues recorded for the test. - /// The default value is `0`. + /// - knownIssueCount: The number of known issues encountered by the end + /// of the test. case pass(knownIssueCount: Int = 0) - /// The symbol to use when a test passes with one or more warnings. - @_spi(Experimental) - case passWithWarnings - /// The symbol to use when a test fails. case fail @@ -66,8 +62,6 @@ extension Event.Symbol { } else { ("\u{10105B}", "checkmark.diamond.fill") } - case .passWithWarnings: - ("\u{100123}", "questionmark.diamond.fill") case .fail: ("\u{100884}", "xmark.diamond.fill") case .difference: @@ -128,9 +122,6 @@ extension Event.Symbol { // Unicode: HEAVY CHECK MARK return "\u{2714}" } - case .passWithWarnings: - // Unicode: QUESTION MARK - return "\u{003F}" case .fail: // Unicode: HEAVY BALLOT X return "\u{2718}" @@ -166,9 +157,6 @@ extension Event.Symbol { // Unicode: SQUARE ROOT return "\u{221A}" } - case .passWithWarnings: - // Unicode: QUESTION MARK - return "\u{003F}" case .fail: // Unicode: MULTIPLICATION SIGN return "\u{00D7}" diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 5d7449b7b..dae58400a 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -79,32 +79,6 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind - /// An enumeration representing the level of severity of a recorded issue. - /// - /// The supported levels, in increasing order of severity, are: - /// - /// - ``warning`` - /// - ``error`` - @_spi(Experimental) - public enum Severity: Sendable { - /// The severity level for an issue which should be noted but is not - /// necessarily an error. - /// - /// An issue with warning severity does not cause the test it's associated - /// with to be marked as a failure, but is noted in the results. - case warning - - /// The severity level for an issue which represents an error in a test. - /// - /// An issue with error severity causes the test it's associated with to be - /// marked as a failure. - case error - } - - /// The severity of this issue. - @_spi(Experimental) - public var severity: Severity - /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -123,20 +97,16 @@ public struct Issue: Sendable { /// /// - Parameters: /// - kind: The kind of issue this value represents. - /// - severity: The severity of this issue. The default value is - /// ``Severity-swift.enum/error``. /// - comments: An array of comments describing the issue. This array may be /// empty. /// - sourceContext: A ``SourceContext`` indicating where and how this issue /// occurred. init( kind: Kind, - severity: Severity = .error, comments: [Comment], sourceContext: SourceContext ) { self.kind = kind - self.severity = severity self.comments = comments self.sourceContext = sourceContext } @@ -184,31 +154,27 @@ public struct Issue: Sendable { } } -extension Issue.Severity: Comparable {} - // MARK: - CustomStringConvertible, CustomDebugStringConvertible extension Issue: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return String(describing: kind) } - return "\(kind) (\(severity))\(joinedComments)" + let joinedComments = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind): \(joinedComments)" } public var debugDescription: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" } - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" + let joinedComments: String = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" } } @@ -268,17 +234,6 @@ extension Issue.Kind: CustomStringConvertible { } } -extension Issue.Severity: CustomStringConvertible { - public var description: String { - switch self { - case .warning: - "warning" - case .error: - "error" - } - } -} - #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting @@ -289,10 +244,6 @@ extension Issue { /// The kind of issue this value represents. public var kind: Kind.Snapshot - /// The severity of this issue. - @_spi(Experimental) - public var severity: Severity - /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -317,22 +268,10 @@ extension Issue { self.kind = Issue.Kind.Snapshot(snapshotting: issue.kind) self.comments = issue.comments } - self.severity = issue.severity self.sourceContext = issue.sourceContext self.isKnown = issue.isKnown } - public init(from decoder: any Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - self.kind = try container.decode(Issue.Kind.Snapshot.self, forKey: .kind) - self.comments = try container.decode([Comment].self, forKey: .comments) - self.sourceContext = try container.decode(SourceContext.self, forKey: .sourceContext) - self.isKnown = try container.decode(Bool.self, forKey: .isKnown) - - // Severity is a new field, so fall back to .error if it's not present. - self.severity = try container.decodeIfPresent(Issue.Severity.self, forKey: .severity) ?? .error - } - /// The error which was associated with this issue, if any. /// /// The value of this property is non-`nil` when ``kind-swift.property`` is @@ -356,8 +295,6 @@ extension Issue { } } -extension Issue.Severity: Codable {} - extension Issue.Kind { /// Serializable kinds of issues which may be recorded. @_spi(ForToolsIntegrationOnly) @@ -541,25 +478,23 @@ extension Issue.Kind { extension Issue.Snapshot: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return String(describing: kind) } - return "\(kind) (\(severity))\(joinedComments)" + let joinedComments = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind): \(joinedComments)" } public var debugDescription: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" } - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" + let joinedComments: String = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" } } diff --git a/Sources/Testing/Running/Configuration+EventHandling.swift b/Sources/Testing/Running/Configuration+EventHandling.swift index e3c189f8b..025f07d2c 100644 --- a/Sources/Testing/Running/Configuration+EventHandling.swift +++ b/Sources/Testing/Running/Configuration+EventHandling.swift @@ -38,23 +38,3 @@ extension Configuration { return eventHandler(event, contextCopy) } } - -extension Configuration.EventHandlingOptions { - /// Determine whether the specified event should be handled according to the - /// options in this instance. - /// - /// - Parameters: - /// - event: The event to consider handling. - /// - /// - Returns: Whether or not the event should be handled or suppressed. - func shouldHandleEvent(_ event: borrowing Event) -> Bool { - switch event.kind { - case let .issueRecorded(issue): - issue.severity > .warning || isWarningIssueRecordedEventEnabled - case .expectationChecked: - isExpectationCheckedEventEnabled - default: - true - } - } -} diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index a917c2f5b..f4ae59813 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -178,33 +178,14 @@ public struct Configuration: Sendable { // MARK: - Event handling - /// A type describing options to use when delivering events to this - /// configuration's event handler - public struct EventHandlingOptions: Sendable { - /// Whether or not events of the kind ``Event/Kind-swift.enum/issueRecorded(_:)`` - /// containing issues with warning (or lower) severity should be delivered - /// to the event handler of the configuration these options are applied to. - /// - /// By default, events matching this criteria are not delivered to event - /// handlers since this is an experimental feature. - /// - /// - Warning: Warning issues are not yet an approved feature. - @_spi(Experimental) - public var isWarningIssueRecordedEventEnabled: Bool = false - - /// Whether or not events of the kind - /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to - /// the event handler of the configuration these options are applied to. - /// - /// By default, events of this kind are not delivered to event handlers - /// because they occur frequently in a typical test run and can generate - /// significant back-pressure on the event handler. - public var isExpectationCheckedEventEnabled: Bool = false - } - - /// The options to use when delivering events to this configuration's event - /// handler. - public var eventHandlingOptions: EventHandlingOptions = .init() + /// Whether or not events of the kind + /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to + /// this configuration's ``eventHandler`` closure. + /// + /// By default, events of this kind are not delivered to event handlers + /// because they occur frequently in a typical test run and can generate + /// significant backpressure on the event handler. + public var deliverExpectationCheckedEvents: Bool = false /// The event handler to which events should be passed when they occur. public var eventHandler: Event.Handler = { _, _ in } @@ -344,14 +325,4 @@ extension Configuration { } } #endif - - @available(*, deprecated, message: "Set eventHandlingOptions.isExpectationCheckedEventEnabled instead.") - public var deliverExpectationCheckedEvents: Bool { - get { - eventHandlingOptions.isExpectationCheckedEventEnabled - } - set { - eventHandlingOptions.isExpectationCheckedEventEnabled = newValue - } - } } diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index 9ae299412..f69e13cd6 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -132,7 +132,7 @@ extension Configuration { /// - Returns: A unique number identifying `self` that can be /// passed to `_removeFromAll(identifiedBy:)`` to unregister it. private func _addToAll() -> UInt64 { - if eventHandlingOptions.isExpectationCheckedEventEnabled { + if deliverExpectationCheckedEvents { Self._deliverExpectationCheckedEventsCount.increment() } return Self._all.withLock { all in @@ -152,14 +152,16 @@ extension Configuration { let configuration = Self._all.withLock { all in all.instances.removeValue(forKey: id) } - if let configuration, configuration.eventHandlingOptions.isExpectationCheckedEventEnabled { + if let configuration, configuration.deliverExpectationCheckedEvents { Self._deliverExpectationCheckedEventsCount.decrement() } } /// An atomic counter that tracks the number of "current" configurations that - /// have set ``EventHandlingOptions/isExpectationCheckedEventEnabled`` to - /// `true`. + /// have set ``deliverExpectationCheckedEvents`` to `true`. + /// + /// On older Apple platforms, this property is not available and ``all`` is + /// directly consulted instead (which is less efficient.) private static let _deliverExpectationCheckedEventsCount = Locked(rawValue: 0) /// Whether or not events of the kind @@ -169,8 +171,7 @@ extension Configuration { /// /// To determine if an individual instance of ``Configuration`` is listening /// for these events, consult the per-instance - /// ``Configuration/EventHandlingOptions/isExpectationCheckedEventEnabled`` - /// property. + /// ``Configuration/deliverExpectationCheckedEvents`` property. static var deliverExpectationCheckedEvents: Bool { _deliverExpectationCheckedEventsCount.rawValue > 0 } diff --git a/Tests/TestingTests/ConfigurationTests.swift b/Tests/TestingTests/ConfigurationTests.swift deleted file mode 100644 index a735d8ac5..000000000 --- a/Tests/TestingTests/ConfigurationTests.swift +++ /dev/null @@ -1,25 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -@_spi(ForToolsIntegrationOnly) import Testing - -@Suite("Configuration Tests") -struct ConfigurationTests { - @Test - @available(*, deprecated, message: "Testing a deprecated SPI.") - func deliverExpectationCheckedEventsProperty() throws { - var configuration = Configuration() - #expect(!configuration.deliverExpectationCheckedEvents) - #expect(!configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) - - configuration.deliverExpectationCheckedEvents = true - #expect(configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) - } -} diff --git a/Tests/TestingTests/EntryPointTests.swift b/Tests/TestingTests/EntryPointTests.swift deleted file mode 100644 index eae7d4b7e..000000000 --- a/Tests/TestingTests/EntryPointTests.swift +++ /dev/null @@ -1,81 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing -private import _TestingInternals - -@Suite("Entry point tests") -struct EntryPointTests { - @Test("Entry point filter with filtering of hidden tests enabled") - func hiddenTests() async throws { - var arguments = __CommandLineArguments_v0() - arguments.filter = ["_someHiddenTest"] - arguments.includeHiddenTests = true - arguments.eventStreamVersion = 0 - arguments.verbosity = .min - - await confirmation("Test event started", expectedCount: 1) { testMatched in - _ = await entryPoint(passing: arguments) { event, context in - if case .testStarted = event.kind { - testMatched() - } - } - } - } - - @Test("Entry point with WarningIssues feature enabled exits with success if all issues have severity < .error") - func warningIssues() async throws { - var arguments = __CommandLineArguments_v0() - arguments.filter = ["_recordWarningIssue"] - arguments.includeHiddenTests = true - arguments.eventStreamVersion = 0 - arguments.verbosity = .min - - let exitCode = await confirmation("Test matched", expectedCount: 1) { testMatched in - await entryPoint(passing: arguments) { event, context in - if case .testStarted = event.kind { - testMatched() - } else if case let .issueRecorded(issue) = event.kind { - Issue.record("Unexpected issue \(issue) was recorded.") - } - } - } - #expect(exitCode == EXIT_SUCCESS) - } - - @Test("Entry point with WarningIssues feature enabled propagates warning issues and exits with success if all issues have severity < .error") - func warningIssuesEnabled() async throws { - var arguments = __CommandLineArguments_v0() - arguments.filter = ["_recordWarningIssue"] - arguments.includeHiddenTests = true - arguments.eventStreamVersion = 0 - arguments.isWarningIssueRecordedEventEnabled = true - arguments.verbosity = .min - - let exitCode = await confirmation("Warning issue recorded", expectedCount: 1) { issueRecorded in - await entryPoint(passing: arguments) { event, context in - if case let .issueRecorded(issue) = event.kind { - #expect(issue.severity == .warning) - issueRecorded() - } - } - } - #expect(exitCode == EXIT_SUCCESS) - } -} - -// MARK: - Fixtures - -@Test(.hidden) private func _someHiddenTest() {} - -@Test(.hidden) private func _recordWarningIssue() { - // Intentionally _only_ record issues with warning (or lower) severity. - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() -} diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 6b5b9bd81..97619b755 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -59,7 +59,7 @@ struct EventRecorderTests { } var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.ConsoleOutputRecorder(options: options, writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -98,7 +98,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -123,7 +123,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -183,20 +183,15 @@ struct EventRecorderTests { @Test( "Issue counts are summed correctly on test end", arguments: [ - ("f()", #".* Test f\(\) failed after .+ seconds with 5 issues \(including 3 known issues\)\."#), - ("g()", #".* Test g\(\) failed after .+ seconds with 2 issues \(including 1 known issue\)\."#), - ("h()", #".* Test h\(\) passed after .+ seconds with 1 warning\."#), - ("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#), - ("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#), - ("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#), + ("f()", false, (total: 5, expected: 3)), + ("g()", false, (total: 2, expected: 1)), + ("PredictablyFailingTests", true, (total: 7, expected: 4)), ] ) - func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws { + func issueCountSummingAtTestEnd(testName: String, isSuite: Bool, issueCount: (total: Int, expected: Int)) async throws { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -209,13 +204,28 @@ struct EventRecorderTests { print(buffer, terminator: "") } - let expectedSuffixRegex = try Regex(expectedPattern) - #expect(try buffer - .split(whereSeparator: \.isNewline) - .compactMap(expectedSuffixRegex.wholeMatch(in:)) - .first != nil, - "buffer: \(buffer)" + let testFailureRegex = Regex { + One(.anyGraphemeCluster) + " \(isSuite ? "Suite" : "Test") \(testName) failed " + ZeroOrMore(.any) + " with " + Capture { OneOrMore(.digit) } transform: { Int($0) } + " issue" + Optionally("s") + " (including " + Capture { OneOrMore(.digit) } transform: { Int($0) } + " known issue" + Optionally("s") + ")." + } + let match = try #require( + buffer + .split(whereSeparator: \.isNewline) + .compactMap(testFailureRegex.wholeMatch(in:)) + .first ) + #expect(issueCount.total == match.output.1) + #expect(issueCount.expected == match.output.2) } #endif @@ -284,51 +294,8 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 9) - #expect(match.output.2 == 5) - } - - @Test("Issue counts are summed correctly on run end for a test with only warning issues") - @available(_regexAPI, *) - func warningIssueCountSummingAtRunEnd() async throws { - let stream = Stream() - - var configuration = Configuration() - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true - let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) - configuration.eventHandler = { event, context in - eventRecorder.record(event, in: context) - } - - await runTestFunction(named: "h()", in: PredictablyFailingTests.self, configuration: configuration) - - let buffer = stream.buffer.rawValue - if testsWithSignificantIOAreEnabled { - print(buffer, terminator: "") - } - - let runFailureRegex = Regex { - One(.anyGraphemeCluster) - " Test run with " - OneOrMore(.digit) - " test" - Optionally("s") - " passed " - ZeroOrMore(.any) - " with " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " warning" - Optionally("s") - "." - } - let match = try #require( - buffer - .split(whereSeparator: \.isNewline) - .compactMap(runFailureRegex.wholeMatch(in:)) - .first, - "buffer: \(buffer)" - ) - #expect(match.output.1 == 1) + #expect(match.output.1 == 7) + #expect(match.output.2 == 4) } #endif @@ -341,7 +308,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.JUnitXMLRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -543,26 +510,4 @@ struct EventRecorderTests { #expect(Bool(false)) } } - - @Test(.hidden) func h() { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - } - - @Test(.hidden) func i() { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - #expect(Bool(false)) - } - - @Test(.hidden) func j() { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - withKnownIssue { - #expect(Bool(false)) - } - } - - @Test(.hidden) func k() { - withKnownIssue { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - } - } } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index f73676d42..8b7d86358 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -301,7 +301,7 @@ final class IssueTests: XCTestCase { let expectationChecked = expectation(description: "expectation checked") var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return @@ -1124,12 +1124,12 @@ final class IssueTests: XCTestCase { do { let sourceLocation = SourceLocation.init(fileID: "FakeModule/FakeFile.swift", filePath: "", line: 9999, column: 1) let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: sourceLocation)) - XCTAssertEqual(issue.description, "A system failure occurred (error): Some issue") - XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1 (error): Some issue") + XCTAssertEqual(issue.description, "A system failure occurred: Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1: Some issue") } do { let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: nil)) - XCTAssertEqual(issue.debugDescription, "A system failure occurred (error): Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred: Some issue") } } diff --git a/Tests/TestingTests/Runner.RuntimeStateTests.swift b/Tests/TestingTests/Runner.RuntimeStateTests.swift index 1576c49e7..e4ee33079 100644 --- a/Tests/TestingTests/Runner.RuntimeStateTests.swift +++ b/Tests/TestingTests/Runner.RuntimeStateTests.swift @@ -34,7 +34,7 @@ struct Runner_RuntimeStateTests { // an event to be posted during the test below without causing any real // issues to be recorded or otherwise confuse the testing harness. var configuration = Configuration.current ?? .init() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true await Configuration.withCurrent(configuration) { await withTaskGroup(of: Void.self) { group in diff --git a/Tests/TestingTests/RunnerTests.swift b/Tests/TestingTests/RunnerTests.swift index 335f8be37..857cfdd81 100644 --- a/Tests/TestingTests/RunnerTests.swift +++ b/Tests/TestingTests/RunnerTests.swift @@ -426,7 +426,7 @@ final class RunnerTests: XCTestCase { func testExpectationCheckedEventHandlingWhenDisabled() async { var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = false + configuration.deliverExpectationCheckedEvents = false configuration.eventHandler = { event, _ in if case .expectationChecked = event.kind { XCTFail("Expectation checked event was posted unexpectedly") @@ -459,7 +459,7 @@ final class RunnerTests: XCTestCase { #endif var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return From fb93dd4bdc76d392b4ce3833cb545b241e63dcc2 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Feb 2025 10:18:59 -0600 Subject: [PATCH 078/234] Introduce a severity level for issues, and a 'warning' severity (un-revert #931) (#952) This un-reverts #950, effectively reintroducing the changes recently landed in #931. The revert was needed because it revealed a latent bug in the Swift compiler, tracked by https://github.com/swiftlang/swift/issues/79304. I reproduced that failure and included a workaround in the second commit on this PR. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../ABI/EntryPoints/ABIEntryPoint.swift | 6 +- .../Testing/ABI/EntryPoints/EntryPoint.swift | 37 +++++- .../ABI/v0/Encoded/ABIv0.EncodedIssue.swift | 19 +++ .../ABI/v0/Encoded/ABIv0.EncodedMessage.swift | 3 + Sources/Testing/Events/Event.swift | 5 +- .../Event.ConsoleOutputRecorder.swift | 2 + .../Event.HumanReadableOutputRecorder.swift | 61 +++++---- .../Events/Recorder/Event.Symbol.swift | 16 ++- Sources/Testing/Issues/Issue.swift | 113 +++++++++++++---- .../Running/Configuration+EventHandling.swift | 20 +++ Sources/Testing/Running/Configuration.swift | 45 +++++-- .../Testing/Running/Runner.RuntimeState.swift | 13 +- Tests/TestingTests/ConfigurationTests.swift | 25 ++++ Tests/TestingTests/EntryPointTests.swift | 81 ++++++++++++ Tests/TestingTests/EventRecorderTests.swift | 117 +++++++++++++----- Tests/TestingTests/IssueTests.swift | 8 +- .../Runner.RuntimeStateTests.swift | 2 +- Tests/TestingTests/RunnerTests.swift | 4 +- 18 files changed, 468 insertions(+), 109 deletions(-) create mode 100644 Tests/TestingTests/ConfigurationTests.swift create mode 100644 Tests/TestingTests/EntryPointTests.swift diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index cc150740e..143f7b549 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -47,7 +47,7 @@ extension ABIv0 { /// callback. public static var entryPoint: EntryPoint { return { configurationJSON, recordHandler in - try await Testing.entryPoint( + try await _entryPoint( configurationJSON: configurationJSON, recordHandler: recordHandler ) == EXIT_SUCCESS @@ -87,7 +87,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in - try await entryPoint( + try await _entryPoint( configurationJSON: configurationJSON, eventStreamVersionIfNil: -1, recordHandler: recordHandler @@ -104,7 +104,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( /// /// This function will be removed (with its logic incorporated into /// ``ABIv0/entryPoint-swift.type.property``) in a future update. -private func entryPoint( +private func _entryPoint( configurationJSON: UnsafeRawBufferPointer?, eventStreamVersionIfNil: Int? = nil, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index a0a5df2a0..89094c88f 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -20,6 +20,8 @@ private import _TestingInternals /// writes events to the standard error stream in addition to passing them /// to this function. /// +/// - Returns: An exit code representing the result of running tests. +/// /// External callers cannot call this function directly. The can use /// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable /// version of this function. @@ -40,7 +42,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // Set up the event handler. configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown { + if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error { exitCode.withLock { exitCode in exitCode = EXIT_FAILURE } @@ -270,6 +272,13 @@ public struct __CommandLineArguments_v0: Sendable { /// The value(s) of the `--skip` argument. public var skip: [String]? + /// Whether or not to include tests with the `.hidden` trait when constructing + /// a test filter based on these arguments. + /// + /// This property is intended for use in testing the testing library itself. + /// It is not parsed as a command-line argument. + var includeHiddenTests: Bool? + /// The value of the `--repetitions` argument. public var repetitions: Int? @@ -278,6 +287,13 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--experimental-attachments-path` argument. public var experimentalAttachmentsPath: String? + + /// Whether or not the experimental warning issue severity feature should be + /// enabled. + /// + /// This property is intended for use in testing the testing library itself. + /// It is not parsed as a command-line argument. + var isWarningIssueRecordedEventEnabled: Bool? } extension __CommandLineArguments_v0: Codable { @@ -517,6 +533,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding)) configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) } + if args.includeHiddenTests == true { + configuration.testFilter.includeHiddenTests = true + } // Set up the iteration policy for the test run. var repetitionPolicy: Configuration.RepetitionPolicy = .once @@ -547,6 +566,22 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr configuration.exitTestHandler = ExitTest.handlerForEntryPoint() #endif + // Warning issues (experimental). + if args.isWarningIssueRecordedEventEnabled == true { + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true + } else { + switch args.eventStreamVersion { + case .some(...0): + // If the event stream version was explicitly specified to a value < 1, + // disable the warning issue event to maintain legacy behavior. + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false + default: + // Otherwise the requested event stream version is ≥ 1, so don't change + // the warning issue event setting. + break + } + } + return configuration } diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift index 2bf1c8462..97c051d28 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift @@ -16,6 +16,19 @@ extension ABIv0 { /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. struct EncodedIssue: Sendable { + /// An enumeration representing the level of severity of a recorded issue. + /// + /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``. + enum Severity: String, Sendable { + case warning + case error + } + + /// The severity of this issue. + /// + /// - Warning: Severity is not yet part of the JSON schema. + var _severity: Severity + /// Whether or not this issue is known to occur. var isKnown: Bool @@ -33,6 +46,11 @@ extension ABIv0 { var _error: EncodedError? init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { + _severity = switch issue.severity { + case .warning: .warning + case .error: .error + } + isKnown = issue.isKnown sourceLocation = issue.sourceLocation if let backtrace = issue.sourceContext.backtrace { @@ -48,3 +66,4 @@ extension ABIv0 { // MARK: - Codable extension ABIv0.EncodedIssue: Codable {} +extension ABIv0.EncodedIssue.Severity: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift index 5cfbf647c..cf44f0af0 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift @@ -25,6 +25,7 @@ extension ABIv0 { case `default` case skip case pass + case passWithWarnings = "_passWithWarnings" case passWithKnownIssue case fail case difference @@ -44,6 +45,8 @@ extension ABIv0 { } else { .pass } + case .passWithWarnings: + .passWithWarnings case .fail: .fail case .difference: diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 60e564d5a..b81f1c2c7 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -290,10 +290,7 @@ extension Event { if let configuration = configuration ?? Configuration.current { // The caller specified a configuration, or the current task has an // associated configuration. Post to either configuration's event handler. - switch kind { - case .expectationChecked where !configuration.deliverExpectationCheckedEvents: - break - default: + if configuration.eventHandlingOptions.shouldHandleEvent(self) { configuration.handleEvent(self, in: context) } } else { diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index cce3a732c..b375b2da1 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -162,6 +162,8 @@ extension Event.Symbol { return "\(_ansiEscapeCodePrefix)90m\(symbolCharacter)\(_resetANSIEscapeCode)" } return "\(_ansiEscapeCodePrefix)92m\(symbolCharacter)\(_resetANSIEscapeCode)" + case .passWithWarnings: + return "\(_ansiEscapeCodePrefix)93m\(symbolCharacter)\(_resetANSIEscapeCode)" case .fail: return "\(_ansiEscapeCodePrefix)91m\(symbolCharacter)\(_resetANSIEscapeCode)" case .warning: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 98303f11c..be92fe932 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -56,8 +56,9 @@ extension Event { /// The instant at which the test started. var startInstant: Test.Clock.Instant - /// The number of issues recorded for the test. - var issueCount = 0 + /// The number of issues recorded for the test, grouped by their + /// level of severity. + var issueCount: [Issue.Severity: Int] = [:] /// The number of known issues recorded for the test. var knownIssueCount = 0 @@ -114,27 +115,36 @@ extension Event.HumanReadableOutputRecorder { /// - graph: The graph to walk while counting issues. /// /// - Returns: A tuple containing the number of issues recorded in `graph`. - private func _issueCounts(in graph: Graph?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { + private func _issueCounts(in graph: Graph?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { guard let graph else { - return (0, 0, 0, "") + return (0, 0, 0, 0, "") } - let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=) + let errorIssueCount = graph.compactMap { $0.value?.issueCount[.error] }.reduce(into: 0, +=) + let warningIssueCount = graph.compactMap { $0.value?.issueCount[.warning] }.reduce(into: 0, +=) let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=) - let totalIssueCount = issueCount + knownIssueCount + let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount // Construct a string describing the issue counts. - let description = switch (issueCount > 0, knownIssueCount > 0) { - case (true, true): + let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) { + case (true, true, true): + " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))" + case (true, false, true): " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))" - case (false, true): + case (false, true, true): + " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))" + case (false, false, true): " with \(knownIssueCount.counting("known issue"))" - case (true, false): + case (true, true, false): + " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))" + case (true, false, false): " with \(totalIssueCount.counting("issue"))" - case(false, false): + case(false, true, false): + " with \(warningIssueCount.counting("warning"))" + case(false, false, false): "" } - return (issueCount, knownIssueCount, totalIssueCount, description) + return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } } @@ -267,7 +277,8 @@ extension Event.HumanReadableOutputRecorder { if issue.isKnown { testData.knownIssueCount += 1 } else { - testData.issueCount += 1 + let issueCount = testData.issueCount[issue.severity] ?? 0 + testData.issueCount[issue.severity] = issueCount + 1 } context.testData[id] = testData @@ -355,7 +366,7 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - return if issues.issueCount > 0 { + return if issues.errorIssueCount > 0 { CollectionOfOne( Message( symbol: .fail, @@ -363,7 +374,7 @@ extension Event.HumanReadableOutputRecorder { ) ) + _formattedComments(for: test) } else { - [ + [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)." @@ -400,13 +411,19 @@ extension Event.HumanReadableOutputRecorder { "" } let symbol: Event.Symbol - let known: String + let subject: String if issue.isKnown { symbol = .pass(knownIssueCount: 1) - known = " known" + subject = "a known issue" } else { - symbol = .fail - known = "n" + switch issue.severity { + case .warning: + symbol = .passWithWarnings + subject = "a warning" + case .error: + symbol = .fail + subject = "an issue" + } } var additionalMessages = [Message]() @@ -435,13 +452,13 @@ extension Event.HumanReadableOutputRecorder { let primaryMessage: Message = if parameterCount == 0 { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } else { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } @@ -498,7 +515,7 @@ extension Event.HumanReadableOutputRecorder { let runStartInstant = context.runStartInstant ?? instant let duration = runStartInstant.descriptionOfDuration(to: instant) - return if issues.issueCount > 0 { + return if issues.errorIssueCount > 0 { [ Message( symbol: .fail, diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index 0f50ed95c..3a3f6df8e 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -22,10 +22,14 @@ extension Event { /// The symbol to use when a test passes. /// /// - Parameters: - /// - knownIssueCount: The number of known issues encountered by the end - /// of the test. + /// - knownIssueCount: The number of known issues recorded for the test. + /// The default value is `0`. case pass(knownIssueCount: Int = 0) + /// The symbol to use when a test passes with one or more warnings. + @_spi(Experimental) + case passWithWarnings + /// The symbol to use when a test fails. case fail @@ -62,6 +66,8 @@ extension Event.Symbol { } else { ("\u{10105B}", "checkmark.diamond.fill") } + case .passWithWarnings: + ("\u{100123}", "questionmark.diamond.fill") case .fail: ("\u{100884}", "xmark.diamond.fill") case .difference: @@ -122,6 +128,9 @@ extension Event.Symbol { // Unicode: HEAVY CHECK MARK return "\u{2714}" } + case .passWithWarnings: + // Unicode: QUESTION MARK + return "\u{003F}" case .fail: // Unicode: HEAVY BALLOT X return "\u{2718}" @@ -157,6 +166,9 @@ extension Event.Symbol { // Unicode: SQUARE ROOT return "\u{221A}" } + case .passWithWarnings: + // Unicode: QUESTION MARK + return "\u{003F}" case .fail: // Unicode: MULTIPLICATION SIGN return "\u{00D7}" diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index dae58400a..5d7449b7b 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -79,6 +79,32 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind + /// An enumeration representing the level of severity of a recorded issue. + /// + /// The supported levels, in increasing order of severity, are: + /// + /// - ``warning`` + /// - ``error`` + @_spi(Experimental) + public enum Severity: Sendable { + /// The severity level for an issue which should be noted but is not + /// necessarily an error. + /// + /// An issue with warning severity does not cause the test it's associated + /// with to be marked as a failure, but is noted in the results. + case warning + + /// The severity level for an issue which represents an error in a test. + /// + /// An issue with error severity causes the test it's associated with to be + /// marked as a failure. + case error + } + + /// The severity of this issue. + @_spi(Experimental) + public var severity: Severity + /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -97,16 +123,20 @@ public struct Issue: Sendable { /// /// - Parameters: /// - kind: The kind of issue this value represents. + /// - severity: The severity of this issue. The default value is + /// ``Severity-swift.enum/error``. /// - comments: An array of comments describing the issue. This array may be /// empty. /// - sourceContext: A ``SourceContext`` indicating where and how this issue /// occurred. init( kind: Kind, + severity: Severity = .error, comments: [Comment], sourceContext: SourceContext ) { self.kind = kind + self.severity = severity self.comments = comments self.sourceContext = sourceContext } @@ -154,27 +184,31 @@ public struct Issue: Sendable { } } +extension Issue.Severity: Comparable {} + // MARK: - CustomStringConvertible, CustomDebugStringConvertible extension Issue: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - if comments.isEmpty { - return String(describing: kind) + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind): \(joinedComments)" + return "\(kind) (\(severity))\(joinedComments)" } public var debugDescription: String { - if comments.isEmpty { - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments: String = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" } } @@ -234,6 +268,17 @@ extension Issue.Kind: CustomStringConvertible { } } +extension Issue.Severity: CustomStringConvertible { + public var description: String { + switch self { + case .warning: + "warning" + case .error: + "error" + } + } +} + #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting @@ -244,6 +289,10 @@ extension Issue { /// The kind of issue this value represents. public var kind: Kind.Snapshot + /// The severity of this issue. + @_spi(Experimental) + public var severity: Severity + /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -268,10 +317,22 @@ extension Issue { self.kind = Issue.Kind.Snapshot(snapshotting: issue.kind) self.comments = issue.comments } + self.severity = issue.severity self.sourceContext = issue.sourceContext self.isKnown = issue.isKnown } + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.kind = try container.decode(Issue.Kind.Snapshot.self, forKey: .kind) + self.comments = try container.decode([Comment].self, forKey: .comments) + self.sourceContext = try container.decode(SourceContext.self, forKey: .sourceContext) + self.isKnown = try container.decode(Bool.self, forKey: .isKnown) + + // Severity is a new field, so fall back to .error if it's not present. + self.severity = try container.decodeIfPresent(Issue.Severity.self, forKey: .severity) ?? .error + } + /// The error which was associated with this issue, if any. /// /// The value of this property is non-`nil` when ``kind-swift.property`` is @@ -295,6 +356,8 @@ extension Issue { } } +extension Issue.Severity: Codable {} + extension Issue.Kind { /// Serializable kinds of issues which may be recorded. @_spi(ForToolsIntegrationOnly) @@ -478,23 +541,25 @@ extension Issue.Kind { extension Issue.Snapshot: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - if comments.isEmpty { - return String(describing: kind) + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind): \(joinedComments)" + return "\(kind) (\(severity))\(joinedComments)" } public var debugDescription: String { - if comments.isEmpty { - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" + let joinedComments = if comments.isEmpty { + "" + } else { + ": " + comments.lazy + .map(\.rawValue) + .joined(separator: "\n") } - let joinedComments: String = comments.lazy - .map(\.rawValue) - .joined(separator: "\n") - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" } } diff --git a/Sources/Testing/Running/Configuration+EventHandling.swift b/Sources/Testing/Running/Configuration+EventHandling.swift index 025f07d2c..e3c189f8b 100644 --- a/Sources/Testing/Running/Configuration+EventHandling.swift +++ b/Sources/Testing/Running/Configuration+EventHandling.swift @@ -38,3 +38,23 @@ extension Configuration { return eventHandler(event, contextCopy) } } + +extension Configuration.EventHandlingOptions { + /// Determine whether the specified event should be handled according to the + /// options in this instance. + /// + /// - Parameters: + /// - event: The event to consider handling. + /// + /// - Returns: Whether or not the event should be handled or suppressed. + func shouldHandleEvent(_ event: borrowing Event) -> Bool { + switch event.kind { + case let .issueRecorded(issue): + issue.severity > .warning || isWarningIssueRecordedEventEnabled + case .expectationChecked: + isExpectationCheckedEventEnabled + default: + true + } + } +} diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index f4ae59813..a917c2f5b 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -178,14 +178,33 @@ public struct Configuration: Sendable { // MARK: - Event handling - /// Whether or not events of the kind - /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to - /// this configuration's ``eventHandler`` closure. - /// - /// By default, events of this kind are not delivered to event handlers - /// because they occur frequently in a typical test run and can generate - /// significant backpressure on the event handler. - public var deliverExpectationCheckedEvents: Bool = false + /// A type describing options to use when delivering events to this + /// configuration's event handler + public struct EventHandlingOptions: Sendable { + /// Whether or not events of the kind ``Event/Kind-swift.enum/issueRecorded(_:)`` + /// containing issues with warning (or lower) severity should be delivered + /// to the event handler of the configuration these options are applied to. + /// + /// By default, events matching this criteria are not delivered to event + /// handlers since this is an experimental feature. + /// + /// - Warning: Warning issues are not yet an approved feature. + @_spi(Experimental) + public var isWarningIssueRecordedEventEnabled: Bool = false + + /// Whether or not events of the kind + /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to + /// the event handler of the configuration these options are applied to. + /// + /// By default, events of this kind are not delivered to event handlers + /// because they occur frequently in a typical test run and can generate + /// significant back-pressure on the event handler. + public var isExpectationCheckedEventEnabled: Bool = false + } + + /// The options to use when delivering events to this configuration's event + /// handler. + public var eventHandlingOptions: EventHandlingOptions = .init() /// The event handler to which events should be passed when they occur. public var eventHandler: Event.Handler = { _, _ in } @@ -325,4 +344,14 @@ extension Configuration { } } #endif + + @available(*, deprecated, message: "Set eventHandlingOptions.isExpectationCheckedEventEnabled instead.") + public var deliverExpectationCheckedEvents: Bool { + get { + eventHandlingOptions.isExpectationCheckedEventEnabled + } + set { + eventHandlingOptions.isExpectationCheckedEventEnabled = newValue + } + } } diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index f69e13cd6..9ae299412 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -132,7 +132,7 @@ extension Configuration { /// - Returns: A unique number identifying `self` that can be /// passed to `_removeFromAll(identifiedBy:)`` to unregister it. private func _addToAll() -> UInt64 { - if deliverExpectationCheckedEvents { + if eventHandlingOptions.isExpectationCheckedEventEnabled { Self._deliverExpectationCheckedEventsCount.increment() } return Self._all.withLock { all in @@ -152,16 +152,14 @@ extension Configuration { let configuration = Self._all.withLock { all in all.instances.removeValue(forKey: id) } - if let configuration, configuration.deliverExpectationCheckedEvents { + if let configuration, configuration.eventHandlingOptions.isExpectationCheckedEventEnabled { Self._deliverExpectationCheckedEventsCount.decrement() } } /// An atomic counter that tracks the number of "current" configurations that - /// have set ``deliverExpectationCheckedEvents`` to `true`. - /// - /// On older Apple platforms, this property is not available and ``all`` is - /// directly consulted instead (which is less efficient.) + /// have set ``EventHandlingOptions/isExpectationCheckedEventEnabled`` to + /// `true`. private static let _deliverExpectationCheckedEventsCount = Locked(rawValue: 0) /// Whether or not events of the kind @@ -171,7 +169,8 @@ extension Configuration { /// /// To determine if an individual instance of ``Configuration`` is listening /// for these events, consult the per-instance - /// ``Configuration/deliverExpectationCheckedEvents`` property. + /// ``Configuration/EventHandlingOptions/isExpectationCheckedEventEnabled`` + /// property. static var deliverExpectationCheckedEvents: Bool { _deliverExpectationCheckedEventsCount.rawValue > 0 } diff --git a/Tests/TestingTests/ConfigurationTests.swift b/Tests/TestingTests/ConfigurationTests.swift new file mode 100644 index 000000000..a735d8ac5 --- /dev/null +++ b/Tests/TestingTests/ConfigurationTests.swift @@ -0,0 +1,25 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Configuration Tests") +struct ConfigurationTests { + @Test + @available(*, deprecated, message: "Testing a deprecated SPI.") + func deliverExpectationCheckedEventsProperty() throws { + var configuration = Configuration() + #expect(!configuration.deliverExpectationCheckedEvents) + #expect(!configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) + + configuration.deliverExpectationCheckedEvents = true + #expect(configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) + } +} diff --git a/Tests/TestingTests/EntryPointTests.swift b/Tests/TestingTests/EntryPointTests.swift new file mode 100644 index 000000000..eae7d4b7e --- /dev/null +++ b/Tests/TestingTests/EntryPointTests.swift @@ -0,0 +1,81 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +private import _TestingInternals + +@Suite("Entry point tests") +struct EntryPointTests { + @Test("Entry point filter with filtering of hidden tests enabled") + func hiddenTests() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_someHiddenTest"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.verbosity = .min + + await confirmation("Test event started", expectedCount: 1) { testMatched in + _ = await entryPoint(passing: arguments) { event, context in + if case .testStarted = event.kind { + testMatched() + } + } + } + } + + @Test("Entry point with WarningIssues feature enabled exits with success if all issues have severity < .error") + func warningIssues() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_recordWarningIssue"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.verbosity = .min + + let exitCode = await confirmation("Test matched", expectedCount: 1) { testMatched in + await entryPoint(passing: arguments) { event, context in + if case .testStarted = event.kind { + testMatched() + } else if case let .issueRecorded(issue) = event.kind { + Issue.record("Unexpected issue \(issue) was recorded.") + } + } + } + #expect(exitCode == EXIT_SUCCESS) + } + + @Test("Entry point with WarningIssues feature enabled propagates warning issues and exits with success if all issues have severity < .error") + func warningIssuesEnabled() async throws { + var arguments = __CommandLineArguments_v0() + arguments.filter = ["_recordWarningIssue"] + arguments.includeHiddenTests = true + arguments.eventStreamVersion = 0 + arguments.isWarningIssueRecordedEventEnabled = true + arguments.verbosity = .min + + let exitCode = await confirmation("Warning issue recorded", expectedCount: 1) { issueRecorded in + await entryPoint(passing: arguments) { event, context in + if case let .issueRecorded(issue) = event.kind { + #expect(issue.severity == .warning) + issueRecorded() + } + } + } + #expect(exitCode == EXIT_SUCCESS) + } +} + +// MARK: - Fixtures + +@Test(.hidden) private func _someHiddenTest() {} + +@Test(.hidden) private func _recordWarningIssue() { + // Intentionally _only_ record issues with warning (or lower) severity. + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() +} diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 97619b755..6b5b9bd81 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -59,7 +59,7 @@ struct EventRecorderTests { } var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(options: options, writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -98,7 +98,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -123,7 +123,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -183,15 +183,20 @@ struct EventRecorderTests { @Test( "Issue counts are summed correctly on test end", arguments: [ - ("f()", false, (total: 5, expected: 3)), - ("g()", false, (total: 2, expected: 1)), - ("PredictablyFailingTests", true, (total: 7, expected: 4)), + ("f()", #".* Test f\(\) failed after .+ seconds with 5 issues \(including 3 known issues\)\."#), + ("g()", #".* Test g\(\) failed after .+ seconds with 2 issues \(including 1 known issue\)\."#), + ("h()", #".* Test h\(\) passed after .+ seconds with 1 warning\."#), + ("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#), + ("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#), + ("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#), ] ) - func issueCountSummingAtTestEnd(testName: String, isSuite: Bool, issueCount: (total: Int, expected: Int)) async throws { + func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws { let stream = Stream() var configuration = Configuration() + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -204,28 +209,13 @@ struct EventRecorderTests { print(buffer, terminator: "") } - let testFailureRegex = Regex { - One(.anyGraphemeCluster) - " \(isSuite ? "Suite" : "Test") \(testName) failed " - ZeroOrMore(.any) - " with " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " issue" - Optionally("s") - " (including " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " known issue" - Optionally("s") - ")." - } - let match = try #require( - buffer - .split(whereSeparator: \.isNewline) - .compactMap(testFailureRegex.wholeMatch(in:)) - .first + let expectedSuffixRegex = try Regex(expectedPattern) + #expect(try buffer + .split(whereSeparator: \.isNewline) + .compactMap(expectedSuffixRegex.wholeMatch(in:)) + .first != nil, + "buffer: \(buffer)" ) - #expect(issueCount.total == match.output.1) - #expect(issueCount.expected == match.output.2) } #endif @@ -294,8 +284,51 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 7) - #expect(match.output.2 == 4) + #expect(match.output.1 == 9) + #expect(match.output.2 == 5) + } + + @Test("Issue counts are summed correctly on run end for a test with only warning issues") + @available(_regexAPI, *) + func warningIssueCountSummingAtRunEnd() async throws { + let stream = Stream() + + var configuration = Configuration() + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true + let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) + configuration.eventHandler = { event, context in + eventRecorder.record(event, in: context) + } + + await runTestFunction(named: "h()", in: PredictablyFailingTests.self, configuration: configuration) + + let buffer = stream.buffer.rawValue + if testsWithSignificantIOAreEnabled { + print(buffer, terminator: "") + } + + let runFailureRegex = Regex { + One(.anyGraphemeCluster) + " Test run with " + OneOrMore(.digit) + " test" + Optionally("s") + " passed " + ZeroOrMore(.any) + " with " + Capture { OneOrMore(.digit) } transform: { Int($0) } + " warning" + Optionally("s") + "." + } + let match = try #require( + buffer + .split(whereSeparator: \.isNewline) + .compactMap(runFailureRegex.wholeMatch(in:)) + .first, + "buffer: \(buffer)" + ) + #expect(match.output.1 == 1) } #endif @@ -308,7 +341,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true let eventRecorder = Event.JUnitXMLRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -510,4 +543,26 @@ struct EventRecorderTests { #expect(Bool(false)) } } + + @Test(.hidden) func h() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + } + + @Test(.hidden) func i() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + #expect(Bool(false)) + } + + @Test(.hidden) func j() { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + withKnownIssue { + #expect(Bool(false)) + } + } + + @Test(.hidden) func k() { + withKnownIssue { + Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() + } + } } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 8b7d86358..f73676d42 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -301,7 +301,7 @@ final class IssueTests: XCTestCase { let expectationChecked = expectation(description: "expectation checked") var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return @@ -1124,12 +1124,12 @@ final class IssueTests: XCTestCase { do { let sourceLocation = SourceLocation.init(fileID: "FakeModule/FakeFile.swift", filePath: "", line: 9999, column: 1) let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: sourceLocation)) - XCTAssertEqual(issue.description, "A system failure occurred: Some issue") - XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1: Some issue") + XCTAssertEqual(issue.description, "A system failure occurred (error): Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1 (error): Some issue") } do { let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: nil)) - XCTAssertEqual(issue.debugDescription, "A system failure occurred: Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred (error): Some issue") } } diff --git a/Tests/TestingTests/Runner.RuntimeStateTests.swift b/Tests/TestingTests/Runner.RuntimeStateTests.swift index e4ee33079..1576c49e7 100644 --- a/Tests/TestingTests/Runner.RuntimeStateTests.swift +++ b/Tests/TestingTests/Runner.RuntimeStateTests.swift @@ -34,7 +34,7 @@ struct Runner_RuntimeStateTests { // an event to be posted during the test below without causing any real // issues to be recorded or otherwise confuse the testing harness. var configuration = Configuration.current ?? .init() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true await Configuration.withCurrent(configuration) { await withTaskGroup(of: Void.self) { group in diff --git a/Tests/TestingTests/RunnerTests.swift b/Tests/TestingTests/RunnerTests.swift index 857cfdd81..335f8be37 100644 --- a/Tests/TestingTests/RunnerTests.swift +++ b/Tests/TestingTests/RunnerTests.swift @@ -426,7 +426,7 @@ final class RunnerTests: XCTestCase { func testExpectationCheckedEventHandlingWhenDisabled() async { var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = false + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = false configuration.eventHandler = { event, _ in if case .expectationChecked = event.kind { XCTFail("Expectation checked event was posted unexpectedly") @@ -459,7 +459,7 @@ final class RunnerTests: XCTestCase { #endif var configuration = Configuration() - configuration.deliverExpectationCheckedEvents = true + configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return From 74a8c52fb312f99e951d5fa2c0662d702e0711e4 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 12 Feb 2025 13:39:48 -0500 Subject: [PATCH 079/234] Some fixes/cleanup to TestContent.md (#948) Proofreading is your friend. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index d6eacf20f..9b981e5c0 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -75,13 +75,13 @@ struct SWTTestContentRecord { }; ``` -Do not use the `__TestContentRecord` typealias defined in the testing library. -This type exists to support the testing library's macros and may change in the -future (e.g. to accomodate a generic argument or to make use of one of the -reserved fields.) +Do not use the `__TestContentRecord` or `__TestContentRecordAccessor` typealias +defined in the testing library. These types exist to support the testing +library's macros and may change in the future (e.g. to accomodate a generic +argument or to make use of a reserved field.) -Instead, define your own copy of this type where needed—you can copy the -definition above _verbatim_. If your test record type's `context` field (as +Instead, define your own copy of these types where needed—you can copy the +definitions above _verbatim_. If your test record type's `context` field (as described below) is a pointer type, make sure to change its type in your version of `TestContentRecord` accordingly so that, on systems with pointer authentication enabled, the pointer is correctly resigned at load time. @@ -108,16 +108,16 @@ other fields in that record are undefined. #### The accessor field The function `accessor` is a C function. When called, it initializes the memory -at its argument `outValue` to an instance of some Swift type and returns `true`, -or returns `false` if it could not generate the relevant content. On successful -return, the caller is responsible for deinitializing the memory at `outValue` -when done with it. +at `outValue` to an instance of the appropriate type and returns `true`; or, if +it could not generate the relevant content, it leaves the memory uninitialized +and returns `false`. On successful return, the caller is responsible for +deinitializing the memory at `outValue` when done with it. If `accessor` is `nil`, the test content record is ignored. The testing library may, in the future, define record kinds that do not provide an accessor function (that is, they represent pure compile-time information only.) -The second argument to this function, `type`, is a pointer to the type[^mightNotBeSwift] +The third argument to this function, `type`, is a pointer to the type[^mightNotBeSwift] (not a bitcast Swift type) of the value expected to be written to `outValue`. This argument helps to prevent memory corruption if two copies of Swift Testing or a third-party library are inadvertently loaded into the same process. If the @@ -131,9 +131,10 @@ accessor function must return `false` and must not modify `outValue`. other than Swift is beyond the scope of this document. With that in mind, it is _technically_ feasible for a test content accessor to be written in (for example) C++, expect the `type` argument to point to a C++ value of type - `std::type_info`, and write a C++ class instance to `outValue`. + [`std::type_info`](https://en.cppreference.com/w/cpp/types/type_info), and + write a C++ class instance to `outValue` using [placement `new`](https://en.cppreference.com/w/cpp/language/new#Placement_new). -The third argument to this function, `hint`, is an optional input that can be +The fourth argument to this function, `hint`, is an optional input that can be passed to help the accessor function determine if its corresponding test content record matches what the caller is looking for. If the caller passes `nil` as the `hint` argument, the accessor behaves as if it matched (that is, no additional @@ -170,8 +171,8 @@ by `type`, and the value pointed to by `hint` depend on the kind of record: > and/or [`withUnsafePointer(to:_:)`](https://developer.apple.com/documentation/swift/withunsafepointer(to:_:)-35wrn) > to ensure the pointers passed to `accessor` are large enough and are > well-aligned. If they are not large enough to contain values of the -> appropriate types (per above), or if `hint` points to uninitialized or -> incorrectly-typed memory, the result is undefined. +> appropriate types (per above), or if `type` or `hint` points to uninitialized +> or incorrectly-typed memory, the result is undefined. #### The context field From 7493d6e3600ce249519d405d6ea7db780c858ea0 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 12 Feb 2025 16:30:34 -0500 Subject: [PATCH 080/234] Move JSON ABI source files to prepare for v1. (#953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At this time, we expect most of the "ABIv1" JSON schema will be the same as the current "ABIv0" one, so this PR reshuffles the source files supporting it so they can be more readily reused between the two. Future PRs will start adding actual "ABIv1" functionality—this is just a bookkeeping PR. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/JSON.md | 2 +- ...aming.swift => ABI.Record+Streaming.swift} | 2 +- .../ABIv0.Record.swift => ABI.Record.swift} | 8 ++--- Sources/Testing/ABI/ABI.swift | 30 +++++++++++++++++++ .../ABI.EncodedAttachment.swift} | 4 +-- .../ABI.EncodedBacktrace.swift} | 4 +-- .../ABI.EncodedError.swift} | 8 ++--- .../ABI.EncodedEvent.swift} | 6 ++-- .../ABI.EncodedInstant.swift} | 4 +-- .../ABI.EncodedIssue.swift} | 6 ++-- .../ABI.EncodedMessage.swift} | 6 ++-- .../ABI.EncodedTest.swift} | 10 +++---- .../ABI/EntryPoints/ABIEntryPoint.swift | 12 ++++---- .../Testing/ABI/EntryPoints/EntryPoint.swift | 4 +-- Sources/Testing/ABI/v0/ABIv0.swift | 13 -------- Sources/Testing/CMakeLists.txt | 22 +++++++------- Sources/Testing/ExitTests/ExitTest.swift | 4 +-- Tests/TestingTests/ABIEntryPointTests.swift | 10 +++---- Tests/TestingTests/SwiftPMTests.swift | 4 +-- 19 files changed, 88 insertions(+), 71 deletions(-) rename Sources/Testing/ABI/{v0/ABIv0.Record+Streaming.swift => ABI.Record+Streaming.swift} (99%) rename Sources/Testing/ABI/{v0/ABIv0.Record.swift => ABI.Record.swift} (92%) create mode 100644 Sources/Testing/ABI/ABI.swift rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedAttachment.swift => Encoded/ABI.EncodedAttachment.swift} (94%) rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedBacktrace.swift => Encoded/ABI.EncodedBacktrace.swift} (96%) rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedError.swift => Encoded/ABI.EncodedError.swift} (90%) rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedEvent.swift => Encoded/ABI.EncodedEvent.swift} (97%) rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedInstant.swift => Encoded/ABI.EncodedInstant.swift} (95%) rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedIssue.swift => Encoded/ABI.EncodedIssue.swift} (95%) rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedMessage.swift => Encoded/ABI.EncodedMessage.swift} (95%) rename Sources/Testing/ABI/{v0/Encoded/ABIv0.EncodedTest.swift => Encoded/ABI.EncodedTest.swift} (95%) delete mode 100644 Sources/Testing/ABI/v0/ABIv0.swift diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index 23dd5b1dc..2fac0c3fb 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -13,7 +13,7 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors This document outlines the JSON schemas used by the testing library for its ABI entry point and for the `--event-stream-output-path` command-line argument. For more information about the ABI entry point, see the documentation for -[ABIv0.EntryPoint](https://github.com/search?q=repo%3Aapple%2Fswift-testing%EntryPoint&type=code). +[ABI.v0.EntryPoint](https://github.com/search?q=repo%3Aapple%2Fswift-testing%EntryPoint&type=code). ## Modified Backus-Naur form diff --git a/Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift similarity index 99% rename from Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift rename to Sources/Testing/ABI/ABI.Record+Streaming.swift index 0c0b4ddba..482d5af18 100644 --- a/Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) && (!SWT_NO_FILE_IO || !SWT_NO_ABI_ENTRY_POINT) -extension ABIv0.Record { +extension ABI.Record { /// Post-process encoded JSON and write it to a file. /// /// - Parameters: diff --git a/Sources/Testing/ABI/v0/ABIv0.Record.swift b/Sources/Testing/ABI/ABI.Record.swift similarity index 92% rename from Sources/Testing/ABI/v0/ABIv0.Record.swift rename to Sources/Testing/ABI/ABI.Record.swift index 520728b76..6dcccd6f9 100644 --- a/Sources/Testing/ABI/v0/ABIv0.Record.swift +++ b/Sources/Testing/ABI/ABI.Record.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of records for the ABI entry point /// and event stream output. /// @@ -48,7 +48,7 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.Record: Codable { +extension ABI.Record: Codable { private enum CodingKeys: String, CodingKey { case version case kind @@ -73,10 +73,10 @@ extension ABIv0.Record: Codable { version = try container.decode(Int.self, forKey: .version) switch try container.decode(String.self, forKey: .kind) { case "test": - let test = try container.decode(ABIv0.EncodedTest.self, forKey: .payload) + let test = try container.decode(ABI.EncodedTest.self, forKey: .payload) kind = .test(test) case "event": - let event = try container.decode(ABIv0.EncodedEvent.self, forKey: .payload) + let event = try container.decode(ABI.EncodedEvent.self, forKey: .payload) kind = .event(event) case let kind: throw DecodingError.dataCorrupted(.init(codingPath: decoder.codingPath, debugDescription: "Unrecognized record kind '\(kind)'")) diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift new file mode 100644 index 000000000..cd952acca --- /dev/null +++ b/Sources/Testing/ABI/ABI.swift @@ -0,0 +1,30 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A namespace for ABI symbols. +@_spi(ForToolsIntegrationOnly) +public enum ABI: Sendable {} + +// MARK: - + +@_spi(ForToolsIntegrationOnly) +extension ABI { + /// A namespace for ABI version 0 symbols. + public enum v0: Sendable {} + + /// A namespace for ABI version 1 symbols. + @_spi(Experimental) + public enum v1: Sendable {} +} + +/// A namespace for ABI version 0 symbols. +@_spi(ForToolsIntegrationOnly) +@available(*, deprecated, renamed: "ABI.v0") +public typealias ABIv0 = ABI.v0 diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedAttachment.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift similarity index 94% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedAttachment.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift index c3e739281..e9cbdeacd 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedAttachment.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Attachment`` for the ABI entry /// point and event stream output. /// @@ -29,4 +29,4 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.EncodedAttachment: Codable {} +extension ABI.EncodedAttachment: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedBacktrace.swift similarity index 96% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedBacktrace.swift index 171cae5d0..9146a040a 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedBacktrace.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Backtrace`` for the ABI entry /// point and event stream output. /// @@ -33,7 +33,7 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.EncodedBacktrace: Codable { +extension ABI.EncodedBacktrace: Codable { func encode(to encoder: any Encoder) throws { try symbolicatedAddresses.encode(to: encoder) } diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedError.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedError.swift similarity index 90% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedError.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedError.swift index c85dd1ba4..eb492b7d1 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedError.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedError.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Error`` for the ABI entry point /// and event stream output. /// @@ -39,7 +39,7 @@ extension ABIv0 { // MARK: - Error -extension ABIv0.EncodedError: Error { +extension ABI.EncodedError: Error { var _domain: String { domain } @@ -56,11 +56,11 @@ extension ABIv0.EncodedError: Error { // MARK: - Codable -extension ABIv0.EncodedError: Codable {} +extension ABI.EncodedError: Codable {} // MARK: - CustomTestStringConvertible -extension ABIv0.EncodedError: CustomTestStringConvertible { +extension ABI.EncodedError: CustomTestStringConvertible { var testDescription: String { description } diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift similarity index 97% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index fd9dc464a..f502873f3 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Event`` for the ABI entry point /// and event stream output. /// @@ -109,5 +109,5 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.EncodedEvent: Codable {} -extension ABIv0.EncodedEvent.Kind: Codable {} +extension ABI.EncodedEvent: Codable {} +extension ABI.EncodedEvent.Kind: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedInstant.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedInstant.swift similarity index 95% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedInstant.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedInstant.swift index 0693df519..1f05e4241 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedInstant.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedInstant.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Test/Clock/Instant`` for the /// ABI entry point and event stream output. /// @@ -37,4 +37,4 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.EncodedInstant: Codable {} +extension ABI.EncodedInstant: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift similarity index 95% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift index 97c051d28..3529dfc98 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Issue`` for the ABI entry point /// and event stream output. /// @@ -65,5 +65,5 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.EncodedIssue: Codable {} -extension ABIv0.EncodedIssue.Severity: Codable {} +extension ABI.EncodedIssue: Codable {} +extension ABI.EncodedIssue.Severity: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift similarity index 95% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift index cf44f0af0..a3aa9575d 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of /// ``Event/HumanReadableOutputRecorder/Message`` for the ABI entry point and /// event stream output. @@ -76,5 +76,5 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.EncodedMessage: Codable {} -extension ABIv0.EncodedMessage.Symbol: Codable {} +extension ABI.EncodedMessage: Codable {} +extension ABI.EncodedMessage.Symbol: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift similarity index 95% rename from Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedTest.swift rename to Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index e7ad7dbb5..533c8bb12 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Test`` for the ABI entry point /// and event stream output. /// @@ -92,7 +92,7 @@ extension ABIv0 { } } -extension ABIv0 { +extension ABI { /// A type implementing the JSON encoding of ``Test/Case`` for the ABI entry /// point and event stream output. /// @@ -120,6 +120,6 @@ extension ABIv0 { // MARK: - Codable -extension ABIv0.EncodedTest: Codable {} -extension ABIv0.EncodedTest.Kind: Codable {} -extension ABIv0.EncodedTestCase: Codable {} +extension ABI.EncodedTest: Codable {} +extension ABI.EncodedTest.Kind: Codable {} +extension ABI.EncodedTestCase: Codable {} diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index 143f7b549..e0063bdd4 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -11,7 +11,7 @@ #if canImport(Foundation) && !SWT_NO_ABI_ENTRY_POINT private import _TestingInternals -extension ABIv0 { +extension ABI.v0 { /// The type of the entry point to the testing library used by tools that want /// to remain version-agnostic regarding the testing library. /// @@ -62,7 +62,7 @@ extension ABIv0 { /// untyped pointer. @_cdecl("swt_abiv0_getEntryPoint") @usableFromInline func abiv0_getEntryPoint() -> UnsafeRawPointer { - unsafeBitCast(ABIv0.entryPoint, to: UnsafeRawPointer.self) + unsafeBitCast(ABI.v0.entryPoint, to: UnsafeRawPointer.self) } #if !SWT_NO_SNAPSHOT_TYPES @@ -72,7 +72,7 @@ extension ABIv0 { /// Beta 1. /// /// This type will be removed in a future update. -@available(*, deprecated, message: "Use ABIv0.EntryPoint instead.") +@available(*, deprecated, message: "Use ABI.v0.EntryPoint instead.") typealias ABIEntryPoint_v0 = @Sendable ( _ argumentsJSON: UnsafeRawBufferPointer?, _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void @@ -82,7 +82,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( /// Xcode 16 Beta 1. /// /// This function will be removed in a future update. -@available(*, deprecated, message: "Use ABIv0.entryPoint (swt_abiv0_getEntryPoint()) instead.") +@available(*, deprecated, message: "Use ABI.v0.entryPoint (swt_abiv0_getEntryPoint()) instead.") @_cdecl("swt_copyABIEntryPoint_v0") @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) @@ -124,8 +124,8 @@ private func _entryPoint( let exitCode = await entryPoint(passing: args, eventHandler: eventHandler) // To maintain compatibility with Xcode 16 Beta 1, suppress custom exit codes. - // (This is also needed by ABIv0.entryPoint to correctly treat the no-tests as - // a successful run.) + // (This is also needed by ABI.v0.entryPoint to correctly treat the no-tests + // as a successful run.) if exitCode == EXIT_NO_TESTS_FOUND { return EXIT_SUCCESS } diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 89094c88f..f6e2cd602 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -612,8 +612,8 @@ func eventHandlerForStreamingEvents( // will be removed in a future update. Do not use it. eventHandlerForStreamingEventSnapshots(to: eventHandler) #endif - case nil, 0: - ABIv0.Record.eventHandler(encodeAsJSONLines: encodeAsJSONLines, forwardingTo: eventHandler) + case nil, 0, 1: + ABI.Record.eventHandler(encodeAsJSONLines: encodeAsJSONLines, forwardingTo: eventHandler) case let .some(unsupportedVersion): throw _EntryPointError.invalidArgument("--event-stream-version", value: "\(unsupportedVersion)") } diff --git a/Sources/Testing/ABI/v0/ABIv0.swift b/Sources/Testing/ABI/v0/ABIv0.swift deleted file mode 100644 index 0e8f5db9f..000000000 --- a/Sources/Testing/ABI/v0/ABIv0.swift +++ /dev/null @@ -1,13 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// A namespace for ABI version 0 symbols. -@_spi(ForToolsIntegrationOnly) -public enum ABIv0: Sendable {} diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 16efd0ee6..e40cb1b0b 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -10,17 +10,17 @@ add_library(Testing ABI/EntryPoints/ABIEntryPoint.swift ABI/EntryPoints/EntryPoint.swift ABI/EntryPoints/SwiftPMEntryPoint.swift - ABI/v0/ABIv0.Record.swift - ABI/v0/ABIv0.Record+Streaming.swift - ABI/v0/ABIv0.swift - ABI/v0/Encoded/ABIv0.EncodedAttachment.swift - ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift - ABI/v0/Encoded/ABIv0.EncodedError.swift - ABI/v0/Encoded/ABIv0.EncodedEvent.swift - ABI/v0/Encoded/ABIv0.EncodedInstant.swift - ABI/v0/Encoded/ABIv0.EncodedIssue.swift - ABI/v0/Encoded/ABIv0.EncodedMessage.swift - ABI/v0/Encoded/ABIv0.EncodedTest.swift + ABI/ABI.Record.swift + ABI/ABI.Record+Streaming.swift + ABI/ABI.swift + ABI/Encoded/ABI.EncodedAttachment.swift + ABI/Encoded/ABI.EncodedBacktrace.swift + ABI/Encoded/ABI.EncodedError.swift + ABI/Encoded/ABI.EncodedEvent.swift + ABI/Encoded/ABI.EncodedInstant.swift + ABI/Encoded/ABI.EncodedIssue.swift + ABI/Encoded/ABI.EncodedMessage.swift + ABI/Encoded/ABI.EncodedTest.swift Attachments/Attachable.swift Attachments/AttachableContainer.swift Attachments/Attachment.swift diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index f6ea88d22..416a0fc29 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -444,7 +444,7 @@ extension ExitTest { // Encode events as JSON and write them to the back channel file handle. // Only forward issue-recorded events. (If we start handling other kinds of // events in the future, we can forward them too.) - let eventHandler = ABIv0.Record.eventHandler(encodeAsJSONLines: true) { json in + let eventHandler = ABI.Record.eventHandler(encodeAsJSONLines: true) { json in _ = try? _backChannelForEntryPoint?.withLock { try _backChannelForEntryPoint?.write(json) try _backChannelForEntryPoint?.write("\n") @@ -692,7 +692,7 @@ extension ExitTest { /// /// - Throws: Any error encountered attempting to decode or process the JSON. private static func _processRecord(_ recordJSON: UnsafeRawBufferPointer, fromBackChannel backChannel: borrowing FileHandle) throws { - let record = try JSON.decode(ABIv0.Record.self, from: recordJSON) + let record = try JSON.decode(ABI.Record.self, from: recordJSON) if case let .event(event) = record.kind, let issue = event.issue { // Translate the issue back into a "real" issue and record it diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index 86ede749e..0f856e6bf 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -27,7 +27,7 @@ struct ABIEntryPointTests { arguments.verbosity = .min let result = try await _invokeEntryPointV0Experimental(passing: arguments) { recordJSON in - let record = try! JSON.decode(ABIv0.Record.self, from: recordJSON) + let record = try! JSON.decode(ABI.Record.self, from: recordJSON) _ = record.version } @@ -89,7 +89,7 @@ struct ABIEntryPointTests { arguments.verbosity = .min let result = try await _invokeEntryPointV0(passing: arguments) { recordJSON in - let record = try! JSON.decode(ABIv0.Record.self, from: recordJSON) + let record = try! JSON.decode(ABI.Record.self, from: recordJSON) _ = record.version } @@ -117,7 +117,7 @@ struct ABIEntryPointTests { try await confirmation("Test matched", expectedCount: 1...) { testMatched in _ = try await _invokeEntryPointV0(passing: arguments) { recordJSON in - let record = try! JSON.decode(ABIv0.Record.self, from: recordJSON) + let record = try! JSON.decode(ABI.Record.self, from: recordJSON) if case .test = record.kind { testMatched() } else { @@ -145,7 +145,7 @@ struct ABIEntryPointTests { ) } #endif - let abiEntryPoint = unsafeBitCast(abiv0_getEntryPoint(), to: ABIv0.EntryPoint.self) + let abiEntryPoint = unsafeBitCast(abiv0_getEntryPoint(), to: ABI.v0.EntryPoint.self) let argumentsJSON = try JSON.withEncoding(of: arguments) { argumentsJSON in let result = UnsafeMutableRawBufferPointer.allocate(byteCount: argumentsJSON.count, alignment: 1) @@ -172,7 +172,7 @@ struct ABIEntryPointTests { #if !SWT_NO_DYNAMIC_LINKING private func withTestingLibraryImageAddress(_ body: (ImageAddress?) throws -> R) throws -> R { - let addressInTestingLibrary = unsafeBitCast(ABIv0.entryPoint, to: UnsafeRawPointer.self) + let addressInTestingLibrary = unsafeBitCast(ABI.v0.entryPoint, to: UnsafeRawPointer.self) var testingLibraryAddress: ImageAddress? #if SWT_TARGET_OS_APPLE diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 6b114efe6..770057b04 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -226,12 +226,12 @@ struct SwiftPMTests { #expect(args.parallel == false) } - func decodeABIv0RecordStream(fromFileAtPath path: String) throws -> [ABIv0.Record] { + func decodeABIv0RecordStream(fromFileAtPath path: String) throws -> [ABI.Record] { try FileHandle(forReadingAtPath: path).readToEnd() .split(whereSeparator: \.isASCIINewline) .map { line in try line.withUnsafeBytes { line in - try JSON.decode(ABIv0.Record.self, from: line) + try JSON.decode(ABI.Record.self, from: line) } } } From e1fd7c7efd521e11890b08a16a2d37fc2e26a05c Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Feb 2025 16:30:35 -0600 Subject: [PATCH 081/234] Require sendable userInfo values in JSON.withEncoding(of:userInfo:_:) (#955) swift-foundation recently landed a change (in https://github.com/swiftlang/swift-foundation/pull/764) which requires `any Sendable` values in `JSONEncoder.userInfo`. This causes a build failure in swift-testing: ``` JSON.swift:44:28: error: type 'Any' does not conform to the 'Sendable' protocol 42 | 43 | // Set user info keys that clients want to use during encoding. 44 | encoder.userInfo.merge(userInfo, uniquingKeysWith: { _, rhs in rhs}) | `- error: type 'Any' does not conform to the 'Sendable' protocol ``` This PR adjusts our `userInfo:` parameter require `any Sendable` values. The values we were passing to this utility were already sendable, luckily. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Support/JSON.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Support/JSON.swift b/Sources/Testing/Support/JSON.swift index f23e51384..76c7b7f07 100644 --- a/Sources/Testing/Support/JSON.swift +++ b/Sources/Testing/Support/JSON.swift @@ -29,7 +29,7 @@ enum JSON { /// - Returns: Whatever is returned by `body`. /// /// - Throws: Whatever is thrown by `body` or by the encoding process. - static func withEncoding(of value: some Encodable, userInfo: [CodingUserInfoKey: Any] = [:], _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + static func withEncoding(of value: some Encodable, userInfo: [CodingUserInfoKey: any Sendable] = [:], _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { #if canImport(Foundation) let encoder = JSONEncoder() From a4070bc42901f9b35a49e21912c8d2c2b2e60408 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Feb 2025 17:27:36 -0600 Subject: [PATCH 082/234] Enable Library Evolution in package builds for public library targets (#951) This modifies `Package.swift` to enable Library Evolution for builds of the package. ### Motivation: I recently landed a change (#931) which passed our project-level CI but later failed in Swift CI. The difference ended up being due to the latter building with Library Evolution (LE) enabled, whereas our project-level CI builds via SwiftPM and does not enable LE. The change was reverted (#950) but this revealed a gap in our testing strategy. We should always build these targets with LE enabled. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes rdar://144655439 --- Package.swift | 27 ++++++++++++++++++--- Sources/Testing/Running/Configuration.swift | 2 +- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Package.swift b/Package.swift index 4a22b98e2..43337b0dc 100644 --- a/Package.swift +++ b/Package.swift @@ -55,7 +55,9 @@ let package = Package( ], exclude: ["CMakeLists.txt", "Testing.swiftcrossimport"], cxxSettings: .packageSettings, - swiftSettings: .packageSettings, + swiftSettings: .packageSettings + [ + .enableLibraryEvolution(), + ], linkerSettings: [ .linkedLibrary("execinfo", .when(platforms: [.custom("freebsd"), .openbsd])) ] @@ -114,7 +116,9 @@ let package = Package( "Testing", ], path: "Sources/Overlays/_Testing_CoreGraphics", - swiftSettings: .packageSettings + swiftSettings: .packageSettings + [ + .enableLibraryEvolution(), + ] ), .target( name: "_Testing_Foundation", @@ -123,7 +127,12 @@ let package = Package( ], path: "Sources/Overlays/_Testing_Foundation", exclude: ["CMakeLists.txt"], - swiftSettings: .packageSettings + swiftSettings: .packageSettings + [ + // The Foundation module only has Library Evolution enabled on Apple + // platforms, and since this target's module publicly imports Foundation, + // it can only enable Library Evolution itself on those platforms. + .enableLibraryEvolution(applePlatformsOnly: true), + ] ), ], @@ -186,6 +195,18 @@ extension Array where Element == PackageDescription.SwiftSetting { } } +extension PackageDescription.SwiftSetting { + /// Create a Swift setting which enables Library Evolution, optionally + /// constraining it to only Apple platforms. + /// + /// - Parameters: + /// - applePlatformsOnly: Whether to constrain this setting to only Apple + /// platforms. + static func enableLibraryEvolution(applePlatformsOnly: Bool = false) -> Self { + unsafeFlags(["-enable-library-evolution"], .when(platforms: applePlatformsOnly ? [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS] : [])) + } +} + extension Array where Element == PackageDescription.CXXSetting { /// Settings intended to be applied to every C++ target in this package. /// Analogous to project-level build settings in an Xcode project. diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index a917c2f5b..30a2ce303 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -218,7 +218,7 @@ public struct Configuration: Sendable { /// property is pre-configured. Otherwise, the default value of this property /// records an issue indicating that it has not been configured. @_spi(Experimental) - public var exitTestHandler: ExitTest.Handler = { _ in + public var exitTestHandler: ExitTest.Handler = { exitTest in throw SystemError(description: "Exit test support has not been implemented by the current testing infrastructure.") } #endif From 1cf8c8391c2912dc74e8a1507d0606f422533894 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Feb 2025 17:37:38 -0600 Subject: [PATCH 083/234] Declare a package name for CMake builds and remove @_spi(ForSwiftTestingOnly) (#954) This removes usage of `@_spi(ForSwiftTestingOnly)` throughout the codebase and adjusts the CMake build rules to allow adoption of Swift's `package` access level. The technical constraints which prevented adopting that feature have been resolved. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/SPI.md | 14 -------------- .../Attachment+AttachableAsCGImage.swift | 2 +- .../Attachments/ImageAttachmentError.swift | 3 +-- .../Attachments/Attachment+URL.swift | 2 +- Sources/Testing/Attachments/Attachment.swift | 3 +-- Tests/TestingTests/AttachmentTests.swift | 2 +- cmake/modules/shared/CompilerSettings.cmake | 2 ++ 7 files changed, 7 insertions(+), 21 deletions(-) diff --git a/Documentation/SPI.md b/Documentation/SPI.md index 534d6e51f..766043f44 100644 --- a/Documentation/SPI.md +++ b/Documentation/SPI.md @@ -39,14 +39,6 @@ external tools, _both_ groups are specified. Such SPI is not generally meant to be promoted to public API, but is still experimental until tools authors have a chance to evaluate it. -For interfaces internal to Swift Testing that must be available across targets, -the SPI group `@_spi(ForSwiftTestingOnly)` is used. They _should_ be marked -`package` and may be in the future, but are currently exported due to technical -constraints when Swift Testing is built using CMake. - -> [!WARNING] -> Never use symbols marked `@_spi(ForSwiftTestingOnly)`. - ## SPI stability The testing library does **not** guarantee SPI stability for either group of @@ -59,12 +51,6 @@ to newer interfaces. SPI marked `@_spi(Experimental)` should be assumed to be unstable. It may be modified or removed at any time. -SPI marked `@_spi(ForSwiftTestingOnly)` is unstable and subject to change at any -time. - -> [!WARNING] -> Never use symbols marked `@_spi(ForSwiftTestingOnly)`. - ## API and ABI stability When Swift Testing reaches its 1.0 release, API changes will follow the same diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift index ddc876442..ca520e0c0 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift @@ -9,7 +9,7 @@ // #if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) -@_spi(ForSwiftTestingOnly) @_spi(Experimental) public import Testing +@_spi(Experimental) public import Testing public import UniformTypeIdentifiers diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift index b63831317..f957888b7 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/ImageAttachmentError.swift @@ -10,8 +10,7 @@ #if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) /// A type representing an error that can occur when attaching an image. -@_spi(ForSwiftTestingOnly) -public enum ImageAttachmentError: Error, CustomStringConvertible { +package enum ImageAttachmentError: Error, CustomStringConvertible { /// The image could not be converted to an instance of `CGImage`. case couldNotCreateCGImage diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index 305bdd35b..dbf7e2688 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) -@_spi(Experimental) @_spi(ForSwiftTestingOnly) public import Testing +@_spi(Experimental) public import Testing public import Foundation #if !SWT_NO_PROCESS_SPAWNING && os(Windows) diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index f69df3679..ef7ae5537 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -37,8 +37,7 @@ public struct Attachment: ~Copyable where AttachableValue: Atta public var fileSystemPath: String? /// The default preferred name to use if the developer does not supply one. - @_spi(ForSwiftTestingOnly) - public static var defaultPreferredName: String { + package static var defaultPreferredName: String { "untitled" } diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 98ecc668d..3fa979b35 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -16,7 +16,7 @@ import Foundation #endif #if canImport(CoreGraphics) import CoreGraphics -@_spi(Experimental) @_spi(ForSwiftTestingOnly) import _Testing_CoreGraphics +@_spi(Experimental) import _Testing_CoreGraphics #endif #if canImport(UniformTypeIdentifiers) import UniformTypeIdentifiers diff --git a/cmake/modules/shared/CompilerSettings.cmake b/cmake/modules/shared/CompilerSettings.cmake index 49e2579fe..eb9da4162 100644 --- a/cmake/modules/shared/CompilerSettings.cmake +++ b/cmake/modules/shared/CompilerSettings.cmake @@ -8,6 +8,8 @@ # Settings intended to be applied to every Swift target in this project. # Analogous to project-level build settings in an Xcode project. +add_compile_options( + "SHELL:$<$:-package-name org.swift.testing>") add_compile_options( "SHELL:$<$:-Xfrontend -require-explicit-sendable>") add_compile_options( From b2ef481c09187f46eaea8b989e2ec35128709504 Mon Sep 17 00:00:00 2001 From: Ben Barham Date: Wed, 12 Feb 2025 21:25:03 -0800 Subject: [PATCH 084/234] =?UTF-8?q?Revert=20"Require=20sendable=20userInfo?= =?UTF-8?q?=20values=20in=20JSON.withEncoding(of:userInfo:=5F:)=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e1fd7c7efd521e11890b08a16a2d37fc2e26a05c. --- Sources/Testing/Support/JSON.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Support/JSON.swift b/Sources/Testing/Support/JSON.swift index 76c7b7f07..f23e51384 100644 --- a/Sources/Testing/Support/JSON.swift +++ b/Sources/Testing/Support/JSON.swift @@ -29,7 +29,7 @@ enum JSON { /// - Returns: Whatever is returned by `body`. /// /// - Throws: Whatever is thrown by `body` or by the encoding process. - static func withEncoding(of value: some Encodable, userInfo: [CodingUserInfoKey: any Sendable] = [:], _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + static func withEncoding(of value: some Encodable, userInfo: [CodingUserInfoKey: Any] = [:], _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { #if canImport(Foundation) let encoder = JSONEncoder() From 44684c1ede59bb9aad200728173f948d800163e1 Mon Sep 17 00:00:00 2001 From: Graham Lee Date: Thu, 13 Feb 2025 13:15:04 +0000 Subject: [PATCH 085/234] Cleaning recently changed docs (#924) Improving the style and explanation of recent additions to the documentation. ### Motivation: The recent changes to test traits are great, and come with some very nice documentation. I've done what I can to improve the style and the wording to make it easier to understand, and to make the style consistent with other documentation in the project. ### Modifications: Changed the documentation for `Trait` and associated types. ### Result: There are no behavioral changes in this PR, but I hope that the existing behavior is easier to use with better documentation. ### Checklist: - [X] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [X] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Testing.docc/Traits.md | 8 +- Sources/Testing/Testing.docc/Traits/Trait.md | 21 +- Sources/Testing/Traits/Bug.swift | 22 +- Sources/Testing/Traits/Comment.swift | 13 +- Sources/Testing/Traits/ConditionTrait.swift | 85 ++++---- .../Testing/Traits/ParallelizationTrait.swift | 19 +- Sources/Testing/Traits/TimeLimitTrait.swift | 51 ++--- Sources/Testing/Traits/Trait.swift | 203 +++++++++--------- 8 files changed, 206 insertions(+), 216 deletions(-) diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index 5fadb2bdc..413b4327c 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -10,14 +10,14 @@ See https://swift.org/LICENSE.txt for license information See https://swift.org/CONTRIBUTORS.txt for Swift project authors --> -Add traits to tests to annotate them or customize their behavior. +Annotate test functions and suites, and customize their behavior. ## Overview Pass built-in traits to test functions or suite types to comment, categorize, -classify, and modify runtime behaviors. You can also use the ``Trait``, ``TestTrait``, -and ``SuiteTrait`` protocols to create your own types that customize the -behavior of test functions. +classify, and modify the runtime behavior of test suites and test functions. +Implement the ``TestTrait``, and ``SuiteTrait`` protocols to create your own +types that customize the behavior of your tests. ## Topics diff --git a/Sources/Testing/Testing.docc/Traits/Trait.md b/Sources/Testing/Testing.docc/Traits/Trait.md index f0e84aaeb..d6422167d 100644 --- a/Sources/Testing/Testing.docc/Traits/Trait.md +++ b/Sources/Testing/Testing.docc/Traits/Trait.md @@ -20,17 +20,15 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors - ``Trait/disabled(if:_:sourceLocation:)`` - ``Trait/disabled(_:sourceLocation:_:)`` -### Limiting the running time of tests +### Controlling how tests are run - ``Trait/timeLimit(_:)-4kzjp`` - -### Running tests serially or in parallel - - ``Trait/serialized`` - -### Categorizing tests + +### Categorizing tests and adding information - ``Trait/tags(_:)`` +- ``Trait/comments`` ### Associating bugs @@ -38,16 +36,9 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors - ``Trait/bug(_:id:_:)-10yf5`` - ``Trait/bug(_:id:_:)-3vtpl`` -### Adding information to tests - -- ``Trait/comments`` - -### Preparing internal state - -- ``Trait/prepare(for:)-3s3zo`` - -### Providing custom execution scope for tests +### Running code before and after a test or suite - ``TestScoping`` - ``Trait/scopeProvider(for:testCase:)-cjmg`` - ``Trait/TestScopeProvider`` +- ``Trait/prepare(for:)-3s3zo`` diff --git a/Sources/Testing/Traits/Bug.swift b/Sources/Testing/Traits/Bug.swift index 48a718dfa..14f541557 100644 --- a/Sources/Testing/Traits/Bug.swift +++ b/Sources/Testing/Traits/Bug.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type representing a bug report tracked by a test. +/// A type that represents a bug report tracked by a test. /// /// To add this trait to a test, use one of the following functions: /// @@ -16,7 +16,7 @@ /// - ``Trait/bug(_:id:_:)-10yf5`` /// - ``Trait/bug(_:id:_:)-3vtpl`` public struct Bug { - /// A URL linking to more information about the bug, if available. + /// A URL that links to more information about the bug, if available. /// /// The value of this property represents a URL conforming to /// [RFC 3986](https://www.ietf.org/rfc/rfc3986.txt). @@ -59,42 +59,42 @@ extension Bug: TestTrait, SuiteTrait { } extension Trait where Self == Bug { - /// Construct a bug to track with a test. + /// Constructs a bug to track with a test. /// /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking + /// - url: A URL that refers to this bug in the associated bug-tracking /// system. /// - title: Optionally, the human-readable title of the bug. /// - /// - Returns: An instance of ``Bug`` representing the specified bug. + /// - Returns: An instance of ``Bug`` that represents the specified bug. public static func bug(_ url: _const String, _ title: Comment? = nil) -> Self { Self(url: url, title: title) } - /// Construct a bug to track with a test. + /// Constructs a bug to track with a test. /// /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking + /// - url: A URL that refers to this bug in the associated bug-tracking /// system. /// - id: The unique identifier of this bug in its associated bug-tracking /// system. /// - title: Optionally, the human-readable title of the bug. /// - /// - Returns: An instance of ``Bug`` representing the specified bug. + /// - Returns: An instance of ``Bug`` that represents the specified bug. public static func bug(_ url: _const String? = nil, id: some Numeric, _ title: Comment? = nil) -> Self { Self(url: url, id: String(describing: id), title: title) } - /// Construct a bug to track with a test. + /// Constructs a bug to track with a test. /// /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking + /// - url: A URL that refers to this bug in the associated bug-tracking /// system. /// - id: The unique identifier of this bug in its associated bug-tracking /// system. /// - title: Optionally, the human-readable title of the bug. /// - /// - Returns: An instance of ``Bug`` representing the specified bug. + /// - Returns: An instance of ``Bug`` that represents the specified bug. public static func bug(_ url: _const String? = nil, id: _const String, _ title: Comment? = nil) -> Self { Self(url: url, id: id, title: title) } diff --git a/Sources/Testing/Traits/Comment.swift b/Sources/Testing/Traits/Comment.swift index 0b282e634..a36cec1aa 100644 --- a/Sources/Testing/Traits/Comment.swift +++ b/Sources/Testing/Traits/Comment.swift @@ -8,22 +8,21 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type representing a comment related to a test. +/// A type that represents a comment related to a test. /// -/// This type may be used to provide context or background information about a +/// Use this type to provide context or background information about a /// test's purpose, explain how a complex test operates, or include details /// which may be helpful when diagnosing issues recorded by a test. /// /// To add a comment to a test or suite, add a code comment before its `@Test` /// or `@Suite` attribute. See for more details. /// -/// - Note: This type is not intended to reference bugs related to a test. -/// Instead, use ``Trait/bug(_:_:)``, ``Trait/bug(_:id:_:)-10yf5``, or -/// ``Trait/bug(_:id:_:)-3vtpl``. +/// - Note: To reference bugs related to a test, use ``Trait/bug(_:_:)``, +/// ``Trait/bug(_:id:_:)-10yf5``, or ``Trait/bug(_:id:_:)-3vtpl``. public struct Comment: RawRepresentable, Sendable { - /// The single comment string contained in this instance. + /// The single comment string that this comment contains. /// - /// To obtain the complete set of comments applied to a test, see + /// To get the complete set of comments applied to a test, see /// ``Test/comments``. public var rawValue: String diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index 8e1117f8a..245f8e98f 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -8,8 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type that defines a condition which must be satisfied for a test to be -/// enabled. +/// A type that defines a condition which must be satisfied for the testing +/// library to enable a test. /// /// To add this trait to a test, use one of the following functions: /// @@ -19,10 +19,10 @@ /// - ``Trait/disabled(if:_:sourceLocation:)`` /// - ``Trait/disabled(_:sourceLocation:_:)`` public struct ConditionTrait: TestTrait, SuiteTrait { - /// An enumeration describing the kinds of conditions that can be represented - /// by an instance of this type. + /// An enumeration that describes the conditions that an instance of this type + /// can represent. enum Kind: Sendable { - /// The trait is conditional on the result of calling a function. + /// Enabling the test is conditional on the result of calling a function. /// /// - Parameters: /// - body: The function to call. The result of this function determines @@ -39,7 +39,8 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// - body: The function to call. The result of this function determines /// whether or not the condition was met. /// - /// - Returns: An instance of this type. + /// - Returns: A trait that marks a test's enabled status as the result of + /// calling a function. static func conditional(_ body: @escaping @Sendable () async throws -> Bool) -> Self { conditional { () -> (Bool, comment: Comment?) in return (try await body(), nil) @@ -49,14 +50,14 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// The trait is unconditional and always has the same result. /// /// - Parameters: - /// - value: Whether or not the condition was met. + /// - value: Whether or not the test is enabled. case unconditional(_ value: Bool) } - /// The kind of condition represented by this instance. + /// The kind of condition represented by this trait. var kind: Kind - /// Whether or not this trait has a condition that is evaluated at runtime. + /// Whether this trait's condition is constant, or evaluated at runtime. /// /// If this trait was created using a function such as /// ``disabled(_:sourceLocation:)`` that unconditionally enables or disables a @@ -77,7 +78,7 @@ public struct ConditionTrait: TestTrait, SuiteTrait { public var comments: [Comment] - /// The source location where this trait was specified. + /// The source location where this trait is specified. public var sourceLocation: SourceLocation public func prepare(for test: Test) async throws { @@ -110,18 +111,17 @@ public struct ConditionTrait: TestTrait, SuiteTrait { // MARK: - extension Trait where Self == ConditionTrait { - /// Construct a condition trait that causes a test to be disabled if it - /// returns `false`. + /// Constructs a condition trait that disables a test if it returns `false`. /// /// - Parameters: - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `true`, the test is allowed to run. Otherwise, - /// the test is skipped. - /// - comment: An optional, user-specified comment describing this trait. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `true`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the - /// specified closure. + /// - Returns: An instance of ``ConditionTrait`` that evaluates the + /// closure you provide. // // @Comment { // - Bug: `condition` cannot be `async` without making this function @@ -136,18 +136,17 @@ extension Trait where Self == ConditionTrait { Self(kind: .conditional(condition), comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that causes a test to be disabled if it - /// returns `false`. + /// Constructs a condition trait that disables a test if it returns `false`. /// /// - Parameters: - /// - comment: An optional, user-specified comment describing this trait. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `true`, the test is allowed to run. Otherwise, - /// the test is skipped. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `true`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the - /// specified closure. + /// - Returns: An instance of ``ConditionTrait`` that evaluates the + /// closure you provide. public static func enabled( _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, @@ -156,13 +155,13 @@ extension Trait where Self == ConditionTrait { Self(kind: .conditional(condition), comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that disables a test unconditionally. + /// Constructs a condition trait that disables a test unconditionally. /// /// - Parameters: - /// - comment: An optional, user-specified comment describing this trait. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. /// - /// - Returns: An instance of ``ConditionTrait`` that will always disable the + /// - Returns: An instance of ``ConditionTrait`` that always disables the /// test to which it is added. public static func disabled( _ comment: Comment? = nil, @@ -171,18 +170,17 @@ extension Trait where Self == ConditionTrait { Self(kind: .unconditional(false), comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that causes a test to be disabled if it - /// returns `true`. + /// Constructs a condition trait that disables a test if its value is true. /// /// - Parameters: - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `false`, the test is allowed to run. Otherwise, - /// the test is skipped. - /// - comment: An optional, user-specified comment describing this trait. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `false`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the - /// specified closure. + /// - Returns: An instance of ``ConditionTrait`` that evaluates the + /// closure you provide. // // @Comment { // - Bug: `condition` cannot be `async` without making this function @@ -197,17 +195,16 @@ extension Trait where Self == ConditionTrait { Self(kind: .conditional { !(try condition()) }, comments: Array(comment), sourceLocation: sourceLocation) } - /// Construct a condition trait that causes a test to be disabled if it - /// returns `true`. + /// Constructs a condition trait that disables a test if its value is true. /// /// - Parameters: - /// - comment: An optional, user-specified comment describing this trait. + /// - comment: An optional comment that describes this trait. /// - sourceLocation: The source location of the trait. - /// - condition: A closure containing the trait's custom condition logic. If - /// this closure returns `false`, the test is allowed to run. Otherwise, - /// the test is skipped. + /// - condition: A closure that contains the trait's custom condition logic. + /// If this closure returns `false`, the trait allows the test to run. + /// Otherwise, the testing library skips the test. /// - /// - Returns: An instance of ``ConditionTrait`` that will evaluate the + /// - Returns: An instance of ``ConditionTrait`` that evaluates the /// specified closure. public static func disabled( _ comment: Comment? = nil, diff --git a/Sources/Testing/Traits/ParallelizationTrait.swift b/Sources/Testing/Traits/ParallelizationTrait.swift index df34f4d63..9eb1bd2a5 100644 --- a/Sources/Testing/Traits/ParallelizationTrait.swift +++ b/Sources/Testing/Traits/ParallelizationTrait.swift @@ -8,20 +8,21 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type that affects whether or not a test or suite is parallelized. +/// A type that defines whether the testing library runs this test serially +/// or in parallel. /// -/// When added to a parameterized test function, this trait causes that test to -/// run its cases serially instead of in parallel. When added to a -/// non-parameterized test function, this trait has no effect. +/// When you add this trait to a parameterized test function, that test runs its +/// cases serially instead of in parallel. This trait has no effect when you +/// apply it to a non-parameterized test function. /// -/// When added to a test suite, this trait causes that suite to run its +/// When you add this trait to a test suite, that suite runs its /// contained test functions (including their cases, when parameterized) and -/// sub-suites serially instead of in parallel. Any children of sub-suites are -/// also run serially. +/// sub-suites serially instead of in parallel. If the sub-suites have children, +/// they also run serially. /// /// This trait does not affect the execution of a test relative to its peers or -/// to unrelated tests. This trait has no effect if test parallelization is -/// globally disabled (by, for example, passing `--no-parallel` to the +/// to unrelated tests. This trait has no effect if you disable test +/// parallelization globally (for example, by passing `--no-parallel` to the /// `swift test` command.) /// /// To add this trait to a test, use ``Trait/serialized``. diff --git a/Sources/Testing/Traits/TimeLimitTrait.swift b/Sources/Testing/Traits/TimeLimitTrait.swift index 2dc86de20..4e84a1f92 100644 --- a/Sources/Testing/Traits/TimeLimitTrait.swift +++ b/Sources/Testing/Traits/TimeLimitTrait.swift @@ -10,17 +10,16 @@ /// A type that defines a time limit to apply to a test. /// -/// To add this trait to a test, use one of the following functions: -/// -/// - ``Trait/timeLimit(_:)-4kzjp`` +/// To add this trait to a test, use ``Trait/timeLimit(_:)-4kzjp``. @available(_clockAPI, *) public struct TimeLimitTrait: TestTrait, SuiteTrait { /// A type representing the duration of a time limit applied to a test. /// - /// This type is intended for use specifically for specifying test timeouts - /// with ``TimeLimitTrait``. It is used instead of Swift's built-in `Duration` - /// type because test timeouts do not support high-precision, arbitrarily - /// short durations. The smallest allowed unit of time is minutes. + /// Use this type to specify a test timeout with ``TimeLimitTrait``. + /// `TimeLimitTrait` uses this type instead of Swift's built-in `Duration` + /// type because the testing library doesn't support high-precision, + /// arbitrarily short durations for test timeouts. The smallest unit of time + /// you can specify in a `Duration` is minutes. public struct Duration: Sendable { /// The underlying Swift `Duration` which this time limit duration /// represents. @@ -29,8 +28,7 @@ public struct TimeLimitTrait: TestTrait, SuiteTrait { /// Construct a time limit duration given a number of minutes. /// /// - Parameters: - /// - minutes: The number of minutes the resulting duration should - /// represent. + /// - minutes: The length of the duration in minutes. /// /// - Returns: A duration representing the specified number of minutes. public static func minutes(_ minutes: some BinaryInteger) -> Self { @@ -97,26 +95,25 @@ extension Trait where Self == TimeLimitTrait { /// - Returns: An instance of ``TimeLimitTrait``. /// /// Test timeouts do not support high-precision, arbitrarily short durations - /// due to variability in testing environments. The time limit must be at - /// least one minute, and can only be expressed in increments of one minute. + /// due to variability in testing environments. You express the duration in + /// minutes, with a minimum duration of one minute. /// - /// When this trait is associated with a test, that test must complete within - /// a time limit of, at most, `timeLimit`. If the test runs longer, an issue - /// of kind ``Issue/Kind/timeLimitExceeded(timeLimitComponents:)`` is - /// recorded. This timeout is treated as a test failure. + /// When you associate this trait with a test, that test must complete within + /// a time limit of, at most, `timeLimit`. If the test runs longer, the + /// testing library records a + /// ``Issue/Kind/timeLimitExceeded(timeLimitComponents:)`` issue, which it + /// treats as a test failure. /// - /// The time limit amount specified by `timeLimit` may be reduced if the - /// testing library is configured to enforce a maximum per-test limit. When - /// such a maximum is set, the effective time limit of the test this trait is - /// applied to will be the lesser of `timeLimit` and that maximum. This is a - /// policy which may be configured on a global basis by the tool responsible - /// for launching the test process. Refer to that tool's documentation for - /// more details. + /// The testing library can use a shorter time limit than that specified by + /// `timeLimit` if you configure it to enforce a maximum per-test limit. When + /// you configure a maximum per-test limit, the time limit of the test this + /// trait is applied to is the shorter of `timeLimit` and the maximum per-test + /// limit. For information on configuring maximum per-test limits, consult the + /// documentation for the tool you use to run your tests. /// /// If a test is parameterized, this time limit is applied to each of its /// test cases individually. If a test has more than one time limit associated - /// with it, the shortest one is used. A test run may also be configured with - /// a maximum time limit per test case. + /// with it, the testing library uses the shortest time limit. public static func timeLimit(_ timeLimit: Self.Duration) -> Self { return Self(timeLimit: timeLimit.underlyingDuration) } @@ -185,11 +182,9 @@ extension TimeLimitTrait.Duration { @available(_clockAPI, *) extension Test { - /// The maximum amount of time the cases of this test may run for. - /// - /// Time limits are associated with tests using this trait: + /// The maximum amount of time this test's cases may run for. /// - /// - ``Trait/timeLimit(_:)-4kzjp`` + /// Associate a time limit with tests by using ``Trait/timeLimit(_:)-4kzjp``. /// /// If a test has more than one time limit associated with it, the value of /// this property is the shortest one. If a test has no time limits associated diff --git a/Sources/Testing/Traits/Trait.swift b/Sources/Testing/Traits/Trait.swift index 4c942f52a..7ebbb38d4 100644 --- a/Sources/Testing/Traits/Trait.swift +++ b/Sources/Testing/Traits/Trait.swift @@ -12,46 +12,49 @@ /// test suite. /// /// The testing library defines a number of traits that can be added to test -/// functions and to test suites. Developers can define their own traits by -/// creating types that conform to ``TestTrait`` and/or ``SuiteTrait``. +/// functions and to test suites. Define your own traits by +/// creating types that conform to ``TestTrait`` or ``SuiteTrait``: /// -/// When creating a custom trait type, the type should conform to ``TestTrait`` -/// if it can be added to test functions, ``SuiteTrait`` if it can be added to -/// test suites, and both ``TestTrait`` and ``SuiteTrait`` if it can be added to -/// both test functions _and_ test suites. +/// - term ``TestTrait``: Conform to this type in traits that you add to test +/// functions. +/// - term ``SuiteTrait``: Conform to this type in traits that you add to test +/// suites. +/// +/// You can add a trait that conforms to both ``TestTrait`` and ``SuiteTrait`` +/// to test functions and test suites. public protocol Trait: Sendable { - /// Prepare to run the test to which this trait was added. + /// Prepare to run the test that has this trait. /// /// - Parameters: - /// - test: The test to which this trait was added. + /// - test: The test that has this trait. /// - /// - Throws: Any error that would prevent the test from running. If an error - /// is thrown from this method, the test will be skipped and the error will - /// be recorded as an ``Issue``. + /// - Throws: Any error that prevents the test from running. If an error + /// is thrown from this method, the test is skipped and the error is + /// recorded as an ``Issue``. /// - /// This method is called after all tests and their traits have been - /// discovered by the testing library, but before any test has begun running. - /// It may be used to prepare necessary internal state, or to influence + /// The testing library calls this method after it discovers all tests and + /// their traits, and before it begins to run any tests. + /// Use this method to prepare necessary internal state, or to determine /// whether the test should run. /// /// The default implementation of this method does nothing. func prepare(for test: Test) async throws - /// The user-provided comments for this trait, if any. + /// The user-provided comments for this trait. /// - /// By default, the value of this property is an empty array. + /// The default value of this property is an empty array. var comments: [Comment] { get } /// The type of the test scope provider for this trait. /// - /// The default type is `Never`, which cannot be instantiated. The - /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with this - /// default type must return `nil`, meaning that trait will not provide a - /// custom scope for the tests it's applied to. + /// The default type is `Never`, which can't be instantiated. The + /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with + /// `Never` as its test scope provider type must return `nil`, meaning that + /// the trait doesn't provide a custom scope for tests it's applied to. associatedtype TestScopeProvider: TestScoping = Never - /// Get this trait's scope provider for the specified test and/or test case, - /// if any. + /// Get this trait's scope provider for the specified test and optional test + /// case. /// /// - Parameters: /// - test: The test for which a scope provider is being requested. @@ -59,28 +62,30 @@ public protocol Trait: Sendable { /// if any. When `test` represents a suite, the value of this argument is /// `nil`. /// - /// - Returns: A value conforming to ``Trait/TestScopeProvider`` which may be - /// used to provide custom scoping for `test` and/or `testCase`, or `nil` if - /// they should not have any custom scope. + /// - Returns: A value conforming to ``Trait/TestScopeProvider`` which you + /// use to provide custom scoping for `test` or `testCase`. Returns `nil` if + /// the trait doesn't provide any custom scope for the test or test case. /// /// If this trait's type conforms to ``TestScoping``, the default value - /// returned by this method depends on `test` and/or `testCase`: + /// returned by this method depends on the values of`test` and `testCase`: /// /// - If `test` represents a suite, this trait must conform to ``SuiteTrait``. /// If the value of this suite trait's ``SuiteTrait/isRecursive`` property - /// is `true`, then this method returns `nil`; otherwise, it returns `self`. - /// This means that by default, a suite trait will _either_ provide its - /// custom scope once for the entire suite, or once per-test function it - /// contains. - /// - Otherwise `test` represents a test function. If `testCase` is `nil`, - /// this method returns `nil`; otherwise, it returns `self`. This means that - /// by default, a trait which is applied to or inherited by a test function - /// will provide its custom scope once for each of that function's cases. - /// - /// A trait may explicitly implement this method to further customize the - /// default behaviors above. For example, if a trait should provide custom + /// is `true`, then this method returns `nil`, and the suite trait + /// provides its custom scope once for each test function the test suite + /// contains. If the value of ``SuiteTrait/isRecursive`` is `false`, this + /// method returns `self`, and the suite trait provides its custom scope + /// once for the entire test suite. + /// - If `test` represents a test function, this trait also conforms to + /// ``TestTrait``. If `testCase` is `nil`, this method returns `nil`; + /// otherwise, it returns `self`. This means that by default, a trait which + /// is applied to or inherited by a test function provides its custom scope + /// once for each of that function's cases. + /// + /// A trait may override this method to further customize the + /// default behaviors above. For example, if a trait needs to provide custom /// test scope both once per-suite and once per-test function in that suite, - /// it may implement the method and return a non-`nil` scope provider under + /// it implements the method to return a non-`nil` scope provider under /// those conditions. /// /// A trait may also implement this method and return `nil` if it determines @@ -92,41 +97,39 @@ public protocol Trait: Sendable { /// If this trait's type does not conform to ``TestScoping`` and its /// associated ``Trait/TestScopeProvider`` type is the default `Never`, then /// this method returns `nil` by default. This means that instances of this - /// trait will not provide a custom scope for tests to which they're applied. + /// trait don't provide a custom scope for tests to which they're applied. func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider? } -/// A protocol that allows providing a custom execution scope for a test -/// function (and each of its cases) or a test suite by performing custom code -/// before or after it runs. +/// A protocol that tells the test runner to run custom code before or after it +/// runs a test suite or test function. /// -/// Types conforming to this protocol may be used in conjunction with a -/// ``Trait``-conforming type by implementing the -/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, allowing custom traits -/// to provide custom scope for tests. Consolidating common set-up and tear-down -/// logic for tests which have similar needs allows each test function to be -/// more succinct with less repetitive boilerplate so it can focus on what makes -/// it unique. +/// Provide custom scope for tests by implementing the +/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, returning a type that +/// conforms to this protocol. Create a custom scope to consolidate common +/// set-up and tear-down logic for tests which have similar needs, which allows +/// each test function to focus on the unique aspects of its test. public protocol TestScoping: Sendable { /// Provide custom execution scope for a function call which is related to the - /// specified test and/or test case. + /// specified test or test case. /// /// - Parameters: - /// - test: The test under which `function` is being performed. - /// - testCase: The test case, if any, under which `function` is being - /// performed. When invoked on a suite, the value of this argument is - /// `nil`. + /// - test: The test which `function` encapsulates. + /// - testCase: The test case, if any, which `function` encapsulates. + /// When invoked on a suite, the value of this argument is `nil`. /// - function: The function to perform. If `test` represents a test suite, /// this function encapsulates running all the tests in that suite. If /// `test` represents a test function, this function is the body of that - /// test function (including all cases if it is parameterized.) + /// test function (including all cases if the test function is + /// parameterized.) /// - /// - Throws: Whatever is thrown by `function`, or an error preventing this - /// type from providing a custom scope correctly. An error thrown from this - /// method is recorded as an issue associated with `test`. If an error is - /// thrown before `function` is called, the corresponding test will not run. + /// - Throws: Any error that `function` throws, or an error that prevents this + /// type from providing a custom scope correctly. The testing library + /// records an error thrown from this method as an issue associated with + /// `test`. If an error is thrown before this method calls `function`, the + /// corresponding test doesn't run. /// - /// When the testing library is preparing to run a test, it starts by finding + /// When the testing library prepares to run a test, it starts by finding /// all traits applied to that test, including those inherited from containing /// suites. It begins with inherited suite traits, sorting them /// outermost-to-innermost, and if the test is a function, it then adds all @@ -136,44 +139,45 @@ public protocol TestScoping: Sendable { /// this method on all non-`nil` scope providers, giving each an opportunity /// to perform arbitrary work before or after invoking `function`. /// - /// This method should either invoke `function` once before returning or throw - /// an error if it is unable to provide a custom scope. + /// This method should either invoke `function` once before returning, + /// or throw an error if it's unable to provide a custom scope. /// /// Issues recorded by this method are associated with `test`. func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws } extension Trait where Self: TestScoping { - /// Get this trait's scope provider for the specified test and/or test case, - /// if any. + /// Get this trait's scope provider for the specified test or test case. /// /// - Parameters: - /// - test: The test for which a scope provider is being requested. - /// - testCase: The test case for which a scope provider is being requested, - /// if any. When `test` represents a suite, the value of this argument is + /// - test: The test for which the testing library requests a + /// scope provider. + /// - testCase: The test case for which the testing library requests a scope + /// provider, if any. When `test` represents a suite, the value of this argument is /// `nil`. /// - /// This default implementation is used when this trait type conforms to - /// ``TestScoping`` and its return value is discussed in - /// ``Trait/scopeProvider(for:testCase:)-cjmg``. + /// The testing library uses this implementation of + /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type conforms + /// to ``TestScoping``. public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { testCase == nil ? nil : self } } extension SuiteTrait where Self: TestScoping { - /// Get this trait's scope provider for the specified test and/or test case, - /// if any. + /// Get this trait's scope provider for the specified test and optional test + /// case. /// /// - Parameters: - /// - test: The test for which a scope provider is being requested. - /// - testCase: The test case for which a scope provider is being requested, - /// if any. When `test` represents a suite, the value of this argument is - /// `nil`. - /// - /// This default implementation is used when this trait type conforms to - /// ``TestScoping`` and its return value is discussed in - /// ``Trait/scopeProvider(for:testCase:)-cjmg``. + /// - test: The test for which the testing library requests a scope + /// provider. + /// - testCase: The test case for which the testing library requests a scope + /// provider, if any. When `test` represents a suite, the value of this + /// argument is `nil`. + /// + /// The testing library uses this implementation of + /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type conforms + /// to both ``SuiteTrait`` and ``TestScoping``. public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { if test.isSuite { isRecursive ? nil : self @@ -187,22 +191,25 @@ extension Never: TestScoping { public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws {} } -/// A protocol describing traits that can be added to a test function. +/// A protocol describing a trait that you can add to a test function. /// -/// The testing library defines a number of traits that can be added to test -/// functions. Developers can also define their own traits by creating types -/// that conform to this protocol and/or to the ``SuiteTrait`` protocol. +/// The testing library defines a number of traits that you can add to test +/// functions. You can also define your own traits by creating types +/// that conform to this protocol, or to the ``SuiteTrait`` protocol. public protocol TestTrait: Trait {} -/// A protocol describing traits that can be added to a test suite. +/// A protocol describing a trait that you can add to a test suite. /// -/// The testing library defines a number of traits that can be added to test -/// suites. Developers can also define their own traits by creating types that -/// conform to this protocol and/or to the ``TestTrait`` protocol. +/// The testing library defines a number of traits that you can add to test +/// suites. You can also define your own traits by creating types that +/// conform to this protocol, or to the ``TestTrait`` protocol. public protocol SuiteTrait: Trait { /// Whether this instance should be applied recursively to child test suites - /// and test functions or should only be applied to the test suite to which it - /// was directly added. + /// and test functions. + /// + /// If the value is `true`, then the testing library applies this trait + /// recursively to child test suites and test functions. Otherwise, it only + /// applies the trait to the test suite to which you added the trait. /// /// By default, traits are not recursively applied to children. var isRecursive: Bool { get } @@ -217,18 +224,18 @@ extension Trait { } extension Trait where TestScopeProvider == Never { - /// Get this trait's scope provider for the specified test and/or test case, - /// if any. + /// Get this trait's scope provider for the specified test or test case. /// /// - Parameters: - /// - test: The test for which a scope provider is being requested. - /// - testCase: The test case for which a scope provider is being requested, - /// if any. When `test` represents a suite, the value of this argument is + /// - test: The test for which the testing library requests a + /// scope provider. + /// - testCase: The test case for which the testing library requests a scope + /// provider, if any. When `test` represents a suite, the value of this argument is /// `nil`. /// - /// This default implementation is used when this trait type's associated - /// ``Trait/TestScopeProvider`` type is the default value of `Never`, and its - /// return value is discussed in ``Trait/scopeProvider(for:testCase:)-cjmg``. + /// The testing library uses this implementation of + /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type's + /// associated ``Trait/TestScopeProvider`` type is `Never`. public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? { nil } From d328594973784ceaa81c1db1371e56accc19b20d Mon Sep 17 00:00:00 2001 From: Ben Barham Date: Thu, 13 Feb 2025 08:41:24 -0800 Subject: [PATCH 086/234] Revert "Revert "Require sendable userInfo values in JSON.withEncoding(of:userInfo:_:)"" --- Sources/Testing/Support/JSON.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Support/JSON.swift b/Sources/Testing/Support/JSON.swift index f23e51384..76c7b7f07 100644 --- a/Sources/Testing/Support/JSON.swift +++ b/Sources/Testing/Support/JSON.swift @@ -29,7 +29,7 @@ enum JSON { /// - Returns: Whatever is returned by `body`. /// /// - Throws: Whatever is thrown by `body` or by the encoding process. - static func withEncoding(of value: some Encodable, userInfo: [CodingUserInfoKey: Any] = [:], _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + static func withEncoding(of value: some Encodable, userInfo: [CodingUserInfoKey: any Sendable] = [:], _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { #if canImport(Foundation) let encoder = JSONEncoder() From 243bba90f0cec10236ce57e6ac73f70e504e6b01 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 17 Feb 2025 10:33:10 -0600 Subject: [PATCH 087/234] Update proposals to reflect implementation status, Swift version, and forum discussion links (#961) This updates some details in the accepted proposals in the repository, to better reflect their current status and match the convention used by proposals in the [swift-evolution](https://github.com/swiftlang/swift-evolution) repo. ## Modifications - Update the _Status_ field of all accepted proposals which have been fully implemented to "Implemented". - Indicate the Swift version in which those proposals were implemented. - Add any missing links to Forum reviews or acceptance posts. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/Proposals/0001-refactor-bug-inits.md | 2 +- Documentation/Proposals/0002-json-abi.md | 2 +- Documentation/Proposals/0003-make-serialized-trait-api.md | 2 +- ...-constrain-the-granularity-of-test-time-limit-durations.md | 2 +- Documentation/Proposals/0005-ranged-confirmations.md | 2 +- .../Proposals/0006-return-errors-from-expect-throws.md | 4 ++-- Documentation/Proposals/0007-test-scoping-traits.md | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Documentation/Proposals/0001-refactor-bug-inits.md b/Documentation/Proposals/0001-refactor-bug-inits.md index 66aecfe5b..0a4f00566 100644 --- a/Documentation/Proposals/0001-refactor-bug-inits.md +++ b/Documentation/Proposals/0001-refactor-bug-inits.md @@ -2,7 +2,7 @@ * Proposal: [SWT-0001](0001-refactor-bug-inits.md) * Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Accepted** +* Status: **Implemented (Swift 6.0)** * Implementation: [swiftlang/swift-testing#401](https://github.com/swiftlang/swift-testing/pull/401) * Review: ([pitch](https://forums.swift.org/t/pitch-dedicated-bug-functions-for-urls-and-ids/71842)), ([acceptance](https://forums.swift.org/t/swt-0001-dedicated-bug-functions-for-urls-and-ids/71842/2)) diff --git a/Documentation/Proposals/0002-json-abi.md b/Documentation/Proposals/0002-json-abi.md index b394116df..0af939972 100644 --- a/Documentation/Proposals/0002-json-abi.md +++ b/Documentation/Proposals/0002-json-abi.md @@ -2,7 +2,7 @@ * Proposal: [SWT-0002](0002-json-abi.md) * Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Accepted** +* Status: **Implemented (Swift 6.0)** * Implementation: [swiftlang/swift-testing#383](https://github.com/swiftlang/swift-testing/pull/383), [swiftlang/swift-testing#402](https://github.com/swiftlang/swift-testing/pull/402) * Review: ([pitch](https://forums.swift.org/t/pitch-a-stable-json-based-abi-for-tools-integration/72627)), ([acceptance](https://forums.swift.org/t/pitch-a-stable-json-based-abi-for-tools-integration/72627/4)) diff --git a/Documentation/Proposals/0003-make-serialized-trait-api.md b/Documentation/Proposals/0003-make-serialized-trait-api.md index 6967275da..dc2a98c28 100644 --- a/Documentation/Proposals/0003-make-serialized-trait-api.md +++ b/Documentation/Proposals/0003-make-serialized-trait-api.md @@ -2,7 +2,7 @@ * Proposal: [SWT-0003](0003-make-serialized-trait-api.md) * Authors: [Dennis Weissmann](https://github.com/dennisweissmann) -* Status: **Accepted** +* Status: **Implemented (Swift 6.0)** * Implementation: [swiftlang/swift-testing#535](https://github.com/swiftlang/swift-testing/pull/535) * Review: diff --git a/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md b/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md index c77bb3eea..8818dba71 100644 --- a/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md +++ b/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md @@ -3,7 +3,7 @@ * Proposal: [SWT-0004](0004-constrain-the-granularity-of-test-time-limit-durations.md) * Authors: [Dennis Weissmann](https://github.com/dennisweissmann) -* Status: **Accepted** +* Status: **Implemented (Swift 6.0)** * Implementation: [swiftlang/swift-testing#534](https://github.com/swiftlang/swift-testing/pull/534) * Review: diff --git a/Documentation/Proposals/0005-ranged-confirmations.md b/Documentation/Proposals/0005-ranged-confirmations.md index fef9d7675..df1db331b 100644 --- a/Documentation/Proposals/0005-ranged-confirmations.md +++ b/Documentation/Proposals/0005-ranged-confirmations.md @@ -2,7 +2,7 @@ * Proposal: [SWT-0005](0005-ranged-confirmations.md) * Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Accepted** +* Status: **Implemented (Swift 6.1)** * Bug: rdar://138499457 * Implementation: [swiftlang/swift-testing#598](https://github.com/swiftlang/swift-testing/pull/598), [swiftlang/swift-testing#689](https://github.com/swiftlang/swift-testing/pull689) * Review: ([pitch](https://forums.swift.org/t/pitch-range-based-confirmations/74589)), diff --git a/Documentation/Proposals/0006-return-errors-from-expect-throws.md b/Documentation/Proposals/0006-return-errors-from-expect-throws.md index 7eee79538..502a18e17 100644 --- a/Documentation/Proposals/0006-return-errors-from-expect-throws.md +++ b/Documentation/Proposals/0006-return-errors-from-expect-throws.md @@ -2,10 +2,10 @@ * Proposal: [SWT-0006](0006-return-errors-from-expect-throws.md) * Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Awaiting review** +* Status: **Implemented (Swift 6.1)** * Bug: rdar://138235250 * Implementation: [swiftlang/swift-testing#780](https://github.com/swiftlang/swift-testing/pull/780) -* Review: ([pitch](https://forums.swift.org/t/pitch-returning-errors-from-expect-throws/75567)) +* Review: ([pitch](https://forums.swift.org/t/pitch-returning-errors-from-expect-throws/75567)), ([acceptance](https://forums.swift.org/t/pitch-returning-errors-from-expect-throws/75567/5)) ## Introduction diff --git a/Documentation/Proposals/0007-test-scoping-traits.md b/Documentation/Proposals/0007-test-scoping-traits.md index 3b0471550..9d01bbbc7 100644 --- a/Documentation/Proposals/0007-test-scoping-traits.md +++ b/Documentation/Proposals/0007-test-scoping-traits.md @@ -2,9 +2,9 @@ * Proposal: [SWT-0007](0007-test-scoping-traits.md) * Authors: [Stuart Montgomery](https://github.com/stmontgomery) -* Status: **Awaiting review** +* Status: **Implemented (Swift 6.1)** * Implementation: [swiftlang/swift-testing#733](https://github.com/swiftlang/swift-testing/pull/733), [swiftlang/swift-testing#86](https://github.com/swiftlang/swift-testing/pull/86) -* Review: ([pitch](https://forums.swift.org/t/pitch-custom-test-execution-traits/75055)) +* Review: ([pitch](https://forums.swift.org/t/pitch-custom-test-execution-traits/75055)), ([review](https://forums.swift.org/t/proposal-test-scoping-traits/76676)), ([acceptance](https://forums.swift.org/t/proposal-test-scoping-traits/76676/3)) ### Revision history From 47cacd296dd17307715bf1b19ca90859d45da9bb Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 17 Feb 2025 11:58:39 -0600 Subject: [PATCH 088/234] Improve the project's GitHub Issue templates (#962) This includes several improvements to this project's GitHub Issue templates. ### Motivation: Some of these changes are meant to guide users to provide more useful information when filing issues, or point them to other places when appropriate. Other changes are meant to help contributors and maintainers better organize issues in the project. ### Modifications: - Add a `config.yml` to add several related links in the New Issue template chooser to related places like the our Forums category, the bug report links for related tools, and documentation. - Modify the "Bug Report" template: - Rename it to "Report a bug" and clarify its description. - Modify its fields to more clearly explain each question and consolidate some. - Modify the "Feature Request" template: - Rename it to "Request a change" and clarify its description. - Modify fields to be more relevant to a change request. - Add a "Task" template. - Added a (newly created) `triage-needed` label to the "Bug Report" and "Request a Change" templates so we can distinguish those which have not yet been triaged. - Add notes to the top of several templates about how they're not meant for Xcode feedback, and directing major ideas to the Forums instead. - Add copyright header to all template-related files. - Add numbers to the template files to control their display order. - Switch to kebab-case file names for the templates. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .github/ISSUE_TEMPLATE/01-bug-report.yml | 74 ++++++++++++++++++++ .github/ISSUE_TEMPLATE/02-change-request.yml | 68 ++++++++++++++++++ .github/ISSUE_TEMPLATE/03-task.yml | 36 ++++++++++ .github/ISSUE_TEMPLATE/BUG_REPORT.yml | 39 ----------- .github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml | 39 ----------- .github/ISSUE_TEMPLATE/config.yml | 43 ++++++++++++ 6 files changed, 221 insertions(+), 78 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/01-bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/02-change-request.yml create mode 100644 .github/ISSUE_TEMPLATE/03-task.yml delete mode 100644 .github/ISSUE_TEMPLATE/BUG_REPORT.yml delete mode 100644 .github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/01-bug-report.yml b/.github/ISSUE_TEMPLATE/01-bug-report.yml new file mode 100644 index 000000000..ec821ae82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/01-bug-report.yml @@ -0,0 +1,74 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for Swift project authors + +name: 🪲 Report a bug +description: > + Report a deviation from expected or documented behavior. +labels: [bug, triage-needed] +body: + - type: markdown + attributes: + value: > + This repository hosts the Swift Testing library and its documentation. + It does _not_ track feedback for Xcode and other closed source Apple + developer software such as XCTest; please direct that to + [Feedback Assistant](https://developer.apple.com/bug-reporting) instead. + - type: textarea + attributes: + label: Description + description: > + A concise description of what causes the problem, in human language. + Though not required, it may help us to more accurately triage the issue + as well as understand a non-trivial test case. + validations: + required: false + - type: textarea + attributes: + label: Reproduction + description: > + Provide an example, preferably in a Markdown code block, and explain how + to build or run it to reproduce the problem. If the problem is a poor or + unexpected diagnostic, fix-it, or other output, please show this output + as is. For example, paste it from the terminal. Consider reducing the + example to the smallest amount of code possible — a smaller example is + easier to reason about and more appealing to contributors. + value: | + ```swift + + ``` + validations: + required: true + - type: textarea + attributes: + label: Expected behavior + description: Describe the behavior you expected. + validations: + required: true + - type: textarea + attributes: + label: Environment + description: > + Provide details about the environment in which this problem occurs. + Include the versions of Swift Testing and the Swift toolchain. If you + suspect the problem might be specific to a particular platform, please + specify the platform and OS version as well. + placeholder: | + Swift Testing version: (shown in `swift test` output) + $ swift --version + $ uname -a + validations: + required: true + - type: textarea + attributes: + label: Additional information + description: > + Any complementary information that could help others to work around the + problem, and us to better understand the problem and its impact. For + example, a link to a discussion or post that motivated this report. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/02-change-request.yml b/.github/ISSUE_TEMPLATE/02-change-request.yml new file mode 100644 index 000000000..4647a4560 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/02-change-request.yml @@ -0,0 +1,68 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for Swift project authors + +name: 🌟 Request a change +description: > + Request a feature, API, improvement, or other change. +labels: [enhancement, triage-needed] +body: + - type: markdown + attributes: + value: > + This repository hosts the Swift Testing library and its documentation. + It does _not_ track feedback for Xcode and other closed source Apple + developer software such as XCTest; please direct that to + [Feedback Assistant](https://developer.apple.com/bug-reporting) instead. + + ___ + + Swift Testing is guided by a community-driven evolution process. + Submitting this form is not a guarantee that the request will be + considered or implemented. If the request implies an addition, removal, + or change to the features of Swift Testing or its public interfaces, + please consider socializing it on the + [Swift forums](https://forums.swift.org/c/related-projects/swift-testing) + instead. The Swift forums are the preferred space for sharing ideas and + discussing them with the Swift community. + - type: textarea + attributes: + label: Motivation + description: > + Describe the problems that this idea seeks to address. If the problem is + that some common pattern is currently hard to express, show how one can + currently get a similar effect and describe its drawbacks. If it's + completely new functionality that cannot be emulated, motivate why this + new functionality would help create better Swift tests. + validations: + required: true + - type: textarea + attributes: + label: Proposed solution + description: > + Describe the proposed solution to the problem. Provide examples and + describe how they work. Show how this solution is better than current + workarounds: is it cleaner, safer, or more efficient? + validations: + required: true + - type: textarea + attributes: + label: Alternatives considered + description: > + Any alternative approaches that were considered, and why the _proposed + solution_ was chosen instead. + validations: + required: false + - type: textarea + attributes: + label: Additional information + description: > + Any complementary information that could be valuable to an author of a + formal proposal, an implementor, or future discussions. For example, a + link to a discussion or post that motivated this request. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/03-task.yml b/.github/ISSUE_TEMPLATE/03-task.yml new file mode 100644 index 000000000..92ed000bc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/03-task.yml @@ -0,0 +1,36 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for Swift project authors + +name: ⚙️ Track a task +description: > + Tasks can be used to track internal work, extract individual subtasks from a + larger issue, or can serve as umbrella issues themselves. +labels: [] +body: + - type: markdown + attributes: + value: > + This repository hosts the Swift Testing library and its documentation. + It does _not_ track feedback for Xcode and other closed source Apple + developer software such as XCTest; please direct that to + [Feedback Assistant](https://developer.apple.com/bug-reporting) instead. + - type: textarea + attributes: + label: Description + description: > + A comprehensive description of the task, in human language. + validations: + required: true + - type: textarea + attributes: + label: Additional information + description: > + Any complementary information that could be valuable to an implementor. + For example, a link to a discussion or post that motivated this task. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.yml b/.github/ISSUE_TEMPLATE/BUG_REPORT.yml deleted file mode 100644 index 2cace0484..000000000 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Bug Report -description: Something isn't working as expected -labels: [bug] -body: -- type: textarea - attributes: - label: Description - validations: - required: true -- type: textarea - attributes: - label: Expected behavior - description: What you expected to happen. - validations: - required: false -- type: textarea - attributes: - label: Actual behavior - description: What actually happened. - validations: - required: false -- type: textarea - attributes: - label: Steps to reproduce - placeholder: | - 1. ... - 2. ... - validations: - required: false -- type: input - attributes: - label: swift-testing version/commit hash - validations: - required: false -- type: textarea - attributes: - label: Swift & OS version (output of `swift --version ; uname -a`) - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml deleted file mode 100644 index 0f9332815..000000000 --- a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Feature Request -description: A suggestion for a new feature -labels: [enhancement] -body: -- type: textarea - attributes: - label: Description - validations: - required: true -- type: textarea - attributes: - label: Expected behavior - description: What you expected to happen. - validations: - required: false -- type: textarea - attributes: - label: Actual behavior - description: What actually happened. - validations: - required: false -- type: textarea - attributes: - label: Steps to reproduce - placeholder: | - 1. ... - 2. ... - validations: - required: false -- type: input - attributes: - label: swift-testing version/commit hash - validations: - required: false -- type: textarea - attributes: - label: Swift & OS version (output of `swift --version && uname -a`) - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..3add3b3e3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,43 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for Swift project authors + +blank_issues_enabled: true +contact_links: + - name: 🌐 Discuss an idea + url: https://forums.swift.org/c/related-projects/swift-testing + about: > + Share an idea with the Swift Testing community. + - name: 📄 Formally propose a change + url: https://github.com/swiftlang/swift-testing/blob/main/Documentation/Proposals/0000-proposal-template.md + about: > + Formally propose an addition, removal, or change to the APIs or features + of Swift Testing. + - name: 🙋 Ask a question + url: https://forums.swift.org/c/related-projects/swift-testing + about: > + Ask a question about or get help with Swift Testing. Beginner questions + welcome! + - name: 🪲 Report an issue with Swift Package Manager + url: https://github.com/swiftlang/swift-package-manager/issues/new/choose + about: > + Report an issue with Swift Package Manager, which includes the + "swift test" CLI tool. + - name: 🪲 Report an issue with Apple software using Feedback Assistant + url: https://developer.apple.com/bug-reporting + about: > + Report an issue with Xcode or other closed source Apple developer + software such as XCTest. + - name: 🪲 Report an issue with the VS Code Swift plugin + url: https://github.com/swiftlang/vscode-swift/issues/new/choose + about: > + Report an issue with the Swift plugin for VS Code, which integrates with + Swift Testing. + - name: 📖 Learn about Swift Testing + url: https://swiftpackageindex.com/swiftlang/swift-testing/main/documentation/testing + about: > + Read the official Swift Testing documentation. From 16f58461685fce4d0139a610b30432e8047ce334 Mon Sep 17 00:00:00 2001 From: Graham Lee Date: Tue, 18 Feb 2025 15:58:12 +0000 Subject: [PATCH 089/234] Add an overview of test serialization to the XCTest migration guide (#960) Describe test serialization in the article about migrating tests from XCTest. ### Motivation: Because XCTest runs tests in a suite serially by default, tests that people migrate from XCTest may encounter issues if they run in parallel. ### Modifications: Add an example of serializing a test suite to the migration guide, along with links to the docs about test parallelization. ### Result: The documentation on migrating tests from XCTest includes guidance on serializing tests within a suite. ### Checklist: - [X] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [X] If public symbols are renamed or modified, DocC references should be updated. --- .../Testing.docc/MigratingFromXCTest.md | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index 133daa49c..2e356ef19 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -273,7 +273,7 @@ with optional expressions to unwrap them: ### Record issues -Finally, XCTest has a function, [`XCTFail()`](https://developer.apple.com/documentation/xctest/1500970-xctfail), +XCTest has a function, [`XCTFail()`](https://developer.apple.com/documentation/xctest/1500970-xctfail), that causes a test to fail immediately and unconditionally. This function is useful when the syntax of the language prevents the use of an `XCTAssert()` function. To record an unconditional issue using the testing library, use the @@ -692,6 +692,56 @@ of issues: } } +### Run tests sequentially + +By default, the testing library runs all tests in a suite in parallel. The +default behavior of XCTest is to run each test in a suite sequentially. If your +tests use shared state such as global variables, you may see unexpected +behavior including unreliable test outcomes when you run tests in parallel. + +Annotate your test suite with ``Trait/serialized`` to run tests within that +suite serially: + +@Row { + @Column { + ```swift + // Before + class RefrigeratorTests : XCTestCase { + func testLightComesOn() throws { + try FoodTruck.shared.refrigerator.openDoor() + XCTAssertEqual(FoodTruck.shared.refrigerator.lightState == .on) + } + + func testLightGoesOut() throws { + try FoodTruck.shared.refrigerator.openDoor() + try FoodTruck.shared.refrigerator.closeDoor() + XCTAssertEqual(FoodTruck.shared.refrigerator.lightState == .off) + } + } + ``` + } + @Column { + ```swift + // After + @Suite(.serialized) + class RefrigeratorTests { + @Test func lightComesOn() throws { + try FoodTruck.shared.refrigerator.openDoor() + #expect(FoodTruck.shared.refrigerator.lightState == .on) + } + + @Test func lightGoesOut() throws { + try FoodTruck.shared.refrigerator.openDoor() + try FoodTruck.shared.refrigerator.closeDoor() + #expect(FoodTruck.shared.refrigerator.lightState == .off) + } + } + ``` + } +} + +For more information, see . + ## See Also - From 85bfa51cfb277537ccf5ce0b97f516966fc1e18e Mon Sep 17 00:00:00 2001 From: Graham Lee Date: Tue, 18 Feb 2025 16:57:10 +0000 Subject: [PATCH 090/234] Corrects use of XCTAssertEqual macro in parallelization examples. (#963) Fixes a typo in the "before" example of adopting test serialization. ### Motivation: The example of XCTest tests that need to run in series doesn't compile. ### Modifications: Correct the code in the example. ### Result: A documentation change that means the code snippet in the example is correct. ### Checklist: - [X] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [X] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Testing.docc/MigratingFromXCTest.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index 2e356ef19..44d91b5b0 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -709,13 +709,13 @@ suite serially: class RefrigeratorTests : XCTestCase { func testLightComesOn() throws { try FoodTruck.shared.refrigerator.openDoor() - XCTAssertEqual(FoodTruck.shared.refrigerator.lightState == .on) + XCTAssertEqual(FoodTruck.shared.refrigerator.lightState, .on) } func testLightGoesOut() throws { try FoodTruck.shared.refrigerator.openDoor() try FoodTruck.shared.refrigerator.closeDoor() - XCTAssertEqual(FoodTruck.shared.refrigerator.lightState == .off) + XCTAssertEqual(FoodTruck.shared.refrigerator.lightState, .off) } } ``` From 7efce3917b6d0091532c2a9adc2329d58ab73b96 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 19 Feb 2025 10:23:26 -0500 Subject: [PATCH 091/234] Plumb the schema version through the JSON ABI layers. (#956) This PR plumbs the schema version (0 or 1) through the Swift types and functions that produce our JSON output, allowing them to change behaviour based on which version is in use. As a proof-of-concept as much as anything else, this PR also adds an (unsupported) `_tags` field to the JSON output for a test record, but only with ABI version 1. I've split this PR into two commits: the first plumbs the version through as an integer argument, while the second uses the type system to represent different ABI versions. The latter uses a protocol that we can extend so that different ABI versions have differing behaviours (although that's too coarse-grained for things like `_tags`.) ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Testing/ABI/ABI.Record+Streaming.swift | 105 +++++------------- Sources/Testing/ABI/ABI.Record.swift | 38 ++++--- Sources/Testing/ABI/ABI.swift | 66 ++++++++++- .../ABI/Encoded/ABI.EncodedAttachment.swift | 2 +- .../ABI/Encoded/ABI.EncodedBacktrace.swift | 2 +- .../ABI/Encoded/ABI.EncodedError.swift | 2 +- .../ABI/Encoded/ABI.EncodedEvent.swift | 14 +-- .../ABI/Encoded/ABI.EncodedInstant.swift | 2 +- .../ABI/Encoded/ABI.EncodedIssue.swift | 7 +- .../ABI/Encoded/ABI.EncodedMessage.swift | 2 +- .../Testing/ABI/Encoded/ABI.EncodedTest.swift | 22 +++- .../ABI/EntryPoints/ABIEntryPoint.swift | 34 +++--- .../Testing/ABI/EntryPoints/EntryPoint.swift | 74 ++++++++---- Sources/Testing/ExitTests/ExitTest.swift | 14 ++- Tests/TestingTests/ABIEntryPointTests.swift | 25 ++++- Tests/TestingTests/SwiftPMTests.swift | 56 +++++++--- 16 files changed, 290 insertions(+), 175 deletions(-) diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 482d5af18..4f2f2944d 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) && (!SWT_NO_FILE_IO || !SWT_NO_ABI_ENTRY_POINT) -extension ABI.Record { +extension ABI.Version { /// Post-process encoded JSON and write it to a file. /// /// - Parameters: @@ -43,25 +43,6 @@ extension ABI.Record { } } - /// Create an event handler that encodes events as JSON and forwards them to - /// an ABI-friendly event handler. - /// - /// - Parameters: - /// - encodeAsJSONLines: Whether or not to ensure JSON passed to - /// `eventHandler` is encoded as JSON Lines (i.e. that it does not contain - /// extra newlines.) - /// - eventHandler: The event handler to forward events to. See - /// ``ABIv0/EntryPoint-swift.typealias`` for more information. - /// - /// - Returns: An event handler. - /// - /// The resulting event handler outputs data as JSON. For each event handled - /// by the resulting event handler, a JSON object representing it and its - /// associated context is created and is passed to `eventHandler`. - /// - /// Note that ``configurationForEntryPoint(from:)`` calls this function and - /// performs additional postprocessing before writing JSON data to ensure it - /// does not contain any newline characters. static func eventHandler( encodeAsJSONLines: Bool, forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void @@ -75,12 +56,12 @@ extension ABI.Record { let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder() return { [eventHandler = eventHandlerCopy] event, context in if case .testDiscovered = event.kind, let test = context.test { - try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in + try? JSON.withEncoding(of: ABI.Record(encoding: test)) { testJSON in eventHandler(testJSON) } } else { let messages = humanReadableOutputRecorder.record(event, in: context, verbosity: 0) - if let eventRecord = Self(encoding: event, in: context, messages: messages) { + if let eventRecord = ABI.Record(encoding: event, in: context, messages: messages) { try? JSON.withEncoding(of: eventRecord, eventHandler) } } @@ -89,63 +70,33 @@ extension ABI.Record { } #if !SWT_NO_SNAPSHOT_TYPES -// MARK: - Experimental event streaming +// MARK: - Xcode 16 Beta 1 compatibility -/// A type containing an event snapshot and snapshots of the contents of an -/// event context suitable for streaming over JSON. -/// -/// This type is not part of the public interface of the testing library. -/// External adopters are not necessarily written in Swift and are expected to -/// decode the JSON produced for this type in implementation-specific ways. -/// -/// - Warning: This type supports early Xcode 16 betas and will be removed in a -/// future update. -struct EventAndContextSnapshot { - /// A snapshot of the event. - var event: Event.Snapshot - - /// A snapshot of the event context. - var eventContext: Event.Context.Snapshot -} - -extension EventAndContextSnapshot: Codable {} +extension ABI.Xcode16Beta1 { + static func eventHandler( + encodeAsJSONLines: Bool, + forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void + ) -> Event.Handler { + return { event, context in + if case .testDiscovered = event.kind { + // Discard events of this kind rather than forwarding them to avoid a + // crash in Xcode 16 Beta 1 (which does not expect any events to occur + // before .runStarted.) + return + } -/// Create an event handler that encodes events as JSON and forwards them to an -/// ABI-friendly event handler. -/// -/// - Parameters: -/// - eventHandler: The event handler to forward events to. See -/// ``ABIv0/EntryPoint-swift.typealias`` for more information. -/// -/// - Returns: An event handler. -/// -/// The resulting event handler outputs data as JSON. For each event handled by -/// the resulting event handler, a JSON object representing it and its -/// associated context is created and is passed to `eventHandler`. -/// -/// Note that ``configurationForEntryPoint(from:)`` calls this function and -/// performs additional postprocessing before writing JSON data to ensure it -/// does not contain any newline characters. -/// -/// - Warning: This function supports early Xcode 16 betas and will be removed -/// in a future update. -func eventHandlerForStreamingEventSnapshots( - to eventHandler: @escaping @Sendable (_ eventAndContextJSON: UnsafeRawBufferPointer) -> Void -) -> Event.Handler { - return { event, context in - if case .testDiscovered = event.kind { - // Discard events of this kind rather than forwarding them to avoid a - // crash in Xcode 16 Beta 1 (which does not expect any events to occur - // before .runStarted.) - return - } - let snapshot = EventAndContextSnapshot( - event: Event.Snapshot(snapshotting: event), - eventContext: Event.Context.Snapshot(snapshotting: context) - ) - try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in - eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in - eventHandler(eventAndContextJSON) + struct EventAndContextSnapshot: Codable { + var event: Event.Snapshot + var eventContext: Event.Context.Snapshot + } + let snapshot = EventAndContextSnapshot( + event: Event.Snapshot(snapshotting: event), + eventContext: Event.Context.Snapshot(snapshotting: context) + ) + try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in + eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in + eventHandler(eventAndContextJSON) + } } } } diff --git a/Sources/Testing/ABI/ABI.Record.swift b/Sources/Testing/ABI/ABI.Record.swift index 6dcccd6f9..74ac7f9aa 100644 --- a/Sources/Testing/ABI/ABI.Record.swift +++ b/Sources/Testing/ABI/ABI.Record.swift @@ -15,19 +15,14 @@ extension ABI { /// This type is not part of the public interface of the testing library. It /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. - struct Record: Sendable { - /// The version of this record. - /// - /// The value of this property corresponds to the ABI version i.e. `0`. - var version = 0 - + struct Record: Sendable where V: ABI.Version { /// An enumeration describing the various kinds of record. enum Kind: Sendable { /// A test record. - case test(EncodedTest) + case test(EncodedTest) /// An event record. - case event(EncodedEvent) + case event(EncodedEvent) } /// The kind of record. @@ -38,7 +33,7 @@ extension ABI { } init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) { - guard let event = EncodedEvent(encoding: event, in: eventContext, messages: messages) else { + guard let event = EncodedEvent(encoding: event, in: eventContext, messages: messages) else { return nil } kind = .event(event) @@ -57,7 +52,7 @@ extension ABI.Record: Codable { func encode(to encoder: any Encoder) throws { var container = encoder.container(keyedBy: CodingKeys.self) - try container.encode(version, forKey: .version) + try container.encode(V.versionNumber, forKey: .version) switch kind { case let .test(test): try container.encode("test", forKey: .kind) @@ -70,16 +65,31 @@ extension ABI.Record: Codable { init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) - version = try container.decode(Int.self, forKey: .version) + + let versionNumber = try container.decode(Int.self, forKey: .version) + if versionNumber != V.versionNumber { + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath + CollectionOfOne(CodingKeys.version as any CodingKey), + debugDescription: "Unexpected record version \(versionNumber) (expected \(V.versionNumber))." + ) + ) + } + switch try container.decode(String.self, forKey: .kind) { case "test": - let test = try container.decode(ABI.EncodedTest.self, forKey: .payload) + let test = try container.decode(ABI.EncodedTest.self, forKey: .payload) kind = .test(test) case "event": - let event = try container.decode(ABI.EncodedEvent.self, forKey: .payload) + let event = try container.decode(ABI.EncodedEvent.self, forKey: .payload) kind = .event(event) case let kind: - throw DecodingError.dataCorrupted(.init(codingPath: decoder.codingPath, debugDescription: "Unrecognized record kind '\(kind)'")) + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath + CollectionOfOne(CodingKeys.kind as any CodingKey), + debugDescription: "Unrecognized record kind '\(kind)'" + ) + ) } } } diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index cd952acca..7926eb324 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -12,16 +12,70 @@ @_spi(ForToolsIntegrationOnly) public enum ABI: Sendable {} -// MARK: - +// MARK: - ABI version abstraction + +extension ABI { + /// A protocol describing the types that represent different ABI versions. + protocol Version: Sendable { + /// The numeric representation of this ABI version. + static var versionNumber: Int { get } + + /// Create an event handler that encodes events as JSON and forwards them to + /// an ABI-friendly event handler. + /// + /// - Parameters: + /// - encodeAsJSONLines: Whether or not to ensure JSON passed to + /// `eventHandler` is encoded as JSON Lines (i.e. that it does not + /// contain extra newlines.) + /// - eventHandler: The event handler to forward events to. + /// + /// - Returns: An event handler. + /// + /// The resulting event handler outputs data as JSON. For each event handled + /// by the resulting event handler, a JSON object representing it and its + /// associated context is created and is passed to `eventHandler`. + static func eventHandler( + encodeAsJSONLines: Bool, + forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void + ) -> Event.Handler + } + + /// The current supported ABI version (ignoring any experimental versions.) + typealias CurrentVersion = v0 +} + +// MARK: - Concrete ABI versions -@_spi(ForToolsIntegrationOnly) extension ABI { - /// A namespace for ABI version 0 symbols. - public enum v0: Sendable {} +#if !SWT_NO_SNAPSHOT_TYPES + /// A namespace and version type for Xcode 16 Beta 1 compatibility. + /// + /// - Warning: This type will be removed in a future update. + enum Xcode16Beta1: Sendable, Version { + static var versionNumber: Int { + -1 + } + } +#endif + + /// A namespace and type for ABI version 0 symbols. + public enum v0: Sendable, Version { + static var versionNumber: Int { + 0 + } + } - /// A namespace for ABI version 1 symbols. + /// A namespace and type for ABI version 1 symbols. + /// + /// @Metadata { + /// @Available("Swift Testing ABI", introduced: 1) + /// } @_spi(Experimental) - public enum v1: Sendable {} + public enum v1: Sendable, Version { + static var versionNumber: Int { + 1 + } + } } /// A namespace for ABI version 0 symbols. diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift index e9cbdeacd..7668f778a 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedAttachment.swift @@ -17,7 +17,7 @@ extension ABI { /// expected to write their own decoders. /// /// - Warning: Attachments are not yet part of the JSON schema. - struct EncodedAttachment: Sendable { + struct EncodedAttachment: Sendable where V: ABI.Version { /// The path where the attachment was written. var path: String? diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedBacktrace.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedBacktrace.swift index 9146a040a..fcfa5cc37 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedBacktrace.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedBacktrace.swift @@ -17,7 +17,7 @@ extension ABI { /// expected to write their own decoders. /// /// - Warning: Backtraces are not yet part of the JSON schema. - struct EncodedBacktrace: Sendable { + struct EncodedBacktrace: Sendable where V: ABI.Version { /// The frames in the backtrace. var symbolicatedAddresses: [Backtrace.SymbolicatedAddress] diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedError.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedError.swift index eb492b7d1..3a299cd3f 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedError.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedError.swift @@ -17,7 +17,7 @@ extension ABI { /// expected to write their own decoders. /// /// - Warning: Errors are not yet part of the JSON schema. - struct EncodedError: Sendable { + struct EncodedError: Sendable where V: ABI.Version { /// The error's description var description: String diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index f502873f3..b8bafdde1 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -15,7 +15,7 @@ extension ABI { /// This type is not part of the public interface of the testing library. It /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. - struct EncodedEvent: Sendable { + struct EncodedEvent: Sendable where V: ABI.Version { /// An enumeration describing the various kinds of event. /// /// Note that the set of encodable events is a subset of all events @@ -38,13 +38,13 @@ extension ABI { var kind: Kind /// The instant at which the event occurred. - var instant: EncodedInstant + var instant: EncodedInstant /// The issue that occurred, if any. /// /// The value of this property is `nil` unless the value of the /// ``kind-swift.property`` property is ``Kind-swift.enum/issueRecorded``. - var issue: EncodedIssue? + var issue: EncodedIssue? /// The value that was attached, if any. /// @@ -52,19 +52,19 @@ extension ABI { /// ``kind-swift.property`` property is ``Kind-swift.enum/valueAttached``. /// /// - Warning: Attachments are not yet part of the JSON schema. - var _attachment: EncodedAttachment? + var _attachment: EncodedAttachment? /// Human-readable messages associated with this event that can be presented /// to the user. - var messages: [EncodedMessage] + var messages: [EncodedMessage] /// The ID of the test associated with this event, if any. - var testID: EncodedTest.ID? + var testID: EncodedTest.ID? /// The ID of the test case associated with this event, if any. /// /// - Warning: Test cases are not yet part of the JSON schema. - var _testCase: EncodedTestCase? + var _testCase: EncodedTestCase? init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) { switch event.kind { diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedInstant.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedInstant.swift index 1f05e4241..9a71ddbfe 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedInstant.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedInstant.swift @@ -15,7 +15,7 @@ extension ABI { /// This type is not part of the public interface of the testing library. It /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. - struct EncodedInstant: Sendable { + struct EncodedInstant: Sendable where V: ABI.Version { /// The number of seconds since the system-defined suspending epoch. /// /// For more information, see [`SuspendingClock`](https://developer.apple.com/documentation/swift/suspendingclock). diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift index 3529dfc98..0ea218cc8 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift @@ -15,7 +15,7 @@ extension ABI { /// This type is not part of the public interface of the testing library. It /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. - struct EncodedIssue: Sendable { + struct EncodedIssue: Sendable where V: ABI.Version { /// An enumeration representing the level of severity of a recorded issue. /// /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``. @@ -38,19 +38,18 @@ extension ABI { /// The backtrace where this issue occurred, if available. /// /// - Warning: Backtraces are not yet part of the JSON schema. - var _backtrace: EncodedBacktrace? + var _backtrace: EncodedBacktrace? /// The error associated with this issue, if applicable. /// /// - Warning: Errors are not yet part of the JSON schema. - var _error: EncodedError? + var _error: EncodedError? init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { _severity = switch issue.severity { case .warning: .warning case .error: .error } - isKnown = issue.isKnown sourceLocation = issue.sourceLocation if let backtrace = issue.sourceContext.backtrace { diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift index a3aa9575d..8f993ecb7 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift @@ -16,7 +16,7 @@ extension ABI { /// This type is not part of the public interface of the testing library. It /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. - struct EncodedMessage: Sendable { + struct EncodedMessage: Sendable where V: ABI.Version { /// A type implementing the JSON encoding of ``Event/Symbol`` for the ABI /// entry point and event stream output. /// diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index 533c8bb12..8a6d4518a 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -17,7 +17,7 @@ extension ABI { /// This type is not part of the public interface of the testing library. It /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. - struct EncodedTest: Sendable { + struct EncodedTest: Sendable where V: ABI.Version { /// An enumeration describing the various kinds of test. enum Kind: String, Sendable { /// A test suite. @@ -65,7 +65,7 @@ extension ABI { /// The test cases in this test, if it is a parameterized test function. /// /// - Warning: Test cases are not yet part of the JSON schema. - var _testCases: [EncodedTestCase]? + var _testCases: [EncodedTestCase]? /// Whether or not the test is parameterized. /// @@ -73,6 +73,15 @@ extension ABI { /// is `nil`. var isParameterized: Bool? + /// The tags associated with the test. + /// + /// - Warning: Tags are not yet part of the JSON schema. + /// + /// @Metadata { + /// @Available("Swift Testing ABI", introduced: 1) + /// } + var _tags: [String]? + init(encoding test: borrowing Test) { if test.isSuite { kind = .suite @@ -88,6 +97,13 @@ extension ABI { displayName = test.displayName sourceLocation = test.sourceLocation id = ID(encoding: test.id) + + if V.versionNumber >= 1 { + let tags = test.tags + if !tags.isEmpty { + _tags = tags.map(String.init(describing:)) + } + } } } } @@ -103,7 +119,7 @@ extension ABI { /// expected to write their own decoders. /// /// - Warning: Test cases are not yet part of the JSON schema. - struct EncodedTestCase: Sendable { + struct EncodedTestCase: Sendable where V: ABI.Version { var id: String var displayName: String diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index e0063bdd4..b63bc6da3 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -56,9 +56,9 @@ extension ABI.v0 { } /// An exported C function that is the equivalent of -/// ``ABIv0/entryPoint-swift.type.property``. +/// ``ABI/v0/entryPoint-swift.type.property``. /// -/// - Returns: The value of ``ABIv0/entryPoint-swift.type.property`` cast to an +/// - Returns: The value of ``ABI/v0/entryPoint-swift.type.property`` cast to an /// untyped pointer. @_cdecl("swt_abiv0_getEntryPoint") @usableFromInline func abiv0_getEntryPoint() -> UnsafeRawPointer { @@ -68,24 +68,26 @@ extension ABI.v0 { #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Xcode 16 Beta 1 compatibility -/// An older signature for ``ABIv0/EntryPoint-swift.typealias`` used by Xcode 16 -/// Beta 1. -/// -/// This type will be removed in a future update. -@available(*, deprecated, message: "Use ABI.v0.EntryPoint instead.") -typealias ABIEntryPoint_v0 = @Sendable ( - _ argumentsJSON: UnsafeRawBufferPointer?, - _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void -) async throws -> CInt +extension ABI.Xcode16Beta1 { + /// An older signature for ``ABI/v0/EntryPoint-swift.typealias`` used by Xcode + /// 16 Beta 1. + /// + /// - Warning: This type will be removed in a future update. + @available(*, deprecated, message: "Use ABI.v0.EntryPoint instead.") + typealias EntryPoint = @Sendable ( + _ argumentsJSON: UnsafeRawBufferPointer?, + _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void + ) async throws -> CInt +} -/// An older signature for ``ABIv0/entryPoint-swift.type.property`` used by +/// An older signature for ``ABI/v0/entryPoint-swift.type.property`` used by /// Xcode 16 Beta 1. /// -/// This function will be removed in a future update. +/// - Warning: This function will be removed in a future update. @available(*, deprecated, message: "Use ABI.v0.entryPoint (swt_abiv0_getEntryPoint()) instead.") @_cdecl("swt_copyABIEntryPoint_v0") @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { - let result = UnsafeMutablePointer.allocate(capacity: 1) + let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in try await _entryPoint( configurationJSON: configurationJSON, @@ -99,11 +101,11 @@ typealias ABIEntryPoint_v0 = @Sendable ( // MARK: - -/// A common implementation for ``ABIv0/entryPoint-swift.type.property`` and +/// A common implementation for ``ABI/v0/entryPoint-swift.type.property`` and /// ``copyABIEntryPoint_v0()`` that provides Xcode 16 Beta 1 compatibility. /// /// This function will be removed (with its logic incorporated into -/// ``ABIv0/entryPoint-swift.type.property``) in a future update. +/// ``ABI/v0/entryPoint-swift.type.property``) in a future update. private func _entryPoint( configurationJSON: UnsafeRawBufferPointer?, eventStreamVersionIfNil: Int? = nil, diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index f6e2cd602..d2ffd54b1 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -23,8 +23,8 @@ private import _TestingInternals /// - Returns: An exit code representing the result of running tests. /// /// External callers cannot call this function directly. The can use -/// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable -/// version of this function. +/// ``ABI/v0/entryPoint-swift.type.property`` to get a reference to an +/// ABI-stable version of this function. func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Handler?) async -> CInt { let exitCode = Locked(rawValue: EXIT_SUCCESS) @@ -259,8 +259,8 @@ public struct __CommandLineArguments_v0: Sendable { /// ``eventStreamOutput``. /// /// The corresponding stable schema is used to encode events to the event - /// stream (for example, ``ABIv0/Record`` is used if the value of this - /// property is `0`.) + /// stream. ``ABI/Record`` is used if the value of this property is `0` or + /// higher. /// /// If the value of this property is `nil`, the testing library assumes that /// the newest available schema should be used. @@ -359,15 +359,34 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum // respected (it should be the least "surprising" outcome of passing both.) } - // Event stream output (experimental) + // Event stream output if let eventOutputIndex = args.firstIndex(of: "--event-stream-output-path") ?? args.firstIndex(of: "--experimental-event-stream-output"), !isLastArgument(at: eventOutputIndex) { result.eventStreamOutputPath = args[args.index(after: eventOutputIndex)] } - // Event stream output (experimental) - if let eventOutputVersionIndex = args.firstIndex(of: "--event-stream-version") ?? args.firstIndex(of: "--experimental-event-stream-version"), - !isLastArgument(at: eventOutputVersionIndex) { - result.eventStreamVersion = Int(args[args.index(after: eventOutputVersionIndex)]) + // Event stream version + do { + var eventOutputVersionIndex: Array.Index? + var allowExperimental = false + eventOutputVersionIndex = args.firstIndex(of: "--event-stream-version") + if eventOutputVersionIndex == nil { + eventOutputVersionIndex = args.firstIndex(of: "--experimental-event-stream-version") + if eventOutputVersionIndex != nil { + allowExperimental = true + } + } + if let eventOutputVersionIndex, !isLastArgument(at: eventOutputVersionIndex) { + result.eventStreamVersion = Int(args[args.index(after: eventOutputVersionIndex)]) + + // If the caller specified an experimental ABI version, they must + // explicitly use --experimental-event-stream-version, otherwise it's + // treated as unsupported. + if let eventStreamVersion = result.eventStreamVersion, + eventStreamVersion > ABI.CurrentVersion.versionNumber, + !allowExperimental { + throw _EntryPointError.experimentalABIVersion(eventStreamVersion) + } + } } #endif @@ -590,32 +609,40 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr /// specified ABI version. /// /// - Parameters: -/// - version: The ABI version to use. +/// - versionNumber: The numeric value of the ABI version to use. /// - encodeAsJSONLines: Whether or not to ensure JSON passed to /// `eventHandler` is encoded as JSON Lines (i.e. that it does not contain /// extra newlines.) -/// - eventHandler: The event handler to forward encoded events to. The +/// - targetEventHandler: The event handler to forward encoded events to. The /// encoding of events depends on `version`. /// /// - Returns: An event handler. /// /// - Throws: If `version` is not a supported ABI version. func eventHandlerForStreamingEvents( - version: Int?, + version versionNumber: Int?, encodeAsJSONLines: Bool, - forwardingTo eventHandler: @escaping @Sendable (UnsafeRawBufferPointer) -> Void + forwardingTo targetEventHandler: @escaping @Sendable (UnsafeRawBufferPointer) -> Void ) throws -> Event.Handler { - switch version { + func eventHandler(for version: (some ABI.Version).Type) -> Event.Handler { + return version.eventHandler(encodeAsJSONLines: encodeAsJSONLines, forwardingTo: targetEventHandler) + } + + return switch versionNumber { + case nil: + eventHandler(for: ABI.CurrentVersion.self) #if !SWT_NO_SNAPSHOT_TYPES case -1: // Legacy support for Xcode 16 betas. Support for this undocumented version // will be removed in a future update. Do not use it. - eventHandlerForStreamingEventSnapshots(to: eventHandler) + eventHandler(for: ABI.Xcode16Beta1.self) #endif - case nil, 0, 1: - ABI.Record.eventHandler(encodeAsJSONLines: encodeAsJSONLines, forwardingTo: eventHandler) - case let .some(unsupportedVersion): - throw _EntryPointError.invalidArgument("--event-stream-version", value: "\(unsupportedVersion)") + case 0: + eventHandler(for: ABI.v0.self) + case 1: + eventHandler(for: ABI.v1.self) + case let .some(unsupportedVersionNumber): + throw _EntryPointError.invalidArgument("--event-stream-version", value: "\(unsupportedVersionNumber)") } } #endif @@ -772,6 +799,13 @@ private enum _EntryPointError: Error { /// - name: The name of the argument. /// - value: The invalid value. case invalidArgument(_ name: String, value: String) + + /// The specified ABI version is experimental, but the caller did not + /// use `--experimental-event-stream-version` to specify it. + /// + /// - Parameters: + /// - versionNumber: The experimental ABI version number. + case experimentalABIVersion(_ versionNumber: Int) } extension _EntryPointError: CustomStringConvertible { @@ -781,6 +815,8 @@ extension _EntryPointError: CustomStringConvertible { explanation case let .invalidArgument(name, value): #"Invalid value "\#(value)" for argument \#(name)"# + case let .experimentalABIVersion(versionNumber): + "Event stream version \(versionNumber) is experimental. Use --experimental-event-stream-version to enable it." } } } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 416a0fc29..7a3597bfa 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -348,6 +348,16 @@ func callExitTest( // MARK: - SwiftPM/tools integration +extension ABI { + /// The ABI version to use for encoding and decoding events sent over the back + /// channel. + /// + /// The back channel always uses the latest ABI version (even if experimental) + /// since both the producer and consumer use this exact version of the testing + /// library. + fileprivate typealias BackChannelVersion = v1 +} + @_spi(Experimental) @_spi(ForToolsIntegrationOnly) extension ExitTest { /// A handler that is invoked when an exit test starts. @@ -444,7 +454,7 @@ extension ExitTest { // Encode events as JSON and write them to the back channel file handle. // Only forward issue-recorded events. (If we start handling other kinds of // events in the future, we can forward them too.) - let eventHandler = ABI.Record.eventHandler(encodeAsJSONLines: true) { json in + let eventHandler = ABI.BackChannelVersion.eventHandler(encodeAsJSONLines: true) { json in _ = try? _backChannelForEntryPoint?.withLock { try _backChannelForEntryPoint?.write(json) try _backChannelForEntryPoint?.write("\n") @@ -692,7 +702,7 @@ extension ExitTest { /// /// - Throws: Any error encountered attempting to decode or process the JSON. private static func _processRecord(_ recordJSON: UnsafeRawBufferPointer, fromBackChannel backChannel: borrowing FileHandle) throws { - let record = try JSON.decode(ABI.Record.self, from: recordJSON) + let record = try JSON.decode(ABI.Record.self, from: recordJSON) if case let .event(event) = record.kind, let issue = event.issue { // Translate the issue back into a "real" issue and record it diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index 0f856e6bf..86a875e77 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -27,8 +27,8 @@ struct ABIEntryPointTests { arguments.verbosity = .min let result = try await _invokeEntryPointV0Experimental(passing: arguments) { recordJSON in - let record = try! JSON.decode(ABI.Record.self, from: recordJSON) - _ = record.version + let record = try! JSON.decode(ABI.Record.self, from: recordJSON) + _ = record.kind } #expect(result == EXIT_SUCCESS) @@ -62,7 +62,7 @@ struct ABIEntryPointTests { ) } #endif - let abiEntryPoint = copyABIEntryPoint_v0().assumingMemoryBound(to: ABIEntryPoint_v0.self) + let abiEntryPoint = copyABIEntryPoint_v0().assumingMemoryBound(to: ABI.Xcode16Beta1.EntryPoint.self) defer { abiEntryPoint.deinitialize(count: 1) abiEntryPoint.deallocate() @@ -89,8 +89,8 @@ struct ABIEntryPointTests { arguments.verbosity = .min let result = try await _invokeEntryPointV0(passing: arguments) { recordJSON in - let record = try! JSON.decode(ABI.Record.self, from: recordJSON) - _ = record.version + let record = try! JSON.decode(ABI.Record.self, from: recordJSON) + _ = record.kind } #expect(result) @@ -117,7 +117,7 @@ struct ABIEntryPointTests { try await confirmation("Test matched", expectedCount: 1...) { testMatched in _ = try await _invokeEntryPointV0(passing: arguments) { recordJSON in - let record = try! JSON.decode(ABI.Record.self, from: recordJSON) + let record = try! JSON.decode(ABI.Record.self, from: recordJSON) if case .test = record.kind { testMatched() } else { @@ -167,6 +167,19 @@ struct ABIEntryPointTests { _ = try JSON.decode(__CommandLineArguments_v0.self, from: emptyBuffer) } } + + @Test func decodeWrongRecordVersion() throws { + let record = ABI.Record(encoding: Test {}) + let error = try JSON.withEncoding(of: record) { recordJSON in + try #require(throws: DecodingError.self) { + _ = try JSON.decode(ABI.Record.self, from: recordJSON) + } + } + guard case let .dataCorrupted(context) = error else { + throw error + } + #expect(context.debugDescription == "Unexpected record version 1 (expected 0).") + } #endif } diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 770057b04..77e52319e 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -226,22 +226,24 @@ struct SwiftPMTests { #expect(args.parallel == false) } - func decodeABIv0RecordStream(fromFileAtPath path: String) throws -> [ABI.Record] { - try FileHandle(forReadingAtPath: path).readToEnd() - .split(whereSeparator: \.isASCIINewline) - .map { line in - try line.withUnsafeBytes { line in - try JSON.decode(ABI.Record.self, from: line) - } - } - } - @Test("--event-stream-output-path argument (writes to a stream and can be read back)", arguments: [ - ("--event-stream-output-path", "--event-stream-version", "0"), - ("--experimental-event-stream-output", "--experimental-event-stream-version", "0"), + ("--event-stream-output-path", "--event-stream-version", 0), + ("--experimental-event-stream-output", "--experimental-event-stream-version", 0), + ("--experimental-event-stream-output", "--experimental-event-stream-version", 1), ]) - func eventStreamOutput(outputArgumentName: String, versionArgumentName: String, version: String) async throws { + func eventStreamOutput(outputArgumentName: String, versionArgumentName: String, version: Int) async throws { + switch version { + case 0: + try await eventStreamOutput(outputArgumentName: outputArgumentName, versionArgumentName: versionArgumentName, version: ABI.v0.self) + case 1: + try await eventStreamOutput(outputArgumentName: outputArgumentName, versionArgumentName: versionArgumentName, version: ABI.v1.self) + default: + Issue.record("Unreachable event stream version \(version)") + } + } + + func eventStreamOutput(outputArgumentName: String, versionArgumentName: String, version: V.Type) async throws where V: ABI.Version { // Test that JSON records are successfully streamed to a file and can be // read back into memory and decoded. let tempDirPath = try temporaryDirectory() @@ -250,8 +252,8 @@ struct SwiftPMTests { _ = remove(temporaryFilePath) } do { - let configuration = try configurationForEntryPoint(withArguments: ["PATH", outputArgumentName, temporaryFilePath, versionArgumentName, version]) - let test = Test {} + let configuration = try configurationForEntryPoint(withArguments: ["PATH", outputArgumentName, temporaryFilePath, versionArgumentName, "\(version.versionNumber)"]) + let test = Test(.tags(.blue)) {} let eventContext = Event.Context(test: test, testCase: nil, configuration: nil) configuration.handleEvent(Event(.testDiscovered, testID: test.id, testCaseID: nil), in: eventContext) @@ -264,7 +266,14 @@ struct SwiftPMTests { configuration.handleEvent(Event(.runEnded, testID: nil, testCaseID: nil), in: eventContext) } - let decodedRecords = try decodeABIv0RecordStream(fromFileAtPath: temporaryFilePath) + let decodedRecords = try FileHandle(forReadingAtPath: temporaryFilePath).readToEnd() + .split(whereSeparator: \.isASCIINewline) + .map { line in + try line.withUnsafeBytes { line in + try JSON.decode(ABI.Record.self, from: line) + } + } + let testRecords = decodedRecords.compactMap { record in if case let .test(test) = record.kind { return test @@ -272,6 +281,13 @@ struct SwiftPMTests { return nil } #expect(testRecords.count == 1) + for testRecord in testRecords { + if version.versionNumber >= 1 { + #expect(testRecord._tags != nil) + } else { + #expect(testRecord._tags == nil) + } + } let eventRecords = decodedRecords.compactMap { record in if case let .event(event) = record.kind { return event @@ -280,6 +296,14 @@ struct SwiftPMTests { } #expect(eventRecords.count == 4) } + + @Test("Experimental ABI version requires --experimental-event-stream-version argument") + func experimentalABIVersionNeedsExperimentalFlag() { + #expect(throws: (any Error).self) { + let experimentalVersion = ABI.CurrentVersion.versionNumber + 1 + _ = try configurationForEntryPoint(withArguments: ["PATH", "--event-stream-version", "\(experimentalVersion)"]) + } + } #endif #endif From 99ed4b3d2bce5d2736aa61a06b4cb4c7a3f7aa30 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 19 Feb 2025 11:39:47 -0500 Subject: [PATCH 092/234] Remove references to Xcode 16 Beta 1 (just Xcode 16, please.) (#966) This PR replaces references to Xcode 16 Beta 1, which are generally centred around our JSON ABI, with references to Xcode 16. All versions of Xcode 16 use a legacy JSON schema, not just Beta 1. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Testing/ABI/ABI.Record+Streaming.swift | 8 ++++---- Sources/Testing/ABI/ABI.swift | 4 ++-- .../ABI/EntryPoints/ABIEntryPoint.swift | 20 +++++++++---------- .../Testing/ABI/EntryPoints/EntryPoint.swift | 6 +++--- Tests/TestingTests/ABIEntryPointTests.swift | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 4f2f2944d..4c26b44ad 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -70,9 +70,9 @@ extension ABI.Version { } #if !SWT_NO_SNAPSHOT_TYPES -// MARK: - Xcode 16 Beta 1 compatibility +// MARK: - Xcode 16 compatibility -extension ABI.Xcode16Beta1 { +extension ABI.Xcode16 { static func eventHandler( encodeAsJSONLines: Bool, forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void @@ -80,8 +80,8 @@ extension ABI.Xcode16Beta1 { return { event, context in if case .testDiscovered = event.kind { // Discard events of this kind rather than forwarding them to avoid a - // crash in Xcode 16 Beta 1 (which does not expect any events to occur - // before .runStarted.) + // crash in Xcode 16 (which does not expect any events to occur before + // .runStarted.) return } diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index 7926eb324..13365ce69 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -48,10 +48,10 @@ extension ABI { extension ABI { #if !SWT_NO_SNAPSHOT_TYPES - /// A namespace and version type for Xcode 16 Beta 1 compatibility. + /// A namespace and version type for Xcode 16 compatibility. /// /// - Warning: This type will be removed in a future update. - enum Xcode16Beta1: Sendable, Version { + enum Xcode16: Sendable, Version { static var versionNumber: Int { -1 } diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index b63bc6da3..7014ec066 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -66,11 +66,11 @@ extension ABI.v0 { } #if !SWT_NO_SNAPSHOT_TYPES -// MARK: - Xcode 16 Beta 1 compatibility +// MARK: - Xcode 16 compatibility -extension ABI.Xcode16Beta1 { - /// An older signature for ``ABI/v0/EntryPoint-swift.typealias`` used by Xcode - /// 16 Beta 1. +extension ABI.Xcode16 { + /// An older signature for ``ABI/v0/EntryPoint-swift.typealias`` used by + /// Xcode 16. /// /// - Warning: This type will be removed in a future update. @available(*, deprecated, message: "Use ABI.v0.EntryPoint instead.") @@ -81,13 +81,13 @@ extension ABI.Xcode16Beta1 { } /// An older signature for ``ABI/v0/entryPoint-swift.type.property`` used by -/// Xcode 16 Beta 1. +/// Xcode 16. /// /// - Warning: This function will be removed in a future update. @available(*, deprecated, message: "Use ABI.v0.entryPoint (swt_abiv0_getEntryPoint()) instead.") @_cdecl("swt_copyABIEntryPoint_v0") @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { - let result = UnsafeMutablePointer.allocate(capacity: 1) + let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in try await _entryPoint( configurationJSON: configurationJSON, @@ -102,7 +102,7 @@ extension ABI.Xcode16Beta1 { // MARK: - /// A common implementation for ``ABI/v0/entryPoint-swift.type.property`` and -/// ``copyABIEntryPoint_v0()`` that provides Xcode 16 Beta 1 compatibility. +/// ``copyABIEntryPoint_v0()`` that provides Xcode 16 compatibility. /// /// This function will be removed (with its logic incorporated into /// ``ABI/v0/entryPoint-swift.type.property``) in a future update. @@ -125,9 +125,9 @@ private func _entryPoint( let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, encodeAsJSONLines: false, forwardingTo: recordHandler) let exitCode = await entryPoint(passing: args, eventHandler: eventHandler) - // To maintain compatibility with Xcode 16 Beta 1, suppress custom exit codes. - // (This is also needed by ABI.v0.entryPoint to correctly treat the no-tests - // as a successful run.) + // To maintain compatibility with Xcode 16, suppress custom exit codes. (This + // is also needed by ABI.v0.entryPoint to correctly treat the no-tests as a + // successful run.) if exitCode == EXIT_NO_TESTS_FOUND { return EXIT_SUCCESS } diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index d2ffd54b1..f36bda694 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -633,9 +633,9 @@ func eventHandlerForStreamingEvents( eventHandler(for: ABI.CurrentVersion.self) #if !SWT_NO_SNAPSHOT_TYPES case -1: - // Legacy support for Xcode 16 betas. Support for this undocumented version - // will be removed in a future update. Do not use it. - eventHandler(for: ABI.Xcode16Beta1.self) + // Legacy support for Xcode 16. Support for this undocumented version will + // be removed in a future update. Do not use it. + eventHandler(for: ABI.Xcode16.self) #endif case 0: eventHandler(for: ABI.v0.self) diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index 86a875e77..47f74af3c 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -62,7 +62,7 @@ struct ABIEntryPointTests { ) } #endif - let abiEntryPoint = copyABIEntryPoint_v0().assumingMemoryBound(to: ABI.Xcode16Beta1.EntryPoint.self) + let abiEntryPoint = copyABIEntryPoint_v0().assumingMemoryBound(to: ABI.Xcode16.EntryPoint.self) defer { abiEntryPoint.deinitialize(count: 1) abiEntryPoint.deallocate() From dca49276b8b263abfc9e06488d0d5e345b377ad6 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 19 Feb 2025 12:19:12 -0500 Subject: [PATCH 093/234] Remove `ABI.Version.eventHandler(...)` protocol requirement when the target does not support JSON event streaming. --- Sources/Testing/ABI/ABI.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index 13365ce69..3106d2a72 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -20,6 +20,7 @@ extension ABI { /// The numeric representation of this ABI version. static var versionNumber: Int { get } +#if canImport(Foundation) && (!SWT_NO_FILE_IO || !SWT_NO_ABI_ENTRY_POINT) /// Create an event handler that encodes events as JSON and forwards them to /// an ABI-friendly event handler. /// @@ -38,6 +39,7 @@ extension ABI { encodeAsJSONLines: Bool, forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void ) -> Event.Handler +#endif } /// The current supported ABI version (ignoring any experimental versions.) From eaae48cd4a8735db71f18e3e7e783c69d28ddd43 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 20 Feb 2025 09:47:38 -0500 Subject: [PATCH 094/234] Avoid calling `unsafeBitCast(_:to:)` to cast C function pointers. (#967) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR replaces calls to `unsafeBitCast(_:to:)` (where the value being cast is a C function pointer) with calls to an internal function that checks that the value being cast is actually a C function pointer. This is intended to make the code in question more self-documenting (by making it clear what kind of cast is being performed.) The function does then call `unsafeBitCast(_:to:)`, but it's in one place only instead of many places—the call sites are all now self-documenting. Also fixed one place we're using `unsafeBitCast(_:to:)` to cast a metatype where, as of Swift 6.1, we don't need to anymore. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/SpawnProcess.swift | 2 +- Sources/Testing/ExitTests/WaitFor.swift | 2 +- .../Testing/Parameterization/TypeInfo.swift | 48 ++++++++++++++++++- .../Testing/SourceAttribution/Backtrace.swift | 2 +- Sources/Testing/Support/Environment.swift | 4 +- Sources/Testing/Support/GetSymbol.swift | 4 +- Sources/Testing/Support/Versions.swift | 2 +- Tests/TestingTests/ABIEntryPointTests.swift | 4 +- 8 files changed, 56 insertions(+), 12 deletions(-) diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 2d4a67442..4726e8f41 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -34,7 +34,7 @@ typealias ProcessID = Never /// `__GLIBC_PREREQ()` is insufficient because `_DEFAULT_SOURCE` may not be /// defined at the point spawn.h is first included. private let _posix_spawn_file_actions_addclosefrom_np = symbol(named: "posix_spawn_file_actions_addclosefrom_np").map { - unsafeBitCast($0, to: (@convention(c) (UnsafeMutablePointer, CInt) -> CInt).self) + castCFunction(at: $0, to: (@convention(c) (UnsafeMutablePointer, CInt) -> CInt).self) } #endif diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index c6841c580..ccd6512b6 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -101,7 +101,7 @@ private nonisolated(unsafe) let _waitThreadNoChildrenCondition = { /// only declared if `_GNU_SOURCE` is set, but setting it causes build errors /// due to conflicts with Swift's Glibc module. private let _pthread_setname_np = symbol(named: "pthread_setname_np").map { - unsafeBitCast($0, to: (@convention(c) (pthread_t, UnsafePointer) -> CInt).self) + castCFunction(at: $0, to: (@convention(c) (pthread_t, UnsafePointer) -> CInt).self) } #endif diff --git a/Sources/Testing/Parameterization/TypeInfo.swift b/Sources/Testing/Parameterization/TypeInfo.swift index eeda19418..b0ed814b9 100644 --- a/Sources/Testing/Parameterization/TypeInfo.swift +++ b/Sources/Testing/Parameterization/TypeInfo.swift @@ -265,9 +265,11 @@ extension TypeInfo { } switch _kind { case let .type(type): - // _mangledTypeName() works with move-only types, but its signature has - // not been updated yet. SEE: rdar://134278607 +#if compiler(>=6.1) + return _mangledTypeName(type) +#else return _mangledTypeName(unsafeBitCast(type, to: Any.Type.self)) +#endif case let .nameOnly(_, _, mangledName): return mangledName } @@ -412,3 +414,45 @@ extension TypeInfo: Codable { } extension TypeInfo.EncodedForm: Codable {} + +// MARK: - Custom casts + +/// Cast the given data pointer to a C function pointer. +/// +/// - Parameters: +/// - address: The C function pointer as an untyped data pointer. +/// - type: The type of the C function. This type must be a function type with +/// the "C" convention (i.e. `@convention (...) -> ...`). +/// +/// - Returns: `address` cast to the given C function type. +/// +/// This function serves to make code that casts C function pointers more +/// self-documenting. In debug builds, it checks that `type` is a C function +/// type. In release builds, it behaves the same as `unsafeBitCast(_:to:)`. +func castCFunction(at address: UnsafeRawPointer, to type: T.Type) -> T { +#if DEBUG + if let mangledName = TypeInfo(describing: T.self).mangledName { + precondition(mangledName.last == "C", "\(#function) should only be used to cast a pointer to a C function type.") + } +#endif + return unsafeBitCast(address, to: type) +} + +/// Cast the given C function pointer to a data pointer. +/// +/// - Parameters: +/// - function: The C function pointer. +/// +/// - Returns: `function` cast to an untyped data pointer. +/// +/// This function serves to make code that casts C function pointers more +/// self-documenting. In debug builds, it checks that `function` is a C function +/// pointer. In release builds, it behaves the same as `unsafeBitCast(_:to:)`. +func castCFunction(_ function: T, to _: UnsafeRawPointer.Type) -> UnsafeRawPointer { +#if DEBUG + if let mangledName = TypeInfo(describing: T.self).mangledName { + precondition(mangledName.last == "C", "\(#function) should only be used to cast a C function.") + } +#endif + return unsafeBitCast(function, to: UnsafeRawPointer.self) +} diff --git a/Sources/Testing/SourceAttribution/Backtrace.swift b/Sources/Testing/SourceAttribution/Backtrace.swift index 78227e3da..97815755e 100644 --- a/Sources/Testing/SourceAttribution/Backtrace.swift +++ b/Sources/Testing/SourceAttribution/Backtrace.swift @@ -339,7 +339,7 @@ extension Backtrace { #if _runtime(_ObjC) && !SWT_NO_DYNAMIC_LINKING if Environment.flag(named: "SWT_FOUNDATION_ERROR_BACKTRACING_ENABLED") == true { let _CFErrorSetCallStackCaptureEnabled = symbol(named: "_CFErrorSetCallStackCaptureEnabled").map { - unsafeBitCast($0, to: (@convention(c) (DarwinBoolean) -> DarwinBoolean).self) + castCFunction(at: $0, to: (@convention(c) (DarwinBoolean) -> DarwinBoolean).self) } _ = _CFErrorSetCallStackCaptureEnabled?(true) return _CFErrorSetCallStackCaptureEnabled != nil diff --git a/Sources/Testing/Support/Environment.swift b/Sources/Testing/Support/Environment.swift index ec2ee9c74..f09efc1ed 100644 --- a/Sources/Testing/Support/Environment.swift +++ b/Sources/Testing/Support/Environment.swift @@ -74,7 +74,7 @@ enum Environment { /// system, the value of this property is `nil`. private static let _environ_lock_np = { symbol(named: "environ_lock_np").map { - unsafeBitCast($0, to: (@convention(c) () -> Void).self) + castCFunction(at: $0, to: (@convention(c) () -> Void).self) } }() @@ -84,7 +84,7 @@ enum Environment { /// system, the value of this property is `nil`. private static let _environ_unlock_np = { symbol(named: "environ_unlock_np").map { - unsafeBitCast($0, to: (@convention(c) () -> Void).self) + castCFunction(at: $0, to: (@convention(c) () -> Void).self) } }() #endif diff --git a/Sources/Testing/Support/GetSymbol.swift b/Sources/Testing/Support/GetSymbol.swift index b0f057088..5fb329143 100644 --- a/Sources/Testing/Support/GetSymbol.swift +++ b/Sources/Testing/Support/GetSymbol.swift @@ -66,13 +66,13 @@ func symbol(in handle: ImageAddress? = nil, named symbolName: String) -> UnsafeR // If the caller supplied a module, use it. if let handle { return GetProcAddress(handle, symbolName).map { - unsafeBitCast($0, to: UnsafeRawPointer.self) + castCFunction($0, to: UnsafeRawPointer.self) } } return HMODULE.all.lazy .compactMap { GetProcAddress($0, symbolName) } - .map { unsafeBitCast($0, to: UnsafeRawPointer.self) } + .map { castCFunction($0, to: UnsafeRawPointer.self) } .first } #else diff --git a/Sources/Testing/Support/Versions.swift b/Sources/Testing/Support/Versions.swift index 5e974c6f1..1eb7f4e48 100644 --- a/Sources/Testing/Support/Versions.swift +++ b/Sources/Testing/Support/Versions.swift @@ -65,7 +65,7 @@ let operatingSystemVersion: String = { // basically always lies on Windows 10, so don't bother calling it on a // fallback path. let RtlGetVersion = symbol(in: GetModuleHandleA("ntdll.dll"), named: "RtlGetVersion").map { - unsafeBitCast($0, to: (@convention(c) (UnsafeMutablePointer) -> NTSTATUS).self) + castCFunction(at: $0, to: (@convention(c) (UnsafeMutablePointer) -> NTSTATUS).self) } if let RtlGetVersion { var versionInfo = OSVERSIONINFOW() diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index 47f74af3c..9fcda9223 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -57,7 +57,7 @@ struct ABIEntryPointTests { let copyABIEntryPoint_v0 = try withTestingLibraryImageAddress { testingLibrary in try #require( symbol(in: testingLibrary, named: "swt_copyABIEntryPoint_v0").map { - unsafeBitCast($0, to: (@convention(c) () -> UnsafeMutableRawPointer).self) + castCFunction(at: $0, to: (@convention(c) () -> UnsafeMutableRawPointer).self) } ) } @@ -140,7 +140,7 @@ struct ABIEntryPointTests { let abiv0_getEntryPoint = try withTestingLibraryImageAddress { testingLibrary in try #require( symbol(in: testingLibrary, named: "swt_abiv0_getEntryPoint").map { - unsafeBitCast($0, to: (@convention(c) () -> UnsafeRawPointer).self) + castCFunction(at: $0, to: (@convention(c) () -> UnsafeRawPointer).self) } ) } From 5b60b0ab217f7a1615192657b2a6f84c10e8f2bc Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 20 Feb 2025 13:57:37 -0500 Subject: [PATCH 095/234] Don't hard-code numeric representations of ABI versions. (#968) This PR replaces some hard-coded ABI version numbers (`-1`, `0`, and `1`) with symbolic references. It also removes a common code path between the Xcode 16-compatible C entry point and the formal "v0" one so that deleting the Xcode 16 legacy paths will be easier later. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Testing/ABI/Encoded/ABI.EncodedTest.swift | 2 +- .../ABI/EntryPoints/ABIEntryPoint.swift | 68 +++++++------------ .../Testing/ABI/EntryPoints/EntryPoint.swift | 6 +- Tests/TestingTests/SwiftPMTests.swift | 12 ++-- 4 files changed, 34 insertions(+), 54 deletions(-) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index 8a6d4518a..51d01781d 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -98,7 +98,7 @@ extension ABI { sourceLocation = test.sourceLocation id = ID(encoding: test.id) - if V.versionNumber >= 1 { + if V.versionNumber >= ABI.v1.versionNumber { let tags = test.tags if !tags.isEmpty { _tags = tags.map(String.init(describing:)) diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index 7014ec066..f3f50a1be 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -47,10 +47,17 @@ extension ABI.v0 { /// callback. public static var entryPoint: EntryPoint { return { configurationJSON, recordHandler in - try await _entryPoint( - configurationJSON: configurationJSON, - recordHandler: recordHandler - ) == EXIT_SUCCESS + let args = try configurationJSON.map { configurationJSON in + try JSON.decode(__CommandLineArguments_v0.self, from: configurationJSON) + } + let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, encodeAsJSONLines: false, forwardingTo: recordHandler) + + switch await Testing.entryPoint(passing: args, eventHandler: eventHandler) { + case EXIT_SUCCESS, EXIT_NO_TESTS_FOUND: + return true + default: + return false + } } } } @@ -89,48 +96,21 @@ extension ABI.Xcode16 { @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in - try await _entryPoint( - configurationJSON: configurationJSON, - eventStreamVersionIfNil: -1, - recordHandler: recordHandler - ) + var args = try configurationJSON.map { configurationJSON in + try JSON.decode(__CommandLineArguments_v0.self, from: configurationJSON) + } + if args?.eventStreamVersion == nil { + args?.eventStreamVersion = ABI.Xcode16.versionNumber + } + let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, encodeAsJSONLines: false, forwardingTo: recordHandler) + + var exitCode = await Testing.entryPoint(passing: args, eventHandler: eventHandler) + if exitCode == EXIT_NO_TESTS_FOUND { + exitCode = EXIT_SUCCESS + } + return exitCode } return .init(result) } #endif - -// MARK: - - -/// A common implementation for ``ABI/v0/entryPoint-swift.type.property`` and -/// ``copyABIEntryPoint_v0()`` that provides Xcode 16 compatibility. -/// -/// This function will be removed (with its logic incorporated into -/// ``ABI/v0/entryPoint-swift.type.property``) in a future update. -private func _entryPoint( - configurationJSON: UnsafeRawBufferPointer?, - eventStreamVersionIfNil: Int? = nil, - recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void -) async throws -> CInt { - var args = try configurationJSON.map { configurationJSON in - try JSON.decode(__CommandLineArguments_v0.self, from: configurationJSON) - } - - // If the caller needs a nil event stream version to default to a specific - // JSON schema, apply it here as if they'd specified it in the configuration - // JSON blob. - if let eventStreamVersionIfNil, args?.eventStreamVersion == nil { - args?.eventStreamVersion = eventStreamVersionIfNil - } - - let eventHandler = try eventHandlerForStreamingEvents(version: args?.eventStreamVersion, encodeAsJSONLines: false, forwardingTo: recordHandler) - let exitCode = await entryPoint(passing: args, eventHandler: eventHandler) - - // To maintain compatibility with Xcode 16, suppress custom exit codes. (This - // is also needed by ABI.v0.entryPoint to correctly treat the no-tests as a - // successful run.) - if exitCode == EXIT_NO_TESTS_FOUND { - return EXIT_SUCCESS - } - return exitCode -} #endif diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index f36bda694..c72542d65 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -632,14 +632,14 @@ func eventHandlerForStreamingEvents( case nil: eventHandler(for: ABI.CurrentVersion.self) #if !SWT_NO_SNAPSHOT_TYPES - case -1: + case ABI.Xcode16.versionNumber: // Legacy support for Xcode 16. Support for this undocumented version will // be removed in a future update. Do not use it. eventHandler(for: ABI.Xcode16.self) #endif - case 0: + case ABI.v0.versionNumber: eventHandler(for: ABI.v0.self) - case 1: + case ABI.v1.versionNumber: eventHandler(for: ABI.v1.self) case let .some(unsupportedVersionNumber): throw _EntryPointError.invalidArgument("--event-stream-version", value: "\(unsupportedVersionNumber)") diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 77e52319e..6e7be0f15 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -228,15 +228,15 @@ struct SwiftPMTests { @Test("--event-stream-output-path argument (writes to a stream and can be read back)", arguments: [ - ("--event-stream-output-path", "--event-stream-version", 0), - ("--experimental-event-stream-output", "--experimental-event-stream-version", 0), - ("--experimental-event-stream-output", "--experimental-event-stream-version", 1), + ("--event-stream-output-path", "--event-stream-version", ABI.v0.versionNumber), + ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v0.versionNumber), + ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v1.versionNumber), ]) func eventStreamOutput(outputArgumentName: String, versionArgumentName: String, version: Int) async throws { switch version { - case 0: + case ABI.v0.versionNumber: try await eventStreamOutput(outputArgumentName: outputArgumentName, versionArgumentName: versionArgumentName, version: ABI.v0.self) - case 1: + case ABI.v1.versionNumber: try await eventStreamOutput(outputArgumentName: outputArgumentName, versionArgumentName: versionArgumentName, version: ABI.v1.self) default: Issue.record("Unreachable event stream version \(version)") @@ -282,7 +282,7 @@ struct SwiftPMTests { } #expect(testRecords.count == 1) for testRecord in testRecords { - if version.versionNumber >= 1 { + if version.versionNumber >= ABI.v1.versionNumber { #expect(testRecord._tags != nil) } else { #expect(testRecord._tags == nil) From e76a44fb6e54beb6caa684630da368f05aa01390 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 20 Feb 2025 15:40:47 -0500 Subject: [PATCH 096/234] Silence a warning on the latest 6.2-dev toolchain when building a test. (#969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are getting a (sort of spurious but also not really) warning when building a test where we have a particularly weird bit of code: ```swift func f() -> Never? { nil } try #require(f()) // ⚠️ warning: will never be executed ``` The code in question _will_ be executed, but will always throw an `ExpectationFailedError`. The compiler's getting a bit confused because the expansion of the macro includes a call to a function that returns `T`, where `T` is the type of the expression being unwrapped. Since `T` is `Never` in this context, the compiler infers that that function does not return. The compiler's inference is correct, but it's unclear what exactly it thinks will never be executed here. In any event, the warning is spurious and does not indicate a problem with the test, so I'm changing the return type of the problematic function from `Never?` to `Int?` to make the compiler happy. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Tests/TestingTests/IssueTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index f73676d42..d22bf9fba 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -144,7 +144,7 @@ final class IssueTests: XCTestCase { static func f(_ x: Int) -> Bool { false } static func g(label x: Int) -> Bool { false } static func h(_ x: () -> Void) -> Bool { false } - static func j(_ x: Int) -> Never? { nil } + static func j(_ x: Int) -> Int? { nil } static func k(_ x: inout Int) -> Bool { false } static func m(_ x: Bool) -> Bool { false } static func n(_ x: Int) throws -> Bool { false } From c488e8fe58ef4f26dee7945bce2cab15225245bb Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 21 Feb 2025 17:05:14 -0500 Subject: [PATCH 097/234] Restrict use of a new overload of `contains()` to Apple platforms that have it. (#970) We have one test that uses a newer overload of [`contains()`](https://developer.apple.com/documentation/swift/collection/contains(_:)) that doesn't exist on older Apple platforms. This PR ensures that our use of that method is constrained to platforms that have it. ### Checklist: - [ ] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [ ] If public symbols are renamed or modified, DocC references should be updated. --- Tests/TestingTests/AttachmentTests.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 3fa979b35..0a220552a 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -301,7 +301,9 @@ struct AttachmentTests { #expect(buffer.count > 32) #expect(buffer[0] == UInt8(ascii: "P")) #expect(buffer[1] == UInt8(ascii: "K")) - #expect(buffer.contains("loremipsum.txt".utf8)) + if #available(_regexAPI, *) { + #expect(buffer.contains("loremipsum.txt".utf8)) + } } valueAttached() } From f9e1701fc5ec5eaf9fc82c5e69c2e17f7d7ab950 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Tue, 25 Feb 2025 22:13:17 +0100 Subject: [PATCH 098/234] Show the number of test cases that ran at the end of a test (#972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation: To have the number of test cases at the end of the test in for parameterized tests, so the logs don’t easily get buried. ### Modifications: Log the number of test cases for parameterized tests at the end of a test. ### Result: If the user runs the parameterized tests, an extra piece of information (with * test cases) would get bound to the end log. Resolves #943. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Event.HumanReadableOutputRecorder.swift | 16 +++++- Tests/TestingTests/EventRecorderTests.swift | 57 ++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index be92fe932..0e856facf 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -62,6 +62,9 @@ extension Event { /// The number of known issues recorded for the test. var knownIssueCount = 0 + + /// The number of test cases for the test. + var testCasesCount = 0 } /// Data tracked on a per-test basis. @@ -281,6 +284,10 @@ extension Event.HumanReadableOutputRecorder { testData.issueCount[issue.severity] = issueCount + 1 } context.testData[id] = testData + + case .testCaseStarted: + let test = test! + context.testData[test.id.keyPathRepresentation]?.testCasesCount += 1 default: // These events do not manipulate the context structure. @@ -366,18 +373,23 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) + let testCasesCount = if test.isParameterized { + " with \(testData.testCasesCount.counting("test case"))" + } else { + "" + } return if issues.errorIssueCount > 0 { CollectionOfOne( Message( symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) failed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) failed after \(duration)\(issues.description)." ) ) + _formattedComments(for: test) } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) passed after \(duration)\(issues.description)." ) ] } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 6b5b9bd81..7712cd973 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -178,6 +178,44 @@ struct EventRecorderTests { .first != nil ) } + + @available(_regexAPI, *) + @Test( + "Log the total number of test cases in parameterized tests at the end of the test run", + arguments: [ + ("f()", #".* Test f\(\) failed after .*"#), + ("h()", #".* Test h\(\) passed after .+"#), + ("l(_:)", #".* Test l\(_:\) with .+ test cases passed after.*"#), + ("m(_:)", #".* Test m\(_:\) with .+ test cases failed after.*"#), + ("n(_:)", #".* Test n\(_:\) with .+ test case passed after.*"#), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#), + ] + ) + func numberOfTestCasesAtTestEnd(testName: String, expectedPattern: String) async throws { + let stream = Stream() + + var configuration = Configuration() + let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) + configuration.eventHandler = { event, context in + eventRecorder.record(event, in: context) + } + + await runTest(for: PredictablyFailingTests.self, configuration: configuration) + + let buffer = stream.buffer.rawValue + if testsWithSignificantIOAreEnabled { + print(buffer, terminator: "") + } + + let aurgmentRegex = try Regex(expectedPattern) + + #expect( + (try buffer + .split(whereSeparator: \.isNewline) + .compactMap(aurgmentRegex.wholeMatch(in:)) + .first) != nil + ) + } @available(_regexAPI, *) @Test( @@ -189,7 +227,7 @@ struct EventRecorderTests { ("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#), ("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#), ("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 16 issues \(including 3 warnings and 6 known issues\)\."#), ] ) func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws { @@ -284,7 +322,7 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 9) + #expect(match.output.1 == 12) #expect(match.output.2 == 5) } @@ -565,4 +603,19 @@ struct EventRecorderTests { Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() } } + + @Test(.hidden, arguments: [1, 2, 3]) + func l(_ arg: Int) { + #expect(arg > 0) + } + + @Test(.hidden, arguments: [1, 2, 3]) + func m(_ arg: Int) { + #expect(arg < 0) + } + + @Test(.hidden, arguments: [1]) + func n(_ arg: Int) { + #expect(arg > 0) + } } From a7b5435933149b679ad4998ed33e0393425ef4e2 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 25 Feb 2025 16:32:49 -0500 Subject: [PATCH 099/234] Change the return types of test content accessors and add `ExitTest.current`. (#974) This PR changes the signatures of the accessor functions for the test content metadata section so that we're not using underscored types. Instead, a `Test.Generator` type replaces `Test._Record` and the bare function type we were using for `Test` and `ExitTest` is promoted to (still experimental) API instead of hiding behind underscores. This change allows the types produced by accessor functions to always be nominal types, avoids the iffy substitution of `_Record` for `@Sendable () async -> Test` at runtime, and simplifies the internal `TestContent` protocol by eliminating the need for an associated type to stand in for the conforming type. This change leaves `ExitTest` as a nominal, but empty, type. I've added `ExitTest.current` to make it useful so that test code, especially libraries, can detect if it's running inside an exit test or not. I will separately update the exit test proposal PR to include this type and property. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 21 ++-- Sources/Testing/Discovery.swift | 16 +-- Sources/Testing/ExitTests/ExitTest.swift | 114 ++++++++++-------- .../ExpectationChecking+Macro.swift | 2 +- Sources/Testing/Test+Discovery+Legacy.swift | 2 +- Sources/Testing/Test+Discovery.swift | 26 +--- Sources/TestingMacros/ConditionMacro.swift | 6 +- Tests/TestingTests/ExitTestTests.swift | 8 ++ 8 files changed, 92 insertions(+), 103 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index 9b981e5c0..89412b29d 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -143,26 +143,21 @@ filtering is performed.) The concrete Swift type of the value written to `outValue`, the type pointed to by `type`, and the value pointed to by `hint` depend on the kind of record: -- For test or suite declarations (kind `0x74657374`), the accessor produces an - asynchronous Swift function[^notAccessorSignature] that returns an instance of - `Testing.Test`: +- For test or suite declarations (kind `0x74657374`), the accessor produces a + structure of type `Testing.Test.Generator` that the testing library can use + to generate the corresponding test[^notAccessorSignature]. - ```swift - @Sendable () async -> Test - ``` - - [^notAccessorSignature]: This signature is not the signature of `accessor`, - but of the Swift function reference it writes to `outValue`. This level of - indirection is necessary because loading a test or suite declaration is an - asynchronous operation, but C functions cannot be `async`. + [^notAccessorSignature]: This level of indirection is necessary because + loading a test or suite declaration is an asynchronous operation, but C + functions cannot be `async`. Test content records of this kind do not specify a type for `hint`. Always pass `nil`. - For exit test declarations (kind `0x65786974`), the accessor produces a - structure describing the exit test (of type `Testing.__ExitTest`.) + structure describing the exit test (of type `Testing.ExitTest`.) - Test content records of this kind accept a `hint` of type `Testing.__ExitTest.ID`. + Test content records of this kind accept a `hint` of type `Testing.ExitTest.ID`. They only produce a result if they represent an exit test declared with the same ID (or if `hint` is `nil`.) diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift index d80d33826..38f4dfa58 100644 --- a/Sources/Testing/Discovery.swift +++ b/Sources/Testing/Discovery.swift @@ -74,20 +74,6 @@ protocol TestContent: ~Copyable { /// By default, this type equals `Never`, indicating that this type of test /// content does not support hinting during discovery. associatedtype TestContentAccessorHint: Sendable = Never - - /// The type to pass (by address) as the accessor function's `type` argument. - /// - /// The default value of this property is `Self.self`. A conforming type can - /// override the default implementation to substitute another type (e.g. if - /// the conforming type is not public but records are created during macro - /// expansion and can only reference public types.) - static var testContentAccessorTypeArgument: any ~Copyable.Type { get } -} - -extension TestContent where Self: ~Copyable { - static var testContentAccessorTypeArgument: any ~Copyable.Type { - self - } } // MARK: - Individual test content records @@ -142,7 +128,7 @@ struct TestContentRecord: Sendable where T: TestContent & ~Copyable { return nil } - return withUnsafePointer(to: T.testContentAccessorTypeArgument) { type in + return withUnsafePointer(to: T.self) { type in withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in let initialized = if let hint { withUnsafePointer(to: hint) { hint in diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 7a3597bfa..75102abe2 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -24,52 +24,31 @@ private import _TestingInternals /// A type describing an exit test. /// -/// Instances of this type describe an exit test defined by the test author and -/// discovered or called at runtime. Tools that implement custom exit test -/// handling will encounter instances of this type in two contexts: -/// -/// - When the current configuration's exit test handler, set with -/// ``Configuration/exitTestHandler``, is called; and -/// - When, in a child process, they need to look up the exit test to call. -/// -/// If you are writing tests, you don't usually need to interact directly with -/// an instance of this type. To create an exit test, use the +/// Instances of this type describe exit tests you create using the /// ``expect(exitsWith:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:_:sourceLocation:performing:)`` macro. -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) -#if SWT_NO_EXIT_TESTS -@available(*, unavailable, message: "Exit tests are not available on this platform.") -#endif -public typealias ExitTest = __ExitTest - -/// A type describing an exit test. -/// -/// - Warning: This type is used to implement the `#expect(exitsWith:)` macro. -/// Do not use it directly. Tools can use the SPI ``ExitTest`` typealias if -/// needed. +/// ``require(exitsWith:_:sourceLocation:performing:)`` macro. You don't usually +/// need to interact directly with an instance of this type. @_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -public struct __ExitTest: Sendable, ~Copyable { - /// A type whose instances uniquely identify instances of `__ExitTest`. +public struct ExitTest: Sendable, ~Copyable { + /// A type whose instances uniquely identify instances of ``ExitTest``. + @_spi(ForToolsIntegrationOnly) public struct ID: Sendable, Equatable, Codable { /// An underlying UUID (stored as two `UInt64` values to avoid relying on /// `UUID` from Foundation or any platform-specific interfaces.) private var _lo: UInt64 private var _hi: UInt64 - /// Initialize an instance of this type. - /// - /// - Warning: This member is used to implement the `#expect(exitsWith:)` - /// macro. Do not use it directly. - public init(__uuid uuid: (UInt64, UInt64)) { + init(_ uuid: (UInt64, UInt64)) { self._lo = uuid.0 self._hi = uuid.1 } } /// A value that uniquely identifies this instance. + @_spi(ForToolsIntegrationOnly) public var id: ID /// The body closure of the exit test. @@ -77,7 +56,7 @@ public struct __ExitTest: Sendable, ~Copyable { /// Do not invoke this closure directly. Instead, invoke ``callAsFunction()`` /// to run the exit test. Running the exit test will always terminate the /// current process. - fileprivate var body: @Sendable () async throws -> Void + fileprivate var body: @Sendable () async throws -> Void = {} /// Storage for ``observedValues``. /// @@ -113,21 +92,52 @@ public struct __ExitTest: Sendable, ~Copyable { _observedValues = newValue } } +} + +#if !SWT_NO_EXIT_TESTS +// MARK: - Current + +@_spi(Experimental) +extension ExitTest { + /// A container type to hold the current exit test. + /// + /// This class is temporarily necessary until `ManagedBuffer` is updated to + /// support storing move-only values. For more information, see [SE-NNNN](https://github.com/swiftlang/swift-evolution/pull/2657). + private final class _CurrentContainer: Sendable { + /// The exit test represented by this container. + /// + /// The value of this property must be optional to avoid a copy when reading + /// the value in ``ExitTest/current``. + let exitTest: ExitTest? + + init(exitTest: borrowing ExitTest) { + self.exitTest = ExitTest(id: exitTest.id, body: exitTest.body, _observedValues: exitTest._observedValues) + } + } + + /// Storage for ``current``. + private static let _current = Locked<_CurrentContainer?>() - /// Initialize an exit test at runtime. + /// The exit test that is running in the current process, if any. /// - /// - Warning: This initializer is used to implement the `#expect(exitsWith:)` - /// macro. Do not use it directly. - public init( - __identifiedBy id: ID, - body: @escaping @Sendable () async throws -> Void = {} - ) { - self.id = id - self.body = body + /// If the current process was created to run an exit test, the value of this + /// property describes that exit test. If this process is the parent process + /// of an exit test, or if no exit test is currently running, the value of + /// this property is `nil`. + /// + /// The value of this property is constant across all tasks in the current + /// process. + public static var current: ExitTest? { + _read { + if let current = _current.rawValue { + yield current.exitTest + } else { + yield nil + } + } } } -#if !SWT_NO_EXIT_TESTS // MARK: - Invocation @_spi(Experimental) @_spi(ForToolsIntegrationOnly) @@ -180,8 +190,7 @@ extension ExitTest { /// This function invokes the closure originally passed to /// `#expect(exitsWith:)` _in the current process_. That closure is expected /// to terminate the process; if it does not, the testing library will - /// terminate the process in a way that causes the corresponding expectation - /// to fail. + /// terminate the process as if its `main()` function returned naturally. public consuming func callAsFunction() async -> Never { Self._disableCrashReporting() @@ -209,6 +218,11 @@ extension ExitTest { } #endif + // Set ExitTest.current before the test body runs. + Self._current.withLock { current in + current = _CurrentContainer(exitTest: self) + } + do { try await body() } catch { @@ -247,11 +261,15 @@ extension ExitTest { } } +#if !SWT_NO_LEGACY_TEST_DISCOVERY // Call the legacy lookup function that discovers tests embedded in types. return types(withNamesContaining: exitTestContainerTypeNameMagic).lazy .compactMap { $0 as? any __ExitTestContainer.Type } - .first { $0.__id == id } - .map { ExitTest(__identifiedBy: $0.__id, body: $0.__body) } + .first { ID($0.__id) == id } + .map { ExitTest(id: ID($0.__id), body: $0.__body) } +#else + return nil +#endif } } @@ -280,7 +298,7 @@ extension ExitTest { /// `await #expect(exitsWith:) { }` invocations regardless of calling /// convention. func callExitTest( - identifiedBy exitTestID: ExitTest.ID, + identifiedBy exitTestID: (UInt64, UInt64), exitsWith expectedExitCondition: ExitCondition, observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, @@ -295,7 +313,7 @@ func callExitTest( var result: ExitTestArtifacts do { - var exitTest = ExitTest(__identifiedBy: exitTestID) + var exitTest = ExitTest(id: ExitTest.ID(exitTestID)) exitTest.observedValues = observedValues result = try await configuration.exitTestHandler(exitTest) @@ -426,10 +444,10 @@ extension ExitTest { /// configurations is undefined. static func findInEnvironmentForEntryPoint() -> Self? { // Find the ID of the exit test to run, if any, in the environment block. - var id: __ExitTest.ID? + var id: ExitTest.ID? if var idString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") { id = try? idString.withUTF8 { idBuffer in - try JSON.decode(__ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) + try JSON.decode(ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) } } guard let id else { diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 267fddba4..550f6039c 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1147,7 +1147,7 @@ public func __checkClosureCall( /// `#require()` macros. Do not call it directly. @_spi(Experimental) public func __checkClosureCall( - identifiedBy exitTestID: __ExitTest.ID, + identifiedBy exitTestID: (UInt64, UInt64), exitsWith expectedExitCondition: ExitCondition, observing observedValues: [any PartialKeyPath & Sendable], performing body: @convention(thin) () -> Void, diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index 6ee080051..301b1e955 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -33,7 +33,7 @@ let testContainerTypeNameMagic = "__🟠$test_container__" @_spi(Experimental) public protocol __ExitTestContainer { /// The unique identifier of the exit test. - static var __id: __ExitTest.ID { get } + static var __id: (UInt64, UInt64) { get } /// The body function of the exit test. static var __body: @Sendable () async throws -> Void { get } diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index d515bc98f..d42f00be6 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -18,25 +18,12 @@ extension Test { /// indirect `async` accessor function rather than directly producing /// instances of ``Test``, but functions are non-nominal types and cannot /// directly conform to protocols. - /// - /// - Note: This helper type must have the exact in-memory layout of the - /// `async` accessor function. Do not add any additional cases or associated - /// values. The layout of this type is [guaranteed](https://github.com/swiftlang/swift/blob/main/docs/ABI/TypeLayout.rst#fragile-enum-layout) - /// by the Swift ABI. - /* @frozen */ private enum _Record: TestContent { + fileprivate struct Generator: TestContent, RawRepresentable { static var testContentKind: UInt32 { 0x74657374 } - static var testContentAccessorTypeArgument: any ~Copyable.Type { - Generator.self - } - - /// The type of the actual (asynchronous) generator function. - typealias Generator = @Sendable () async -> Test - - /// The actual (asynchronous) accessor function. - case generator(Generator) + var rawValue: @Sendable () async -> Test } /// All available ``Test`` instances in the process, according to the runtime. @@ -65,15 +52,10 @@ extension Test { // Walk all test content and gather generator functions, then call them in // a task group and collate their results. if useNewMode { - let generators = _Record.allTestContentRecords().lazy.compactMap { record in - if case let .generator(generator) = record.load() { - return generator - } - return nil // currently unreachable, but not provably so - } + let generators = Generator.allTestContentRecords().lazy.compactMap { $0.load() } await withTaskGroup(of: Self.self) { taskGroup in for generator in generators { - taskGroup.addTask(operation: generator) + taskGroup.addTask { await generator.rawValue() } } result = await taskGroup.reduce(into: result) { $0.insert($1) } } diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 0b39d2f78..f687aa631 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -435,7 +435,7 @@ extension ExitTestConditionMacro { // TODO: use UUID() here if we can link to Foundation let exitTestID = (UInt64.random(in: 0 ... .max), UInt64.random(in: 0 ... .max)) - let exitTestIDExpr: ExprSyntax = "Testing.__ExitTest.ID(__uuid: (\(literal: exitTestID.0), \(literal: exitTestID.1)))" + let exitTestIDExpr: ExprSyntax = "(\(literal: exitTestID.0), \(literal: exitTestID.1))" var decls = [DeclSyntax]() @@ -444,7 +444,7 @@ extension ExitTestConditionMacro { let bodyThunkName = context.makeUniqueName("") decls.append( """ - @Sendable func \(bodyThunkName)() async throws -> Void { + @Sendable func \(bodyThunkName)() async throws -> Swift.Void { return try await Testing.__requiringTry(Testing.__requiringAwait(\(bodyArgumentExpr.trimmed)))() } """ @@ -457,7 +457,7 @@ extension ExitTestConditionMacro { """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") enum \(enumName): Testing.__ExitTestContainer, Sendable { - static var __id: Testing.__ExitTest.ID { + static var __id: (Swift.UInt64, Swift.UInt64) { \(exitTestIDExpr) } static var __body: @Sendable () async throws -> Void { diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 0c5a9dcab..00d54f3f6 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -416,6 +416,14 @@ private import _TestingInternals fatalError() } } + + @Test("ExitTest.current property") + func currentProperty() async { + #expect((ExitTest.current == nil) as Bool) + await #expect(exitsWith: .success) { + #expect((ExitTest.current != nil) as Bool) + } + } } // MARK: - Fixtures From e12e2436c0f352c5a52d84ec1b19c81162037c47 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 28 Feb 2025 11:22:12 -0500 Subject: [PATCH 100/234] Delete sidecar DocC files. (#979) This PR deletes the sidecar DocC files we've created that include custom availability annotations for Swift and Xcode and/or deprecation summaries. The latest DocC builds allow us to specify this metadata directly on Swift symbols. Also add Xcode 16.3 availability for symbols added in Swift 6.1. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Expectations/Expectation+Macro.swift | 34 ++++++++++++++++ Sources/Testing/Issues/Confirmation.swift | 5 +++ .../AvailabilityStubs/ExpectComplexThrows.md | 28 ------------- .../AvailabilityStubs/Issues/Confirmation.md | 5 --- .../AvailabilityStubs/RequireComplexThrows.md | 28 ------------- ...opeProvider-default-implementation-Self.md | 15 ------- .../Traits/TestScopeProvider.md | 15 ------- .../Traits/TestScoping-provideScope.md | 15 ------- .../AvailabilityStubs/Traits/TestScoping.md | 15 ------- ...peProvider-default-implementation-Never.md | 15 ------- ...opeProvider-default-implementation-Self.md | 15 ------- ...rait-scopeProvider-protocol-requirement.md | 15 ------- Sources/Testing/Traits/Trait.swift | 39 +++++++++++++++++++ 13 files changed, 78 insertions(+), 166 deletions(-) delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Issues/Confirmation.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md delete mode 100644 Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 7e82b2144..a3c484043 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -375,6 +375,23 @@ public macro require( /// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. If the thrown /// error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), /// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.0) +/// @Available(Xcode, introduced: 16.0) +/// } +/// +/// @DeprecationSummary { +/// Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` +/// or ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: +/// +/// ```swift +/// let error = #expect(throws: FoodTruckError.self) { +/// ... +/// } +/// #expect(error?.napkinCount == 0) +/// ``` +/// } @available(swift, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") @discardableResult @freestanding(expression) public macro expect( @@ -427,6 +444,23 @@ public macro require( /// /// If `expression` should _never_ throw, simply invoke the code without using /// this macro. The test will then fail if an error is thrown. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.0) +/// @Available(Xcode, introduced: 16.0) +/// } +/// +/// @DeprecationSummary { +/// Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` +/// or ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: +/// +/// ```swift +/// let error = try #require(throws: FoodTruckError.self) { +/// ... +/// } +/// #expect(error.napkinCount == 0) +/// ``` +/// } @available(swift, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") @discardableResult @freestanding(expression) public macro require( diff --git a/Sources/Testing/Issues/Confirmation.swift b/Sources/Testing/Issues/Confirmation.swift index 04fbfa766..21baec505 100644 --- a/Sources/Testing/Issues/Confirmation.swift +++ b/Sources/Testing/Issues/Confirmation.swift @@ -161,6 +161,11 @@ public func confirmation( /// /// If an exact count is expected, use /// ``confirmation(_:expectedCount:isolation:sourceLocation:_:)-5mqz2`` instead. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.1) +/// @Available(Xcode, introduced: 16.3) +/// } public func confirmation( _ comment: Comment? = nil, expectedCount: some RangeExpression & Sequence & Sendable, diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md deleted file mode 100644 index 3114002d2..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/ExpectComplexThrows.md +++ /dev/null @@ -1,28 +0,0 @@ -# ``expect(_:sourceLocation:performing:throws:)`` - - - -@Metadata { - @Available(Swift, introduced: 6.0) - @Available(Xcode, introduced: 16.0) -} - -@DeprecationSummary { - Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` or - ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: - - ```swift - let error = #expect(throws: FoodTruckError.self) { - ... - } - #expect(error?.napkinCount == 0) - ``` -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Issues/Confirmation.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Issues/Confirmation.md deleted file mode 100644 index 3ce9bcbdd..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Issues/Confirmation.md +++ /dev/null @@ -1,5 +0,0 @@ -# ``Testing/confirmation(_:expectedCount:isolation:sourceLocation:_:)-l3il`` - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md b/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md deleted file mode 100644 index 291ac0d32..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/RequireComplexThrows.md +++ /dev/null @@ -1,28 +0,0 @@ -# ``require(_:sourceLocation:performing:throws:)`` - - - -@Metadata { - @Available(Swift, introduced: 6.0) - @Available(Xcode, introduced: 16.0) -} - -@DeprecationSummary { - Examine the result of ``require(throws:_:sourceLocation:performing:)-7n34r`` - or ``require(throws:_:sourceLocation:performing:)-4djuw`` instead: - - ```swift - let error = try #require(throws: FoodTruckError.self) { - ... - } - #expect(error.napkinCount == 0) - ``` -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md deleted file mode 100644 index 6136ee8cb..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/SuiteTrait-scopeProvider-default-implementation-Self.md +++ /dev/null @@ -1,15 +0,0 @@ -# ``Trait/scopeProvider(for:testCase:)-1z8kh`` - - - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md deleted file mode 100644 index 25281bfe6..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScopeProvider.md +++ /dev/null @@ -1,15 +0,0 @@ -# ``Trait/TestScopeProvider`` - - - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md deleted file mode 100644 index 809fb833e..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping-provideScope.md +++ /dev/null @@ -1,15 +0,0 @@ -# ``TestScoping/provideScope(for:testCase:performing:)`` - - - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md deleted file mode 100644 index a0ab00e1f..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/TestScoping.md +++ /dev/null @@ -1,15 +0,0 @@ -# ``TestScoping`` - - - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md deleted file mode 100644 index 8e903eb3b..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Never.md +++ /dev/null @@ -1,15 +0,0 @@ -# ``Trait/scopeProvider(for:testCase:)-9fxg4`` - - - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md deleted file mode 100644 index 0ff33a204..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-default-implementation-Self.md +++ /dev/null @@ -1,15 +0,0 @@ -# ``Trait/scopeProvider(for:testCase:)-inmj`` - - - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md b/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md deleted file mode 100644 index 5fcff2667..000000000 --- a/Sources/Testing/Testing.docc/AvailabilityStubs/Traits/Trait-scopeProvider-protocol-requirement.md +++ /dev/null @@ -1,15 +0,0 @@ -# ``Trait/scopeProvider(for:testCase:)`` - - - -@Metadata { - @Available(Swift, introduced: 6.1) -} diff --git a/Sources/Testing/Traits/Trait.swift b/Sources/Testing/Traits/Trait.swift index 7ebbb38d4..70aafe90e 100644 --- a/Sources/Testing/Traits/Trait.swift +++ b/Sources/Testing/Traits/Trait.swift @@ -51,6 +51,11 @@ public protocol Trait: Sendable { /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with /// `Never` as its test scope provider type must return `nil`, meaning that /// the trait doesn't provide a custom scope for tests it's applied to. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.1) + /// @Available(Xcode, introduced: 16.3) + /// } associatedtype TestScopeProvider: TestScoping = Never /// Get this trait's scope provider for the specified test and optional test @@ -98,6 +103,11 @@ public protocol Trait: Sendable { /// associated ``Trait/TestScopeProvider`` type is the default `Never`, then /// this method returns `nil` by default. This means that instances of this /// trait don't provide a custom scope for tests to which they're applied. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.1) + /// @Available(Xcode, introduced: 16.3) + /// } func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider? } @@ -109,6 +119,11 @@ public protocol Trait: Sendable { /// conforms to this protocol. Create a custom scope to consolidate common /// set-up and tear-down logic for tests which have similar needs, which allows /// each test function to focus on the unique aspects of its test. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.1) +/// @Available(Xcode, introduced: 16.3) +/// } public protocol TestScoping: Sendable { /// Provide custom execution scope for a function call which is related to the /// specified test or test case. @@ -143,6 +158,11 @@ public protocol TestScoping: Sendable { /// or throw an error if it's unable to provide a custom scope. /// /// Issues recorded by this method are associated with `test`. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.1) + /// @Available(Xcode, introduced: 16.3) + /// } func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws } @@ -159,6 +179,11 @@ extension Trait where Self: TestScoping { /// The testing library uses this implementation of /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type conforms /// to ``TestScoping``. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.1) + /// @Available(Xcode, introduced: 16.3) + /// } public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { testCase == nil ? nil : self } @@ -178,6 +203,11 @@ extension SuiteTrait where Self: TestScoping { /// The testing library uses this implementation of /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type conforms /// to both ``SuiteTrait`` and ``TestScoping``. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.1) + /// @Available(Xcode, introduced: 16.3) + /// } public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { if test.isSuite { isRecursive ? nil : self @@ -188,6 +218,10 @@ extension SuiteTrait where Self: TestScoping { } extension Never: TestScoping { + /// @Metadata { + /// @Available(Swift, introduced: 6.1) + /// @Available(Xcode, introduced: 16.3) + /// } public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws {} } @@ -236,6 +270,11 @@ extension Trait where TestScopeProvider == Never { /// The testing library uses this implementation of /// ``Trait/scopeProvider(for:testCase:)-cjmg`` when the trait type's /// associated ``Trait/TestScopeProvider`` type is `Never`. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.1) + /// @Available(Xcode, introduced: 16.3) + /// } public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? { nil } From 66708a7b8650457610725bebbaa3c303c2930f64 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 28 Feb 2025 11:22:29 -0500 Subject: [PATCH 101/234] Note in documentation when `#expect(throws:)` started returning a value. (#980) DocC doesn't let us annotate our return type's availability separately from our macros' availability, and we can't include both the old symbols and new ones because it is ambiguous at compile time, so this PR explains in prose when the return types were added. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Expectations/Expectation+Macro.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index a3c484043..134ffad37 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -160,6 +160,9 @@ public macro require( /// is running in the current task. Any value returned by `expression` is /// discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), /// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. /// @@ -224,6 +227,9 @@ public macro require( /// is running in the current task and an instance of ``ExpectationFailedError`` /// is thrown. Any value returned by `expression` is discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), /// use ``require(throws:_:sourceLocation:performing:)-4djuw`` instead. /// @@ -286,6 +292,9 @@ public macro require( /// not equal to `error`, an ``Issue`` is recorded for the test that is running /// in the current task. Any value returned by `expression` is discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only be an instance of a particular type, use /// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. @discardableResult @@ -327,6 +336,9 @@ public macro require( /// in the current task and an instance of ``ExpectationFailedError`` is thrown. /// Any value returned by `expression` is discarded. /// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// /// If the thrown error need only be an instance of a particular type, use /// ``require(throws:_:sourceLocation:performing:)-7n34r`` instead. @discardableResult From 44022f41c7a025d1dc583676a6a1ce4f6e547040 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 28 Feb 2025 13:43:00 -0500 Subject: [PATCH 102/234] Rename `ExitTestArtifacts` and split `ExitCondition` in twain. (#975) This PR renames `ExitTestArtifacts` to `ExitTest.Result` and splits `ExitCondition` into two types: `ExitTest.Condition` which can be passed to `#expect(exitsWith:)` and `StatusAtExit` which represents the raw, possibly platform-specific status reported by the kernel when a child process terminates. The latter type is not nested in `ExitTest` because it can be used independently of exit tests and we may want to use it in the future for things like multi-process parallelization, but if a platform supports spawning processes but not exit tests, nesting it in `ExitTest` would make it unavailable. I considered several names for `StatusAtExit`: - `ExitStatus`: too easily confusable with exit _codes_ such as `EXIT_SUCCESS`; - `ProcessStatus`: we don't say "process" in our API surface elsewhere; - `Status`: too generic - `ExitReason`: "status" is a more widely-used term of art for this concept. Foundation uses `terminationStatus` to represent the raw integer value and `Process.TerminationReason` to represent whether it's an exit code or signal. We don't use "termination" in Swift Testing's API anywhere. I settled on `StatusAtExit` because it was distinct and makes it clear that it represents the status of a process _at exit time_ (as opposed to while running, e.g. `enum ProcessStatus { case running; case suspended; case terminated }`. Updates to the exit tests proposal document will follow in a separate PR. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/CMakeLists.txt | 5 +- Sources/Testing/ExitTests/ExitCondition.swift | 236 ------------------ .../ExitTests/ExitTest.Condition.swift | 152 +++++++++++ .../Testing/ExitTests/ExitTest.Result.swift | 86 +++++++ Sources/Testing/ExitTests/ExitTest.swift | 63 ++--- .../Testing/ExitTests/ExitTestArtifacts.swift | 90 ------- Sources/Testing/ExitTests/SpawnProcess.swift | 2 +- Sources/Testing/ExitTests/StatusAtExit.swift | 73 ++++++ Sources/Testing/ExitTests/WaitFor.swift | 16 +- .../Expectations/Expectation+Macro.swift | 28 +-- .../ExpectationChecking+Macro.swift | 6 +- Sources/TestingMacros/ConditionMacro.swift | 8 +- Tests/TestingTests/ExitTestTests.swift | 72 ++---- 13 files changed, 392 insertions(+), 445 deletions(-) delete mode 100644 Sources/Testing/ExitTests/ExitCondition.swift create mode 100644 Sources/Testing/ExitTests/ExitTest.Condition.swift create mode 100644 Sources/Testing/ExitTests/ExitTest.Result.swift delete mode 100644 Sources/Testing/ExitTests/ExitTestArtifacts.swift create mode 100644 Sources/Testing/ExitTests/StatusAtExit.swift diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index e40cb1b0b..2f1e94c1a 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -31,10 +31,11 @@ add_library(Testing Events/Recorder/Event.JUnitXMLRecorder.swift Events/Recorder/Event.Symbol.swift Events/TimeValue.swift - ExitTests/ExitCondition.swift ExitTests/ExitTest.swift - ExitTests/ExitTestArtifacts.swift + ExitTests/ExitTest.Condition.swift + ExitTests/ExitTest.Result.swift ExitTests/SpawnProcess.swift + ExitTests/StatusAtExit.swift ExitTests/WaitFor.swift Expectations/Expectation.swift Expectations/Expectation+Macro.swift diff --git a/Sources/Testing/ExitTests/ExitCondition.swift b/Sources/Testing/ExitTests/ExitCondition.swift deleted file mode 100644 index 19f884303..000000000 --- a/Sources/Testing/ExitTests/ExitCondition.swift +++ /dev/null @@ -1,236 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -private import _TestingInternals - -/// An enumeration describing possible conditions under which a process will -/// exit. -/// -/// Values of this type are used to describe the conditions under which an exit -/// test is expected to pass or fail by passing them to -/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. -@_spi(Experimental) -#if SWT_NO_PROCESS_SPAWNING -@available(*, unavailable, message: "Exit tests are not available on this platform.") -#endif -public enum ExitCondition: Sendable { - /// The process terminated successfully with status `EXIT_SUCCESS`. - public static var success: Self { - // Strictly speaking, the C standard treats 0 as a successful exit code and - // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern - // operating system defines EXIT_SUCCESS to any value other than 0, so the - // distinction is academic. - .exitCode(EXIT_SUCCESS) - } - - /// The process terminated abnormally with any status other than - /// `EXIT_SUCCESS` or with any signal. - case failure - - /// The process terminated with the given exit code. - /// - /// - Parameters: - /// - exitCode: The exit code yielded by the process. - /// - /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), - /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their - /// own non-standard exit codes: - /// - /// | Platform | Header | - /// |-|-| - /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | - /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | - /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | - /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | - /// - /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by - /// the process is yielded to the parent process. Linux and other POSIX-like - /// systems may only reliably report the low unsigned 8 bits (0–255) of - /// the exit code. - case exitCode(_ exitCode: CInt) - - /// The process terminated with the given signal. - /// - /// - Parameters: - /// - signal: The signal that terminated the process. - /// - /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). - /// Platforms may additionally define their own non-standard signal codes: - /// - /// | Platform | Header | - /// |-|-| - /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | - /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | - /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | - /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | - case signal(_ signal: CInt) -} - -// MARK: - Equatable - -@_spi(Experimental) -#if SWT_NO_PROCESS_SPAWNING -@available(*, unavailable, message: "Exit tests are not available on this platform.") -#endif -extension Optional { - /// Check whether or not two exit conditions are equal. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are equal. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are _exactly_ equal, - /// use the ``===(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs == rhs) // prints "true" - /// print(lhs === rhs) // prints "false" - /// ``` - /// - /// This special behavior means that the ``==(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a == b` implies that `a != b` is `false`. - public static func ==(lhs: Self, rhs: Self) -> Bool { -#if !SWT_NO_PROCESS_SPAWNING - return switch (lhs, rhs) { - case let (.failure, .exitCode(exitCode)), let (.exitCode(exitCode), .failure): - exitCode != EXIT_SUCCESS - case (.failure, .signal), (.signal, .failure): - // All terminating signals are considered failures. - true - default: - lhs === rhs - } -#else - fatalError("Unsupported") -#endif - } - - /// Check whether or not two exit conditions are _not_ equal. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are _not_ equal. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are not _exactly_ - /// equal, use the ``!==(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs != rhs) // prints "false" - /// print(lhs !== rhs) // prints "true" - /// ``` - /// - /// This special behavior means that the ``!=(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a == b` implies that `a != b` is `false`. - public static func !=(lhs: Self, rhs: Self) -> Bool { -#if !SWT_NO_PROCESS_SPAWNING - !(lhs == rhs) -#else - fatalError("Unsupported") -#endif - } - - /// Check whether or not two exit conditions are identical. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are identical. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are _exactly_ equal, - /// use the ``===(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs == rhs) // prints "true" - /// print(lhs === rhs) // prints "false" - /// ``` - /// - /// This special behavior means that the ``==(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a === b` implies that `a !== b` is `false`. - public static func ===(lhs: Self, rhs: Self) -> Bool { - return switch (lhs, rhs) { - case (.none, .none): - true - case (.failure, .failure): - true - case let (.exitCode(lhs), .exitCode(rhs)): - lhs == rhs - case let (.signal(lhs), .signal(rhs)): - lhs == rhs - default: - false - } - } - - /// Check whether or not two exit conditions are _not_ identical. - /// - /// - Parameters: - /// - lhs: One value to compare. - /// - rhs: Another value to compare. - /// - /// - Returns: Whether or not `lhs` and `rhs` are _not_ identical. - /// - /// Two exit conditions can be compared; if either instance is equal to - /// ``ExitCondition/failure``, it will compare equal to any instance except - /// ``ExitCondition/success``. To check if two instances are not _exactly_ - /// equal, use the ``!==(_:_:)`` operator: - /// - /// ```swift - /// let lhs: ExitCondition = .failure - /// let rhs: ExitCondition = .signal(SIGINT) - /// print(lhs != rhs) // prints "false" - /// print(lhs !== rhs) // prints "true" - /// ``` - /// - /// This special behavior means that the ``!=(_:_:)`` operator is not - /// transitive, and does not satisfy the requirements of - /// [`Equatable`](https://developer.apple.com/documentation/swift/equatable) - /// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable). - /// - /// For any values `a` and `b`, `a === b` implies that `a !== b` is `false`. - public static func !==(lhs: Self, rhs: Self) -> Bool { -#if !SWT_NO_PROCESS_SPAWNING - !(lhs === rhs) -#else - fatalError("Unsupported") -#endif - } -} diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift new file mode 100644 index 000000000..10f2a6ff0 --- /dev/null +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -0,0 +1,152 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest { + /// The possible conditions under which an exit test will complete. + /// + /// Values of this type are used to describe the conditions under which an + /// exit test is expected to pass or fail by passing them to + /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + public struct Condition: Sendable { + /// An enumeration describing the possible conditions for an exit test. + private enum _Kind: Sendable, Equatable { + /// The exit test must exit with a particular exit status. + case statusAtExit(StatusAtExit) + + /// The exit test must exit with any failure. + case failure + } + + /// The kind of condition. + private var _kind: _Kind + } +} + +// MARK: - + +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest.Condition { + /// A condition that matches when a process terminates successfully with exit + /// code `EXIT_SUCCESS`. + public static var success: Self { + // Strictly speaking, the C standard treats 0 as a successful exit code and + // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern + // operating system defines EXIT_SUCCESS to any value other than 0, so the + // distinction is academic. +#if !SWT_NO_EXIT_TESTS + .exitCode(EXIT_SUCCESS) +#else + fatalError("Unsupported") +#endif + } + + /// A condition that matches when a process terminates abnormally with any + /// exit code other than `EXIT_SUCCESS` or with any signal. + public static var failure: Self { + Self(_kind: .failure) + } + + public init(_ statusAtExit: StatusAtExit) { + self.init(_kind: .statusAtExit(statusAtExit)) + } + + /// Creates a condition that matches when a process terminates with a given + /// exit code. + /// + /// - Parameters: + /// - exitCode: The exit code yielded by the process. + /// + /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), + /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their + /// own non-standard exit codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | + /// + /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by + /// the process is yielded to the parent process. Linux and other POSIX-like + /// systems may only reliably report the low unsigned 8 bits (0–255) of + /// the exit code. + public static func exitCode(_ exitCode: CInt) -> Self { +#if !SWT_NO_EXIT_TESTS + Self(.exitCode(exitCode)) +#else + fatalError("Unsupported") +#endif + } + + /// Creates a condition that matches when a process terminates with a given + /// signal. + /// + /// - Parameters: + /// - signal: The signal that terminated the process. + /// + /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). + /// Platforms may additionally define their own non-standard signal codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + public static func signal(_ signal: CInt) -> Self { +#if !SWT_NO_EXIT_TESTS + Self(.signal(signal)) +#else + fatalError("Unsupported") +#endif + } +} + +// MARK: - Comparison + +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest.Condition { + /// Check whether or not an exit test condition matches a given exit status. + /// + /// - Parameters: + /// - statusAtExit: An exit status to compare against. + /// + /// - Returns: Whether or not `self` and `statusAtExit` represent the same + /// exit condition. + /// + /// Two exit test conditions can be compared; if either instance is equal to + /// ``failure``, it will compare equal to any instance except ``success``. + func isApproximatelyEqual(to statusAtExit: StatusAtExit) -> Bool { + return switch (self._kind, statusAtExit) { + case let (.failure, .exitCode(exitCode)): + exitCode != EXIT_SUCCESS + case (.failure, .signal): + // All terminating signals are considered failures. + true + default: + self._kind == .statusAtExit(statusAtExit) + } + } +} diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift new file mode 100644 index 000000000..beb2d56fc --- /dev/null +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -0,0 +1,86 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest { + /// A type representing the result of an exit test after it has exited and + /// returned control to the calling test function. + /// + /// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and + /// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return + /// instances of this type. + public struct Result: Sendable { + /// The exit condition the exit test exited with. + /// + /// When the exit test passes, the value of this property is equal to the + /// exit status reported by the process that hosted the exit test. + public var statusAtExit: StatusAtExit + + /// All bytes written to the standard output stream of the exit test before + /// it exited. + /// + /// The value of this property may contain any arbitrary sequence of bytes, + /// including sequences that are not valid UTF-8 and cannot be decoded by + /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) + /// instead. + /// + /// When checking the value of this property, keep in mind that the standard + /// output stream is globally accessible, and any code running in an exit + /// test may write to it including including the operating system and any + /// third-party dependencies you have declared in your package. Rather than + /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) + /// to check if expected output is present. + /// + /// To enable gathering output from the standard output stream during an + /// exit test, pass `\.standardOutputContent` in the `observedValues` + /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)`` + /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// + /// If you did not request standard output content when running an exit + /// test, the value of this property is the empty array. + public var standardOutputContent: [UInt8] = [] + + /// All bytes written to the standard error stream of the exit test before + /// it exited. + /// + /// The value of this property may contain any arbitrary sequence of bytes, + /// including sequences that are not valid UTF-8 and cannot be decoded by + /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) + /// instead. + /// + /// When checking the value of this property, keep in mind that the standard + /// error stream is globally accessible, and any code running in an exit + /// test may write to it including including the operating system and any + /// third-party dependencies you have declared in your package. Rather than + /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) + /// to check if expected output is present. + /// + /// To enable gathering output from the standard error stream during an exit + /// test, pass `\.standardErrorContent` in the `observedValues` argument of + /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// + /// If you did not request standard error content when running an exit test, + /// the value of this property is the empty array. + public var standardErrorContent: [UInt8] = [] + + @_spi(ForToolsIntegrationOnly) + public init(statusAtExit: StatusAtExit) { + self.statusAtExit = statusAtExit + } + } +} diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 75102abe2..9425de432 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -25,9 +25,9 @@ private import _TestingInternals /// A type describing an exit test. /// /// Instances of this type describe exit tests you create using the -/// ``expect(exitsWith:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:_:sourceLocation:performing:)`` macro. You don't usually -/// need to interact directly with an instance of this type. +/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` +/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` macro. You +/// don't usually need to interact directly with an instance of this type. @_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") @@ -62,8 +62,8 @@ public struct ExitTest: Sendable, ~Copyable { /// /// Key paths are not sendable because the properties they refer to may or may /// not be, so this property needs to be `nonisolated(unsafe)`. It is safe to - /// use it in this fashion because `ExitTestArtifacts` is sendable. - fileprivate var _observedValues = [any PartialKeyPath & Sendable]() + /// use it in this fashion because ``ExitTest/Result`` is sendable. + fileprivate var _observedValues = [any PartialKeyPath & Sendable]() /// Key paths representing results from within this exit test that should be /// observed and returned to the caller. @@ -74,17 +74,17 @@ public struct ExitTest: Sendable, ~Copyable { /// this property to determine what information you need to preserve from your /// child process. /// - /// The value of this property always includes ``ExitTestArtifacts/exitCondition`` + /// The value of this property always includes ``ExitTest/Result/statusAtExit`` /// even if the test author does not specify it. /// /// Within a child process running an exit test, the value of this property is /// otherwise unspecified. @_spi(ForToolsIntegrationOnly) - public var observedValues: [any PartialKeyPath & Sendable] { + public var observedValues: [any PartialKeyPath & Sendable] { get { var result = _observedValues - if !result.contains(\.exitCondition) { // O(n), but n <= 3 (no Set needed) - result.append(\.exitCondition) + if !result.contains(\.statusAtExit) { // O(n), but n <= 3 (no Set needed) + result.append(\.statusAtExit) } return result } @@ -283,7 +283,7 @@ extension ExitTest { /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTestArtifacts/exitCondition`` property is always returned. +/// ``ExitTest/Result/statusAtExit`` property is always returned. /// - expression: The expression, corresponding to `condition`, that is being /// evaluated (if available at compile time.) /// - comments: An array of comments describing the expectation. This array @@ -299,19 +299,19 @@ extension ExitTest { /// convention. func callExitTest( identifiedBy exitTestID: (UInt64, UInt64), - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable], + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result { guard let configuration = Configuration.current ?? Configuration.all.first else { preconditionFailure("A test must be running on the current task to use #expect(exitsWith:).") } - var result: ExitTestArtifacts + var result: ExitTest.Result do { var exitTest = ExitTest(id: ExitTest.ID(exitTestID)) exitTest.observedValues = observedValues @@ -320,8 +320,8 @@ func callExitTest( #if os(Windows) // For an explanation of this magic, see the corresponding logic in // ExitTest.callAsFunction(). - if case let .exitCode(exitCode) = result.exitCondition, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { - result.exitCondition = .signal(exitCode & STATUS_CODE_MASK) + if case let .exitCode(exitCode) = result.statusAtExit, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { + result.statusAtExit = .signal(exitCode & STATUS_CODE_MASK) } #endif } catch { @@ -346,17 +346,22 @@ func callExitTest( // For lack of a better way to handle an exit test failing in this way, // we record the system issue above, then let the expectation fail below by // reporting an exit condition that's the inverse of the expected one. - result = ExitTestArtifacts(exitCondition: expectedExitCondition == .failure ? .success : .failure) + let statusAtExit: StatusAtExit = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) { + .exitCode(EXIT_SUCCESS) + } else { + .exitCode(EXIT_FAILURE) + } + result = ExitTest.Result(statusAtExit: statusAtExit) } // How did the exit test actually exit? - let actualExitCondition = result.exitCondition + let actualStatusAtExit = result.statusAtExit // Plumb the exit test's result through the general expectation machinery. return __checkValue( - expectedExitCondition == actualExitCondition, + expectedExitCondition.isApproximatelyEqual(to: actualStatusAtExit), expression: expression, - expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualExitCondition), + expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualStatusAtExit), mismatchedExitConditionDescription: String(describingForTest: expectedExitCondition), comments: comments(), isRequired: isRequired, @@ -404,7 +409,7 @@ extension ExitTest { /// are available or the child environment is otherwise terminated. The parent /// environment is then responsible for interpreting those results and /// recording any issues that occur. - public typealias Handler = @Sendable (_ exitTest: borrowing ExitTest) async throws -> ExitTestArtifacts + public typealias Handler = @Sendable (_ exitTest: borrowing ExitTest) async throws -> ExitTest.Result /// The back channel file handle set up by the parent process. /// @@ -581,7 +586,7 @@ extension ExitTest { childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) } - typealias ResultUpdater = @Sendable (inout ExitTestArtifacts) -> Void + typealias ResultUpdater = @Sendable (inout ExitTest.Result) -> Void return try await withThrowingTaskGroup(of: ResultUpdater?.self) { taskGroup in // Set up stdout and stderr streams. By POSIX convention, stdin/stdout // are line-buffered by default and stderr is unbuffered by default. @@ -641,8 +646,8 @@ extension ExitTest { // Await termination of the child process. taskGroup.addTask { - let exitCondition = try await wait(for: processID) - return { $0.exitCondition = exitCondition } + let statusAtExit = try await wait(for: processID) + return { $0.statusAtExit = statusAtExit } } // Read back the stdout and stderr streams. @@ -669,10 +674,10 @@ extension ExitTest { return nil } - // Collate the various bits of the result. The exit condition .failure - // here is just a placeholder and will be replaced by the result of one - // of the tasks above. - var result = ExitTestArtifacts(exitCondition: .failure) + // Collate the various bits of the result. The exit condition used here + // is just a placeholder and will be replaced by the result of one of + // the tasks above. + var result = ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) for try await update in taskGroup { update?(&result) } diff --git a/Sources/Testing/ExitTests/ExitTestArtifacts.swift b/Sources/Testing/ExitTests/ExitTestArtifacts.swift deleted file mode 100644 index 6e1710e2a..000000000 --- a/Sources/Testing/ExitTests/ExitTestArtifacts.swift +++ /dev/null @@ -1,90 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// A type representing the result of an exit test after it has exited and -/// returned control to the calling test function. -/// -/// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and -/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return -/// instances of this type. -/// -/// - Warning: The name of this type is still unstable and subject to change. -@_spi(Experimental) -#if SWT_NO_EXIT_TESTS -@available(*, unavailable, message: "Exit tests are not available on this platform.") -#endif -public struct ExitTestArtifacts: Sendable { - /// The exit condition the exit test exited with. - /// - /// When the exit test passes, the value of this property is equal to the - /// value of the `expectedExitCondition` argument passed to - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or to - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. You can - /// compare two instances of ``ExitCondition`` with - /// ``/Swift/Optional/==(_:_:)``. - public var exitCondition: ExitCondition - - /// All bytes written to the standard output stream of the exit test before - /// it exited. - /// - /// The value of this property may contain any arbitrary sequence of bytes, - /// including sequences that are not valid UTF-8 and cannot be decoded by - /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). - /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) - /// instead. - /// - /// When checking the value of this property, keep in mind that the standard - /// output stream is globally accessible, and any code running in an exit - /// test may write to it including including the operating system and any - /// third-party dependencies you have declared in your package. Rather than - /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), - /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) - /// to check if expected output is present. - /// - /// To enable gathering output from the standard output stream during an - /// exit test, pass `\.standardOutputContent` in the `observedValues` - /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)`` - /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``. - /// - /// If you did not request standard output content when running an exit test, - /// the value of this property is the empty array. - public var standardOutputContent: [UInt8] = [] - - /// All bytes written to the standard error stream of the exit test before - /// it exited. - /// - /// The value of this property may contain any arbitrary sequence of bytes, - /// including sequences that are not valid UTF-8 and cannot be decoded by - /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). - /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) - /// instead. - /// - /// When checking the value of this property, keep in mind that the standard - /// output stream is globally accessible, and any code running in an exit - /// test may write to it including including the operating system and any - /// third-party dependencies you have declared in your package. Rather than - /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), - /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) - /// to check if expected output is present. - /// - /// To enable gathering output from the standard error stream during an exit - /// test, pass `\.standardErrorContent` in the `observedValues` argument of - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. - /// - /// If you did not request standard error content when running an exit test, - /// the value of this property is the empty array. - public var standardErrorContent: [UInt8] = [] - - @_spi(ForToolsIntegrationOnly) - public init(exitCondition: ExitCondition) { - self.exitCondition = exitCondition - } -} diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 4726e8f41..c824baa4e 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information diff --git a/Sources/Testing/ExitTests/StatusAtExit.swift b/Sources/Testing/ExitTests/StatusAtExit.swift new file mode 100644 index 000000000..26514ffa5 --- /dev/null +++ b/Sources/Testing/ExitTests/StatusAtExit.swift @@ -0,0 +1,73 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// An enumeration describing possible status a process will yield on exit. +/// +/// You can convert an instance of this type to an instance of +/// ``ExitTest/Condition`` using ``ExitTest/Condition/init(_:)``. That value +/// can then be used to describe the condition under which an exit test is +/// expected to pass or fail by passing it to +/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or +/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. +@_spi(Experimental) +#if SWT_NO_PROCESS_SPAWNING +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +public enum StatusAtExit: Sendable { + /// The process terminated with the given exit code. + /// + /// - Parameters: + /// - exitCode: The exit code yielded by the process. + /// + /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), + /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their + /// own non-standard exit codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | + /// + /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by + /// the process is yielded to the parent process. Linux and other POSIX-like + /// systems may only reliably report the low unsigned 8 bits (0–255) of + /// the exit code. + case exitCode(_ exitCode: CInt) + + /// The process terminated with the given signal. + /// + /// - Parameters: + /// - signal: The signal that terminated the process. + /// + /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). + /// Platforms may additionally define their own non-standard signal codes: + /// + /// | Platform | Header | + /// |-|-| + /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | + /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | + /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | + /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + case signal(_ signal: CInt) +} + +// MARK: - Equatable + +@_spi(Experimental) +#if SWT_NO_PROCESS_SPAWNING +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension StatusAtExit: Equatable {} diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index ccd6512b6..cc611158f 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -20,7 +20,7 @@ internal import _TestingInternals /// /// - Throws: If the exit status of the process with ID `pid` cannot be /// determined (i.e. it does not represent an exit condition.) -private func _blockAndWait(for pid: consuming pid_t) throws -> ExitCondition { +private func _blockAndWait(for pid: consuming pid_t) throws -> StatusAtExit { let pid = consume pid // Get the exit status of the process or throw an error (other than EINTR.) @@ -61,7 +61,7 @@ private func _blockAndWait(for pid: consuming pid_t) throws -> ExitCondition { /// - Note: The open-source implementation of libdispatch available on Linux /// and other platforms does not support `DispatchSourceProcess`. Those /// platforms use an alternate implementation below. -func wait(for pid: consuming pid_t) async throws -> ExitCondition { +func wait(for pid: consuming pid_t) async throws -> StatusAtExit { let pid = consume pid let source = DispatchSource.makeProcessSource(identifier: pid, eventMask: .exit) @@ -80,7 +80,7 @@ func wait(for pid: consuming pid_t) async throws -> ExitCondition { } #elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) /// A mapping of awaited child PIDs to their corresponding Swift continuations. -private let _childProcessContinuations = LockedWith]>() +private let _childProcessContinuations = LockedWith]>() /// A condition variable used to suspend the waiter thread created by /// `_createWaitThread()` when there are no child processes to await. @@ -202,7 +202,7 @@ private let _createWaitThread: Void = { /// /// On Apple platforms, the libdispatch-based implementation above is more /// efficient because it does not need to permanently reserve a thread. -func wait(for pid: consuming pid_t) async throws -> ExitCondition { +func wait(for pid: consuming pid_t) async throws -> StatusAtExit { let pid = consume pid // Ensure the waiter thread is running. @@ -239,7 +239,7 @@ func wait(for pid: consuming pid_t) async throws -> ExitCondition { /// This implementation of `wait(for:)` calls `RegisterWaitForSingleObject()` to /// wait for `processHandle`, suspends the calling task until the waiter's /// callback is called, then calls `GetExitCodeProcess()`. -func wait(for processHandle: consuming HANDLE) async throws -> ExitCondition { +func wait(for processHandle: consuming HANDLE) async throws -> StatusAtExit { let processHandle = consume processHandle defer { _ = CloseHandle(processHandle) @@ -276,13 +276,13 @@ func wait(for processHandle: consuming HANDLE) async throws -> ExitCondition { guard GetExitCodeProcess(processHandle, &status) else { // The child process terminated but we couldn't get its status back. // Assume generic failure. - return .failure + return .exitCode(EXIT_FAILURE) } return .exitCode(CInt(bitPattern: .init(status))) } #else #warning("Platform-specific implementation missing: cannot wait for child processes to exit") -func wait(for processID: consuming Never) async throws -> ExitCondition {} +func wait(for processID: consuming Never) async throws -> StatusAtExit {} #endif #endif diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 134ffad37..5111fbddd 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -490,13 +490,13 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTestArtifacts/exitCondition`` property is always returned. +/// ``ExitTest/Result/statusAtExit`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. /// - expression: The expression to be evaluated. /// -/// - Returns: If the exit test passes, an instance of ``ExitTestArtifacts`` +/// - Returns: If the exit test passes, an instance of ``ExitTest/Result`` /// describing the state of the exit test when it exited. If the exit test /// fails, the result is `nil`. /// @@ -529,8 +529,8 @@ public macro require( /// process is terminated. /// /// Once the child process terminates, the parent process resumes and compares -/// its exit status against `exitCondition`. If they match, the exit test has -/// passed; otherwise, it has failed and an issue is recorded. +/// its exit status against `expectedExitCondition`. If they match, the exit +/// test has passed; otherwise, it has failed and an issue is recorded. /// /// ## Child process output /// @@ -588,12 +588,12 @@ public macro require( #endif @discardableResult @freestanding(expression) public macro expect( - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable] = [], + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void -) -> ExitTestArtifacts? = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") +) -> ExitTest.Result? = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") /// Check that an expression causes the process to terminate in a given fashion /// and throw an error if it did not. @@ -602,13 +602,13 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTestArtifacts/exitCondition`` property is always returned. +/// ``ExitTest/Result/statusAtExit`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. /// - expression: The expression to be evaluated. /// -/// - Returns: An instance of ``ExitTestArtifacts`` describing the state of the +/// - Returns: An instance of ``ExitTest/Result`` describing the state of the /// exit test when it exited. /// /// - Throws: An instance of ``ExpectationFailedError`` if the exit condition of @@ -643,8 +643,8 @@ public macro require( /// process is terminated. /// /// Once the child process terminates, the parent process resumes and compares -/// its exit status against `exitCondition`. If they match, the exit test has -/// passed; otherwise, it has failed and an issue is recorded. +/// its exit status against `expectedExitCondition`. If they match, the exit +/// test has passed; otherwise, it has failed and an issue is recorded. /// /// ## Child process output /// @@ -700,9 +700,9 @@ public macro require( #endif @discardableResult @freestanding(expression) public macro require( - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable] = [], + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void -) -> ExitTestArtifacts = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") +) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 550f6039c..7254ad049 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1148,15 +1148,15 @@ public func __checkClosureCall( @_spi(Experimental) public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64), - exitsWith expectedExitCondition: ExitCondition, - observing observedValues: [any PartialKeyPath & Sendable], + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable], performing body: @convention(thin) () -> Void, expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result { await callExitTest( identifiedBy: exitTestID, exitsWith: expectedExitCondition, diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index f687aa631..b4f5af1c3 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -415,15 +415,15 @@ extension ExitTestConditionMacro { _ = try Base.expansion(of: macro, in: context) var arguments = argumentList(of: macro, in: context) - let expectedExitConditionIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("exitsWith") } - guard let expectedExitConditionIndex else { - fatalError("Could not find the exit condition for this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + let requirementIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("exitsWith") } + guard let requirementIndex else { + fatalError("Could not find the requirement for this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } let observationListIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("observing") } if observationListIndex == nil { arguments.insert( Argument(label: "observing", expression: ArrayExprSyntax(expressions: [])), - at: arguments.index(after: expectedExitConditionIndex) + at: arguments.index(after: requirementIndex) ) } let trailingClosureIndex = arguments.firstIndex { $0.label?.tokenKind == _trailingClosureLabel.tokenKind } diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 00d54f3f6..bc3425e0a 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -88,29 +88,15 @@ private import _TestingInternals // Mock an exit test where the process exits successfully. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) } await Test { await #expect(exitsWith: .success) {} }.run(configuration: configuration) - // Mock an exit test where the process exits with a generic failure. - configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .failure) - } - await Test { - await #expect(exitsWith: .failure) {} - }.run(configuration: configuration) - await Test { - await #expect(exitsWith: .exitCode(EXIT_FAILURE)) {} - }.run(configuration: configuration) - await Test { - await #expect(exitsWith: .signal(SIGABRT)) {} - }.run(configuration: configuration) - // Mock an exit test where the process exits with a particular error code. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .exitCode(123)) + return ExitTest.Result(statusAtExit: .exitCode(123)) } await Test { await #expect(exitsWith: .failure) {} @@ -118,7 +104,7 @@ private import _TestingInternals // Mock an exit test where the process exits with a signal. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .signal(SIGABRT)) + return ExitTest.Result(statusAtExit: .signal(SIGABRT)) } await Test { await #expect(exitsWith: .signal(SIGABRT)) {} @@ -140,7 +126,7 @@ private import _TestingInternals // Mock exit tests that were expected to fail but passed. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) } await Test { await #expect(exitsWith: .failure) {} @@ -154,7 +140,7 @@ private import _TestingInternals // Mock exit tests that unexpectedly signalled. configuration.exitTestHandler = { _ in - return ExitTestArtifacts(exitCondition: .signal(SIGABRT)) + return ExitTest.Result(statusAtExit: .signal(SIGABRT)) } await Test { await #expect(exitsWith: .exitCode(EXIT_SUCCESS)) {} @@ -239,36 +225,6 @@ private import _TestingInternals } #endif - @Test("Exit condition matching operators (==, !=, ===, !==)") - func exitConditionMatching() { - #expect(Optional.none == Optional.none) - #expect(Optional.none === Optional.none) - #expect(Optional.none !== .some(.success)) - #expect(Optional.none !== .some(.failure)) - - #expect(ExitCondition.success == .success) - #expect(ExitCondition.success === .success) - #expect(ExitCondition.success == .exitCode(EXIT_SUCCESS)) - #expect(ExitCondition.success === .exitCode(EXIT_SUCCESS)) - #expect(ExitCondition.success != .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.success !== .exitCode(EXIT_FAILURE)) - - #expect(ExitCondition.failure == .failure) - #expect(ExitCondition.failure === .failure) - - #expect(ExitCondition.exitCode(EXIT_FAILURE &+ 1) != .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.exitCode(EXIT_FAILURE &+ 1) !== .exitCode(EXIT_FAILURE)) - - #expect(ExitCondition.success != .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.success !== .exitCode(EXIT_FAILURE)) - #expect(ExitCondition.success != .signal(SIGINT)) - #expect(ExitCondition.success !== .signal(SIGINT)) - #expect(ExitCondition.signal(SIGINT) == .signal(SIGINT)) - #expect(ExitCondition.signal(SIGINT) === .signal(SIGINT)) - #expect(ExitCondition.signal(SIGTERM) != .signal(SIGINT)) - #expect(ExitCondition.signal(SIGTERM) !== .signal(SIGINT)) - } - @MainActor static func someMainActorFunction() { MainActor.assertIsolated() } @@ -289,21 +245,21 @@ private import _TestingInternals var result = await #expect(exitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.exitCondition === .success) + #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) result = await #expect(exitsWith: .exitCode(123)) { exit(123) } - #expect(result?.exitCondition === .exitCode(123)) + #expect(result?.statusAtExit == .exitCode(123)) // Test that basic passing exit tests produce the correct results (#require) result = try await #require(exitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.exitCondition === .success) + #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) result = try await #require(exitsWith: .exitCode(123)) { exit(123) } - #expect(result?.exitCondition === .exitCode(123)) + #expect(result?.statusAtExit == .exitCode(123)) } @Test("Result is nil on failure") @@ -322,7 +278,7 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTestArtifacts(exitCondition: .exitCode(123)) + ExitTest.Result(statusAtExit: .exitCode(123)) } await Test { @@ -345,7 +301,7 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTestArtifacts(exitCondition: .failure) + ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) } await Test { @@ -392,7 +348,7 @@ private import _TestingInternals try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.exitCondition === .success) + #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.contains("STANDARD OUTPUT".utf8)) #expect(result.standardErrorContent.isEmpty) @@ -401,7 +357,7 @@ private import _TestingInternals try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.exitCondition === .success) + #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.isEmpty) #expect(result.standardErrorContent.contains("STANDARD ERROR".utf8.reversed())) } @@ -409,7 +365,7 @@ private import _TestingInternals @Test("Arguments to the macro are not captured during expansion (do not need to be literals/const)") func argumentsAreNotCapturedDuringMacroExpansion() async throws { let unrelatedSourceLocation = #_sourceLocation - func nonConstExitCondition() async throws -> ExitCondition { + func nonConstExitCondition() async throws -> ExitTest.Condition { .failure } await #expect(exitsWith: try await nonConstExitCondition(), sourceLocation: unrelatedSourceLocation) { From bc10850903728e165ec5861bf7b3579bb42c449d Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 28 Feb 2025 14:59:21 -0600 Subject: [PATCH 103/234] Use more distinct characters for 'skip' and 'pass w/known issues' symbols (#983) This refines the characters used for the `.fail` and `.pass(knownIssueCount:)` symbols to make them more distinct, especially when rendered in a console without ANSI colors enabled. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Events/Recorder/Event.Symbol.swift | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index 3a3f6df8e..846fb2d4d 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -62,7 +62,7 @@ extension Event.Symbol { ("\u{10065F}", "arrow.triangle.turn.up.right.diamond.fill") case let .pass(knownIssueCount): if knownIssueCount > 0 { - ("\u{100884}", "xmark.diamond.fill") + ("\u{100883}", "xmark.diamond") } else { ("\u{10105B}", "checkmark.diamond.fill") } @@ -118,8 +118,8 @@ extension Event.Symbol { // Unicode: WHITE DIAMOND return "\u{25C7}" case .skip: - // Unicode: HEAVY BALLOT X - return "\u{2718}" + // Unicode: HEAVY ROUND-TIPPED RIGHTWARDS ARROW + return "\u{279C}" case let .pass(knownIssueCount): if knownIssueCount > 0 { // Unicode: HEAVY BALLOT X @@ -156,8 +156,8 @@ extension Event.Symbol { // Unicode: LOZENGE return "\u{25CA}" case .skip: - // Unicode: MULTIPLICATION SIGN - return "\u{00D7}" + // Unicode: HEAVY ROUND-TIPPED RIGHTWARDS ARROW + return "\u{279C}" case let .pass(knownIssueCount): if knownIssueCount > 0 { // Unicode: MULTIPLICATION SIGN From 0c84d57bf1ef92c6e08a579012da39f156fb0f1b Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 28 Feb 2025 15:42:24 -0600 Subject: [PATCH 104/234] Add a utility tool for previewing event symbol rendering (#984) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a development-only utility executable target to the package which allows previewing the event symbols rendered using various console output styles. Screenshot 2025-02-28 at 2 14 47 PM ### Motivation: This is a tool I made and found useful while working on #983. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 10 ++ .../SymbolShowcase/SymbolShowcaseMain.swift | 117 ++++++++++++++++++ .../Event.ConsoleOutputRecorder.swift | 2 +- 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 Sources/SymbolShowcase/SymbolShowcaseMain.swift diff --git a/Package.swift b/Package.swift index 43337b0dc..132bb8ef7 100644 --- a/Package.swift +++ b/Package.swift @@ -134,6 +134,16 @@ let package = Package( .enableLibraryEvolution(applePlatformsOnly: true), ] ), + + // Utility targets: These are utilities intended for use when developing + // this package, not for distribution. + .executableTarget( + name: "SymbolShowcase", + dependencies: [ + "Testing", + ], + swiftSettings: .packageSettings + ), ], cxxLanguageStandard: .cxx20 diff --git a/Sources/SymbolShowcase/SymbolShowcaseMain.swift b/Sources/SymbolShowcase/SymbolShowcaseMain.swift new file mode 100644 index 000000000..74816779d --- /dev/null +++ b/Sources/SymbolShowcase/SymbolShowcaseMain.swift @@ -0,0 +1,117 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +import Foundation + +/// A type which acts as the main entry point for this executable target. +@main enum SymbolShowcaseMain { + static func main() { + let nameColumnWidth = symbols.reduce(into: 0) { $0 = max($0, $1.0.count) } + 4 + let styleColumnWidth = styles.reduce(into: 0) { $0 = max($0, $1.label.count) } + 2 + let totalHeaderWidth = nameColumnWidth + (styleColumnWidth * styles.count) + + // Print the table header. + print("Name".padding(toLength: nameColumnWidth), terminator: "") + for style in styles { + print(style.label.padding(toLength: styleColumnWidth), terminator: "") + } + print() + print(String(repeating: "=", count: totalHeaderWidth)) + + // Print a row for each symbol, with a preview of each style. + for (label, symbol) in symbols { + print(label.padding(toLength: nameColumnWidth), terminator: "") + for style in styles { + print(style.string(for: symbol), terminator: "") + print("".padding(toLength: styleColumnWidth - 1), terminator: "") + } + print() + } + } + + /// The symbols to preview. + fileprivate static var symbols: KeyValuePairs { + [ + "Default": .default, + "Pass": .pass(knownIssueCount: 0), + "Pass w/known issues": .pass(knownIssueCount: 1), + "Pass with warnings": .passWithWarnings, + "Skip": .skip, + "Fail": .fail, + "Difference": .difference, + "Warning": .warning, + "Details": .details, + "Attachment": .attachment, + ] + } + + /// The styles to preview. + fileprivate static var styles: [Style] { + var styles: [Style] = [ + Style(label: "Unicode", usesColor: false), + Style(label: "w/color", usesColor: true), + ] + +#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) + styles.append(contentsOf: [ + Style(label: "SF Symbols", usesColor: false, usesSFSymbols: true), + Style(label: "w/color", usesColor: true, usesSFSymbols: true), + ]) +#endif + + return styles + } +} + +/// A type representing a style of symbol to preview. +fileprivate struct Style { + /// The label for this style, displayed in its column header. + var label: String + + /// Whether this style should render symbols using ANSI color. + var usesColor: Bool + +#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) + /// Whether this style should use SF Symbols. + var usesSFSymbols: Bool = false +#endif + + /// Return a string for the specified symbol based on this style's options. + /// + /// - Parameters: + /// - symbol: The symbol to format into a string. + /// + /// - Returns: A formatted string representing the specified symbol. + func string(for symbol: Event.Symbol) -> String { + var options = Event.ConsoleOutputRecorder.Options() + options.useANSIEscapeCodes = usesColor + options.ansiColorBitDepth = usesColor ? 8 : 1 +#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst)) + options.useSFSymbols = usesSFSymbols +#endif + + return symbol.stringValue(options: options) + } +} + +extension String { + /// Returns a new string formed from this String by either removing characters + /// from the end, or by appending as many occurrences as necessary of a given + /// pad string. + /// + /// - Parameters: + /// - newLength: The length to pad to. + /// + /// - Returns: A padded string. + fileprivate func padding(toLength newLength: Int) -> Self { + padding(toLength: newLength, withPad: " ", startingAt: 0) + } +} diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index b375b2da1..ea48e7ad1 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -136,7 +136,7 @@ extension Event.Symbol { /// /// - Returns: A string representation of `self` appropriate for writing to /// a stream. - fileprivate func stringValue(options: Event.ConsoleOutputRecorder.Options) -> String { + package func stringValue(options: Event.ConsoleOutputRecorder.Options) -> String { let useColorANSIEscapeCodes = options.useANSIEscapeCodes && options.ansiColorBitDepth >= 4 var symbolCharacter = String(unicodeCharacter) From 69a1e26edd482177139f816a422956bb18b60a9f Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 28 Feb 2025 15:56:17 -0600 Subject: [PATCH 105/234] JUnit XML recorder should ignore warning issues (#986) This updates `Event.JUnitXMLRecorder` to ignore `Issue` instances whose `severity` is less than `.error` (such as `.warning`). ### Motivation: The concept of issue severity was recently added in #931 (but was reverted and re-landed in #952), and that did not adjust the JUnit XML recorder logic. The JUnit XML schema we currently attempt to adhere to does not appear to have a way to represent non-fatal issues, so I think it would be best for now to ignore these issues. ### Modifications: - Implement the fix and a validating unit test. - (Drive-by) Fix a nearby test I noticed wasn't actually working as intended and wasn't properly validating the fix it was intended to. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Recorder/Event.JUnitXMLRecorder.swift | 2 +- Tests/TestingTests/EventRecorderTests.swift | 30 +++++++++++++++---- .../TestSupport/TestingAdditions.swift | 6 ++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.JUnitXMLRecorder.swift b/Sources/Testing/Events/Recorder/Event.JUnitXMLRecorder.swift index 0b9b283e3..c761753d2 100644 --- a/Sources/Testing/Events/Recorder/Event.JUnitXMLRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.JUnitXMLRecorder.swift @@ -122,7 +122,7 @@ extension Event.JUnitXMLRecorder { } return nil case let .issueRecorded(issue): - if issue.isKnown { + if issue.isKnown || issue.severity < .error { return nil } if let id = test?.id { diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 7712cd973..8ac7f6728 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -467,15 +467,35 @@ struct EventRecorderTests { @Test("JUnitXMLRecorder counts issues without associated tests") func junitRecorderCountsIssuesWithoutTests() async throws { let issue = Issue(kind: .unconditional) - let event = Event(.issueRecorded(issue), testID: nil, testCaseID: nil) let context = Event.Context(test: nil, testCase: nil, configuration: nil) - let recorder = Event.JUnitXMLRecorder { string in - if string.contains(" Date: Mon, 3 Mar 2025 13:52:28 -0500 Subject: [PATCH 106/234] Diagnose when using a non-escapable type as suite. (#988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR introduces a custom compile-time diagnostic when attempting to use a non-escapable type as a suite. For example: ```swift @Suite struct NumberOfBeesTests: ~Escapable { @Test borrowing func countBees() { ... } // 🛑 Attribute 'Test' cannot be applied to a function within structure 'NumberOfBeesTests' because its conformance to 'Escapable' has been suppressed } ``` Values with non-escapable type cannot currently be initialized nor returned from a function, and we need to be able to do both in order to correctly implement the `@Test` macro. This change does not diagnose if `@Suite` is applied to such a type but does not contain any test functions, because we do compile successfully in that case and this sort of pattern remains valid: ```swift @Suite struct MyTests: ~Escapable { @Suite struct EndToEndTests { ... } } ``` ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../TypeSyntaxProtocolAdditions.swift | 1 + .../Support/DiagnosticMessage.swift | 54 ++++++++----------- .../TestingMacros/TestDeclarationMacro.swift | 20 ++++++- .../TestDeclarationMacroTests.swift | 6 +++ 4 files changed, 48 insertions(+), 33 deletions(-) diff --git a/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift b/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift index e6e381e94..e1bd346ed 100644 --- a/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift @@ -54,6 +54,7 @@ extension TypeSyntaxProtocol { let nameWithoutGenericParameters = tokens(viewMode: .fixedUp) .prefix { $0.tokenKind != .leftAngle } .filter { $0.tokenKind != .period } + .filter { $0.tokenKind != .leftParen && $0.tokenKind != .rightParen } .map(\.textWithoutBackticks) .joined(separator: ".") diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index aa26b9dc7..a474d2801 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -321,65 +321,57 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage { /// generic. /// - attribute: The `@Test` or `@Suite` attribute. /// - decl: The declaration in question (contained in `node`.) + /// - escapableNonConformance: The suppressed conformance to `Escapable` for + /// `decl`, if present. /// /// - Returns: A diagnostic message. - static func containingNodeUnsupported(_ node: some SyntaxProtocol, genericBecauseOf genericClause: Syntax? = nil, whenUsing attribute: AttributeSyntax, on decl: some SyntaxProtocol) -> Self { + static func containingNodeUnsupported(_ node: some SyntaxProtocol, genericBecauseOf genericClause: Syntax? = nil, whenUsing attribute: AttributeSyntax, on decl: some SyntaxProtocol, withSuppressedConformanceToEscapable escapableNonConformance: SuppressedTypeSyntax? = nil) -> Self { // Avoid using a syntax node from a lexical context (it won't have source // location information.) let syntax: Syntax = if let genericClause, attribute.root == genericClause.root { // Prefer the generic clause if available as the root cause. genericClause + } else if let escapableNonConformance, attribute.root == escapableNonConformance.root { + // Then the ~Escapable conformance if present. + Syntax(escapableNonConformance) } else if attribute.root == node.root { - // Second choice is the unsupported containing node. + // Next best choice is the unsupported containing node. Syntax(node) } else { // Finally, fall back to the attribute, which we assume is not detached. Syntax(attribute) } + + // Figure out the message to present. + var message = "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true))" let generic = if genericClause != nil { " generic" } else { "" } if let functionDecl = node.as(FunctionDeclSyntax.self) { - let functionName = functionDecl.completeName - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within\(generic) function '\(functionName)'", - severity: .error - ) + message += " within\(generic) function '\(functionDecl.completeName)'" } else if let namedDecl = node.asProtocol((any NamedDeclSyntax).self) { - let declName = namedDecl.name.textWithoutBackticks - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within\(generic) \(_kindString(for: node)) '\(declName)'", - severity: .error - ) + message += " within\(generic) \(_kindString(for: node)) '\(namedDecl.name.textWithoutBackticks)'" } else if let extensionDecl = node.as(ExtensionDeclSyntax.self) { // Subtly different phrasing from the NamedDeclSyntax case above. - let nodeKind = if genericClause != nil { - "a generic extension to type" + if genericClause != nil { + message += " within a generic extension to type '\(extensionDecl.extendedType.trimmedDescription)'" } else { - "an extension to type" + message += " within an extension to type '\(extensionDecl.extendedType.trimmedDescription)'" } - let declGroupName = extensionDecl.extendedType.trimmedDescription - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within \(nodeKind) '\(declGroupName)'", - severity: .error - ) } else { - let nodeKind = if genericClause != nil { - "a generic \(_kindString(for: node))" + if genericClause != nil { + message += " within a generic \(_kindString(for: node))" } else { - _kindString(for: node, includeA: true) + message += " within \(_kindString(for: node, includeA: true))" } - return Self( - syntax: syntax, - message: "Attribute \(_macroName(attribute)) cannot be applied to \(_kindString(for: decl, includeA: true)) within \(nodeKind)", - severity: .error - ) } + if escapableNonConformance != nil { + message += " because its conformance to 'Escapable' has been suppressed" + } + + return Self(syntax: syntax, message: message, severity: .error) } /// Create a diagnostic message stating that the given attribute cannot be diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 1a3f2c448..1b9f995bc 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -61,14 +61,15 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } // Check if the lexical context is appropriate for a suite or test. - diagnostics += diagnoseIssuesWithLexicalContext(context.lexicalContext, containing: declaration, attribute: testAttribute) + let lexicalContext = context.lexicalContext + diagnostics += diagnoseIssuesWithLexicalContext(lexicalContext, containing: declaration, attribute: testAttribute) // Suites inheriting from XCTestCase are not supported. We are a bit // conservative here in this check and only check the immediate context. // Presumably, if there's an intermediate lexical context that is *not* a // type declaration, then it must be a function or closure (disallowed // elsewhere) and thus the test function is not a member of any type. - if let containingTypeDecl = context.lexicalContext.first?.asProtocol((any DeclGroupSyntax).self), + if let containingTypeDecl = lexicalContext.first?.asProtocol((any DeclGroupSyntax).self), containingTypeDecl.inherits(fromTypeNamed: "XCTestCase", inModuleNamed: "XCTest") { diagnostics.append(.containingNodeUnsupported(containingTypeDecl, whenUsing: testAttribute, on: declaration)) } @@ -118,6 +119,21 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } } + // Disallow non-escapable types as suites. In order to support them, the + // compiler team needs to finish implementing the lifetime dependency + // feature so that `init()`, ``__requiringTry()`, and `__requiringAwait()` + // can be correctly expressed. + if let containingType = lexicalContext.first?.asProtocol((any DeclGroupSyntax).self), + let inheritedTypes = containingType.inheritanceClause?.inheritedTypes { + let escapableNonConformances = inheritedTypes + .map(\.type) + .compactMap { $0.as(SuppressedTypeSyntax.self) } + .filter { $0.type.isNamed("Escapable", inModuleNamed: "Swift") } + for escapableNonConformance in escapableNonConformances { + diagnostics.append(.containingNodeUnsupported(containingType, whenUsing: testAttribute, on: function, withSuppressedConformanceToEscapable: escapableNonConformance)) + } + } + return !diagnostics.lazy.map(\.severity).contains(.error) } diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index a77acfea1..96eb9075c 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -140,6 +140,12 @@ struct TestDeclarationMacroTests { "Attribute 'Test' cannot be applied to a function within a generic extension to type 'T!'", "extension T! { @Suite struct S {} }": "Attribute 'Suite' cannot be applied to a structure within a generic extension to type 'T!'", + "struct S: ~Escapable { @Test func f() {} }": + "Attribute 'Test' cannot be applied to a function within structure 'S' because its conformance to 'Escapable' has been suppressed", + "struct S: ~Swift.Escapable { @Test func f() {} }": + "Attribute 'Test' cannot be applied to a function within structure 'S' because its conformance to 'Escapable' has been suppressed", + "struct S: ~(Escapable) { @Test func f() {} }": + "Attribute 'Test' cannot be applied to a function within structure 'S' because its conformance to 'Escapable' has been suppressed", ] ) func apiMisuseErrors(input: String, expectedMessage: String) throws { From 46fdaaf8e78fd9770a4afe2ad2c0b1ecc382a3eb Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 3 Mar 2025 14:36:23 -0500 Subject: [PATCH 107/234] Suppress `.unsafeFlags()` in Package.swift when tagging for release. (#991) This PR removes the unsafe flags we specify in our Package.swift manifest when the package has been tagged in Git (which indicates it's a release or prerelease.) This allows a package to add Swift Testing as a package dependency without breaking its own ability to be added as a package dependency due to the use of unsafe flags. ### Checklist: - [ ] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [ ] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 70 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/Package.swift b/Package.swift index 132bb8ef7..5be4d4e1c 100644 --- a/Package.swift +++ b/Package.swift @@ -13,6 +13,13 @@ import PackageDescription import CompilerPluginSupport +/// Information about the current state of the package's git repository. +let git = Context.gitInformation + +/// Whether or not this package is being built for development rather than +/// distribution as a package dependency. +let buildingForDevelopment = (git?.currentTag == nil) + let package = Package( name: "swift-testing", @@ -55,9 +62,7 @@ let package = Package( ], exclude: ["CMakeLists.txt", "Testing.swiftcrossimport"], cxxSettings: .packageSettings, - swiftSettings: .packageSettings + [ - .enableLibraryEvolution(), - ], + swiftSettings: .packageSettings + .enableLibraryEvolution(), linkerSettings: [ .linkedLibrary("execinfo", .when(platforms: [.custom("freebsd"), .openbsd])) ] @@ -86,10 +91,8 @@ let package = Package( .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), ], exclude: ["CMakeLists.txt"], - swiftSettings: .packageSettings + [ - // When building as a package, the macro plugin always builds as an - // executable rather than a library. - .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), + swiftSettings: .packageSettings + { + var result = [PackageDescription.SwiftSetting]() // The only target which needs the ability to import this macro // implementation target's module is its unit test target. Users of the @@ -97,8 +100,12 @@ let package = Package( // Testing module. This target's module is never distributed to users, // but as an additional guard against accidental misuse, this specifies // the unit test target as the only allowable client. - .unsafeFlags(["-Xfrontend", "-allowable-client", "-Xfrontend", "TestingMacrosTests"]), - ] + if buildingForDevelopment { + result.append(.unsafeFlags(["-Xfrontend", "-allowable-client", "-Xfrontend", "TestingMacrosTests"])) + } + + return result + }() ), // "Support" targets: These contain C family code and are used exclusively @@ -116,9 +123,7 @@ let package = Package( "Testing", ], path: "Sources/Overlays/_Testing_CoreGraphics", - swiftSettings: .packageSettings + [ - .enableLibraryEvolution(), - ] + swiftSettings: .packageSettings + .enableLibraryEvolution() ), .target( name: "_Testing_Foundation", @@ -127,12 +132,10 @@ let package = Package( ], path: "Sources/Overlays/_Testing_Foundation", exclude: ["CMakeLists.txt"], - swiftSettings: .packageSettings + [ - // The Foundation module only has Library Evolution enabled on Apple - // platforms, and since this target's module publicly imports Foundation, - // it can only enable Library Evolution itself on those platforms. - .enableLibraryEvolution(applePlatformsOnly: true), - ] + // The Foundation module only has Library Evolution enabled on Apple + // platforms, and since this target's module publicly imports Foundation, + // it can only enable Library Evolution itself on those platforms. + swiftSettings: .packageSettings + .enableLibraryEvolution(applePlatformsOnly: true), ), // Utility targets: These are utilities intended for use when developing @@ -167,14 +170,23 @@ extension Array where Element == PackageDescription.SwiftSetting { /// Settings intended to be applied to every Swift target in this package. /// Analogous to project-level build settings in an Xcode project. static var packageSettings: Self { - availabilityMacroSettings + [ - .unsafeFlags(["-require-explicit-sendable"]), + var result = availabilityMacroSettings + + if buildingForDevelopment { + result.append(.unsafeFlags(["-require-explicit-sendable"])) + } + + result += [ .enableUpcomingFeature("ExistentialAny"), .enableExperimentalFeature("SuppressedAssociatedTypes"), .enableExperimentalFeature("AccessLevelOnImport"), .enableUpcomingFeature("InternalImportsByDefault"), + // When building as a package, the macro plugin always builds as an + // executable rather than a library. + .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), + .define("SWT_TARGET_OS_APPLE", .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS])), .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), @@ -183,6 +195,8 @@ extension Array where Element == PackageDescription.SwiftSetting { .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] + + return result } /// Settings which define commonly-used OS availability macros. @@ -203,9 +217,7 @@ extension Array where Element == PackageDescription.SwiftSetting { .enableExperimentalFeature("AvailabilityMacro=_distantFuture:macOS 99.0, iOS 99.0, watchOS 99.0, tvOS 99.0, visionOS 99.0"), ] } -} -extension PackageDescription.SwiftSetting { /// Create a Swift setting which enables Library Evolution, optionally /// constraining it to only Apple platforms. /// @@ -213,7 +225,17 @@ extension PackageDescription.SwiftSetting { /// - applePlatformsOnly: Whether to constrain this setting to only Apple /// platforms. static func enableLibraryEvolution(applePlatformsOnly: Bool = false) -> Self { - unsafeFlags(["-enable-library-evolution"], .when(platforms: applePlatformsOnly ? [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS] : [])) + var result = [PackageDescription.SwiftSetting]() + + if buildingForDevelopment { + var condition: BuildSettingCondition? + if applePlatformsOnly { + condition = .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS]) + } + result.append(.unsafeFlags(["-enable-library-evolution"], condition)) + } + + return result } } @@ -232,7 +254,7 @@ extension Array where Element == PackageDescription.CXXSetting { ] // Capture the testing library's version as a C++ string constant. - if let git = Context.gitInformation { + if let git { let testingLibraryVersion = if let tag = git.currentTag { tag } else if git.hasUncommittedChanges { From 270ee571bfbbf18e2a3688acc7c6535b916b9116 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 4 Mar 2025 09:37:16 -0500 Subject: [PATCH 108/234] Fix typo affecting the 6.0 compiler --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 5be4d4e1c..a34eed92f 100644 --- a/Package.swift +++ b/Package.swift @@ -135,7 +135,7 @@ let package = Package( // The Foundation module only has Library Evolution enabled on Apple // platforms, and since this target's module publicly imports Foundation, // it can only enable Library Evolution itself on those platforms. - swiftSettings: .packageSettings + .enableLibraryEvolution(applePlatformsOnly: true), + swiftSettings: .packageSettings + .enableLibraryEvolution(applePlatformsOnly: true) ), // Utility targets: These are utilities intended for use when developing From f1892c3ae27105552931bd66cd94495b08e06763 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 4 Mar 2025 12:39:26 -0600 Subject: [PATCH 109/234] Relocate the proposal template to swift-evolution and update documentation accordingly (#976) The feature and API proposal template in this repo is being relocated to swift-evolution. (See https://github.com/swiftlang/swift-evolution/pull/2709.) This removes the file from this repo and updates the documentation Readme file accordingly. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Proposals/0000-proposal-template.md | 184 ------- .../Proposals/0001-refactor-bug-inits.md | 174 +----- Documentation/Proposals/0002-json-abi.md | 425 +-------------- .../0003-make-serialized-trait-api.md | 157 +----- ...ranularity-of-test-time-limit-durations.md | 209 +------ .../Proposals/0005-ranged-confirmations.md | 192 +------ .../0006-return-errors-from-expect-throws.md | 269 +-------- .../Proposals/0007-test-scoping-traits.md | 516 +----------------- Documentation/README.md | 7 +- 9 files changed, 57 insertions(+), 2076 deletions(-) delete mode 100644 Documentation/Proposals/0000-proposal-template.md diff --git a/Documentation/Proposals/0000-proposal-template.md b/Documentation/Proposals/0000-proposal-template.md deleted file mode 100644 index 342c04000..000000000 --- a/Documentation/Proposals/0000-proposal-template.md +++ /dev/null @@ -1,184 +0,0 @@ -# Feature name - -* Proposal: [SWT-NNNN](NNNN-filename.md) -* Authors: [Author 1](https://github.com/author1), [Author 2](https://github.com/author2) -* Status: **Awaiting implementation** or **Awaiting review** -* Bug: _if applicable_ [swiftlang/swift-testing#NNNNN](https://github.com/swiftlang/swift-testing/issues/NNNNN) -* Implementation: [swiftlang/swift-testing#NNNNN](https://github.com/swiftlang/swift-testing/pull/NNNNN) -* Previous Proposal: _if applicable_ [SWT-XXXX](XXXX-filename.md) -* Previous Revision: _if applicable_ [1](https://github.com/swiftlang/swift-testing/blob/...commit-ID.../Documentation/Proposals/NNNN-filename.md) -* Review: ([pitch](https://forums.swift.org/...)) - -When filling out this template, you should delete or replace all of the text -except for the section headers and the header fields above. For example, you -should delete everything from this paragraph down to the Introduction section -below. - -As a proposal author, you should fill out all of the header fields. Delete any -header fields marked _if applicable_ that are not applicable to your proposal. - -When sharing a link to the proposal while it is still a PR, be sure to share a -live link to the proposal, not an exact commit, so that readers will always see -the latest version when you make changes. On GitHub, you can find this link by -browsing the PR branch: from the PR page, click the "username wants to merge ... -from username:my-branch-name" link and find the proposal file in that branch. - -`Status` should reflect the current implementation status while the proposal is -still a PR. The proposal cannot be reviewed until an implementation is available, -but early readers should see the correct status. - -`Bug` should be used when this proposal is fixing a bug with significant -discussion in the bug report. It is not necessary to link bugs that do not -contain significant discussion or that merely duplicate discussion linked -somewhere else. Do not link bugs from private bug trackers. - -`Implementation` should link to the PR(s) implementing the feature. If the -proposal has not been implemented yet, or if it simply codifies existing -behavior, just say that. If the implementation has already been committed to the -main branch (as an experimental feature or SPI), mention that. If the -implementation is spread across multiple PRs, just link to the most important -ones. - -`Previous Proposal` should be used when there is a specific line of succession -between this proposal and another proposal. For example, this proposal might -have been removed from a previous proposal so that it can be reviewed separately, -or this proposal might supersede a previous proposal in some way that was felt -to exceed the scope of a "revision". Include text briefly explaining the -relationship, such as "Supersedes SWT-1234" or "Extracted from SWT-01234". If -possible, link to a post explaining the relationship, such as a review decision -that asked for part of the proposal to be split off. Otherwise, you can just -link to the previous proposal. - -`Previous Revision` should be added after a major substantive revision of a -proposal that has undergone review. It links to the previously reviewed revision. -It is not necessary to add or update this field after minor editorial changes. - -`Review` is a history of all discussion threads about this proposal, in -chronological order. Use these standardized link names: `pitch` `review` -`revision` `acceptance` `rejection`. If there are multiple such threads, spell -the ordinal out: `first pitch` `second review` etc. - -## Introduction - -A short description of what the feature is. Try to keep it to a single-paragraph -"elevator pitch" so the reader understands what problem this proposal is -addressing. - -## Motivation - -Describe the problems that this proposal seeks to address. If the problem is -that some common pattern is currently hard to express, show how one can -currently get a similar effect and describe its drawbacks. If it's completely -new functionality that cannot be emulated, motivate why this new functionality -would help Swift developers test their code more effectively. - -## Proposed solution - -Describe your solution to the problem. Provide examples and describe how they -work. Show how your solution is better than current workarounds: is it cleaner, -safer, or more efficient? - -This section doesn't have to be comprehensive. Focus on the most important parts -of the proposal and make arguments about why the proposal is better than the -status quo. - -## Detailed design - -Describe the design of the solution in detail. If it includes new API, show the -full API and its documentation comments detailing what it does. If it involves -new macro logic, describe the behavior changes and include a succinct example of -the additions or modifications to the macro expansion code. The detail in this -section should be sufficient for someone who is *not* one of the authors to be -able to reasonably implement the feature. - -## Source compatibility - -Describe the impact of this proposal on source compatibility. As a general rule, -all else being equal, test code that worked in previous releases of the testing -library should work in new releases. That means both that it should continue to -build and that it should continue to behave dynamically the same as it did -before. - -This is not an absolute guarantee, and the testing library administrators will -consider intentional compatibility breaks if their negative impact can be shown -to be small and the current behavior is causing substantial problems in practice. - -For proposals that affect testing library API, consider the impact on existing -clients. If clients provide a similar API, will type-checking find the right one? -If the feature overloads an existing API, is it problematic that existing users -of that API might start resolving to the new API? - -## Integration with supporting tools - -In this section, describe how this proposal affects tools which integrate with -the testing library. Some features depend on supporting tools gaining awareness -of the new feature for users to realize new benefits. Other features do not -strictly require integration but bring improvement opportunities which are worth -considering. Use this section to discuss any impact on tools. - -This section does need not to include details of how this proposal may be -integrated with _specific_ tools, but it should consider the general ways that -tools might support this feature and note any accompanying SPI intended for -tools which are included in the implementation. Note that tools may evolve -independently and have differing release schedules than the testing library, so -special care should be taken to ensure compatibility across versions according -to the needs of each tool. - -## Future directions - -Describe any interesting proposals that could build on this proposal in the -future. This is especially important when these future directions inform the -design of the proposal, for example by making sure an interface meant for tools -integration can be extended to include additional information. - -The rest of the proposal should generally not talk about future directions -except by referring to this section. It is important not to confuse reviewers -about what is covered by this specific proposal. If there's a larger vision that -needs to be explained in order to understand this proposal, consider starting a -discussion thread on the forums to capture your broader thoughts. - -Avoid making affirmative statements in this section, such as "we will" or even -"we should". Describe the proposals neutrally as possibilities to be considered -in the future. - -Consider whether any of these future directions should really just be part of -the current proposal. It's important to make focused, self-contained proposals -that can be incrementally implemented and reviewed, but it's also good when -proposals feel "complete" rather than leaving significant gaps in their design. -An an example from the Swift project, when -[SE-0193](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0193-cross-module-inlining-and-specialization.md) -introduced the `@inlinable` attribute, it also included the `@usableFromInline` -attribute so that declarations used in inlinable functions didn't have to be -`public`. This was a relatively small addition to the proposal which avoided -creating a serious usability problem for many adopters of `@inlinable`. - -## Alternatives considered - -Describe alternative approaches to addressing the same problem. This is an -important part of most proposal documents. Reviewers are often familiar with -other approaches prior to review and may have reasons to prefer them. This -section is your first opportunity to try to convince them that your approach is -the right one, and even if you don't fully succeed, you can help set the terms -of the conversation and make the review a much more productive exchange of ideas. - -You should be fair about other proposals, but you do not have to be neutral; -after all, you are specifically proposing something else. Describe any -advantages these alternatives might have, but also be sure to explain the -disadvantages that led you to prefer the approach in this proposal. - -You should update this section during the pitch phase to discuss any -particularly interesting alternatives raised by the community. You do not need -to list every idea raised during the pitch, just the ones you think raise points -that are worth discussing. Of course, if you decide the alternative is more -compelling than what's in the current proposal, you should change the main -proposal; be sure to then discuss your previous proposal in this section and -explain why the new idea is better. - -## Acknowledgments - -If significant changes or improvements suggested by members of the community -were incorporated into the proposal as it developed, take a moment here to thank -them for their contributions. This is a collaborative process, and everyone's -input should receive recognition! - -Generally, you should not acknowledge anyone who is listed as a co-author. diff --git a/Documentation/Proposals/0001-refactor-bug-inits.md b/Documentation/Proposals/0001-refactor-bug-inits.md index 0a4f00566..18927db22 100644 --- a/Documentation/Proposals/0001-refactor-bug-inits.md +++ b/Documentation/Proposals/0001-refactor-bug-inits.md @@ -1,168 +1,10 @@ # Dedicated `.bug()` functions for URLs and IDs -* Proposal: [SWT-0001](0001-refactor-bug-inits.md) -* Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Implemented (Swift 6.0)** -* Implementation: [swiftlang/swift-testing#401](https://github.com/swiftlang/swift-testing/pull/401) -* Review: ([pitch](https://forums.swift.org/t/pitch-dedicated-bug-functions-for-urls-and-ids/71842)), ([acceptance](https://forums.swift.org/t/swt-0001-dedicated-bug-functions-for-urls-and-ids/71842/2)) - -## Introduction - -One of the features of swift-testing is a test traits system that allows -associating metadata with a test suite or test function. One trait in -particular, `.bug()`, has the potential for integration with development tools -but needs some refinement before integration would be practical. - -## Motivation - -A test author can associate a bug (AKA issue, problem, ticket, etc.) with a test -using the `.bug()` trait, to which they pass an "identifier" for the bug. The -swift-testing team's intent here was that a test author would pass the unique -identifier of the bug in the test author's preferred bug-tracking system (e.g. -GitHub Issues, Bugzilla, etc.) and that any tooling built around this trait -would be able to infer where the bug was located and how to view it. - -It became clear immediately that a generic system for looking up bugs by unique -identifier in an arbitrary and unspecified database wouldn't be a workable -solution. So we modified the description of `.bug()` to explain that, if the -identifier passed to it was a valid URL, then it would be "interpreted" as a URL -and that tools could be designed to open that URL as needed. - -This design change then placed the burden of parsing each `.bug()` trait and -potentially mapping it to a URL on tools. swift-testing itself avoids linking to -or using Foundation API such as `URL`, so checking for a valid URL inside the -testing library was not feasible either. - -## Proposed solution - -To solve the underlying problem and allow test authors to specify a URL when -available, or just an opaque identifier otherwise, we propose splitting the -`.bug()` function up into two overloads: - -- The first overload takes a URL string and additional optional metadata; -- The second overload takes a bug identifier as an opaque string or integer and, - optionally, a URL string. - -Test authors are then free to specify any combination of URL and opaque -identifier depending on the information they have available and their specific -needs. Tools authors are free to consume either or both of these properties and -present them where appropriate. - -## Detailed design - -The `Bug` trait type and `.bug()` trait factory function shall be refactored -thusly: - -```swift -/// A type representing a bug report tracked by a test. -/// -/// To add this trait to a test, use one of the following functions: -/// -/// - ``Trait/bug(_:_:)`` -/// - ``Trait/bug(_:id:_:)-10yf5`` -/// - ``Trait/bug(_:id:_:)-3vtpl`` -public struct Bug: TestTrait, SuiteTrait, Equatable, Hashable, Codable { - /// A URL linking to more information about the bug, if available. - /// - /// The value of this property represents a URL conforming to - /// [RFC 3986](https://www.ietf.org/rfc/rfc3986.txt). - public var url: String? - - /// A unique identifier in this bug's associated bug-tracking system, if - /// available. - /// - /// For more information on how the testing library interprets bug - /// identifiers, see . - public var id: String? - - /// The human-readable title of the bug, if specified by the test author. - public var title: Comment? -} - -extension Trait where Self == Bug { - /// Construct a bug to track with a test. - /// - /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking - /// system. - /// - title: Optionally, the human-readable title of the bug. - /// - /// - Returns: An instance of ``Bug`` representing the specified bug. - public static func bug(_ url: _const String, _ title: Comment? = nil) -> Self - - /// Construct a bug to track with a test. - /// - /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking - /// system. - /// - id: The unique identifier of this bug in its associated bug-tracking - /// system. - /// - title: Optionally, the human-readable title of the bug. - /// - /// - Returns: An instance of ``Bug`` representing the specified bug. - public static func bug(_ url: _const String? = nil, id: some Numeric, _ title: Comment? = nil) -> Self - - /// Construct a bug to track with a test. - /// - /// - Parameters: - /// - url: A URL referring to this bug in the associated bug-tracking - /// system. - /// - id: The unique identifier of this bug in its associated bug-tracking - /// system. - /// - title: Optionally, the human-readable title of the bug. - /// - /// - Returns: An instance of ``Bug`` representing the specified bug. - public static func bug(_ url: _const String? = nil, id: _const String, _ title: Comment? = nil) -> Self -} -``` - -The `@Test` and `@Suite` macros have already been modified so that they perform -basic validation of a URL string passed as input and emit a diagnostic if the -URL string appears malformed. - -## Source compatibility - -This change is expected to be source-breaking for test authors who have already -adopted the existing `.bug()` functions. This change is source-breaking for code -that directly refers to these functions by their signatures. This change is -source-breaking for code that uses the `identifier` property of the `Bug` type -or expects it to contain a URL. - -## Integration with supporting tools - -Tools that integrate with swift-testing and provide lists of tests or record -results after tests have run can use the `Bug` trait on tests to present -relevant identifiers and/or URLs to users. - -Tools that use the experimental event stream output feature of the testing -library will need a JSON schema for bug traits on tests. This work is tracked in -a separate upcoming proposal. - -## Alternatives considered - -- Inferring whether or not a bug identifier was a URL by parsing it at runtime - in tools. As discussed above, this option would require every tool that - integrates with swift-testing to provide its own URL-parsing logic. - -- Using different argument labels (e.g. the label `url` for the URL argument - and/or no label for the `id` argument.) We felt that URLs, which are - recognizable by their general structure, did not need labels. At least one - argument must have a label to avoid ambiguous resolution of the `.bug()` - function at compile time. - -- Inferring whether or not a bug identifier was a URL by parsing it at compile- - time or at runtime using `Foundation.URL` or libcurl. swift-testing actively - avoids linking to Foundation if at all possible, and libcurl would be a - platform-specific solution (Windows doesn't ship with libcurl, but does have - `InternetCrackUrlW()` whose parsing engine differs.) We also run the risk of - inappropriately interpreting some arbitrary bug identifier as a URL when it is - not meant to be parsed that way. - -- Removing the `.bug()` trait. We see this particular trait as having strong - potential for integration with tools and for use by test authors; removing it - because we can't reliably parse URLs would be unfortunate. - -## Acknowledgments - -Thanks to the swift-testing team and managers for their contributions! Thanks to -our community for the initial feedback around this feature. +> [!NOTE] +> This proposal was accepted before Swift Testing began using the Swift +> evolution review process. Its original identifier was SWT-0001 but its prefix +> has been changed to "ST" and it has been relocated to the +> [swift-evolution](https://github.com/swiftlang/swift-evolution) repository. + +To view this proposal, see +[ST-0001: Dedicated `.bug()` functions for URLs and IDs](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0001-refactor-bug-inits.md). diff --git a/Documentation/Proposals/0002-json-abi.md b/Documentation/Proposals/0002-json-abi.md index 0af939972..8fd1e3242 100644 --- a/Documentation/Proposals/0002-json-abi.md +++ b/Documentation/Proposals/0002-json-abi.md @@ -1,423 +1,10 @@ # A stable JSON-based ABI for tools integration -* Proposal: [SWT-0002](0002-json-abi.md) -* Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Implemented (Swift 6.0)** -* Implementation: [swiftlang/swift-testing#383](https://github.com/swiftlang/swift-testing/pull/383), - [swiftlang/swift-testing#402](https://github.com/swiftlang/swift-testing/pull/402) -* Review: ([pitch](https://forums.swift.org/t/pitch-a-stable-json-based-abi-for-tools-integration/72627)), ([acceptance](https://forums.swift.org/t/pitch-a-stable-json-based-abi-for-tools-integration/72627/4)) - -## Introduction - -One of the core components of Swift Testing is its ability to interoperate with -Xcode 16, VS Code, and other tools. Swift Testing has been fully open-sourced -across all platforms supported by Swift, and can be added as a package -dependency (or—eventually—linked from the Swift toolchain.) - -## Motivation - -Because Swift Testing may be used in various forms, and because integration with -various tools is critical to its success, we need it to have a stable interface -that can be used regardless of how it's been added to a package. There are a few -patterns in particular we know we need to support: - -- An IDE (e.g. Xcode 16) that builds and links its own copy of Swift Testing: - the copy used by the IDE might be the same as the copy that tests use, in - which case interoperation is trivial, but it may also be distinct if the tests - use Swift Testing as a package dependency. - - In the case of Xcode 16, Swift Testing is built as a framework much like - XCTest and is automatically linked by test targets in an Xcode project or - Swift package, but if the test target specifies a package dependency on Swift - Testing, that dependency will take priority when the test code is compiled. - -- An IDE (e.g. VS Code) that does _not_ link directly to Swift Testing (and - perhaps, as with VS Code, cannot because it is not natively compiled): such an - IDE needs a way to configure and invoke test code and then to read events back - as they occur, but cannot touch the Swift symbols used by the tests. - - In the case of VS Code, because it is implemented using TypeScript, it is not - able to directly link to Swift Testing or other Swift libraries. In order for - it to interpret events from a test run like "test started" or "issue - recorded", it needs to receive those events in a format it can understand. - -Tools integration is important to the success of Swift Testing. The more tools -provide integrations for it, the more likely developers are to adopt it. The -more developers adopt, the more tests are written. And the more tests are -written, the better our lives as software engineers will be. - -## Proposed solution - -We propose defining and implementing a stable ABI for using Swift Testing that -can be reliably adopted by various IDEs and other tools. There are two aspects -of this ABI we need to implement: - -- A stable entry point function that can be resolved dynamically at runtime (on - platforms with dynamic loaders such as Darwin, Linux, and Windows.) This - function needs a signature that will not change over time and which will take - input and pass back asynchronous output in a format that a wide variety of - tools will be able to interpret (whether they are written in Swift or not.) - - This function should be implemented in Swift as it is expected to be used by - code that can call into Swift, but which cannot rely on the specific binary - minutiae of a given copy of Swift Testing. - -- A stable format for input that can be passed to the entry point function and - which can also be passed at the command line; and a stable format for output - that can be consumed by tools to interpret test results. - - Some tools cannot directly link to Swift code and must instead rely on - command-line invocations of `swift test`. These tools will be able to pass - their test configuration and options as an argument in the stable format and - will be able to receive event information in the same stable format via a - dedicated channel such as a file or named pipe. - -> [!NOTE] -> This document proposes defining a stable format for input and output, but only -> actually defines the JSON schema for _output_. We intend to define the schema -> for input in a subsequent proposal. -> -> In the interim, early adopters can encode an instance of Swift Testing's -> `__CommandLineArguments_v0` type using `JSONEncoder`. - -## Detailed design - -We propose defining the stable input and output format using JSON as it is -widely supported across platforms and languages. The proposed JSON schema for -output is defined [here](../ABI/JSON.md). - -### Example output - -The proposed schema is a sequence of JSON objects written to an event handler or -file stream. When a test run starts, Swift Testing first emits a sequence of -JSON objects representing each test that is part of the planned run. For -example, this is the JSON representation of Swift Testing's own `canGetStdout()` -test function: - -```json -{ - "kind": "test", - "payload": { - "displayName": "Can get stdout", - "id": "TestingTests.FileHandleTests/canGetStdout()/FileHandleTests.swift:33:4", - "isParameterized": false, - "kind": "function", - "name": "canGetStdout()", - "sourceLocation": { - "column": 4, - "fileID": "TestingTests/FileHandleTests.swift", - "line": 33 - } - }, - "version": 0 -} -``` - -A tool that is observing this data stream can build a map or dictionary of test -IDs to comprehensive test details if needed. Once all tests in the planned run -have been written out, testing begins. Swift Testing writes a sequence of JSON -objects representing various events such as "test started" or "issue recorded". -For example, here is an abridged sequence of events generated for a test that -records a failed expectation: - -```json -{ - "kind": "event", - "payload": { - "instant": { - "absolute": 266418.545786299, - "since1970": 1718302639.76747 - }, - "kind": "testStarted", - "messages": [ - { - "symbol": "default", - "text": "Test \"Can get stdout\" started." - } - ], - "testID": "TestingTests.FileHandleTests/canGetStdout()/FileHandleTests.swift:33:4" - }, - "version": 0 -} - -{ - "kind": "event", - "payload": { - "instant": { - "absolute": 266636.524236724, - "since1970": 1718302857.74857 - }, - "issue": { - "isKnown": false, - "sourceLocation": { - "column": 7, - "fileID": "TestingTests/FileHandleTests.swift", - "line": 29 - } - }, - "kind": "issueRecorded", - "messages": [ - { - "symbol": "fail", - "text": "Expectation failed: (EOF → -1) == (feof(fileHandle) → 0)" - } - ], - "testID": "TestingTests.FileHandleTests/canGetStdout()/FileHandleTests.swift:33:4" - }, - "version": 0 -} - -{ - "kind": "event", - "payload": { - "instant": { - "absolute": 266636.524741106, - "since1970": 1718302857.74908 - }, - "kind": "testEnded", - "messages": [ - { - "symbol": "fail", - "text": "Test \"Can get stdout\" failed after 0.001 seconds with 1 issue." - } - ], - "testID": "TestingTests.FileHandleTests/canGetStdout()/FileHandleTests.swift:33:4" - }, - "version": 0 -} -``` - -Each event includes zero or more "messages" that Swift Testing intends to -present to the user. These messages contain human-readable text as well as -abstractly-specified symbols that correspond to the output written to the -standard error stream of the test process. Tools can opt to present these -messages in whatever ways are appropriate for their interfaces. - -### Invoking from the command line - -When invoking `swift test`, we propose adding three new arguments to Swift -Package Manager: - -| Argument | Value Type | Description | -|---|:-:|---| -| `--configuration-path` | File system path | Specifies a path to a file, named pipe, etc. containing test configuration/options. | -| `--event-stream-output-path` | File system path | Specifies a path to a file, named pipe, etc. to which output should be written. | -| `--event-stream-version` | Integer | Specifies the version of the stable JSON schema to use for output. | - -The process for adding arguments to Swift Package Manager is separate from the -process for Swift Testing API changes, so the names of these arguments are -speculative and are subject to change as part of the Swift Package Manager -review process. - -If `--configuration-path` is specified, Swift Testing will open it for reading -and attempt to decode its contents as JSON. If `--event-stream-output-path` is -specified, Swift Testing will open it for writing and will write a sequence of -[JSON Lines](https://jsonlines.org) to it representing the data and events -produced by the test run. `--event-stream-version` determines the stable schema -used for output; pass `0` to match the schema proposed in this document. - > [!NOTE] -> If `--event-stream-output-path` is specified but `--event-stream-version` is -> not, the format _currently_ used is based on direct JSON encodings of the -> internal Swift structures used by Swift Testing. This format is necessary to -> support Xcode 16 Beta 1. In the future, the default value of this argument -> will be assumed to equal the newest available JSON schema version (`0` as of -> this document's acceptance, i.e. the JSON schema will match what we are -> proposing here until a new schema supersedes it.) -> -> Tools authors that rely on the JSON schema are strongly advised to specify a -> version rather than relying on this behavior to avoid breaking changes in the -> future. - -On platforms that support them, callers can use a named pipe with -`--event-stream-output-path` to get live results back from the test run rather -than needing to wait until the file is closed by the test process. Named pipes -can be created on Darwin or Linux with the POSIX [`mkfifo()`](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/mkfifo.2.html) -function or on Windows with the [`CreateNamedPipe()`](https://learn.microsoft.com/en-us/windows/win32/api/namedpipeapi/nf-namedpipeapi-createnamedpipew) -function. - -If `--configuration-path` is specified in addition to explicit command-line -options like `--no-parallel`, the explicit command-line options take priority. - -### Invoking from Swift - -Tools that can link to and call Swift directly have the option of instantiating -the tools-only SPI type `Runner`, however this is only possible if the tools and -the test target link to the exact same copy of Swift Testing. To support tools -that may link to a different copy (intentionally or otherwise), we propose -adding an exported symbol to the Swift Testing library with the following Swift -signature: - -```swift -@_spi(ForToolsIntegrationOnly) -public enum ABIv0 { - /* ... */ - - /// The type of the entry point to the testing library used by tools that want - /// to remain version-agnostic regarding the testing library. - /// - /// - Parameters: - /// - configurationJSON: A buffer to memory representing the test - /// configuration and options. If `nil`, a new instance is synthesized - /// from the command-line arguments to the current process. - /// - recordHandler: A JSON record handler to which is passed a buffer to - /// memory representing each record as described in `ABI/JSON.md`. - /// - /// - Returns: Whether or not the test run finished successfully. - /// - /// - Throws: Any error that occurred prior to running tests. Errors that are - /// thrown while tests are running are handled by the testing library. - public typealias EntryPoint = @convention(thin) @Sendable ( - _ configurationJSON: UnsafeRawBufferPointer?, - _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void - ) async throws -> Bool - - /// The entry point to the testing library used by tools that want to remain - /// version-agnostic regarding the testing library. - /// - /// The value of this property is a Swift function that can be used by tools - /// that do not link directly to the testing library and wish to invoke tests - /// in a binary that has been loaded into the current process. The value of - /// this property is accessible from C and C++ as a function with name - /// `"swt_abiv0_getEntryPoint"` and can be dynamically looked up at runtime - /// using `dlsym()` or a platform equivalent. - /// - /// The value of this property can be thought of as equivalent to - /// `swift test --event-stream-output-path` except that, instead of streaming - /// JSON records to a named pipe or file, it streams them to an in-process - /// callback. - public static var entryPoint: EntryPoint { get } -} -``` - -The inputs and outputs to this function are typed as `UnsafeRawBufferPointer` -rather than `Data` because the latter is part of Foundation, and adding a public -dependency on a Foundation type would make it very difficult for Foundation to -adopt Swift Testing. It is a goal of the Swift Testing team to keep our Swift -dependency list as small as possible. - -### Invoking from C or C++ - -We expect most tools that need to make use of this entry point will not be able -to directly link to the exported Swift symbol and will instead need to look it -up at runtime using a platform-specific interface such as [`dlsym()`](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dlsym.3.html) -or [`GetProcAddress()`](https://learn.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-getprocaddress). -The `ABIv0.entryPoint` property's getter will be exported to C and C++ as: - -```c++ -extern "C" const void *_Nonnull swt_abiv0_getEntryPoint(void); -``` - -The value returned from this C function is a direct representation of the value -of `ABIv0.entryPoint` and can be cast back to its Swift function type using -[`unsafeBitCast(_:to:)`](https://developer.apple.com/documentation/swift/unsafebitcast%28_%3Ato%3A%29). - -On platforms where data-pointer-to-function-pointer conversion is disallowed per -the C standard, this operation is unsupported. See §6.3.2.3 and §J.5.7 of -[the C standard](https://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf). - -> [!NOTE] -> Swift Testing is statically linked into the main executable when it is -> included as a package dependency. On Linux and other platforms that use the -> ELF executable format, symbol information for the main executable may not be -> available at runtime unless the `--export-dynamic` flag is passed to the -> linker. - -## Source compatibility - -The changes proposed in this document are additive. - -## Integration with supporting tools - -Tools are able to use the proposed additions as described above. - -## Future directions - -- Extending the JSON schema to cover _input_ as well as _output_. As discussed, - we will do so in a subsequent proposal. - -- Extending the JSON schema to include richer information about events such as - specific mismatched values in `#expect()` calls. This information is complex - and we need to take care to model it efficiently and clearly. - -- Adding Markdown or other formats to event messages. Rich text can be used by - tools to emphasize values, switch to code voice, provide improved - accessibility, etc. - -- Adding additional entry points for different access patterns. We anticipate - that a Swift function and a command-line interface are sufficient to cover - most real-world use cases, but it may be the case that tools could use other - mechanisms for starting test runs such as: - - Pure C or Objective-C interfaces; - - A WebAssembly and/or JavaScript [`async`-compatible](https://github.com/WebAssembly/component-model/blob/2f447274b5028f54c549cb4e28ceb493a471dd4b/design/mvp/Async.md) - interface; - - Platform-specific interfaces; or - - Direct bindings to other languages like Rust, Go, C#, etc. - -## Alternatives considered - -- Doing nothing. If we made no changes, we would be effectively requiring - developers to use Xcode for all Swift Testing development and would be - requiring third-party tools to parse human-readable command-line output. This - approach would run counter to several of the Swift project's high-level goals - and would not represent a true cross-platform solution. - -- Using direct JSON encodings of Swift Testing's internal types to represent - output. We initially attempted this and you can see the results in the Swift - Testing repository if you look for "snapshot" types. A major downside became - apparent quickly: these data types don't make for particularly usable JSON - unless you're using `JSONDecoder` to convert back to them, and the default - JSON encodings produced with `JSONEncoder` are not stable if we e.g. add - enumeration cases with associated values or add non-optional fields to types. - -- Using a format other than JSON. We considered using XML, YAML, Apple property - lists, and a few other formats. JSON won out pretty quickly though: it is - widely supported across platforms and languages and it is trivial to create - Swift structures that encode to a well-designed JSON schema using - `JSONEncoder`. Property lists would be just as easy to create, but it is a - proprietary format and would not be trivially decodable on non-Apple platforms - or using non-Apple tools. - -- Exposing the C interface as a function that returns heap-allocated memory - containing a Swift function reference. This allows us to emit a "thick" Swift - function but requires callers to manually manage the resulting memory, and it - may be difficult to reason about code that requires an extra level of pointer - indirection. By having the C entry point function return a thin Swift function - instead, the caller need only bitcast it and can call it directly, and the - equivalent Swift interface can simply be a property getter rather than a - function call. - -- Exposing the C interface as a function that takes a callback and a completion - handler as might traditionally used by Objective-C callers, of the form: - - ```c++ - extern "C" void swt_abiv0_entryPoint( - __attribute__((__noescape__)) const void *_Nullable configurationJSON, - size_t configurationJSONLength, - void *_Null_unspecified context, - void (*_Nonnull recordHandler)( - __attribute__((__noescape__)) const void *recordJSON, - size_t recordJSONLength, - void *_Null_unspecified context - ), - void (*_Nonnull completionHandler)( - _Bool success, - void *_Null_unspecified context - ) - ); - ``` - - The known clients of the native entry point function are all able to call - Swift code and do not need this sort of interface. If there are other clients - that would need the entry point to use a signature like this one, it would be - straightforward to implement it in a future amendment to this proposal. - -## Acknowledgments - -Thanks much to [Dennis Weissmann](https://github.com/dennisweissmann) for his -tireless work in this area and to [Paul LeMarquand](https://github.com/plemarquand) -for putting up with my incessant revisions and nitpicking while he worked on -VS Code's Swift Testing support. +> This proposal was accepted before Swift Testing began using the Swift +> evolution review process. Its original identifier was SWT-0002 but its prefix +> has been changed to "ST" and it has been relocated to the +> [swift-evolution](https://github.com/swiftlang/swift-evolution) repository. -Thanks to the rest of the Swift Testing team for reviewing this proposal and the -JSON schema and to the community for embracing Swift Testing! +To view this proposal, see +[ST-0002: A stable JSON-based ABI for tools integration](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0002-json-abi.md). diff --git a/Documentation/Proposals/0003-make-serialized-trait-api.md b/Documentation/Proposals/0003-make-serialized-trait-api.md index dc2a98c28..4d96ece70 100644 --- a/Documentation/Proposals/0003-make-serialized-trait-api.md +++ b/Documentation/Proposals/0003-make-serialized-trait-api.md @@ -1,151 +1,10 @@ # Make .serialized trait API -* Proposal: [SWT-0003](0003-make-serialized-trait-api.md) -* Authors: [Dennis Weissmann](https://github.com/dennisweissmann) -* Status: **Implemented (Swift 6.0)** -* Implementation: -[swiftlang/swift-testing#535](https://github.com/swiftlang/swift-testing/pull/535) -* Review: -([pitch](https://forums.swift.org/t/pitch-make-serialized-trait-public-api/73147)), -([acceptance](https://forums.swift.org/t/pitch-make-serialized-trait-public-api/73147/5)) - -## Introduction - -We propose promoting the existing `.serialized` trait to public API. This trait -enables developers to designate tests or test suites to run serially, ensuring -sequential execution where necessary. - -## Motivation - -The Swift Testing library defaults to parallel execution of tests, promoting -efficiency and isolation. However, certain test scenarios demand strict -sequential execution due to shared state or complex dependencies between tests. -The `.serialized` trait provides a solution by allowing developers to enforce -serial execution for specific tests or suites. - -While global actors ensure that only one task associated with that actor runs -at any given time, thus preventing concurrent access to actor state, tasks can -yield and allow other tasks to proceed, potentially interleaving execution. -That means global actors do not ensure that a specific test runs entirely to -completion before another begins. A testing library requires a construct that -guarantees that each annotated test runs independently and completely (in its -suite), one after another, without interleaving. - -## Proposed Solution - -We propose exposing the `.serialized` trait as a public API. This attribute can -be applied to individual test functions or entire test suites, modifying the -test execution behavior to enforce sequential execution where specified. - -Annotating just a single test in a suite does not enforce any serialization -behavior - the testing library encourages parallelization and the bar to -degrade overall performance of test execution should be high. -Additionally, traits apply inwards - it would be unexpected to impact the exact -conditions of a another test in a suite without applying a trait to the suite -itself. -Thus, this trait should only be applied to suites (to enforce serial execution -of all tests inside it) or parameterized tests. If applied to just a test this -trait does not have any effect. - -## Detailed Design - -The `.serialized` trait functions as an attribute that alters the execution -scheduling of tests. When applied, it ensures that tests or suites annotated -with `.serialized` run serially. - -```swift -/// A type that affects whether or not a test or suite is parallelized. -/// -/// When added to a parameterized test function, this trait causes that test to -/// run its cases serially instead of in parallel. When applied to a -/// non-parameterized test function, this trait has no effect. When applied to a -/// test suite, this trait causes that suite to run its contained test functions -/// and sub-suites serially instead of in parallel. -/// -/// This trait is recursively applied: if it is applied to a suite, any -/// parameterized tests or test suites contained in that suite are also -/// serialized (as are any tests contained in those suites, and so on.) -/// -/// This trait does not affect the execution of a test relative to its peers or -/// to unrelated tests. This trait has no effect if test parallelization is -/// globally disabled (by, for example, passing `--no-parallel` to the -/// `swift test` command.) -/// -/// To add this trait to a test, use ``Trait/serialized``. -public struct ParallelizationTrait: TestTrait, SuiteTrait {} - -extension Trait where Self == ParallelizationTrait { - /// A trait that serializes the test to which it is applied. - /// - /// ## See Also - /// - /// - ``ParallelizationTrait`` - public static var serialized: Self { get } -} -``` - -The call site looks like this: - -```swift -@Test(.serialized, arguments: Food.allCases) func prepare(food: Food) { - // This function will be invoked serially, once per food, because it has the - // .serialized trait. -} - -@Suite(.serialized) struct FoodTruckTests { - @Test(arguments: Condiment.allCases) func refill(condiment: Condiment) { - // This function will be invoked serially, once per condiment, because the - // containing suite has the .serialized trait. - } - - @Test func startEngine() async throws { - // This function will not run while refill(condiment:) is running. One test - // must end before the other will start. - } -} - -@Suite struct FoodTruckTests { - @Test(.serialized) func startEngine() async throws { - // This function will not run serially - it's not a parameterized test and - // the suite is not annotated with the `.serialized` trait. - } - - @Test func prepareFood() async throws { - // It doesn't matter if this test is `.serialized` or not, traits applied - // to other tests won't affect this test don't impact other tests. - } -} -``` - -## Source Compatibility - -Introducing `.serialized` as a public API does not have any impact on existing -code. Tests will continue to run in parallel by default unless explicitly -marked with `.serialized`. - -## Integration with Supporting Tools - -N/A. - -## Future Directions - -There might be asks for more advanced and complex ways to affect parallelization -which include ways to specify dependencies between tests ie. "Require `foo()` to -run before `bar()`". - -## Alternatives Considered - -Alternative approaches, such as relying solely on global actors for test -isolation, were considered. However, global actors do not provide the -deterministic, sequential execution required for certain testing scenarios. The -`.serialized` trait offers a more explicit and flexible mechanism, ensuring -that each designated test or suite runs to completion without interruption. - -Various more complex parallelization and serialization options were discussed -and considered but ultimately disregarded in favor of this simple yet powerful -implementation. - -## Acknowledgments - -Thanks to the swift-testing team and managers for their contributions! Thanks -to our community for the initial feedback around this feature. +> [!NOTE] +> This proposal was accepted before Swift Testing began using the Swift +> evolution review process. Its original identifier was SWT-0003 but its prefix +> has been changed to "ST" and it has been relocated to the +> [swift-evolution](https://github.com/swiftlang/swift-evolution) repository. + +To view this proposal, see +[ST-0003: Make .serialized trait API](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0003-make-serialized-trait-api.md). diff --git a/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md b/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md index 8818dba71..8bee19894 100644 --- a/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md +++ b/Documentation/Proposals/0004-constrain-the-granularity-of-test-time-limit-durations.md @@ -1,203 +1,10 @@ # Constrain the granularity of test time limit durations -* Proposal: -[SWT-0004](0004-constrain-the-granularity-of-test-time-limit-durations.md) -* Authors: [Dennis Weissmann](https://github.com/dennisweissmann) -* Status: **Implemented (Swift 6.0)** -* Implementation: -[swiftlang/swift-testing#534](https://github.com/swiftlang/swift-testing/pull/534) -* Review: -([pitch](https://forums.swift.org/t/pitch-constrain-the-granularity-of-test-time-limit-durations/73146)), -([acceptance](https://forums.swift.org/t/pitch-constrain-the-granularity-of-test-time-limit-durations/73146/3)) - -## Introduction - -Sometimes tests might get into a state (either due the test code itself or due -to the code they're testing) where they don't make forward progress and hang. -Swift Testing provides a way to handle these issues using the TimeLimit trait: - -```swift -@Test(.timeLimit(.minutes(60)) -func testFunction() { ... } -``` - -Currently there exist multiple overloads for the `.timeLimit` trait: one that -takes a `Swift.Duration` which allows for arbitrary `Duration` values to be -passed, and one that takes a `TimeLimitTrait.Duration` which constrains the -minimum time limit as well as the increment to 1 minute. - -## Motivation - -Small time limit values in particular cause more harm than good due to tests -running in environments with drastically differing performance characteristics. -Particularly when running in CI systems or on virtualized hardware tests can -run much slower than at desk. -Swift Testing should help developers use a reasonable time limit value in its -API without developers having to refer to the documentation. - -It is crucial to emphasize that unit tests failing due to exceeding their -timeout should be exceptionally rare. At the same time, a spurious unit test -failure caused by a short timeout can be surprisingly costly, potentially -leading to an entire CI pipeline being rerun. Determining an appropriate -timeout for a specific test can be a challenging task. - -Additionally, when the system intentionally runs multiple tests simultaneously -to optimize resource utilization, the scheduler becomes the arbiter of test -execution. Consequently, the test may take significantly longer than -anticipated, potentially due to external factors beyond the control of the code -under test. - -A unit test should be capable of failing due to hanging, but it should not fail -due to being slow, unless the developer has explicitly indicated that it -should, effectively transforming it into a performance test. - -The time limit feature is *not* intended to be used to apply small timeouts to -tests to ensure test runtime doesn't regress by small amounts. This feature is -intended to be used to guard against hangs and pathologically long running -tests. - -## Proposed Solution - -We propose changing the `.timeLimit` API to accept values of a new `Duration` -type defined in `TimeLimitTrait` which only allows for `.minute` values to be -passed. -This type already exists as SPI and this proposal is seeking to making it API. - -## Detailed Design - -The `TimeLimitTrait.Duration` struct only has one factory method: -```swift -public static func minutes(_ minutes: some BinaryInteger) -> Self -``` - -That ensures 2 things: -1. It's impossible to create short time limits (under a minute). -2. It's impossible to create high-precision increments of time. - -Both of these features are important to ensure the API is self documenting and -conveying the intended purpose. - -For parameterized tests these time limits apply to each individual test case. - -The `TimeLimitTrait.Duration` struct is declared as follows: - -```swift -/// A type that defines a time limit to apply to a test. -/// -/// To add this trait to a test, use one of the following functions: -/// -/// - ``Trait/timeLimit(_:)`` -@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) -public struct TimeLimitTrait: TestTrait, SuiteTrait { - /// A type representing the duration of a time limit applied to a test. - /// - /// This type is intended for use specifically for specifying test timeouts - /// with ``TimeLimitTrait``. It is used instead of Swift's built-in `Duration` - /// type because test timeouts do not support high-precision, arbitrarily - /// short durations. The smallest allowed unit of time is minutes. - public struct Duration: Sendable { - - /// Construct a time limit duration given a number of minutes. - /// - /// - Parameters: - /// - minutes: The number of minutes the resulting duration should - /// represent. - /// - /// - Returns: A duration representing the specified number of minutes. - public static func minutes(_ minutes: some BinaryInteger) -> Self - } - - /// The maximum amount of time a test may run for before timing out. - public var timeLimit: Swift.Duration { get set } -} -``` - -The extension on `Trait` that allows for `.timeLimit(...)` to work is defined -like this: - -```swift -/// Construct a time limit trait that causes a test to time out if it runs for -/// too long. -/// -/// - Parameters: -/// - timeLimit: The maximum amount of time the test may run for. -/// -/// - Returns: An instance of ``TimeLimitTrait``. -/// -/// Test timeouts do not support high-precision, arbitrarily short durations -/// due to variability in testing environments. The time limit must be at -/// least one minute, and can only be expressed in increments of one minute. -/// -/// When this trait is associated with a test, that test must complete within -/// a time limit of, at most, `timeLimit`. If the test runs longer, an issue -/// of kind ``Issue/Kind/timeLimitExceeded(timeLimitComponents:)`` is -/// recorded. This timeout is treated as a test failure. -/// -/// The time limit amount specified by `timeLimit` may be reduced if the -/// testing library is configured to enforce a maximum per-test limit. When -/// such a maximum is set, the effective time limit of the test this trait is -/// applied to will be the lesser of `timeLimit` and that maximum. This is a -/// policy which may be configured on a global basis by the tool responsible -/// for launching the test process. Refer to that tool's documentation for -/// more details. -/// -/// If a test is parameterized, this time limit is applied to each of its -/// test cases individually. If a test has more than one time limit associated -/// with it, the shortest one is used. A test run may also be configured with -/// a maximum time limit per test case. -public static func timeLimit(_ timeLimit: Self.Duration) -> Self -``` - -And finally, the call site of the API looks like this: - -```swift -@Test(.timeLimit(.minutes(60)) -func serve100CustomersInOneHour() async { - for _ in 0 ..< 100 { - let customer = await Customer.next() - await customer.order() - ... - } -} -``` - -The `TimeLimitTrait.Duration` struct has various `unavailable` overloads that -are included for diagnostic purposes only. They are all documented and -annotated like this: - -```swift -/// Construct a time limit duration given a number of . -/// -/// This function is unavailable and is provided for diagnostic purposes only. -@available(*, unavailable, message: "Time limit must be specified in minutes") -``` - -## Source Compatibility - -This impacts clients that have adopted the `.timeLimit` trait and use overloads -of the trait that accept an arbitrary `Swift.Duration` except if they used the -`minutes` overload. - -## Integration with Supporting Tools - -N/A - -## Future Directions - -We could allow more finegrained time limits in the future that scale with the -performance of the test host device. -Or take a more manual approach where we detect the type of environment -(like CI vs local) and provide a way to use different timeouts depending on the -environment. - -## Alternatives Considered - -We have considered using `Swift.Duration` as the currency type for this API but -decided against it to avoid common pitfalls and misuses of this feature such as -providing very small time limits that lead to flaky tests in different -environments. - -## Acknowledgments - -The authors acknowledge valuable contributions and feedback from the Swift -Testing community during the development of this proposal. +> [!NOTE] +> This proposal was accepted before Swift Testing began using the Swift +> evolution review process. Its original identifier was SWT-0004 but its prefix +> has been changed to "ST" and it has been relocated to the +> [swift-evolution](https://github.com/swiftlang/swift-evolution) repository. + +To view this proposal, see +[ST-0004: Constrain the granularity of test time limit durations](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0004-constrain-the-granularity-of-test-time-limit-durations.md). diff --git a/Documentation/Proposals/0005-ranged-confirmations.md b/Documentation/Proposals/0005-ranged-confirmations.md index df1db331b..8777d6009 100644 --- a/Documentation/Proposals/0005-ranged-confirmations.md +++ b/Documentation/Proposals/0005-ranged-confirmations.md @@ -1,186 +1,10 @@ # Range-based confirmations -* Proposal: [SWT-0005](0005-ranged-confirmations.md) -* Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Implemented (Swift 6.1)** -* Bug: rdar://138499457 -* Implementation: [swiftlang/swift-testing#598](https://github.com/swiftlang/swift-testing/pull/598), [swiftlang/swift-testing#689](https://github.com/swiftlang/swift-testing/pull689) -* Review: ([pitch](https://forums.swift.org/t/pitch-range-based-confirmations/74589)), - ([acceptance](https://forums.swift.org/t/pitch-range-based-confirmations/74589/7)) - -## Introduction - -Swift Testing includes [an interface](https://swiftpackageindex.com/swiftlang/swift-testing/main/documentation/testing/confirmation(_:expectedcount:isolation:sourcelocation:_:)) -for checking that some asynchronous event occurs a given number of times -(typically exactly once or never at all.) This proposal enhances that interface -to allow arbitrary ranges of event counts so that a test can be written against -code that may not always fire said event the exact same number of times. - -## Motivation - -Some tests rely on fixtures or external state that is not perfectly -deterministic. For example, consider a test that checks that clicking the mouse -button will generate a `.mouseClicked` event. Such a test might use the -`confirmation()` interface: - -```swift -await confirmation(expectedCount: 1) { mouseClicked in - var eventLoop = EventLoop() - eventLoop.eventHandler = { event in - if event == .mouseClicked { - mouseClicked() - } - } - await eventLoop.simulate(.mouseClicked) -} -``` - -But what happens if the user _actually_ clicks a mouse button while this test is -running? That might trigger a _second_ `.mouseClicked` event, and then the test -will fail spuriously. - -## Proposed solution - -If the test author could instead indicate to Swift Testing that their test will -generate _one or more_ events, they could avoid spurious failures: - -```swift -await confirmation(expectedCount: 1...) { mouseClicked in - ... -} -``` - -With this proposal, we add an overload of `confirmation()` that takes any range -expression instead of a single integer value (which is still accepted via the -existing overload.) - -## Detailed design - -A new overload of `confirmation()` is added: - -```swift -/// Confirm that some event occurs during the invocation of a function. -/// -/// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - expectedCount: A range of integers indicating the number of times the -/// expected event should occur when `body` is invoked. -/// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to which any recorded issues should -/// be attributed. -/// - body: The function to invoke. -/// -/// - Returns: Whatever is returned by `body`. -/// -/// - Throws: Whatever is thrown by `body`. -/// -/// Use confirmations to check that an event occurs while a test is running in -/// complex scenarios where `#expect()` and `#require()` are insufficient. For -/// example, a confirmation may be useful when an expected event occurs: -/// -/// - In a context that cannot be awaited by the calling function such as an -/// event handler or delegate callback; -/// - More than once, or never; or -/// - As a callback that is invoked as part of a larger operation. -/// -/// To use a confirmation, pass a closure containing the work to be performed. -/// The testing library will then pass an instance of ``Confirmation`` to the -/// closure. Every time the event in question occurs, the closure should call -/// the confirmation: -/// -/// ```swift -/// let minBuns = 5 -/// let maxBuns = 10 -/// await confirmation( -/// "Baked between \(minBuns) and \(maxBuns) buns", -/// expectedCount: minBuns ... maxBuns -/// ) { bunBaked in -/// foodTruck.eventHandler = { event in -/// if event == .baked(.cinnamonBun) { -/// bunBaked() -/// } -/// } -/// await foodTruck.bakeTray(of: .cinnamonBun) -/// } -/// ``` -/// -/// When the closure returns, the testing library checks if the confirmation's -/// preconditions have been met, and records an issue if they have not. -/// -/// If an exact count is expected, use -/// ``confirmation(_:expectedCount:isolation:sourceLocation:_:)`` instead. -public func confirmation( - _ comment: Comment? = nil, - expectedCount: some RangeExpression & Sequence Sendable, - isolation: isolated (any Actor)? = #isolation, - sourceLocation: SourceLocation = #_sourceLocation, - _ body: (Confirmation) async throws -> sending R -) async rethrows -> R -``` - -### Ranges without lower bounds - -Certain types of range, specifically [`PartialRangeUpTo`](https://developer.apple.com/documentation/swift/partialrangeupto) -and [`PartialRangeThrough`](https://developer.apple.com/documentation/swift/partialrangethrough), -may have surprising behavior when used with this new interface because they -implicitly include `0`. If a test author writes `...10`, do they mean "zero to -ten" or "one to ten"? The programmatic meaning is the former, but some test -authors might mean the latter. If an event does not occur, a test using -`confirmation()` and this `expectedCount` value would pass when the test author -meant for it to fail. - -The unbounded range (`...`) type `UnboundedRange` is effectively useless when -used with this interface and any use of it here is almost certainly a programmer -error. - -`PartialRangeUpTo` and `PartialRangeThrough` conform to `RangeExpression`, but -not to `Sequence`, so they will be rejected at compile time. `UnboundedRange` is -a non-nominal type and will not match either. We will provide unavailable -overloads of `confirmation()` for these types with messages that explain why -they are unavailable, e.g.: - -```swift -@available(*, unavailable, message: "Unbounded range '...' has no effect when used with a confirmation.") -public func confirmation( - _ comment: Comment? = nil, - expectedCount: UnboundedRange, - isolation: isolated (any Actor)? = #isolation, - sourceLocation: SourceLocation = #_sourceLocation, - _ body: (Confirmation) async throws -> R -) async rethrows -> R -``` - -## Source compatibility - -This change is additive. Existing tests are unaffected. - -Code that refers to `confirmation(_:expectedCount:isolation:sourceLocation:_:)` -by symbol name may need to add a contextual type to disambiguate the two -overloads at compile time. - -## Integration with supporting tools - -The type of the associated value `expected` for the `Issue.Kind` case -`confirmationMiscounted(actual:expected:)` will change from `Int` to -`any RangeExpression & Sendable`[^1]. Tools that implement event handlers and -distinguish between `Issue.Kind` cases are advised not to assume the type of -this value is `Int`. - -## Alternatives considered - -- Doing nothing. We have identified real-world use cases for this interface - including in Swift Testing’s own test target. -- Allowing the use of any value as the `expectedCount` argument so long as it - conforms to a protocol `ExpectedCount` (we'd have range types and `Int` - conform by default.) It was unclear what this sort of flexibility would let - us do, and posed challenges for encoding and decoding events and issues when - using the JSON event stream interface. - -## Acknowledgments - -Thanks to the testing team for their help preparing this pitch! - -[^1]: In the future, this type will change to - `any RangeExpression & Sendable`. Compiler support is required - ([96960993](rdar://96960993)). +> [!NOTE] +> This proposal was accepted before Swift Testing began using the Swift +> evolution review process. Its original identifier was SWT-0005 but its prefix +> has been changed to "ST" and it has been relocated to the +> [swift-evolution](https://github.com/swiftlang/swift-evolution) repository. + +To view this proposal, see +[ST-0005: Range-based confirmations](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0005-ranged-confirmations.md). diff --git a/Documentation/Proposals/0006-return-errors-from-expect-throws.md b/Documentation/Proposals/0006-return-errors-from-expect-throws.md index 502a18e17..7fe1f6b5c 100644 --- a/Documentation/Proposals/0006-return-errors-from-expect-throws.md +++ b/Documentation/Proposals/0006-return-errors-from-expect-throws.md @@ -1,267 +1,10 @@ # Return errors from `#expect(throws:)` -* Proposal: [SWT-0006](0006-return-errors-from-expect-throws.md) -* Authors: [Jonathan Grynspan](https://github.com/grynspan) -* Status: **Implemented (Swift 6.1)** -* Bug: rdar://138235250 -* Implementation: [swiftlang/swift-testing#780](https://github.com/swiftlang/swift-testing/pull/780) -* Review: ([pitch](https://forums.swift.org/t/pitch-returning-errors-from-expect-throws/75567)), ([acceptance](https://forums.swift.org/t/pitch-returning-errors-from-expect-throws/75567/5)) - -## Introduction - -Swift Testing includes overloads of `#expect()` and `#require()` that can be -used to assert that some code throws an error. They are useful when validating -that your code's failure cases are correctly detected and handled. However, for -more complex validation cases, they aren't particularly ergonomic. This proposal -seeks to resolve that issue by having these overloads return thrown errors for -further inspection. - -## Motivation - -We offer three variants of `#expect(throws:)`: - -- One that takes an error type, and matches any error of the same type; -- One that takes an error _instance_ (conforming to `Equatable`) and matches any - error that compares equal to it; and -- One that takes a trailing closure and allows test authors to write arbitrary - validation logic. - -The third overload has proven to be somewhat problematic. First, it yields the -error to its closure as an instance of `any Error`, which typically forces the -developer to cast it before doing any useful comparisons. Second, the test -author must return `true` to indicate the error matched and `false` to indicate -it didn't, which can be both logically confusing and difficult to express -concisely: - -```swift -try #require { - let potato = try Sack.randomPotato() - try potato.turnIntoFrenchFries() -} throws: { error in - guard let error = error as PotatoError else { - return false - } - guard case .potatoNotPeeled = error else { - return false - } - return error.variety != .russet -} -``` - -The first impulse many test authors have here is to use `#expect()` in the -second closure, but it doesn't return the necessary boolean value _and_ it can -result in multiple issues being recorded in a test when there's really only one. - -## Proposed solution - -I propose deprecating [`#expect(_:sourceLocation:performing:throws:)`](https://developer.apple.com/documentation/testing/expect(_:sourcelocation:performing:throws:)) -and [`#require(_:sourceLocation:performing:throws:)`](https://developer.apple.com/documentation/testing/require(_:sourcelocation:performing:throws:)) -and modifying the other overloads so that, on success, they return the errors -that were thrown. - -## Detailed design - -All overloads of `#expect(throws:)` and `#require(throws:)` will be updated to -return an instance of the error type specified by their arguments, with the -problematic overloads returning `any Error` since more precise type information -is not statically available. The problematic overloads will also be deprecated: - -```diff ---- a/Sources/Testing/Expectations/Expectation+Macro.swift -+++ b/Sources/Testing/Expectations/Expectation+Macro.swift -+@discardableResult - @freestanding(expression) public macro expect( - throws errorType: E.Type, - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing expression: () async throws -> R --) -+) -> E? where E: Error - -+@discardableResult - @freestanding(expression) public macro require( - throws errorType: E.Type, - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing expression: () async throws -> R --) where E: Error -+) -> E where E: Error - -+@discardableResult - @freestanding(expression) public macro expect( - throws error: E, - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing expression: () async throws -> R --) where E: Error & Equatable -+) -> E? where E: Error & Equatable - -+@discardableResult - @freestanding(expression) public macro require( - throws error: E, - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing expression: () async throws -> R --) where E: Error & Equatable -+) -> E where E: Error & Equatable - -+@available(swift, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") -+@discardableResult - @freestanding(expression) public macro expect( - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing expression: () async throws -> R, - throws errorMatcher: (any Error) async throws -> Bool --) -+) -> (any Error)? - -+@available(swift, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") -+@discardableResult - @freestanding(expression) public macro require( - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing expression: () async throws -> R, - throws errorMatcher: (any Error) async throws -> Bool --) -+) -> any Error -``` - -(More detailed information about the deprecations will be provided via DocC.) - -The `#expect(throws:)` overloads return an optional value that is `nil` if the -expectation failed, while the `#require(throws:)` overloads return non-optional -values and throw instances of `ExpectationFailedError` on failure (as before.) - > [!NOTE] -> Instances of `ExpectationFailedError` thrown by `#require(throws:)` on failure -> are not returned as that would defeat the purpose of using `#require(throws:)` -> instead of `#expect(throws:)`. - -Test authors will be able to use the result of the above functions to verify -that the thrown error is correct: - -```swift -let error = try #require(throws: PotatoError.self) { - let potato = try Sack.randomPotato() - try potato.turnIntoFrenchFries() -} -#expect(error == .potatoNotPeeled) -#expect(error.variety != .russet) -``` - -The new code is more concise than the old code and avoids boilerplate casting -from `any Error`. - -## Source compatibility - -In most cases, this change does not affect source compatibility. Swift does not -allow forming references to macros at runtime, so we don't need to worry about -type mismatches assigning one to some local variable. - -We have identified two scenarios where a new warning will be emitted. - -### Inferred return type from macro invocation - -The return type of the macro may be used by the compiler to infer the return -type of an enclosing closure. If the return value is then discarded, the -compiler may emit a warning: - -```swift -func pokePotato(_ pPotato: UnsafePointer) throws { ... } - -let potato = Potato() -try await Task.sleep(for: .months(3)) -withUnsafePointer(to: potato) { pPotato in - // ^ ^ ^ ⚠️ Result of call to 'withUnsafePointer(to:_:)' is unused - #expect(throws: PotatoError.rotten) { - try pokePotato(pPotato) - } -} -``` - -This warning can be suppressed by assigning the result of the macro invocation -or the result of the function call to `_`: - -```swift -withUnsafePointer(to: potato) { pPotato in - _ = #expect(throws: PotatoError.rotten) { - try pokePotato(pPotato) - } -} -``` - -### Use of `#require(throws:)` in a generic context with `Never.self` - -If `#require(throws:)` (but not `#expect(throws:)`) is used in a generic context -where the type of thrown error is a generic parameter, and the type is resolved -to `Never`, there is no valid value for the invocation to return: - -```swift -func wrapper(throws type: E.Type, _ body: () throws -> Void) throws -> E { - return try #require(throws: type) { - try body() - } -} -let error = try #require(throws: Never.self) { ... } -``` - -We don't think this particular pattern is common (and outside of our own test -target, I'd be surprised if anybody's attempted it yet.) However, we do need to -handle it gracefully. If this pattern is encountered, Swift Testing will record -an "API Misused" issue for the current test and advise the test author to switch -to `#expect(throws:)` or to not pass `Never.self` here. - -## Integration with supporting tools - -N/A - -## Future directions - -- Adopting [typed throws](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0413-typed-throws.md) - to statically require that the error thrown from test code is of the correct - type. - - If we adopted typed throws in the signatures of these macros, it would force - adoption of typed throws in the code under test even when it may not be - appropriate. For example, if we adopted typed throws, the following code would - not compile: - - ```swift - func cook(_ food: consuming some Food) throws { ... } - - let error: PotatoError? = #expect(throws: PotatoError.self) { - var potato = Potato() - potato.fossilize() - try cook(potato) // 🛑 ERROR: Invalid conversion of thrown error type - // 'any Error' to 'PotatoError' - } - ``` - - We believe it may be possible to overload these macros or their expansions so - that the code sample above _does_ compile and behave as intended. We intend to - experiment further with this idea and potentially revisit typed throws support - in a future proposal. - -## Alternatives considered - -- Leaving the existing implementation and signatures in place. We've had - sufficient feedback about the ergonomics of this API that we want to address - the problem. - -- Having the return type of the macros be `any Error` and returning _any_ error - that was thrown even on mismatch. This would make the ergonomics of the - subsequent test code less optimal because the test author would need to cast - the error to the appropriate type before inspecting it. - - There's a philosophical argument to be made here that if a mismatched error is - thrown, then the test has already failed and is in an inconsistent state, so - we should allow the test to fail rather than return what amounts to "bad - output". - - If the test author wants to inspect any arbitrary thrown error, they can - specify `(any Error).self` instead of a concrete error type. - -## Acknowledgments +> This proposal was accepted before Swift Testing began using the Swift +> evolution review process. Its original identifier was SWT-0006 but its prefix +> has been changed to "ST" and it has been relocated to the +> [swift-evolution](https://github.com/swiftlang/swift-evolution) repository. -Thanks to the team and to [@jakepetroules](https://github.com/jakepetroules) for -starting the discussion that ultimately led to this proposal. +To view this proposal, see +[ST-0006: Return errors from `#expect(throws:)`](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0006-return-errors-from-expect-throws.md). diff --git a/Documentation/Proposals/0007-test-scoping-traits.md b/Documentation/Proposals/0007-test-scoping-traits.md index 9d01bbbc7..969f163fc 100644 --- a/Documentation/Proposals/0007-test-scoping-traits.md +++ b/Documentation/Proposals/0007-test-scoping-traits.md @@ -1,510 +1,10 @@ # Test Scoping Traits -* Proposal: [SWT-0007](0007-test-scoping-traits.md) -* Authors: [Stuart Montgomery](https://github.com/stmontgomery) -* Status: **Implemented (Swift 6.1)** -* Implementation: [swiftlang/swift-testing#733](https://github.com/swiftlang/swift-testing/pull/733), [swiftlang/swift-testing#86](https://github.com/swiftlang/swift-testing/pull/86) -* Review: ([pitch](https://forums.swift.org/t/pitch-custom-test-execution-traits/75055)), ([review](https://forums.swift.org/t/proposal-test-scoping-traits/76676)), ([acceptance](https://forums.swift.org/t/proposal-test-scoping-traits/76676/3)) - -### Revision history - -* **v1**: Initial pitch. -* **v2**: Dropped 'Custom' prefix from the proposed API names (although kept the - word in certain documentation passages where it clarified behavior). -* **v3**: Changed the `Trait` requirement from a property to a method which - accepts the test and/or test case, and modify its default implementations such - that custom behavior is either performed per-suite or per-test case by default. -* **v4**: Renamed the APIs to use "scope" as the base verb instead of "execute". - -## Introduction - -This introduces API which enables a `Trait`-conforming type to provide a custom -execution scope for test functions and suites, including running code before or -after them. - -## Motivation - -One of the primary motivations for the trait system in Swift Testing, as -[described in the vision document](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md#trait-extensibility), -is to provide a way to customize the behavior of tests which have things in -common. If all the tests in a given suite type need the same custom behavior, -`init` and/or `deinit` (if applicable) can be used today. But if only _some_ of -the tests in a suite need custom behavior, or tests across different levels of -the suite hierarchy need it, traits would be a good place to encapsulate common -logic since they can be applied granularly per-test or per-suite. This aspect of -the vision for traits hasn't been realized yet, though: the `Trait` protocol -does not offer a way for a trait to customize the execution of the tests or -suites it's applied to. - -Customizing a test's behavior typically means running code either before or -after it runs, or both. Consolidating common set-up and tear-down logic allows -each test function to be more succinct with less repetitive boilerplate so it -can focus on what makes it unique. - -## Proposed solution - -At a high level, this proposal entails adding API to the `Trait` protocol -allowing a conforming type to opt-in to providing a custom execution scope for a -test. We discuss how that capability should be exposed to trait types below. - -### Supporting scoped access - -There are different approaches one could take to expose hooks for a trait to -customize test behavior. To illustrate one of them, consider the following -example of a `@Test` function with a custom trait whose purpose is to set mock -API credentials for the duration of each test it's applied to: - -```swift -@Test(.mockAPICredentials) -func example() { - // ... -} - -struct MockAPICredentialsTrait: TestTrait { ... } - -extension Trait where Self == MockAPICredentialsTrait { - static var mockAPICredentials: Self { ... } -} -``` - -In this hypothetical example, the current API credentials are stored via a -static property on an `APICredentials` type which is part of the module being -tested: - -```swift -struct APICredentials { - var apiKey: String - - static var shared: Self? -} -``` - -One way that this custom trait could customize the API credentials during each -test is if the `Trait` protocol were to expose a pair of method requirements -which were then called before and after the test, respectively: - -```swift -public protocol Trait: Sendable { - // ... - func setUp() async throws - func tearDown() async throws -} - -extension Trait { - // ... - public func setUp() async throws { /* No-op */ } - public func tearDown() async throws { /* No-op */ } -} -``` - -The custom trait type could adopt these using code such as the following: - -```swift -extension MockAPICredentialsTrait { - func setUp() { - APICredentials.shared = .init(apiKey: "...") - } - - func tearDown() { - APICredentials.shared = nil - } -} -``` - -Many testing systems use this pattern, including XCTest. However, this approach -encourages the use of global mutable state such as the `APICredentials.shared` -variable, and this limits the testing library's ability to parallelize test -execution, which is -[another part of the Swift Testing vision](https://github.com/swiftlang/swift-evolution/blob/main/visions/swift-testing.md#parallelization-and-concurrency). - -The use of nonisolated static variables is generally discouraged now, and in -Swift 6 the above `APICredentials.shared` property produces an error. One way -to resolve that is to change it to a `@TaskLocal` variable, as this would be -concurrency-safe and still allow tests accessing this state to run in parallel: - -```swift -extension APICredentials { - @TaskLocal static var current: Self? -} -``` - -Binding task local values requires using the scoped access -[`TaskLocal.withValue()`](https://developer.apple.com/documentation/swift/tasklocal/withvalue(_:operation:isolation:file:line:)) -API though, and that would not be possible if `Trait` exposed separate methods -like `setUp()` and `tearDown()`. - -For these reasons, I believe it's important to expose this trait capability -using a single, scoped access-style API which accepts a closure. A simplified -version of that idea might look like this: - -```swift -public protocol Trait: Sendable { - // ... - - // Simplified example, not the actual proposal - func executeTest(_ body: @Sendable () async throws -> Void) async throws -} - -extension MockAPICredentialsTrait { - func executeTest(_ body: @Sendable () async throws -> Void) async throws { - let mockCredentials = APICredentials(apiKey: "...") - try await APICredentials.$current.withValue(mockCredentials) { - try await body() - } - } -} -``` - -### Avoiding unnecessarily lengthy backtraces - -A scoped access-style API has some potential downsides. To apply this approach -to a test function, the scoped call of a trait must wrap the invocation of that -test function, and every _other_ trait applied to that same test which offers -custom behavior _also_ must wrap the other traits' calls in a nesting fashion. -To visualize this, imagine a test function with multiple traits: - -```swift -@Test(.traitA, .traitB, .traitC) -func exampleTest() { - // ... -} -``` - -If all three of those traits provide a custom scope for tests, then each of them -needs to wrap the call to the next one, and the last trait needs to wrap the -invocation of the test, illustrated by the following: - -``` -TraitA.executeTest { - TraitB.executeTest { - TraitC.executeTest { - exampleTest() - } - } -} -``` - -Tests may have an arbitrary number of traits applied to them, including those -inherited from containing suite types. A naïve implementation in which _every_ -trait is given the opportunity to customize test behavior by calling its scoped -access API might cause unnecessarily lengthy backtraces that make debugging the -body of tests more difficult. Or worse: if the number of traits is great enough, -it could cause a stack overflow. - -In practice, most traits probably do _not_ need to provide a custom scope for -the tests they're applied to, so to mitigate these downsides it's important that -there be some way to distinguish traits which customize test behavior. That way, -the testing library can limit these scoped access calls to only traits which -need it. - -### Avoiding unnecessary (re-)execution - -Traits can be applied to either test functions or suites, and traits applied to -suites can optionally support inheritance by implementing the `isRecursive` -property of the `SuiteTrait` protocol. When a trait is directly applied to a -test function, if the trait customizes the behavior of tests it's applied to, it -should be given the opportunity to perform its custom behavior once for every -invocation of that test function. In particular, if the test function is -parameterized and runs multiple times, then the trait applied to it should -perform its custom behavior once for every invocation. This should not be -surprising to users, since it's consistent with the behavior of `init` and -`deinit` for an instance `@Test` method. - -It may be useful for certain kinds of traits to perform custom logic once for -_all_ the invocations of a parameterized test. Although this should be possible, -we believe it shouldn't be the default since it could lead to work being -repeated multiple times needlessly, or unintentional state sharing across tests, -unless the trait is implemented carefully to avoid those problems. - -When a trait conforms to `SuiteTrait` and is applied to a suite, the question of -when its custom scope (if any) should be applied is less obvious. Some suite -traits support inheritance and are recursively applied to all the test functions -they contain (including transitively, via sub-suites). Other suite traits don't -support inheritance, and only affect the specific suite they're applied to. -(It's also worth noting that a sub-suite _can_ have the same non-recursive suite -trait one of its ancestors has, as long as it's applied explicitly.) - -As a general rule of thumb, we believe most traits will either want to perform -custom logic once for _all_ children or once for _each_ child, not both. -Therefore, when it comes to suite traits, the default behavior should depend on -whether it supports inheritance: a recursive suite trait should by default -perform custom logic before each test, and a non-recursive one per-suite. But -the APIs should be flexible enough to support both, for advanced traits which -need it. - -## Detailed design - -I propose the following new APIs: - -- A new protocol `TestScoping` with a single required `provideScope(...)` method. - This will be called to provide scope for a test, and allows the conforming - type to perform custom logic before or after. -- A new method `scopeProvider(for:testCase:)` on the `Trait` protocol whose - result type is an `Optional` value of a type conforming to `TestScoping`. A - `nil` value returned by this method will skip calling the `provideScope(...)` - method. -- A default implementation of `Trait.scopeProvider(...)` which returns `nil`. -- A conditional implementation of `Trait.scopeProvider(...)` which returns `self` - in the common case where the trait type conforms to `TestScoping` itself. - -Since the `scopeProvider(...)` method's return type is optional and returns `nil` -by default, the testing library cannot invoke the `provideScope(...)` method -unless a trait customizes test behavior. This avoids the "unnecessarily lengthy -backtraces" problem above. - -Below are the proposed interfaces: - -```swift -/// A protocol that allows providing a custom execution scope for a test -/// function (and each of its cases) or a test suite by performing custom code -/// before or after it runs. -/// -/// Types conforming to this protocol may be used in conjunction with a -/// ``Trait``-conforming type by implementing the -/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, allowing custom traits -/// to provide custom scope for tests. Consolidating common set-up and tear-down -/// logic for tests which have similar needs allows each test function to be -/// more succinct with less repetitive boilerplate so it can focus on what makes -/// it unique. -public protocol TestScoping: Sendable { - /// Provide custom execution scope for a function call which is related to the - /// specified test and/or test case. - /// - /// - Parameters: - /// - test: The test under which `function` is being performed. - /// - testCase: The test case, if any, under which `function` is being - /// performed. When invoked on a suite, the value of this argument is - /// `nil`. - /// - function: The function to perform. If `test` represents a test suite, - /// this function encapsulates running all the tests in that suite. If - /// `test` represents a test function, this function is the body of that - /// test function (including all cases if it is parameterized.) - /// - /// - Throws: Whatever is thrown by `function`, or an error preventing this - /// type from providing a custom scope correctly. An error thrown from this - /// method is recorded as an issue associated with `test`. If an error is - /// thrown before `function` is called, the corresponding test will not run. - /// - /// When the testing library is preparing to run a test, it starts by finding - /// all traits applied to that test, including those inherited from containing - /// suites. It begins with inherited suite traits, sorting them - /// outermost-to-innermost, and if the test is a function, it then adds all - /// traits applied directly to that functions in the order they were applied - /// (left-to-right). It then asks each trait for its scope provider (if any) - /// by calling ``Trait/scopeProvider(for:testCase:)-cjmg``. Finally, it calls - /// this method on all non-`nil` scope providers, giving each an opportunity - /// to perform arbitrary work before or after invoking `function`. - /// - /// This method should either invoke `function` once before returning or throw - /// an error if it is unable to provide a custom scope. - /// - /// Issues recorded by this method are associated with `test`. - func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws -} - -public protocol Trait: Sendable { - // ... - - /// The type of the test scope provider for this trait. - /// - /// The default type is `Never`, which cannot be instantiated. The - /// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with this - /// default type must return `nil`, meaning that trait will not provide a - /// custom scope for the tests it's applied to. - associatedtype TestScopeProvider: TestScoping = Never - - /// Get this trait's scope provider for the specified test and/or test case, - /// if any. - /// - /// - Parameters: - /// - test: The test for which a scope provider is being requested. - /// - testCase: The test case for which a scope provider is being requested, - /// if any. When `test` represents a suite, the value of this argument is - /// `nil`. - /// - /// - Returns: A value conforming to ``Trait/TestScopeProvider`` which may be - /// used to provide custom scoping for `test` and/or `testCase`, or `nil` if - /// they should not have any custom scope. - /// - /// If this trait's type conforms to ``TestScoping``, the default value - /// returned by this method depends on `test` and/or `testCase`: - /// - /// - If `test` represents a suite, this trait must conform to ``SuiteTrait``. - /// If the value of this suite trait's ``SuiteTrait/isRecursive`` property - /// is `true`, then this method returns `nil`; otherwise, it returns `self`. - /// This means that by default, a suite trait will _either_ provide its - /// custom scope once for the entire suite, or once per-test function it - /// contains. - /// - Otherwise `test` represents a test function. If `testCase` is `nil`, - /// this method returns `nil`; otherwise, it returns `self`. This means that - /// by default, a trait which is applied to or inherited by a test function - /// will provide its custom scope once for each of that function's cases. - /// - /// A trait may explicitly implement this method to further customize the - /// default behaviors above. For example, if a trait should provide custom - /// test scope both once per-suite and once per-test function in that suite, - /// it may implement the method and return a non-`nil` scope provider under - /// those conditions. - /// - /// A trait may also implement this method and return `nil` if it determines - /// that it does not need to provide a custom scope for a particular test at - /// runtime, even if the test has the trait applied. This can improve - /// performance and make diagnostics clearer by avoiding an unnecessary call - /// to ``TestScoping/provideScope(for:testCase:performing:)``. - /// - /// If this trait's type does not conform to ``TestScoping`` and its - /// associated ``Trait/TestScopeProvider`` type is the default `Never`, then - /// this method returns `nil` by default. This means that instances of this - /// trait will not provide a custom scope for tests to which they're applied. - func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider? -} - -extension Trait where Self: TestScoping { - // Returns `nil` if `testCase` is `nil`, else `self`. - public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? -} - -extension SuiteTrait where Self: TestScoping { - // If `test` is a suite, returns `nil` if `isRecursive` is `true`, else `self`. - // Otherwise, `test` is a function and this returns `nil` if `testCase` is - // `nil`, else `self`. - public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? -} - -extension Trait where TestScopeProvider == Never { - // Returns `nil`. - public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? -} - -extension Never: TestScoping {} -``` - -Here is a complete example of the usage scenario described earlier, showcasing -the proposed APIs: - -```swift -@Test(.mockAPICredentials) -func example() { - // ...validate API usage, referencing `APICredentials.current`... -} - -struct MockAPICredentialsTrait: TestTrait, TestScoping { - func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { - let mockCredentials = APICredentials(apiKey: "...") - try await APICredentials.$current.withValue(mockCredentials) { - try await function() - } - } -} - -extension Trait where Self == MockAPICredentialsTrait { - static var mockAPICredentials: Self { - Self() - } -} -``` - -## Source compatibility - -The proposed APIs are purely additive. - -This proposal will replace the existing `CustomExecutionTrait` SPI, and after -further refactoring we anticipate it will obsolete the need for the -`SPIAwareTrait` SPI as well. - -## Integration with supporting tools - -Although some built-in traits are relevant to supporting tools (such as -SourceKit-LSP statically discovering `.tags` traits), custom test behaviors are -only relevant within the test executable process while tests are running. We -don't anticipate any particular need for this feature to integrate with -supporting tools. - -## Future directions - -### Access to suite type instances - -Some test authors have expressed interest in allowing custom traits to access -the instance of a suite type for `@Test` instance methods, so the trait could -inspect or mutate the instance. Currently, only instance-level members of a -suite type (including `init`, `deinit`, and the test function itself) can access -`self`, so this would grant traits applied to an instance test method access to -the instance as well. This is certainly interesting, but poses several technical -challenges that puts it out of scope of this proposal. - -### Convenience trait for setting task locals - -Some reviewers of this proposal pointed out that the hypothetical usage example -shown earlier involving setting a task local value while a test is executing -will likely become a common use of these APIs. To streamline that pattern, it -would be very natural to add a built-in trait type which facilitates this. I -have prototyped this idea and plan to add it once this new trait functionality -lands. - -## Alternatives considered - -### Separate set up & tear down methods on `Trait` - -This idea was discussed in [Supporting scoped access](#supporting-scoped-access) -above, and as mentioned there, the primary problem with this approach is that it -cannot be used with scoped access-style APIs, including (importantly) -`TaskLocal.withValue()`. For that reason, it prevents using that common Swift -concurrency technique and reduces the potential for test parallelization. - -### Add `provideScope(...)` directly to the `Trait` protocol - -The proposed `provideScope(...)` method could be added as a requirement of the -`Trait` protocol instead of being part of a separate `TestScoping` protocol, and -it could have a default implementation which directly invokes the passed-in -closure. But this approach would suffer from the lengthy backtrace problem -described above. - -### Extend the `Trait` protocol - -The original, experimental implementation of this feature included a protocol -named`CustomExecutionTrait` which extended `Trait` and had roughly the same -method requirement as the `TestScoping` protocol proposed above. This design -worked, provided scoped access, and avoided the lengthy backtrace problem. - -After evaluating the design and usage of this SPI though, it seemed unfortunate -to structure it as a sub-protocol of `Trait` because it means that the full -capabilities of the trait system are spread across multiple protocols. In the -proposed design, the ability to return a test scoping provider is exposed via -the main `Trait` protocol, and it relies on an associated type to conditionally -opt-in to custom test behavior. In other words, the proposed design expresses -custom test behavior as just a _capability_ that a trait may have, rather than a -distinct sub-type of trait. - -Also, the implementation of this approach within the testing library was not -ideal as it required a conditional `trait as? CustomExecutionTrait` downcast at -runtime, in contrast to the simpler and more performant Optional property of the -proposed API. - -### API names - -We first considered "execute" as the base verb for the proposed new concept, but -felt this wasn't appropriate since these trait types are not "the executor" of -tests, they merely customize behavior and provide scope(s) for tests to run -within. Also, the term "executor" has prior art in Swift Concurrency, and -although that word is used in other contexts too, it may be helpful to avoid -potential confusion with concurrency executors. - -We also considered "run" as the base verb for the proposed new concept instead -of "execute", which would imply the names `TestRunning`, `TestRunner`, -`runner(for:testCase)`, and `run(_:for:testCase:)`. The word "run" is used in -many other contexts related to testing though, such as the `Runner` SPI type and -more casually to refer to a run which occurred of a test, in the past tense, so -overloading this term again may cause confusion. - -## Acknowledgments - -Thanks to [Dennis Weissmann](https://github.com/dennisweissmann) for originally -implementing this as SPI, and for helping promote its usefulness. - -Thanks to [Jonathan Grynspan](https://github.com/grynspan) for exploring ideas -to refine the API, and considering alternatives to avoid unnecessarily long -backtraces. - -Thanks to [Brandon Williams](https://github.com/mbrandonw) for feedback on the -Forum pitch thread which ultimately led to the refinements described in the -"Avoiding unnecessary (re-)execution" section. +> [!NOTE] +> This proposal was accepted before Swift Testing began using the Swift +> evolution review process. Its original identifier was SWT-0007 but its prefix +> has been changed to "ST" and it has been relocated to the +> [swift-evolution](https://github.com/swiftlang/swift-evolution) repository. + +To view this proposal, see +[ST-0007: Test Scoping Traits](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0007-test-scoping-traits.md). diff --git a/Documentation/README.md b/Documentation/README.md index e41bc9568..3ef579986 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -29,8 +29,11 @@ The [Vision document](https://github.com/swiftlang/swift-evolution/blob/main/vis for Swift Testing offers a comprehensive discussion of the project's design principles and goals. -The [`Proposals`](Proposals/) directory contains API proposals that have been -accepted and merged into Swift Testing. +Feature and API proposals for Swift Testing are stored in the +[swift-evolution](https://github.com/swiftlang/swift-evolution) repository in +the `proposals/testing/` subdirectory, and new proposals should use the +[testing template](https://github.com/swiftlang/swift-evolution/blob/main/proposal-templates/0000-swift-testing-template.md) +there. ## Development and contribution From f3c93297a02ebb4312af2e20962e1410be8b1dcb Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 4 Mar 2025 14:23:47 -0500 Subject: [PATCH 110/234] Disable snapshot types on Android. (#994) Snapshot types are used by Xcode 16. They are not needed on non-Apple platforms. This PR disables them on Android by setting `SWT_NO_SNAPSHOT_TYPES`. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index a34eed92f..f952e4080 100644 --- a/Package.swift +++ b/Package.swift @@ -191,7 +191,7 @@ extension Array where Element == PackageDescription.SwiftSetting { .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi])), + .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android])), .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] @@ -248,7 +248,7 @@ extension Array where Element == PackageDescription.CXXSetting { result += [ .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi])), + .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android])), .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), .define("SWT_NO_PIPES", .when(platforms: [.wasi])), ] From ade64fddfa53ad510a1bef12d13f8eeb083a755a Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 4 Mar 2025 16:07:47 -0600 Subject: [PATCH 111/234] Begin installing textual .swiftinterface instead of binary .swiftmodule in toolchain builds for non-Apple platforms (#982) This modifies the CMake rules to begin installing textual .swiftinterface files instead of binary .swiftmodule files when performing a toolchain build for non-Apple platforms. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes rdar://140587787 --- cmake/modules/SwiftModuleInstallation.cmake | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/cmake/modules/SwiftModuleInstallation.cmake b/cmake/modules/SwiftModuleInstallation.cmake index 15537250f..d1f5b096e 100644 --- a/cmake/modules/SwiftModuleInstallation.cmake +++ b/cmake/modules/SwiftModuleInstallation.cmake @@ -86,19 +86,9 @@ function(_swift_testing_install_target module) install(FILES $/${module_name}.swiftdoc DESTINATION "${module_dir}" RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftdoc) - if(APPLE) - # Only Darwin has stable ABI. - install(FILES $/${module_name}.swiftinterface - DESTINATION "${module_dir}" - RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftinterface) - else() - # Only install the binary .swiftmodule on platforms which do not have a - # stable ABI. Other platforms will use the textual .swiftinterface - # (installed above) and this limits access to this module's SPIs. - install(FILES $/${module_name}.swiftmodule - DESTINATION "${module_dir}" - RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftmodule) - endif() + install(FILES $/${module_name}.swiftinterface + DESTINATION "${module_dir}" + RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftinterface) endfunction() # Install the specified .swiftcrossimport directory for the specified declaring From 162e8e81f4d1e5c910dcd6ef3bed8c0b81061f29 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Fri, 21 Feb 2025 15:14:49 -0800 Subject: [PATCH 112/234] build: restructure the installation location Clean up some of the module property handling to be more declarative. This reduces the operations that are being done on targets in favour of defaults. Adopt the latest best practices for handling module installation. Introduce the new `SwiftTesting_INSTALL_NESTED_SUBDIR` option to allow installation into platform/architecture subdirectory allowing multi-architecture installations for platforms like Windows and Android. This is currently opt-in and requires a newer toolchain (something within the last ~2w) to detect the defaults. The values can be overridden by the user if desired. --- CMakeLists.txt | 19 +++--- cmake/modules/PlatformInfo.cmake | 48 +++++++++++++ cmake/modules/SwiftModuleInstallation.cmake | 75 ++------------------- 3 files changed, 65 insertions(+), 77 deletions(-) create mode 100644 cmake/modules/PlatformInfo.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1be9a4bed..ea6fa132a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,18 +32,21 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +set(CMAKE_INSTALL_RPATH $,"@loader_path/..","$ORIGIN">) +set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH YES) + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDLL) set(CMAKE_CXX_STANDARD 20) set(CMAKE_Swift_LANGUAGE_VERSION 6) set(CMAKE_Swift_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/swift) -if(NOT SWIFT_SYSTEM_NAME) - if(CMAKE_SYSTEM_NAME STREQUAL Darwin) - set(SWIFT_SYSTEM_NAME macosx) - else() - set(SWIFT_SYSTEM_NAME "$") - endif() -endif() - +include(PlatformInfo) include(SwiftModuleInstallation) + +option(SwiftTesting_INSTALL_NESTED_SUBDIR "Install libraries under a platform and architecture subdirectory" NO) +set(SwiftTesting_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${SwiftTesting_PLATFORM_SUBDIR}$<$,$>>:/testing>$<$:/${SwiftTesting_ARCH_SUBDIR}>") +set(SwiftTesting_INSTALL_SWIFTMODULEDIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${SwiftTesting_PLATFORM_SUBDIR}$<$,$>>:/testing>$<$:/${SwiftTesting_PLATFORM_SUBDIR}>") + +add_compile_options($<$:-no-toolchain-stdlib-rpath>) + add_subdirectory(Sources) diff --git a/cmake/modules/PlatformInfo.cmake b/cmake/modules/PlatformInfo.cmake new file mode 100644 index 000000000..94c60ef28 --- /dev/null +++ b/cmake/modules/PlatformInfo.cmake @@ -0,0 +1,48 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +set(print_target_info_invocation "${CMAKE_Swift_COMPILER}" -print-target-info) +if(CMAKE_Swift_COMPILER_TARGET) + list(APPEND print_target_info_invocation -target ${CMAKE_Swift_COMPILER_TARGET}) +endif() +execute_process(COMMAND ${print_target_info_invocation} OUTPUT_VARIABLE target_info_json) +message(CONFIGURE_LOG "Swift Target Info: ${print_target_info_invocation}\n" +"${target_info_json}") + +if(NOT SwiftTesting_MODULE_TRIPLE) + string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") + set(SwiftTesting_MODULE_TRIPLE "${module_triple}" CACHE STRING "Triple used for installed swift{doc,module,interface} files") + mark_as_advanced(SwiftTesting_MODULE_TRIPLE) + + message(CONFIGURE_LOG "Swift Module Triple: ${module_triple}") +endif() + +if(NOT SwiftTesting_PLATFORM_SUBDIR) + string(JSON platform GET "${target_info_json}" "target" "platform") + if(NOT platform) + if(NOT SWIFT_SYSTEM_NAME) + if(CMAKE_SYSTEM_NAME STREQUAL Darwin) + set(platform macosx) + else() + set(platform $) + endif() + endif() + endif() + set(SwiftTesting_PLATFORM_SUBDIR "${platform}" CACHE STRING "Platform name used for installed swift{doc,module,interface} files") + mark_as_advanced(SwiftTesting_PLATFORM_SUBDIR) + + message(CONFIGURE_LOG "Swift Platform: ${platform}") +endif() + +if(NOT SwiftTesting_ARCH_SUBDIR) + string(JSON arch GET "${target_info_json}" "target" "arch") + set(SwiftTesting_ARCH_SUBDIR "${arch}" CACHE STRING "Architecture used for setting the architecture subdirectory") + mark_as_advanced(SwiftTesting_ARCH_SUBDIR) + + message(CONFIGURE_LOG "Swift Architecture: ${arch}") +endif() diff --git a/cmake/modules/SwiftModuleInstallation.cmake b/cmake/modules/SwiftModuleInstallation.cmake index d1f5b096e..f9bade57d 100644 --- a/cmake/modules/SwiftModuleInstallation.cmake +++ b/cmake/modules/SwiftModuleInstallation.cmake @@ -6,62 +6,13 @@ # See http://swift.org/LICENSE.txt for license information # See http://swift.org/CONTRIBUTORS.txt for Swift project authors -# Returns the os name in a variable -# -# Usage: -# get_swift_host_os(result_var_name) -# -# -# Sets ${result_var_name} with the converted OS name derived from -# CMAKE_SYSTEM_NAME. -function(get_swift_host_os result_var_name) - set(${result_var_name} ${SWIFT_SYSTEM_NAME} PARENT_SCOPE) -endfunction() - -# Returns the path to the Swift Testing library installation directory -# -# Usage: -# get_swift_testing_install_lib_dir(type result_var_name) -# -# Arguments: -# type: The type of the library (STATIC_LIBRARY, SHARED_LIBRARY, or EXECUTABLE). -# Typically, the value of the TYPE target property. -# result_var_name: The name of the variable to set -function(get_swift_testing_install_lib_dir type result_var_name) - get_swift_host_os(swift_os) - if(type STREQUAL STATIC_LIBRARY) - set(swift swift_static) - else() - set(swift swift) - endif() - - if(APPLE) - set(${result_var_name} "lib/${swift}/${swift_os}/testing" PARENT_SCOPE) - else() - set(${result_var_name} "lib/${swift}/${swift_os}" PARENT_SCOPE) - endif() -endfunction() - function(_swift_testing_install_target module) - target_compile_options(${module} PRIVATE "-no-toolchain-stdlib-rpath") - - if(APPLE) - set_target_properties(${module} PROPERTIES - INSTALL_RPATH "@loader_path/.." - INSTALL_REMOVE_ENVIRONMENT_RPATH ON) - else() - set_target_properties(${module} PROPERTIES - INSTALL_RPATH "$ORIGIN" - INSTALL_REMOVE_ENVIRONMENT_RPATH ON) - endif() - - get_target_property(type ${module} TYPE) - get_swift_testing_install_lib_dir(${type} lib_destination_dir) - install(TARGETS ${module} - ARCHIVE DESTINATION "${lib_destination_dir}" - LIBRARY DESTINATION "${lib_destination_dir}" + ARCHIVE DESTINATION "${SwiftTesting_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${SwiftTesting_INSTALL_LIBDIR}" RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + + get_target_property(type ${module} TYPE) if(type STREQUAL EXECUTABLE) return() endif() @@ -71,18 +22,7 @@ function(_swift_testing_install_target module) set(module_name ${module}) endif() - if(NOT SwiftTesting_MODULE_TRIPLE) - set(module_triple_command "${CMAKE_Swift_COMPILER}" -print-target-info) - if(CMAKE_Swift_COMPILER_TARGET) - list(APPEND module_triple_command -target ${CMAKE_Swift_COMPILER_TARGET}) - endif() - execute_process(COMMAND ${module_triple_command} OUTPUT_VARIABLE target_info_json) - string(JSON module_triple GET "${target_info_json}" "target" "moduleTriple") - set(SwiftTesting_MODULE_TRIPLE "${module_triple}" CACHE STRING "swift module triple used for installed swiftmodule and swiftinterface files") - mark_as_advanced(SwiftTesting_MODULE_TRIPLE) - endif() - - set(module_dir "${lib_destination_dir}/${module_name}.swiftmodule") + set(module_dir ${SwiftTesting_INSTALL_SWIFTMODULEDIR}/${module_name}.swiftmodule) install(FILES $/${module_name}.swiftdoc DESTINATION "${module_dir}" RENAME ${SwiftTesting_MODULE_TRIPLE}.swiftdoc) @@ -104,9 +44,6 @@ endfunction() # swiftcrossimport_dir: The path to the source .swiftcrossimport directory # which will be installed. function(_swift_testing_install_swiftcrossimport module swiftcrossimport_dir) - get_target_property(type ${module} TYPE) - get_swift_testing_install_lib_dir(${type} lib_destination_dir) - install(DIRECTORY "${swiftcrossimport_dir}" - DESTINATION "${lib_destination_dir}") + DESTINATION "${SwiftTesting_INSTALL_SWIFTMODULEDIR}") endfunction() From 4dd5548c6a4a83ca3738e0e2ab2150afc94f5d2b Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 5 Mar 2025 08:42:21 -0500 Subject: [PATCH 113/234] Add a new _TestDiscovery library/target. (#981) This PR factors out our test discovery logic into a separate module that can be imported and linked to without needing to build all of Swift Syntax and Swift Testing. This allows test library developers to start experimenting with using the new (still experimental) test content section without needing to link to a package copy of Swift Testing. Resolves rdar://145694068. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 79 +++++- Package.swift | 34 ++- Sources/CMakeLists.txt | 1 + Sources/Testing/CMakeLists.txt | 7 +- Sources/Testing/Discovery+Macro.swift | 48 ++++ Sources/Testing/Discovery.swift | 171 ------------- Sources/Testing/ExitTests/ExitTest.swift | 3 +- .../Support/Additions/WinSDKAdditions.swift | 79 ------ Sources/Testing/Support/GetSymbol.swift | 3 + Sources/Testing/Test+Discovery+Legacy.swift | 17 -- Sources/Testing/Test+Discovery.swift | 3 +- .../Additions/WinSDKAdditions.swift | 90 +++++++ Sources/_TestDiscovery/CMakeLists.txt | 32 +++ .../DiscoverableAsTestContent.swift | 42 ++++ .../SectionBounds.swift} | 43 +++- .../_TestDiscovery/TestContentRecord.swift | 235 ++++++++++++++++++ Tests/TestingTests/MiscellaneousTests.swift | 25 +- 17 files changed, 610 insertions(+), 302 deletions(-) create mode 100644 Sources/Testing/Discovery+Macro.swift delete mode 100644 Sources/Testing/Discovery.swift create mode 100644 Sources/_TestDiscovery/Additions/WinSDKAdditions.swift create mode 100644 Sources/_TestDiscovery/CMakeLists.txt create mode 100644 Sources/_TestDiscovery/DiscoverableAsTestContent.swift rename Sources/{Testing/Discovery+Platform.swift => _TestDiscovery/SectionBounds.swift} (92%) create mode 100644 Sources/_TestDiscovery/TestContentRecord.swift diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index 89412b29d..3e3efc512 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -200,7 +200,9 @@ Third-party test content should set the `kind` field to a unique value only used by that tool, or used by that tool in collaboration with other compatible tools. At runtime, Swift Testing ignores test content records with unrecognized `kind` values. To reserve a new unique `kind` value, open a [GitHub issue](https://github.com/swiftlang/swift-testing/issues/new/choose) -against Swift Testing. +against Swift Testing. The value you reserve does not need to be representable +as a [FourCC](https://en.wikipedia.org/wiki/FourCC) value, but it can be helpful +for debugging purposes. The layout of third-party test content records must be compatible with that of `TestContentRecord` as specified above. Third-party tools are ultimately @@ -213,3 +215,78 @@ TODO: elaborate further, give examples TODO: standardize a mechanism for third parties to produce `Test` instances since we don't have a public initializer for the `Test` type. --> + +## Discovering previously-emitted test content + + + +To add test content discovery support to your package, add a dependency on the +`_TestDiscovery` module in the `swift-testing` package (not the copy of Swift +Testing included with the Swift toolchain or Xcode), then import the module with +SPI enabled: + +```swift +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) import _TestDiscovery +``` + +> [!IMPORTANT] +> Don't add a dependency on the `swift-testing` package's `Testing` module. If +> you add a dependency on this module, it will cause you to build and link Swift +> Testing every time you build your package. You only need the `_TestDiscovery` +> module in order to discover your own test content types. + +After importing `_TestDiscovery`, find the type in your module that should be +discoverable at runtime and add conformance to the `DiscoverableAsTestContent` +protocol: + +```swift +extension FoodTruckDiagnostic: DiscoverableAsTestContent { + static var testContentKind: UInt32 { /* Your `kind` value here. */ } +} +``` + +This type does not need to be publicly visible. However, if the values produced +by your accessor functions are members of a public type, you may be able to +simplify your code by using the same type. + +If you have defined a custom `context` type other than `UInt`, you can specify +it here by setting the associated `TestContentContext` type. If you have defined +a custom `hint` type for your accessor functions, you can set +`TestContentAccessorHint`: + +```swift +extension FoodTruckDiagnostic: DiscoverableAsTestContent { + static var testContentKind: UInt32 { /* Your `kind` value here. */ } + + typealias TestContentContext = UnsafePointer + typealias TestContentAccessorHint = String +} +``` + +If you customize `TestContentContext`, be aware that the type you specify must +have the same stride and alignment as `UInt`. + +When you are done configuring your type's protocol conformance, you can then +enumerate all test content records matching it as instances of +`TestContentRecord`. + +You can use the `context` property to access the `context` field of the record +(as emitted into the test content section). The testing library will +automatically cast the value of the field to an instance of `TestContentContext` +for you. + +If you find a record you wish to resolve to an instance of your conforming type, +call its `load()` function. `load()` calls the record's accessor function and, +if you have set a hint type, lets you pass an optional instance of that type: + +```swift +for diagnosticRecord in FoodTruckDiagnostic.allTestContentRecords() { + if diagnosticRecord.context.pointee == .briansBranMuffins { + if let diagnostic = diagnosticRecord.load(withHint: "...") { + diagnostic.run() + } + } +} +``` diff --git a/Package.swift b/Package.swift index f952e4080..11adfc14e 100644 --- a/Package.swift +++ b/Package.swift @@ -32,22 +32,36 @@ let package = Package( .visionOS(.v1), ], - products: [ - { + products: { + var result = [Product]() + #if os(Windows) + result.append( .library( name: "Testing", type: .dynamic, // needed so Windows exports ABI entry point symbols targets: ["Testing"] ) + ) #else + result.append( .library( name: "Testing", targets: ["Testing"] ) + ) #endif - }() - ], + + result.append( + .library( + name: "_TestDiscovery", + type: .static, + targets: ["_TestDiscovery"] + ) + ) + + return result + }(), dependencies: [ .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "601.0.0-latest"), @@ -57,6 +71,7 @@ let package = Package( .target( name: "Testing", dependencies: [ + "_TestDiscovery", "_TestingInternals", "TestingMacros", ], @@ -108,13 +123,20 @@ let package = Package( }() ), - // "Support" targets: These contain C family code and are used exclusively - // by other targets above, not directly included in product libraries. + // "Support" targets: These targets are not meant to be used directly by + // test authors. .target( name: "_TestingInternals", exclude: ["CMakeLists.txt"], cxxSettings: .packageSettings ), + .target( + name: "_TestDiscovery", + dependencies: ["_TestingInternals",], + exclude: ["CMakeLists.txt"], + cxxSettings: .packageSettings, + swiftSettings: .packageSettings + ), // Cross-import overlays (not supported by Swift Package Manager) .target( diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 1f3cf3680..1211ffcc1 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -103,6 +103,7 @@ endif() include(AvailabilityDefinitions) include(CompilerSettings) +add_subdirectory(_TestDiscovery) add_subdirectory(_TestingInternals) add_subdirectory(Overlays) add_subdirectory(Testing) diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 2f1e94c1a..7e07636d5 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -83,8 +83,7 @@ add_library(Testing Support/Locked.swift Support/Locked+Platform.swift Support/Versions.swift - Discovery.swift - Discovery+Platform.swift + Discovery+Macro.swift Test.ID.Selection.swift Test.ID.swift Test.swift @@ -107,6 +106,7 @@ add_library(Testing Traits/TimeLimitTrait.swift Traits/Trait.swift) target_link_libraries(Testing PRIVATE + _TestDiscovery _TestingInternals) if(NOT APPLE) if(NOT CMAKE_SYSTEM_NAME STREQUAL WASI) @@ -121,8 +121,9 @@ if(NOT APPLE) endif() if(NOT BUILD_SHARED_LIBS) # When building a static library, tell clients to autolink the internal - # library. + # libraries. target_compile_options(Testing PRIVATE + "SHELL:-Xfrontend -public-autolink-library -Xfrontend _TestDiscovery" "SHELL:-Xfrontend -public-autolink-library -Xfrontend _TestingInternals") endif() add_dependencies(Testing diff --git a/Sources/Testing/Discovery+Macro.swift b/Sources/Testing/Discovery+Macro.swift new file mode 100644 index 000000000..391278983 --- /dev/null +++ b/Sources/Testing/Discovery+Macro.swift @@ -0,0 +1,48 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) internal import _TestDiscovery + +/// A shadow declaration of `_TestDiscovery.DiscoverableAsTestContent` that +/// allows us to add public conformances to it without causing the +/// `_TestDiscovery` module to appear in `Testing.private.swiftinterface`. +/// +/// This protocol is not part of the public interface of the testing library. +protocol DiscoverableAsTestContent: _TestDiscovery.DiscoverableAsTestContent, ~Copyable {} + +/// The type of the accessor function used to access a test content record. +/// +/// The signature of this function type must match that of the corresponding +/// type in the `_TestDiscovery` module. For more information, see +/// `ABI/TestContent.md`. +/// +/// - Warning: This type is used to implement the `@Test` macro. Do not use it +/// directly. +public typealias __TestContentRecordAccessor = @convention(c) ( + _ outValue: UnsafeMutableRawPointer, + _ type: UnsafeRawPointer, + _ hint: UnsafeRawPointer? +) -> CBool + +/// The content of a test content record. +/// +/// The layout of this type must match that of the corresponding type +/// in the `_TestDiscovery` module. For more information, see +/// `ABI/TestContent.md`. +/// +/// - Warning: This type is used to implement the `@Test` macro. Do not use it +/// directly. +public typealias __TestContentRecord = ( + kind: UInt32, + reserved1: UInt32, + accessor: __TestContentRecordAccessor?, + context: UInt, + reserved2: UInt +) diff --git a/Sources/Testing/Discovery.swift b/Sources/Testing/Discovery.swift deleted file mode 100644 index 38f4dfa58..000000000 --- a/Sources/Testing/Discovery.swift +++ /dev/null @@ -1,171 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -private import _TestingInternals - -/// The type of the accessor function used to access a test content record. -/// -/// - Parameters: -/// - outValue: A pointer to uninitialized memory large enough to contain the -/// corresponding test content record's value. -/// - type: A pointer to the expected type of `outValue`. Use `load(as:)` to -/// get the Swift type, not `unsafeBitCast(_:to:)`. -/// - hint: An optional pointer to a hint value. -/// -/// - Returns: Whether or not `outValue` was initialized. The caller is -/// responsible for deinitializing `outValue` if it was initialized. -/// -/// - Warning: This type is used to implement the `@Test` macro. Do not use it -/// directly. -public typealias __TestContentRecordAccessor = @convention(c) ( - _ outValue: UnsafeMutableRawPointer, - _ type: UnsafeRawPointer, - _ hint: UnsafeRawPointer? -) -> CBool - -/// The content of a test content record. -/// -/// - Parameters: -/// - kind: The kind of this record. -/// - reserved1: Reserved for future use. -/// - accessor: A function which, when called, produces the test content. -/// - context: Kind-specific context for this record. -/// - reserved2: Reserved for future use. -/// -/// - Warning: This type is used to implement the `@Test` macro. Do not use it -/// directly. -public typealias __TestContentRecord = ( - kind: UInt32, - reserved1: UInt32, - accessor: __TestContentRecordAccessor?, - context: UInt, - reserved2: UInt -) - -// MARK: - - -/// A protocol describing a type that can be stored as test content at compile -/// time and later discovered at runtime. -/// -/// This protocol is used to bring some Swift type safety to the ABI described -/// in `ABI/TestContent.md`. Refer to that document for more information about -/// this protocol's requirements. -/// -/// This protocol is not part of the public interface of the testing library. In -/// the future, we could make it public if we want to support runtime discovery -/// of test content by second- or third-party code. -protocol TestContent: ~Copyable { - /// The unique "kind" value associated with this type. - /// - /// The value of this property is reserved for each test content type. See - /// `ABI/TestContent.md` for a list of values and corresponding types. - static var testContentKind: UInt32 { get } - - /// A type of "hint" passed to ``allTestContentRecords()`` to help the testing - /// library find the correct result. - /// - /// By default, this type equals `Never`, indicating that this type of test - /// content does not support hinting during discovery. - associatedtype TestContentAccessorHint: Sendable = Never -} - -// MARK: - Individual test content records - -/// A type describing a test content record of a particular (known) type. -/// -/// Instances of this type can be created by calling -/// ``TestContent/allTestContentRecords()`` on a type that conforms to -/// ``TestContent``. -/// -/// This type is not part of the public interface of the testing library. In the -/// future, we could make it public if we want to support runtime discovery of -/// test content by second- or third-party code. -struct TestContentRecord: Sendable where T: TestContent & ~Copyable { - /// The base address of the image containing this instance, if known. - /// - /// On platforms such as WASI that statically link to the testing library, the - /// value of this property is always `nil`. - /// - /// - Note: The value of this property is distinct from the pointer returned - /// by `dlopen()` (on platforms that have that function) and cannot be used - /// with interfaces such as `dlsym()` that expect such a pointer. - nonisolated(unsafe) var imageAddress: UnsafeRawPointer? - - /// The underlying test content record loaded from a metadata section. - private var _record: __TestContentRecord - - fileprivate init(imageAddress: UnsafeRawPointer?, record: __TestContentRecord) { - self.imageAddress = imageAddress - self._record = record - } - - /// The context value for this test content record. - var context: UInt { - _record.context - } - - /// Load the value represented by this record. - /// - /// - Parameters: - /// - hint: An optional hint value. If not `nil`, this value is passed to - /// the accessor function of the underlying test content record. - /// - /// - Returns: An instance of the test content type `T`, or `nil` if the - /// underlying test content record did not match `hint` or otherwise did not - /// produce a value. - /// - /// If this function is called more than once on the same instance, a new - /// value is created on each call. - func load(withHint hint: T.TestContentAccessorHint? = nil) -> T? { - guard let accessor = _record.accessor else { - return nil - } - - return withUnsafePointer(to: T.self) { type in - withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in - let initialized = if let hint { - withUnsafePointer(to: hint) { hint in - accessor(buffer.baseAddress!, type, hint) - } - } else { - accessor(buffer.baseAddress!, type, nil) - } - guard initialized else { - return nil - } - return buffer.baseAddress!.move() - } - } - } -} - -// MARK: - Enumeration of test content records - -extension TestContent where Self: ~Copyable { - /// Get all test content of this type known to Swift and found in the current - /// process. - /// - /// - Returns: A sequence of instances of ``TestContentRecord``. Only test - /// content records matching this ``TestContent`` type's requirements are - /// included in the sequence. - /// - /// - Bug: This function returns an instance of `AnySequence` instead of an - /// opaque type due to a compiler crash. ([143080508](rdar://143080508)) - static func allTestContentRecords() -> AnySequence> { - let result = SectionBounds.all(.testContent).lazy.flatMap { sb in - sb.buffer.withMemoryRebound(to: __TestContentRecord.self) { records in - records.lazy - .filter { $0.kind == testContentKind } - .map { TestContentRecord(imageAddress: sb.imageAddress, record: $0) } - } - } - return AnySequence(result) - } -} diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 9425de432..a28e2eede 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -8,6 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) private import _TestDiscovery private import _TestingInternals #if !SWT_NO_EXIT_TESTS @@ -237,7 +238,7 @@ extension ExitTest { // MARK: - Discovery -extension ExitTest: TestContent { +extension ExitTest: DiscoverableAsTestContent { static var testContentKind: UInt32 { 0x65786974 } diff --git a/Sources/Testing/Support/Additions/WinSDKAdditions.swift b/Sources/Testing/Support/Additions/WinSDKAdditions.swift index 18d08bfcd..488d52dd6 100644 --- a/Sources/Testing/Support/Additions/WinSDKAdditions.swift +++ b/Sources/Testing/Support/Additions/WinSDKAdditions.swift @@ -50,83 +50,4 @@ let STATUS_SIGNAL_CAUGHT_BITS = { return result }() - -// MARK: - HMODULE members - -extension HMODULE { - /// A helper type that manages state for ``HMODULE/all``. - private final class _AllState { - /// The toolhelp snapshot. - var snapshot: HANDLE? - - /// The module iterator. - var me = MODULEENTRY32W() - - deinit { - if let snapshot { - CloseHandle(snapshot) - } - } - } - - /// All modules loaded in the current process. - /// - /// - Warning: It is possible for one or more modules in this sequence to be - /// unloaded while you are iterating over it. To minimize the risk, do not - /// discard the sequence until iteration is complete. Modules containing - /// Swift code can never be safely unloaded. - static var all: some Sequence { - sequence(state: _AllState()) { state in - if let snapshot = state.snapshot { - // We have already iterated over the first module. Return the next one. - if Module32NextW(snapshot, &state.me) { - return state.me.hModule - } - } else { - // Create a toolhelp snapshot that lists modules. - guard let snapshot = CreateToolhelp32Snapshot(DWORD(TH32CS_SNAPMODULE), 0) else { - return nil - } - state.snapshot = snapshot - - // Initialize the iterator for use by the resulting sequence and return - // the first module. - state.me.dwSize = DWORD(MemoryLayout.stride(ofValue: state.me)) - if Module32FirstW(snapshot, &state.me) { - return state.me.hModule - } - } - - // Reached the end of the iteration. - return nil - } - } - - /// Get the NT header corresponding to this module. - /// - /// - Parameters: - /// - body: The function to invoke. A pointer to the module's NT header is - /// passed to this function, or `nil` if it could not be found. - /// - /// - Returns: Whatever is returned by `body`. - /// - /// - Throws: Whatever is thrown by `body`. - func withNTHeader(_ body: (UnsafePointer?) throws -> R) rethrows -> R { - // Get the DOS header (to which the HMODULE directly points, conveniently!) - // and check it's sufficiently valid for us to walk. The DOS header then - // tells us where to find the NT header. - try withMemoryRebound(to: IMAGE_DOS_HEADER.self, capacity: 1) { dosHeader in - guard dosHeader.pointee.e_magic == IMAGE_DOS_SIGNATURE, - let e_lfanew = Int(exactly: dosHeader.pointee.e_lfanew), e_lfanew > 0 else { - return try body(nil) - } - - let ntHeader = (UnsafeRawPointer(dosHeader) + e_lfanew).assumingMemoryBound(to: IMAGE_NT_HEADERS.self) - guard ntHeader.pointee.Signature == IMAGE_NT_SIGNATURE else { - return try body(nil) - } - return try body(ntHeader) - } - } -} #endif diff --git a/Sources/Testing/Support/GetSymbol.swift b/Sources/Testing/Support/GetSymbol.swift index 5fb329143..02346c2a8 100644 --- a/Sources/Testing/Support/GetSymbol.swift +++ b/Sources/Testing/Support/GetSymbol.swift @@ -8,6 +8,9 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +#if os(Windows) +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) private import _TestDiscovery +#endif internal import _TestingInternals #if !SWT_NO_DYNAMIC_LINKING diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index 301b1e955..dfb8d84c5 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -43,20 +43,3 @@ public protocol __ExitTestContainer { /// `__ExitTestContainer` protocol. let exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" #endif - -// MARK: - - -/// Get all types known to Swift found in the current process whose names -/// contain a given substring. -/// -/// - Parameters: -/// - nameSubstring: A string which the names of matching classes all contain. -/// -/// - Returns: A sequence of Swift types whose names contain `nameSubstring`. -func types(withNamesContaining nameSubstring: String) -> some Sequence { - SectionBounds.all(.typeMetadata).lazy.flatMap { sb in - stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy - .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: nameSubstring) } - .map { unsafeBitCast($0, to: Any.Type.self) } - } -} diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index d42f00be6..0d0695f6c 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -8,6 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) private import _TestDiscovery private import _TestingInternals extension Test { @@ -18,7 +19,7 @@ extension Test { /// indirect `async` accessor function rather than directly producing /// instances of ``Test``, but functions are non-nominal types and cannot /// directly conform to protocols. - fileprivate struct Generator: TestContent, RawRepresentable { + fileprivate struct Generator: DiscoverableAsTestContent, RawRepresentable { static var testContentKind: UInt32 { 0x74657374 } diff --git a/Sources/_TestDiscovery/Additions/WinSDKAdditions.swift b/Sources/_TestDiscovery/Additions/WinSDKAdditions.swift new file mode 100644 index 000000000..20019ebda --- /dev/null +++ b/Sources/_TestDiscovery/Additions/WinSDKAdditions.swift @@ -0,0 +1,90 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if os(Windows) +package import _TestingInternals + +extension HMODULE { + /// A helper type that manages state for ``HMODULE/all``. + private final class _AllState { + /// The toolhelp snapshot. + var snapshot: HANDLE? + + /// The module iterator. + var me = MODULEENTRY32W() + + deinit { + if let snapshot { + CloseHandle(snapshot) + } + } + } + + /// All modules loaded in the current process. + /// + /// - Warning: It is possible for one or more modules in this sequence to be + /// unloaded while you are iterating over it. To minimize the risk, do not + /// discard the sequence until iteration is complete. Modules containing + /// Swift code can never be safely unloaded. + package static var all: some Sequence { + sequence(state: _AllState()) { state in + if let snapshot = state.snapshot { + // We have already iterated over the first module. Return the next one. + if Module32NextW(snapshot, &state.me) { + return state.me.hModule + } + } else { + // Create a toolhelp snapshot that lists modules. + guard let snapshot = CreateToolhelp32Snapshot(DWORD(TH32CS_SNAPMODULE), 0) else { + return nil + } + state.snapshot = snapshot + + // Initialize the iterator for use by the resulting sequence and return + // the first module. + state.me.dwSize = DWORD(MemoryLayout.stride(ofValue: state.me)) + if Module32FirstW(snapshot, &state.me) { + return state.me.hModule + } + } + + // Reached the end of the iteration. + return nil + } + } + + /// Get the NT header corresponding to this module. + /// + /// - Parameters: + /// - body: The function to invoke. A pointer to the module's NT header is + /// passed to this function, or `nil` if it could not be found. + /// + /// - Returns: Whatever is returned by `body`. + /// + /// - Throws: Whatever is thrown by `body`. + func withNTHeader(_ body: (UnsafePointer?) throws -> R) rethrows -> R { + // Get the DOS header (to which the HMODULE directly points, conveniently!) + // and check it's sufficiently valid for us to walk. The DOS header then + // tells us where to find the NT header. + try withMemoryRebound(to: IMAGE_DOS_HEADER.self, capacity: 1) { dosHeader in + guard dosHeader.pointee.e_magic == IMAGE_DOS_SIGNATURE, + let e_lfanew = Int(exactly: dosHeader.pointee.e_lfanew), e_lfanew > 0 else { + return try body(nil) + } + + let ntHeader = (UnsafeRawPointer(dosHeader) + e_lfanew).assumingMemoryBound(to: IMAGE_NT_HEADERS.self) + guard ntHeader.pointee.Signature == IMAGE_NT_SIGNATURE else { + return try body(nil) + } + return try body(ntHeader) + } + } +} +#endif diff --git a/Sources/_TestDiscovery/CMakeLists.txt b/Sources/_TestDiscovery/CMakeLists.txt new file mode 100644 index 000000000..fa287d120 --- /dev/null +++ b/Sources/_TestDiscovery/CMakeLists.txt @@ -0,0 +1,32 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +add_library(_TestDiscovery STATIC + Additions/WinSDKAdditions.swift + DiscoverableAsTestContent.swift + SectionBounds.swift + TestContentRecord.swift) + +target_link_libraries(_TestDiscovery PRIVATE + _TestingInternals) + +target_compile_options(_TestDiscovery PRIVATE + -enable-library-evolution + -emit-module-interface -emit-module-interface-path $/_TestDiscovery.swiftinterface) +set(CMAKE_STATIC_LIBRARY_PREFIX_Swift "lib") + +_swift_testing_install_target(_TestDiscovery) + +if(NOT BUILD_SHARED_LIBS) + # When building a static library, install the internal library archive + # alongside the main library. In shared library builds, the internal library + # is linked into the main library and does not need to be installed separately. + get_swift_testing_install_lib_dir(STATIC_LIBRARY lib_destination_dir) + install(TARGETS _TestDiscovery + ARCHIVE DESTINATION ${lib_destination_dir}) +endif() diff --git a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift new file mode 100644 index 000000000..719b8dbe9 --- /dev/null +++ b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift @@ -0,0 +1,42 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A protocol describing a type that can be represented by a test content +/// record, stored in the test content section of a Swift binary at compile +/// time, and dynamically discovered at runtime. +/// +/// Types conforming to this protocol must also conform to [`Sendable`](https://developer.apple.com/documentation/swift/sendable) +/// because they may be discovered within any isolation context or within +/// multiple isolation contexts running concurrently. +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +public protocol DiscoverableAsTestContent: Sendable, ~Copyable { + /// The value of the `kind` field in test content records associated with this + /// type. + /// + /// The value of this property is reserved for each test content type. See + /// `ABI/TestContent.md` for a list of values and corresponding types. + static var testContentKind: UInt32 { get } + + /// The type of the `context` field in test content records associated with + /// this type. + /// + /// By default, this type equals `UInt`. This type can be set to some other + /// type with the same stride and alignment as `UInt`. Using a type with + /// different stride or alignment will result in a failure when trying to + /// discover test content records associated with this type. + associatedtype TestContentContext: BitwiseCopyable = UInt + + /// A type of "hint" passed to ``allTestContentRecords()`` to help the testing + /// library find the correct result. + /// + /// By default, this type equals `Never`, indicating that this type of test + /// content does not support hinting during discovery. + associatedtype TestContentAccessorHint = Never +} diff --git a/Sources/Testing/Discovery+Platform.swift b/Sources/_TestDiscovery/SectionBounds.swift similarity index 92% rename from Sources/Testing/Discovery+Platform.swift rename to Sources/_TestDiscovery/SectionBounds.swift index 3da974386..1fc379258 100644 --- a/Sources/Testing/Discovery+Platform.swift +++ b/Sources/_TestDiscovery/SectionBounds.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -internal import _TestingInternals +private import _TestingInternals #if _runtime(_ObjC) private import ObjectiveC #endif @@ -68,19 +68,30 @@ extension SectionBounds.Kind { /// An array containing all of the test content section bounds known to the /// testing library. -private let _sectionBounds = Locked<[SectionBounds.Kind: [SectionBounds]]>() +private nonisolated(unsafe) let _sectionBounds = { + let result = ManagedBuffer<[SectionBounds.Kind: [SectionBounds]], pthread_mutex_t>.create( + minimumCapacity: 1, + makingHeaderWith: { _ in [:] } + ) + + result.withUnsafeMutablePointers { sectionBounds, lock in + _ = pthread_mutex_init(lock, nil) -/// A call-once function that initializes `_sectionBounds` and starts listening -/// for loaded Mach headers. -private let _startCollectingSectionBounds: Void = { - // Ensure _sectionBounds is initialized before we touch libobjc or dyld. - _sectionBounds.withLock { sectionBounds in let imageCount = Int(clamping: _dyld_image_count()) for kind in SectionBounds.Kind.allCases { - sectionBounds[kind, default: []].reserveCapacity(imageCount) + sectionBounds.pointee[kind, default: []].reserveCapacity(imageCount) } } + return result +}() + +/// A call-once function that initializes `_sectionBounds` and starts listening +/// for loaded Mach headers. +private let _startCollectingSectionBounds: Void = { + // Ensure _sectionBounds is initialized before we touch libobjc or dyld. + _ = _sectionBounds + func addSectionBounds(from mh: UnsafePointer) { #if _pointerBitWidth(_64) let mh = UnsafeRawPointer(mh).assumingMemoryBound(to: mach_header_64.self) @@ -102,8 +113,12 @@ private let _startCollectingSectionBounds: Void = { if let start = getsectiondata(mh, segmentName.utf8Start, sectionName.utf8Start, &size), size > 0 { let buffer = UnsafeRawBufferPointer(start: start, count: Int(clamping: size)) let sb = SectionBounds(imageAddress: mh, buffer: buffer) - _sectionBounds.withLock { sectionBounds in - sectionBounds[kind]!.append(sb) + _sectionBounds.withUnsafeMutablePointers { sectionBounds, lock in + pthread_mutex_lock(lock) + defer { + pthread_mutex_unlock(lock) + } + sectionBounds.pointee[kind]!.append(sb) } } } @@ -129,7 +144,13 @@ private let _startCollectingSectionBounds: Void = { /// content sections in the current process. private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { _startCollectingSectionBounds - return _sectionBounds.rawValue[kind]! + return _sectionBounds.withUnsafeMutablePointers { sectionBounds, lock in + pthread_mutex_lock(lock) + defer { + pthread_mutex_unlock(lock) + } + return sectionBounds.pointee[kind]! + } } #elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift new file mode 100644 index 000000000..f576a8160 --- /dev/null +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -0,0 +1,235 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +// MARK: Low-level structure + +/// The type of the accessor function used to access a test content record. +/// +/// - Parameters: +/// - outValue: A pointer to uninitialized memory large enough to contain the +/// corresponding test content record's value. +/// - type: A pointer to the expected type of `outValue`. Use `load(as:)` to +/// get the Swift type, not `unsafeBitCast(_:to:)`. +/// - hint: An optional pointer to a hint value. +/// +/// - Returns: Whether or not `outValue` was initialized. The caller is +/// responsible for deinitializing `outValue` if it was initialized. +private typealias _TestContentRecordAccessor = @convention(c) ( + _ outValue: UnsafeMutableRawPointer, + _ type: UnsafeRawPointer, + _ hint: UnsafeRawPointer? +) -> CBool + +/// The content of a test content record. +/// +/// - Parameters: +/// - kind: The kind of this record. +/// - reserved1: Reserved for future use. +/// - accessor: A function which, when called, produces the test content. +/// - context: Kind-specific context for this record. +/// - reserved2: Reserved for future use. +private typealias _TestContentRecord = ( + kind: UInt32, + reserved1: UInt32, + accessor: _TestContentRecordAccessor?, + context: UInt, + reserved2: UInt +) + +extension DiscoverableAsTestContent where Self: ~Copyable { + /// Check that the layout of this structure in memory matches its expected + /// layout in the test content section. + /// + /// It is not currently possible to perform this validation at compile time. + /// ([swift-#79667](https://github.com/swiftlang/swift/issues/79667)) + fileprivate static func validateMemoryLayout() { + precondition(MemoryLayout.stride == MemoryLayout.stride, "'\(self).TestContentContext' aka '\(TestContentContext.self)' must have the same stride as 'UInt'.") + precondition(MemoryLayout.alignment == MemoryLayout.alignment, "'\(self).TestContentContext' aka '\(TestContentContext.self)' must have the same alignment as 'UInt'.") + } +} + +// MARK: - Individual test content records + +/// A type describing a test content record of a particular (known) type. +/// +/// Instances of this type can be created by calling +/// ``DiscoverableAsTestContent/allTestContentRecords()`` on a type that +/// conforms to ``DiscoverableAsTestContent``. +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyable { + /// The base address of the image containing this instance, if known. + /// + /// The type of this pointer is platform-dependent: + /// + /// | Platform | Pointer Type | + /// |-|-| + /// | macOS, iOS, watchOS, tvOS, visionOS | `UnsafePointer` | + /// | Linux, FreeBSD, Android | `UnsafePointer` | + /// | OpenBSD | `UnsafePointer` | + /// | Windows | `HMODULE` | + /// + /// On platforms such as WASI that statically link to the testing library, the + /// value of this property is always `nil`. + /// + /// - Note: The value of this property is distinct from the pointer returned + /// by `dlopen()` (on platforms that have that function) and cannot be used + /// with interfaces such as `dlsym()` that expect such a pointer. + public private(set) nonisolated(unsafe) var imageAddress: UnsafeRawPointer? + + /// The address of the underlying test content record loaded from a metadata + /// section. + private nonisolated(unsafe) var _recordAddress: UnsafePointer<_TestContentRecord> + + fileprivate init(imageAddress: UnsafeRawPointer?, recordAddress: UnsafePointer<_TestContentRecord>) { + self.imageAddress = imageAddress + self._recordAddress = recordAddress + } + + /// The type of the ``context`` property. + public typealias Context = T.TestContentContext + + /// The context of this test content record. + public var context: Context { + T.validateMemoryLayout() + return withUnsafeBytes(of: _recordAddress.pointee.context) { context in + context.load(as: Context.self) + } + } + + /// The type of the `hint` argument to ``load(withHint:)``. + public typealias Hint = T.TestContentAccessorHint + + /// Load the value represented by this record. + /// + /// - Parameters: + /// - hint: An optional hint value. If not `nil`, this value is passed to + /// the accessor function of the underlying test content record. + /// + /// - Returns: An instance of the test content type `T`, or `nil` if the + /// underlying test content record did not match `hint` or otherwise did not + /// produce a value. + /// + /// The result of this function is not cached. If this function is called more + /// than once on the same instance, the testing library calls the underlying + /// test content record's accessor function each time. + public func load(withHint hint: Hint? = nil) -> T? { + guard let accessor = _recordAddress.pointee.accessor else { + return nil + } + + return withUnsafePointer(to: T.self) { type in + withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in + let initialized = if let hint { + withUnsafePointer(to: hint) { hint in + accessor(buffer.baseAddress!, type, hint) + } + } else { + accessor(buffer.baseAddress!, type, nil) + } + guard initialized else { + return nil + } + return buffer.baseAddress!.move() + } + } + } +} + +// Test content sections effectively exist outside any Swift isolation context. +// We can only be (reasonably) sure that the data backing the test content +// record is concurrency-safe if all fields in the test content record are. The +// pointers stored in this structure are read-only and come from a loaded image, +// and all fields of `_TestContentRecord` as we define it are sendable. However, +// the custom `Context` type may or may not be sendable (it could validly be a +// pointer to more data, for instance.) +extension TestContentRecord: Sendable where Context: Sendable {} + +// MARK: - CustomStringConvertible + +extension TestContentRecord: CustomStringConvertible { + /// This test content type's kind value as an ASCII string (of the form + /// `"abcd"`) if it looks like it might be a [FourCC](https://en.wikipedia.org/wiki/FourCC) + /// value, or `nil` if not. + private static var _asciiKind: String? { + return withUnsafeBytes(of: T.testContentKind.bigEndian) { bytes in + if bytes.allSatisfy(Unicode.ASCII.isASCII) { + let characters = String(decoding: bytes, as: Unicode.ASCII.self) + let allAlphanumeric = characters.allSatisfy { $0.isLetter || $0.isWholeNumber } + if allAlphanumeric { + return characters + } + } + return nil + } + } + + public var description: String { + let typeName = String(describing: Self.self) + let hexKind = "0x" + String(T.testContentKind, radix: 16) + let kind = Self._asciiKind.map { asciiKind in + "'\(asciiKind)' (\(hexKind))" + } ?? hexKind + let recordAddress = imageAddress.map { imageAddress in + let recordAddressDelta = UnsafeRawPointer(_recordAddress) - imageAddress + return "\(imageAddress)+0x\(String(recordAddressDelta, radix: 16))" + } ?? "\(_recordAddress)" + return "<\(typeName) \(recordAddress)> { kind: \(kind), context: \(context) }" + } +} + +// MARK: - Enumeration of test content records + +extension DiscoverableAsTestContent where Self: ~Copyable { + /// Get all test content of this type known to Swift and found in the current + /// process. + /// + /// - Returns: A sequence of instances of ``TestContentRecord``. Only test + /// content records matching this ``TestContent`` type's requirements are + /// included in the sequence. + /// + /// @Comment { + /// - Bug: This function returns an instance of `AnySequence` instead of an + /// opaque type due to a compiler crash. ([143080508](rdar://143080508)) + /// } + public static func allTestContentRecords() -> AnySequence> { + validateMemoryLayout() + let result = SectionBounds.all(.testContent).lazy.flatMap { sb in + sb.buffer.withMemoryRebound(to: _TestContentRecord.self) { records in + (0 ..< records.count).lazy + .map { (records.baseAddress! + $0) as UnsafePointer<_TestContentRecord> } + .filter { $0.pointee.kind == testContentKind } + .map { TestContentRecord(imageAddress: sb.imageAddress, recordAddress: $0) } + } + } + return AnySequence(result) + } +} + +// MARK: - Legacy test content discovery + +private import _TestingInternals + +/// Get all types known to Swift found in the current process whose names +/// contain a given substring. +/// +/// - Parameters: +/// - nameSubstring: A string which the names of matching classes all contain. +/// +/// - Returns: A sequence of Swift types whose names contain `nameSubstring`. +/// +/// - Warning: Do not adopt this functionality in new code. It is for use by +/// Swift Testing along its legacy discovery codepath only. +package func types(withNamesContaining nameSubstring: String) -> some Sequence { + SectionBounds.all(.typeMetadata).lazy.flatMap { sb in + stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy + .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: nameSubstring) } + .map { unsafeBitCast($0, to: Any.Type.self) } + } +} diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 79db275a7..ec7fdd12d 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -9,6 +9,7 @@ // @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) import _TestDiscovery private import _TestingInternals @Test(/* name unspecified */ .hidden) @@ -581,7 +582,7 @@ struct MiscellaneousTests { } #if !SWT_NO_DYNAMIC_LINKING && hasFeature(SymbolLinkageMarkers) - struct DiscoverableTestContent: TestContent { + struct MyTestContent: Testing.DiscoverableAsTestContent { typealias TestContentAccessorHint = UInt32 var value: UInt32 @@ -616,7 +617,7 @@ struct MiscellaneousTests { 0xABCD1234, 0, { outValue, type, hint in - guard type.load(as: Any.Type.self) == DiscoverableTestContent.self else { + guard type.load(as: Any.Type.self) == MyTestContent.self else { return false } if let hint, hint.load(as: TestContentAccessorHint.self) != expectedHint { @@ -632,10 +633,10 @@ struct MiscellaneousTests { @Test func testDiscovery() async { // Check the type of the test record sequence (it should be lazy.) - let allRecordsSeq = DiscoverableTestContent.allTestContentRecords() + let allRecordsSeq = MyTestContent.allTestContentRecords() #if SWT_FIXED_143080508 #expect(allRecordsSeq is any LazySequenceProtocol) - #expect(!(allRecordsSeq is [TestContentRecord])) + #expect(!(allRecordsSeq is [TestContentRecord])) #endif // It should have exactly one matching record (because we only emitted one.) @@ -644,22 +645,22 @@ struct MiscellaneousTests { // Can find a single test record #expect(allRecords.contains { record in - record.load()?.value == DiscoverableTestContent.expectedValue - && record.context == DiscoverableTestContent.expectedContext + record.load()?.value == MyTestContent.expectedValue + && record.context == MyTestContent.expectedContext }) // Can find a test record with matching hint #expect(allRecords.contains { record in - let hint = DiscoverableTestContent.expectedHint - return record.load(withHint: hint)?.value == DiscoverableTestContent.expectedValue - && record.context == DiscoverableTestContent.expectedContext + let hint = MyTestContent.expectedHint + return record.load(withHint: hint)?.value == MyTestContent.expectedValue + && record.context == MyTestContent.expectedContext }) // Doesn't find a test record with a mismatched hint #expect(!allRecords.contains { record in - let hint = ~DiscoverableTestContent.expectedHint - return record.load(withHint: hint)?.value == DiscoverableTestContent.expectedValue - && record.context == DiscoverableTestContent.expectedContext + let hint = ~MyTestContent.expectedHint + return record.load(withHint: hint)?.value == MyTestContent.expectedValue + && record.context == MyTestContent.expectedContext }) } #endif From ca0e513603762aaa72cba6ff13b313e58785422b Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Wed, 5 Mar 2025 16:38:18 -0800 Subject: [PATCH 114/234] build: update missed callsites of `get_swift_testing_install_lib_dir` This has been replaced with a global variable. Update the callsites to use that instead. --- Sources/_TestDiscovery/CMakeLists.txt | 3 +-- Sources/_TestingInternals/CMakeLists.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Sources/_TestDiscovery/CMakeLists.txt b/Sources/_TestDiscovery/CMakeLists.txt index fa287d120..74d951682 100644 --- a/Sources/_TestDiscovery/CMakeLists.txt +++ b/Sources/_TestDiscovery/CMakeLists.txt @@ -26,7 +26,6 @@ if(NOT BUILD_SHARED_LIBS) # When building a static library, install the internal library archive # alongside the main library. In shared library builds, the internal library # is linked into the main library and does not need to be installed separately. - get_swift_testing_install_lib_dir(STATIC_LIBRARY lib_destination_dir) install(TARGETS _TestDiscovery - ARCHIVE DESTINATION ${lib_destination_dir}) + ARCHIVE DESTINATION "${SwiftTesting_INSTALL_LIBDIR}") endif() diff --git a/Sources/_TestingInternals/CMakeLists.txt b/Sources/_TestingInternals/CMakeLists.txt index 972a0c56b..16713ab27 100644 --- a/Sources/_TestingInternals/CMakeLists.txt +++ b/Sources/_TestingInternals/CMakeLists.txt @@ -32,7 +32,6 @@ if(NOT BUILD_SHARED_LIBS) # When building a static library, install the internal library archive # alongside the main library. In shared library builds, the internal library # is linked into the main library and does not need to be installed separately. - get_swift_testing_install_lib_dir(STATIC_LIBRARY lib_destination_dir) install(TARGETS _TestingInternals - ARCHIVE DESTINATION ${lib_destination_dir}) + ARCHIVE DESTINATION "${SwiftTesting_INSTALL_LIBDIR}") endif() From 46841e7964db48cd34a91a0ee0a6a18fc0824fce Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Thu, 6 Mar 2025 22:23:55 -0800 Subject: [PATCH 115/234] [CMake] Fix quote in a generator expression --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ea6fa132a..80c922eb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -set(CMAKE_INSTALL_RPATH $,"@loader_path/..","$ORIGIN">) +set(CMAKE_INSTALL_RPATH "$,@loader_path/..,$ORIGIN>") set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH YES) set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDLL) From ad8a13f4e8efcbd53b41e7f7d021536be8c9c460 Mon Sep 17 00:00:00 2001 From: Zev Eisenberg Date: Fri, 7 Mar 2025 08:36:38 -0500 Subject: [PATCH 116/234] Fix link to Swift Forums (#1001) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e7381a31..01b8a4fd6 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ func mentionedContinents(videoName: String) async throws { Swift Testing works on all major platforms supported by Swift, including Apple platforms, Linux, and Windows, so your tests can behave more consistently when moving between platforms. It’s developed as open source and discussed on the -[Swift Forums](https://forums.swift.org/c/related-projects/swift-testing) so the +[Swift Forums](https://forums.swift.org/c/development/swift-testing/103) so the very best ideas, from anywhere, can help shape the future of testing in Swift. The table below describes the current level of support that Swift Testing has From 23af7e5f52519f85b175cf593e774b25770b15f8 Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Fri, 7 Mar 2025 12:16:57 -0500 Subject: [PATCH 117/234] Fix some broken links in Documentation (#1003) I periodically run a broken link checker over swiftlang repositories. There were a few links pointing to outdated locations in swift-testing. --- Documentation/ABI/JSON.md | 2 +- Documentation/WASI.md | 2 +- Sources/Testing/Testing.docc/MigratingFromXCTest.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index 2fac0c3fb..f313ddc00 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -13,7 +13,7 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors This document outlines the JSON schemas used by the testing library for its ABI entry point and for the `--event-stream-output-path` command-line argument. For more information about the ABI entry point, see the documentation for -[ABI.v0.EntryPoint](https://github.com/search?q=repo%3Aapple%2Fswift-testing%EntryPoint&type=code). +[ABI.v0.EntryPoint](https://github.com/search?q=repo%3Aswiftlang%2Fswift-testing%20EntryPoint&type=code). ## Modified Backus-Naur form diff --git a/Documentation/WASI.md b/Documentation/WASI.md index f4176f755..46b8800ef 100644 --- a/Documentation/WASI.md +++ b/Documentation/WASI.md @@ -14,7 +14,7 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors because it provides instructions the reader must follow directly. --> To run tests for WebAssembly, install a Swift SDK for WebAssembly by following -[these instructions](https://book.swiftwasm.org/getting-started/setup-snapshot.html). +[these instructions](https://book.swiftwasm.org/getting-started/setup.html). Because `swift test` doesn't know what WebAssembly environment you'd like to use to run your tests, building tests and running them are two separate steps. To diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index 44d91b5b0..e3a9d961f 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -397,7 +397,7 @@ Wherever possible, prefer to use Swift concurrency to validate asynchronous conditions. For example, if it's necessary to determine the result of an asynchronous Swift function, it can be awaited with `await`. For a function that takes a completion handler but which doesn't use `await`, a Swift -[continuation](https://developer.apple.com/documentation/swift/withcheckedcontinuation(function:_:)) +[continuation](https://developer.apple.com/documentation/swift/withcheckedcontinuation(isolation:function:_:)) can be used to convert the call into an `async`-compatible one. Some tests, especially those that test asynchronously-delivered events, cannot From f27305afd66cb3402d3066f8b2f9e97e8f6e298b Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 7 Mar 2025 19:42:09 -0500 Subject: [PATCH 118/234] Raise `types(withNamesContaining:)` to SPI rather than `package`. (#1004) This PR makes `types(withNamesContaining:)` available as pre-deprecated SPI. --- Sources/_TestDiscovery/TestContentRecord.swift | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index f576a8160..35e8392f6 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -223,10 +223,9 @@ private import _TestingInternals /// - nameSubstring: A string which the names of matching classes all contain. /// /// - Returns: A sequence of Swift types whose names contain `nameSubstring`. -/// -/// - Warning: Do not adopt this functionality in new code. It is for use by -/// Swift Testing along its legacy discovery codepath only. -package func types(withNamesContaining nameSubstring: String) -> some Sequence { +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@available(swift, deprecated: 100000.0, message: "Do not adopt this functionality in new code. It will be removed in a future release.") +public func types(withNamesContaining nameSubstring: String) -> some Sequence { SectionBounds.all(.typeMetadata).lazy.flatMap { sb in stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: nameSubstring) } From fc6f68c426399d19300e2f3269ecc0193cadf9d2 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 10 Mar 2025 10:12:31 -0500 Subject: [PATCH 119/234] Remove the "Result" section of this project's Pull Request template (#1005) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This removes the "Result" section from this project's GitHub Pull Request template. ### Motivation: I frequently find that in practice, the other sections of the PR template cover the result our outcome of the PR already, so the result section isn't needed and I delete it. ### Result: This section no longer exists! 😅 🪦 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .github/PULL_REQUEST_TEMPLATE.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fc4607581..8e8b5bf8d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,10 +8,6 @@ _[Explain here the context, and why you're making that change. What is the probl _[Describe the modifications you've done.]_ -### Result: - -_[After your change, what will change.]_ - ### Checklist: - [ ] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). From 67f6c4c0297154d0b7c2744d0789a7a81f38530d Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sun, 9 Mar 2025 10:35:47 -0400 Subject: [PATCH 120/234] Refine type-based test discovery mechanism to use test content records. This PR changes how we discover tests in the type metadata section to more closely align with how the test content section works. This change will allow for a smoother transition to the test content section and our use of `@section` by using the same underlying structures (as much as is feasible.) Client code that uses the new `_TestDiscovery` module will need fewer changes to adapt. --- Sources/Testing/ExitTests/ExitTest.swift | 47 +++++- Sources/Testing/Test+Discovery+Legacy.swift | 53 +++--- Sources/Testing/Test+Discovery.swift | 39 ++++- Sources/TestingMacros/CMakeLists.txt | 2 + Sources/TestingMacros/ConditionMacro.swift | 28 +++- .../TestingMacros/SuiteDeclarationMacro.swift | 65 +++++--- .../IntegerLiteralExprSyntaxAdditions.swift | 18 ++ .../Additions/TokenSyntaxAdditions.swift | 11 ++ .../Support/AttributeDiscovery.swift | 30 +++- .../Support/TestContentGeneration.swift | 74 +++++++++ .../TestingMacros/TestDeclarationMacro.swift | 95 ++++++----- .../_TestDiscovery/TestContentRecord.swift | 154 +++++++++++++++--- .../TestDeclarationMacroTests.swift | 2 +- .../TestingTests/TypeNameConflictTests.swift | 2 +- 14 files changed, 477 insertions(+), 143 deletions(-) create mode 100644 Sources/TestingMacros/Support/Additions/IntegerLiteralExprSyntaxAdditions.swift create mode 100644 Sources/TestingMacros/Support/TestContentGeneration.swift diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index a28e2eede..d8c254522 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -244,6 +244,40 @@ extension ExitTest: DiscoverableAsTestContent { } typealias TestContentAccessorHint = ID + + /// Store the exit test into the given memory. + /// + /// - Parameters: + /// - outValue: The uninitialized memory to store the exit test into. + /// - id: The unique identifier of the exit test to store. + /// - body: The body closure of the exit test to store. + /// - typeAddress: A pointer to the expected type of the exit test as passed + /// to the test content record calling this function. + /// - hintAddress: A pointer to an instance of ``ID`` to use as a hint. + /// + /// - Returns: Whether or not an exit test was stored into `outValue`. + /// + /// - Warning: This function is used to implement the `#expect(exitsWith:)` + /// macro. Do not use it directly. + public static func __store( + _ id: (UInt64, UInt64), + _ body: @escaping @Sendable () async throws -> Void, + into outValue: UnsafeMutableRawPointer, + asTypeAt typeAddress: UnsafeRawPointer, + withHintAt hintAddress: UnsafeRawPointer? = nil + ) -> CBool { + let callerExpectedType = TypeInfo(describing: typeAddress.load(as: Any.Type.self)) + let selfType = TypeInfo(describing: Self.self) + guard callerExpectedType == selfType else { + return false + } + let id = ID(id) + if let hintedID = hintAddress?.load(as: ID.self), hintedID != id { + return false + } + outValue.initializeMemory(as: Self.self, to: Self(id: id, body: body)) + return true + } } @_spi(Experimental) @_spi(ForToolsIntegrationOnly) @@ -262,15 +296,14 @@ extension ExitTest { } } -#if !SWT_NO_LEGACY_TEST_DISCOVERY // Call the legacy lookup function that discovers tests embedded in types. - return types(withNamesContaining: exitTestContainerTypeNameMagic).lazy - .compactMap { $0 as? any __ExitTestContainer.Type } - .first { ID($0.__id) == id } - .map { ExitTest(id: ID($0.__id), body: $0.__body) } -#else + for record in Self.allTypeMetadataBasedTestContentRecords() { + if let exitTest = record.load(withHint: id) { + return exitTest + } + } + return nil -#endif } } diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index dfb8d84c5..f974e3829 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -8,38 +8,35 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -private import _TestingInternals +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) internal import _TestDiscovery -/// A protocol describing a type that contains tests. +/// A shadow declaration of `_TestDiscovery.TestContentRecordContainer` that +/// allows us to add public conformances to it without causing the +/// `_TestDiscovery` module to appear in `Testing.private.swiftinterface`. /// -/// - Warning: This protocol is used to implement the `@Test` macro. Do not use -/// it directly. -@_alwaysEmitConformanceMetadata -public protocol __TestContainer { - /// The set of tests contained by this type. - static var __tests: [Test] { get async } -} - -/// A string that appears within all auto-generated types conforming to the -/// `__TestContainer` protocol. -let testContainerTypeNameMagic = "__🟠$test_container__" +/// This protocol is not part of the public interface of the testing library. +protocol TestContentRecordContainer: _TestDiscovery.TestContentRecordContainer {} -#if !SWT_NO_EXIT_TESTS -/// A protocol describing a type that contains an exit test. +/// An abstract base class describing a type that contains tests. /// -/// - Warning: This protocol is used to implement the `#expect(exitsWith:)` -/// macro. Do not use it directly. -@_alwaysEmitConformanceMetadata -@_spi(Experimental) -public protocol __ExitTestContainer { - /// The unique identifier of the exit test. - static var __id: (UInt64, UInt64) { get } +/// - Warning: This class is used to implement the `@Test` macro. Do not use it +/// directly. +open class __TestContentRecordContainer: TestContentRecordContainer { + /// The corresponding test content record. + /// + /// - Warning: This property is used to implement the `@Test` macro. Do not + /// use it directly. + open nonisolated class var __testContentRecord: __TestContentRecord { + (0, 0, nil, 0, 0) + } - /// The body function of the exit test. - static var __body: @Sendable () async throws -> Void { get } + static func storeTestContentRecord(to outTestContentRecord: UnsafeMutableRawPointer) -> Bool { + outTestContentRecord.withMemoryRebound(to: __TestContentRecord.self, capacity: 1) { outTestContentRecord in + outTestContentRecord.initialize(to: __testContentRecord) + return true + } + } } -/// A string that appears within all auto-generated types conforming to the -/// `__ExitTestContainer` protocol. -let exitTestContainerTypeNameMagic = "__🟠$exit_test_body__" -#endif +@available(*, unavailable) +extension __TestContentRecordContainer: Sendable {} diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 0d0695f6c..5c2d86f32 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -19,7 +19,7 @@ extension Test { /// indirect `async` accessor function rather than directly producing /// instances of ``Test``, but functions are non-nominal types and cannot /// directly conform to protocols. - fileprivate struct Generator: DiscoverableAsTestContent, RawRepresentable { + struct Generator: DiscoverableAsTestContent, RawRepresentable { static var testContentKind: UInt32 { 0x74657374 } @@ -27,6 +27,30 @@ extension Test { var rawValue: @Sendable () async -> Test } + /// Store the test generator function into the given memory. + /// + /// - Parameters: + /// - generator: The generator function to store. + /// - outValue: The uninitialized memory to store `generator` into. + /// - typeAddress: A pointer to the expected type of `generator` as passed + /// to the test content record calling this function. + /// + /// - Returns: Whether or not `generator` was stored into `outValue`. + /// + /// - Warning: This function is used to implement the `@Test` macro. Do not + /// use it directly. + public static func __store( + _ generator: @escaping @Sendable () async -> Test, + into outValue: UnsafeMutableRawPointer, + asTypeAt typeAddress: UnsafeRawPointer + ) -> CBool { + guard typeAddress.load(as: Any.Type.self) == Generator.self else { + return false + } + outValue.initializeMemory(as: Generator.self, to: .init(rawValue: generator)) + return true + } + /// All available ``Test`` instances in the process, according to the runtime. /// /// The order of values in this sequence is unspecified. @@ -64,15 +88,12 @@ extension Test { // Perform legacy test discovery if needed. if useLegacyMode && result.isEmpty { - let types = types(withNamesContaining: testContainerTypeNameMagic).lazy - .compactMap { $0 as? any __TestContainer.Type } - await withTaskGroup(of: [Self].self) { taskGroup in - for type in types { - taskGroup.addTask { - await type.__tests - } + let generators = Generator.allTypeMetadataBasedTestContentRecords().lazy.compactMap { $0.load() } + await withTaskGroup(of: Self.self) { taskGroup in + for generator in generators { + taskGroup.addTask { await generator.rawValue() } } - result = await taskGroup.reduce(into: result) { $0.formUnion($1) } + result = await taskGroup.reduce(into: result) { $0.insert($1) } } } diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index 4fc8b3b58..b0d809665 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -87,6 +87,7 @@ target_sources(TestingMacros PRIVATE Support/Additions/DeclGroupSyntaxAdditions.swift Support/Additions/EditorPlaceholderExprSyntaxAdditions.swift Support/Additions/FunctionDeclSyntaxAdditions.swift + Support/Additions/IntegerLiteralExprSyntaxAdditions.swift Support/Additions/MacroExpansionContextAdditions.swift Support/Additions/TokenSyntaxAdditions.swift Support/Additions/TriviaPieceAdditions.swift @@ -103,6 +104,7 @@ target_sources(TestingMacros PRIVATE Support/DiagnosticMessage+Diagnosing.swift Support/SourceCodeCapturing.swift Support/SourceLocationGeneration.swift + Support/TestContentGeneration.swift TagMacro.swift TestDeclarationMacro.swift TestingMacrosMain.swift) diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index b4f5af1c3..c1c04069b 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -452,16 +452,32 @@ extension ExitTestConditionMacro { // Create a local type that can be discovered at runtime and which contains // the exit test body. - let enumName = context.makeUniqueName("__🟠$exit_test_body__") + let className = context.makeUniqueName("__🟠$") + let testContentRecordDecl = makeTestContentRecordDecl( + named: .identifier("testContentRecord"), + in: TypeSyntax(IdentifierTypeSyntax(name: className)), + ofKind: .exitTest, + accessingWith: .identifier("accessor") + ) + decls.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - enum \(enumName): Testing.__ExitTestContainer, Sendable { - static var __id: (Swift.UInt64, Swift.UInt64) { - \(exitTestIDExpr) + final class \(className): Testing.__TestContentRecordContainer { + private nonisolated static let accessor: Testing.__TestContentRecordAccessor = { outValue, type, hint in + Testing.ExitTest.__store( + \(exitTestIDExpr), + \(bodyThunkName), + into: outValue, + asTypeAt: type, + withHintAt: hint + ) } - static var __body: @Sendable () async throws -> Void { - \(bodyThunkName) + + \(testContentRecordDecl) + + override nonisolated class var __testContentRecord: Testing.__TestContentRecord { + testContentRecord } } """ diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index c9fb6bb08..a65a7a420 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -25,7 +25,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { guard _diagnoseIssues(with: declaration, suiteAttribute: node, in: context) else { return [] } - return _createTestContainerDecls(for: declaration, suiteAttribute: node, in: context) + return _createSuiteDecls(for: declaration, suiteAttribute: node, in: context) } public static func expansion( @@ -97,8 +97,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { return !diagnostics.lazy.map(\.severity).contains(.error) } - /// Create a declaration for a type that conforms to the `__TestContainer` - /// protocol and which contains the given suite type. + /// Create the declarations necessary to discover a suite at runtime. /// /// - Parameters: /// - declaration: The type declaration the result should encapsulate. @@ -107,7 +106,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { /// /// - Returns: An array of declarations providing runtime information about /// the test suite type `declaration`. - private static func _createTestContainerDecls( + private static func _createSuiteDecls( for declaration: some DeclGroupSyntax, suiteAttribute: AttributeSyntax, in context: some MacroExpansionContext @@ -127,28 +126,48 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { // Parse the @Suite attribute. let attributeInfo = AttributeInfo(byParsing: suiteAttribute, on: declaration, in: context) - // The emitted type must be public or the compiler can optimize it away - // (since it is not actually used anywhere that the compiler can see.) - // - // The emitted type must be deprecated to avoid causing warnings in client - // code since it references the suite metatype, which may be deprecated - // to allow test functions to validate deprecated APIs. The emitted type is - // also annotated unavailable, since it's meant only for use by the testing - // library at runtime. The compiler does not allow combining 'unavailable' - // and 'deprecated' into a single availability attribute: rdar://111329796 - let typeName = declaration.type.tokens(viewMode: .fixedUp).map(\.textWithoutBackticks).joined() - let enumName = context.makeUniqueName("__🟠$test_container__suite__\(typeName)") + let generatorName = context.makeUniqueName("generator") + result.append( + """ + @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") + @Sendable private static func \(generatorName)() async -> Testing.Test { + .__type( + \(declaration.type.trimmed).self, + \(raw: attributeInfo.functionArgumentList(in: context)) + ) + } + """ + ) + + let accessorName = context.makeUniqueName("accessor") + result.append( + """ + @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") + private nonisolated static let \(accessorName): Testing.__TestContentRecordAccessor = { outValue, type, _ in + Testing.Test.__store(\(generatorName), into: outValue, asTypeAt: type) + } + """ + ) + + let testContentRecordName = context.makeUniqueName("testContentRecord") + result.append( + makeTestContentRecordDecl( + named: testContentRecordName, + in: declaration.type, + ofKind: .testDeclaration, + accessingWith: accessorName, + context: attributeInfo.testContentRecordFlags + ) + ) + + // Emit a type that contains a reference to the test content record. + let className = context.makeUniqueName("__🟠$") result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - enum \(enumName): Testing.__TestContainer { - static var __tests: [Testing.Test] { - get async {[ - .__type( - \(declaration.type.trimmed).self, - \(raw: attributeInfo.functionArgumentList(in: context)) - ) - ]} + private final class \(className): Testing.__TestContentRecordContainer { + override nonisolated class var __testContentRecord: Testing.__TestContentRecord { + \(testContentRecordName) } } """ diff --git a/Sources/TestingMacros/Support/Additions/IntegerLiteralExprSyntaxAdditions.swift b/Sources/TestingMacros/Support/Additions/IntegerLiteralExprSyntaxAdditions.swift new file mode 100644 index 000000000..e2310b44f --- /dev/null +++ b/Sources/TestingMacros/Support/Additions/IntegerLiteralExprSyntaxAdditions.swift @@ -0,0 +1,18 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +import SwiftSyntax + +extension IntegerLiteralExprSyntax { + init(_ value: some BinaryInteger, radix: IntegerLiteralExprSyntax.Radix = .decimal) { + let stringValue = "\(radix.literalPrefix)\(String(value, radix: radix.size))" + self.init(literal: .integerLiteral(stringValue)) + } +} diff --git a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift index 12e6abb24..2be9977d5 100644 --- a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift @@ -47,3 +47,14 @@ extension TokenSyntax { return nil } } + +/// The `static` keyword, if `typeName` is not `nil`. +/// +/// - Parameters: +/// - typeName: The name of the type containing the macro being expanded. +/// +/// - Returns: A token representing the `static` keyword, or one representing +/// nothing if `typeName` is `nil`. +func staticKeyword(for typeName: TypeSyntax?) -> TokenSyntax { + (typeName != nil) ? .keyword(.static) : .unknown("") +} diff --git a/Sources/TestingMacros/Support/AttributeDiscovery.swift b/Sources/TestingMacros/Support/AttributeDiscovery.swift index dce4bddd3..84d96cf84 100644 --- a/Sources/TestingMacros/Support/AttributeDiscovery.swift +++ b/Sources/TestingMacros/Support/AttributeDiscovery.swift @@ -16,8 +16,8 @@ import SwiftSyntaxMacros /// /// If the developer specified Self.something as an argument to the `@Test` or /// `@Suite` attribute, we will currently incorrectly infer Self as equalling -/// the `__TestContainer` type we emit rather than the type containing the -/// test. This class strips off `Self.` wherever that occurs. +/// the container type that we emit rather than the type containing the test. +/// This class strips off `Self.` wherever that occurs. /// /// Note that this operation is technically incorrect if a subexpression of the /// attribute declares a type and refers to it with `Self`. We accept this @@ -60,6 +60,9 @@ struct AttributeInfo { /// The attribute node that was parsed to produce this instance. var attribute: AttributeSyntax + /// The declaration to which ``attribute`` was attached. + var declaration: DeclSyntax + /// The display name of the attribute, if present. var displayName: StringLiteralExprSyntax? @@ -85,6 +88,21 @@ struct AttributeInfo { /// as the canonical source location of the test or suite. var sourceLocation: ExprSyntax + /// Flags to apply to the test content record generated from this instance. + var testContentRecordFlags: UInt32 { + var result = UInt32(0) + + if declaration.is(FunctionDeclSyntax.self) { + if hasFunctionArguments { + result |= 1 << 1 /* is parameterized */ + } + } else { + result |= 1 << 0 /* suite decl */ + } + + return result + } + /// Create an instance of this type by parsing a `@Test` or `@Suite` /// attribute. /// @@ -92,13 +110,11 @@ struct AttributeInfo { /// - attribute: The attribute whose arguments should be extracted. If this /// attribute is not a `@Test` or `@Suite` attribute, the result is /// unspecified. - /// - declaration: The declaration to which `attribute` is attached. For - /// technical reasons, this argument is only constrained to - /// `SyntaxProtocol`, however an instance of a type conforming to - /// `DeclSyntaxProtocol & WithAttributesSyntax` is expected. + /// - declaration: The declaration to which `attribute` is attached. /// - context: The macro context in which the expression is being parsed. - init(byParsing attribute: AttributeSyntax, on declaration: some SyntaxProtocol, in context: some MacroExpansionContext) { + init(byParsing attribute: AttributeSyntax, on declaration: some DeclSyntaxProtocol, in context: some MacroExpansionContext) { self.attribute = attribute + self.declaration = DeclSyntax(declaration) var displayNameArgument: LabeledExprListSyntax.Element? var nonDisplayNameArguments: [Argument] = [] diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift new file mode 100644 index 000000000..646bd97d4 --- /dev/null +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -0,0 +1,74 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +import SwiftSyntax +import SwiftSyntaxMacros + +/// An enumeration representing the different kinds of test content known to the +/// testing library. +/// +/// When adding cases to this enumeration, be sure to also update the +/// corresponding enumeration in TestContent.md. +enum TestContentKind: UInt32 { + /// A test or suite declaration. + case testDeclaration = 0x74657374 + + /// An exit test. + case exitTest = 0x65786974 + + /// This kind value as a comment (`/* 'abcd' */`) if it looks like it might be + /// a [FourCC](https://en.wikipedia.org/wiki/FourCC) value, or `nil` if not. + var commentRepresentation: Trivia { + switch self { + case .testDeclaration: + .blockComment("/* 'test' */") + case .exitTest: + .blockComment("/* 'exit' */") + } + } +} + +/// Make a test content record that can be discovered at runtime by the testing +/// library. +/// +/// - Parameters: +/// - name: The name of the record declaration to use in Swift source. The +/// value of this argument should be unique in the context in which the +/// declaration will be emitted. +/// - typeName: The name of the type enclosing the resulting declaration, or +/// `nil` if it will not be emitted into a type's scope. +/// - kind: The kind of test content record being emitted. +/// - accessorName: The Swift name of an `@convention(c)` function to emit +/// into the resulting record. +/// - context: A value to emit as the `context` field of the test content +/// record. +/// +/// - Returns: A variable declaration that, when emitted into Swift source, will +/// cause the linker to emit data in a location that is discoverable at +/// runtime. +func makeTestContentRecordDecl(named name: TokenSyntax, in typeName: TypeSyntax? = nil, ofKind kind: TestContentKind, accessingWith accessorName: TokenSyntax, context: UInt32 = 0) -> DeclSyntax { + let kindExpr = IntegerLiteralExprSyntax(kind.rawValue, radix: .hex) + let contextExpr = if context == 0 { + IntegerLiteralExprSyntax(0) + } else { + IntegerLiteralExprSyntax(context, radix: .binary) + } + + return """ + @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") + private nonisolated \(staticKeyword(for: typeName)) let \(name): Testing.__TestContentRecord = ( + \(kindExpr), \(kind.commentRepresentation) + 0, + \(accessorName), + \(contextExpr), + 0 + ) + """ +} diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 1b9f995bc..c085ffc42 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -28,7 +28,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { let functionDecl = declaration.cast(FunctionDeclSyntax.self) let typeName = context.typeOfLexicalContext - return _createTestContainerDecls(for: functionDecl, on: typeName, testAttribute: node, in: context) + return _createTestDecls(for: functionDecl, on: typeName, testAttribute: node, in: context) } public static var formatMode: FormatMode { @@ -364,8 +364,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { return thunkDecl.cast(FunctionDeclSyntax.self) } - /// Create a declaration for a type that conforms to the `__TestContainer` - /// protocol and which contains a test for the given function. + /// Create the declarations necessary to discover a test at runtime. /// /// - Parameters: /// - functionDecl: The function declaration the result should encapsulate. @@ -376,7 +375,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { /// /// - Returns: An array of declarations providing runtime information about /// the test function `functionDecl`. - private static func _createTestContainerDecls( + private static func _createTestDecls( for functionDecl: FunctionDeclSyntax, on typeName: TypeSyntax?, testAttribute: AttributeSyntax, @@ -421,16 +420,14 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { // Create the expression that returns the Test instance for the function. var testsBody: CodeBlockItemListSyntax = """ - return [ - .__function( - named: \(literal: functionDecl.completeName.trimmedDescription), - in: \(typeNameExpr), - xcTestCompatibleSelector: \(selectorExpr ?? "nil"), - \(raw: attributeInfo.functionArgumentList(in: context)), - parameters: \(raw: functionDecl.testFunctionParameterList), - testFunction: \(thunkDecl.name) - ) - ] + return .__function( + named: \(literal: functionDecl.completeName.trimmedDescription), + in: \(typeNameExpr), + xcTestCompatibleSelector: \(selectorExpr ?? "nil"), + \(raw: attributeInfo.functionArgumentList(in: context)), + parameters: \(raw: functionDecl.testFunctionParameterList), + testFunction: \(thunkDecl.name) + ) """ // If this function has arguments, then it can only be referenced (let alone @@ -446,16 +443,14 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { result.append( """ @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") - private \(_staticKeyword(for: typeName)) nonisolated func \(unavailableTestName)() async -> [Testing.Test] { - [ - .__function( - named: \(literal: functionDecl.completeName.trimmedDescription), - in: \(typeNameExpr), - xcTestCompatibleSelector: \(selectorExpr ?? "nil"), - \(raw: attributeInfo.functionArgumentList(in: context)), - testFunction: {} - ) - ] + private \(staticKeyword(for: typeName)) nonisolated func \(unavailableTestName)() async -> Testing.Test { + .__function( + named: \(literal: functionDecl.completeName.trimmedDescription), + in: \(typeNameExpr), + xcTestCompatibleSelector: \(selectorExpr ?? "nil"), + \(raw: attributeInfo.functionArgumentList(in: context)), + testFunction: {} + ) } """ ) @@ -470,25 +465,45 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { ) } - // The emitted type must be public or the compiler can optimize it away - // (since it is not actually used anywhere that the compiler can see.) - // - // The emitted type must be deprecated to avoid causing warnings in client - // code since it references the test function thunk, which is itself - // deprecated to allow test functions to validate deprecated APIs. The - // emitted type is also annotated unavailable, since it's meant only for use - // by the testing library at runtime. The compiler does not allow combining - // 'unavailable' and 'deprecated' into a single availability attribute: - // rdar://111329796 - let enumName = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟠$test_container__function__") + let generatorName = context.makeUniqueName(thunking: functionDecl, withPrefix: "generator") + result.append( + """ + @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") + @Sendable private \(staticKeyword(for: typeName)) func \(generatorName)() async -> Testing.Test { + \(raw: testsBody) + } + """ + ) + + let accessorName = context.makeUniqueName(thunking: functionDecl, withPrefix: "accessor") + result.append( + """ + @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") + private \(staticKeyword(for: typeName)) nonisolated let \(accessorName): Testing.__TestContentRecordAccessor = { outValue, type, _ in + Testing.Test.__store(\(generatorName), into: outValue, asTypeAt: type) + } + """ + ) + + let testContentRecordName = context.makeUniqueName(thunking: functionDecl, withPrefix: "testContentRecord") + result.append( + makeTestContentRecordDecl( + named: testContentRecordName, + in: typeName, + ofKind: .testDeclaration, + accessingWith: accessorName, + context: attributeInfo.testContentRecordFlags + ) + ) + + // Emit a type that contains a reference to the test content record. + let className = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟠$") result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - enum \(enumName): Testing.__TestContainer { - static var __tests: [Testing.Test] { - get async { - \(raw: testsBody) - } + private final class \(className): Testing.__TestContentRecordContainer { + override nonisolated class var __testContentRecord: Testing.__TestContentRecord { + \(testContentRecordName) } } """ diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index 35e8392f6..ed5c42238 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -83,13 +83,40 @@ public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyabl /// with interfaces such as `dlsym()` that expect such a pointer. public private(set) nonisolated(unsafe) var imageAddress: UnsafeRawPointer? - /// The address of the underlying test content record loaded from a metadata - /// section. - private nonisolated(unsafe) var _recordAddress: UnsafePointer<_TestContentRecord> + /// A type defining storage for the underlying test content record. + private enum _RecordStorage: @unchecked Sendable { + /// The test content record is stored by address. + case atAddress(UnsafePointer<_TestContentRecord>) + + /// The test content record is stored in-place. + case inline(_TestContentRecord) + } + + /// Storage for `_record`. + private var _recordStorage: _RecordStorage + + /// The underlying test content record. + private var _record: _TestContentRecord { + _read { + switch _recordStorage { + case let .atAddress(recordAddress): + yield recordAddress.pointee + case let .inline(record): + yield record + } + } + } fileprivate init(imageAddress: UnsafeRawPointer?, recordAddress: UnsafePointer<_TestContentRecord>) { + precondition(recordAddress.pointee.kind == T.testContentKind) self.imageAddress = imageAddress - self._recordAddress = recordAddress + self._recordStorage = .atAddress(recordAddress) + } + + fileprivate init(imageAddress: UnsafeRawPointer?, record: _TestContentRecord) { + precondition(record.kind == T.testContentKind) + self.imageAddress = imageAddress + self._recordStorage = .inline(record) } /// The type of the ``context`` property. @@ -98,7 +125,7 @@ public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyabl /// The context of this test content record. public var context: Context { T.validateMemoryLayout() - return withUnsafeBytes(of: _recordAddress.pointee.context) { context in + return withUnsafeBytes(of: _record.context) { context in context.load(as: Context.self) } } @@ -120,7 +147,7 @@ public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyabl /// than once on the same instance, the testing library calls the underlying /// test content record's accessor function each time. public func load(withHint hint: Hint? = nil) -> T? { - guard let accessor = _recordAddress.pointee.accessor else { + guard let accessor = _record.accessor else { return nil } @@ -176,11 +203,16 @@ extension TestContentRecord: CustomStringConvertible { let kind = Self._asciiKind.map { asciiKind in "'\(asciiKind)' (\(hexKind))" } ?? hexKind - let recordAddress = imageAddress.map { imageAddress in - let recordAddressDelta = UnsafeRawPointer(_recordAddress) - imageAddress - return "\(imageAddress)+0x\(String(recordAddressDelta, radix: 16))" - } ?? "\(_recordAddress)" - return "<\(typeName) \(recordAddress)> { kind: \(kind), context: \(context) }" + switch _recordStorage { + case let .atAddress(recordAddress): + let recordAddress = imageAddress.map { imageAddress in + let recordAddressDelta = UnsafeRawPointer(recordAddress) - imageAddress + return "\(imageAddress)+0x\(String(recordAddressDelta, radix: 16))" + } ?? "\(recordAddress)" + return "<\(typeName) \(recordAddress)> { kind: \(kind), context: \(context) }" + case .inline: + return "<\(typeName)> { kind: \(kind), context: \(context) }" + } } } @@ -216,19 +248,99 @@ extension DiscoverableAsTestContent where Self: ~Copyable { private import _TestingInternals -/// Get all types known to Swift found in the current process whose names -/// contain a given substring. +/// A protocol describing a type, emitted at compile time or macro expansion +/// time, that represents a single test content record. /// -/// - Parameters: -/// - nameSubstring: A string which the names of matching classes all contain. +/// Use this protocol to make discoverable any test content records contained in +/// the type metadata section (the "legacy discovery mechanism"). For example, +/// if you have creasted a test content record named `myRecord` and your test +/// content record typealias is named `MyRecordType`: +/// +/// ```swift +/// private enum MyRecordContainer: TestContentRecordContainer { +/// nonisolated static func storeTestContentRecord(to outTestContentRecord: UnsafeMutableRawPointer) -> Bool { +/// outTestContentRecord.initializeMemory(as: MyRecordType.self, to: myRecord) +/// return true +/// } +/// } +/// ``` +/// +/// Then, at discovery time, call ``DiscoverableAsTestContent/allTypeMetadataBasedTestContentRecords()`` +/// to look up `myRecord`. /// -/// - Returns: A sequence of Swift types whose names contain `nameSubstring`. +/// Types that represent test content and that should be discoverable at runtime +/// should not conform to this protocol. Instead, they should conform to +/// ``DiscoverableAsTestContent``. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_alwaysEmitConformanceMetadata @available(swift, deprecated: 100000.0, message: "Do not adopt this functionality in new code. It will be removed in a future release.") -public func types(withNamesContaining nameSubstring: String) -> some Sequence { - SectionBounds.all(.typeMetadata).lazy.flatMap { sb in - stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy - .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: nameSubstring) } - .map { unsafeBitCast($0, to: Any.Type.self) } +public protocol TestContentRecordContainer { + /// Store this container's corresponding test content record to memory. + /// + /// - Parameters: + /// - outTestContentRecord: A pointer to uninitialized memory large enough + /// to hold a test content record. The memory is untyped so that client + /// code can use a custom definition of the test content record tuple + /// type. + /// + /// - Returns: Whether or not `outTestContentRecord` was initialized. If this + /// function returns `true`, the caller is responsible for deinitializing + /// said memory after it is done using it. + nonisolated static func storeTestContentRecord(to outTestContentRecord: UnsafeMutableRawPointer) -> Bool +} + +extension DiscoverableAsTestContent where Self: ~Copyable { + /// Make a test content record of this type from the given test content record + /// container type if it matches this type's requirements. + /// + /// - Parameters: + /// - containerType: The test content record container type. + /// - sb: The section bounds containing `containerType` and, thus, the test + /// content record. + /// + /// - Returns: A new test content record value, or `nil` if `containerType` + /// failed to store a record or if the record's kind did not match this + /// type's ``testContentKind`` property. + private static func _makeTestContentRecord(from containerType: (some TestContentRecordContainer).Type, in sb: SectionBounds) -> TestContentRecord? { + withUnsafeTemporaryAllocation(of: _TestContentRecord.self, capacity: 1) { buffer in + // Load the record from the container type. + guard containerType.storeTestContentRecord(to: buffer.baseAddress!) else { + return nil + } + let record = buffer.baseAddress!.move() + + // Make sure that the record's kind matches. + guard record.kind == Self.testContentKind else { + return nil + } + + // Construct the TestContentRecord instance from the record. + return TestContentRecord(imageAddress: sb.imageAddress, record: record) + } + } + + /// Get all test content of this type known to Swift and found in the current + /// process using the legacy discovery mechanism. + /// + /// - Returns: A sequence of instances of ``TestContentRecord``. Only test + /// content records matching this ``TestContent`` type's requirements are + /// included in the sequence. + /// + /// @Comment { + /// - Bug: This function returns an instance of `AnySequence` instead of an + /// opaque type due to a compiler crash. ([143080508](rdar://143080508)) + /// } + @available(swift, deprecated: 100000.0, message: "Do not adopt this functionality in new code. It will be removed in a future release.") + public static func allTypeMetadataBasedTestContentRecords() -> AnySequence> { + validateMemoryLayout() + + let result = SectionBounds.all(.typeMetadata).lazy.flatMap { sb in + stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy + .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: "__🟠$") } + .map { unsafeBitCast($0, to: Any.Type.self) } + .compactMap { $0 as? any TestContentRecordContainer.Type } + .compactMap { _makeTestContentRecord(from: $0, in: sb) } + } + return AnySequence(result) } } diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index 96eb9075c..6ac3542a9 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -408,7 +408,7 @@ struct TestDeclarationMacroTests { func differentFunctionTypes(input: String, expectedTypeName: String?, otherCode: String?) throws { let (output, _) = try parse(input) - #expect(output.contains("__TestContainer")) + #expect(output.contains("__TestContentRecordContainer")) if let expectedTypeName { #expect(output.contains(expectedTypeName)) } diff --git a/Tests/TestingTests/TypeNameConflictTests.swift b/Tests/TestingTests/TypeNameConflictTests.swift index e3698cb4f..7a0bc7961 100644 --- a/Tests/TestingTests/TypeNameConflictTests.swift +++ b/Tests/TestingTests/TypeNameConflictTests.swift @@ -27,7 +27,7 @@ struct TypeNameConflictTests { // MARK: - Fixtures fileprivate struct SourceLocation {} -fileprivate struct __TestContainer {} +fileprivate struct __TestContentRecordContainer {} fileprivate struct __XCTestCompatibleSelector {} fileprivate func __forward(_ value: R) async throws { From 1a3f430f2510dc133b817e9b8df7a93bea000890 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Mar 2025 13:59:44 -0400 Subject: [PATCH 121/234] Incorporate feedback; use a different emoji to avoid namespace stompin' on Xcode 16 symbols --- Sources/Testing/ExitTests/ExitTest.swift | 2 +- Sources/TestingMacros/ConditionMacro.swift | 2 +- Sources/TestingMacros/SuiteDeclarationMacro.swift | 2 +- Sources/TestingMacros/TestDeclarationMacro.swift | 2 +- Sources/_TestDiscovery/TestContentRecord.swift | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index d8c254522..5b800f0c0 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -248,9 +248,9 @@ extension ExitTest: DiscoverableAsTestContent { /// Store the exit test into the given memory. /// /// - Parameters: - /// - outValue: The uninitialized memory to store the exit test into. /// - id: The unique identifier of the exit test to store. /// - body: The body closure of the exit test to store. + /// - outValue: The uninitialized memory to store the exit test into. /// - typeAddress: A pointer to the expected type of the exit test as passed /// to the test content record calling this function. /// - hintAddress: A pointer to an instance of ``ID`` to use as a hint. diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index c1c04069b..01cac9a3a 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -452,7 +452,7 @@ extension ExitTestConditionMacro { // Create a local type that can be discovered at runtime and which contains // the exit test body. - let className = context.makeUniqueName("__🟠$") + let className = context.makeUniqueName("__🟡$") let testContentRecordDecl = makeTestContentRecordDecl( named: .identifier("testContentRecord"), in: TypeSyntax(IdentifierTypeSyntax(name: className)), diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index a65a7a420..be08de84a 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -161,7 +161,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { ) // Emit a type that contains a reference to the test content record. - let className = context.makeUniqueName("__🟠$") + let className = context.makeUniqueName("__🟡$") result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index c085ffc42..9ec56af90 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -497,7 +497,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { ) // Emit a type that contains a reference to the test content record. - let className = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟠$") + let className = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟡$") result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index ed5c42238..e2bdd7830 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -336,7 +336,7 @@ extension DiscoverableAsTestContent where Self: ~Copyable { let result = SectionBounds.all(.typeMetadata).lazy.flatMap { sb in stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy - .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: "__🟠$") } + .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: "__🟡$") } .map { unsafeBitCast($0, to: Any.Type.self) } .compactMap { $0 as? any TestContentRecordContainer.Type } .compactMap { _makeTestContentRecord(from: $0, in: sb) } From ffedee989400abf791777a5723b84099048baf3a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Mar 2025 14:34:39 -0400 Subject: [PATCH 122/234] Don't make emitted classes private --- Sources/TestingMacros/SuiteDeclarationMacro.swift | 2 +- Sources/TestingMacros/TestDeclarationMacro.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index be08de84a..b47109291 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -165,7 +165,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - private final class \(className): Testing.__TestContentRecordContainer { + final class \(className): Testing.__TestContentRecordContainer { override nonisolated class var __testContentRecord: Testing.__TestContentRecord { \(testContentRecordName) } diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 9ec56af90..21faed78f 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -501,7 +501,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - private final class \(className): Testing.__TestContentRecordContainer { + final class \(className): Testing.__TestContentRecordContainer { override nonisolated class var __testContentRecord: Testing.__TestContentRecord { \(testContentRecordName) } From f5690dca86fcf76e5704ade5ccfa9a33487d203e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 10 Mar 2025 14:54:11 -0400 Subject: [PATCH 123/234] Tweaks --- Sources/Testing/Test+Discovery+Legacy.swift | 1 + Sources/Testing/Test+Discovery.swift | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index f974e3829..0be944ad9 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -15,6 +15,7 @@ /// `_TestDiscovery` module to appear in `Testing.private.swiftinterface`. /// /// This protocol is not part of the public interface of the testing library. +@_alwaysEmitConformanceMetadata protocol TestContentRecordContainer: _TestDiscovery.TestContentRecordContainer {} /// An abstract base class describing a type that contains tests. diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 5c2d86f32..a8cc831c4 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -19,7 +19,7 @@ extension Test { /// indirect `async` accessor function rather than directly producing /// instances of ``Test``, but functions are non-nominal types and cannot /// directly conform to protocols. - struct Generator: DiscoverableAsTestContent, RawRepresentable { + fileprivate struct Generator: DiscoverableAsTestContent, RawRepresentable { static var testContentKind: UInt32 { 0x74657374 } From a5dfbc2507eb7b745f164ec52bc5f6d92b9c38e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 10 Mar 2025 20:07:56 +0100 Subject: [PATCH 124/234] [CMake] Set CMP0157 to OLD when targeting Android with the Windows toolchain (#1009) There is no early swift-driver build for the Windows toolchain. As a result, swift-testing fails to build properly when CMP0157 is set to NEW due to object files not being generated. This sets CMP0157 to OLD when targeting Android with the Windows toolchain until the early swift-driver is available on Windows. This is analog to https://github.com/swiftlang/swift-corelibs-foundation/pull/5180 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80c922eb2..c59e2f35d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,14 @@ cmake_minimum_required(VERSION 3.19.6...3.29) if(POLICY CMP0157) - cmake_policy(SET CMP0157 NEW) + if(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows AND CMAKE_SYSTEM_NAME STREQUAL Android) + # CMP0157 causes builds to fail when targetting Android with the Windows + # toolchain, because the early swift-driver isn't (yet) available. Disable + # it for now. + cmake_policy(SET CMP0157 OLD) + else() + cmake_policy(SET CMP0157 NEW) + endif() endif() project(SwiftTesting From 7d33b87488175af5a2b832fa99e547dbc27ca261 Mon Sep 17 00:00:00 2001 From: Kyle Date: Mon, 10 Mar 2025 17:37:38 +0800 Subject: [PATCH 125/234] Add ConditionTraitTests to cover the Swift 6.0 compiler issue for custom trait + closure + macro See https://github.com/swiftlang/swift-testing/issues/1006#issuecomment-2708665563 --- .../Traits/ConditionTraitTests.swift | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 Tests/TestingTests/Traits/ConditionTraitTests.swift diff --git a/Tests/TestingTests/Traits/ConditionTraitTests.swift b/Tests/TestingTests/Traits/ConditionTraitTests.swift new file mode 100644 index 000000000..5d70c8e87 --- /dev/null +++ b/Tests/TestingTests/Traits/ConditionTraitTests.swift @@ -0,0 +1,44 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable import Testing + +@Suite("Condition Trait Tests", .tags(.traitRelated)) +struct ConditionTraitTests { + #if compiler(>=6.1) + @Test( + ".enabled trait", + .enabled { true }, + .bug("https://github.com/swiftlang/swift/issues/76409", "Verify the custom trait with closure causes @Test macro to fail is fixed") + ) + func enabledTraitClosure() throws {} + #endif + + @Test( + ".enabled if trait", + .enabled(if: true) + ) + func enabledTraitIf() throws {} + + #if compiler(>=6.1) + @Test( + ".disabled trait", + .disabled { false }, + .bug("https://github.com/swiftlang/swift/issues/76409", "Verify the custom trait with closure causes @Test macro to fail is fixed") + ) + func disabledTraitClosure() throws {} + #endif + + @Test( + ".disabled if trait", + .disabled(if: false) + ) + func disabledTraitIf() throws {} +} From 1e55aa7ef7628c6dded55871484864323fe66e91 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 11 Mar 2025 15:10:23 -0400 Subject: [PATCH 126/234] Store test content in a custom metadata section. (#880) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR uses the experimental symbol linkage margers feature in the Swift compiler to emit metadata about tests (and exit tests) into a dedicated section of the test executable being built. At runtime, we discover that section and read out the tests from it. This has several benefits over our current model, which involves walking Swift's type metadata table looking for types that conform to a protocol: 1. We don't need to define that protocol as public API in Swift Testing, 1. We don't need to emit type metadata (much larger than what we really need) for every test function, 1. We don't need to duplicate a large chunk of the Swift ABI sources in order to walk the type metadata table correctly, and 1. Almost all the new code is written in Swift, whereas the code it is intended to replace could not be fully represented in Swift and needed to be written in C++. This change will be necessary to support Embedded Swift because there is no type metadata section emitted for embedded targets. The change also opens up the possibility of supporting generic types in the future because we can emit metadata without needing to emit a nested type (which is not always valid in a generic context.) That's a "future direction" and not covered by this PR specifically. I've defined a layout for entries in the new `swift5_tests` section that should be flexible enough for us in the short-to-medium term and which lets us define additional arbitrary test content record types. The layout of this section is covered in depth in the new [TestContent.md](https://github.com/swiftlang/swift-testing/blob/main/Documentation/ABI/TestContent.md) article. This functionality is only available if a test target enables the experimental `"SymbolLinkageMarkers"` feature and only if Swift Testing is used as a package (not as a toolchain component.) We continue to emit protocol-conforming types for now—that code will be removed if and when the experimental feature is properly supported (modulo us adopting relevant changes to the feature's API.) ## See Also https://github.com/swiftlang/swift-testing/issues/735 https://github.com/swiftlang/swift-testing/issues/764 https://github.com/swiftlang/swift/issues/76698 https://github.com/swiftlang/swift/pull/78411 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/Porting.md | 8 +- Package.swift | 10 ++- Sources/Testing/ExitTests/ExitTest.swift | 2 + Sources/Testing/Test+Discovery+Legacy.swift | 2 + Sources/Testing/Test+Discovery.swift | 6 ++ Sources/TestingMacros/ConditionMacro.swift | 75 ++++++++++++------- .../TestingMacros/SuiteDeclarationMacro.swift | 6 ++ .../Support/TestContentGeneration.swift | 12 +++ .../TestingMacros/TestDeclarationMacro.swift | 19 ++--- Sources/_TestDiscovery/SectionBounds.swift | 8 ++ .../_TestDiscovery/TestContentRecord.swift | 2 + Sources/_TestingInternals/Discovery.cpp | 12 +++ .../TestDeclarationMacroTests.swift | 5 ++ Tests/TestingTests/MiscellaneousTests.swift | 17 +++++ 14 files changed, 140 insertions(+), 44 deletions(-) diff --git a/Documentation/Porting.md b/Documentation/Porting.md index ce179d53d..8b230ff22 100644 --- a/Documentation/Porting.md +++ b/Documentation/Porting.md @@ -145,8 +145,10 @@ to load that information: + let resourceName: Str255 = switch kind { + case .testContent: + "__swift5_tests" ++#if !SWT_NO_LEGACY_TEST_DISCOVERY + case .typeMetadata: + "__swift5_types" ++#endif + } + + let oldRefNum = CurResFile() @@ -219,14 +221,18 @@ diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals +#elif defined(macintosh) +extern "C" const char testContentSectionBegin __asm__("..."); +extern "C" const char testContentSectionEnd __asm__("..."); ++#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) +extern "C" const char typeMetadataSectionBegin __asm__("..."); +extern "C" const char typeMetadataSectionEnd __asm__("..."); ++#endif #else #warning Platform-specific implementation missing: Runtime test discovery unavailable (static) static const char testContentSectionBegin = 0; static const char& testContentSectionEnd = testContentSectionBegin; + #if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) static const char typeMetadataSectionBegin = 0; - static const char& typeMetadataSectionEnd = testContentSectionBegin; + static const char& typeMetadataSectionEnd = typeMetadataSectionBegin; + #endif #endif ``` diff --git a/Package.swift b/Package.swift index 11adfc14e..e2257af1a 100644 --- a/Package.swift +++ b/Package.swift @@ -89,10 +89,7 @@ let package = Package( "_Testing_CoreGraphics", "_Testing_Foundation", ], - swiftSettings: .packageSettings + [ - // For testing test content section discovery only - .enableExperimentalFeature("SymbolLinkageMarkers"), - ] + swiftSettings: .packageSettings ), .macro( @@ -205,6 +202,11 @@ extension Array where Element == PackageDescription.SwiftSetting { .enableExperimentalFeature("AccessLevelOnImport"), .enableUpcomingFeature("InternalImportsByDefault"), + // This setting is enabled in the package, but not in the toolchain build + // (via CMake). Enabling it is dependent on acceptance of the @section + // proposal via Swift Evolution. + .enableExperimentalFeature("SymbolLinkageMarkers"), + // When building as a package, the macro plugin always builds as an // executable rather than a library. .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 5b800f0c0..61b35b9fd 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -296,12 +296,14 @@ extension ExitTest { } } +#if !SWT_NO_LEGACY_TEST_DISCOVERY // Call the legacy lookup function that discovers tests embedded in types. for record in Self.allTypeMetadataBasedTestContentRecords() { if let exitTest = record.load(withHint: id) { return exitTest } } +#endif return nil } diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index 0be944ad9..711f95e73 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -8,6 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +#if !SWT_NO_LEGACY_TEST_DISCOVERY @_spi(Experimental) @_spi(ForToolsIntegrationOnly) internal import _TestDiscovery /// A shadow declaration of `_TestDiscovery.TestContentRecordContainer` that @@ -41,3 +42,4 @@ open class __TestContentRecordContainer: TestContentRecordContainer { @available(*, unavailable) extension __TestContentRecordContainer: Sendable {} +#endif diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index a8cc831c4..2568353f8 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -65,6 +65,7 @@ extension Test { // the legacy and new mechanisms, but we can set an environment variable // to explicitly select one or the other. When we remove legacy support, // we can also remove this enumeration and environment variable check. +#if !SWT_NO_LEGACY_TEST_DISCOVERY let (useNewMode, useLegacyMode) = switch Environment.flag(named: "SWT_USE_LEGACY_TEST_DISCOVERY") { case .none: (true, true) @@ -73,6 +74,9 @@ extension Test { case .some(false): (true, false) } +#else + let useNewMode = true +#endif // Walk all test content and gather generator functions, then call them in // a task group and collate their results. @@ -86,6 +90,7 @@ extension Test { } } +#if !SWT_NO_LEGACY_TEST_DISCOVERY // Perform legacy test discovery if needed. if useLegacyMode && result.isEmpty { let generators = Generator.allTypeMetadataBasedTestContentRecords().lazy.compactMap { $0.load() } @@ -96,6 +101,7 @@ extension Test { result = await taskGroup.reduce(into: result) { $0.insert($1) } } } +#endif return result } diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 01cac9a3a..43d28aa53 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -11,6 +11,10 @@ public import SwiftSyntax public import SwiftSyntaxMacros +#if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY +#error("Platform-specific misconfiguration: either SymbolLinkageMarkers or legacy test discovery is required to expand #expect(exitsWith:)") +#endif + /// A protocol containing the common implementation for the expansions of the /// `#expect()` and `#require()` macros. /// @@ -452,42 +456,59 @@ extension ExitTestConditionMacro { // Create a local type that can be discovered at runtime and which contains // the exit test body. - let className = context.makeUniqueName("__🟡$") - let testContentRecordDecl = makeTestContentRecordDecl( - named: .identifier("testContentRecord"), - in: TypeSyntax(IdentifierTypeSyntax(name: className)), - ofKind: .exitTest, - accessingWith: .identifier("accessor") - ) - - decls.append( - """ - @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - final class \(className): Testing.__TestContentRecordContainer { - private nonisolated static let accessor: Testing.__TestContentRecordAccessor = { outValue, type, hint in - Testing.ExitTest.__store( - \(exitTestIDExpr), - \(bodyThunkName), - into: outValue, - asTypeAt: type, - withHintAt: hint - ) - } - - \(testContentRecordDecl) + let enumName = context.makeUniqueName("") + do { + // Create the test content record. + let testContentRecordDecl = makeTestContentRecordDecl( + named: .identifier("testContentRecord"), + in: TypeSyntax(IdentifierTypeSyntax(name: enumName)), + ofKind: .exitTest, + accessingWith: .identifier("accessor") + ) + // Create another local type for legacy test discovery. + var recordDecl: DeclSyntax? +#if !SWT_NO_LEGACY_TEST_DISCOVERY + let className = context.makeUniqueName("__🟡$") + recordDecl = """ + private final class \(className): Testing.__TestContentRecordContainer { override nonisolated class var __testContentRecord: Testing.__TestContentRecord { - testContentRecord + \(enumName).testContentRecord } } """ - ) +#endif + + decls.append( + """ + @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") + enum \(enumName) { + private nonisolated static let accessor: Testing.__TestContentRecordAccessor = { outValue, type, hint in + Testing.ExitTest.__store( + \(exitTestIDExpr), + \(bodyThunkName), + into: outValue, + asTypeAt: type, + withHintAt: hint + ) + } + + \(testContentRecordDecl) + + \(recordDecl) + } + """ + ) + } arguments[trailingClosureIndex].expression = ExprSyntax( ClosureExprSyntax { for decl in decls { - CodeBlockItemSyntax(item: .decl(decl)) - .with(\.trailingTrivia, .newline) + CodeBlockItemSyntax( + leadingTrivia: .newline, + item: .decl(decl), + trailingTrivia: .newline + ) } } ) diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index b47109291..c90606577 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -11,6 +11,10 @@ public import SwiftSyntax public import SwiftSyntaxMacros +#if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY +#error("Platform-specific misconfiguration: either SymbolLinkageMarkers or legacy test discovery is required to expand @Suite") +#endif + /// A type describing the expansion of the `@Suite` attribute macro. /// /// This type is used to implement the `@Suite` attribute macro. Do not use it @@ -160,6 +164,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { ) ) +#if !SWT_NO_LEGACY_TEST_DISCOVERY // Emit a type that contains a reference to the test content record. let className = context.makeUniqueName("__🟡$") result.append( @@ -172,6 +177,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { } """ ) +#endif return result } diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index 646bd97d4..391a468b1 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -62,6 +62,18 @@ func makeTestContentRecordDecl(named name: TokenSyntax, in typeName: TypeSyntax? } return """ + #if hasFeature(SymbolLinkageMarkers) + #if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) + @_section("__DATA_CONST,__swift5_tests") + #elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) + @_section("swift5_tests") + #elseif os(Windows) + @_section(".sw5test$B") + #else + @__testing(warning: "Platform-specific implementation missing: test content section name unavailable") + #endif + @_used + #endif @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") private nonisolated \(staticKeyword(for: typeName)) let \(name): Testing.__TestContentRecord = ( \(kindExpr), \(kind.commentRepresentation) diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 21faed78f..3c6b12fe0 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -11,6 +11,10 @@ public import SwiftSyntax public import SwiftSyntaxMacros +#if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY +#error("Platform-specific misconfiguration: either SymbolLinkageMarkers or legacy test discovery is required to expand @Test") +#endif + /// A type describing the expansion of the `@Test` attribute macro. /// /// This type is used to implement the `@Test` attribute macro. Do not use it @@ -188,17 +192,6 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { return FunctionParameterClauseSyntax(parameters: parameterList) } - /// The `static` keyword, if `typeName` is not `nil`. - /// - /// - Parameters: - /// - typeName: The name of the type containing the macro being expanded. - /// - /// - Returns: A token representing the `static` keyword, or one representing - /// nothing if `typeName` is `nil`. - private static func _staticKeyword(for typeName: TypeSyntax?) -> TokenSyntax { - (typeName != nil) ? .keyword(.static) : .unknown("") - } - /// Create a thunk function with a normalized signature that calls a /// developer-supplied test function. /// @@ -356,7 +349,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { let thunkName = context.makeUniqueName(thunking: functionDecl) let thunkDecl: DeclSyntax = """ @available(*, deprecated, message: "This function is an implementation detail of the testing library. Do not use it directly.") - @Sendable private \(_staticKeyword(for: typeName)) func \(thunkName)\(thunkParamsExpr) async throws -> Void { + @Sendable private \(staticKeyword(for: typeName)) func \(thunkName)\(thunkParamsExpr) async throws -> Void { \(thunkBody) } """ @@ -496,6 +489,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { ) ) +#if !SWT_NO_LEGACY_TEST_DISCOVERY // Emit a type that contains a reference to the test content record. let className = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟡$") result.append( @@ -508,6 +502,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } """ ) +#endif return result } diff --git a/Sources/_TestDiscovery/SectionBounds.swift b/Sources/_TestDiscovery/SectionBounds.swift index 1fc379258..212edbfbf 100644 --- a/Sources/_TestDiscovery/SectionBounds.swift +++ b/Sources/_TestDiscovery/SectionBounds.swift @@ -27,8 +27,10 @@ struct SectionBounds: Sendable { /// The test content metadata section. case testContent +#if !SWT_NO_LEGACY_TEST_DISCOVERY /// The type metadata section. case typeMetadata +#endif } /// All section bounds of the given kind found in the current process. @@ -60,8 +62,10 @@ extension SectionBounds.Kind { switch self { case .testContent: ("__DATA_CONST", "__swift5_tests") +#if !SWT_NO_LEGACY_TEST_DISCOVERY case .typeMetadata: ("__TEXT", "__swift5_types") +#endif } } } @@ -186,8 +190,10 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { let range = switch context.pointee.kind { case .testContent: sections.swift5_tests +#if !SWT_NO_LEGACY_TEST_DISCOVERY case .typeMetadata: sections.swift5_type_metadata +#endif } let start = UnsafeRawPointer(bitPattern: range.start) let size = Int(clamping: range.length) @@ -276,8 +282,10 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> some Sequence
+#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) #include #include #include +#endif #if defined(SWT_NO_DYNAMIC_LINKING) #pragma mark - Statically-linked section bounds @@ -21,24 +23,32 @@ #if defined(__APPLE__) extern "C" const char testContentSectionBegin __asm("section$start$__DATA_CONST$__swift5_tests"); extern "C" const char testContentSectionEnd __asm("section$end$__DATA_CONST$__swift5_tests"); +#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) extern "C" const char typeMetadataSectionBegin __asm__("section$start$__TEXT$__swift5_types"); extern "C" const char typeMetadataSectionEnd __asm__("section$end$__TEXT$__swift5_types"); +#endif #elif defined(__wasi__) extern "C" const char testContentSectionBegin __asm__("__start_swift5_tests"); extern "C" const char testContentSectionEnd __asm__("__stop_swift5_tests"); +#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) extern "C" const char typeMetadataSectionBegin __asm__("__start_swift5_type_metadata"); extern "C" const char typeMetadataSectionEnd __asm__("__stop_swift5_type_metadata"); +#endif #else #warning Platform-specific implementation missing: Runtime test discovery unavailable (static) static const char testContentSectionBegin = 0; static const char& testContentSectionEnd = testContentSectionBegin; +#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) static const char typeMetadataSectionBegin = 0; static const char& typeMetadataSectionEnd = typeMetadataSectionBegin; #endif +#endif static constexpr const char *const staticallyLinkedSectionBounds[][2] = { { &testContentSectionBegin, &testContentSectionEnd }, +#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) { &typeMetadataSectionBegin, &typeMetadataSectionEnd }, +#endif }; void swt_getStaticallyLinkedSectionBounds(size_t kind, const void **outSectionBegin, size_t *outByteCount) { @@ -48,6 +58,7 @@ void swt_getStaticallyLinkedSectionBounds(size_t kind, const void **outSectionBe } #endif +#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) #pragma mark - Swift ABI #if defined(__PTRAUTH_INTRINSICS__) @@ -221,3 +232,4 @@ const void *swt_getTypeFromTypeMetadataRecord(const void *recordAddress, const c return nullptr; } +#endif diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index 6ac3542a9..b0028c438 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -408,7 +408,12 @@ struct TestDeclarationMacroTests { func differentFunctionTypes(input: String, expectedTypeName: String?, otherCode: String?) throws { let (output, _) = try parse(input) +#if hasFeature(SymbolLinkageMarkers) + #expect(output.contains("@_section")) +#endif +#if !SWT_NO_LEGACY_TEST_DISCOVERY #expect(output.contains("__TestContentRecordContainer")) +#endif if let expectedTypeName { #expect(output.contains(expectedTypeName)) } diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index ec7fdd12d..a6c62fdbc 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -664,4 +664,21 @@ struct MiscellaneousTests { }) } #endif + +#if !SWT_NO_LEGACY_TEST_DISCOVERY && hasFeature(SymbolLinkageMarkers) + @Test("Legacy test discovery finds the same number of tests") func discoveredTestCount() async { + let oldFlag = Environment.variable(named: "SWT_USE_LEGACY_TEST_DISCOVERY") + defer { + Environment.setVariable(oldFlag, named: "SWT_USE_LEGACY_TEST_DISCOVERY") + } + + Environment.setVariable("1", named: "SWT_USE_LEGACY_TEST_DISCOVERY") + let testsWithOldCode = await Array(Test.all).count + + Environment.setVariable("0", named: "SWT_USE_LEGACY_TEST_DISCOVERY") + let testsWithNewCode = await Array(Test.all).count + + #expect(testsWithOldCode == testsWithNewCode) + } +#endif } From 87ceeb4d5429bb4b6a05e5c41619f4bf06e1a1a8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 11 Mar 2025 20:56:25 -0400 Subject: [PATCH 127/234] Fix crash when loading test content records from the type metadata section. (#1015) Looks like #880 and/or #1010 caused a regression: the compiler appears to be dead-code-stripping the classes we emit to contain test content records. This PR changes the design back to using a protocol so that the members we need are always covered by `@_alwaysEmitConformanceMetadata`. This makes `_TestDiscovery` a little harder to use with legacy lookup, but it's all experimental and eventually going to be removed anyway. Resolves rdar://146809312. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Test+Discovery+Legacy.swift | 42 ++++---- Sources/TestingMacros/ConditionMacro.swift | 6 +- .../TestingMacros/SuiteDeclarationMacro.swift | 6 +- .../TestingMacros/TestDeclarationMacro.swift | 6 +- .../DiscoverableAsTestContent.swift | 13 +++ .../_TestDiscovery/TestContentRecord.swift | 100 +++++------------- 6 files changed, 70 insertions(+), 103 deletions(-) diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index 711f95e73..a3374b315 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -11,35 +11,37 @@ #if !SWT_NO_LEGACY_TEST_DISCOVERY @_spi(Experimental) @_spi(ForToolsIntegrationOnly) internal import _TestDiscovery -/// A shadow declaration of `_TestDiscovery.TestContentRecordContainer` that -/// allows us to add public conformances to it without causing the -/// `_TestDiscovery` module to appear in `Testing.private.swiftinterface`. -/// -/// This protocol is not part of the public interface of the testing library. -@_alwaysEmitConformanceMetadata -protocol TestContentRecordContainer: _TestDiscovery.TestContentRecordContainer {} - -/// An abstract base class describing a type that contains tests. +/// A protocol base class describing a type that contains tests. /// /// - Warning: This class is used to implement the `@Test` macro. Do not use it /// directly. -open class __TestContentRecordContainer: TestContentRecordContainer { - /// The corresponding test content record. +@_alwaysEmitConformanceMetadata +public protocol __TestContentRecordContainer { + /// The test content record associated with this container. /// /// - Warning: This property is used to implement the `@Test` macro. Do not /// use it directly. - open nonisolated class var __testContentRecord: __TestContentRecord { - (0, 0, nil, 0, 0) - } + nonisolated static var __testContentRecord: __TestContentRecord { get } +} + +extension DiscoverableAsTestContent where Self: ~Copyable { + /// Get all test content of this type known to Swift and found in the current + /// process using the legacy discovery mechanism. + /// + /// - Returns: A sequence of instances of ``TestContentRecord``. Only test + /// content records matching this ``TestContent`` type's requirements are + /// included in the sequence. + static func allTypeMetadataBasedTestContentRecords() -> AnySequence> { + return allTypeMetadataBasedTestContentRecords { type, buffer in + guard let type = type as? any __TestContentRecordContainer.Type else { + return false + } - static func storeTestContentRecord(to outTestContentRecord: UnsafeMutableRawPointer) -> Bool { - outTestContentRecord.withMemoryRebound(to: __TestContentRecord.self, capacity: 1) { outTestContentRecord in - outTestContentRecord.initialize(to: __testContentRecord) + buffer.withMemoryRebound(to: __TestContentRecord.self) { buffer in + buffer.baseAddress!.initialize(to: type.__testContentRecord) + } return true } } } - -@available(*, unavailable) -extension __TestContentRecordContainer: Sendable {} #endif diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 43d28aa53..b7a1526c1 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -469,10 +469,10 @@ extension ExitTestConditionMacro { // Create another local type for legacy test discovery. var recordDecl: DeclSyntax? #if !SWT_NO_LEGACY_TEST_DISCOVERY - let className = context.makeUniqueName("__🟡$") + let legacyEnumName = context.makeUniqueName("__🟡$") recordDecl = """ - private final class \(className): Testing.__TestContentRecordContainer { - override nonisolated class var __testContentRecord: Testing.__TestContentRecord { + enum \(legacyEnumName): Testing.__TestContentRecordContainer { + nonisolated static var __testContentRecord: Testing.__TestContentRecord { \(enumName).testContentRecord } } diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index c90606577..791d08b82 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -166,12 +166,12 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { #if !SWT_NO_LEGACY_TEST_DISCOVERY // Emit a type that contains a reference to the test content record. - let className = context.makeUniqueName("__🟡$") + let enumName = context.makeUniqueName("__🟡$") result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - final class \(className): Testing.__TestContentRecordContainer { - override nonisolated class var __testContentRecord: Testing.__TestContentRecord { + enum \(enumName): Testing.__TestContentRecordContainer { + nonisolated static var __testContentRecord: Testing.__TestContentRecord { \(testContentRecordName) } } diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 3c6b12fe0..c94769d21 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -491,12 +491,12 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { #if !SWT_NO_LEGACY_TEST_DISCOVERY // Emit a type that contains a reference to the test content record. - let className = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟡$") + let enumName = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟡$") result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") - final class \(className): Testing.__TestContentRecordContainer { - override nonisolated class var __testContentRecord: Testing.__TestContentRecord { + enum \(enumName): Testing.__TestContentRecordContainer { + nonisolated static var __testContentRecord: Testing.__TestContentRecord { \(testContentRecordName) } } diff --git a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift index 719b8dbe9..96a8c7698 100644 --- a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift +++ b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift @@ -39,4 +39,17 @@ public protocol DiscoverableAsTestContent: Sendable, ~Copyable { /// By default, this type equals `Never`, indicating that this type of test /// content does not support hinting during discovery. associatedtype TestContentAccessorHint = Never + +#if !SWT_NO_LEGACY_TEST_DISCOVERY + /// A string present in the names of types containing test content records + /// associated with this type. + @available(swift, deprecated: 100000.0, message: "Do not adopt this functionality in new code. It will be removed in a future release.") + static var _testContentTypeNameHint: String { get } +#endif +} + +extension DiscoverableAsTestContent where Self: ~Copyable { + public static var _testContentTypeNameHint: String { + "__🟡$" + } } diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index 8fee1827b..4c7ec8d4d 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -249,80 +249,17 @@ extension DiscoverableAsTestContent where Self: ~Copyable { private import _TestingInternals -/// A protocol describing a type, emitted at compile time or macro expansion -/// time, that represents a single test content record. -/// -/// Use this protocol to make discoverable any test content records contained in -/// the type metadata section (the "legacy discovery mechanism"). For example, -/// if you have creasted a test content record named `myRecord` and your test -/// content record typealias is named `MyRecordType`: -/// -/// ```swift -/// private enum MyRecordContainer: TestContentRecordContainer { -/// nonisolated static func storeTestContentRecord(to outTestContentRecord: UnsafeMutableRawPointer) -> Bool { -/// outTestContentRecord.initializeMemory(as: MyRecordType.self, to: myRecord) -/// return true -/// } -/// } -/// ``` -/// -/// Then, at discovery time, call ``DiscoverableAsTestContent/allTypeMetadataBasedTestContentRecords()`` -/// to look up `myRecord`. -/// -/// Types that represent test content and that should be discoverable at runtime -/// should not conform to this protocol. Instead, they should conform to -/// ``DiscoverableAsTestContent``. -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) -@_alwaysEmitConformanceMetadata -@available(swift, deprecated: 100000.0, message: "Do not adopt this functionality in new code. It will be removed in a future release.") -public protocol TestContentRecordContainer { - /// Store this container's corresponding test content record to memory. - /// - /// - Parameters: - /// - outTestContentRecord: A pointer to uninitialized memory large enough - /// to hold a test content record. The memory is untyped so that client - /// code can use a custom definition of the test content record tuple - /// type. - /// - /// - Returns: Whether or not `outTestContentRecord` was initialized. If this - /// function returns `true`, the caller is responsible for deinitializing - /// said memory after it is done using it. - nonisolated static func storeTestContentRecord(to outTestContentRecord: UnsafeMutableRawPointer) -> Bool -} - extension DiscoverableAsTestContent where Self: ~Copyable { - /// Make a test content record of this type from the given test content record - /// container type if it matches this type's requirements. - /// - /// - Parameters: - /// - containerType: The test content record container type. - /// - sb: The section bounds containing `containerType` and, thus, the test - /// content record. - /// - /// - Returns: A new test content record value, or `nil` if `containerType` - /// failed to store a record or if the record's kind did not match this - /// type's ``testContentKind`` property. - private static func _makeTestContentRecord(from containerType: (some TestContentRecordContainer).Type, in sb: SectionBounds) -> TestContentRecord? { - withUnsafeTemporaryAllocation(of: _TestContentRecord.self, capacity: 1) { buffer in - // Load the record from the container type. - guard containerType.storeTestContentRecord(to: buffer.baseAddress!) else { - return nil - } - let record = buffer.baseAddress!.move() - - // Make sure that the record's kind matches. - guard record.kind == Self.testContentKind else { - return nil - } - - // Construct the TestContentRecord instance from the record. - return TestContentRecord(imageAddress: sb.imageAddress, record: record) - } - } - /// Get all test content of this type known to Swift and found in the current /// process using the legacy discovery mechanism. /// + /// - Parameters: + /// - baseType: The type which all discovered container types must + /// conform to or subclass. + /// - loader: A function that is called once per type conforming to or + /// subclassing `baseType`. This function should load the corresponding + /// test content record into the buffer passed to it. + /// /// - Returns: A sequence of instances of ``TestContentRecord``. Only test /// content records matching this ``TestContent`` type's requirements are /// included in the sequence. @@ -332,15 +269,30 @@ extension DiscoverableAsTestContent where Self: ~Copyable { /// opaque type due to a compiler crash. ([143080508](rdar://143080508)) /// } @available(swift, deprecated: 100000.0, message: "Do not adopt this functionality in new code. It will be removed in a future release.") - public static func allTypeMetadataBasedTestContentRecords() -> AnySequence> { + public static func allTypeMetadataBasedTestContentRecords( + loadingWith loader: @escaping @Sendable (Any.Type, UnsafeMutableRawBufferPointer) -> Bool + ) -> AnySequence> { validateMemoryLayout() + let typeNameHint = _testContentTypeNameHint + let kind = testContentKind + let loader: @Sendable (Any.Type) -> _TestContentRecord? = { type in + withUnsafeTemporaryAllocation(of: _TestContentRecord.self, capacity: 1) { buffer in + // Load the record from the container type. + guard loader(type, .init(buffer)) else { + return nil + } + return buffer.baseAddress!.move() + } + } + let result = SectionBounds.all(.typeMetadata).lazy.flatMap { sb in stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy - .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: "__🟡$") } + .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: typeNameHint) } .map { unsafeBitCast($0, to: Any.Type.self) } - .compactMap { $0 as? any TestContentRecordContainer.Type } - .compactMap { _makeTestContentRecord(from: $0, in: sb) } + .compactMap(loader) + .filter { $0.kind == kind } + .map { TestContentRecord(imageAddress: sb.imageAddress, record: $0) } } return AnySequence(result) } From 3e8add397fd9244658d7c67fa2c6b6b312b8e2ed Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 11 Mar 2025 21:06:31 -0400 Subject: [PATCH 128/234] Fix a couple of comment typos --- Sources/Testing/Test+Discovery+Legacy.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index a3374b315..8ff878338 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -11,10 +11,10 @@ #if !SWT_NO_LEGACY_TEST_DISCOVERY @_spi(Experimental) @_spi(ForToolsIntegrationOnly) internal import _TestDiscovery -/// A protocol base class describing a type that contains tests. +/// A protocol describing a type that contains tests. /// -/// - Warning: This class is used to implement the `@Test` macro. Do not use it -/// directly. +/// - Warning: This protocol is used to implement the `@Test` macro. Do not use +/// it directly. @_alwaysEmitConformanceMetadata public protocol __TestContentRecordContainer { /// The test content record associated with this container. From 77fa2613831673278de5380447996460a1526101 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Mar 2025 11:46:07 -0500 Subject: [PATCH 129/234] Enable upcoming feature 'MemberImportVisibility' and fix issues it reveals (#1020) This enables the `MemberImportVisibility` upcoming Swift feature described in [SE-0444: Member import visibility](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0444-member-import-visibility.md), then fixes the new issues that enabling it reveals. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 2 ++ Sources/Testing/ABI/ABI.Record+Streaming.swift | 2 ++ Sources/TestingMacros/ConditionMacro.swift | 1 + Sources/TestingMacros/PragmaMacro.swift | 2 ++ Sources/TestingMacros/SuiteDeclarationMacro.swift | 2 ++ .../Support/Additions/MacroExpansionContextAdditions.swift | 1 + .../Support/Additions/TokenSyntaxAdditions.swift | 1 + Sources/TestingMacros/Support/Argument.swift | 1 + Sources/TestingMacros/Support/AttributeDiscovery.swift | 1 + Sources/TestingMacros/Support/AvailabilityGuards.swift | 1 + Sources/TestingMacros/Support/CommentParsing.swift | 1 + .../TestingMacros/Support/ConditionArgumentParsing.swift | 1 + .../Support/DiagnosticMessage+Diagnosing.swift | 2 ++ Sources/TestingMacros/Support/DiagnosticMessage.swift | 2 ++ Sources/TestingMacros/Support/SourceCodeCapturing.swift | 2 ++ .../TestingMacros/Support/SourceLocationGeneration.swift | 1 + Sources/TestingMacros/Support/TestContentGeneration.swift | 1 + Sources/TestingMacros/TagMacro.swift | 1 + Sources/TestingMacros/TestDeclarationMacro.swift | 2 ++ Tests/TestingMacrosTests/PragmaMacroTests.swift | 2 ++ Tests/TestingMacrosTests/TestDeclarationMacroTests.swift | 1 + Tests/TestingMacrosTests/TestSupport/Parse.swift | 1 + Tests/TestingTests/AttachmentTests.swift | 6 ++++-- 23 files changed, 35 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index e2257af1a..3a66ef039 100644 --- a/Package.swift +++ b/Package.swift @@ -202,6 +202,8 @@ extension Array where Element == PackageDescription.SwiftSetting { .enableExperimentalFeature("AccessLevelOnImport"), .enableUpcomingFeature("InternalImportsByDefault"), + .enableUpcomingFeature("MemberImportVisibility"), + // This setting is enabled in the package, but not in the toolchain build // (via CMake). Enabling it is dependent on acceptance of the @section // proposal via Swift Evolution. diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 4c26b44ad..7b86cb438 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -9,6 +9,8 @@ // #if canImport(Foundation) && (!SWT_NO_FILE_IO || !SWT_NO_ABI_ENTRY_POINT) +private import Foundation + extension ABI.Version { /// Post-process encoded JSON and write it to a file. /// diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index b7a1526c1..616e9abbc 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -9,6 +9,7 @@ // public import SwiftSyntax +import SwiftSyntaxBuilder public import SwiftSyntaxMacros #if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY diff --git a/Sources/TestingMacros/PragmaMacro.swift b/Sources/TestingMacros/PragmaMacro.swift index 783440764..b34325e87 100644 --- a/Sources/TestingMacros/PragmaMacro.swift +++ b/Sources/TestingMacros/PragmaMacro.swift @@ -8,6 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +import SwiftDiagnostics +import SwiftParser public import SwiftSyntax public import SwiftSyntaxMacros diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index 791d08b82..8cfe5e902 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -8,7 +8,9 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +import SwiftDiagnostics public import SwiftSyntax +import SwiftSyntaxBuilder public import SwiftSyntaxMacros #if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY diff --git a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift index 8bcf2522a..3b31caf72 100644 --- a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros import SwiftDiagnostics diff --git a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift index 2be9977d5..447a18dee 100644 --- a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift @@ -8,6 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +import SwiftParser import SwiftSyntax extension TokenSyntax { diff --git a/Sources/TestingMacros/Support/Argument.swift b/Sources/TestingMacros/Support/Argument.swift index 44eeeabe7..e81e58cf7 100644 --- a/Sources/TestingMacros/Support/Argument.swift +++ b/Sources/TestingMacros/Support/Argument.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder /// A type describing an argument to a function, closure, etc. /// diff --git a/Sources/TestingMacros/Support/AttributeDiscovery.swift b/Sources/TestingMacros/Support/AttributeDiscovery.swift index 84d96cf84..a61989aef 100644 --- a/Sources/TestingMacros/Support/AttributeDiscovery.swift +++ b/Sources/TestingMacros/Support/AttributeDiscovery.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros /// A syntax rewriter that removes leading `Self.` tokens from member access diff --git a/Sources/TestingMacros/Support/AvailabilityGuards.swift b/Sources/TestingMacros/Support/AvailabilityGuards.swift index a61be5772..e9f4ba762 100644 --- a/Sources/TestingMacros/Support/AvailabilityGuards.swift +++ b/Sources/TestingMacros/Support/AvailabilityGuards.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros /// A structure describing a single platform/version pair from an `@available()` diff --git a/Sources/TestingMacros/Support/CommentParsing.swift b/Sources/TestingMacros/Support/CommentParsing.swift index 1c16c10b8..fbfc27609 100644 --- a/Sources/TestingMacros/Support/CommentParsing.swift +++ b/Sources/TestingMacros/Support/CommentParsing.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder /// Find a common whitespace prefix among all lines in a string and trim it. /// diff --git a/Sources/TestingMacros/Support/ConditionArgumentParsing.swift b/Sources/TestingMacros/Support/ConditionArgumentParsing.swift index 30f0cd430..edf9a23c3 100644 --- a/Sources/TestingMacros/Support/ConditionArgumentParsing.swift +++ b/Sources/TestingMacros/Support/ConditionArgumentParsing.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros /// The result of parsing the condition argument passed to `#expect()` or diff --git a/Sources/TestingMacros/Support/DiagnosticMessage+Diagnosing.swift b/Sources/TestingMacros/Support/DiagnosticMessage+Diagnosing.swift index e6682dc8f..bec994f82 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage+Diagnosing.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage+Diagnosing.swift @@ -9,7 +9,9 @@ // import SwiftDiagnostics +import SwiftParser import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros extension AttributeInfo { diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index a474d2801..e49cfa497 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -9,7 +9,9 @@ // import SwiftDiagnostics +import SwiftParser import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros import SwiftSyntaxMacroExpansion diff --git a/Sources/TestingMacros/Support/SourceCodeCapturing.swift b/Sources/TestingMacros/Support/SourceCodeCapturing.swift index 9fd687e8f..3e18e4713 100644 --- a/Sources/TestingMacros/Support/SourceCodeCapturing.swift +++ b/Sources/TestingMacros/Support/SourceCodeCapturing.swift @@ -8,7 +8,9 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +import SwiftParser import SwiftSyntax +import SwiftSyntaxBuilder /// Get a swift-syntax expression initializing an instance of `__Expression` /// from an arbitrary syntax node. diff --git a/Sources/TestingMacros/Support/SourceLocationGeneration.swift b/Sources/TestingMacros/Support/SourceLocationGeneration.swift index c93b0d9f0..1adf96b78 100644 --- a/Sources/TestingMacros/Support/SourceLocationGeneration.swift +++ b/Sources/TestingMacros/Support/SourceLocationGeneration.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros /// Get an expression initializing an instance of ``SourceLocation`` from an diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index 391a468b1..a8a5d28a2 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -9,6 +9,7 @@ // import SwiftSyntax +import SwiftSyntaxBuilder import SwiftSyntaxMacros /// An enumeration representing the different kinds of test content known to the diff --git a/Sources/TestingMacros/TagMacro.swift b/Sources/TestingMacros/TagMacro.swift index b94149c09..624f812cd 100644 --- a/Sources/TestingMacros/TagMacro.swift +++ b/Sources/TestingMacros/TagMacro.swift @@ -9,6 +9,7 @@ // public import SwiftSyntax +import SwiftSyntaxBuilder public import SwiftSyntaxMacros /// A type describing the expansion of the `@Tag` attribute macro. diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index c94769d21..1503081f0 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -8,7 +8,9 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +import SwiftDiagnostics public import SwiftSyntax +import SwiftSyntaxBuilder public import SwiftSyntaxMacros #if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY diff --git a/Tests/TestingMacrosTests/PragmaMacroTests.swift b/Tests/TestingMacrosTests/PragmaMacroTests.swift index 0d430c036..bba101754 100644 --- a/Tests/TestingMacrosTests/PragmaMacroTests.swift +++ b/Tests/TestingMacrosTests/PragmaMacroTests.swift @@ -11,8 +11,10 @@ import Testing @testable import TestingMacros +import SwiftDiagnostics import SwiftParser import SwiftSyntax +import SwiftSyntaxBuilder @Suite("PragmaMacro Tests") struct PragmaMacroTests { diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index b0028c438..13ae3d180 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -11,6 +11,7 @@ import Testing @testable import TestingMacros +import SwiftBasicFormat import SwiftDiagnostics import SwiftParser import SwiftSyntax diff --git a/Tests/TestingMacrosTests/TestSupport/Parse.swift b/Tests/TestingMacrosTests/TestSupport/Parse.swift index ecff8de58..2b30df42e 100644 --- a/Tests/TestingMacrosTests/TestSupport/Parse.swift +++ b/Tests/TestingMacrosTests/TestSupport/Parse.swift @@ -10,6 +10,7 @@ @testable import TestingMacros +import SwiftBasicFormat import SwiftDiagnostics import SwiftOperators import SwiftParser diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 0a220552a..5a36fd4b6 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -91,7 +91,8 @@ struct AttachmentTests { // Write the attachment to disk, then read it back. let filePath = try attachment.write(toFileInDirectoryAtPath: temporaryDirectory(), appending: suffixes.next()!) createdFilePaths.append(filePath) - let fileName = try #require(filePath.split { $0 == "/" || $0 == #"\"# }.last) + let filePathComponents = filePath.split { $0 == "/" || $0 == #"\"# } + let fileName = try #require(filePathComponents.last) if i == 0 { #expect(fileName == baseFileName) } else { @@ -118,7 +119,8 @@ struct AttachmentTests { defer { remove(filePath) } - let fileName = try #require(filePath.split { $0 == "/" || $0 == #"\"# }.last) + let filePathComponents = filePath.split { $0 == "/" || $0 == #"\"# } + let fileName = try #require(filePathComponents.last) #expect(fileName == "loremipsum-\(suffix).tgz.gif.jpeg.html") try compare(attachableValue, toContentsOfFileAtPath: filePath) } From 0018a425358edb01916a89a4f09be1fc1628c55a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 12 Mar 2025 12:50:14 -0400 Subject: [PATCH 130/234] Don't emit `@_section` attributes unless the macro target has `SymbolLinkageMarkers`. (#1018) This PR suppresses our use of `@_section` and `@_used` unless the macro target is built with `SymbolLinkageMarkers` enabled. Resolves rdar://146819169. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Support/TestContentGeneration.swift | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index a8a5d28a2..8b53cabd8 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -62,7 +62,19 @@ func makeTestContentRecordDecl(named name: TokenSyntax, in typeName: TypeSyntax? IntegerLiteralExprSyntax(context, radix: .binary) } - return """ + var result: DeclSyntax = """ + @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") + private nonisolated \(staticKeyword(for: typeName)) let \(name): Testing.__TestContentRecord = ( + \(kindExpr), \(kind.commentRepresentation) + 0, + \(accessorName), + \(contextExpr), + 0 + ) + """ + +#if hasFeature(SymbolLinkageMarkers) + result = """ #if hasFeature(SymbolLinkageMarkers) #if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) @_section("__DATA_CONST,__swift5_tests") @@ -75,13 +87,9 @@ func makeTestContentRecordDecl(named name: TokenSyntax, in typeName: TypeSyntax? #endif @_used #endif - @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") - private nonisolated \(staticKeyword(for: typeName)) let \(name): Testing.__TestContentRecord = ( - \(kindExpr), \(kind.commentRepresentation) - 0, - \(accessorName), - \(contextExpr), - 0 - ) + \(result) """ +#endif + + return result } From b3d3fd761243622767e457935b5c497b577fb114 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 12 Mar 2025 12:51:01 -0400 Subject: [PATCH 131/234] Add a dedicated `TestContentKind` type to `_TestDiscovery`. (#1019) This PR adds a `TestContentKind` type to `_TestDiscovery` that represents the `kind` field of a record and allows initialization from a string literal (i.e. a FourCC) or an integer literal: ```swift extension MyType: DiscoverableAsTestContent { static var testContentKind: TestContentKind { "moof" } } ``` At this time, it's not possible to use this type directly in test content records emitted by our macros because the type cannot be made public, but it allows for types conforming to `DiscoverableAsTestContent` to be a bit more expressive. Test library authors who are experimenting with `_TestDiscovery` may opt to export the symbols from `_TestDiscovery` and thus could use the type directly in their emitted records (but keep in mind this module is still experimental!) The new type is `@frozen` and mostly `@inlinable` because its layout is set in stone by virtue of the test content record `kind` field having a fixed layout. Resolves rdar://146855125. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 4 +- Sources/Testing/ExitTests/ExitTest.swift | 6 +- Sources/Testing/Test+Discovery.swift | 4 +- .../Support/TestContentGeneration.swift | 3 + Sources/_TestDiscovery/CMakeLists.txt | 1 + .../DiscoverableAsTestContent.swift | 2 +- Sources/_TestDiscovery/TestContentKind.swift | 102 +++++++++++ .../_TestDiscovery/TestContentRecord.swift | 36 ++-- Tests/TestingTests/DiscoveryTests.swift | 161 ++++++++++++++++++ Tests/TestingTests/MiscellaneousTests.swift | 101 ----------- 10 files changed, 287 insertions(+), 133 deletions(-) create mode 100644 Sources/_TestDiscovery/TestContentKind.swift create mode 100644 Tests/TestingTests/DiscoveryTests.swift diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index 3e3efc512..6cb388533 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -243,7 +243,7 @@ protocol: ```swift extension FoodTruckDiagnostic: DiscoverableAsTestContent { - static var testContentKind: UInt32 { /* Your `kind` value here. */ } + static var testContentKind: TestContentKind { /* Your `kind` value here. */ } } ``` @@ -258,7 +258,7 @@ a custom `hint` type for your accessor functions, you can set ```swift extension FoodTruckDiagnostic: DiscoverableAsTestContent { - static var testContentKind: UInt32 { /* Your `kind` value here. */ } + static var testContentKind: TestContentKind { /* Your `kind` value here. */ } typealias TestContentContext = UnsafePointer typealias TestContentAccessorHint = String diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 61b35b9fd..69346b74e 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -239,11 +239,11 @@ extension ExitTest { // MARK: - Discovery extension ExitTest: DiscoverableAsTestContent { - static var testContentKind: UInt32 { - 0x65786974 + fileprivate static var testContentKind: TestContentKind { + "exit" } - typealias TestContentAccessorHint = ID + fileprivate typealias TestContentAccessorHint = ID /// Store the exit test into the given memory. /// diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 2568353f8..5d1b204ae 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -20,8 +20,8 @@ extension Test { /// instances of ``Test``, but functions are non-nominal types and cannot /// directly conform to protocols. fileprivate struct Generator: DiscoverableAsTestContent, RawRepresentable { - static var testContentKind: UInt32 { - 0x74657374 + static var testContentKind: TestContentKind { + "test" } var rawValue: @Sendable () async -> Test diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index 8b53cabd8..c6ea40357 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -17,6 +17,9 @@ import SwiftSyntaxMacros /// /// When adding cases to this enumeration, be sure to also update the /// corresponding enumeration in TestContent.md. +/// +/// - Bug: This type should be imported directly from `_TestDiscovery` instead +/// of being redefined (differently) here. enum TestContentKind: UInt32 { /// A test or suite declaration. case testDeclaration = 0x74657374 diff --git a/Sources/_TestDiscovery/CMakeLists.txt b/Sources/_TestDiscovery/CMakeLists.txt index 74d951682..7d6059792 100644 --- a/Sources/_TestDiscovery/CMakeLists.txt +++ b/Sources/_TestDiscovery/CMakeLists.txt @@ -10,6 +10,7 @@ add_library(_TestDiscovery STATIC Additions/WinSDKAdditions.swift DiscoverableAsTestContent.swift SectionBounds.swift + TestContentKind.swift TestContentRecord.swift) target_link_libraries(_TestDiscovery PRIVATE diff --git a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift index 96a8c7698..a4b400bad 100644 --- a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift +++ b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift @@ -22,7 +22,7 @@ public protocol DiscoverableAsTestContent: Sendable, ~Copyable { /// /// The value of this property is reserved for each test content type. See /// `ABI/TestContent.md` for a list of values and corresponding types. - static var testContentKind: UInt32 { get } + static var testContentKind: TestContentKind { get } /// The type of the `context` field in test content records associated with /// this type. diff --git a/Sources/_TestDiscovery/TestContentKind.swift b/Sources/_TestDiscovery/TestContentKind.swift new file mode 100644 index 000000000..645b06424 --- /dev/null +++ b/Sources/_TestDiscovery/TestContentKind.swift @@ -0,0 +1,102 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +/// A type representing a test content record's `kind` field. +/// +/// Test content kinds are 32-bit unsigned integers and are stored as such when +/// test content records are emitted at compile time. +/// +/// This type lets you represent a kind value as an integer literal or as a +/// string literal in Swift code. In particular, when adding a conformance to +/// the ``DiscoverableAsTestContent`` protocol, the protocol's +/// ``DiscoverableAsTestContent/testContentKind`` property must be an instance +/// of this type. +/// +/// For a list of reserved values, or to reserve a value for your own use, see +/// `ABI/TestContent.md`. +/// +/// @Comment { +/// This type is `@frozen` and most of its members are `@inlinable` because it +/// represents the underlying `kind` field which has a fixed layout. In the +/// future, we may want to use this type in test content records, but that +/// will require the type be publicly visible and that `@const` is implemented +/// in the compiler. +/// } +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@frozen public struct TestContentKind: Sendable, RawRepresentable { + public var rawValue: UInt32 + + @inlinable public init(rawValue: UInt32) { + self.rawValue = rawValue + } +} + +// MARK: - Equatable, Hashable + +extension TestContentKind: Equatable, Hashable { + @inlinable public static func ==(lhs: Self, rhs: Self) -> Bool { + lhs.rawValue == rhs.rawValue + } + + @inlinable public func hash(into hasher: inout Hasher) { + hasher.combine(rawValue) + } +} + +// MARK: - Codable + +extension TestContentKind: Codable {} + +// MARK: - ExpressibleByStringLiteral, ExpressibleByIntegerLiteral + +extension TestContentKind: ExpressibleByStringLiteral, ExpressibleByIntegerLiteral { + @inlinable public init(stringLiteral stringValue: StaticString) { + precondition(stringValue.utf8CodeUnitCount == MemoryLayout.stride, #""\#(stringValue)".utf8CodeUnitCount = \#(stringValue.utf8CodeUnitCount), expected \#(MemoryLayout.stride)"#) + let rawValue = stringValue.withUTF8Buffer { stringValue in + let bigEndian = UnsafeRawBufferPointer(stringValue).loadUnaligned(as: UInt32.self) + return UInt32(bigEndian: bigEndian) + } + self.init(rawValue: rawValue) + } + + @inlinable public init(integerLiteral: UInt32) { + self.init(rawValue: integerLiteral) + } +} + +// MARK: - CustomStringConvertible + +extension TestContentKind: CustomStringConvertible { + /// This test content type's kind value as an ASCII string (of the form + /// `"abcd"`) if it looks like it might be a [FourCC](https://en.wikipedia.org/wiki/FourCC) + /// value, or `nil` if not. + private var _fourCCValue: String? { + withUnsafeBytes(of: rawValue.bigEndian) { bytes in + if bytes.allSatisfy(Unicode.ASCII.isASCII) { + let characters = String(decoding: bytes, as: Unicode.ASCII.self) + let allAlphanumeric = characters.allSatisfy { $0.isLetter || $0.isWholeNumber } + if allAlphanumeric { + return characters + } + } + return nil + } + } + + public var description: String { + let hexValue = "0x" + String(rawValue, radix: 16) + if let fourCCValue = _fourCCValue { + return "'\(fourCCValue)' (\(hexValue))" + } + return hexValue + } +} diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index 4c7ec8d4d..d515620cf 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -108,17 +108,22 @@ public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyabl } fileprivate init(imageAddress: UnsafeRawPointer?, recordAddress: UnsafePointer<_TestContentRecord>) { - precondition(recordAddress.pointee.kind == T.testContentKind) + precondition(recordAddress.pointee.kind == T.testContentKind.rawValue) self.imageAddress = imageAddress self._recordStorage = .atAddress(recordAddress) } fileprivate init(imageAddress: UnsafeRawPointer?, record: _TestContentRecord) { - precondition(record.kind == T.testContentKind) + precondition(record.kind == T.testContentKind.rawValue) self.imageAddress = imageAddress self._recordStorage = .inline(record) } + /// The kind of this test content record. + public var kind: TestContentKind { + TestContentKind(rawValue: _record.kind) + } + /// The type of the ``context`` property. public typealias Context = T.TestContentContext @@ -181,28 +186,8 @@ extension TestContentRecord: Sendable where Context: Sendable {} // MARK: - CustomStringConvertible extension TestContentRecord: CustomStringConvertible { - /// This test content type's kind value as an ASCII string (of the form - /// `"abcd"`) if it looks like it might be a [FourCC](https://en.wikipedia.org/wiki/FourCC) - /// value, or `nil` if not. - private static var _asciiKind: String? { - return withUnsafeBytes(of: T.testContentKind.bigEndian) { bytes in - if bytes.allSatisfy(Unicode.ASCII.isASCII) { - let characters = String(decoding: bytes, as: Unicode.ASCII.self) - let allAlphanumeric = characters.allSatisfy { $0.isLetter || $0.isWholeNumber } - if allAlphanumeric { - return characters - } - } - return nil - } - } - public var description: String { let typeName = String(describing: Self.self) - let hexKind = "0x" + String(T.testContentKind, radix: 16) - let kind = Self._asciiKind.map { asciiKind in - "'\(asciiKind)' (\(hexKind))" - } ?? hexKind switch _recordStorage { case let .atAddress(recordAddress): let recordAddress = imageAddress.map { imageAddress in @@ -232,11 +217,14 @@ extension DiscoverableAsTestContent where Self: ~Copyable { /// } public static func allTestContentRecords() -> AnySequence> { validateMemoryLayout() + + let kind = testContentKind.rawValue + let result = SectionBounds.all(.testContent).lazy.flatMap { sb in sb.buffer.withMemoryRebound(to: _TestContentRecord.self) { records in (0 ..< records.count).lazy .map { (records.baseAddress! + $0) as UnsafePointer<_TestContentRecord> } - .filter { $0.pointee.kind == testContentKind } + .filter { $0.pointee.kind == kind } .map { TestContentRecord(imageAddress: sb.imageAddress, recordAddress: $0) } } } @@ -275,7 +263,7 @@ extension DiscoverableAsTestContent where Self: ~Copyable { validateMemoryLayout() let typeNameHint = _testContentTypeNameHint - let kind = testContentKind + let kind = testContentKind.rawValue let loader: @Sendable (Any.Type) -> _TestContentRecord? = { type in withUnsafeTemporaryAllocation(of: _TestContentRecord.self, capacity: 1) { buffer in // Load the record from the container type. diff --git a/Tests/TestingTests/DiscoveryTests.swift b/Tests/TestingTests/DiscoveryTests.swift new file mode 100644 index 000000000..5af7b26f1 --- /dev/null +++ b/Tests/TestingTests/DiscoveryTests.swift @@ -0,0 +1,161 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) import _TestDiscovery +#if canImport(Foundation) +private import Foundation +#endif + +@Suite("Runtime Test Discovery Tests") +struct DiscoveryTests { + @Test func testContentKind() { + let kind1: TestContentKind = "abcd" + let kind2: TestContentKind = 0x61626364 + #expect(kind1 == kind2) + #expect(String(describing: kind1) == String(describing: kind2)) + #expect(String(describing: kind1) == "'abcd' (0x61626364)") + + let kind3: TestContentKind = 0xFF123456 + #expect(kind1 != kind3) + #expect(kind2 != kind3) + #expect(String(describing: kind1) != String(describing: kind3)) + #expect(String(describing: kind3).lowercased() == "0xff123456") + } + +#if canImport(Foundation) + @Test func testContentKindCodableConformance() throws { + let kind1: TestContentKind = "moof" + let data = try JSONEncoder().encode(kind1) + let uint32 = try JSONDecoder().decode(UInt32.self, from: data) + let kind2 = try JSONDecoder().decode(TestContentKind.self, from: data) + #expect(uint32 == kind2.rawValue) + } +#endif + + @Test func utf8TestContentKind() { + let kind: TestContentKind = "\u{1F3B6}" + #expect(kind.rawValue == 0xF09F8EB6) + #expect(String(describing: kind).uppercased() == "0XF09F8EB6") + } + +#if !SWT_NO_EXIT_TESTS + @Test("TestContentKind rejects bad string literals") + func badTestContentKindLiteral() async { + await #expect(exitsWith: .failure) { + _ = "abc" as TestContentKind + } + await #expect(exitsWith: .failure) { + _ = "abcde" as TestContentKind + } + } +#endif + +#if !SWT_NO_DYNAMIC_LINKING && hasFeature(SymbolLinkageMarkers) + struct MyTestContent: Testing.DiscoverableAsTestContent { + typealias TestContentAccessorHint = UInt32 + + var value: UInt32 + + static var testContentKind: TestContentKind { + TestContentKind(rawValue: record.kind) + } + + static var expectedHint: TestContentAccessorHint { + 0x01020304 + } + + static var expectedValue: UInt32 { + 0xCAFEF00D + } + + static var expectedContext: UInt { + record.context + } + +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) + @_section("__DATA_CONST,__swift5_tests") +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) + @_section("swift5_tests") +#elseif os(Windows) + @_section(".sw5test$B") +#else + @__testing(warning: "Platform-specific implementation missing: test content section name unavailable") +#endif + @_used + private static let record: __TestContentRecord = ( + 0xABCD1234, + 0, + { outValue, type, hint in + guard type.load(as: Any.Type.self) == MyTestContent.self else { + return false + } + if let hint, hint.load(as: TestContentAccessorHint.self) != expectedHint { + return false + } + _ = outValue.initializeMemory(as: Self.self, to: .init(value: expectedValue)) + return true + }, + UInt(truncatingIfNeeded: UInt64(0x0204060801030507)), + 0 + ) + } + + @Test func testDiscovery() async { + // Check the type of the test record sequence (it should be lazy.) + let allRecordsSeq = MyTestContent.allTestContentRecords() +#if SWT_FIXED_143080508 + #expect(allRecordsSeq is any LazySequenceProtocol) + #expect(!(allRecordsSeq is [TestContentRecord])) +#endif + + // It should have exactly one matching record (because we only emitted one.) + let allRecords = Array(allRecordsSeq) + #expect(allRecords.count == 1) + + // Can find a single test record + #expect(allRecords.contains { record in + record.load()?.value == MyTestContent.expectedValue + && record.context == MyTestContent.expectedContext + }) + + // Can find a test record with matching hint + #expect(allRecords.contains { record in + let hint = MyTestContent.expectedHint + return record.load(withHint: hint)?.value == MyTestContent.expectedValue + && record.context == MyTestContent.expectedContext + }) + + // Doesn't find a test record with a mismatched hint + #expect(!allRecords.contains { record in + let hint = ~MyTestContent.expectedHint + return record.load(withHint: hint)?.value == MyTestContent.expectedValue + && record.context == MyTestContent.expectedContext + }) + } +#endif + +#if !SWT_NO_LEGACY_TEST_DISCOVERY && hasFeature(SymbolLinkageMarkers) + @Test("Legacy test discovery finds the same number of tests") func discoveredTestCount() async { + let oldFlag = Environment.variable(named: "SWT_USE_LEGACY_TEST_DISCOVERY") + defer { + Environment.setVariable(oldFlag, named: "SWT_USE_LEGACY_TEST_DISCOVERY") + } + + Environment.setVariable("1", named: "SWT_USE_LEGACY_TEST_DISCOVERY") + let testsWithOldCode = await Array(Test.all).count + + Environment.setVariable("0", named: "SWT_USE_LEGACY_TEST_DISCOVERY") + let testsWithNewCode = await Array(Test.all).count + + #expect(testsWithOldCode == testsWithNewCode) + } +#endif +} diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index a6c62fdbc..1f18f20a9 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -580,105 +580,4 @@ struct MiscellaneousTests { } #expect(duration < .seconds(1)) } - -#if !SWT_NO_DYNAMIC_LINKING && hasFeature(SymbolLinkageMarkers) - struct MyTestContent: Testing.DiscoverableAsTestContent { - typealias TestContentAccessorHint = UInt32 - - var value: UInt32 - - static var testContentKind: UInt32 { - record.kind - } - - static var expectedHint: TestContentAccessorHint { - 0x01020304 - } - - static var expectedValue: UInt32 { - 0xCAFEF00D - } - - static var expectedContext: UInt { - record.context - } - -#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) - @_section("__DATA_CONST,__swift5_tests") -#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) - @_section("swift5_tests") -#elseif os(Windows) - @_section(".sw5test$B") -#else - @__testing(warning: "Platform-specific implementation missing: test content section name unavailable") -#endif - @_used - private static let record: __TestContentRecord = ( - 0xABCD1234, - 0, - { outValue, type, hint in - guard type.load(as: Any.Type.self) == MyTestContent.self else { - return false - } - if let hint, hint.load(as: TestContentAccessorHint.self) != expectedHint { - return false - } - _ = outValue.initializeMemory(as: Self.self, to: .init(value: expectedValue)) - return true - }, - UInt(truncatingIfNeeded: UInt64(0x0204060801030507)), - 0 - ) - } - - @Test func testDiscovery() async { - // Check the type of the test record sequence (it should be lazy.) - let allRecordsSeq = MyTestContent.allTestContentRecords() -#if SWT_FIXED_143080508 - #expect(allRecordsSeq is any LazySequenceProtocol) - #expect(!(allRecordsSeq is [TestContentRecord])) -#endif - - // It should have exactly one matching record (because we only emitted one.) - let allRecords = Array(allRecordsSeq) - #expect(allRecords.count == 1) - - // Can find a single test record - #expect(allRecords.contains { record in - record.load()?.value == MyTestContent.expectedValue - && record.context == MyTestContent.expectedContext - }) - - // Can find a test record with matching hint - #expect(allRecords.contains { record in - let hint = MyTestContent.expectedHint - return record.load(withHint: hint)?.value == MyTestContent.expectedValue - && record.context == MyTestContent.expectedContext - }) - - // Doesn't find a test record with a mismatched hint - #expect(!allRecords.contains { record in - let hint = ~MyTestContent.expectedHint - return record.load(withHint: hint)?.value == MyTestContent.expectedValue - && record.context == MyTestContent.expectedContext - }) - } -#endif - -#if !SWT_NO_LEGACY_TEST_DISCOVERY && hasFeature(SymbolLinkageMarkers) - @Test("Legacy test discovery finds the same number of tests") func discoveredTestCount() async { - let oldFlag = Environment.variable(named: "SWT_USE_LEGACY_TEST_DISCOVERY") - defer { - Environment.setVariable(oldFlag, named: "SWT_USE_LEGACY_TEST_DISCOVERY") - } - - Environment.setVariable("1", named: "SWT_USE_LEGACY_TEST_DISCOVERY") - let testsWithOldCode = await Array(Test.all).count - - Environment.setVariable("0", named: "SWT_USE_LEGACY_TEST_DISCOVERY") - let testsWithNewCode = await Array(Test.all).count - - #expect(testsWithOldCode == testsWithNewCode) - } -#endif } From 4399cba961ad04adc45f533d0c1a950e6670e7e1 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 12 Mar 2025 15:10:50 -0400 Subject: [PATCH 132/234] Add a `reserved` argument to the test content record accessor signature. (#1017) This PR adds a `reserved` argument to `__TestContentRecordAccessor` for our future use. Resolves rdar://146818672. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 13 +++++++++---- Sources/Testing/Discovery+Macro.swift | 3 ++- Sources/TestingMacros/ConditionMacro.swift | 2 +- Sources/TestingMacros/SuiteDeclarationMacro.swift | 2 +- Sources/TestingMacros/TestDeclarationMacro.swift | 2 +- Sources/_TestDiscovery/TestContentRecord.swift | 7 ++++--- Tests/TestingTests/DiscoveryTests.swift | 2 +- 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index 6cb388533..cb68a2d6e 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -44,7 +44,8 @@ testing library have the following layout: typealias Accessor = @convention(c) ( _ outValue: UnsafeMutableRawPointer, _ type: UnsafeRawPointer, - _ hint: UnsafeRawPointer? + _ hint: UnsafeRawPointer?, + _ reserved: UInt ) -> CBool typealias TestContentRecord = ( @@ -63,7 +64,8 @@ If needed, this type can be represented in C as a structure: typedef bool (* SWTAccessor)( void *outValue, const void *type, - const void *_Nullable hint + const void *_Nullable hint, + uintptr_t reserved ); struct SWTTestContentRecord { @@ -117,7 +119,7 @@ If `accessor` is `nil`, the test content record is ignored. The testing library may, in the future, define record kinds that do not provide an accessor function (that is, they represent pure compile-time information only.) -The third argument to this function, `type`, is a pointer to the type[^mightNotBeSwift] +The second argument to this function, `type`, is a pointer to the type[^mightNotBeSwift] (not a bitcast Swift type) of the value expected to be written to `outValue`. This argument helps to prevent memory corruption if two copies of Swift Testing or a third-party library are inadvertently loaded into the same process. If the @@ -134,12 +136,15 @@ accessor function must return `false` and must not modify `outValue`. [`std::type_info`](https://en.cppreference.com/w/cpp/types/type_info), and write a C++ class instance to `outValue` using [placement `new`](https://en.cppreference.com/w/cpp/language/new#Placement_new). -The fourth argument to this function, `hint`, is an optional input that can be +The third argument to this function, `hint`, is an optional input that can be passed to help the accessor function determine if its corresponding test content record matches what the caller is looking for. If the caller passes `nil` as the `hint` argument, the accessor behaves as if it matched (that is, no additional filtering is performed.) +The fourth argument to this function, `reserved`, is reserved for future use. +Accessor functions should assume it is `0` and must not access it. + The concrete Swift type of the value written to `outValue`, the type pointed to by `type`, and the value pointed to by `hint` depend on the kind of record: diff --git a/Sources/Testing/Discovery+Macro.swift b/Sources/Testing/Discovery+Macro.swift index 391278983..97b925e55 100644 --- a/Sources/Testing/Discovery+Macro.swift +++ b/Sources/Testing/Discovery+Macro.swift @@ -28,7 +28,8 @@ protocol DiscoverableAsTestContent: _TestDiscovery.DiscoverableAsTestContent, ~C public typealias __TestContentRecordAccessor = @convention(c) ( _ outValue: UnsafeMutableRawPointer, _ type: UnsafeRawPointer, - _ hint: UnsafeRawPointer? + _ hint: UnsafeRawPointer?, + _ reserved: UInt ) -> CBool /// The content of a test content record. diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 616e9abbc..89498e1ec 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -484,7 +484,7 @@ extension ExitTestConditionMacro { """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") enum \(enumName) { - private nonisolated static let accessor: Testing.__TestContentRecordAccessor = { outValue, type, hint in + private nonisolated static let accessor: Testing.__TestContentRecordAccessor = { outValue, type, hint, _ in Testing.ExitTest.__store( \(exitTestIDExpr), \(bodyThunkName), diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index 8cfe5e902..60a276689 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -149,7 +149,7 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { result.append( """ @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") - private nonisolated static let \(accessorName): Testing.__TestContentRecordAccessor = { outValue, type, _ in + private nonisolated static let \(accessorName): Testing.__TestContentRecordAccessor = { outValue, type, _, _ in Testing.Test.__store(\(generatorName), into: outValue, asTypeAt: type) } """ diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 1503081f0..2a4da4e3c 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -474,7 +474,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { result.append( """ @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") - private \(staticKeyword(for: typeName)) nonisolated let \(accessorName): Testing.__TestContentRecordAccessor = { outValue, type, _ in + private \(staticKeyword(for: typeName)) nonisolated let \(accessorName): Testing.__TestContentRecordAccessor = { outValue, type, _, _ in Testing.Test.__store(\(generatorName), into: outValue, asTypeAt: type) } """ diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index d515620cf..5235d9c64 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -24,7 +24,8 @@ private typealias _TestContentRecordAccessor = @convention(c) ( _ outValue: UnsafeMutableRawPointer, _ type: UnsafeRawPointer, - _ hint: UnsafeRawPointer? + _ hint: UnsafeRawPointer?, + _ reserved: UInt ) -> CBool /// The content of a test content record. @@ -160,10 +161,10 @@ public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyabl withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in let initialized = if let hint { withUnsafePointer(to: hint) { hint in - accessor(buffer.baseAddress!, type, hint) + accessor(buffer.baseAddress!, type, hint, 0) } } else { - accessor(buffer.baseAddress!, type, nil) + accessor(buffer.baseAddress!, type, nil, 0) } guard initialized else { return nil diff --git a/Tests/TestingTests/DiscoveryTests.swift b/Tests/TestingTests/DiscoveryTests.swift index 5af7b26f1..a730f8b53 100644 --- a/Tests/TestingTests/DiscoveryTests.swift +++ b/Tests/TestingTests/DiscoveryTests.swift @@ -93,7 +93,7 @@ struct DiscoveryTests { private static let record: __TestContentRecord = ( 0xABCD1234, 0, - { outValue, type, hint in + { outValue, type, hint, _ in guard type.load(as: Any.Type.self) == MyTestContent.self else { return false } From db50ace240f5a9e73e6422fb25ae54f393849edd Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 13 Mar 2025 15:09:13 -0500 Subject: [PATCH 133/234] Enable 'MemberImportVisibility' upcoming feature in CMake rules (#1022) Follow-on to #1020 to enable this upcoming feature in the CMake rules, too. Resolves rdar://146876960. - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- cmake/modules/shared/CompilerSettings.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/modules/shared/CompilerSettings.cmake b/cmake/modules/shared/CompilerSettings.cmake index eb9da4162..f526caae3 100644 --- a/cmake/modules/shared/CompilerSettings.cmake +++ b/cmake/modules/shared/CompilerSettings.cmake @@ -17,7 +17,8 @@ add_compile_options( "SHELL:$<$:-Xfrontend -enable-experimental-feature -Xfrontend SuppressedAssociatedTypes>") add_compile_options( "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend ExistentialAny>" - "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend InternalImportsByDefault>") + "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend InternalImportsByDefault>" + "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend MemberImportVisibility>") # Platform-specific definitions. if(APPLE) From cabf4d419cf9c8c96b131a50f9ee9da3ea67a1af Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 13 Mar 2025 15:10:49 -0500 Subject: [PATCH 134/234] Continue encoding value for deprecated property in runner plan snapshot to avoid decoding error in clients (#1023) This fixes a regression in the encoding of the `Runner.Plan` snapshot types, which can manifest when using Xcode 16. ### Motivation: The `isParallelizationEnabled` property was deprecated and changed from a stored property to a derived one in #901. That caused the value to no longer be encoded in `Runner.Plan.Action.RunOptions`, which can cause decoding errors in versions of Xcode which expect it to still be present. ### Modifications: - Manually implement `Codable` conformance for the affected type to begin including a hardcoded value. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes rdar://146284519 --- Sources/Testing/Running/Runner.Plan.swift | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index 26cb00d14..c89fdecb5 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -15,7 +15,7 @@ extension Runner { public enum Action: Sendable { /// A type describing options to apply to actions of case /// ``Runner/Plan/Action/run(options:)`` when they are run. - public struct RunOptions: Sendable, Codable { + public struct RunOptions: Sendable { /// Whether or not this step should be run in parallel with other tests. /// /// By default, all steps in a runner plan are run in parallel if the @@ -347,6 +347,29 @@ extension Runner.Plan { } } +extension Runner.Plan.Action.RunOptions: Codable { + private enum CodingKeys: CodingKey { + case isParallelizationEnabled + } + + public init(from decoder: any Decoder) throws { + // No-op. This initializer cannot be synthesized since `CodingKeys` includes + // a case representing a non-stored property. See comment about the + // `isParallelizationEnabled` property in `encode(to:)`. + self.init() + } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + + // The `isParallelizationEnabled` property was removed after this type was + // first introduced. Its value was never actually used in a meaningful way + // by known clients, but its absence can cause decoding errors, so to avoid + // such problems, continue encoding a hardcoded value. + try container.encode(false, forKey: .isParallelizationEnabled) + } +} + #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting From efc39a5f14f657bd25155a0ab6cac96c4981a44d Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Fri, 14 Mar 2025 09:45:45 -0700 Subject: [PATCH 135/234] build: avoid doubly nesting the swift module (#1025) We always install the swift module into the platform directory. The additional nesting is not required. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c59e2f35d..38aeda617 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,7 +52,7 @@ include(SwiftModuleInstallation) option(SwiftTesting_INSTALL_NESTED_SUBDIR "Install libraries under a platform and architecture subdirectory" NO) set(SwiftTesting_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${SwiftTesting_PLATFORM_SUBDIR}$<$,$>>:/testing>$<$:/${SwiftTesting_ARCH_SUBDIR}>") -set(SwiftTesting_INSTALL_SWIFTMODULEDIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${SwiftTesting_PLATFORM_SUBDIR}$<$,$>>:/testing>$<$:/${SwiftTesting_PLATFORM_SUBDIR}>") +set(SwiftTesting_INSTALL_SWIFTMODULEDIR "${CMAKE_INSTALL_LIBDIR}/swift$<$>:_static>/${SwiftTesting_PLATFORM_SUBDIR}$<$,$>>:/testing>") add_compile_options($<$:-no-toolchain-stdlib-rpath>) From 939737a09c604a4d3696f8cc69187fbaa88bdeba Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 14 Mar 2025 15:12:15 -0500 Subject: [PATCH 136/234] Represent non-encodable test argument values in Test.Case.ID (#1000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This expands `Test.Case.ID` to represent combinations of arguments which aren't fully encodable, and uniquely distinguishes test cases associated with the same parameterized test function which have otherwise identical IDs. ### Motivation: It is possible to declare a parameterized test function with a collection of arguments that includes the same element more than once. As a trivial example: ```swift @Test(arguments: [1, 1]) func repeatedArg(value: Int) { ... } ``` This can happen in more realistic scenarios if you dynamically construct a collection of arguments that accidentally or intentionally includes the same value more than once. The testing library attempts to form a unique identifier for each argument passed to a parameterized test. It does so by checking whether the value conforms to `Encodable`, or one of the other protocols mentioned in the documentation (see [Run selected test cases](https://swiftpackageindex.com/swiftlang/swift-testing/main/documentation/testing/parameterizedtesting#Run-selected-test-cases)). One problem with the current implementation is that if the value _doesn't_ conform to one of those known protocols, the testing library gives up and doesn't produce a unique identifier for that test case at all. Specifically, in that situation the `argumentIDs` property of `Test.Case.ID` will have a value of `nil`. Another problem is that if an argument is passed more than once, the derived identifiers for each argument's test case will be the same and the result reporting will be ambiguous at best; or worse, the lack of a unique identifier could cause an integrated tool to misbehave or crash. (Current versions of Xcode 16 experience this issue non-deterministically for projects which have test parallelization enabled.) To solve this, there needs to be a way to deterministically distinguish test cases which, from the testing library's perspective, appear identical—either because they actually are the same value, or because they encode to the same representation. ### Modifications: - Add a new `isStable` boolean property to `Test.Case.Argument.ID` representing whether or not the testing library was able to encode a stable representation of the argument value it identifies. - Add a new SPI property named `discriminator` to `Test.Case` which distinguishes test cases associated with the same test function whose arguments are identical. - Add a corresponding property to `Test.Case.ID` with the same name, `discriminator`. - Change the `Test.Case.ID.argumentIDs` property to non-Optional, so that it always has a value even if one or more argument IDs is non-stable. - Add a derived boolean property `isStable` to `Test.Case.ID` whose value is `true` iff all of its argument IDs are stable. - Add `Hashable` conformance to `Test.Case`, since the reason for avoiding such conformance has been resolved. - Add new tests. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Resolves #995 Resolves rdar://119522099 --- .../Testing/ABI/Encoded/ABI.EncodedTest.swift | 6 +- .../Event.HumanReadableOutputRecorder.swift | 12 +- .../CustomTestArgumentEncodable.swift | 13 +- .../Test.Case.Generator.swift | 29 ++- .../Parameterization/Test.Case.ID.swift | 119 +++++++++-- .../Testing/Parameterization/Test.Case.swift | 188 +++++++++++++++--- Tests/TestingTests/EventTests.swift | 2 +- .../Test.Case.Argument.IDTests.swift | 32 +-- .../Test.Case.ArgumentTests.swift | 40 ++-- .../Test.Case.GeneratorTests.swift | 31 +++ Tests/TestingTests/Test.CaseTests.swift | 177 +++++++++++++++++ .../TestingTests/TestCaseSelectionTests.swift | 5 +- .../TestSupport/TestingAdditions.swift | 23 +++ 13 files changed, 576 insertions(+), 101 deletions(-) create mode 100644 Tests/TestingTests/Test.Case.GeneratorTests.swift create mode 100644 Tests/TestingTests/Test.CaseTests.swift diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index 51d01781d..cda558f83 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -124,9 +124,13 @@ extension ABI { var displayName: String init(encoding testCase: borrowing Test.Case) { + guard let arguments = testCase.arguments else { + preconditionFailure("Attempted to initialize an EncodedTestCase encoding a test case which is not parameterized: \(testCase). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + } + // TODO: define an encodable form of Test.Case.ID id = String(describing: testCase.id) - displayName = testCase.arguments.lazy + displayName = arguments.lazy .map(\.value) .map(String.init(describingForTest:)) .joined(separator: ", ") diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 0e856facf..f585495a9 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -171,8 +171,14 @@ extension Test.Case { /// - Parameters: /// - includeTypeNames: Whether the qualified type name of each argument's /// runtime type should be included. Defaults to `false`. + /// + /// - Returns: A string containing the arguments of this test case formatted + /// for presentation, or an empty string if this test cases is + /// non-parameterized. fileprivate func labeledArguments(includingQualifiedTypeNames includeTypeNames: Bool = false) -> String { - arguments.lazy + guard let arguments else { return "" } + + return arguments.lazy .map { argument in let valueDescription = String(describingForTest: argument.value) @@ -494,14 +500,14 @@ extension Event.HumanReadableOutputRecorder { return result case .testCaseStarted: - guard let testCase = eventContext.testCase, testCase.isParameterized else { + guard let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { break } return [ Message( symbol: .default, - stringValue: "Passing \(testCase.arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)" + stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)" ) ] diff --git a/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift b/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift index 58d738f11..90b3ff292 100644 --- a/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift +++ b/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift @@ -44,18 +44,25 @@ public protocol CustomTestArgumentEncodable: Sendable { } extension Test.Case.Argument.ID { - /// Initialize this instance with an ID for the specified test argument. + /// Initialize an ID instance with the specified test argument value. /// /// - Parameters: /// - value: The value of a test argument for which to get an ID. /// - parameter: The parameter of the test function to which this argument /// value was passed. /// - /// - Returns: `nil` if an ID cannot be formed from the specified test + /// - Returns: `nil` if a stable ID cannot be formed from the specified test /// argument value. /// /// - Throws: Any error encountered while attempting to encode `value`. /// + /// If a stable representation of `value` can be encoded successfully, the + /// value of this instance's `bytes` property will be the the bytes of that + /// encoded JSON representation and this instance may be considered stable. If + /// no stable representation of `value` can be obtained, `nil` is returned. If + /// a stable representation was obtained but failed to encode, the error + /// resulting from the encoding attempt is thrown. + /// /// This function is not part of the public interface of the testing library. /// /// ## See Also @@ -83,7 +90,7 @@ extension Test.Case.Argument.ID { return nil } - self = .init(bytes: try Self._encode(encodableValue, parameter: parameter)) + self.init(bytes: try Self._encode(encodableValue, parameter: parameter)) #else nil #endif diff --git a/Sources/Testing/Parameterization/Test.Case.Generator.swift b/Sources/Testing/Parameterization/Test.Case.Generator.swift index d4d583e48..d30e3a7d3 100644 --- a/Sources/Testing/Parameterization/Test.Case.Generator.swift +++ b/Sources/Testing/Parameterization/Test.Case.Generator.swift @@ -62,7 +62,7 @@ extension Test.Case { // A beautiful hack to give us the right number of cases: iterate over a // collection containing a single Void value. self.init(sequence: CollectionOfOne(())) { _ in - Test.Case(arguments: [], body: testFunction) + Test.Case(body: testFunction) } } @@ -257,7 +257,32 @@ extension Test.Case { extension Test.Case.Generator: Sequence { func makeIterator() -> some IteratorProtocol { - _sequence.lazy.map(_mapElement).makeIterator() + let state = ( + iterator: _sequence.makeIterator(), + testCaseIDs: [Test.Case.ID: Int](minimumCapacity: underestimatedCount) + ) + + return sequence(state: state) { state in + guard let element = state.iterator.next() else { + return nil + } + + var testCase = _mapElement(element) + + if testCase.isParameterized { + // Store the original, unmodified test case ID. We're about to modify a + // property which affects it, and we want to update state based on the + // original one. + let testCaseID = testCase.id + + // Ensure test cases with identical IDs each have a unique discriminator. + let discriminator = state.testCaseIDs[testCaseID, default: 0] + testCase.discriminator = discriminator + state.testCaseIDs[testCaseID] = discriminator + 1 + } + + return testCase + } } var underestimatedCount: Int { diff --git a/Sources/Testing/Parameterization/Test.Case.ID.swift b/Sources/Testing/Parameterization/Test.Case.ID.swift index 26b57fdf8..b29268104 100644 --- a/Sources/Testing/Parameterization/Test.Case.ID.swift +++ b/Sources/Testing/Parameterization/Test.Case.ID.swift @@ -15,27 +15,41 @@ extension Test.Case { /// parameterized test function. They are not necessarily unique across two /// different ``Test`` instances. @_spi(ForToolsIntegrationOnly) - public struct ID: Sendable, Equatable, Hashable { + public struct ID: Sendable { /// The IDs of the arguments of this instance's associated ``Test/Case``, in /// the order they appear in ``Test/Case/arguments``. /// - /// The value of this property is `nil` if _any_ of the associated test - /// case's arguments has a `nil` ID. + /// The value of this property is `nil` for the ID of the single test case + /// associated with a non-parameterized test function. public var argumentIDs: [Argument.ID]? - public init(argumentIDs: [Argument.ID]?) { + /// A number used to distinguish this test case from others associated with + /// the same parameterized test function whose arguments have the same ID. + /// + /// The value of this property is `nil` for the ID of the single test case + /// associated with a non-parameterized test function. + /// + /// ## See Also + /// + /// - ``Test/Case/discriminator`` + public var discriminator: Int? + + /// Whether or not this test case ID is considered stable across successive + /// runs. + public var isStable: Bool + + init(argumentIDs: [Argument.ID]?, discriminator: Int?, isStable: Bool) { + precondition((argumentIDs == nil) == (discriminator == nil)) + self.argumentIDs = argumentIDs + self.discriminator = discriminator + self.isStable = isStable } } @_spi(ForToolsIntegrationOnly) public var id: ID { - let argumentIDs = arguments.compactMap(\.id) - guard argumentIDs.count == arguments.count else { - return ID(argumentIDs: nil) - } - - return ID(argumentIDs: argumentIDs) + ID(argumentIDs: arguments.map { $0.map(\.id) }, discriminator: discriminator, isStable: isStable) } } @@ -43,22 +57,83 @@ extension Test.Case { extension Test.Case.ID: CustomStringConvertible { public var description: String { - "argumentIDs: \(String(describing: argumentIDs))" + if let argumentIDs, let discriminator { + "Parameterized test case ID: argumentIDs: \(argumentIDs), discriminator: \(discriminator), isStable: \(isStable)" + } else { + "Non-parameterized test case ID" + } } } // MARK: - Codable -extension Test.Case.ID: Codable {} +extension Test.Case.ID: Codable { + private enum CodingKeys: String, CodingKey { + /// A coding key for ``Test/Case/ID/argumentIDs``. + /// + /// This case's string value is non-standard because ``legacyArgumentIDs`` + /// already used "argumentIDs" and this needs to be different. + case argumentIDs = "argIDs" -// MARK: - Equatable + /// A coding key for ``Test/Case/ID/discriminator``. + case discriminator -// We cannot safely implement Equatable for Test.Case because its values are -// type-erased. It does conform to `Identifiable`, but its ID type is composed -// of the IDs of its arguments, and those IDs are not always available (for -// example, if the type of an argument is not Codable). Thus, we cannot check -// for equality of test cases based on this, because if two test cases had -// different arguments, but the type of those arguments is not Codable, they -// both will have a `nil` ID and would incorrectly be considered equal. -// -// `Test.Case.ID` is Equatable, however. + /// A coding key for ``Test/Case/ID/isStable``. + case isStable + + /// A coding key for the legacy representation of ``Test/Case/ID/argumentIDs``. + /// + /// This case's string value is non-standard in order to maintain + /// legacy compatibility with its original value. + case legacyArgumentIDs = "argumentIDs" + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + if container.contains(.isStable) { + // `isStable` is present, so we're decoding an instance encoded using the + // newest style: every property can be decoded straightforwardly. + try self.init( + argumentIDs: container.decodeIfPresent([Test.Case.Argument.ID].self, forKey: .argumentIDs), + discriminator: container.decodeIfPresent(Int.self, forKey: .discriminator), + isStable: container.decode(Bool.self, forKey: .isStable) + ) + } else if container.contains(.legacyArgumentIDs) { + // `isStable` is absent, so we're decoding using the old style. Since the + // legacy `argumentIDs` is present, the representation should be + // considered stable. + let decodedArgumentIDs = try container.decode([Test.Case.Argument.ID].self, forKey: .legacyArgumentIDs) + let argumentIDs = decodedArgumentIDs.isEmpty ? nil : decodedArgumentIDs + + // Discriminator should be `nil` for the ID of a non-parameterized test + // case, but can default to 0 for the ID of a parameterized test case. + let discriminator = argumentIDs == nil ? nil : 0 + + self.init(argumentIDs: argumentIDs, discriminator: discriminator, isStable: true) + } else { + // This is the old style, and since `argumentIDs` is absent, we know this + // ID represents a parameterized test case which is non-stable. + self.init(argumentIDs: [.init(bytes: [])], discriminator: 0, isStable: false) + } + } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + + try container.encode(isStable, forKey: .isStable) + try container.encodeIfPresent(discriminator, forKey: .discriminator) + try container.encodeIfPresent(argumentIDs, forKey: .argumentIDs) + + // Encode the legacy representation of `argumentIDs`. + if argumentIDs == nil { + try container.encode([Test.Case.Argument.ID](), forKey: .legacyArgumentIDs) + } else if isStable, let argumentIDs = argumentIDs { + try container.encode(argumentIDs, forKey: .legacyArgumentIDs) + } + } +} + +// MARK: - Equatable, Hashable + +extension Test.Case.ID: Equatable, Hashable {} diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 80ff101da..ab9183cf8 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -15,6 +15,30 @@ extension Test { /// Tests that are _not_ parameterized map to a single instance of /// ``Test/Case``. public struct Case: Sendable { + /// An enumeration describing the various kinds of test cases. + private enum _Kind: Sendable { + /// A test case associated with a non-parameterized test function. + /// + /// There is only one test case with this kind associated with each + /// non-parameterized test function. + case nonParameterized + + /// A test case associated with a parameterized test function. + /// + /// - Parameters: + /// - arguments: The arguments passed to the parameterized test function + /// this test case is associated with. + /// - discriminator: A number used to distinguish this test case from + /// others associated with the same parameterized test function whose + /// arguments have the same ID. + /// - isStable: Whether or not this test case is considered stable + /// across successive runs. + case parameterized(arguments: [Argument], discriminator: Int, isStable: Bool) + } + + /// The kind of this test case. + private var _kind: _Kind + /// A type representing an argument passed to a parameter of a parameterized /// test function. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) @@ -26,41 +50,36 @@ extension Test { /// The raw bytes of this instance's identifier. public var bytes: [UInt8] - public init(bytes: [UInt8]) { - self.bytes = bytes + init(bytes: some Sequence) { + self.bytes = Array(bytes) } } - /// The ID of this parameterized test argument, if any. + /// The value of this parameterized test argument. + public var value: any Sendable + + /// The ID of this parameterized test argument. /// /// The uniqueness of this value is narrow: it is considered unique only /// within the scope of the parameter of the test function this argument /// was passed to. /// - /// The value of this property is `nil` when an ID cannot be formed. This - /// may occur if the type of ``value`` does not conform to one of the - /// protocols used for encoding a stable and unique representation of the - /// value. - /// /// ## See Also /// /// - ``CustomTestArgumentEncodable`` - @_spi(ForToolsIntegrationOnly) - public var id: ID? { - // FIXME: Capture the error and propagate to the user, not as a test - // failure but as an advisory warning. A missing argument ID will - // prevent re-running the test case, but is not a blocking issue. - try? Argument.ID(identifying: value, parameter: parameter) - } - - /// The value of this parameterized test argument. - public var value: any Sendable + public var id: ID /// The parameter of the test function to which this argument was passed. public var parameter: Parameter + + init(id: ID, value: any Sendable, parameter: Parameter) { + self.id = id + self.value = value + self.parameter = parameter + } } - /// The arguments passed to this test case. + /// The arguments passed to this test case, if any. /// /// If the argument was a tuple but its elements were passed to distinct /// parameters of the test function, each element of the tuple will be @@ -70,19 +89,87 @@ extension Test { /// represented as one ``Argument`` instance. /// /// Non-parameterized test functions will have a single test case instance, - /// and the value of this property will be an empty array for such test - /// cases. + /// and the value of this property will be `nil` for such test cases. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) - public var arguments: [Argument] + public var arguments: [Argument]? { + switch _kind { + case .nonParameterized: + nil + case let .parameterized(arguments, _, _): + arguments + } + } - init( - arguments: [Argument], - body: @escaping @Sendable () async throws -> Void - ) { - self.arguments = arguments + /// A number used to distinguish this test case from others associated with + /// the same parameterized test function whose arguments have the same ID. + /// + /// As an example, imagine the same argument is passed more than once to a + /// parameterized test: + /// + /// ```swift + /// @Test(arguments: [1, 1]) + /// func example(x: Int) { ... } + /// ``` + /// + /// There will be two ``Test/Case`` instances associated with this test + /// function. Each will represent one instance of the repeated argument `1`, + /// and each will have a different value for this property. + /// + /// The value of this property for successive runs of the same test are not + /// guaranteed to be the same. The value of this property may be equal for + /// two test cases associated with the same test if the IDs of their + /// arguments are different. The value of this property is `nil` for the + /// single test case associated with a non-parameterized test function. + @_spi(Experimental) @_spi(ForToolsIntegrationOnly) + public internal(set) var discriminator: Int? { + get { + switch _kind { + case .nonParameterized: + nil + case let .parameterized(_, discriminator, _): + discriminator + } + } + set { + switch _kind { + case .nonParameterized: + precondition(newValue == nil, "A non-nil discriminator may only be set for a test case which is parameterized.") + case let .parameterized(arguments, _, isStable): + guard let newValue else { + preconditionFailure("A nil discriminator may only be set for a test case which is not parameterized.") + } + _kind = .parameterized(arguments: arguments, discriminator: newValue, isStable: isStable) + } + } + } + + /// Whether or not this test case is considered stable across successive + /// runs. + @_spi(Experimental) @_spi(ForToolsIntegrationOnly) + public var isStable: Bool { + switch _kind { + case .nonParameterized: + true + case let .parameterized(_, _, isStable): + isStable + } + } + + private init(kind: _Kind, body: @escaping @Sendable () async throws -> Void) { + self._kind = kind self.body = body } + /// Initialize a test case for a non-parameterized test function. + /// + /// - Parameters: + /// - body: The body closure of this test case. + /// + /// The resulting test case will have zero arguments. + init(body: @escaping @Sendable () async throws -> Void) { + self.init(kind: .nonParameterized, body: body) + } + /// Initialize a test case by pairing values with their corresponding /// parameters to form the ``arguments`` array. /// @@ -95,15 +182,50 @@ extension Test { parameters: [Parameter], body: @escaping @Sendable () async throws -> Void ) { + var isStable = true + let arguments = zip(values, parameters).map { value, parameter in - Argument(value: value, parameter: parameter) + var stableArgumentID: Argument.ID? + + // Attempt to get a stable, encoded representation of this value if no + // such attempts for previous values have failed. + if isStable { + do { + stableArgumentID = try .init(identifying: value, parameter: parameter) + } catch { + // FIXME: Capture the error and propagate to the user, not as a test + // failure but as an advisory warning. A missing stable argument ID + // will prevent re-running the test case, but isn't a blocking issue. + } + } + + let argumentID: Argument.ID + if let stableArgumentID { + argumentID = stableArgumentID + } else { + // If we couldn't get a stable representation of at least one value, + // give up and consider the overall test case non-stable. This allows + // skipping unnecessary work later: if any individual argument doesn't + // have a stable ID, there's no point encoding the values which _are_ + // encodable. + isStable = false + argumentID = .init(bytes: String(describingForTest: value).utf8) + } + + return Argument(id: argumentID, value: value, parameter: parameter) } - self.init(arguments: arguments, body: body) + + self.init(kind: .parameterized(arguments: arguments, discriminator: 0, isStable: isStable), body: body) } /// Whether or not this test case is from a parameterized test. public var isParameterized: Bool { - !arguments.isEmpty + switch _kind { + case .nonParameterized: + false + case .parameterized: + true + } } /// The body closure of this test case. @@ -188,7 +310,11 @@ extension Test.Case { /// - testCase: The original test case to snapshot. public init(snapshotting testCase: borrowing Test.Case) { id = testCase.id - arguments = testCase.arguments.map(Test.Case.Argument.Snapshot.init) + arguments = if let arguments = testCase.arguments { + arguments.map(Test.Case.Argument.Snapshot.init) + } else { + [] + } } } } diff --git a/Tests/TestingTests/EventTests.swift b/Tests/TestingTests/EventTests.swift index 941dcadb9..653ad2a87 100644 --- a/Tests/TestingTests/EventTests.swift +++ b/Tests/TestingTests/EventTests.swift @@ -57,7 +57,7 @@ struct EventTests { let testID = Test.ID(moduleName: "ModuleName", nameComponents: ["NameComponent1", "NameComponent2"], sourceLocation: #_sourceLocation) - let testCaseID = Test.Case.ID(argumentIDs: nil) + let testCaseID = Test.Case.ID(argumentIDs: nil, discriminator: nil, isStable: true) let event = Event(kind, testID: testID, testCaseID: testCaseID, instant: .now) let eventSnapshot = Event.Snapshot(snapshotting: event) let decoded = try JSON.encodeAndDecode(eventSnapshot) diff --git a/Tests/TestingTests/Test.Case.Argument.IDTests.swift b/Tests/TestingTests/Test.Case.Argument.IDTests.swift index ced76adac..052213912 100644 --- a/Tests/TestingTests/Test.Case.Argument.IDTests.swift +++ b/Tests/TestingTests/Test.Case.Argument.IDTests.swift @@ -20,10 +20,10 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) - #expect(String(decoding: argumentID.bytes, as: UTF8.self) == "123") + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) + #expect(String(decoding: argument.id.bytes, as: UTF8.self) == "123") } @Test("One CustomTestArgumentEncodable parameter") @@ -34,11 +34,11 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) #if canImport(Foundation) - let decodedArgument = try argumentID.bytes.withUnsafeBufferPointer { argumentID in + let decodedArgument = try argument.id.bytes.withUnsafeBufferPointer { argumentID in try JSON.decode(MyCustomTestArgument.self, from: .init(argumentID)) } #expect(decodedArgument == MyCustomTestArgument(x: 123, y: "abc")) @@ -53,10 +53,10 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) - #expect(String(decoding: argumentID.bytes, as: UTF8.self) == #""abc""#) + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) + #expect(String(decoding: argument.id.bytes, as: UTF8.self) == #""abc""#) } @Test("One RawRepresentable parameter") @@ -67,10 +67,10 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) - #expect(String(decoding: argumentID.bytes, as: UTF8.self) == #""abc""#) + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) + #expect(String(decoding: argument.id.bytes, as: UTF8.self) == #""abc""#) } } diff --git a/Tests/TestingTests/Test.Case.ArgumentTests.swift b/Tests/TestingTests/Test.Case.ArgumentTests.swift index a5c5e7462..4ea9925d6 100644 --- a/Tests/TestingTests/Test.Case.ArgumentTests.swift +++ b/Tests/TestingTests/Test.Case.ArgumentTests.swift @@ -19,10 +19,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? String == "value") #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") @@ -38,17 +38,17 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 2) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 2) do { - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? String == "value") #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") } do { - let argument = testCase.arguments[1] + let argument = arguments[1] #expect(argument.value as? Int == 123) #expect(argument.parameter.index == 1) #expect(argument.parameter.firstName == "y") @@ -65,10 +65,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? (String) == ("value")) #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") @@ -84,10 +84,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] let value = try #require(argument.value as? (String, Int)) #expect(value.0 == "value") #expect(value.1 == 123) @@ -105,17 +105,17 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 2) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 2) do { - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? String == "value") #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") } do { - let argument = testCase.arguments[1] + let argument = arguments[1] #expect(argument.value as? Int == 123) #expect(argument.parameter.index == 1) #expect(argument.parameter.firstName == "y") @@ -132,10 +132,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] let value = try #require(argument.value as? (String, Int)) #expect(value.0 == "value") #expect(value.1 == 123) diff --git a/Tests/TestingTests/Test.Case.GeneratorTests.swift b/Tests/TestingTests/Test.Case.GeneratorTests.swift new file mode 100644 index 000000000..0c2afd0ef --- /dev/null +++ b/Tests/TestingTests/Test.Case.GeneratorTests.swift @@ -0,0 +1,31 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Test.Case.Generator Tests") +struct Test_Case_GeneratorTests { + @Test func uniqueDiscriminators() throws { + let generator = Test.Case.Generator( + arguments: [1, 1, 1], + parameters: [Test.Parameter(index: 0, firstName: "x", type: Int.self)], + testFunction: { _ in } + ) + + let testCases = Array(generator) + #expect(testCases.count == 3) + + let firstCase = try #require(testCases.first) + #expect(firstCase.id.discriminator == 0) + + let discriminators = Set(testCases.map(\.id.discriminator)) + #expect(discriminators.count == 3) + } +} diff --git a/Tests/TestingTests/Test.CaseTests.swift b/Tests/TestingTests/Test.CaseTests.swift new file mode 100644 index 000000000..f9e955fe9 --- /dev/null +++ b/Tests/TestingTests/Test.CaseTests.swift @@ -0,0 +1,177 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +#if canImport(Foundation) +private import Foundation +#endif + +@Suite("Test.Case Tests") +struct Test_CaseTests { + @Test func nonParameterized() throws { + let testCase = Test.Case(body: {}) + #expect(testCase.id.argumentIDs == nil) + #expect(testCase.id.discriminator == nil) + } + + @Test func singleStableArgument() throws { + let testCase = Test.Case( + values: [1], + parameters: [Test.Parameter(index: 0, firstName: "x", type: Int.self)], + body: {} + ) + #expect(testCase.id.isStable) + } + + @Test func twoStableArguments() throws { + let testCase = Test.Case( + values: [1, "a"], + parameters: [ + Test.Parameter(index: 0, firstName: "x", type: Int.self), + Test.Parameter(index: 1, firstName: "y", type: String.self), + ], + body: {} + ) + #expect(testCase.id.isStable) + } + + @Test("Two arguments: one non-stable, followed by one stable") + func nonStableAndStableArgument() throws { + let testCase = Test.Case( + values: [NonCodable(), IssueRecordingEncodable()], + parameters: [ + Test.Parameter(index: 0, firstName: "x", type: NonCodable.self), + Test.Parameter(index: 1, firstName: "y", type: IssueRecordingEncodable.self), + ], + body: {} + ) + #expect(!testCase.id.isStable) + } + + @Suite("Test.Case.ID Tests") + struct IDTests { +#if canImport(Foundation) + @Test(arguments: [ + Test.Case.ID(argumentIDs: nil, discriminator: nil, isStable: true), + Test.Case.ID(argumentIDs: [.init(bytes: "x".utf8)], discriminator: 0, isStable: false), + Test.Case.ID(argumentIDs: [.init(bytes: #""abc""#.utf8)], discriminator: 0, isStable: true), + ]) + func roundTripping(id: Test.Case.ID) throws { + #expect(try JSON.encodeAndDecode(id) == id) + } + + @Test func legacyDecoding_stable() throws { + let encodedData = Data(""" + {"argumentIDs": [ + {"bytes": [1]} + ]} + """.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + + let argumentIDs = try #require(testCaseID.argumentIDs) + #expect(argumentIDs.count == 1) + } + + @Test func legacyDecoding_nonStable() throws { + let encodedData = Data("{}".utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(!testCaseID.isStable) + + let argumentIDs = try #require(testCaseID.argumentIDs) + #expect(argumentIDs.count == 1) + } + + @Test func legacyDecoding_nonParameterized() throws { + let encodedData = Data(#"{"argumentIDs": []}"#.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + #expect(testCaseID.argumentIDs == nil) + #expect(testCaseID.discriminator == nil) + } + + @Test func newDecoding_nonParameterized() throws { + let encodedData = Data(#"{"isStable": true}"#.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + #expect(testCaseID.argumentIDs == nil) + #expect(testCaseID.discriminator == nil) + } + + @Test func newDecoding_parameterizedStable() throws { + let encodedData = Data(""" + { + "isStable": true, + "argIDs": [ + {"bytes": [1]} + ], + "discriminator": 0 + } + """.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + #expect(testCaseID.argumentIDs?.count == 1) + #expect(testCaseID.discriminator == 0) + } + + @Test func newEncoding_nonParameterized() throws { + let id = Test.Case.ID(argumentIDs: nil, discriminator: nil, isStable: true) + let legacyID = try JSON.withEncoding(of: id) { data in + try JSON.decode(_LegacyTestCaseID.self, from: data) + } + let argumentIDs = try #require(legacyID.argumentIDs) + #expect(argumentIDs.isEmpty) + } + + @Test func newEncoding_parameterizedNonStable() throws { + let id = Test.Case.ID( + argumentIDs: [.init(bytes: "x".utf8)], + discriminator: 0, + isStable: false + ) + let legacyID = try JSON.withEncoding(of: id) { data in + try JSON.decode(_LegacyTestCaseID.self, from: data) + } + #expect(legacyID.argumentIDs == nil) + } + + @Test func newEncoding_parameterizedStable() throws { + let id = Test.Case.ID( + argumentIDs: [.init(bytes: #""abc""#.utf8)], + discriminator: 0, + isStable: true + ) + let legacyID = try JSON.withEncoding(of: id) { data in + try JSON.decode(_LegacyTestCaseID.self, from: data) + } + let argumentIDs = try #require(legacyID.argumentIDs) + #expect(argumentIDs.count == 1) + let argumentID = try #require(argumentIDs.first) + #expect(String(decoding: argumentID.bytes, as: UTF8.self) == #""abc""#) + } +#endif + } +} + +// MARK: - Fixtures, helpers + +private struct NonCodable {} + +private struct IssueRecordingEncodable: Encodable { + func encode(to encoder: any Encoder) throws { + Issue.record("Unexpected attempt to encode an instance of \(Self.self)") + } +} + +/// A fixture type which implements legacy decoding for ``Test/Case/ID``. +private struct _LegacyTestCaseID: Decodable { + var argumentIDs: [Test.Case.Argument.ID]? +} diff --git a/Tests/TestingTests/TestCaseSelectionTests.swift b/Tests/TestingTests/TestCaseSelectionTests.swift index de8b10c66..d24ddf8f6 100644 --- a/Tests/TestingTests/TestCaseSelectionTests.swift +++ b/Tests/TestingTests/TestCaseSelectionTests.swift @@ -85,8 +85,9 @@ struct TestCaseSelectionTests { } let selectedTestCase = try #require(fixtureTest.testCases?.first { testCase in - guard let firstArg = testCase.arguments.first?.value as? String, - let secondArg = testCase.arguments.last?.value as? Int + guard let arguments = testCase.arguments, + let firstArg = arguments.first?.value as? String, + let secondArg = arguments.last?.value as? Int else { return false } diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 5a0121444..6807fd62a 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -9,10 +9,15 @@ // @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + #if canImport(XCTest) import XCTest #endif +#if canImport(Foundation) +import Foundation +#endif + extension Tag { /// A tag indicating that a test is related to a trait. @Tag static var traitRelated: Self @@ -348,6 +353,24 @@ extension JSON { try JSON.decode(T.self, from: data) } } + +#if canImport(Foundation) + /// Decode a value from JSON data. + /// + /// - Parameters: + /// - type: The type of value to decode. + /// - jsonRepresentation: Data of the JSON encoding of the value to decode. + /// + /// - Returns: An instance of `T` decoded from `jsonRepresentation`. + /// + /// - Throws: Whatever is thrown by the decoding process. + @_disfavoredOverload + static func decode(_ type: T.Type, from jsonRepresentation: Data) throws -> T where T: Decodable { + try jsonRepresentation.withUnsafeBytes { bytes in + try JSON.decode(type, from: bytes) + } + } +#endif } @available(_clockAPI, *) From 6bc562dc3baa219bb3a6555b30493c4166795979 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 17 Mar 2025 13:00:29 -0500 Subject: [PATCH 137/234] Adjust a timing-sensitive TimeLimitTrait test to avoid flakiness in CI (#1029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adjusts the `TimeLimitTraitTests.cancelledTestExitsEarly()` test function to make it less sensitive to running on resource-constrained CI systems. ### Motivation: Recently on a few PRs I have seen flaky results from this test which are unrelated to my work. They typically show up as console output such as: ``` ✘ Test "Cancelled tests can exit early (cancellation checking works)" recorded an issue at TimeLimitTraitTests.swift:193:5: Expectation failed: (timeAwaited → 5.108262525 seconds) < (.seconds(5) → 5.0 seconds) ``` This indicates the test is just a little too sensitive to specific timing. ### Modifications: - Lengthen the "full" sleep duration the test would run if it wasn't cancelled, which it is. - Extend the much shorter duration it allows for waiting, to avoid false positives when running on resource-constrained CI systems. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Tests/TestingTests/Traits/TimeLimitTraitTests.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift index 00c9cbb44..b29ccb93c 100644 --- a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift +++ b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift @@ -184,13 +184,17 @@ struct TimeLimitTraitTests { await withTaskGroup(of: Void.self) { taskGroup in taskGroup.addTask { await Test { - try await Test.Clock.sleep(for: .seconds(60)) + try await Test.Clock.sleep(for: .seconds(60) * 60) }.run() } taskGroup.cancelAll() } } - #expect(timeAwaited < .seconds(5)) // less than the 60 second sleep + + // Expect that the time awaited is significantly less than the duration of + // the sleep above. To avoid flakiness in CI, allow for a somewhat long + // wait, but still much less than the full sleep duration. + #expect(timeAwaited < .seconds(60)) } @available(_clockAPI, *) From cf94bdc1cdf6bb552b5ae8432e00b291b1950143 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 19 Mar 2025 15:06:46 -0400 Subject: [PATCH 138/234] Rewrite static section bounds discovery in Swift. (#1012) This PR rewrites the static section bounds discovery code in Swift (replacing the current C++ implementation.) We are making use of `@_silgen_name` here to name the section bounds symbols that are defined by the linker, so we'll want to get the core team's approval before merging. ### Why do we need to rewrite this code in Swift? In Embedded Swift, there is only one (statically linked) image in the process containing Swift code, and if the target is a microcontroller or similar, it probably doesn't even have a dynamic linker/loader in the first place for us to query. So we need, on Embedded Swift targets, to perform static discovery of exactly one test content section. (This is potentially something we can make configurable via the target's toolset in a future PR.) The bounds of the test content section are currently defined in C++ using `__asm__` to reference linker-defined symbols. However, our C++ target can't reliably tell if it is being built for Embedded Swift. That knowledge is treated by the Swift toolchain as a Swift language feature and can be checked in Swift with `hasFeature(Embedded)`, but there's no C++ equivalent. If the implementation is written in Swift, we can know at compile time whether or not we're dynamically or statically linking to a test target (and therefore whether we need to do dynamic or static lookup of images, customized toolset aside.) Rewriting this logic in Swift reduces our C++ code size (and our direct dependencies on the C++ STL) which is also a nice bonus. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/Porting.md | 57 +++++++------ Sources/_TestDiscovery/SectionBounds.swift | 79 ++++++++++++++++--- .../_TestDiscovery/TestContentRecord.swift | 3 +- Sources/_TestingInternals/Discovery.cpp | 46 +---------- Sources/_TestingInternals/include/Discovery.h | 17 +--- 5 files changed, 104 insertions(+), 98 deletions(-) diff --git a/Documentation/Porting.md b/Documentation/Porting.md index 8b230ff22..6e83e0eb0 100644 --- a/Documentation/Porting.md +++ b/Documentation/Porting.md @@ -136,8 +136,8 @@ emits section information into the resource fork on Classic, you would use the to load that information: ```diff ---- a/Sources/Testing/Discovery+Platform.swift -+++ b/Sources/Testing/Discovery+Platform.swift +--- a/Sources/_TestDiscovery/SectionBounds.swift ++++ b/Sources/_TestDiscovery/SectionBounds.swift // ... +#elseif os(Classic) @@ -176,10 +176,13 @@ to load that information: + } while noErr == GetNextResourceFile(refNum, &refNum)) + return result +} - #else - private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { - #warning("Platform-specific implementation missing: Runtime test discovery unavailable (dynamic)") - return [] ++ + #elseif !SWT_NO_DYNAMIC_LINKING + // MARK: - Missing dynamic implementation + + private func _sectionBounds(_ kind: SectionBounds.Kind) -> EmptyCollection { + #warning("Platform-specific implementation missing: Runtime test discovery unavailable (dynamic)") + return EmptyCollection() } #endif ``` @@ -211,36 +214,42 @@ start with `"SWT_"`). If your platform does not support dynamic linking and loading, you will need to use static linkage instead. Define the `"SWT_NO_DYNAMIC_LINKING"` compiler conditional for your platform in both `Package.swift` and -`CompilerSettings.cmake`, then define the symbols `testContentSectionBegin`, -`testContentSectionEnd`, `typeMetadataSectionBegin`, and -`typeMetadataSectionEnd` in `Discovery.cpp`. +`CompilerSettings.cmake`, then define the symbols `_testContentSectionBegin`, +`_testContentSectionEnd`, `_typeMetadataSectionBegin`, and +`_typeMetadataSectionEnd` in `SectionBounds.swift`: ```diff -diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp +--- a/Sources/_TestDiscovery/SectionBounds.swift ++++ b/Sources/_TestDiscovery/SectionBounds.swift // ... -+#elif defined(macintosh) -+extern "C" const char testContentSectionBegin __asm__("..."); -+extern "C" const char testContentSectionEnd __asm__("..."); -+#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) -+extern "C" const char typeMetadataSectionBegin __asm__("..."); -+extern "C" const char typeMetadataSectionEnd __asm__("..."); ++#elseif os(Classic) ++@_silgen_name(raw: "...") private nonisolated(unsafe) var _testContentSectionBegin: _SectionBound ++@_silgen_name(raw: "...") private nonisolated(unsafe) var _testContentSectionEnd: _SectionBound ++#if !SWT_NO_LEGACY_TEST_DISCOVERY ++@_silgen_name(raw: "...") private nonisolated(unsafe) var _typeMetadataSectionBegin: _SectionBound ++@_silgen_name(raw: "...") private nonisolated(unsafe) var _typeMetadataSectionEnd: _SectionBound +#endif #else - #warning Platform-specific implementation missing: Runtime test discovery unavailable (static) - static const char testContentSectionBegin = 0; - static const char& testContentSectionEnd = testContentSectionBegin; - #if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) - static const char typeMetadataSectionBegin = 0; - static const char& typeMetadataSectionEnd = typeMetadataSectionBegin; + #warning("Platform-specific implementation missing: Runtime test discovery unavailable (static)") + private nonisolated(unsafe) let _testContentSectionBegin = UnsafeMutableRawPointer.allocate(byteCount: 1, alignment: 16) + private nonisolated(unsafe) let _testContentSectionEnd = _testContentSectionBegin + #if !SWT_NO_LEGACY_TEST_DISCOVERY + private nonisolated(unsafe) let _typeMetadataSectionBegin = UnsafeMutableRawPointer.allocate(byteCount: 1, alignment: 16) + private nonisolated(unsafe) let _typeMetadataSectionEnd = _typeMetadataSectionBegin #endif #endif + // ... ``` These symbols must have unique addresses corresponding to the first byte of the test content section and the first byte _after_ the test content section, respectively. Their linker-level names will be platform-dependent: refer to the linker documentation for your platform to determine what names to place in the -`__asm__` attribute applied to each. +`@_silgen_name` attribute applied to each. + +If your target platform statically links Swift Testing but the linker does not +define section bounds symbols, please reach out to us in the Swift forums for +advice. ## C++ stub implementations @@ -332,7 +341,7 @@ to include the necessary linker flags. ## Adding CI jobs for the new platform The Swift project maintains a set of CI jobs that target various platforms. To -add CI jobs for Swift Testing or the Swift toolchain, please contain the CI +add CI jobs for Swift Testing or the Swift toolchain, please contact the CI maintainers on the Swift forums. If you wish to host your own CI jobs, let us know: we'd be happy to run them as diff --git a/Sources/_TestDiscovery/SectionBounds.swift b/Sources/_TestDiscovery/SectionBounds.swift index 212edbfbf..1b4cbaa2a 100644 --- a/Sources/_TestDiscovery/SectionBounds.swift +++ b/Sources/_TestDiscovery/SectionBounds.swift @@ -23,7 +23,7 @@ struct SectionBounds: Sendable { /// An enumeration describing the different sections discoverable by the /// testing library. - enum Kind: Int, Equatable, Hashable, CaseIterable { + enum Kind: Equatable, Hashable, CaseIterable { /// The test content metadata section. case testContent @@ -45,8 +45,7 @@ struct SectionBounds: Sendable { } } -#if !SWT_NO_DYNAMIC_LINKING -#if SWT_TARGET_OS_APPLE +#if SWT_TARGET_OS_APPLE && !SWT_NO_DYNAMIC_LINKING // MARK: - Apple implementation extension SectionBounds.Kind { @@ -157,7 +156,7 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { } } -#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) +#elseif (os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android)) && !SWT_NO_DYNAMIC_LINKING // MARK: - ELF implementation private import SwiftShims // For MetadataSections @@ -278,6 +277,9 @@ private func _findSection(named sectionName: String, in hModule: HMODULE) -> Sec /// /// - Returns: An array of structures describing the bounds of all known test /// content sections in the current process. +/// +/// This implementation is always used on Windows (even when the testing library +/// is statically linked.) private func _sectionBounds(_ kind: SectionBounds.Kind) -> some Sequence { let sectionName = switch kind { case .testContent: @@ -289,7 +291,10 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> some Sequence
some Sequence
[SectionBounds] { - #warning("Platform-specific implementation missing: Runtime test discovery unavailable (dynamic)") - return [] +private func _sectionBounds(_ kind: SectionBounds.Kind) -> EmptyCollection { +#warning("Platform-specific implementation missing: Runtime test discovery unavailable (dynamic)") + return EmptyCollection() } -#endif + #else // MARK: - Statically-linked implementation +/// A type representing the upper or lower bound of a metadata section. +/// +/// This type is move-only and instances of it are declared as global mutable +/// variables below to ensure that they have fixed addresses. Use the `..<` +/// operator to get a range of addresses from two instances of this type. +/// +/// On platforms that use static linkage and have well-defined bounds symbols, +/// those symbols are imported into Swift below using the experimental +/// `@_silgen_name` attribute. +private struct _SectionBound: Sendable, ~Copyable { + /// A property that forces the structure to have an in-memory representation. + private var _storage: CChar = 0 + + static func ..<(lhs: inout Self, rhs: inout Self) -> Range { + withUnsafeMutablePointer(to: &lhs) { lhs in + withUnsafeMutablePointer(to: &rhs) { rhs in + UnsafeRawPointer(lhs) ..< UnsafeRawPointer(rhs) + } + } + } +} + +#if SWT_TARGET_OS_APPLE +@_silgen_name(raw: "section$start$__DATA_CONST$__swift5_tests") private nonisolated(unsafe) var _testContentSectionBegin: _SectionBound +@_silgen_name(raw: "section$end$__DATA_CONST$__swift5_tests") private nonisolated(unsafe) var _testContentSectionEnd: _SectionBound +#if !SWT_NO_LEGACY_TEST_DISCOVERY +@_silgen_name(raw: "section$start$__TEXT$__swift5_types") private nonisolated(unsafe) var _typeMetadataSectionBegin: _SectionBound +@_silgen_name(raw: "section$end$__TEXT$__swift5_types") private nonisolated(unsafe) var _typeMetadataSectionEnd: _SectionBound +#endif +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) +@_silgen_name(raw: "__start_swift5_tests") private nonisolated(unsafe) var _testContentSectionBegin: _SectionBound +@_silgen_name(raw: "__stop_swift5_tests") private nonisolated(unsafe) var _testContentSectionEnd: _SectionBound +#if !SWT_NO_LEGACY_TEST_DISCOVERY +@_silgen_name(raw: "__start_swift5_type_metadata") private nonisolated(unsafe) var _typeMetadataSectionBegin: _SectionBound +@_silgen_name(raw: "__stop_swift5_type_metadata") private nonisolated(unsafe) var _typeMetadataSectionEnd: _SectionBound +#endif +#else +#warning("Platform-specific implementation missing: Runtime test discovery unavailable (static)") +private nonisolated(unsafe) let _testContentSectionBegin = UnsafeMutableRawPointer.allocate(byteCount: 1, alignment: 16) +private nonisolated(unsafe) let _testContentSectionEnd = _testContentSectionBegin +#if !SWT_NO_LEGACY_TEST_DISCOVERY +private nonisolated(unsafe) let _typeMetadataSectionBegin = UnsafeMutableRawPointer.allocate(byteCount: 1, alignment: 16) +private nonisolated(unsafe) let _typeMetadataSectionEnd = _typeMetadataSectionBegin +#endif +#endif + /// The common implementation of ``SectionBounds/all(_:)`` for platforms that do /// not support dynamic linking. /// @@ -314,9 +365,13 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> [SectionBounds] { /// - Returns: A structure describing the bounds of the type metadata section /// contained in the same image as the testing library itself. private func _sectionBounds(_ kind: SectionBounds.Kind) -> CollectionOfOne { - var (baseAddress, count): (UnsafeRawPointer?, Int) = (nil, 0) - swt_getStaticallyLinkedSectionBounds(kind.rawValue, &baseAddress, &count) - let buffer = UnsafeRawBufferPointer(start: baseAddress, count: count) + let range = switch kind { + case .testContent: + _testContentSectionBegin ..< _testContentSectionEnd + case .typeMetadata: + _typeMetadataSectionBegin ..< _typeMetadataSectionEnd + } + let buffer = UnsafeRawBufferPointer(start: range.lowerBound, count: range.count) let sb = SectionBounds(imageAddress: nil, buffer: buffer) return CollectionOfOne(sb) } diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index 5235d9c64..25f46fa44 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -276,7 +276,8 @@ extension DiscoverableAsTestContent where Self: ~Copyable { } let result = SectionBounds.all(.typeMetadata).lazy.flatMap { sb in - stride(from: sb.buffer.baseAddress!, to: sb.buffer.baseAddress! + sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy + stride(from: 0, to: sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy + .map { sb.buffer.baseAddress! + $0 } .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: typeNameHint) } .map { unsafeBitCast($0, to: Any.Type.self) } .compactMap(loader) diff --git a/Sources/_TestingInternals/Discovery.cpp b/Sources/_TestingInternals/Discovery.cpp index ad898534f..1e70a038d 100644 --- a/Sources/_TestingInternals/Discovery.cpp +++ b/Sources/_TestingInternals/Discovery.cpp @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,55 +10,11 @@ #include "Discovery.h" -#include #if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) #include #include #include -#endif - -#if defined(SWT_NO_DYNAMIC_LINKING) -#pragma mark - Statically-linked section bounds -#if defined(__APPLE__) -extern "C" const char testContentSectionBegin __asm("section$start$__DATA_CONST$__swift5_tests"); -extern "C" const char testContentSectionEnd __asm("section$end$__DATA_CONST$__swift5_tests"); -#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) -extern "C" const char typeMetadataSectionBegin __asm__("section$start$__TEXT$__swift5_types"); -extern "C" const char typeMetadataSectionEnd __asm__("section$end$__TEXT$__swift5_types"); -#endif -#elif defined(__wasi__) -extern "C" const char testContentSectionBegin __asm__("__start_swift5_tests"); -extern "C" const char testContentSectionEnd __asm__("__stop_swift5_tests"); -#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) -extern "C" const char typeMetadataSectionBegin __asm__("__start_swift5_type_metadata"); -extern "C" const char typeMetadataSectionEnd __asm__("__stop_swift5_type_metadata"); -#endif -#else -#warning Platform-specific implementation missing: Runtime test discovery unavailable (static) -static const char testContentSectionBegin = 0; -static const char& testContentSectionEnd = testContentSectionBegin; -#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) -static const char typeMetadataSectionBegin = 0; -static const char& typeMetadataSectionEnd = typeMetadataSectionBegin; -#endif -#endif - -static constexpr const char *const staticallyLinkedSectionBounds[][2] = { - { &testContentSectionBegin, &testContentSectionEnd }, -#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) - { &typeMetadataSectionBegin, &typeMetadataSectionEnd }, -#endif -}; - -void swt_getStaticallyLinkedSectionBounds(size_t kind, const void **outSectionBegin, size_t *outByteCount) { - auto [sectionBegin, sectionEnd] = staticallyLinkedSectionBounds[kind]; - *outSectionBegin = sectionBegin; - *outByteCount = std::distance(sectionBegin, sectionEnd); -} -#endif - -#if !defined(SWT_NO_LEGACY_TEST_DISCOVERY) #pragma mark - Swift ABI #if defined(__PTRAUTH_INTRINSICS__) diff --git a/Sources/_TestingInternals/include/Discovery.h b/Sources/_TestingInternals/include/Discovery.h index 9bda12b93..25c3603b3 100644 --- a/Sources/_TestingInternals/include/Discovery.h +++ b/Sources/_TestingInternals/include/Discovery.h @@ -1,7 +1,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -30,21 +30,6 @@ SWT_IMPORT_FROM_STDLIB void swift_enumerateAllMetadataSections( ); #endif -#pragma mark - Statically-linked section bounds - -/// Get the bounds of a statically linked section in this image. -/// -/// - Parameters: -/// - kind: The value of `SectionBounds.Kind.rawValue` for the given section. -/// - outSectionBegin: On return, a pointer to the first byte of the section. -/// - outByteCount: On return, the number of bytes in the section. -/// -/// - Note: This symbol is _declared_, but not _defined_, on platforms with -/// dynamic linking because the `SWT_NO_DYNAMIC_LINKING` C++ macro (not the -/// Swift compiler conditional of the same name) is not consistently declared -/// when Swift files import the `_TestingInternals` C++ module. -SWT_EXTERN void swt_getStaticallyLinkedSectionBounds(size_t kind, const void *_Nullable *_Nonnull outSectionBegin, size_t *outByteCount); - #pragma mark - Legacy test discovery /// The size, in bytes, of a Swift type metadata record. From e242d3e21a195c7ea2a43bf6d569de452aa9dac1 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Wed, 19 Mar 2025 12:07:04 -0700 Subject: [PATCH 139/234] build: account for spaces in the toolchain (#1030) If the macro library path contains a space, the invocation will improperly truncate it. Account for this eventuality. --- Sources/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 1211ffcc1..8106eb2a3 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -94,10 +94,10 @@ if(NOT SwiftTesting_MACRO_PATH) elseif(SwiftTesting_MACRO_PATH) if(SwiftTesting_MACRO_PATH MATCHES [[\.(dylib|so|dll)$]]) message(STATUS "TestingMacros: ${SwiftTesting_MACRO_PATH} (shared library)") - add_compile_options("$<$:SHELL:-load-plugin-library ${SwiftTesting_MACRO_PATH}>") + add_compile_options("$<$:SHELL:-load-plugin-library \"${SwiftTesting_MACRO_PATH}\">") else() message(STATUS "TestingMacros: ${SwiftTesting_MACRO_PATH} (executable)") - add_compile_options("$<$:SHELL:-load-plugin-executable ${SwiftTesting_MACRO_PATH}#TestingMacros>") + add_compile_options("$<$:SHELL:-load-plugin-executable \"${SwiftTesting_MACRO_PATH}#TestingMacros\">") endif() endif() From 65ca016b3f8e00c1ef357e4a7aeabb3c0a217b09 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 19 Mar 2025 15:27:58 -0400 Subject: [PATCH 140/234] Insert missing '!SWT_NO_LEGACY_TEST_DISCOVERY' guard --- Sources/_TestDiscovery/SectionBounds.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/_TestDiscovery/SectionBounds.swift b/Sources/_TestDiscovery/SectionBounds.swift index 1b4cbaa2a..1a3ae8e11 100644 --- a/Sources/_TestDiscovery/SectionBounds.swift +++ b/Sources/_TestDiscovery/SectionBounds.swift @@ -368,8 +368,10 @@ private func _sectionBounds(_ kind: SectionBounds.Kind) -> CollectionOfOne Date: Thu, 20 Mar 2025 17:43:55 -0400 Subject: [PATCH 141/234] Change `Attachment.attach()` to `Attachment.record()` and `Attachable.withUnsafeBufferPointer()` to `Attachable.withUnsafeBytes()`. (#1032) This PR updates the `attach()` function to make it static on `Attachment` and renames it `record()`, per the discussion [here](https://forums.swift.org/t/pitch-attachments/78072). It also renames `Attachable.withUnsafeBufferPointer()` to `Attachable.withUnsafeBytes()` to more closely match the convention used by the standard library for raw buffers. Finally, it removes the conditional dependence on the experimental language feature "SuppressedAssociatedTypes" in `AttachableContainer`. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 1 - .../_AttachableImageContainer.swift | 4 +- .../Attachable+Encodable+NSSecureCoding.swift | 4 +- .../Attachments/Attachable+Encodable.swift | 12 +-- .../Attachable+NSSecureCoding.swift | 2 +- .../Attachments/Data+Attachable.swift | 2 +- .../Attachments/_AttachableURLContainer.swift | 2 +- Sources/Testing/Attachments/Attachable.swift | 25 ++--- .../Attachments/AttachableContainer.swift | 14 +-- Sources/Testing/Attachments/Attachment.swift | 91 ++++++++++++++++--- Tests/TestingTests/AttachmentTests.swift | 51 ++++++----- cmake/modules/shared/CompilerSettings.cmake | 3 +- 12 files changed, 134 insertions(+), 77 deletions(-) diff --git a/Package.swift b/Package.swift index 3a66ef039..8085d7bc8 100644 --- a/Package.swift +++ b/Package.swift @@ -197,7 +197,6 @@ extension Array where Element == PackageDescription.SwiftSetting { result += [ .enableUpcomingFeature("ExistentialAny"), - .enableExperimentalFeature("SuppressedAssociatedTypes"), .enableExperimentalFeature("AccessLevelOnImport"), .enableUpcomingFeature("InternalImportsByDefault"), diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift index 9db225826..90d1c0c70 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift @@ -32,7 +32,7 @@ import UniformTypeIdentifiers /// such a requirement, and all image types we care about are non-final /// classes. Thus, the compiler will steadfastly refuse to allow non-final /// classes to conform to the `Attachable` protocol. We could get around this -/// by changing the signature of `withUnsafeBufferPointer()` so that the +/// by changing the signature of `withUnsafeBytes()` so that the /// generic parameter to `Attachment` is not `Self`, but that would defeat /// much of the purpose of making `Attachment` generic in the first place. /// (And no, the language does not let us write `where T: Self` anywhere @@ -132,7 +132,7 @@ extension _AttachableImageContainer: AttachableContainer { image } - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { let data = NSMutableData() // Convert the image to a CGImage. diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift index 80c75b5e9..cd26c24cc 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift @@ -21,8 +21,8 @@ public import Foundation @_spi(Experimental) extension Attachable where Self: Encodable & NSSecureCoding { @_documentation(visibility: private) - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { - try _Testing_Foundation.withUnsafeBufferPointer(encoding: self, for: attachment, body) + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + try _Testing_Foundation.withUnsafeBytes(encoding: self, for: attachment, body) } } #endif diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift index cfae97ca7..812db0b70 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift @@ -12,9 +12,9 @@ @_spi(Experimental) public import Testing private import Foundation -/// A common implementation of ``withUnsafeBufferPointer(for:_:)`` that is -/// used when a type conforms to `Encodable`, whether or not it also conforms -/// to `NSSecureCoding`. +/// A common implementation of ``withUnsafeBytes(for:_:)`` that is used when a +/// type conforms to `Encodable`, whether or not it also conforms to +/// `NSSecureCoding`. /// /// - Parameters: /// - attachableValue: The value to encode. @@ -27,7 +27,7 @@ private import Foundation /// /// - Throws: Whatever is thrown by `body`, or any error that prevented the /// creation of the buffer. -func withUnsafeBufferPointer(encoding attachableValue: borrowing E, for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R where E: Attachable & Encodable { +func withUnsafeBytes(encoding attachableValue: borrowing E, for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R where E: Attachable & Encodable { let format = try EncodingFormat(for: attachment) let data: Data @@ -86,8 +86,8 @@ extension Attachable where Self: Encodable { /// _and_ [`NSSecureCoding`](https://developer.apple.com/documentation/foundation/nssecurecoding), /// the default implementation of this function uses the value's conformance /// to `Encodable`. - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { - try _Testing_Foundation.withUnsafeBufferPointer(encoding: self, for: attachment, body) + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + try _Testing_Foundation.withUnsafeBytes(encoding: self, for: attachment, body) } } #endif diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift index c6916ec39..c2cc28ea0 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift @@ -46,7 +46,7 @@ extension Attachable where Self: NSSecureCoding { /// _and_ [`NSSecureCoding`](https://developer.apple.com/documentation/foundation/nssecurecoding), /// the default implementation of this function uses the value's conformance /// to `Encodable`. - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { let format = try EncodingFormat(for: attachment) var data = try NSKeyedArchiver.archivedData(withRootObject: self, requiringSecureCoding: true) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift index f931e5824..38233cd3c 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift @@ -14,7 +14,7 @@ public import Foundation @_spi(Experimental) extension Data: Attachable { - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } } diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift b/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift index 38f21d4d3..c7a223a51 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift @@ -36,7 +36,7 @@ extension _AttachableURLContainer: AttachableContainer { url } - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try data.withUnsafeBytes(body) } diff --git a/Sources/Testing/Attachments/Attachable.swift b/Sources/Testing/Attachments/Attachable.swift index 4a1d775a5..09a7e0b78 100644 --- a/Sources/Testing/Attachments/Attachable.swift +++ b/Sources/Testing/Attachments/Attachable.swift @@ -11,10 +11,11 @@ /// A protocol describing a type that can be attached to a test report or /// written to disk when a test is run. /// -/// To attach an attachable value to a test report or test run output, use it to -/// initialize a new instance of ``Attachment``, then call -/// ``Attachment/attach(sourceLocation:)``. An attachment can only be attached -/// once. +/// To attach an attachable value to a test, pass it to ``Attachment/record(_:named:sourceLocation:)``. +/// To further configure an attachable value before you attach it, use it to +/// initialize an instance of ``Attachment`` and set its properties before +/// passing it to ``Attachment/record(_:sourceLocation:)``. An attachable +/// value can only be attached to a test once. /// /// The testing library provides default conformances to this protocol for a /// variety of standard library types. Most user-defined types do not need to @@ -63,7 +64,7 @@ public protocol Attachable: ~Copyable { /// the buffer to contain an image in PNG format, JPEG format, etc., but it /// would not be idiomatic for the buffer to contain a textual description of /// the image. - borrowing func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R + borrowing func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R /// Generate a preferred name for the given attachment. /// @@ -99,8 +100,8 @@ extension Attachable where Self: Collection, Element == UInt8 { count } - // We do not provide an implementation of withUnsafeBufferPointer(for:_:) here - // because there is no way in the standard library to statically detect if a + // We do not provide an implementation of withUnsafeBytes(for:_:) here because + // there is no way in the standard library to statically detect if a // collection can provide contiguous storage (_HasContiguousBytes is not API.) // If withContiguousStorageIfAvailable(_:) fails, we don't want to make a // (potentially expensive!) copy of the collection. @@ -120,28 +121,28 @@ extension Attachable where Self: StringProtocol { // developers can attach raw data when needed. @_spi(Experimental) extension Array: Attachable { - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } } @_spi(Experimental) extension ContiguousArray: Attachable { - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } } @_spi(Experimental) extension ArraySlice: Attachable { - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } } @_spi(Experimental) extension String: Attachable { - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { var selfCopy = self return try selfCopy.withUTF8 { utf8 in try body(UnsafeRawBufferPointer(utf8)) @@ -151,7 +152,7 @@ extension String: Attachable { @_spi(Experimental) extension Substring: Attachable { - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { var selfCopy = self return try selfCopy.withUTF8 { utf8 in try body(UnsafeRawBufferPointer(utf8)) diff --git a/Sources/Testing/Attachments/AttachableContainer.swift b/Sources/Testing/Attachments/AttachableContainer.swift index aced49c0f..e4d716e9c 100644 --- a/Sources/Testing/Attachments/AttachableContainer.swift +++ b/Sources/Testing/Attachments/AttachableContainer.swift @@ -12,23 +12,19 @@ /// written to disk when a test is run and which contains another value that it /// stands in for. /// -/// To attach an attachable value to a test report or test run output, use it to -/// initialize a new instance of ``Attachment``, then call -/// ``Attachment/attach(sourceLocation:)``. An attachment can only be attached -/// once. +/// To attach an attachable value to a test, pass it to ``Attachment/record(_:named:sourceLocation:)``. +/// To further configure an attachable value before you attach it, use it to +/// initialize an instance of ``Attachment`` and set its properties before +/// passing it to ``Attachment/record(_:sourceLocation:)``. An attachable +/// value can only be attached to a test once. /// /// A type can conform to this protocol if it represents another type that /// cannot directly conform to ``Attachable``, such as a non-final class or a /// type declared in a third-party module. @_spi(Experimental) public protocol AttachableContainer: Attachable, ~Copyable { -#if hasFeature(SuppressedAssociatedTypes) - /// The type of the attachable value represented by this type. - associatedtype AttachableValue: ~Copyable -#else /// The type of the attachable value represented by this type. associatedtype AttachableValue -#endif /// The attachable value represented by this instance. var attachableValue: AttachableValue { get } diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index ef7ae5537..c7a4fc0c9 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -143,7 +143,7 @@ public struct AnyAttachable: AttachableContainer, Copyable, Sendable { attachableValue.estimatedAttachmentByteCount } - public func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { func open(_ attachableValue: T, for attachment: borrowing Attachment) throws -> R where T: Attachable & Sendable & Copyable { let temporaryAttachment = Attachment( _attachableValue: attachableValue, @@ -151,7 +151,7 @@ public struct AnyAttachable: AttachableContainer, Copyable, Sendable { _preferredName: attachment._preferredName, sourceLocation: attachment.sourceLocation ) - return try temporaryAttachment.withUnsafeBufferPointer(body) + return try temporaryAttachment.withUnsafeBytes(body) } return try open(attachableValue, for: attachment) } @@ -220,25 +220,61 @@ extension Attachment where AttachableValue: AttachableContainer & ~Copyable { #if !SWT_NO_LAZY_ATTACHMENTS extension Attachment where AttachableValue: Sendable & Copyable { - /// Attach this instance to the current test. + /// Attach an attachment to the current test. /// /// - Parameters: + /// - attachment: The attachment to attach. /// - sourceLocation: The source location of the call to this function. /// + /// When attaching a value of a type that does not conform to both + /// [`Sendable`](https://developer.apple.com/documentation/swift/sendable) and + /// [`Copyable`](https://developer.apple.com/documentation/swift/copyable), + /// the testing library encodes it as data immediately. If the value cannot be + /// encoded and an error is thrown, that error is recorded as an issue in the + /// current test and the attachment is not written to the test report or to + /// disk. + /// /// An attachment can only be attached once. @_documentation(visibility: private) - public consuming func attach(sourceLocation: SourceLocation = #_sourceLocation) { - var attachmentCopy = Attachment(self) + public static func record(_ attachment: consuming Self, sourceLocation: SourceLocation = #_sourceLocation) { + var attachmentCopy = Attachment(attachment) attachmentCopy.sourceLocation = sourceLocation Event.post(.valueAttached(attachmentCopy)) } + + /// Attach a value to the current test. + /// + /// - Parameters: + /// - attachableValue: The value to attach. + /// - preferredName: The preferred name of the attachment when writing it to + /// a test report or to disk. If `nil`, the testing library attempts to + /// derive a reasonable filename for the attached value. + /// - sourceLocation: The source location of the call to this function. + /// + /// When attaching a value of a type that does not conform to both + /// [`Sendable`](https://developer.apple.com/documentation/swift/sendable) and + /// [`Copyable`](https://developer.apple.com/documentation/swift/copyable), + /// the testing library encodes it as data immediately. If the value cannot be + /// encoded and an error is thrown, that error is recorded as an issue in the + /// current test and the attachment is not written to the test report or to + /// disk. + /// + /// This function creates a new instance of ``Attachment`` and immediately + /// attaches it to the current test. + /// + /// An attachment can only be attached once. + @_documentation(visibility: private) + public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { + record(Self(attachableValue, named: preferredName), sourceLocation: sourceLocation) + } } #endif extension Attachment where AttachableValue: ~Copyable { - /// Attach this instance to the current test. + /// Attach an attachment to the current test. /// /// - Parameters: + /// - attachment: The attachment to attach. /// - sourceLocation: The source location of the call to this function. /// /// When attaching a value of a type that does not conform to both @@ -250,14 +286,14 @@ extension Attachment where AttachableValue: ~Copyable { /// disk. /// /// An attachment can only be attached once. - public consuming func attach(sourceLocation: SourceLocation = #_sourceLocation) { + public static func record(_ attachment: consuming Self, sourceLocation: SourceLocation = #_sourceLocation) { do { - let attachmentCopy = try withUnsafeBufferPointer { buffer in + let attachmentCopy = try attachment.withUnsafeBytes { buffer in let attachableContainer = AnyAttachable(attachableValue: Array(buffer)) return Attachment( _attachableValue: attachableContainer, - fileSystemPath: fileSystemPath, - _preferredName: preferredName, // invokes preferredName(for:basedOn:) + fileSystemPath: attachment.fileSystemPath, + _preferredName: attachment.preferredName, // invokes preferredName(for:basedOn:) sourceLocation: sourceLocation ) } @@ -267,6 +303,31 @@ extension Attachment where AttachableValue: ~Copyable { Issue(kind: .valueAttachmentFailed(error), comments: [], sourceContext: sourceContext).record() } } + + /// Attach a value to the current test. + /// + /// - Parameters: + /// - attachableValue: The value to attach. + /// - preferredName: The preferred name of the attachment when writing it to + /// a test report or to disk. If `nil`, the testing library attempts to + /// derive a reasonable filename for the attached value. + /// - sourceLocation: The source location of the call to this function. + /// + /// When attaching a value of a type that does not conform to both + /// [`Sendable`](https://developer.apple.com/documentation/swift/sendable) and + /// [`Copyable`](https://developer.apple.com/documentation/swift/copyable), + /// the testing library encodes it as data immediately. If the value cannot be + /// encoded and an error is thrown, that error is recorded as an issue in the + /// current test and the attachment is not written to the test report or to + /// disk. + /// + /// This function creates a new instance of ``Attachment`` and immediately + /// attaches it to the current test. + /// + /// An attachment can only be attached once. + public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { + record(Self(attachableValue, named: preferredName), sourceLocation: sourceLocation) + } } // MARK: - Getting the serialized form of an attachable value (generically) @@ -286,10 +347,10 @@ extension Attachment where AttachableValue: ~Copyable { /// /// The testing library uses this function when writing an attachment to a /// test report or to a file on disk. This function calls the - /// ``Attachable/withUnsafeBufferPointer(for:_:)`` function on this - /// attachment's ``attachableValue-2tnj5`` property. - @inlinable public borrowing func withUnsafeBufferPointer(_ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { - try attachableValue.withUnsafeBufferPointer(for: self, body) + /// ``Attachable/withUnsafeBytes(for:_:)`` function on this attachment's + /// ``attachableValue-2tnj5`` property. + @inlinable public borrowing func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + try attachableValue.withUnsafeBytes(for: self, body) } } @@ -391,7 +452,7 @@ extension Attachment where AttachableValue: ~Copyable { // There should be no code path that leads to this call where the attachable // value is nil. - try withUnsafeBufferPointer { buffer in + try withUnsafeBytes { buffer in try file!.write(buffer) } diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 5a36fd4b6..126633776 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -27,7 +27,7 @@ struct AttachmentTests { @Test func saveValue() { let attachableValue = MyAttachable(string: "") let attachment = Attachment(attachableValue, named: "AttachmentTests.saveValue.html") - attachment.attach() + Attachment.record(attachment) } @Test func description() { @@ -174,41 +174,41 @@ struct AttachmentTests { await Test { let attachment = Attachment(attachableValue, named: "loremipsum.html") - attachment.attach() + Attachment.record(attachment) }.run(configuration: configuration) } } #endif @Test func attachValue() async { - await confirmation("Attachment detected") { valueAttached in + await confirmation("Attachment detected", expectedCount: 2) { valueAttached in var configuration = Configuration() configuration.eventHandler = { event, _ in guard case let .valueAttached(attachment) = event.kind else { return } - #expect(attachment.preferredName == "loremipsum") #expect(attachment.sourceLocation.fileID == #fileID) valueAttached() } await Test { - let attachableValue = MyAttachable(string: "") - Attachment(attachableValue, named: "loremipsum").attach() + let attachableValue1 = MyAttachable(string: "") + Attachment.record(attachableValue1) + let attachableValue2 = MyAttachable(string: "") + Attachment.record(Attachment(attachableValue2)) }.run(configuration: configuration) } } @Test func attachSendableValue() async { - await confirmation("Attachment detected") { valueAttached in + await confirmation("Attachment detected", expectedCount: 2) { valueAttached in var configuration = Configuration() configuration.eventHandler = { event, _ in guard case let .valueAttached(attachment) = event.kind else { return } - #expect(attachment.preferredName == "loremipsum") #expect(attachment.attachableValue is MySendableAttachable) #expect(attachment.sourceLocation.fileID == #fileID) valueAttached() @@ -216,7 +216,8 @@ struct AttachmentTests { await Test { let attachableValue = MySendableAttachable(string: "") - Attachment(attachableValue, named: "loremipsum").attach() + Attachment.record(attachableValue) + Attachment.record(Attachment(attachableValue)) }.run(configuration: configuration) } } @@ -240,7 +241,7 @@ struct AttachmentTests { await Test { var attachableValue = MyAttachable(string: "") attachableValue.errorToThrow = MyError() - Attachment(attachableValue, named: "loremipsum").attach() + Attachment.record(Attachment(attachableValue, named: "loremipsum")) }.run(configuration: configuration) } } @@ -267,7 +268,7 @@ struct AttachmentTests { #expect(attachment.preferredName == temporaryFileName) #expect(throws: Never.self) { - try attachment.withUnsafeBufferPointer { buffer in + try attachment.withUnsafeBytes { buffer in #expect(buffer.count == data.count) } } @@ -276,7 +277,7 @@ struct AttachmentTests { await Test { let attachment = try await Attachment(contentsOf: temporaryURL) - attachment.attach() + Attachment.record(attachment) }.run(configuration: configuration) } } @@ -299,7 +300,7 @@ struct AttachmentTests { } #expect(attachment.preferredName == "\(temporaryDirectoryName).zip") - try! attachment.withUnsafeBufferPointer { buffer in + try! attachment.withUnsafeBytes { buffer in #expect(buffer.count > 32) #expect(buffer[0] == UInt8(ascii: "P")) #expect(buffer[1] == UInt8(ascii: "K")) @@ -312,7 +313,7 @@ struct AttachmentTests { await Test { let attachment = try await Attachment(contentsOf: temporaryURL) - attachment.attach() + Attachment.record(attachment) }.run(configuration: configuration) } } @@ -393,7 +394,7 @@ struct AttachmentTests { } func open(_ attachment: borrowing Attachment) throws where T: Attachable { - try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { bytes in + try attachment.attachableValue.withUnsafeBytes(for: attachment) { bytes in #expect(bytes.first == args.firstCharacter.asciiValue) let decodedStringValue = try args.decode(Data(bytes)) #expect(decodedStringValue == "stringly speaking") @@ -416,7 +417,7 @@ struct AttachmentTests { let attachableValue = MySecureCodingAttachable(string: "stringly speaking") let attachment = Attachment(attachableValue, named: "loremipsum.json") #expect(throws: CocoaError.self) { - try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { _ in } + try attachment.attachableValue.withUnsafeBytes(for: attachment) { _ in } } } @@ -425,7 +426,7 @@ struct AttachmentTests { let attachableValue = MySecureCodingAttachable(string: "stringly speaking") let attachment = Attachment(attachableValue, named: "loremipsum.gif") #expect(throws: CocoaError.self) { - try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { _ in } + try attachment.attachableValue.withUnsafeBytes(for: attachment) { _ in } } } #endif @@ -437,7 +438,7 @@ extension AttachmentTests { func test(_ value: some Attachable) throws { #expect(value.estimatedAttachmentByteCount == 6) let attachment = Attachment(value) - try attachment.withUnsafeBufferPointer { buffer in + try attachment.withUnsafeBytes { buffer in #expect(buffer.elementsEqual("abc123".utf8)) #expect(buffer.count == 6) } @@ -530,10 +531,10 @@ extension AttachmentTests { let image = try Self.cgImage.get() let attachment = Attachment(image, named: "diamond") #expect(attachment.attachableValue === image) - try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { buffer in + try attachment.attachableValue.withUnsafeBytes(for: attachment) { buffer in #expect(buffer.count > 32) } - attachment.attach() + Attachment.record(attachment) } @available(_uttypesAPI, *) @@ -542,7 +543,7 @@ extension AttachmentTests { let image = try Self.cgImage.get() let attachment = Attachment(image, named: "diamond", as: type, encodingQuality: quality) #expect(attachment.attachableValue === image) - try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { buffer in + try attachment.attachableValue.withUnsafeBytes(for: attachment) { buffer in #expect(buffer.count > 32) } if let ext = type?.preferredFilenameExtension { @@ -555,7 +556,7 @@ extension AttachmentTests { @Test func cannotAttachCGImageWithNonImageType() async { await #expect(exitsWith: .failure) { let attachment = Attachment(try Self.cgImage.get(), named: "diamond", as: .mp3) - try attachment.attachableValue.withUnsafeBufferPointer(for: attachment) { _ in } + try attachment.attachableValue.withUnsafeBytes(for: attachment) { _ in } } } #endif @@ -569,7 +570,7 @@ struct MyAttachable: Attachable, ~Copyable { var string: String var errorToThrow: (any Error)? - func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { if let errorToThrow { throw errorToThrow } @@ -587,7 +588,7 @@ extension MyAttachable: Sendable {} struct MySendableAttachable: Attachable, Sendable { var string: String - func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { #expect(attachment.attachableValue.string == string) var string = string return try string.withUTF8 { buffer in @@ -599,7 +600,7 @@ struct MySendableAttachable: Attachable, Sendable { struct MySendableAttachableWithDefaultByteCount: Attachable, Sendable { var string: String - func withUnsafeBufferPointer(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { + func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { var string = string return try string.withUTF8 { buffer in try body(.init(buffer)) diff --git a/cmake/modules/shared/CompilerSettings.cmake b/cmake/modules/shared/CompilerSettings.cmake index f526caae3..0da4216c5 100644 --- a/cmake/modules/shared/CompilerSettings.cmake +++ b/cmake/modules/shared/CompilerSettings.cmake @@ -13,8 +13,7 @@ add_compile_options( add_compile_options( "SHELL:$<$:-Xfrontend -require-explicit-sendable>") add_compile_options( - "SHELL:$<$:-Xfrontend -enable-experimental-feature -Xfrontend AccessLevelOnImport>" - "SHELL:$<$:-Xfrontend -enable-experimental-feature -Xfrontend SuppressedAssociatedTypes>") + "SHELL:$<$:-Xfrontend -enable-experimental-feature -Xfrontend AccessLevelOnImport>") add_compile_options( "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend ExistentialAny>" "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend InternalImportsByDefault>" From eef23405f46b08f7a28145a8669346304bee5429 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 20 Mar 2025 23:06:46 -0400 Subject: [PATCH 142/234] Work around a compiler crash building `Attachment.record()`. (#1033) This PR works around a compiler crash that appeared in CI while building one of the overloads of `Attachment.record()`. Works around rdar://147543560. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Attachments/Attachment.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index c7a4fc0c9..d7c1cddb7 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -265,7 +265,7 @@ extension Attachment where AttachableValue: Sendable & Copyable { /// An attachment can only be attached once. @_documentation(visibility: private) public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { - record(Self(attachableValue, named: preferredName), sourceLocation: sourceLocation) + record(Self(attachableValue, named: preferredName, sourceLocation: sourceLocation), sourceLocation: sourceLocation) } } #endif @@ -326,7 +326,7 @@ extension Attachment where AttachableValue: ~Copyable { /// /// An attachment can only be attached once. public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { - record(Self(attachableValue, named: preferredName), sourceLocation: sourceLocation) + record(Self(attachableValue, named: preferredName, sourceLocation: sourceLocation), sourceLocation: sourceLocation) } } From ee700e25f3eb7d51a96ba806413818bbb571b1c8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 22 Mar 2025 13:08:51 -0400 Subject: [PATCH 143/234] Ensure `Locked+Platform.swift` is not stripped when statically linking. (#1035) All symbols in Locked+Platform.swift are referenced _indirectly_ by the Swift compiler due to `LockedWith` being generic, which can cause symbols from that file (including protocol conformance metadata) to be stripped at link time as if it were unused. This change (hopefully) works around that issue. Works around rdar://147250336. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Support/Locked+Platform.swift | 11 +++++++++++ Sources/Testing/Support/Locked.swift | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/Sources/Testing/Support/Locked+Platform.swift b/Sources/Testing/Support/Locked+Platform.swift index 951e62da8..2b3f5b648 100644 --- a/Sources/Testing/Support/Locked+Platform.swift +++ b/Sources/Testing/Support/Locked+Platform.swift @@ -94,3 +94,14 @@ typealias DefaultLock = Never #warning("Platform-specific implementation missing: locking unavailable") typealias DefaultLock = Never #endif + +#if SWT_NO_DYNAMIC_LINKING +/// A function which, when called by another file, ensures that the file in +/// which ``DefaultLock`` is declared is linked. +/// +/// When static linking is used, the linker may opt to strip some or all of the +/// symbols (including protocol conformance metadata) declared in this file. +/// ``LockedWith`` calls this function in ``LockedWith/init(rawValue:)`` to work +/// around that issue. +func linkLockImplementations() {} +#endif diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index e8b17be7b..c69cfd351 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -66,6 +66,10 @@ struct LockedWith: RawRepresentable where L: Lockable { private nonisolated(unsafe) var _storage: ManagedBuffer init(rawValue: T) { +#if SWT_NO_DYNAMIC_LINKING + linkLockImplementations() +#endif + _storage = _Storage.create(minimumCapacity: 1, makingHeaderWith: { _ in rawValue }) _storage.withUnsafeMutablePointerToElements { lock in L.initializeLock(at: lock) From 2c60dd64337d77fac166600855c94e18b19526b7 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 25 Mar 2025 13:44:32 -0500 Subject: [PATCH 144/234] Remove unused "override comment" from ConditionTrait implementation (#1036) This removes a mechanism in the implementation of `ConditionTrait` for a condition closure to provide an "override" comment. ### Motivation: While reviewing a draft evolution proposal for `ConditionTrait` (https://github.com/swiftlang/swift-evolution/pull/2740) I realized that we don't use this "comment override" mechanism anywhere in the testing library. I believe we did back when it was first added, but it's no longer used. Removing this would allow the public API being proposed above to be simplified: the `evaluate()` method could return `Bool` instead of a tuple. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Traits/ConditionTrait.swift | 33 ++++----------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index 245f8e98f..7efef95a7 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -26,26 +26,8 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// /// - Parameters: /// - body: The function to call. The result of this function determines - /// if the condition is satisfied or not. If this function returns - /// `false` and a comment is also returned, it is used in place of the - /// value of the associated trait's ``ConditionTrait/comment`` property. - /// If this function returns `true`, the returned comment is ignored. - case conditional(_ body: @Sendable () async throws -> (Bool, comment: Comment?)) - - /// Create an instance of this type associated with a trait that is - /// conditional on the result of calling a function. - /// - /// - Parameters: - /// - body: The function to call. The result of this function determines - /// whether or not the condition was met. - /// - /// - Returns: A trait that marks a test's enabled status as the result of - /// calling a function. - static func conditional(_ body: @escaping @Sendable () async throws -> Bool) -> Self { - conditional { () -> (Bool, comment: Comment?) in - return (try await body(), nil) - } - } + /// if the condition is satisfied or not. + case conditional(_ body: @Sendable () async throws -> Bool) /// The trait is unconditional and always has the same result. /// @@ -82,14 +64,11 @@ public struct ConditionTrait: TestTrait, SuiteTrait { public var sourceLocation: SourceLocation public func prepare(for test: Test) async throws { - let result: Bool - var commentOverride: Comment? - - switch kind { + let result = switch kind { case let .conditional(condition): - (result, commentOverride) = try await condition() + try await condition() case let .unconditional(unconditionalValue): - result = unconditionalValue + unconditionalValue } if !result { @@ -99,7 +78,7 @@ public struct ConditionTrait: TestTrait, SuiteTrait { // attempt to get the backtrace of the caught error when creating an issue // for it, however. let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) - throw SkipInfo(comment: commentOverride ?? comments.first, sourceContext: sourceContext) + throw SkipInfo(comment: comments.first, sourceContext: sourceContext) } } From 7ccbd6884d75d8a685055dd2c15c881ea30cfd64 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 25 Mar 2025 13:45:09 -0500 Subject: [PATCH 145/234] Qualify `@__testing(warning:)` usage in macro expansion with module name and enhance a related unit test (#1038) A small enhancement to the `@Test` and `@Suite` macro expansion changes made in #880: qualify the `@__testing(warning:)` usage with the module name. I also took the opportunity to enhance a unit test related to `@__testing(semantics:)`. It already correctly checks the attribute module name if present, and this test simply validates that. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/TestingMacros/Support/TestContentGeneration.swift | 2 +- Tests/TestingMacrosTests/PragmaMacroTests.swift | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index c6ea40357..9c5136d97 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -86,7 +86,7 @@ func makeTestContentRecordDecl(named name: TokenSyntax, in typeName: TypeSyntax? #elseif os(Windows) @_section(".sw5test$B") #else - @__testing(warning: "Platform-specific implementation missing: test content section name unavailable") + @Testing.__testing(warning: "Platform-specific implementation missing: test content section name unavailable") #endif @_used #endif diff --git a/Tests/TestingMacrosTests/PragmaMacroTests.swift b/Tests/TestingMacrosTests/PragmaMacroTests.swift index bba101754..9019085bf 100644 --- a/Tests/TestingMacrosTests/PragmaMacroTests.swift +++ b/Tests/TestingMacrosTests/PragmaMacroTests.swift @@ -22,6 +22,7 @@ struct PragmaMacroTests { let node = """ @Testing.__testing(semantics: "abc123") @__testing(semantics: "def456") + @UnrelatedModule.__testing(semantics: "xyz789") let x = 0 """ as DeclSyntax let nodeWithAttributes = try #require(node.asProtocol((any WithAttributesSyntax).self)) From 1d28fa4bee2136a1f230c8beb0130adaa410c6bc Mon Sep 17 00:00:00 2001 From: David Catmull Date: Fri, 28 Mar 2025 12:54:38 -0600 Subject: [PATCH 146/234] Add ConditionTrait.evaluate() (#909) Add `ConditionTrait.evaluate()` so that a condition can be evaluated independent of a `Test` object. ### Motivation: Currently, the only way a `ConditionTrait` is evaluated is inside the `prepare(for:)` method. This makes it difficult and awkward for third-party libraries to utilize these traits because evaluating a condition would require creating a dummy `Test` to pass to that method. ### Modifications: Add `ConditionTrait.evaluate()`, and `ConditionTrait.Evaluation` enum for the return value. ### Result: Public API allows for evaluating a `ConditionTrait` in any context. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Traits/ConditionTrait.swift | 19 ++++++++++++++---- .../Traits/ConditionTraitTests.swift | 20 ++++++++++++++++++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index 7efef95a7..a3b98c99d 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -62,16 +62,27 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// The source location where this trait is specified. public var sourceLocation: SourceLocation - - public func prepare(for test: Test) async throws { - let result = switch kind { + + /// Evaluate this instance's underlying condition. + /// + /// - Returns: The result of evaluating this instance's underlying condition. + /// + /// The evaluation is performed each time this function is called, and is not + /// cached. + @_spi(Experimental) + public func evaluate() async throws -> Bool { + switch kind { case let .conditional(condition): try await condition() case let .unconditional(unconditionalValue): unconditionalValue } + } + + public func prepare(for test: Test) async throws { + let isEnabled = try await evaluate() - if !result { + if !isEnabled { // We don't need to consider including a backtrace here because it will // primarily contain frames in the testing library, not user code. If an // error was thrown by a condition evaluated above, the caller _should_ diff --git a/Tests/TestingTests/Traits/ConditionTraitTests.swift b/Tests/TestingTests/Traits/ConditionTraitTests.swift index 5d70c8e87..6b5311202 100644 --- a/Tests/TestingTests/Traits/ConditionTraitTests.swift +++ b/Tests/TestingTests/Traits/ConditionTraitTests.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@testable import Testing +@testable @_spi(Experimental) import Testing @Suite("Condition Trait Tests", .tags(.traitRelated)) struct ConditionTraitTests { @@ -41,4 +41,22 @@ struct ConditionTraitTests { .disabled(if: false) ) func disabledTraitIf() throws {} + + @Test + func evaluateCondition() async throws { + let trueUnconditional = ConditionTrait(kind: .unconditional(true), comments: [], sourceLocation: #_sourceLocation) + let falseUnconditional = ConditionTrait.disabled() + let enabledTrue = ConditionTrait.enabled(if: true) + let enabledFalse = ConditionTrait.enabled(if: false) + var result: Bool + + result = try await trueUnconditional.evaluate() + #expect(result) + result = try await falseUnconditional.evaluate() + #expect(!result) + result = try await enabledTrue.evaluate() + #expect(result) + result = try await enabledFalse.evaluate() + #expect(!result) + } } From 4e7a085d627b09748a4bea6d4ccd2f6bbbc648c7 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 31 Mar 2025 13:37:53 -0400 Subject: [PATCH 147/234] Erase environment variables set by exit tests after reading them. (#1044) The environment variables we currently use to pass information from the parent process to the child process are implementation details of exit tests and may change or be removed in a future update. To minimize the risk of code relying on these environment variables, and to avoid accidentally trying to open inherited file descriptors twice, clear the variables after reading them. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 8 ++++ Sources/Testing/Support/Environment.swift | 39 +++++++++++++++++++ .../Support/EnvironmentTests.swift | 39 ------------------- 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 69346b74e..bd5cb95b9 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -457,6 +457,10 @@ extension ExitTest { return nil } + // Erase the environment variable so that it cannot accidentally be opened + // twice (nor, in theory, affect the code of the exit test.) + Environment.setVariable(nil, named: "SWT_EXPERIMENTAL_BACKCHANNEL") + var fd: CInt? #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) fd = CInt(backChannelEnvironmentVariable) @@ -487,6 +491,10 @@ extension ExitTest { // Find the ID of the exit test to run, if any, in the environment block. var id: ExitTest.ID? if var idString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") { + // Clear the environment variable. It's an implementation detail and exit + // test code shouldn't be dependent on it. Use ExitTest.current if needed! + Environment.setVariable(nil, named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") + id = try? idString.withUTF8 { idBuffer in try JSON.decode(ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) } diff --git a/Sources/Testing/Support/Environment.swift b/Sources/Testing/Support/Environment.swift index f09efc1ed..2ab3710a4 100644 --- a/Sources/Testing/Support/Environment.swift +++ b/Sources/Testing/Support/Environment.swift @@ -235,3 +235,42 @@ enum Environment { } } } + +// MARK: - Setting variables + +extension Environment { + /// Set the environment variable with the specified name. + /// + /// - Parameters: + /// - value: The new value for the specified environment variable. Pass + /// `nil` to remove the variable from the current process' environment. + /// - name: The name of the environment variable. + /// + /// - Returns: Whether or not the environment variable was successfully set. + @discardableResult + static func setVariable(_ value: String?, named name: String) -> Bool { +#if SWT_NO_ENVIRONMENT_VARIABLES + simulatedEnvironment.withLock { environment in + environment[name] = value + } + return true +#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) + if let value { + return 0 == setenv(name, value, 1) + } + return 0 == unsetenv(name) +#elseif os(Windows) + name.withCString(encodedAs: UTF16.self) { name in + if let value { + return value.withCString(encodedAs: UTF16.self) { value in + SetEnvironmentVariableW(name, value) + } + } + return SetEnvironmentVariableW(name, nil) + } +#else +#warning("Platform-specific implementation missing: environment variables unavailable") + return false +#endif + } +} diff --git a/Tests/TestingTests/Support/EnvironmentTests.swift b/Tests/TestingTests/Support/EnvironmentTests.swift index 512ebfe7b..43d8ea3f3 100644 --- a/Tests/TestingTests/Support/EnvironmentTests.swift +++ b/Tests/TestingTests/Support/EnvironmentTests.swift @@ -71,42 +71,3 @@ struct EnvironmentTests { #expect(Environment.flag(named: name) == false) } } - -// MARK: - Fixtures - -extension Environment { - /// Set the environment variable with the specified name. - /// - /// - Parameters: - /// - value: The new value for the specified environment variable. Pass - /// `nil` to remove the variable from the current process' environment. - /// - name: The name of the environment variable. - /// - /// - Returns: Whether or not the environment variable was successfully set. - @discardableResult - static func setVariable(_ value: String?, named name: String) -> Bool { -#if SWT_NO_ENVIRONMENT_VARIABLES - simulatedEnvironment.withLock { environment in - environment[name] = value - } - return true -#elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) - if let value { - return 0 == setenv(name, value, 1) - } - return 0 == unsetenv(name) -#elseif os(Windows) - name.withCString(encodedAs: UTF16.self) { name in - if let value { - return value.withCString(encodedAs: UTF16.self) { value in - SetEnvironmentVariableW(name, value) - } - } - return SetEnvironmentVariableW(name, nil) - } -#else -#warning("Platform-specific implementation missing: environment variables unavailable") - return false -#endif - } -} From 1f49df965607d9305e5ff30a13e4f0b50593da66 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 31 Mar 2025 15:07:10 -0400 Subject: [PATCH 148/234] Pause child processes on spawn. (#1042) This PR pauses child processes we spawn before resuming them, which makes it easier to set a breakpoint and attach a debugger to those processes. This functionality is available on macOS and Windows; as far as I know, neither Linux nor POSIX-in-general has API for this. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/SpawnProcess.swift | 24 ++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index c824baa4e..8f8d95db6 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -176,6 +176,11 @@ func spawnExecutable( #warning("Platform-specific implementation missing: cannot close unused file descriptors") #endif +#if SWT_TARGET_OS_APPLE && DEBUG + // Start the process suspended so we can attach a debugger if needed. + flags |= CShort(POSIX_SPAWN_START_SUSPENDED) +#endif + // Set flags; make sure to keep this call below any code that might modify // the flags mask! _ = posix_spawnattr_setflags(attrs, flags) @@ -202,6 +207,10 @@ func spawnExecutable( guard 0 == processSpawned else { throw CError(rawValue: processSpawned) } +#if SWT_TARGET_OS_APPLE && DEBUG + // Resume the process. + _ = kill(pid, SIGCONT) +#endif return pid } } @@ -254,6 +263,12 @@ func spawnExecutable( let commandLine = _escapeCommandLine(CollectionOfOne(executablePath) + arguments) let environ = environment.map { "\($0.key)=\($0.value)" }.joined(separator: "\0") + "\0\0" + var flags = DWORD(CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT | EXTENDED_STARTUPINFO_PRESENT) +#if DEBUG + // Start the process suspended so we can attach a debugger if needed. + flags |= DWORD(CREATE_SUSPENDED) +#endif + return try commandLine.withCString(encodedAs: UTF16.self) { commandLine in try environ.withCString(encodedAs: UTF16.self) { environ in var processInfo = PROCESS_INFORMATION() @@ -264,7 +279,7 @@ func spawnExecutable( nil, nil, true, // bInheritHandles - DWORD(CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT | EXTENDED_STARTUPINFO_PRESENT), + flags, .init(mutating: environ), nil, startupInfo.pointer(to: \.StartupInfo)!, @@ -272,8 +287,13 @@ func spawnExecutable( ) else { throw Win32Error(rawValue: GetLastError()) } - _ = CloseHandle(processInfo.hThread) +#if DEBUG + // Resume the process. + _ = ResumeThread(processInfo.hThread!) +#endif + + _ = CloseHandle(processInfo.hThread) return processInfo.hProcess! } } From 0e3bdfd4f7eed1a21a82d31f486695b8b8aa93fa Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 1 Apr 2025 14:05:53 -0400 Subject: [PATCH 149/234] Emit a diagnostic if an exit test's body closure includes a capture list. (#1046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds a custom diagnostic for `#expect(exitsWith:)` if the passed closure visibly closes over any state (via a capture list). For example: ```swift await #expect(exitsWith: .failure) { [x] in // ... } ``` Produces: > 🛑 Cannot specify a capture clause in closure passed to '#expect(exitsWith:_:)' With a fix-it to remove the capture list. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/TestingMacros/ConditionMacro.swift | 10 +++- .../Support/DiagnosticMessage.swift | 48 +++++++++++++++++++ .../ConditionMacroTests.swift | 17 +++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 89498e1ec..f07fad91f 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -436,7 +436,15 @@ extension ExitTestConditionMacro { fatalError("Could not find the body argument to this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } - let bodyArgumentExpr = arguments[trailingClosureIndex].expression + // Extract the body argument and, if it's a closure with a capture list, + // emit an appropriate diagnostic. + var bodyArgumentExpr = arguments[trailingClosureIndex].expression + bodyArgumentExpr = removeParentheses(from: bodyArgumentExpr) ?? bodyArgumentExpr + if let closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), + let captureClause = closureExpr.signature?.capture, + !captureClause.items.isEmpty { + context.diagnose(.captureClauseUnsupported(captureClause, in: closureExpr, inExitTest: macro)) + } // TODO: use UUID() here if we can link to Foundation let exitTestID = (UInt64.random(in: 0 ... .max), UInt64.random(in: 0 ... .max)) diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index e49cfa497..dc9defe5d 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -764,3 +764,51 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage { var severity: DiagnosticSeverity var fixIts: [FixIt] = [] } + +// MARK: - Captured values + +extension DiagnosticMessage { + /// Create a diagnostic message stating that a capture clause cannot be used + /// in an exit test. + /// + /// - Parameters: + /// - captureClause: The invalid capture clause. + /// - closure: The closure containing `captureClause`. + /// - exitTestMacro: The containing exit test macro invocation. + /// + /// - Returns: A diagnostic message. + static func captureClauseUnsupported(_ captureClause: ClosureCaptureClauseSyntax, in closure: ClosureExprSyntax, inExitTest exitTestMacro: some FreestandingMacroExpansionSyntax) -> Self { + let changes: [FixIt.Change] + if let signature = closure.signature, + Array(signature.with(\.capture, nil).tokens(viewMode: .sourceAccurate)).count == 1 { + // The only remaining token in the signature is `in`, so remove the whole + // signature tree instead of just the capture clause. + changes = [ + .replaceTrailingTrivia(token: closure.leftBrace, newTrivia: ""), + .replace( + oldNode: Syntax(signature), + newNode: Syntax("" as ExprSyntax) + ) + ] + } else { + changes = [ + .replace( + oldNode: Syntax(captureClause), + newNode: Syntax("" as ExprSyntax) + ) + ] + } + + return Self( + syntax: Syntax(captureClause), + message: "Cannot specify a capture clause in closure passed to \(_macroName(exitTestMacro))", + severity: .error, + fixIts: [ + FixIt( + message: MacroExpansionFixItMessage("Remove '\(captureClause.trimmed)'"), + changes: changes + ), + ] + ) + } +} diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index e7307476d..07d84b0f8 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -383,6 +383,23 @@ struct ConditionMacroTests { #expect(diagnostic.message.contains("is redundant")) } + @Test( + "Capture list on an exit test produces a diagnostic", + arguments: [ + "#expectExitTest(exitsWith: x) { [a] in }": + "Cannot specify a capture clause in closure passed to '#expectExitTest(exitsWith:_:)'" + ] + ) + func exitTestCaptureListProducesDiagnostic(input: String, expectedMessage: String) throws { + let (_, diagnostics) = try parse(input) + + #expect(diagnostics.count > 0) + for diagnostic in diagnostics { + #expect(diagnostic.diagMessage.severity == .error) + #expect(diagnostic.message == expectedMessage) + } + } + @Test("Macro expansion is performed within a test function") func macroExpansionInTestFunction() throws { let input = ##""" From f7705437e5010b262a10e2b3f1ce416ac18794a5 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 3 Apr 2025 13:01:59 -0400 Subject: [PATCH 150/234] Replace CRC-32 with SHA-256 in our macro target (#1047) This PR replaces CRC-32 with SHA-256 in our macro target. We currently use CRC-32 to disambiguate test functions that would otherwise generate identical derived symbol names (using `context.makeUniqueName()`.) By replacing it with the first 64 bits of a SHA-256 hash, we reduce the odds of a collision. We also currently use a 128-bit random number as a unique ID for exit tests during macro expansion (which is as unique as it gets, but unstable.) Replacing this random number with half of a SHA-256 hash allows us to generate _stable_ IDs (that are still statistically unique) which can help when debugging an exit test and may also improve cache quality for tools (e.g. an IDE's symbol cache.) Because there is no SHA-256 implementation available in the Swift standard library or other components we can reliably link to in the macro target, I've borrowed the implementation of SHA-256 in swift-tools-support-core. The original version is [here](https://github.com/swiftlang/swift-tools-support-core/blob/main/Sources/TSCBasic/HashAlgorithms.swift). ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 14 +- Sources/TestingMacros/CMakeLists.txt | 2 +- Sources/TestingMacros/ConditionMacro.swift | 51 ++++- .../MacroExpansionContextAdditions.swift | 8 +- Sources/TestingMacros/Support/CRC32.swift | 74 ------- Sources/TestingMacros/Support/SHA256.swift | 184 ++++++++++++++++++ 6 files changed, 245 insertions(+), 88 deletions(-) delete mode 100644 Sources/TestingMacros/Support/CRC32.swift create mode 100644 Sources/TestingMacros/Support/SHA256.swift diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index bd5cb95b9..a38c7592e 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -35,10 +35,20 @@ private import _TestingInternals #endif public struct ExitTest: Sendable, ~Copyable { /// A type whose instances uniquely identify instances of ``ExitTest``. + /// + /// An instance of this type uniquely identifies an exit test within the + /// context of the current test target. You can get an exit test's unique + /// identifier from its ``id`` property. + /// + /// The encoded form of an instance of this type is subject to change over + /// time. Instances of this type are only guaranteed to be decodable by the + /// same version of the testing library that encoded them. @_spi(ForToolsIntegrationOnly) public struct ID: Sendable, Equatable, Codable { - /// An underlying UUID (stored as two `UInt64` values to avoid relying on - /// `UUID` from Foundation or any platform-specific interfaces.) + /// Storage for the underlying bits of the ID. + /// + /// - Note: On Apple platforms, we deploy to OS versions that do not include + /// support for `UInt128`, so we use two `UInt64`s for storage instead. private var _lo: UInt64 private var _hi: UInt64 diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index b0d809665..72184f94b 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -99,9 +99,9 @@ target_sources(TestingMacros PRIVATE Support/AvailabilityGuards.swift Support/CommentParsing.swift Support/ConditionArgumentParsing.swift - Support/CRC32.swift Support/DiagnosticMessage.swift Support/DiagnosticMessage+Diagnosing.swift + Support/SHA256.swift Support/SourceCodeCapturing.swift Support/SourceLocationGeneration.swift Support/TestContentGeneration.swift diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index f07fad91f..c82acd725 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -8,6 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +import SwiftParser public import SwiftSyntax import SwiftSyntaxBuilder public import SwiftSyntaxMacros @@ -446,9 +447,8 @@ extension ExitTestConditionMacro { context.diagnose(.captureClauseUnsupported(captureClause, in: closureExpr, inExitTest: macro)) } - // TODO: use UUID() here if we can link to Foundation - let exitTestID = (UInt64.random(in: 0 ... .max), UInt64.random(in: 0 ... .max)) - let exitTestIDExpr: ExprSyntax = "(\(literal: exitTestID.0), \(literal: exitTestID.1))" + // Generate a unique identifier for this exit test. + let idExpr = _makeExitTestIDExpr(for: macro, in: context) var decls = [DeclSyntax]() @@ -494,7 +494,7 @@ extension ExitTestConditionMacro { enum \(enumName) { private nonisolated static let accessor: Testing.__TestContentRecordAccessor = { outValue, type, hint, _ in Testing.ExitTest.__store( - \(exitTestIDExpr), + \(idExpr), \(bodyThunkName), into: outValue, asTypeAt: type, @@ -525,10 +525,7 @@ extension ExitTestConditionMacro { // Insert the exit test's ID as the first argument. Note that this will // invalidate all indices into `arguments`! arguments.insert( - Argument( - label: "identifiedBy", - expression: exitTestIDExpr - ), + Argument(label: "identifiedBy", expression: idExpr), at: arguments.startIndex ) @@ -541,6 +538,44 @@ extension ExitTestConditionMacro { return try Base.expansion(of: macro, primaryExpression: bodyArgumentExpr, in: context) } + + /// Make an expression representing an exit test ID that can be passed to the + /// `ExitTest.__store()` function at runtime. + /// + /// - Parameters: + /// - macro: The exit test macro being inspected. + /// - context: The macro context in which the expression is being parsed. + /// + /// - Returns: An expression representing the exit test's unique ID. + private static func _makeExitTestIDExpr( + for macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) -> ExprSyntax { + let exitTestID: (UInt64, UInt64) + if let sourceLocation = context.location(of: macro, at: .afterLeadingTrivia, filePathMode: .fileID), + let fileID = sourceLocation.file.as(StringLiteralExprSyntax.self)?.representedLiteralValue, + let line = sourceLocation.line.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue, + let column = sourceLocation.column.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue { + // Hash the entire source location and store as many bits as possible in + // the resulting ID. + let stringValue = "\(fileID):\(line):\(column)" + exitTestID = SHA256.hash(stringValue.utf8).withUnsafeBytes { sha256 in + sha256.loadUnaligned(as: (UInt64, UInt64).self) + } + } else { + // This branch is dead code in production, but is used when we expand a + // macro in our own unit tests because the macro expansion context does + // not have real source location information. + exitTestID.0 = .random(in: 0 ... .max) + exitTestID.1 = .random(in: 0 ... .max) + } + + // Return a tuple of integer literals (which is what the runtime __store() + // function is expecting.) + return """ + (\(IntegerLiteralExprSyntax(exitTestID.0, radix: .hex)), \(IntegerLiteralExprSyntax(exitTestID.1, radix: .hex))) + """ + } } /// A type describing the expansion of the `#expect(exitsWith:)` macro. diff --git a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift index 3b31caf72..322a84f3a 100644 --- a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift @@ -63,10 +63,12 @@ extension MacroExpansionContext { .tokens(viewMode: .fixedUp) .map(\.textWithoutBackticks) .joined() - let crcValue = crc32(identifierCharacters.utf8) - let suffix = String(crcValue, radix: 16, uppercase: false) + let hashValue = SHA256.hash(identifierCharacters.utf8).withUnsafeBytes { sha256 in + sha256.loadUnaligned(as: UInt64.self) + } + let suffix = String(hashValue, radix: 16, uppercase: false) - // If the caller did not specify a prefix and the CRC32 value starts with a + // If the caller did not specify a prefix and the hash value starts with a // digit, include a single-character prefix to ensure that Swift's name // demangling still works correctly. var prefix = prefix diff --git a/Sources/TestingMacros/Support/CRC32.swift b/Sources/TestingMacros/Support/CRC32.swift deleted file mode 100644 index e58c4c7f7..000000000 --- a/Sources/TestingMacros/Support/CRC32.swift +++ /dev/null @@ -1,74 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// The precomputed CRC-32 lookup table. -/// -/// This table is used by the ``crc32(_:)`` function below. It is borrowed from -/// the [Swift standard library](https://github.com/swiftlang/swift/blob/main/stdlib/public/Backtracing/Elf.swift). -private let _crc32Table: [UInt32] = [ - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, - 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, - 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, - 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, - 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, - 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, - 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, - 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, - 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, - 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, - 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, - 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, - 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d -] - -/// Compute the CRC-32 code for a sequence of bytes. -/// -/// - Parameters: -/// - bytes: The bytes for which a CRC-32 code should be computed. -/// -/// - Returns: The CRC-32 code computed for `bytes`. -/// -/// A starting value of `0` is assumed. This function is adapted from the -/// [Swift standard library](https://github.com/swiftlang/swift/blob/main/stdlib/public/Backtracing/Elf.swift). -func crc32(_ bytes: some Sequence) -> UInt32 { - ~bytes.reduce(~0) { crcValue, byte in - _crc32Table[Int(UInt8(truncatingIfNeeded: crcValue) ^ byte)] ^ (crcValue >> 8) - } -} diff --git a/Sources/TestingMacros/Support/SHA256.swift b/Sources/TestingMacros/Support/SHA256.swift new file mode 100644 index 000000000..a4d88a801 --- /dev/null +++ b/Sources/TestingMacros/Support/SHA256.swift @@ -0,0 +1,184 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// The contents of this file were copied more-or-less verbatim from +/// [swift-tools-support-core](https://github.com/swiftlang/swift-tools-support-core/blob/add9e1518ac37a8e52b7612d3eb2f009ae8f6ce8/Sources/TSCBasic/HashAlgorithms.swift). + +/// SHA-256 implementation from Secure Hash Algorithm 2 (SHA-2) set of +/// cryptographic hash functions (FIPS PUB 180-2). +enum SHA256 { + /// The length of the output digest (in bits). + private static let _digestLength = 256 + + /// The size of each blocks (in bits). + private static let _blockBitSize = 512 + + /// The initial hash value. + private static let _initialHashValue: [UInt32] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 + ] + + /// The constants in the algorithm (K). + private static let _konstants: [UInt32] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 + ] + + public static func hash(_ bytes: some Sequence) -> [UInt8] { + var input = Array(bytes) + + // Pad the input. + _pad(&input) + + // Break the input into N 512-bit blocks. + let messageBlocks = input.blocks(size: _blockBitSize / 8) + + /// The hash that is being computed. + var hash = _initialHashValue + + // Process each block. + for block in messageBlocks { + _process(block, hash: &hash) + } + + // Finally, compute the result. + var result = [UInt8](repeating: 0, count: _digestLength / 8) + for (idx, element) in hash.enumerated() { + let pos = idx * 4 + result[pos + 0] = UInt8((element >> 24) & 0xff) + result[pos + 1] = UInt8((element >> 16) & 0xff) + result[pos + 2] = UInt8((element >> 8) & 0xff) + result[pos + 3] = UInt8(element & 0xff) + } + + return result + } + + /// Process and compute hash from a block. + private static func _process(_ block: ArraySlice, hash: inout [UInt32]) { + + // Compute message schedule. + var W = [UInt32](repeating: 0, count: _konstants.count) + for t in 0..> 10) + let σ0 = W[t-15].rotateRight(by: 7) ^ W[t-15].rotateRight(by: 18) ^ (W[t-15] >> 3) + W[t] = σ1 &+ W[t-7] &+ σ0 &+ W[t-16] + } + } + + var a = hash[0] + var b = hash[1] + var c = hash[2] + var d = hash[3] + var e = hash[4] + var f = hash[5] + var g = hash[6] + var h = hash[7] + + // Run the main algorithm. + for t in 0..<_konstants.count { + let Σ1 = e.rotateRight(by: 6) ^ e.rotateRight(by: 11) ^ e.rotateRight(by: 25) + let ch = (e & f) ^ (~e & g) + let t1 = h &+ Σ1 &+ ch &+ _konstants[t] &+ W[t] + + let Σ0 = a.rotateRight(by: 2) ^ a.rotateRight(by: 13) ^ a.rotateRight(by: 22) + let maj = (a & b) ^ (a & c) ^ (b & c) + let t2 = Σ0 &+ maj + + h = g + g = f + f = e + e = d &+ t1 + d = c + c = b + b = a + a = t1 &+ t2 + } + + hash[0] = a &+ hash[0] + hash[1] = b &+ hash[1] + hash[2] = c &+ hash[2] + hash[3] = d &+ hash[3] + hash[4] = e &+ hash[4] + hash[5] = f &+ hash[5] + hash[6] = g &+ hash[6] + hash[7] = h &+ hash[7] + } + + /// Pad the given byte array to be a multiple of 512 bits. + private static func _pad(_ input: inout [UInt8]) { + // Find the bit count of input. + let inputBitLength = input.count * 8 + + // Append the bit 1 at end of input. + input.append(0x80) + + // Find the number of bits we need to append. + // + // inputBitLength + 1 + bitsToAppend ≡ 448 mod 512 + let mod = inputBitLength % 512 + let bitsToAppend = mod < 448 ? 448 - 1 - mod : 512 + 448 - mod - 1 + + // We already appended first 7 bits with 0x80 above. + input += [UInt8](repeating: 0, count: (bitsToAppend - 7) / 8) + + // We need to append 64 bits of input length. + for byte in UInt64(inputBitLength).toByteArray().lazy.reversed() { + input.append(byte) + } + assert((input.count * 8) % 512 == 0, "Expected padded length to be 512.") + } +} + +// MARK:- Helpers + +extension UInt64 { + /// Converts the 64 bit integer into an array of single byte integers. + fileprivate func toByteArray() -> [UInt8] { + var value = self.littleEndian + return withUnsafeBytes(of: &value, Array.init) + } +} + +extension UInt32 { + /// Rotates self by given amount. + fileprivate func rotateRight(by amount: UInt32) -> UInt32 { + return (self >> amount) | (self << (32 - amount)) + } +} + +extension Array { + /// Breaks the array into the given size. + fileprivate func blocks(size: Int) -> AnyIterator> { + var currentIndex = startIndex + return AnyIterator { + if let nextIndex = self.index(currentIndex, offsetBy: size, limitedBy: self.endIndex) { + defer { currentIndex = nextIndex } + return self[currentIndex.. Date: Fri, 4 Apr 2025 16:28:33 -0400 Subject: [PATCH 151/234] Revert "Ensure `Locked+Platform.swift` is not stripped when statically linking. (#1035)" This reverts commit ee700e25f3eb7d51a96ba806413818bbb571b1c8. --- Sources/Testing/Support/Locked+Platform.swift | 11 ----------- Sources/Testing/Support/Locked.swift | 4 ---- 2 files changed, 15 deletions(-) diff --git a/Sources/Testing/Support/Locked+Platform.swift b/Sources/Testing/Support/Locked+Platform.swift index 2b3f5b648..951e62da8 100644 --- a/Sources/Testing/Support/Locked+Platform.swift +++ b/Sources/Testing/Support/Locked+Platform.swift @@ -94,14 +94,3 @@ typealias DefaultLock = Never #warning("Platform-specific implementation missing: locking unavailable") typealias DefaultLock = Never #endif - -#if SWT_NO_DYNAMIC_LINKING -/// A function which, when called by another file, ensures that the file in -/// which ``DefaultLock`` is declared is linked. -/// -/// When static linking is used, the linker may opt to strip some or all of the -/// symbols (including protocol conformance metadata) declared in this file. -/// ``LockedWith`` calls this function in ``LockedWith/init(rawValue:)`` to work -/// around that issue. -func linkLockImplementations() {} -#endif diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index c69cfd351..e8b17be7b 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -66,10 +66,6 @@ struct LockedWith: RawRepresentable where L: Lockable { private nonisolated(unsafe) var _storage: ManagedBuffer init(rawValue: T) { -#if SWT_NO_DYNAMIC_LINKING - linkLockImplementations() -#endif - _storage = _Storage.create(minimumCapacity: 1, makingHeaderWith: { _ in rawValue }) _storage.withUnsafeMutablePointerToElements { lock in L.initializeLock(at: lock) From 0d7a3abe7494f531fe37e29d5f56b0bc872e474a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 5 Apr 2025 16:36:01 -0400 Subject: [PATCH 152/234] [6.2] Update version to 6.2. --- cmake/modules/LibraryVersion.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/LibraryVersion.cmake b/cmake/modules/LibraryVersion.cmake index e50e51182..972307653 100644 --- a/cmake/modules/LibraryVersion.cmake +++ b/cmake/modules/LibraryVersion.cmake @@ -8,7 +8,7 @@ # The current version of the Swift Testing release. For release branches, # remember to remove -dev. -set(SWT_TESTING_LIBRARY_VERSION "6.2-dev") +set(SWT_TESTING_LIBRARY_VERSION "6.2") find_package(Git QUIET) if(Git_FOUND) From 1eba9c04f59dfcfb342c0824b31d3df1d3a34ee5 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 5 Apr 2025 19:18:13 -0400 Subject: [PATCH 153/234] Update the build version to 6.3-dev. (#1052) --- cmake/modules/LibraryVersion.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/LibraryVersion.cmake b/cmake/modules/LibraryVersion.cmake index e50e51182..259ead608 100644 --- a/cmake/modules/LibraryVersion.cmake +++ b/cmake/modules/LibraryVersion.cmake @@ -8,7 +8,7 @@ # The current version of the Swift Testing release. For release branches, # remember to remove -dev. -set(SWT_TESTING_LIBRARY_VERSION "6.2-dev") +set(SWT_TESTING_LIBRARY_VERSION "6.3-dev") find_package(Git QUIET) if(Git_FOUND) From 526ef76567a5e65c3396e0963f0b35bbbd02829f Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 7 Apr 2025 13:56:19 -0400 Subject: [PATCH 154/234] Miscellaneous bookkeeping/cleanup of `_TestDiscovery`. (#1055) This PR has a few minor changes: - The default implementation of `DiscoverableAsTestContent._testContentTypeNameHint` is not present if `SWT_NO_LEGACY_TEST_DISCOVERY` is defined; - Fixed the precondition in `TestContentKind.init(stringLiteral:)` so it doesn't call `utf8CodeUnitCount` on single-character static strings; - The implementation of `TestContentKind._fourCCValue` uses `isprint()` to check if the kind value looks like a FourCC rather than calling ICU-based functions; - Fixed a type in a comment on `SectionBounds.Kind.segmentAndSectionName`; and - Simplified the implementation of `TestContentKind.commentRepresentation` (in the macro target). ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Support/TestContentGeneration.swift | 8 +++----- .../_TestDiscovery/DiscoverableAsTestContent.swift | 2 ++ Sources/_TestDiscovery/SectionBounds.swift | 10 +++++----- Sources/_TestDiscovery/TestContentKind.swift | 13 ++++++------- Sources/_TestingInternals/include/Includes.h | 1 + 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index 9c5136d97..9a2529cee 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -30,12 +30,10 @@ enum TestContentKind: UInt32 { /// This kind value as a comment (`/* 'abcd' */`) if it looks like it might be /// a [FourCC](https://en.wikipedia.org/wiki/FourCC) value, or `nil` if not. var commentRepresentation: Trivia { - switch self { - case .testDeclaration: - .blockComment("/* 'test' */") - case .exitTest: - .blockComment("/* 'exit' */") + let stringValue = withUnsafeBytes(of: self.rawValue.bigEndian) { bytes in + String(decoding: bytes, as: Unicode.ASCII.self) } + return .blockComment("/* '\(stringValue)' */") } } diff --git a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift index a4b400bad..d4b15f8db 100644 --- a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift +++ b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift @@ -48,8 +48,10 @@ public protocol DiscoverableAsTestContent: Sendable, ~Copyable { #endif } +#if !SWT_NO_LEGACY_TEST_DISCOVERY extension DiscoverableAsTestContent where Self: ~Copyable { public static var _testContentTypeNameHint: String { "__🟡$" } } +#endif diff --git a/Sources/_TestDiscovery/SectionBounds.swift b/Sources/_TestDiscovery/SectionBounds.swift index 1a3ae8e11..6b5f0292f 100644 --- a/Sources/_TestDiscovery/SectionBounds.swift +++ b/Sources/_TestDiscovery/SectionBounds.swift @@ -52,11 +52,11 @@ extension SectionBounds.Kind { /// The Mach-O segment and section name for this instance as a pair of /// null-terminated UTF-8 C strings and pass them to a function. /// - /// The values of this property within this function are instances of - /// `StaticString` rather than `String` because the latter's inner storage is - /// sometimes Objective-C-backed and touching it here can cause a recursive - /// access to an internal libobjc lock, whereas `StaticString`'s internal - /// storage is immediately available. + /// The values of this property are instances of `StaticString` rather than + /// `String` because the latter's inner storage is sometimes backed by + /// Objective-C and touching it here can cause a recursive access to an + /// internal libobjc lock, whereas `StaticString`'s internal storage is + /// immediately available. fileprivate var segmentAndSectionName: (segmentName: StaticString, sectionName: StaticString) { switch self { case .testContent: diff --git a/Sources/_TestDiscovery/TestContentKind.swift b/Sources/_TestDiscovery/TestContentKind.swift index 645b06424..4e6955acc 100644 --- a/Sources/_TestDiscovery/TestContentKind.swift +++ b/Sources/_TestDiscovery/TestContentKind.swift @@ -60,8 +60,8 @@ extension TestContentKind: Codable {} extension TestContentKind: ExpressibleByStringLiteral, ExpressibleByIntegerLiteral { @inlinable public init(stringLiteral stringValue: StaticString) { - precondition(stringValue.utf8CodeUnitCount == MemoryLayout.stride, #""\#(stringValue)".utf8CodeUnitCount = \#(stringValue.utf8CodeUnitCount), expected \#(MemoryLayout.stride)"#) let rawValue = stringValue.withUTF8Buffer { stringValue in + precondition(stringValue.count == MemoryLayout.stride, #""\#(stringValue)".utf8CodeUnitCount = \#(stringValue.count), expected \#(MemoryLayout.stride)"#) let bigEndian = UnsafeRawBufferPointer(stringValue).loadUnaligned(as: UInt32.self) return UInt32(bigEndian: bigEndian) } @@ -81,12 +81,11 @@ extension TestContentKind: CustomStringConvertible { /// value, or `nil` if not. private var _fourCCValue: String? { withUnsafeBytes(of: rawValue.bigEndian) { bytes in - if bytes.allSatisfy(Unicode.ASCII.isASCII) { - let characters = String(decoding: bytes, as: Unicode.ASCII.self) - let allAlphanumeric = characters.allSatisfy { $0.isLetter || $0.isWholeNumber } - if allAlphanumeric { - return characters - } + let allPrintableASCII = bytes.allSatisfy { byte in + Unicode.ASCII.isASCII(byte) && 0 != isprint(CInt(byte)) + } + if allPrintableASCII { + return String(decoding: bytes, as: Unicode.ASCII.self) } return nil } diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index 5ba496ee9..bfc87b001 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -26,6 +26,7 @@ /// /// - Note: Avoid including headers that aren't actually used. +#include #include #include /// Guard against including `signal.h` on WASI. The `signal.h` header file From 1ba4e6fa159720128495a86f67c0847f78a511d0 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 7 Apr 2025 13:59:26 -0400 Subject: [PATCH 155/234] Use the entire SHA-256 hash as an exit test ID. (#1053) This PR uses the entire 256 bits of the computed SHA-256 hash for an exit test's ID, not just the first 128. They're there, might as well use them. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 22 +++++---- .../ExpectationChecking+Macro.swift | 2 +- Sources/TestingMacros/ConditionMacro.swift | 49 ++++++++++--------- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index a38c7592e..13e72b691 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -48,13 +48,17 @@ public struct ExitTest: Sendable, ~Copyable { /// Storage for the underlying bits of the ID. /// /// - Note: On Apple platforms, we deploy to OS versions that do not include - /// support for `UInt128`, so we use two `UInt64`s for storage instead. - private var _lo: UInt64 - private var _hi: UInt64 - - init(_ uuid: (UInt64, UInt64)) { - self._lo = uuid.0 - self._hi = uuid.1 + /// support for `UInt128`, so we use four `UInt64`s for storage instead. + private var _0: UInt64 + private var _1: UInt64 + private var _2: UInt64 + private var _3: UInt64 + + init(_ uuid: (UInt64, UInt64, UInt64, UInt64)) { + self._0 = uuid.0 + self._1 = uuid.1 + self._2 = uuid.2 + self._3 = uuid.3 } } @@ -270,7 +274,7 @@ extension ExitTest: DiscoverableAsTestContent { /// - Warning: This function is used to implement the `#expect(exitsWith:)` /// macro. Do not use it directly. public static func __store( - _ id: (UInt64, UInt64), + _ id: (UInt64, UInt64, UInt64, UInt64), _ body: @escaping @Sendable () async throws -> Void, into outValue: UnsafeMutableRawPointer, asTypeAt typeAddress: UnsafeRawPointer, @@ -344,7 +348,7 @@ extension ExitTest { /// `await #expect(exitsWith:) { }` invocations regardless of calling /// convention. func callExitTest( - identifiedBy exitTestID: (UInt64, UInt64), + identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), exitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 7254ad049..aa999395a 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1147,7 +1147,7 @@ public func __checkClosureCall( /// `#require()` macros. Do not call it directly. @_spi(Experimental) public func __checkClosureCall( - identifiedBy exitTestID: (UInt64, UInt64), + identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), exitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], performing body: @convention(thin) () -> Void, diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index c82acd725..f8b87e1fa 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -551,30 +551,35 @@ extension ExitTestConditionMacro { for macro: some FreestandingMacroExpansionSyntax, in context: some MacroExpansionContext ) -> ExprSyntax { - let exitTestID: (UInt64, UInt64) - if let sourceLocation = context.location(of: macro, at: .afterLeadingTrivia, filePathMode: .fileID), - let fileID = sourceLocation.file.as(StringLiteralExprSyntax.self)?.representedLiteralValue, - let line = sourceLocation.line.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue, - let column = sourceLocation.column.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue { - // Hash the entire source location and store as many bits as possible in - // the resulting ID. - let stringValue = "\(fileID):\(line):\(column)" - exitTestID = SHA256.hash(stringValue.utf8).withUnsafeBytes { sha256 in - sha256.loadUnaligned(as: (UInt64, UInt64).self) + withUnsafeTemporaryAllocation(of: UInt64.self, capacity: 4) { exitTestID in + if let sourceLocation = context.location(of: macro, at: .afterLeadingTrivia, filePathMode: .fileID), + let fileID = sourceLocation.file.as(StringLiteralExprSyntax.self)?.representedLiteralValue, + let line = sourceLocation.line.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue, + let column = sourceLocation.column.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue { + // Hash the entire source location and store the entire hash in the + // resulting ID. + let stringValue = "\(fileID):\(line):\(column)" + exitTestID.withMemoryRebound(to: UInt8.self) { exitTestID in + _ = exitTestID.initialize(from: SHA256.hash(stringValue.utf8)) + } + } else { + // This branch is dead code in production, but is used when we expand a + // macro in our own unit tests because the macro expansion context does + // not have real source location information. + for i in 0 ..< exitTestID.count { + exitTestID[i] = .random(in: 0 ... .max) + } } - } else { - // This branch is dead code in production, but is used when we expand a - // macro in our own unit tests because the macro expansion context does - // not have real source location information. - exitTestID.0 = .random(in: 0 ... .max) - exitTestID.1 = .random(in: 0 ... .max) - } - // Return a tuple of integer literals (which is what the runtime __store() - // function is expecting.) - return """ - (\(IntegerLiteralExprSyntax(exitTestID.0, radix: .hex)), \(IntegerLiteralExprSyntax(exitTestID.1, radix: .hex))) - """ + // Return a tuple of integer literals (which is what the runtime __store() + // function is expecting.) + let tupleExpr = TupleExprSyntax { + for uint64 in exitTestID { + LabeledExprSyntax(expression: IntegerLiteralExprSyntax(uint64, radix: .hex)) + } + } + return ExprSyntax(tupleExpr) + } } } From 27d09f01eadab517627c77ed7b5eec7562d3e71a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 7 Apr 2025 14:29:10 -0400 Subject: [PATCH 156/234] Add signal handler for `SIGABRT_COMPAT` on Windows. (#1056) The special-case signal handling implemented for Windows doesn't currently set a handler for `SIGABRT_COMPAT` (which is a synonym of `SIGABRT` with a different value.) This PR adds it to the list of signals we install handlers for. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 13e72b691..6341ce422 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -218,7 +218,7 @@ extension ExitTest { // exit code that is unlikely to be encountered "in the wild" and which // encodes the caught signal. Corresponding code in the parent process looks // for these special exit codes and translates them back to signals. - for sig in [SIGINT, SIGILL, SIGFPE, SIGSEGV, SIGTERM, SIGBREAK, SIGABRT] { + for sig in [SIGINT, SIGILL, SIGFPE, SIGSEGV, SIGTERM, SIGBREAK, SIGABRT, SIGABRT_COMPAT] { _ = signal(sig) { sig in _Exit(STATUS_SIGNAL_CAUGHT_BITS | sig) } From 94c3fc0345459d67573776ba68710cce353b9756 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 8 Apr 2025 00:21:09 -0400 Subject: [PATCH 157/234] [6.2] Miscellaneous bookkeeping/cleanup of `_TestDiscovery`. (#1057) - **Explanation**: Some minor code cleanup in/around the `_TestDiscovery` module. - **Scope**: Internal only - **Issues**: N/A - **Original PRs**: #1055 - **Risk**: Low (nothing obvious) - **Testing**: The usual CI jobs. - **Reviewers**: @briancroom --- .../Support/TestContentGeneration.swift | 8 +++----- .../_TestDiscovery/DiscoverableAsTestContent.swift | 2 ++ Sources/_TestDiscovery/SectionBounds.swift | 10 +++++----- Sources/_TestDiscovery/TestContentKind.swift | 13 ++++++------- Sources/_TestingInternals/include/Includes.h | 1 + 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index 9c5136d97..9a2529cee 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -30,12 +30,10 @@ enum TestContentKind: UInt32 { /// This kind value as a comment (`/* 'abcd' */`) if it looks like it might be /// a [FourCC](https://en.wikipedia.org/wiki/FourCC) value, or `nil` if not. var commentRepresentation: Trivia { - switch self { - case .testDeclaration: - .blockComment("/* 'test' */") - case .exitTest: - .blockComment("/* 'exit' */") + let stringValue = withUnsafeBytes(of: self.rawValue.bigEndian) { bytes in + String(decoding: bytes, as: Unicode.ASCII.self) } + return .blockComment("/* '\(stringValue)' */") } } diff --git a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift index a4b400bad..d4b15f8db 100644 --- a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift +++ b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift @@ -48,8 +48,10 @@ public protocol DiscoverableAsTestContent: Sendable, ~Copyable { #endif } +#if !SWT_NO_LEGACY_TEST_DISCOVERY extension DiscoverableAsTestContent where Self: ~Copyable { public static var _testContentTypeNameHint: String { "__🟡$" } } +#endif diff --git a/Sources/_TestDiscovery/SectionBounds.swift b/Sources/_TestDiscovery/SectionBounds.swift index 1a3ae8e11..6b5f0292f 100644 --- a/Sources/_TestDiscovery/SectionBounds.swift +++ b/Sources/_TestDiscovery/SectionBounds.swift @@ -52,11 +52,11 @@ extension SectionBounds.Kind { /// The Mach-O segment and section name for this instance as a pair of /// null-terminated UTF-8 C strings and pass them to a function. /// - /// The values of this property within this function are instances of - /// `StaticString` rather than `String` because the latter's inner storage is - /// sometimes Objective-C-backed and touching it here can cause a recursive - /// access to an internal libobjc lock, whereas `StaticString`'s internal - /// storage is immediately available. + /// The values of this property are instances of `StaticString` rather than + /// `String` because the latter's inner storage is sometimes backed by + /// Objective-C and touching it here can cause a recursive access to an + /// internal libobjc lock, whereas `StaticString`'s internal storage is + /// immediately available. fileprivate var segmentAndSectionName: (segmentName: StaticString, sectionName: StaticString) { switch self { case .testContent: diff --git a/Sources/_TestDiscovery/TestContentKind.swift b/Sources/_TestDiscovery/TestContentKind.swift index 645b06424..4e6955acc 100644 --- a/Sources/_TestDiscovery/TestContentKind.swift +++ b/Sources/_TestDiscovery/TestContentKind.swift @@ -60,8 +60,8 @@ extension TestContentKind: Codable {} extension TestContentKind: ExpressibleByStringLiteral, ExpressibleByIntegerLiteral { @inlinable public init(stringLiteral stringValue: StaticString) { - precondition(stringValue.utf8CodeUnitCount == MemoryLayout.stride, #""\#(stringValue)".utf8CodeUnitCount = \#(stringValue.utf8CodeUnitCount), expected \#(MemoryLayout.stride)"#) let rawValue = stringValue.withUTF8Buffer { stringValue in + precondition(stringValue.count == MemoryLayout.stride, #""\#(stringValue)".utf8CodeUnitCount = \#(stringValue.count), expected \#(MemoryLayout.stride)"#) let bigEndian = UnsafeRawBufferPointer(stringValue).loadUnaligned(as: UInt32.self) return UInt32(bigEndian: bigEndian) } @@ -81,12 +81,11 @@ extension TestContentKind: CustomStringConvertible { /// value, or `nil` if not. private var _fourCCValue: String? { withUnsafeBytes(of: rawValue.bigEndian) { bytes in - if bytes.allSatisfy(Unicode.ASCII.isASCII) { - let characters = String(decoding: bytes, as: Unicode.ASCII.self) - let allAlphanumeric = characters.allSatisfy { $0.isLetter || $0.isWholeNumber } - if allAlphanumeric { - return characters - } + let allPrintableASCII = bytes.allSatisfy { byte in + Unicode.ASCII.isASCII(byte) && 0 != isprint(CInt(byte)) + } + if allPrintableASCII { + return String(decoding: bytes, as: Unicode.ASCII.self) } return nil } diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index 5ba496ee9..bfc87b001 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -26,6 +26,7 @@ /// /// - Note: Avoid including headers that aren't actually used. +#include #include #include /// Guard against including `signal.h` on WASI. The `signal.h` header file From 3cd81d038463a06626a9db6552d6fd8cf76b4686 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 8 Apr 2025 00:21:23 -0400 Subject: [PATCH 158/234] [6.2] Use the entire SHA-256 hash as an exit test ID. (#1058) - **Explanation**: Uses the full 256 bits from a SHA-256 hash to uniquely identify an exit test instead of truncating to 128 bits. - **Scope**: Exit test IDs - **Issues**: N/A - **Original PRs**: #1053 - **Risk**: Low (everything involved is recompiled) - **Testing**: Normal CI jobs. - **Reviewers**: @briancroom --- Sources/Testing/ExitTests/ExitTest.swift | 22 +++++---- .../ExpectationChecking+Macro.swift | 2 +- Sources/TestingMacros/ConditionMacro.swift | 49 ++++++++++--------- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index a38c7592e..13e72b691 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -48,13 +48,17 @@ public struct ExitTest: Sendable, ~Copyable { /// Storage for the underlying bits of the ID. /// /// - Note: On Apple platforms, we deploy to OS versions that do not include - /// support for `UInt128`, so we use two `UInt64`s for storage instead. - private var _lo: UInt64 - private var _hi: UInt64 - - init(_ uuid: (UInt64, UInt64)) { - self._lo = uuid.0 - self._hi = uuid.1 + /// support for `UInt128`, so we use four `UInt64`s for storage instead. + private var _0: UInt64 + private var _1: UInt64 + private var _2: UInt64 + private var _3: UInt64 + + init(_ uuid: (UInt64, UInt64, UInt64, UInt64)) { + self._0 = uuid.0 + self._1 = uuid.1 + self._2 = uuid.2 + self._3 = uuid.3 } } @@ -270,7 +274,7 @@ extension ExitTest: DiscoverableAsTestContent { /// - Warning: This function is used to implement the `#expect(exitsWith:)` /// macro. Do not use it directly. public static func __store( - _ id: (UInt64, UInt64), + _ id: (UInt64, UInt64, UInt64, UInt64), _ body: @escaping @Sendable () async throws -> Void, into outValue: UnsafeMutableRawPointer, asTypeAt typeAddress: UnsafeRawPointer, @@ -344,7 +348,7 @@ extension ExitTest { /// `await #expect(exitsWith:) { }` invocations regardless of calling /// convention. func callExitTest( - identifiedBy exitTestID: (UInt64, UInt64), + identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), exitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 7254ad049..aa999395a 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1147,7 +1147,7 @@ public func __checkClosureCall( /// `#require()` macros. Do not call it directly. @_spi(Experimental) public func __checkClosureCall( - identifiedBy exitTestID: (UInt64, UInt64), + identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), exitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], performing body: @convention(thin) () -> Void, diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index c82acd725..f8b87e1fa 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -551,30 +551,35 @@ extension ExitTestConditionMacro { for macro: some FreestandingMacroExpansionSyntax, in context: some MacroExpansionContext ) -> ExprSyntax { - let exitTestID: (UInt64, UInt64) - if let sourceLocation = context.location(of: macro, at: .afterLeadingTrivia, filePathMode: .fileID), - let fileID = sourceLocation.file.as(StringLiteralExprSyntax.self)?.representedLiteralValue, - let line = sourceLocation.line.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue, - let column = sourceLocation.column.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue { - // Hash the entire source location and store as many bits as possible in - // the resulting ID. - let stringValue = "\(fileID):\(line):\(column)" - exitTestID = SHA256.hash(stringValue.utf8).withUnsafeBytes { sha256 in - sha256.loadUnaligned(as: (UInt64, UInt64).self) + withUnsafeTemporaryAllocation(of: UInt64.self, capacity: 4) { exitTestID in + if let sourceLocation = context.location(of: macro, at: .afterLeadingTrivia, filePathMode: .fileID), + let fileID = sourceLocation.file.as(StringLiteralExprSyntax.self)?.representedLiteralValue, + let line = sourceLocation.line.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue, + let column = sourceLocation.column.as(IntegerLiteralExprSyntax.self)?.representedLiteralValue { + // Hash the entire source location and store the entire hash in the + // resulting ID. + let stringValue = "\(fileID):\(line):\(column)" + exitTestID.withMemoryRebound(to: UInt8.self) { exitTestID in + _ = exitTestID.initialize(from: SHA256.hash(stringValue.utf8)) + } + } else { + // This branch is dead code in production, but is used when we expand a + // macro in our own unit tests because the macro expansion context does + // not have real source location information. + for i in 0 ..< exitTestID.count { + exitTestID[i] = .random(in: 0 ... .max) + } } - } else { - // This branch is dead code in production, but is used when we expand a - // macro in our own unit tests because the macro expansion context does - // not have real source location information. - exitTestID.0 = .random(in: 0 ... .max) - exitTestID.1 = .random(in: 0 ... .max) - } - // Return a tuple of integer literals (which is what the runtime __store() - // function is expecting.) - return """ - (\(IntegerLiteralExprSyntax(exitTestID.0, radix: .hex)), \(IntegerLiteralExprSyntax(exitTestID.1, radix: .hex))) - """ + // Return a tuple of integer literals (which is what the runtime __store() + // function is expecting.) + let tupleExpr = TupleExprSyntax { + for uint64 in exitTestID { + LabeledExprSyntax(expression: IntegerLiteralExprSyntax(uint64, radix: .hex)) + } + } + return ExprSyntax(tupleExpr) + } } } From e99d47ee4c7e234add19a8368b48ea3453447750 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 8 Apr 2025 13:52:04 -0400 Subject: [PATCH 159/234] [6.2] Add signal handler for `SIGABRT_COMPAT` on Windows. (#1059) - **Explanation**: Make sure all public signals on Windows are handled correctly in exit tests. - **Scope**: Windows exit tests - **Issues**: N/A - **Original PRs**: #1056 - **Risk**: Low - **Testing**: CI jobs - **Reviewers**: @briancroom @stmontgomery @compnerd --- Sources/Testing/ExitTests/ExitTest.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 13e72b691..6341ce422 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -218,7 +218,7 @@ extension ExitTest { // exit code that is unlikely to be encountered "in the wild" and which // encodes the caught signal. Corresponding code in the parent process looks // for these special exit codes and translates them back to signals. - for sig in [SIGINT, SIGILL, SIGFPE, SIGSEGV, SIGTERM, SIGBREAK, SIGABRT] { + for sig in [SIGINT, SIGILL, SIGFPE, SIGSEGV, SIGTERM, SIGBREAK, SIGABRT, SIGABRT_COMPAT] { _ = signal(sig) { sig in _Exit(STATUS_SIGNAL_CAUGHT_BITS | sig) } From 7907a4a153bb89bf709b78349ef1fdd550ff3cff Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 8 Apr 2025 18:15:00 -0400 Subject: [PATCH 160/234] [Experimental] Add Embedded Swift support to the `_TestDiscovery` target. (#1043) This PR adds preliminary/experimental support for Embedded Swift _to the `_TestDiscovery` target only_ when building Swift Testing as a package. To try it out, you must set the environment variable `SWT_EMBEDDED` to `true` before building. Tested with the following incantation using the 2025-03-28 main-branch toolchain: ```sh SWT_EMBEDDED=1 swift build --target _TestDiscovery --triple arm64-apple-macosx ``` ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/TestContent.md | 5 +- Package.swift | 103 +++++++++++++++--- Sources/Testing/ExitTests/ExitTest.swift | 2 + Sources/Testing/Test+Discovery.swift | 2 + Sources/_TestDiscovery/TestContentKind.swift | 2 + .../_TestDiscovery/TestContentRecord.swift | 53 ++++++--- Tests/TestingTests/DiscoveryTests.swift | 2 + 7 files changed, 136 insertions(+), 33 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index cb68a2d6e..be2493530 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -126,7 +126,10 @@ or a third-party library are inadvertently loaded into the same process. If the value at `type` does not match the test content record's expected type, the accessor function must return `false` and must not modify `outValue`. - +When building for **Embedded Swift**, the value passed as `type` by Swift +Testing is unspecified because type metadata pointers are not available in that +environment. + [^mightNotBeSwift]: Although this document primarily deals with Swift, the test content record section is generally language-agnostic. The use of languages diff --git a/Package.swift b/Package.swift index 8085d7bc8..4194416fb 100644 --- a/Package.swift +++ b/Package.swift @@ -20,17 +20,49 @@ let git = Context.gitInformation /// distribution as a package dependency. let buildingForDevelopment = (git?.currentTag == nil) +/// Whether or not this package is being built for Embedded Swift. +/// +/// This value is `true` if `SWT_EMBEDDED` is set in the environment to `true` +/// when `swift build` is invoked. This inference is experimental and is subject +/// to change in the future. +/// +/// - Bug: There is currently no way for us to tell if we are being asked to +/// build for an Embedded Swift target at the package manifest level. +/// ([swift-syntax-#8431](https://github.com/swiftlang/swift-package-manager/issues/8431)) +let buildingForEmbedded: Bool = { + guard let envvar = Context.environment["SWT_EMBEDDED"] else { + return false + } + return Bool(envvar) ?? ((Int(envvar) ?? 0) != 0) +}() + let package = Package( name: "swift-testing", - platforms: [ - .macOS(.v10_15), - .iOS(.v13), - .watchOS(.v6), - .tvOS(.v13), - .macCatalyst(.v13), - .visionOS(.v1), - ], + platforms: { + if !buildingForEmbedded { + [ + .macOS(.v10_15), + .iOS(.v13), + .watchOS(.v6), + .tvOS(.v13), + .macCatalyst(.v13), + .visionOS(.v1), + ] + } else { + // Open-source main-branch toolchains (currently required to build this + // package for Embedded Swift) have higher Apple platform deployment + // targets than we would otherwise require. + [ + .macOS(.v14), + .iOS(.v18), + .watchOS(.v10), + .tvOS(.v18), + .macCatalyst(.v18), + .visionOS(.v1), + ] + } + }(), products: { var result = [Product]() @@ -185,6 +217,31 @@ package.targets.append(contentsOf: [ ]) #endif +extension BuildSettingCondition { + /// Creates a build setting condition that evaluates to `true` for Embedded + /// Swift. + /// + /// - Parameters: + /// - nonEmbeddedCondition: The value to return if the target is not + /// Embedded Swift. If `nil`, the build condition evaluates to `false`. + /// + /// - Returns: A build setting condition that evaluates to `true` for Embedded + /// Swift or is equal to `nonEmbeddedCondition` for non-Embedded Swift. + static func whenEmbedded(or nonEmbeddedCondition: @autoclosure () -> Self? = nil) -> Self? { + if !buildingForEmbedded { + if let nonEmbeddedCondition = nonEmbeddedCondition() { + nonEmbeddedCondition + } else { + // The caller did not supply a fallback. + .when(platforms: []) + } + } else { + // Enable unconditionally because the target is Embedded Swift. + nil + } + } +} + extension Array where Element == PackageDescription.SwiftSetting { /// Settings intended to be applied to every Swift target in this package. /// Analogous to project-level build settings in an Xcode project. @@ -195,6 +252,10 @@ extension Array where Element == PackageDescription.SwiftSetting { result.append(.unsafeFlags(["-require-explicit-sendable"])) } + if buildingForEmbedded { + result.append(.enableExperimentalFeature("Embedded")) + } + result += [ .enableUpcomingFeature("ExistentialAny"), @@ -214,11 +275,14 @@ extension Array where Element == PackageDescription.SwiftSetting { .define("SWT_TARGET_OS_APPLE", .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS])), - .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android])), - .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), - .define("SWT_NO_PIPES", .when(platforms: [.wasi])), + .define("SWT_NO_EXIT_TESTS", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), + .define("SWT_NO_PROCESS_SPAWNING", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), + .define("SWT_NO_SNAPSHOT_TYPES", .whenEmbedded(or: .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android]))), + .define("SWT_NO_DYNAMIC_LINKING", .whenEmbedded(or: .when(platforms: [.wasi]))), + .define("SWT_NO_PIPES", .whenEmbedded(or: .when(platforms: [.wasi]))), + + .define("SWT_NO_LEGACY_TEST_DISCOVERY", .whenEmbedded()), + .define("SWT_NO_LIBDISPATCH", .whenEmbedded()), ] return result @@ -271,11 +335,14 @@ extension Array where Element == PackageDescription.CXXSetting { var result = Self() result += [ - .define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])), - .define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android])), - .define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])), - .define("SWT_NO_PIPES", .when(platforms: [.wasi])), + .define("SWT_NO_EXIT_TESTS", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), + .define("SWT_NO_PROCESS_SPAWNING", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), + .define("SWT_NO_SNAPSHOT_TYPES", .whenEmbedded(or: .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android]))), + .define("SWT_NO_DYNAMIC_LINKING", .whenEmbedded(or: .when(platforms: [.wasi]))), + .define("SWT_NO_PIPES", .whenEmbedded(or: .when(platforms: [.wasi]))), + + .define("SWT_NO_LEGACY_TEST_DISCOVERY", .whenEmbedded()), + .define("SWT_NO_LIBDISPATCH", .whenEmbedded()), ] // Capture the testing library's version as a C++ string constant. diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 6341ce422..cff8eb5a1 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -280,11 +280,13 @@ extension ExitTest: DiscoverableAsTestContent { asTypeAt typeAddress: UnsafeRawPointer, withHintAt hintAddress: UnsafeRawPointer? = nil ) -> CBool { +#if !hasFeature(Embedded) let callerExpectedType = TypeInfo(describing: typeAddress.load(as: Any.Type.self)) let selfType = TypeInfo(describing: Self.self) guard callerExpectedType == selfType else { return false } +#endif let id = ID(id) if let hintedID = hintAddress?.load(as: ID.self), hintedID != id { return false diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 5d1b204ae..35f716525 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -44,9 +44,11 @@ extension Test { into outValue: UnsafeMutableRawPointer, asTypeAt typeAddress: UnsafeRawPointer ) -> CBool { +#if !hasFeature(Embedded) guard typeAddress.load(as: Any.Type.self) == Generator.self else { return false } +#endif outValue.initializeMemory(as: Generator.self, to: .init(rawValue: generator)) return true } diff --git a/Sources/_TestDiscovery/TestContentKind.swift b/Sources/_TestDiscovery/TestContentKind.swift index 4e6955acc..30f201a83 100644 --- a/Sources/_TestDiscovery/TestContentKind.swift +++ b/Sources/_TestDiscovery/TestContentKind.swift @@ -52,9 +52,11 @@ extension TestContentKind: Equatable, Hashable { } } +#if !hasFeature(Embedded) // MARK: - Codable extension TestContentKind: Codable {} +#endif // MARK: - ExpressibleByStringLiteral, ExpressibleByIntegerLiteral diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index 25f46fa44..d893664ee 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -139,6 +139,34 @@ public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyabl /// The type of the `hint` argument to ``load(withHint:)``. public typealias Hint = T.TestContentAccessorHint + /// Invoke an accessor function to load a test content record. + /// + /// - Parameters: + /// - accessor: The accessor function to call. + /// - typeAddress: A pointer to the type of test content record. + /// - hint: An optional hint value. + /// + /// - Returns: An instance of the test content type `T`, or `nil` if the + /// underlying test content record did not match `hint` or otherwise did not + /// produce a value. + /// + /// Do not call this function directly. Instead, call ``load(withHint:)``. + private static func _load(using accessor: _TestContentRecordAccessor, withTypeAt typeAddress: UnsafeRawPointer, withHint hint: Hint? = nil) -> T? { + withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in + let initialized = if let hint { + withUnsafePointer(to: hint) { hint in + accessor(buffer.baseAddress!, typeAddress, hint, 0) + } + } else { + accessor(buffer.baseAddress!, typeAddress, nil, 0) + } + guard initialized else { + return nil + } + return buffer.baseAddress!.move() + } + } + /// Load the value represented by this record. /// /// - Parameters: @@ -157,21 +185,14 @@ public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyabl return nil } - return withUnsafePointer(to: T.self) { type in - withUnsafeTemporaryAllocation(of: T.self, capacity: 1) { buffer in - let initialized = if let hint { - withUnsafePointer(to: hint) { hint in - accessor(buffer.baseAddress!, type, hint, 0) - } - } else { - accessor(buffer.baseAddress!, type, nil, 0) - } - guard initialized else { - return nil - } - return buffer.baseAddress!.move() - } +#if !hasFeature(Embedded) + return withUnsafePointer(to: T.self) { typeAddress in + Self._load(using: accessor, withTypeAt: typeAddress, withHint: hint) } +#else + let typeAddress = UnsafeRawPointer(bitPattern: UInt(T.testContentKind.rawValue)).unsafelyUnwrapped + return Self._load(using: accessor, withTypeAt: typeAddress, withHint: hint) +#endif } } @@ -188,7 +209,11 @@ extension TestContentRecord: Sendable where Context: Sendable {} extension TestContentRecord: CustomStringConvertible { public var description: String { +#if !hasFeature(Embedded) let typeName = String(describing: Self.self) +#else + let typeName = "TestContentRecord" +#endif switch _recordStorage { case let .atAddress(recordAddress): let recordAddress = imageAddress.map { imageAddress in diff --git a/Tests/TestingTests/DiscoveryTests.swift b/Tests/TestingTests/DiscoveryTests.swift index a730f8b53..2b53cd467 100644 --- a/Tests/TestingTests/DiscoveryTests.swift +++ b/Tests/TestingTests/DiscoveryTests.swift @@ -94,9 +94,11 @@ struct DiscoveryTests { 0xABCD1234, 0, { outValue, type, hint, _ in +#if !hasFeature(Embedded) guard type.load(as: Any.Type.self) == MyTestContent.self else { return false } +#endif if let hint, hint.load(as: TestContentAccessorHint.self) != expectedHint { return false } From 6cf0110dce46713fefb98cb48a16934336ee5c97 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 9 Apr 2025 11:45:14 -0400 Subject: [PATCH 161/234] Don't use a class to store the current exit test. (#1065) This PR replaces the private `_CurrentContainer` class used to store the current exit test with a bare pointer. The class is prone to duplicate definitions, but we really just use it as a glorified box type for the move-only `ExitTest`, so an immortal pointer will work just as well. Works around rdar://148837303. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 50 +++++++++++++----------- Sources/Testing/Support/Locked.swift | 4 +- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index cff8eb5a1..93393b69b 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -107,6 +107,18 @@ public struct ExitTest: Sendable, ~Copyable { _observedValues = newValue } } + + /// Make a copy of this instance. + /// + /// - Returns: A copy of this instance. + /// + /// This function is unsafe because if the caller is not careful, it could + /// invoke the same exit test twice. + fileprivate borrowing func unsafeCopy() -> Self { + var result = Self(id: id, body: body) + result._observedValues = _observedValues + return result + } } #if !SWT_NO_EXIT_TESTS @@ -114,24 +126,15 @@ public struct ExitTest: Sendable, ~Copyable { @_spi(Experimental) extension ExitTest { - /// A container type to hold the current exit test. - /// - /// This class is temporarily necessary until `ManagedBuffer` is updated to - /// support storing move-only values. For more information, see [SE-NNNN](https://github.com/swiftlang/swift-evolution/pull/2657). - private final class _CurrentContainer: Sendable { - /// The exit test represented by this container. - /// - /// The value of this property must be optional to avoid a copy when reading - /// the value in ``ExitTest/current``. - let exitTest: ExitTest? - - init(exitTest: borrowing ExitTest) { - self.exitTest = ExitTest(id: exitTest.id, body: exitTest.body, _observedValues: exitTest._observedValues) - } - } - /// Storage for ``current``. - private static let _current = Locked<_CurrentContainer?>() + /// + /// A pointer is used for indirection because `ManagedBuffer` cannot yet hold + /// move-only types. + private static nonisolated(unsafe) var _current: Locked> = { + let current = UnsafeMutablePointer.allocate(capacity: 1) + current.initialize(to: nil) + return Locked(rawValue: current) + }() /// The exit test that is running in the current process, if any. /// @@ -144,11 +147,13 @@ extension ExitTest { /// process. public static var current: ExitTest? { _read { - if let current = _current.rawValue { - yield current.exitTest - } else { - yield nil + // NOTE: Even though this accessor is `_read` and has borrowing semantics, + // we must make a copy so that we don't yield lock-guarded memory to the + // caller (which is not concurrency-safe.) + let currentCopy = _current.withLock { current in + return current.pointee?.unsafeCopy() } + yield currentCopy } } } @@ -235,7 +240,8 @@ extension ExitTest { // Set ExitTest.current before the test body runs. Self._current.withLock { current in - current = _CurrentContainer(exitTest: self) + precondition(current.pointee == nil, "Set the current exit test twice in the same process. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + current.pointee = self.unsafeCopy() } do { diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index e8b17be7b..d1db8ef1f 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -88,7 +88,7 @@ struct LockedWith: RawRepresentable where L: Lockable { /// This function can be used to synchronize access to shared data from a /// synchronous caller. Wherever possible, use actor isolation or other Swift /// concurrency tools. - nonmutating func withLock(_ body: (inout T) throws -> R) rethrows -> R { + nonmutating func withLock(_ body: (inout T) throws -> R) rethrows -> R where R: ~Copyable { try _storage.withUnsafeMutablePointers { rawValue, lock in L.unsafelyAcquireLock(at: lock) defer { @@ -118,7 +118,7 @@ struct LockedWith: RawRepresentable where L: Lockable { /// - Warning: Callers that unlock the lock _must_ lock it again before the /// closure returns. If the lock is not acquired when `body` returns, the /// effect is undefined. - nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { + nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R where R: ~Copyable { try withLock { value in try _storage.withUnsafeMutablePointerToElements { lock in try body(lock, value) From 9810afee6d44c1b0bb89a88bb9e69bc8f865f8d3 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 9 Apr 2025 13:17:53 -0400 Subject: [PATCH 162/234] [6.2] Don't use a class to store the current exit test. (#1066) - **Explanation**: Fixes a spurious runtime warning about duplicate definitions of an internal class. - **Scope**: `ExitTest.current` - **Issues**: rdar://148837303 - **Original PRs**: #1065 - **Risk**: Low - **Testing**: Typical CI testing - **Reviewers**: @stmontgomery @briancroom --- Sources/Testing/ExitTests/ExitTest.swift | 50 +++++++++++++----------- Sources/Testing/Support/Locked.swift | 4 +- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 6341ce422..95c3ec696 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -107,6 +107,18 @@ public struct ExitTest: Sendable, ~Copyable { _observedValues = newValue } } + + /// Make a copy of this instance. + /// + /// - Returns: A copy of this instance. + /// + /// This function is unsafe because if the caller is not careful, it could + /// invoke the same exit test twice. + fileprivate borrowing func unsafeCopy() -> Self { + var result = Self(id: id, body: body) + result._observedValues = _observedValues + return result + } } #if !SWT_NO_EXIT_TESTS @@ -114,24 +126,15 @@ public struct ExitTest: Sendable, ~Copyable { @_spi(Experimental) extension ExitTest { - /// A container type to hold the current exit test. - /// - /// This class is temporarily necessary until `ManagedBuffer` is updated to - /// support storing move-only values. For more information, see [SE-NNNN](https://github.com/swiftlang/swift-evolution/pull/2657). - private final class _CurrentContainer: Sendable { - /// The exit test represented by this container. - /// - /// The value of this property must be optional to avoid a copy when reading - /// the value in ``ExitTest/current``. - let exitTest: ExitTest? - - init(exitTest: borrowing ExitTest) { - self.exitTest = ExitTest(id: exitTest.id, body: exitTest.body, _observedValues: exitTest._observedValues) - } - } - /// Storage for ``current``. - private static let _current = Locked<_CurrentContainer?>() + /// + /// A pointer is used for indirection because `ManagedBuffer` cannot yet hold + /// move-only types. + private static nonisolated(unsafe) var _current: Locked> = { + let current = UnsafeMutablePointer.allocate(capacity: 1) + current.initialize(to: nil) + return Locked(rawValue: current) + }() /// The exit test that is running in the current process, if any. /// @@ -144,11 +147,13 @@ extension ExitTest { /// process. public static var current: ExitTest? { _read { - if let current = _current.rawValue { - yield current.exitTest - } else { - yield nil + // NOTE: Even though this accessor is `_read` and has borrowing semantics, + // we must make a copy so that we don't yield lock-guarded memory to the + // caller (which is not concurrency-safe.) + let currentCopy = _current.withLock { current in + return current.pointee?.unsafeCopy() } + yield currentCopy } } } @@ -235,7 +240,8 @@ extension ExitTest { // Set ExitTest.current before the test body runs. Self._current.withLock { current in - current = _CurrentContainer(exitTest: self) + precondition(current.pointee == nil, "Set the current exit test twice in the same process. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + current.pointee = self.unsafeCopy() } do { diff --git a/Sources/Testing/Support/Locked.swift b/Sources/Testing/Support/Locked.swift index c69cfd351..a4ab92d66 100644 --- a/Sources/Testing/Support/Locked.swift +++ b/Sources/Testing/Support/Locked.swift @@ -92,7 +92,7 @@ struct LockedWith: RawRepresentable where L: Lockable { /// This function can be used to synchronize access to shared data from a /// synchronous caller. Wherever possible, use actor isolation or other Swift /// concurrency tools. - nonmutating func withLock(_ body: (inout T) throws -> R) rethrows -> R { + nonmutating func withLock(_ body: (inout T) throws -> R) rethrows -> R where R: ~Copyable { try _storage.withUnsafeMutablePointers { rawValue, lock in L.unsafelyAcquireLock(at: lock) defer { @@ -122,7 +122,7 @@ struct LockedWith: RawRepresentable where L: Lockable { /// - Warning: Callers that unlock the lock _must_ lock it again before the /// closure returns. If the lock is not acquired when `body` returns, the /// effect is undefined. - nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R { + nonmutating func withUnsafeUnderlyingLock(_ body: (UnsafeMutablePointer, T) throws -> R) rethrows -> R where R: ~Copyable { try withLock { value in try _storage.withUnsafeMutablePointerToElements { lock in try body(lock, value) From 756316c0c456b7dda07986854d1a150142079bad Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 9 Apr 2025 13:03:25 -0500 Subject: [PATCH 163/234] Fully-qualify reference to Swift's `Actor` protocol in macro expansion code for synchronous test functions (#1067) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes a compilation error in code expanded from the `@Test` macro when it's attached to a synchronous (i.e. non-`async`) test function in a context where there is a concrete type named `Actor`. For example, the following code reproduces the error: ```swift // In MyApp public class Actor {} // In test code import Testing import MyApp // ❌ 'any' has no effect on concrete type 'Actor' // - 'isolated' parameter type 'Actor?' does not conform to 'Actor' or 'DistributedActor' @Test func example() /* No 'async' */ {} ``` The macro code includes an unqualified reference to a type by that name, but it's intended to refer to the protocol in Swift's `_Concurrency` module. The fix is to ensure the macro's reference to this protocol is fully-qualified with a module name. This was first reported on the Swift Forums in https://forums.swift.org/t/error-isolated-parameter-type-actor-does-not-conform-to-actor-or-distributedactor/79190. This bug was introduced in #747, which first landed in Swift 6.1 and Xcode 16.3. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/TestingMacros/TestDeclarationMacro.swift | 2 +- Tests/TestingTests/MiscellaneousTests.swift | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 2a4da4e3c..307b1615d 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -328,7 +328,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } FunctionParameterSyntax( firstName: .wildcardToken(), - type: "isolated (any Actor)?" as TypeSyntax, + type: "isolated (any _Concurrency.Actor)?" as TypeSyntax, defaultValue: InitializerClauseSyntax(value: "Testing.__defaultSynchronousIsolationContext" as ExprSyntax) ) } diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 1f18f20a9..b4b12a217 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -139,6 +139,11 @@ struct SendableTests: Sendable { @Suite("Named Sendable test type", .hidden) struct NamedSendableTests: Sendable {} +// This is meant to help detect unqualified usages of the `Actor` protocol from +// Swift's `_Concurrency` module in macro expansion code, since it's possible +// for another module to declare a type with that name. +private class Actor {} + #if !SWT_NO_GLOBAL_ACTORS @Suite(.hidden) @MainActor From 45384e526b9646b07b30ea9f734028fcc7571b45 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 10 Apr 2025 17:30:25 -0400 Subject: [PATCH 164/234] Promote attachments to API (#973) This PR promotes attachments to API and makes the appropriate changes to match what was approved in the review of [ST-0009](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0009-attachments.md). ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/ABI/JSON.md | 7 +- .../Attachment+AttachableAsCGImage.swift | 10 +- ...er.swift => _AttachableImageWrapper.swift} | 8 +- .../Attachable+Encodable+NSSecureCoding.swift | 6 +- .../Attachments/Attachable+Encodable.swift | 11 ++- .../Attachable+NSSecureCoding.swift | 11 ++- .../Attachments/Attachment+URL.swift | 13 ++- .../Attachments/Data+Attachable.swift | 9 +- .../Attachments/EncodingFormat.swift | 2 +- ...iner.swift => _AttachableURLWrapper.swift} | 9 +- .../_Testing_Foundation/CMakeLists.txt | 2 +- .../ABI/Encoded/ABI.EncodedEvent.swift | 8 +- Sources/Testing/Attachments/Attachable.swift | 26 +++-- ...ontainer.swift => AttachableWrapper.swift} | 23 +++-- Sources/Testing/Attachments/Attachment.swift | 99 +++++++++++++------ Sources/Testing/CMakeLists.txt | 2 +- Sources/Testing/Events/Event.swift | 1 - .../Events/Recorder/Event.Symbol.swift | 1 - Sources/Testing/Issues/Issue.swift | 1 - Sources/Testing/Running/Configuration.swift | 1 - Sources/Testing/Testing.docc/Attachments.md | 32 ++++++ Sources/Testing/Testing.docc/Documentation.md | 4 + .../Testing.docc/MigratingFromXCTest.md | 60 +++++++++++ Tests/TestingTests/AttachmentTests.swift | 4 +- 24 files changed, 265 insertions(+), 85 deletions(-) rename Sources/Overlays/_Testing_CoreGraphics/Attachments/{_AttachableImageContainer.swift => _AttachableImageWrapper.swift} (96%) rename Sources/Overlays/_Testing_Foundation/Attachments/{_AttachableURLContainer.swift => _AttachableURLWrapper.swift} (90%) rename Sources/Testing/Attachments/{AttachableContainer.swift => AttachableWrapper.swift} (70%) create mode 100644 Sources/Testing/Testing.docc/Attachments.md diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index f313ddc00..e4ff24a4b 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -188,19 +188,24 @@ sufficient information to display the event in a human-readable format. "kind": , "instant": , ; when the event occurred ["issue": ,] ; the recorded issue (if "kind" is "issueRecorded") + ["attachment": ,] ; the attachment (if kind is "valueAttached") "messages": , ["testID": ,] } ::= "runStarted" | "testStarted" | "testCaseStarted" | "issueRecorded" | "testCaseEnded" | "testEnded" | "testSkipped" | - "runEnded" ; additional event kinds may be added in the future + "runEnded" | "valueAttached"; additional event kinds may be added in the future ::= { "isKnown": , ; is this a known issue or not? ["sourceLocation": ,] ; where the issue occurred, if known } + ::= { + "path": , ; the absolute path to the attachment on disk +} + ::= { "symbol": , "text": , ; the human-readable text of this message diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift index ca520e0c0..ed1e6a2ee 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/Attachment+AttachableAsCGImage.swift @@ -42,9 +42,9 @@ extension Attachment { contentType: (any Sendable)?, encodingQuality: Float, sourceLocation: SourceLocation - ) where AttachableValue == _AttachableImageContainer { - let imageContainer = _AttachableImageContainer(image: attachableValue, encodingQuality: encodingQuality, contentType: contentType) - self.init(imageContainer, named: preferredName, sourceLocation: sourceLocation) + ) where AttachableValue == _AttachableImageWrapper { + let imageWrapper = _AttachableImageWrapper(image: attachableValue, encodingQuality: encodingQuality, contentType: contentType) + self.init(imageWrapper, named: preferredName, sourceLocation: sourceLocation) } /// Initialize an instance of this type that encloses the given image. @@ -79,7 +79,7 @@ extension Attachment { as contentType: UTType?, encodingQuality: Float = 1.0, sourceLocation: SourceLocation = #_sourceLocation - ) where AttachableValue == _AttachableImageContainer { + ) where AttachableValue == _AttachableImageWrapper { self.init(attachableValue: attachableValue, named: preferredName, contentType: contentType, encodingQuality: encodingQuality, sourceLocation: sourceLocation) } @@ -109,7 +109,7 @@ extension Attachment { named preferredName: String? = nil, encodingQuality: Float = 1.0, sourceLocation: SourceLocation = #_sourceLocation - ) where AttachableValue == _AttachableImageContainer { + ) where AttachableValue == _AttachableImageWrapper { self.init(attachableValue: attachableValue, named: preferredName, contentType: nil, encodingQuality: encodingQuality, sourceLocation: sourceLocation) } } diff --git a/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageWrapper.swift similarity index 96% rename from Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift rename to Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageWrapper.swift index 90d1c0c70..7aa1fd139 100644 --- a/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageContainer.swift +++ b/Sources/Overlays/_Testing_CoreGraphics/Attachments/_AttachableImageWrapper.swift @@ -9,7 +9,7 @@ // #if SWT_TARGET_OS_APPLE && canImport(CoreGraphics) -@_spi(Experimental) public import Testing +public import Testing private import CoreGraphics private import ImageIO @@ -48,7 +48,7 @@ import UniformTypeIdentifiers /// /// - [`CGImage`](https://developer.apple.com/documentation/coregraphics/cgimage) @_spi(Experimental) -public struct _AttachableImageContainer: Sendable where Image: AttachableAsCGImage { +public struct _AttachableImageWrapper: Sendable where Image: AttachableAsCGImage { /// The underlying image. /// /// `CGImage` and `UIImage` are sendable, but `NSImage` is not. `NSImage` @@ -127,8 +127,8 @@ public struct _AttachableImageContainer: Sendable where Image: Attachable // MARK: - -extension _AttachableImageContainer: AttachableContainer { - public var attachableValue: Image { +extension _AttachableImageWrapper: AttachableWrapper { + public var wrappedValue: Image { image } diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift index cd26c24cc..46a1e11e6 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) -@_spi(Experimental) public import Testing +public import Testing public import Foundation // This implementation is necessary to let the compiler disambiguate when a type @@ -18,7 +18,9 @@ public import Foundation // (which explicitly document what happens when a type conforms to both // protocols.) -@_spi(Experimental) +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } extension Attachable where Self: Encodable & NSSecureCoding { @_documentation(visibility: private) public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift index 812db0b70..683888801 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) -@_spi(Experimental) public import Testing +public import Testing private import Foundation /// A common implementation of ``withUnsafeBytes(for:_:)`` that is used when a @@ -53,7 +53,10 @@ func withUnsafeBytes(encoding attachableValue: borrowing E, for attachment // Implement the protocol requirements generically for any encodable value by // encoding to JSON. This lets developers provide trivial conformance to the // protocol for types that already support Codable. -@_spi(Experimental) + +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } extension Attachable where Self: Encodable { /// Encode this value into a buffer using either [`PropertyListEncoder`](https://developer.apple.com/documentation/foundation/propertylistencoder) /// or [`JSONEncoder`](https://developer.apple.com/documentation/foundation/jsonencoder), @@ -86,6 +89,10 @@ extension Attachable where Self: Encodable { /// _and_ [`NSSecureCoding`](https://developer.apple.com/documentation/foundation/nssecurecoding), /// the default implementation of this function uses the value's conformance /// to `Encodable`. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try _Testing_Foundation.withUnsafeBytes(encoding: self, for: attachment, body) } diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift index c2cc28ea0..4acbf4960 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift @@ -9,13 +9,16 @@ // #if canImport(Foundation) -@_spi(Experimental) public import Testing +public import Testing public import Foundation // As with Encodable, implement the protocol requirements for // NSSecureCoding-conformant classes by default. The implementation uses // NSKeyedArchiver for encoding. -@_spi(Experimental) + +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } extension Attachable where Self: NSSecureCoding { /// Encode this object using [`NSKeyedArchiver`](https://developer.apple.com/documentation/foundation/nskeyedarchiver) /// into a buffer, then call a function and pass that buffer to it. @@ -46,6 +49,10 @@ extension Attachable where Self: NSSecureCoding { /// _and_ [`NSSecureCoding`](https://developer.apple.com/documentation/foundation/nssecurecoding), /// the default implementation of this function uses the value's conformance /// to `Encodable`. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { let format = try EncodingFormat(for: attachment) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index dbf7e2688..83c3909be 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) -@_spi(Experimental) public import Testing +public import Testing public import Foundation #if !SWT_NO_PROCESS_SPAWNING && os(Windows) @@ -32,8 +32,7 @@ extension URL { } } -@_spi(Experimental) -extension Attachment where AttachableValue == _AttachableURLContainer { +extension Attachment where AttachableValue == _AttachableURLWrapper { #if SWT_TARGET_OS_APPLE /// An operation queue to use for asynchronously reading data from disk. private static let _operationQueue = OperationQueue() @@ -51,6 +50,10 @@ extension Attachment where AttachableValue == _AttachableURLContainer { /// attachment. /// /// - Throws: Any error that occurs attempting to read from `url`. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public init( contentsOf url: URL, named preferredName: String? = nil, @@ -91,8 +94,8 @@ extension Attachment where AttachableValue == _AttachableURLContainer { } #endif - let urlContainer = _AttachableURLContainer(url: url, data: data, isCompressedDirectory: isDirectory) - self.init(urlContainer, named: preferredName, sourceLocation: sourceLocation) + let urlWrapper = _AttachableURLWrapper(url: url, data: data, isCompressedDirectory: isDirectory) + self.init(urlWrapper, named: preferredName, sourceLocation: sourceLocation) } } diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift index 38233cd3c..ce7b719a9 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift @@ -9,11 +9,16 @@ // #if canImport(Foundation) -@_spi(Experimental) public import Testing +public import Testing public import Foundation -@_spi(Experimental) +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } extension Data: Attachable { + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift b/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift index bbbe934ab..49499a8c2 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/EncodingFormat.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) -@_spi(Experimental) import Testing +import Testing import Foundation /// An enumeration describing the encoding formats we support for `Encodable` diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift b/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLWrapper.swift similarity index 90% rename from Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift rename to Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLWrapper.swift index c7a223a51..d6be53c80 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLContainer.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/_AttachableURLWrapper.swift @@ -9,7 +9,7 @@ // #if canImport(Foundation) -@_spi(Experimental) public import Testing +public import Testing public import Foundation /// A wrapper type representing file system objects and URLs that can be @@ -17,8 +17,7 @@ public import Foundation /// /// You do not need to use this type directly. Instead, initialize an instance /// of ``Attachment`` using a file URL. -@_spi(Experimental) -public struct _AttachableURLContainer: Sendable { +public struct _AttachableURLWrapper: Sendable { /// The underlying URL. var url: URL @@ -31,8 +30,8 @@ public struct _AttachableURLContainer: Sendable { // MARK: - -extension _AttachableURLContainer: AttachableContainer { - public var attachableValue: URL { +extension _AttachableURLWrapper: AttachableWrapper { + public var wrappedValue: URL { url } diff --git a/Sources/Overlays/_Testing_Foundation/CMakeLists.txt b/Sources/Overlays/_Testing_Foundation/CMakeLists.txt index 54a340323..9343960ab 100644 --- a/Sources/Overlays/_Testing_Foundation/CMakeLists.txt +++ b/Sources/Overlays/_Testing_Foundation/CMakeLists.txt @@ -7,7 +7,7 @@ # See http://swift.org/CONTRIBUTORS.txt for Swift project authors add_library(_Testing_Foundation - Attachments/_AttachableURLContainer.swift + Attachments/_AttachableURLWrapper.swift Attachments/EncodingFormat.swift Attachments/Attachment+URL.swift Attachments/Attachable+NSSecureCoding.swift diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index b8bafdde1..73e7db2ac 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -27,7 +27,7 @@ extension ABI { case testStarted case testCaseStarted case issueRecorded - case valueAttached = "_valueAttached" + case valueAttached case testCaseEnded case testEnded case testSkipped @@ -50,9 +50,7 @@ extension ABI { /// /// The value of this property is `nil` unless the value of the /// ``kind-swift.property`` property is ``Kind-swift.enum/valueAttached``. - /// - /// - Warning: Attachments are not yet part of the JSON schema. - var _attachment: EncodedAttachment? + var attachment: EncodedAttachment? /// Human-readable messages associated with this event that can be presented /// to the user. @@ -82,7 +80,7 @@ extension ABI { issue = EncodedIssue(encoding: recordedIssue, in: eventContext) case let .valueAttached(attachment): kind = .valueAttached - _attachment = EncodedAttachment(encoding: attachment, in: eventContext) + self.attachment = EncodedAttachment(encoding: attachment, in: eventContext) case .testCaseEnded: if eventContext.test?.isParameterized == false { return nil diff --git a/Sources/Testing/Attachments/Attachable.swift b/Sources/Testing/Attachments/Attachable.swift index 09a7e0b78..be466940b 100644 --- a/Sources/Testing/Attachments/Attachable.swift +++ b/Sources/Testing/Attachments/Attachable.swift @@ -24,9 +24,12 @@ /// A type should conform to this protocol if it can be represented as a /// sequence of bytes that would be diagnostically useful if a test fails. If a /// type cannot conform directly to this protocol (such as a non-final class or -/// a type declared in a third-party module), you can create a container type -/// that conforms to ``AttachableContainer`` to act as a proxy. -@_spi(Experimental) +/// a type declared in a third-party module), you can create a wrapper type that +/// conforms to ``AttachableWrapper`` to act as a proxy. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } public protocol Attachable: ~Copyable { /// An estimate of the number of bytes of memory needed to store this value as /// an attachment. @@ -42,6 +45,10 @@ public protocol Attachable: ~Copyable { /// /// - Complexity: O(1) unless `Self` conforms to `Collection`, in which case /// up to O(_n_) where _n_ is the length of the collection. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } var estimatedAttachmentByteCount: Int? { get } /// Call a function and pass a buffer representing this instance to it. @@ -64,6 +71,10 @@ public protocol Attachable: ~Copyable { /// the buffer to contain an image in PNG format, JPEG format, etc., but it /// would not be idiomatic for the buffer to contain a textual description of /// the image. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } borrowing func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R /// Generate a preferred name for the given attachment. @@ -80,6 +91,10 @@ public protocol Attachable: ~Copyable { /// when adding `attachment` to a test report or persisting it to storage. The /// default implementation of this function returns `suggestedName` without /// any changes. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String } @@ -119,28 +134,24 @@ extension Attachable where Self: StringProtocol { // Implement the protocol requirements for byte arrays and buffers so that // developers can attach raw data when needed. -@_spi(Experimental) extension Array: Attachable { public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } } -@_spi(Experimental) extension ContiguousArray: Attachable { public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } } -@_spi(Experimental) extension ArraySlice: Attachable { public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) } } -@_spi(Experimental) extension String: Attachable { public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { var selfCopy = self @@ -150,7 +161,6 @@ extension String: Attachable { } } -@_spi(Experimental) extension Substring: Attachable { public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { var selfCopy = self diff --git a/Sources/Testing/Attachments/AttachableContainer.swift b/Sources/Testing/Attachments/AttachableWrapper.swift similarity index 70% rename from Sources/Testing/Attachments/AttachableContainer.swift rename to Sources/Testing/Attachments/AttachableWrapper.swift index e4d716e9c..81df52d4d 100644 --- a/Sources/Testing/Attachments/AttachableContainer.swift +++ b/Sources/Testing/Attachments/AttachableWrapper.swift @@ -21,11 +21,22 @@ /// A type can conform to this protocol if it represents another type that /// cannot directly conform to ``Attachable``, such as a non-final class or a /// type declared in a third-party module. -@_spi(Experimental) -public protocol AttachableContainer: Attachable, ~Copyable { - /// The type of the attachable value represented by this type. - associatedtype AttachableValue +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } +public protocol AttachableWrapper: Attachable, ~Copyable { + /// The type of the underlying value represented by this type. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } + associatedtype Wrapped - /// The attachable value represented by this instance. - var attachableValue: AttachableValue { get } + /// The underlying value represented by this instance. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } + var wrappedValue: Wrapped { get } } diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index d7c1cddb7..c11ca7a50 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -18,7 +18,10 @@ private import _TestingInternals /// of some type that conforms to ``Attachable``. Initialize an instance of /// ``Attachment`` with that value and, optionally, a preferred filename to use /// when writing to disk. -@_spi(Experimental) +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } public struct Attachment: ~Copyable where AttachableValue: Attachable & ~Copyable { /// Storage for ``attachableValue-7dyjv``. fileprivate var _attachableValue: AttachableValue @@ -51,6 +54,10 @@ public struct Attachment: ~Copyable where AttachableValue: Atta /// testing library may substitute a different filename as needed. If the /// value of this property has not been explicitly set, the testing library /// will attempt to generate its own value. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public var preferredName: String { let suggestedName = if let _preferredName, !_preferredName.isEmpty { _preferredName @@ -90,6 +97,10 @@ extension Attachment where AttachableValue: ~Copyable { /// - sourceLocation: The source location of the call to this initializer. /// This value is used when recording issues associated with the /// attachment. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public init(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { self._attachableValue = attachableValue self._preferredName = preferredName @@ -97,7 +108,7 @@ extension Attachment where AttachableValue: ~Copyable { } } -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(ForToolsIntegrationOnly) extension Attachment where AttachableValue == AnyAttachable { /// Create a type-erased attachment from an instance of ``Attachment``. /// @@ -105,7 +116,7 @@ extension Attachment where AttachableValue == AnyAttachable { /// - attachment: The attachment to type-erase. fileprivate init(_ attachment: Attachment) { self.init( - _attachableValue: AnyAttachable(attachableValue: attachment.attachableValue), + _attachableValue: AnyAttachable(wrappedValue: attachment.attachableValue), fileSystemPath: attachment.fileSystemPath, _preferredName: attachment._preferredName, sourceLocation: attachment.sourceLocation @@ -114,7 +125,7 @@ extension Attachment where AttachableValue == AnyAttachable { } #endif -/// A type-erased container type that represents any attachable value. +/// A type-erased wrapper type that represents any attachable value. /// /// This type is not generally visible to developers. It is used when posting /// events of kind ``Event/Kind/valueAttached(_:)``. Test tools authors who use @@ -125,54 +136,55 @@ extension Attachment where AttachableValue == AnyAttachable { // Swift's type system requires that this type be at least as visible as // `Event.Kind.valueAttached(_:)`, otherwise it would be declared private. // } -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) -public struct AnyAttachable: AttachableContainer, Copyable, Sendable { +@_spi(ForToolsIntegrationOnly) +public struct AnyAttachable: AttachableWrapper, Copyable, Sendable { #if !SWT_NO_LAZY_ATTACHMENTS - public typealias AttachableValue = any Attachable & Sendable /* & Copyable rdar://137614425 */ + public typealias Wrapped = any Attachable & Sendable /* & Copyable rdar://137614425 */ #else - public typealias AttachableValue = [UInt8] + public typealias Wrapped = [UInt8] #endif - public var attachableValue: AttachableValue + public var wrappedValue: Wrapped - init(attachableValue: AttachableValue) { - self.attachableValue = attachableValue + init(wrappedValue: Wrapped) { + self.wrappedValue = wrappedValue } public var estimatedAttachmentByteCount: Int? { - attachableValue.estimatedAttachmentByteCount + wrappedValue.estimatedAttachmentByteCount } public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { - func open(_ attachableValue: T, for attachment: borrowing Attachment) throws -> R where T: Attachable & Sendable & Copyable { + func open(_ wrappedValue: T, for attachment: borrowing Attachment) throws -> R where T: Attachable & Sendable & Copyable { let temporaryAttachment = Attachment( - _attachableValue: attachableValue, + _attachableValue: wrappedValue, fileSystemPath: attachment.fileSystemPath, _preferredName: attachment._preferredName, sourceLocation: attachment.sourceLocation ) return try temporaryAttachment.withUnsafeBytes(body) } - return try open(attachableValue, for: attachment) + return try open(wrappedValue, for: attachment) } public borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String { - func open(_ attachableValue: T, for attachment: borrowing Attachment) -> String where T: Attachable & Sendable & Copyable { + func open(_ wrappedValue: T, for attachment: borrowing Attachment) -> String where T: Attachable & Sendable & Copyable { let temporaryAttachment = Attachment( - _attachableValue: attachableValue, + _attachableValue: wrappedValue, fileSystemPath: attachment.fileSystemPath, _preferredName: attachment._preferredName, sourceLocation: attachment.sourceLocation ) return temporaryAttachment.preferredName } - return open(attachableValue, for: attachment) + return open(wrappedValue, for: attachment) } } // MARK: - Describing an attachment extension Attachment where AttachableValue: ~Copyable { + @_documentation(visibility: private) public var description: String { let typeInfo = TypeInfo(describing: AttachableValue.self) return #""\#(preferredName)": instance of '\#(typeInfo.unqualifiedName)'"# @@ -180,6 +192,9 @@ extension Attachment where AttachableValue: ~Copyable { } extension Attachment: CustomStringConvertible { + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public var description: String { #""\#(preferredName)": \#(String(describingForTest: attachableValue))"# } @@ -187,9 +202,12 @@ extension Attachment: CustomStringConvertible { // MARK: - Getting an attachable value from an attachment -@_spi(Experimental) extension Attachment where AttachableValue: ~Copyable { /// The value of this attachment. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } @_disfavoredOverload public var attachableValue: AttachableValue { _read { yield _attachableValue @@ -197,21 +215,24 @@ extension Attachment where AttachableValue: ~Copyable { } } -@_spi(Experimental) -extension Attachment where AttachableValue: AttachableContainer & ~Copyable { +extension Attachment where AttachableValue: AttachableWrapper & ~Copyable { /// The value of this attachment. /// - /// When the attachable value's type conforms to ``AttachableContainer``, the - /// value of this property equals the container's underlying attachable value. + /// When the attachable value's type conforms to ``AttachableWrapper``, the + /// value of this property equals the wrapper's underlying attachable value. /// To access the attachable value as an instance of `T` (where `T` conforms - /// to ``AttachableContainer``), specify the type explicitly: + /// to ``AttachableWrapper``), specify the type explicitly: /// /// ```swift /// let attachableValue = attachment.attachableValue as T /// ``` - public var attachableValue: AttachableValue.AttachableValue { + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } + public var attachableValue: AttachableValue.Wrapped { _read { - yield attachableValue.attachableValue + yield attachableValue.wrappedValue } } } @@ -235,6 +256,10 @@ extension Attachment where AttachableValue: Sendable & Copyable { /// disk. /// /// An attachment can only be attached once. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } @_documentation(visibility: private) public static func record(_ attachment: consuming Self, sourceLocation: SourceLocation = #_sourceLocation) { var attachmentCopy = Attachment(attachment) @@ -263,6 +288,10 @@ extension Attachment where AttachableValue: Sendable & Copyable { /// attaches it to the current test. /// /// An attachment can only be attached once. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } @_documentation(visibility: private) public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { record(Self(attachableValue, named: preferredName, sourceLocation: sourceLocation), sourceLocation: sourceLocation) @@ -286,12 +315,16 @@ extension Attachment where AttachableValue: ~Copyable { /// disk. /// /// An attachment can only be attached once. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func record(_ attachment: consuming Self, sourceLocation: SourceLocation = #_sourceLocation) { do { let attachmentCopy = try attachment.withUnsafeBytes { buffer in - let attachableContainer = AnyAttachable(attachableValue: Array(buffer)) + let attachableWrapper = AnyAttachable(wrappedValue: Array(buffer)) return Attachment( - _attachableValue: attachableContainer, + _attachableValue: attachableWrapper, fileSystemPath: attachment.fileSystemPath, _preferredName: attachment.preferredName, // invokes preferredName(for:basedOn:) sourceLocation: sourceLocation @@ -325,6 +358,10 @@ extension Attachment where AttachableValue: ~Copyable { /// attaches it to the current test. /// /// An attachment can only be attached once. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { record(Self(attachableValue, named: preferredName, sourceLocation: sourceLocation), sourceLocation: sourceLocation) } @@ -349,6 +386,10 @@ extension Attachment where AttachableValue: ~Copyable { /// test report or to a file on disk. This function calls the /// ``Attachable/withUnsafeBytes(for:_:)`` function on this attachment's /// ``attachableValue-2tnj5`` property. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } @inlinable public borrowing func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try attachableValue.withUnsafeBytes(for: self, body) } @@ -382,7 +423,7 @@ extension Attachment where AttachableValue: ~Copyable { /// This function is provided as a convenience to allow tools authors to write /// attachments to persistent storage the same way that Swift Package Manager /// does. You are not required to use this function. - @_spi(Experimental) @_spi(ForToolsIntegrationOnly) + @_spi(ForToolsIntegrationOnly) public borrowing func write(toFileInDirectoryAtPath directoryPath: String) throws -> String { try write( toFileInDirectoryAtPath: directoryPath, diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 7e07636d5..81a0c550a 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -22,7 +22,7 @@ add_library(Testing ABI/Encoded/ABI.EncodedMessage.swift ABI/Encoded/ABI.EncodedTest.swift Attachments/Attachable.swift - Attachments/AttachableContainer.swift + Attachments/AttachableWrapper.swift Attachments/Attachment.swift Events/Clock.swift Events/Event.swift diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index b81f1c2c7..0be14ae88 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -102,7 +102,6 @@ public struct Event: Sendable { /// /// - Parameters: /// - attachment: The attachment that was created. - @_spi(Experimental) indirect case valueAttached(_ attachment: Attachment) /// A test ended. diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index 846fb2d4d..3354691fb 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -44,7 +44,6 @@ extension Event { case details /// The symbol to use when describing an instance of ``Attachment``. - @_spi(Experimental) case attachment } } diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 5d7449b7b..d210ecec7 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -65,7 +65,6 @@ public struct Issue: Sendable { /// /// - Parameters: /// - error: The error which was associated with this issue. - @_spi(Experimental) case valueAttachmentFailed(_ error: any Error) /// An issue occurred due to misuse of the testing library. diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 30a2ce303..b8c48aa79 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -237,7 +237,6 @@ public struct Configuration: Sendable { /// The value of this property must refer to a directory on the local file /// system that already exists and which the current user can write to. If it /// is a relative path, it is resolved to an absolute path automatically. - @_spi(Experimental) public var attachmentsPath: String? { get { _attachmentsPath diff --git a/Sources/Testing/Testing.docc/Attachments.md b/Sources/Testing/Testing.docc/Attachments.md new file mode 100644 index 000000000..0da40c201 --- /dev/null +++ b/Sources/Testing/Testing.docc/Attachments.md @@ -0,0 +1,32 @@ +# Attachments + + + +Attach values to tests to help diagnose issues and gather feedback. + +## Overview + +Attach values such as strings and files to tests. Implement the ``Attachable`` +protocol to create your own attachable types. + +## Topics + +### Attaching values to tests + +- ``Attachment`` +- ``Attachable`` +- ``AttachableWrapper`` + + diff --git a/Sources/Testing/Testing.docc/Documentation.md b/Sources/Testing/Testing.docc/Documentation.md index 901c0e3a6..cc4001889 100644 --- a/Sources/Testing/Testing.docc/Documentation.md +++ b/Sources/Testing/Testing.docc/Documentation.md @@ -69,3 +69,7 @@ their problems. ### Test customization - + +### Data collection + +- diff --git a/Sources/Testing/Testing.docc/MigratingFromXCTest.md b/Sources/Testing/Testing.docc/MigratingFromXCTest.md index e3a9d961f..60744ba7a 100644 --- a/Sources/Testing/Testing.docc/MigratingFromXCTest.md +++ b/Sources/Testing/Testing.docc/MigratingFromXCTest.md @@ -742,6 +742,66 @@ suite serially: For more information, see . +### Attach values + +In XCTest, you can create an instance of [`XCTAttachment`](https://developer.apple.com/documentation/xctest/xctattachment) +representing arbitrary data, files, property lists, encodable objects, images, +and other types of information that would be useful to have available if a test +fails. Swift Testing has an ``Attachment`` type that serves much the same +purpose. + +To attach a value from a test to the output of a test run, that value must +conform to the ``Attachable`` protocol. The testing library provides default +conformances for various standard library and Foundation types. + +If you want to attach a value of another type, and that type already conforms to +[`Encodable`](https://developer.apple.com/documentation/swift/encodable) or to +[`NSSecureCoding`](https://developer.apple.com/documentation/foundation/nssecurecoding), +the testing library automatically provides a default implementation when you +import Foundation: + +@Row { + @Column { + ```swift + // Before + import Foundation + + class Tortilla: NSSecureCoding { /* ... */ } + + func testTortillaIntegrity() async { + let tortilla = Tortilla(diameter: .large) + ... + let attachment = XCTAttachment( + archivableObject: tortilla + ) + self.add(attachment) + } + ``` + } + @Column { + ```swift + // After + import Foundation + + struct Tortilla: Codable, Attachable { /* ... */ } + + @Test func tortillaIntegrity() async { + let tortilla = Tortilla(diameter: .large) + ... + Attachment.record(tortilla) + } + ``` + } +} + +If you have a type that does not (or cannot) conform to `Encodable` or +`NSSecureCoding`, or if you want fine-grained control over how it is serialized +when attaching it to a test, you can provide your own implementation of +``Attachable/withUnsafeBytes(for:_:)``. + + + ## See Also - diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 126633776..0281b4091 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -8,11 +8,11 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +@testable @_spi(ForToolsIntegrationOnly) import Testing private import _TestingInternals #if canImport(Foundation) import Foundation -@_spi(Experimental) import _Testing_Foundation +import _Testing_Foundation #endif #if canImport(CoreGraphics) import CoreGraphics From d8d20a5633577cab57f98daa0820ea6022b004ae Mon Sep 17 00:00:00 2001 From: Suzy Ratcliff Date: Thu, 10 Apr 2025 16:25:08 -0700 Subject: [PATCH 165/234] Introduce a severity level when recording issues (#1070) Introduce a severity level when recording issues ### Motivation: In order to create issues that don't fail a test this introduces a parameter to specify the severity of the issue. This is in support of work added here for an issue severity: https://github.com/swiftlang/swift-testing/pull/931 This is experimental. Example usage: `Issue.record("My comment", severity: .warning)` ### Modifications: I modified the `Issue.record` method signature to take in a severity level so that users can create issues that are not failing. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. - [x] Add tests --- Sources/Testing/Issues/Issue+Recording.swift | 51 +++++++++++++++++++- Tests/TestingTests/IssueTests.swift | 42 ++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index aaf721c6a..bd3e9a3bb 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -73,9 +73,31 @@ extension Issue { @discardableResult public static func record( _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation + ) -> Self { + record(comment, severity: .error, sourceLocation: sourceLocation) + } + + /// Record an issue when a running test fails unexpectedly. + /// + /// - Parameters: + /// - comment: A comment describing the expectation. + /// - severity: The severity of the issue. + /// - sourceLocation: The source location to which the issue should be + /// attributed. + /// + /// - Returns: The issue that was recorded. + /// + /// Use this function if, while running a test, an issue occurs that cannot be + /// represented as an expectation (using the ``expect(_:_:sourceLocation:)`` + /// or ``require(_:_:sourceLocation:)-5l63q`` macros.) + @_spi(Experimental) + @discardableResult public static func record( + _ comment: Comment? = nil, + severity: Severity, + sourceLocation: SourceLocation = #_sourceLocation ) -> Self { let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) - let issue = Issue(kind: .unconditional, comments: Array(comment), sourceContext: sourceContext) + let issue = Issue(kind: .unconditional, severity: severity, comments: Array(comment), sourceContext: sourceContext) return issue.record() } } @@ -101,10 +123,35 @@ extension Issue { _ error: any Error, _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation + ) -> Self { + record(error, comment, severity: .error, sourceLocation: sourceLocation) + } + + /// Record a new issue when a running test unexpectedly catches an error. + /// + /// - Parameters: + /// - error: The error that caused the issue. + /// - comment: A comment describing the expectation. + /// - severity: The severity of the issue. + /// - sourceLocation: The source location to which the issue should be + /// attributed. + /// + /// - Returns: The issue that was recorded. + /// + /// This function can be used if an unexpected error is caught while running a + /// test and it should be treated as a test failure. If an error is thrown + /// from a test function, it is automatically recorded as an issue and this + /// function does not need to be used. + @_spi(Experimental) + @discardableResult public static func record( + _ error: any Error, + _ comment: Comment? = nil, + severity: Severity, + sourceLocation: SourceLocation = #_sourceLocation ) -> Self { let backtrace = Backtrace(forFirstThrowOf: error) ?? Backtrace.current() let sourceContext = SourceContext(backtrace: backtrace, sourceLocation: sourceLocation) - let issue = Issue(kind: .errorCaught(error), comments: Array(comment), sourceContext: sourceContext) + let issue = Issue(kind: .errorCaught(error), severity: severity, comments: Array(comment), sourceContext: sourceContext) return issue.record() } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index d22bf9fba..cb7ce28f3 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -1010,6 +1010,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.severity, .error) guard case .unconditional = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1021,6 +1022,25 @@ final class IssueTests: XCTestCase { Issue.record("Custom message") }.run(configuration: configuration) } + + func testWarning() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.severity, .warning) + guard case .unconditional = issue.kind else { + XCTFail("Unexpected issue kind \(issue.kind)") + return + } + } + + await Test { + Issue.record("Custom message", severity: .warning) + }.run(configuration: configuration) + } #if !SWT_NO_UNSTRUCTURED_TASKS func testFailWithoutCurrentTest() async throws { @@ -1048,6 +1068,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.severity, .error) guard case let .errorCaught(error) = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1060,6 +1081,27 @@ final class IssueTests: XCTestCase { Issue.record(MyError(), "Custom message") }.run(configuration: configuration) } + + func testWarningBecauseOfError() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.severity, .warning) + guard case let .errorCaught(error) = issue.kind else { + XCTFail("Unexpected issue kind \(issue.kind)") + return + } + XCTAssertTrue(error is MyError) + } + + await Test { + Issue.record(MyError(), severity: .warning) + Issue.record(MyError(), "Custom message", severity: .warning) + }.run(configuration: configuration) + } func testErrorPropertyValidForThrownErrors() async throws { var configuration = Configuration() From 197d6b3819ef8a47f188f124959bb45e64aaee0f Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 12 Apr 2025 09:55:44 -0400 Subject: [PATCH 166/234] Add support for the `--attachments-path` CLI argument. (#1074) This PR adds support for the `--attachments-path` CLI argument on `swift test` as approved in [ST-0009](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0009-attachments.md). We will maintain support for the older `--experimental-attachments-path` version through Swift 6.2. Resolves rdar://147753783. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ABI/EntryPoints/EntryPoint.swift | 15 ++++++++------- Sources/Testing/Attachments/Attachment.swift | 12 ++++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index c72542d65..4d510ccba 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -285,8 +285,8 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--repeat-until` argument. public var repeatUntil: String? - /// The value of the `--experimental-attachments-path` argument. - public var experimentalAttachmentsPath: String? + /// The value of the `--attachments-path` argument. + public var attachmentsPath: String? /// Whether or not the experimental warning issue severity feature should be /// enabled. @@ -314,7 +314,7 @@ extension __CommandLineArguments_v0: Codable { case skip case repetitions case repeatUntil - case experimentalAttachmentsPath + case attachmentsPath } } @@ -396,8 +396,9 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum } // Attachment output - if let attachmentsPathIndex = args.firstIndex(of: "--experimental-attachments-path"), !isLastArgument(at: attachmentsPathIndex) { - result.experimentalAttachmentsPath = args[args.index(after: attachmentsPathIndex)] + if let attachmentsPathIndex = args.firstIndex(of: "--attachments-path") ?? args.firstIndex(of: "--experimental-attachments-path"), + !isLastArgument(at: attachmentsPathIndex) { + result.attachmentsPath = args[args.index(after: attachmentsPathIndex)] } #endif @@ -509,9 +510,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr } // Attachment output. - if let attachmentsPath = args.experimentalAttachmentsPath { + if let attachmentsPath = args.attachmentsPath { guard fileExists(atPath: attachmentsPath) else { - throw _EntryPointError.invalidArgument("--experimental-attachments-path", value: attachmentsPath) + throw _EntryPointError.invalidArgument("---attachments-path", value: attachmentsPath) } configuration.attachmentsPath = attachmentsPath } diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index c11ca7a50..7468834bf 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -30,9 +30,9 @@ public struct Attachment: ~Copyable where AttachableValue: Atta /// /// If a developer sets the ``Configuration/attachmentsPath`` property of the /// current configuration before running tests, or if a developer passes - /// `--experimental-attachments-path` on the command line, then attachments - /// will be automatically written to disk when they are attached and the value - /// of this property will describe the path where they were written. + /// `--attachments-path` on the command line, then attachments will be + /// automatically written to disk when they are attached and the value of this + /// property will describe the path where they were written. /// /// If no destination path is set, or if an error occurred while writing this /// attachment to disk, the value of this property is `nil`. @@ -412,9 +412,9 @@ extension Attachment where AttachableValue: ~Copyable { /// The attachment is written to a file _within_ `directoryPath`, whose name /// is derived from the value of the ``Attachment/preferredName`` property. /// - /// If you pass `--experimental-attachments-path` to `swift test`, the testing - /// library automatically uses this function to persist attachments to the - /// directory you specify. + /// If you pass `--attachments-path` to `swift test`, the testing library + /// automatically uses this function to persist attachments to the directory + /// you specify. /// /// This function does not get or set the value of the attachment's /// ``fileSystemPath`` property. The caller is responsible for setting the From 55f82edce1c910e45143c456e8c5762565dba76d Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Sat, 12 Apr 2025 09:55:57 -0400 Subject: [PATCH 167/234] Add `CustomStringConvertible` conformance to `ExitTest.Condition` and `StatusAtExit`. (#1073) This PR adds `CustomStringConvertible` conformance to these data/value types so that they are presented reasonably in expectation failure messages or when printed. Strictly speaking, API changes should go through a review, but there's not really anything to review here, so with the consent of the Testing Workgroup we can amend the exit tests proposal to retroactively include these conformances. See also: https://github.com/swiftlang/swift-evolution/pull/2789 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../ExitTests/ExitTest.Condition.swift | 42 +++++++++++++++---- Sources/Testing/ExitTests/StatusAtExit.swift | 16 +++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift index 10f2a6ff0..d2c637d79 100644 --- a/Sources/Testing/ExitTests/ExitTest.Condition.swift +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -27,6 +27,9 @@ extension ExitTest { /// The exit test must exit with a particular exit status. case statusAtExit(StatusAtExit) + /// The exit test must exit successfully. + case success + /// The exit test must exit with any failure. case failure } @@ -46,15 +49,7 @@ extension ExitTest.Condition { /// A condition that matches when a process terminates successfully with exit /// code `EXIT_SUCCESS`. public static var success: Self { - // Strictly speaking, the C standard treats 0 as a successful exit code and - // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern - // operating system defines EXIT_SUCCESS to any value other than 0, so the - // distinction is academic. -#if !SWT_NO_EXIT_TESTS - .exitCode(EXIT_SUCCESS) -#else - fatalError("Unsupported") -#endif + Self(_kind: .success) } /// A condition that matches when a process terminates abnormally with any @@ -122,6 +117,29 @@ extension ExitTest.Condition { } } +// MARK: - CustomStringConvertible + +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest.Condition: CustomStringConvertible { + public var description: String { +#if !SWT_NO_EXIT_TESTS + switch _kind { + case .failure: + ".failure" + case .success: + ".success" + case let .statusAtExit(statusAtExit): + String(describing: statusAtExit) + } +#else + fatalError("Unsupported") +#endif + } +} + // MARK: - Comparison #if SWT_NO_EXIT_TESTS @@ -139,7 +157,13 @@ extension ExitTest.Condition { /// Two exit test conditions can be compared; if either instance is equal to /// ``failure``, it will compare equal to any instance except ``success``. func isApproximatelyEqual(to statusAtExit: StatusAtExit) -> Bool { + // Strictly speaking, the C standard treats 0 as a successful exit code and + // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern + // operating system defines EXIT_SUCCESS to any value other than 0, so the + // distinction is academic. return switch (self._kind, statusAtExit) { + case let (.success, .exitCode(exitCode)): + exitCode == EXIT_SUCCESS case let (.failure, .exitCode(exitCode)): exitCode != EXIT_SUCCESS case (.failure, .signal): diff --git a/Sources/Testing/ExitTests/StatusAtExit.swift b/Sources/Testing/ExitTests/StatusAtExit.swift index 26514ffa5..ea5e287c7 100644 --- a/Sources/Testing/ExitTests/StatusAtExit.swift +++ b/Sources/Testing/ExitTests/StatusAtExit.swift @@ -71,3 +71,19 @@ public enum StatusAtExit: Sendable { @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif extension StatusAtExit: Equatable {} + +// MARK: - CustomStringConvertible +@_spi(Experimental) +#if SWT_NO_PROCESS_SPAWNING +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension StatusAtExit: CustomStringConvertible { + public var description: String { + switch self { + case let .exitCode(exitCode): + ".exitCode(\(exitCode))" + case let .signal(signal): + ".signal(\(signal))" + } + } +} From 72bed50df7d9d3e52e901cb4d96a07358a915128 Mon Sep 17 00:00:00 2001 From: Suzy Ratcliff Date: Mon, 14 Apr 2025 15:53:21 -0700 Subject: [PATCH 168/234] Add isFailure to Issue (#1078) Add `isFailure` var to `Issue` ### Motivation: This adds a variable to `Issue` in order to inspect if an issue isFailing. This will be nice when inspecting an issue after it is recorded. ### Modifications: - Add `isFailure` variable - Update existing code to use `isFailure` ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ABI/EntryPoints/EntryPoint.swift | 2 +- Sources/Testing/Issues/Issue.swift | 15 +++++++++++++++ Tests/TestingTests/IssueTests.swift | 2 ++ Tests/TestingTests/KnownIssueTests.swift | 3 ++- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 4d510ccba..7a2e63003 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -42,7 +42,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // Set up the event handler. configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error { + if case let .issueRecorded(issue) = event.kind, issue.isFailure { exitCode.withLock { exitCode in exitCode = EXIT_FAILURE } diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index d210ecec7..53364c151 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -103,6 +103,21 @@ public struct Issue: Sendable { /// The severity of this issue. @_spi(Experimental) public var severity: Severity + + /// Whether or not this issue should cause the test it's associated with to be + /// considered a failure. + /// + /// The value of this property is `true` for issues which have a severity level of + /// ``Issue/Severity/error`` or greater and are not known issues via + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)``. + /// Otherwise, the value of this property is `false.` + /// + /// Use this property to determine if an issue should be considered a failure, instead of + /// directly comparing the value of the ``severity`` property. + @_spi(Experimental) + public var isFailure: Bool { + return !self.isKnown && self.severity >= .error + } /// Any comments provided by the developer and associated with this issue. /// diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index cb7ce28f3..6ea1a5827 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -1011,6 +1011,7 @@ final class IssueTests: XCTestCase { } XCTAssertFalse(issue.isKnown) XCTAssertEqual(issue.severity, .error) + XCTAssertTrue(issue.isFailure) guard case .unconditional = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1031,6 +1032,7 @@ final class IssueTests: XCTestCase { } XCTAssertFalse(issue.isKnown) XCTAssertEqual(issue.severity, .warning) + XCTAssertFalse(issue.isFailure) guard case .unconditional = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return diff --git a/Tests/TestingTests/KnownIssueTests.swift b/Tests/TestingTests/KnownIssueTests.swift index 448b8e35d..733fbbf01 100644 --- a/Tests/TestingTests/KnownIssueTests.swift +++ b/Tests/TestingTests/KnownIssueTests.swift @@ -10,7 +10,7 @@ #if canImport(XCTest) import XCTest -@testable @_spi(ForToolsIntegrationOnly) import Testing +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing final class KnownIssueTests: XCTestCase { func testIssueIsKnownPropertyIsSetCorrectly() async { @@ -26,6 +26,7 @@ final class KnownIssueTests: XCTestCase { issueRecorded.fulfill() XCTAssertTrue(issue.isKnown) + XCTAssertFalse(issue.isFailure) } await Test { From 7097c2b1e51aac70eb06b37bda8f5c31467ce814 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 15 Apr 2025 12:19:19 -0400 Subject: [PATCH 169/234] Add `unsafe` keyword handling to macro expansions. (#1069) This PR changes the `@Test` and `#expect()` macros so they handle `unsafe` expressions the same way `try` and `await` expressions are handled. The propagation rules for `unsafe` aren't the same as for `try` and `await` (i.e. it doesn't colour the calling function), but the general way we handle the keyword is the same. I haven't attempted to avoid inserting `unsafe` if a function is not marked `@unsafe` as it complicates the necessary logic but has no effects at runtime. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 2 +- Sources/Testing/Test+Macro.swift | 9 + Sources/TestingMacros/CMakeLists.txt | 3 +- Sources/TestingMacros/ConditionMacro.swift | 3 +- .../Support/ConditionArgumentParsing.swift | 20 +-- .../Support/EffectfulExpressionHandling.swift | 159 ++++++++++++++++++ .../TestingMacros/TestDeclarationMacro.swift | 6 +- 7 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 Sources/TestingMacros/Support/EffectfulExpressionHandling.swift diff --git a/Package.swift b/Package.swift index 4194416fb..f515f16a9 100644 --- a/Package.swift +++ b/Package.swift @@ -96,7 +96,7 @@ let package = Package( }(), dependencies: [ - .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "601.0.0-latest"), + .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "602.0.0-latest"), ], targets: [ diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index 86fb42c14..d1ad6623b 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -537,6 +537,15 @@ extension Test { value } +/// A function that abstracts away whether or not the `unsafe` keyword is needed +/// on an expression. +/// +/// - Warning: This function is used to implement the `@Test` macro. Do not use +/// it directly. +@unsafe @inlinable public func __requiringUnsafe(_ value: consuming T) throws -> T where T: ~Copyable { + value +} + /// The current default isolation context. /// /// - Warning: This property is used to implement the `@Test` macro. Do not call diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index 72184f94b..e535f13cf 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -31,7 +31,7 @@ if(SwiftTesting_BuildMacrosAsExecutables) set(FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/_d) FetchContent_Declare(SwiftSyntax GIT_REPOSITORY https://github.com/swiftlang/swift-syntax - GIT_TAG 1cd35348b089ff8966588742c69727205d99f8ed) # 601.0.0-prerelease-2024-11-18 + GIT_TAG 340f8400262d494c7c659cd838223990195d7fed) # 602.0.0-prerelease-2025-04-10 FetchContent_MakeAvailable(SwiftSyntax) endif() @@ -101,6 +101,7 @@ target_sources(TestingMacros PRIVATE Support/ConditionArgumentParsing.swift Support/DiagnosticMessage.swift Support/DiagnosticMessage+Diagnosing.swift + Support/EffectfulExpressionHandling.swift Support/SHA256.swift Support/SourceCodeCapturing.swift Support/SourceLocationGeneration.swift diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index f8b87e1fa..8ae26bf82 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -117,7 +117,6 @@ extension ConditionMacro { var checkArguments = [Argument]() do { if let trailingClosureIndex { - // Include all arguments other than the "comment" and "sourceLocation" // arguments here. checkArguments += macroArguments.indices.lazy @@ -458,7 +457,7 @@ extension ExitTestConditionMacro { decls.append( """ @Sendable func \(bodyThunkName)() async throws -> Swift.Void { - return try await Testing.__requiringTry(Testing.__requiringAwait(\(bodyArgumentExpr.trimmed)))() + return \(applyEffectfulKeywords([.try, .await, .unsafe], to: bodyArgumentExpr))() } """ ) diff --git a/Sources/TestingMacros/Support/ConditionArgumentParsing.swift b/Sources/TestingMacros/Support/ConditionArgumentParsing.swift index edf9a23c3..e0ccda9a7 100644 --- a/Sources/TestingMacros/Support/ConditionArgumentParsing.swift +++ b/Sources/TestingMacros/Support/ConditionArgumentParsing.swift @@ -472,17 +472,6 @@ private func _parseCondition(from expr: ExprSyntax, for macro: some Freestanding return _parseCondition(from: closureExpr, for: macro, in: context) } - // If the condition involves the `try` or `await` keywords, assume we cannot - // expand it. This check cannot handle expressions like - // `try #expect(a.b(c))` where `b()` is throwing because the `try` keyword is - // outside the macro expansion. SEE: rdar://109470248 - let containsTryOrAwait = expr.tokens(viewMode: .sourceAccurate).lazy - .map(\.tokenKind) - .contains { $0 == .keyword(.try) || $0 == .keyword(.await) } - if containsTryOrAwait { - return Condition(expression: expr) - } - if let infixOperator = expr.as(InfixOperatorExprSyntax.self), let op = infixOperator.operator.as(BinaryOperatorExprSyntax.self) { return _parseCondition(from: expr, leftOperand: infixOperator.leftOperand, operator: op, rightOperand: infixOperator.rightOperand, for: macro, in: context) @@ -527,6 +516,15 @@ private func _parseCondition(from expr: ExprSyntax, for macro: some Freestanding /// /// - Returns: An instance of ``Condition`` describing `expr`. func parseCondition(from expr: ExprSyntax, for macro: some FreestandingMacroExpansionSyntax, in context: some MacroExpansionContext) -> Condition { + // If the condition involves the `unsafe`, `try`, or `await` keywords, assume + // we cannot expand it. This check cannot handle expressions like + // `try #expect(a.b(c))` where `b()` is throwing because the `try` keyword is + // outside the macro expansion. SEE: rdar://109470248 + let effectKeywordsToApply = findEffectKeywords(in: expr, context: context) + guard effectKeywordsToApply.intersection([.unsafe, .try, .await]).isEmpty else { + return Condition(expression: expr) + } + _diagnoseTrivialBooleanValue(from: expr, for: macro, in: context) let result = _parseCondition(from: expr, for: macro, in: context) return result diff --git a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift new file mode 100644 index 000000000..b8f6d125d --- /dev/null +++ b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift @@ -0,0 +1,159 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +import SwiftSyntax +import SwiftSyntaxBuilder +import SwiftSyntaxMacros + +// MARK: - Finding effect keywords + +/// A syntax visitor class that looks for effectful keywords in a given +/// expression. +private final class _EffectFinder: SyntaxAnyVisitor { + /// The effect keywords discovered so far. + var effectKeywords: Set = [] + + override func visitAny(_ node: Syntax) -> SyntaxVisitorContinueKind { + switch node.kind { + case .tryExpr: + effectKeywords.insert(.try) + case .awaitExpr: + effectKeywords.insert(.await) + case .consumeExpr: + effectKeywords.insert(.consume) + case .borrowExpr: + effectKeywords.insert(.borrow) + case .unsafeExpr: + effectKeywords.insert(.unsafe) + case .closureExpr, .functionDecl: + // Do not delve into closures or function declarations. + return .skipChildren + case .variableDecl: + // Delve into variable declarations. + return .visitChildren + default: + // Do not delve into declarations other than variables. + if node.isProtocol((any DeclSyntaxProtocol).self) { + return .skipChildren + } + } + + // Recurse into everything else. + return .visitChildren + } +} + +/// Find effectful keywords in a syntax node. +/// +/// - Parameters: +/// - node: The node to inspect. +/// - context: The macro context in which the expression is being parsed. +/// +/// - Returns: A set of effectful keywords such as `await` that are present in +/// `node`. +/// +/// This function does not descend into function declarations or closure +/// expressions because they represent distinct lexical contexts and their +/// effects are uninteresting in the context of `node` unless they are called. +func findEffectKeywords(in node: some SyntaxProtocol, context: some MacroExpansionContext) -> Set { + // TODO: gather any effects from the lexical context once swift-syntax-#3037 and related PRs land + let effectFinder = _EffectFinder(viewMode: .sourceAccurate) + effectFinder.walk(node) + return effectFinder.effectKeywords +} + +// MARK: - Inserting effect keywords/thunks + +/// Make a function call expression to an effectful thunk function provided by +/// the testing library. +/// +/// - Parameters: +/// - thunkName: The unqualified name of the thunk function to call. This +/// token must be the name of a function in the `Testing` module. +/// - expr: The expression to thunk. +/// +/// - Returns: An expression representing a call to the function named +/// `thunkName`, passing `expr`. +private func _makeCallToEffectfulThunk(_ thunkName: TokenSyntax, passing expr: some ExprSyntaxProtocol) -> ExprSyntax { + ExprSyntax( + FunctionCallExprSyntax( + calledExpression: MemberAccessExprSyntax( + base: DeclReferenceExprSyntax(baseName: .identifier("Testing")), + declName: DeclReferenceExprSyntax(baseName: thunkName) + ), + leftParen: .leftParenToken(), + rightParen: .rightParenToken() + ) { + LabeledExprSyntax(expression: expr.trimmed) + } + ) +} + +/// Apply the given effectful keywords (i.e. `try` and `await`) to an expression +/// using thunk functions provided by the testing library. +/// +/// - Parameters: +/// - effectfulKeywords: The effectful keywords to apply. +/// - expr: The expression to apply the keywords and thunk functions to. +/// +/// - Returns: A copy of `expr` if no changes are needed, or an expression that +/// adds the keywords in `effectfulKeywords` to `expr`. +func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some ExprSyntaxProtocol) -> ExprSyntax { + let originalExpr = expr + var expr = ExprSyntax(expr) + + let needAwait = effectfulKeywords.contains(.await) && !expr.is(AwaitExprSyntax.self) + let needTry = effectfulKeywords.contains(.try) && !expr.is(TryExprSyntax.self) + let needUnsafe = effectfulKeywords.contains(.unsafe) && !expr.is(UnsafeExprSyntax.self) + + // First, add thunk function calls. + if needAwait { + expr = _makeCallToEffectfulThunk(.identifier("__requiringAwait"), passing: expr) + } + if needTry { + expr = _makeCallToEffectfulThunk(.identifier("__requiringTry"), passing: expr) + } + if needUnsafe { + expr = _makeCallToEffectfulThunk(.identifier("__requiringUnsafe"), passing: expr) + } + + // Then add keyword expressions. (We do this separately so we end up writing + // `try await __r(__r(self))` instead of `try __r(await __r(self))` which is + // less accepted by the compiler.) + if needAwait { + expr = ExprSyntax( + AwaitExprSyntax( + awaitKeyword: .keyword(.await).with(\.trailingTrivia, .space), + expression: expr + ) + ) + } + if needTry { + expr = ExprSyntax( + TryExprSyntax( + tryKeyword: .keyword(.try).with(\.trailingTrivia, .space), + expression: expr + ) + ) + } + if needUnsafe { + expr = ExprSyntax( + UnsafeExprSyntax( + unsafeKeyword: .keyword(.unsafe).with(\.trailingTrivia, .space), + expression: expr + ) + ) + } + + expr.leadingTrivia = originalExpr.leadingTrivia + expr.trailingTrivia = originalExpr.trailingTrivia + + return expr +} diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 307b1615d..0b2d43f1e 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -246,17 +246,17 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { // detecting isolation to other global actors. lazy var isMainActorIsolated = !functionDecl.attributes(named: "MainActor", inModuleNamed: "_Concurrency").isEmpty var forwardCall: (ExprSyntax) -> ExprSyntax = { - "try await Testing.__requiringTry(Testing.__requiringAwait(\($0)))" + applyEffectfulKeywords([.try, .await, .unsafe], to: $0) } let forwardInit = forwardCall if functionDecl.noasyncAttribute != nil { if isMainActorIsolated { forwardCall = { - "try await MainActor.run { try Testing.__requiringTry(\($0)) }" + "try await MainActor.run { \(applyEffectfulKeywords([.try, .unsafe], to: $0)) }" } } else { forwardCall = { - "try { try Testing.__requiringTry(\($0)) }()" + "try { \(applyEffectfulKeywords([.try, .unsafe], to: $0)) }()" } } } From 4e4885d614343c7c4619ffc30e9cebd1b02a39ce Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 15 Apr 2025 13:33:02 -0400 Subject: [PATCH 170/234] [Experimental] Capturing values in exit tests (#1040) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR implements an experimental form of state capture in exit tests. If you specify a capture list on the test's body closure and explicitly write the type of each captured value, _and_ each value conforms to `Sendable`, `Codable`, and (implicitly) `Copyable`, we'll encode them and send them "over the wire" to the child process: ```swift let a = Int.random(in: 100 ..< 200) await #expect(exitsWith: .failure) { [a = a as Int] in assert(a > 500) } ``` This PR is incomplete. Among other details: - [x] Need to properly transmit the data, not stuff it in an environment variable - [x] Need to implement diagnostics correctly - [x] Need to figure out if `ExitTest.CapturedValue` and `__Expression.Value` have any synergy. _(They do, but it's beyond the scope of this initial/experimental PR.)_ We are ultimately constrained by the language here as we don't have real type information for the captured values, nor can we infer captures by inspecting the syntax of the exit test body (hence the need for an explicit capture list with types.) If we had something like `decltype()` we could apply during macro expansion, you wouldn't need to write `x = x as T` and could just write `x`. The macro would use `decltype()` to produce a thunk function of the form: ```swift @Sendable func __compiler_generated_name__(aʹ: decltype(a), bʹ: decltype(b), cʹ: decltype(c)) async throws { let (a, b, c) = (aʹ, bʹ, cʹ) try await { /* ... */ }() } ``` ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 19 +- .../Testing/ABI/ABI.Record+Streaming.swift | 35 +-- Sources/Testing/CMakeLists.txt | 1 + .../ExitTests/ExitTest.CapturedValue.swift | 168 ++++++++++++ Sources/Testing/ExitTests/ExitTest.swift | 259 ++++++++++++++---- .../ExpectationChecking+Macro.swift | 43 ++- .../Support/Additions/ArrayAdditions.swift | 18 ++ Sources/Testing/Support/JSON.swift | 24 ++ Sources/TestingMacros/CMakeLists.txt | 1 + Sources/TestingMacros/ConditionMacro.swift | 83 ++++-- ...EditorPlaceholderExprSyntaxAdditions.swift | 13 + .../MacroExpansionContextAdditions.swift | 19 +- .../Support/ClosureCaptureListParsing.swift | 88 ++++++ .../Support/DiagnosticMessage.swift | 91 ++++-- .../Support/EffectfulExpressionHandling.swift | 2 +- .../ConditionMacroTests.swift | 36 ++- Tests/TestingTests/ExitTestTests.swift | 77 ++++++ 17 files changed, 833 insertions(+), 144 deletions(-) create mode 100644 Sources/Testing/ExitTests/ExitTest.CapturedValue.swift create mode 100644 Sources/TestingMacros/Support/ClosureCaptureListParsing.swift diff --git a/Package.swift b/Package.swift index f515f16a9..80e6ce797 100644 --- a/Package.swift +++ b/Package.swift @@ -1,9 +1,9 @@ -// swift-tools-version: 6.0 +// swift-tools-version: 6.1 // // This source file is part of the Swift.org open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -95,6 +95,13 @@ let package = Package( return result }(), + traits: [ + .trait( + name: "ExperimentalExitTestValueCapture", + description: "Enable experimental support for capturing values in exit tests" + ), + ], + dependencies: [ .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "602.0.0-latest"), ], @@ -285,6 +292,14 @@ extension Array where Element == PackageDescription.SwiftSetting { .define("SWT_NO_LIBDISPATCH", .whenEmbedded()), ] + // Unconditionally enable 'ExperimentalExitTestValueCapture' when building + // for development. + if buildingForDevelopment { + result += [ + .define("ExperimentalExitTestValueCapture") + ] + } + return result } diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 7b86cb438..1aa1362ec 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -12,39 +12,6 @@ private import Foundation extension ABI.Version { - /// Post-process encoded JSON and write it to a file. - /// - /// - Parameters: - /// - json: The JSON to write. - /// - file: The file to write to. - /// - /// - Throws: Whatever is thrown when writing to `file`. - private static func _asJSONLine(_ json: UnsafeRawBufferPointer, _ eventHandler: (_ recordJSON: UnsafeRawBufferPointer) throws -> Void) rethrows { - // We don't actually expect the JSON encoder to produce output containing - // newline characters, so in debug builds we'll log a diagnostic message. - if _slowPath(json.contains(where: \.isASCIINewline)) { -#if DEBUG && !SWT_NO_FILE_IO - let message = Event.ConsoleOutputRecorder.warning( - "JSON encoder produced one or more newline characters while encoding an event to JSON. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new", - options: .for(.stderr) - ) -#if SWT_TARGET_OS_APPLE - try? FileHandle.stderr.write(message) -#else - print(message) -#endif -#endif - - // Remove the newline characters to conform to JSON lines specification. - var json = Array(json) - json.removeAll(where: \.isASCIINewline) - try json.withUnsafeBytes(eventHandler) - } else { - // No newlines found, no need to copy the buffer. - try eventHandler(json) - } - } - static func eventHandler( encodeAsJSONLines: Bool, forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void @@ -52,7 +19,7 @@ extension ABI.Version { // Encode as JSON Lines if requested. var eventHandlerCopy = eventHandler if encodeAsJSONLines { - eventHandlerCopy = { @Sendable in _asJSONLine($0, eventHandler) } + eventHandlerCopy = { @Sendable in JSON.asJSONLine($0, eventHandler) } } let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder() diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 81a0c550a..b4e865427 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -32,6 +32,7 @@ add_library(Testing Events/Recorder/Event.Symbol.swift Events/TimeValue.swift ExitTests/ExitTest.swift + ExitTests/ExitTest.CapturedValue.swift ExitTests/ExitTest.Condition.swift ExitTests/ExitTest.Result.swift ExitTests/SpawnProcess.swift diff --git a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift new file mode 100644 index 000000000..aeeb13818 --- /dev/null +++ b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift @@ -0,0 +1,168 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !SWT_NO_EXIT_TESTS +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +extension ExitTest { + /// A type representing a value captured by an exit test's body. + /// + /// An instance of this type may represent the actual value that was captured + /// when the exit test was invoked. In the child process created by the + /// current exit test handler, instances will initially only have the type of + /// the value, but not the value itself. + /// + /// Instances of this type are created automatically by the testing library + /// for all elements in an exit test body's capture list and are stored in the + /// exit test's ``capturedValues`` property. For example, given the following + /// exit test: + /// + /// ```swift + /// await #expect(exitsWith: .failure) { [a = a as T, b = b as U, c = c as V] in + /// ... + /// } + /// ``` + /// + /// There are three captured values in its ``capturedValues`` property. These + /// values are captured at the time the exit test is called, as they would be + /// if the closure were called locally. + /// + /// The current exit test handler is responsible for encoding and decoding + /// instances of this type. When the handler is called, it is passed an + /// instance of ``ExitTest``. The handler encodes the values in that + /// instance's ``capturedValues`` property, then passes the encoded forms of + /// those values to the child process. The encoding format and message-passing + /// interface are implementation details of the exit test handler. + /// + /// When the child process calls ``ExitTest/find(identifiedBy:)``, it receives + /// an instance of ``ExitTest`` whose ``capturedValues`` property contains + /// type information but no values. The child process decodes the values it + /// encoded in the parent process and then updates the ``wrappedValue`` + /// property of each element in the array before calling the exit test's body. + public struct CapturedValue: Sendable { + /// An enumeration of the different states a captured value can have. + private enum _Kind: Sendable { + /// The runtime value of the captured value is known. + case wrappedValue(any Codable & Sendable) + + /// Only the type of the captured value is known. + case typeOnly(any (Codable & Sendable).Type) + } + + /// The current state of this instance. + private var _kind: _Kind + + init(wrappedValue: some Codable & Sendable) { + _kind = .wrappedValue(wrappedValue) + } + + init(typeOnly type: (some Codable & Sendable).Type) { + _kind = .typeOnly(type) + } + + /// The underlying value captured by this instance at runtime. + /// + /// In a child process created by the current exit test handler, the value + /// of this property is `nil` until the entry point sets it. + public var wrappedValue: (any Codable & Sendable)? { + get { + if case let .wrappedValue(wrappedValue) = _kind { + return wrappedValue + } + return nil + } + + set { + let type = typeOfWrappedValue + + func validate(_ newValue: T, is expectedType: U.Type) { + assert(newValue is U, "Attempted to set a captured value to an instance of '\(String(describingForTest: T.self))', but an instance of '\(String(describingForTest: U.self))' was expected.") + } + validate(newValue, is: type) + + if let newValue { + _kind = .wrappedValue(newValue) + } else { + _kind = .typeOnly(type) + } + } + } + + /// The type of the underlying value captured by this instance. + /// + /// This type is known at compile time and is always available, even before + /// this instance's ``wrappedValue`` property is set. + public var typeOfWrappedValue: any (Codable & Sendable).Type { + switch _kind { + case let .wrappedValue(wrappedValue): + type(of: wrappedValue) + case let .typeOnly(type): + type + } + } + } +} + +// MARK: - Collection conveniences + +extension Array where Element == ExitTest.CapturedValue { + init(_ wrappedValues: repeat each T) where repeat each T: Codable & Sendable { + self.init() + repeat self.append(ExitTest.CapturedValue(wrappedValue: each wrappedValues)) + } + + init(_ typesOfWrappedValues: repeat (each T).Type) where repeat each T: Codable & Sendable { + self.init() + repeat self.append(ExitTest.CapturedValue(typeOnly: (each typesOfWrappedValues).self)) + } +} + +extension Collection where Element == ExitTest.CapturedValue { + /// Cast the elements in this collection to a tuple of their wrapped values. + /// + /// - Returns: A tuple containing the wrapped values of the elements in this + /// collection. + /// + /// - Throws: If an expected value could not be found or was not of the + /// type the caller expected. + /// + /// This function assumes that the entry point function has already set the + /// ``wrappedValue`` property of each element in this collection. + func takeCapturedValues() throws -> (repeat each T) { + func nextValue( + as type: U.Type, + from capturedValues: inout SubSequence + ) throws -> U { + // Get the next captured value in the collection. If we run out of values + // before running out of parameter pack elements, then something in the + // exit test handler or entry point is likely broken. + guard let wrappedValue = capturedValues.first?.wrappedValue else { + let actualCount = self.count + let expectedCount = parameterPackCount(repeat (each T).self) + fatalError("Found fewer captured values (\(actualCount)) than expected (\(expectedCount)) when passing them to the current exit test.") + } + + // Next loop, get the next element. (We're mutating a subsequence, not + // self, so this is generally an O(1) operation.) + capturedValues = capturedValues.dropFirst() + + // Make sure the value is of the correct type. If it's not, that's also + // probably a problem with the exit test handler or entry point. + guard let wrappedValue = wrappedValue as? U else { + fatalError("Expected captured value at index \(capturedValues.startIndex) with type '\(String(describingForTest: U.self))', but found an instance of '\(String(describingForTest: Swift.type(of: wrappedValue)))' instead.") + } + + return wrappedValue + } + + var capturedValues = self[...] + return (repeat try nextValue(as: (each T).self, from: &capturedValues)) + } +} +#endif diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 93393b69b..503e143c6 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -68,10 +68,13 @@ public struct ExitTest: Sendable, ~Copyable { /// The body closure of the exit test. /// + /// - Parameters: + /// - exitTest: The exit test to which this body closure belongs. + /// /// Do not invoke this closure directly. Instead, invoke ``callAsFunction()`` /// to run the exit test. Running the exit test will always terminate the /// current process. - fileprivate var body: @Sendable () async throws -> Void = {} + fileprivate var body: @Sendable (_ exitTest: inout Self) async throws -> Void = { _ in } /// Storage for ``observedValues``. /// @@ -108,6 +111,19 @@ public struct ExitTest: Sendable, ~Copyable { } } + /// The set of values captured in the parent process before the exit test is + /// called. + /// + /// This property is automatically set by the testing library when using the + /// built-in exit test handler and entry point functions. Do not modify the + /// value of this property unless you are implementing a custom exit test + /// handler or entry point function. + /// + /// The order of values in this array must be the same between the parent and + /// child processes. + @_spi(Experimental) @_spi(ForToolsIntegrationOnly) + public var capturedValues = [CapturedValue]() + /// Make a copy of this instance. /// /// - Returns: A copy of this instance. @@ -117,6 +133,7 @@ public struct ExitTest: Sendable, ~Copyable { fileprivate borrowing func unsafeCopy() -> Self { var result = Self(id: id, body: body) result._observedValues = _observedValues + result.capturedValues = capturedValues return result } } @@ -245,7 +262,7 @@ extension ExitTest { } do { - try await body() + try await body(&self) } catch { _errorInMain(error) } @@ -279,25 +296,39 @@ extension ExitTest: DiscoverableAsTestContent { /// /// - Warning: This function is used to implement the `#expect(exitsWith:)` /// macro. Do not use it directly. - public static func __store( + public static func __store( _ id: (UInt64, UInt64, UInt64, UInt64), - _ body: @escaping @Sendable () async throws -> Void, + _ body: @escaping @Sendable (repeat each T) async throws -> Void, into outValue: UnsafeMutableRawPointer, asTypeAt typeAddress: UnsafeRawPointer, withHintAt hintAddress: UnsafeRawPointer? = nil - ) -> CBool { + ) -> CBool where repeat each T: Codable & Sendable { #if !hasFeature(Embedded) + // Check that the type matches. let callerExpectedType = TypeInfo(describing: typeAddress.load(as: Any.Type.self)) let selfType = TypeInfo(describing: Self.self) guard callerExpectedType == selfType else { return false } #endif + + // Check that the ID matches if provided. let id = ID(id) if let hintedID = hintAddress?.load(as: ID.self), hintedID != id { return false } - outValue.initializeMemory(as: Self.self, to: Self(id: id, body: body)) + + // Wrap the body function in a thunk that decodes any captured state and + // passes it along. + let body: @Sendable (inout Self) async throws -> Void = { exitTest in + let values: (repeat each T) = try exitTest.capturedValues.takeCapturedValues() + try await body(repeat each values) + } + + // Construct and return the instance. + var exitTest = Self(id: id, body: body) + exitTest.capturedValues = Array(repeat (each T).self) + outValue.initializeMemory(as: Self.self, to: exitTest) return true } } @@ -338,6 +369,7 @@ extension ExitTest { /// /// - Parameters: /// - exitTestID: The unique identifier of the exit test. +/// - capturedValues: Any values captured by the exit test. /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The @@ -357,6 +389,7 @@ extension ExitTest { /// convention. func callExitTest( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), + encodingCapturedValues capturedValues: [ExitTest.CapturedValue], exitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, @@ -371,8 +404,12 @@ func callExitTest( var result: ExitTest.Result do { + // Construct a temporary/local exit test to pass to the exit test handler. var exitTest = ExitTest(id: ExitTest.ID(exitTestID)) exitTest.observedValues = observedValues + exitTest.capturedValues = capturedValues + + // Invoke the exit test handler and wait for the child process to terminate. result = try await configuration.exitTestHandler(exitTest) #if os(Windows) @@ -467,15 +504,23 @@ extension ExitTest { /// are available or the child environment is otherwise terminated. The parent /// environment is then responsible for interpreting those results and /// recording any issues that occur. - public typealias Handler = @Sendable (_ exitTest: borrowing ExitTest) async throws -> ExitTest.Result + public typealias Handler = @Sendable (_ exitTest: borrowing Self) async throws -> ExitTest.Result - /// The back channel file handle set up by the parent process. + /// Make a file handle from the string contained in the given environment + /// variable. + /// + /// - Parameters: + /// - name: The name of the environment variable to read. The value of this + /// environment variable should represent the file handle. The exact value + /// is platform-specific but is generally the file descriptor as a string. + /// - mode: The mode to open the file with, such as `"wb"`. /// - /// The value of this property is a file handle open for writing to which - /// events should be written, or `nil` if the file handle could not be - /// resolved. - private static let _backChannelForEntryPoint: FileHandle? = { - guard let backChannelEnvironmentVariable = Environment.variable(named: "SWT_EXPERIMENTAL_BACKCHANNEL") else { + /// - Returns: A new file handle, or `nil` if one could not be created. + /// + /// The effect of calling this function more than once for the same + /// environment variable is undefined. + private static func _makeFileHandle(forEnvironmentVariableNamed name: String, mode: String) -> FileHandle? { + guard let environmentVariable = Environment.variable(named: name) else { return nil } @@ -485,20 +530,55 @@ extension ExitTest { var fd: CInt? #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) - fd = CInt(backChannelEnvironmentVariable) + fd = CInt(environmentVariable) #elseif os(Windows) - if let handle = UInt(backChannelEnvironmentVariable).flatMap(HANDLE.init(bitPattern:)) { - fd = _open_osfhandle(Int(bitPattern: handle), _O_WRONLY | _O_BINARY) + if let handle = UInt(environmentVariable).flatMap(HANDLE.init(bitPattern:)) { + var flags: CInt = switch (mode.contains("r"), mode.contains("w")) { + case (true, true): + _O_RDWR + case (true, false): + _O_RDONLY + case (false, true): + _O_WRONLY + case (false, false): + 0 + } + flags |= _O_BINARY + fd = _open_osfhandle(Int(bitPattern: handle), flags) } #else -#warning("Platform-specific implementation missing: back-channel pipe unavailable") +#warning("Platform-specific implementation missing: additional file descriptors unavailable") #endif guard let fd, fd >= 0 else { return nil } - return try? FileHandle(unsafePOSIXFileDescriptor: fd, mode: "wb") - }() + return try? FileHandle(unsafePOSIXFileDescriptor: fd, mode: mode) + } + + /// Make a string suitable for use as the value of an environment variable + /// that describes the given file handle. + /// + /// - Parameters: + /// - fileHandle: The file handle to represent. + /// + /// - Returns: A string representation of `fileHandle` that can be converted + /// back to a (new) file handle with `_makeFileHandle()`, or `nil` if the + /// file handle could not be converted to a string. + private static func _makeEnvironmentVariable(for fileHandle: borrowing FileHandle) -> String? { +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) + return fileHandle.withUnsafePOSIXFileDescriptor { fd in + fd.map(String.init(describing:)) + } +#elseif os(Windows) + return fileHandle.withUnsafeWindowsHANDLE { handle in + handle.flatMap { String(describing: UInt(bitPattern: $0)) } + } +#else +#warning("Platform-specific implementation missing: additional file descriptors unavailable") + return nil +#endif + } /// Find the exit test function specified in the environment of the current /// process, if any. @@ -533,7 +613,7 @@ extension ExitTest { } // We can't say guard let here because it counts as a consume. - guard _backChannelForEntryPoint != nil else { + guard let backChannel = _makeFileHandle(forEnvironmentVariableNamed: "SWT_EXPERIMENTAL_BACKCHANNEL", mode: "wb") else { return result } @@ -544,9 +624,9 @@ extension ExitTest { // Only forward issue-recorded events. (If we start handling other kinds of // events in the future, we can forward them too.) let eventHandler = ABI.BackChannelVersion.eventHandler(encodeAsJSONLines: true) { json in - _ = try? _backChannelForEntryPoint?.withLock { - try _backChannelForEntryPoint?.write(json) - try _backChannelForEntryPoint?.write("\n") + _ = try? backChannel.withLock { + try backChannel.write(json) + try backChannel.write("\n") } } configuration.eventHandler = { event, eventContext in @@ -555,8 +635,11 @@ extension ExitTest { } } - result.body = { [configuration, body = result.body] in - try await Configuration.withCurrent(configuration, perform: body) + result.body = { [configuration, body = result.body] exitTest in + try await Configuration.withCurrent(configuration) { + try exitTest._decodeCapturedValuesForEntryPoint() + try await body(&exitTest) + } } return result } @@ -626,7 +709,7 @@ extension ExitTest { return result }() - return { exitTest in + @Sendable func result(_ exitTest: borrowing ExitTest) async throws -> ExitTest.Result { let childProcessExecutablePath = try childProcessExecutablePath.get() // Inherit the environment from the parent process and make any necessary @@ -679,37 +762,50 @@ extension ExitTest { var backChannelWriteEnd: FileHandle! try FileHandle.makePipe(readEnd: &backChannelReadEnd, writeEnd: &backChannelWriteEnd) - // Let the child process know how to find the back channel by setting a - // known environment variable to the corresponding file descriptor - // (HANDLE on Windows.) - var backChannelEnvironmentVariable: String? -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) - backChannelEnvironmentVariable = backChannelWriteEnd.withUnsafePOSIXFileDescriptor { fd in - fd.map(String.init(describing:)) - } -#elseif os(Windows) - backChannelEnvironmentVariable = backChannelWriteEnd.withUnsafeWindowsHANDLE { handle in - handle.flatMap { String(describing: UInt(bitPattern: $0)) } - } -#else -#warning("Platform-specific implementation missing: back-channel pipe unavailable") -#endif - if let backChannelEnvironmentVariable { + // Create another pipe to send captured values (and possibly other state + // in the future) to the child process. + var capturedValuesReadEnd: FileHandle! + var capturedValuesWriteEnd: FileHandle! + try FileHandle.makePipe(readEnd: &capturedValuesReadEnd, writeEnd: &capturedValuesWriteEnd) + + // Let the child process know how to find the back channel and + // captured values channel by setting a known environment variable to + // the corresponding file descriptor (HANDLE on Windows) for each. + if let backChannelEnvironmentVariable = _makeEnvironmentVariable(for: backChannelWriteEnd) { childEnvironment["SWT_EXPERIMENTAL_BACKCHANNEL"] = backChannelEnvironmentVariable } + if let capturedValuesEnvironmentVariable = _makeEnvironmentVariable(for: capturedValuesReadEnd) { + childEnvironment["SWT_EXPERIMENTAL_CAPTURED_VALUES"] = capturedValuesEnvironmentVariable + } // Spawn the child process. let processID = try withUnsafePointer(to: backChannelWriteEnd) { backChannelWriteEnd in - try spawnExecutable( - atPath: childProcessExecutablePath, - arguments: childArguments, - environment: childEnvironment, - standardOutput: stdoutWriteEnd, - standardError: stderrWriteEnd, - additionalFileHandles: [backChannelWriteEnd] - ) + try withUnsafePointer(to: capturedValuesReadEnd) { capturedValuesReadEnd in + try spawnExecutable( + atPath: childProcessExecutablePath, + arguments: childArguments, + environment: childEnvironment, + standardOutput: stdoutWriteEnd, + standardError: stderrWriteEnd, + additionalFileHandles: [backChannelWriteEnd, capturedValuesReadEnd] + ) + } } + // Write the captured values blob over the back channel to the child + // process. (If we end up needing to write additional data, we can + // define a full schema for this stream. Fortunately, both endpoints are + // implemented in the same copy of the testing library, so we don't have + // to worry about backwards-compatibility.) + try capturedValuesWriteEnd.withLock { + try exitTest._withEncodedCapturedValuesForEntryPoint { capturedValuesJSON in + try capturedValuesWriteEnd.write(capturedValuesJSON) + try capturedValuesWriteEnd.write("\n") + } + } + capturedValuesReadEnd.close() + capturedValuesWriteEnd.close() + // Await termination of the child process. taskGroup.addTask { let statusAtExit = try await wait(for: processID) @@ -750,6 +846,8 @@ extension ExitTest { return result } } + + return result } /// Read lines from the given back channel file handle and process them as @@ -797,9 +895,7 @@ extension ExitTest { // Translate the issue back into a "real" issue and record it // in the parent process. This translation is, of course, lossy // due to the process boundary, but we make a best effort. - let comments: [Comment] = event.messages.compactMap { message in - message.symbol == .details ? Comment(rawValue: message.text) : nil - } + let comments: [Comment] = event.messages.map(\.text).map(Comment.init(rawValue:)) let issueKind: Issue.Kind = if let error = issue._error { .errorCaught(error) } else { @@ -815,5 +911,62 @@ extension ExitTest { issueCopy.record() } } + + /// Decode this exit test's captured values and update its ``capturedValues`` + /// property. + /// + /// - Throws: If a captured value could not be decoded. + /// + /// This function should only be used when the process was started via the + /// `__swiftPMEntryPoint()` function. The effect of using it under other + /// configurations is undefined. + private mutating func _decodeCapturedValuesForEntryPoint() throws { + // Read the content of the captured values stream provided by the parent + // process above. + guard let fileHandle = Self._makeFileHandle(forEnvironmentVariableNamed: "SWT_EXPERIMENTAL_CAPTURED_VALUES", mode: "rb") else { + return + } + let capturedValuesJSON = try fileHandle.readToEnd() + let capturedValuesJSONLines = capturedValuesJSON.split(whereSeparator: \.isASCIINewline) + assert(capturedValues.count == capturedValuesJSONLines.count, "Expected to decode \(capturedValues.count) captured value(s) for the current exit test, but received \(capturedValuesJSONLines.count). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + + // Walk the list of captured values' types, map them to their JSON blobs, + // and decode them. + capturedValues = try zip(capturedValues, capturedValuesJSONLines).map { capturedValue, capturedValueJSON in + var capturedValue = capturedValue + + func open(_ type: T.Type) throws -> T where T: Codable & Sendable { + return try capturedValueJSON.withUnsafeBytes { capturedValueJSON in + try JSON.decode(type, from: capturedValueJSON) + } + } + capturedValue.wrappedValue = try open(capturedValue.typeOfWrappedValue) + + return capturedValue + } + } + + /// Encode this exit test's captured values in a format suitable for passing + /// to the child process. + /// + /// - Parameters: + /// - body: A function to call. This function is called once per captured + /// value in the exit test. + /// + /// - Throws: Whatever is thrown by `body` or while encoding. + /// + /// This function produces a byte buffer representing each value in this exit + /// test's ``capturedValues`` property and passes each buffer to `body`. + /// + /// This function should only be used when the process was started via the + /// `__swiftPMEntryPoint()` function. The effect of using it under other + /// configurations is undefined. + private borrowing func _withEncodedCapturedValuesForEntryPoint(_ body: (UnsafeRawBufferPointer) throws -> Void) throws -> Void { + for capturedValue in capturedValues { + try JSON.withEncoding(of: capturedValue.wrappedValue!) { capturedValueJSON in + try JSON.asJSONLine(capturedValueJSON, body) + } + } + } } #endif diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index aa999395a..e8767d01f 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1139,9 +1139,8 @@ public func __checkClosureCall( /// Check that an expression always exits (terminates the current process) with /// a given status. /// -/// This overload is used for `await #expect(exitsWith:) { }` invocations. Note -/// that the `body` argument is thin here because it cannot meaningfully capture -/// state from the enclosing context. +/// This overload is used for `await #expect(exitsWith:) { }` invocations that +/// do not capture any state. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. @@ -1149,8 +1148,8 @@ public func __checkClosureCall( public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), exitsWith expectedExitCondition: ExitTest.Condition, - observing observedValues: [any PartialKeyPath & Sendable], - performing body: @convention(thin) () -> Void, + observing observedValues: [any PartialKeyPath & Sendable] = [], + performing _: @convention(thin) () -> Void, expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, @@ -1159,6 +1158,40 @@ public func __checkClosureCall( ) async -> Result { await callExitTest( identifiedBy: exitTestID, + encodingCapturedValues: [], + exitsWith: expectedExitCondition, + observing: observedValues, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +/// Check that an expression always exits (terminates the current process) with +/// a given status. +/// +/// This overload is used for `await #expect(exitsWith:) { }` invocations that +/// capture some values with an explicit capture list. +/// +/// - Warning: This function is used to implement the `#expect()` and +/// `#require()` macros. Do not call it directly. +@_spi(Experimental) +public func __checkClosureCall( + identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), + encodingCapturedValues capturedValues: (repeat each T), + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], + performing _: @convention(thin) () -> Void, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation +) async -> Result where repeat each T: Codable & Sendable { + await callExitTest( + identifiedBy: exitTestID, + encodingCapturedValues: Array(repeat each capturedValues), exitsWith: expectedExitCondition, observing: observedValues, expression: expression, diff --git a/Sources/Testing/Support/Additions/ArrayAdditions.swift b/Sources/Testing/Support/Additions/ArrayAdditions.swift index 462a330fd..eee74037d 100644 --- a/Sources/Testing/Support/Additions/ArrayAdditions.swift +++ b/Sources/Testing/Support/Additions/ArrayAdditions.swift @@ -21,3 +21,21 @@ extension Array { self = optionalValue.map { [$0] } ?? [] } } + +/// Get the number of elements in a parameter pack. +/// +/// - Parameters: +/// - pack: The parameter pack. +/// +/// - Returns: The number of elements in `pack`. +/// +/// - Complexity: O(_n_) where _n_ is the number of elements in `pack`. The +/// compiler may be able to optimize this operation when the types of `pack` +/// are statically known. +func parameterPackCount(_ pack: repeat each T) -> Int { + var result = 0 + for _ in repeat each pack { + result += 1 + } + return result +} diff --git a/Sources/Testing/Support/JSON.swift b/Sources/Testing/Support/JSON.swift index 76c7b7f07..3d656687f 100644 --- a/Sources/Testing/Support/JSON.swift +++ b/Sources/Testing/Support/JSON.swift @@ -50,6 +50,30 @@ enum JSON { #endif } + /// Post-process encoded JSON and write it to a file. + /// + /// - Parameters: + /// - json: The JSON to write. + /// - body: A function to call. A copy of `json` is passed to it with any + /// newlines removed. + /// + /// - Returns: Whatever is returned by `body`. + /// + /// - Throws: Whatever is thrown by `body`. + static func asJSONLine(_ json: UnsafeRawBufferPointer, _ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { + if _slowPath(json.contains(where: \.isASCIINewline)) { + // Remove the newline characters to conform to JSON lines specification. + // This is not actually expected to happen in practice with Foundation's + // JSON encoder. + var json = Array(json) + json.removeAll(where: \.isASCIINewline) + return try json.withUnsafeBytes(body) + } else { + // No newlines found, no need to copy the buffer. + return try body(json) + } + } + /// Decode a value from JSON data. /// /// - Parameters: diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index e535f13cf..c9a579eaf 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -97,6 +97,7 @@ target_sources(TestingMacros PRIVATE Support/Argument.swift Support/AttributeDiscovery.swift Support/AvailabilityGuards.swift + Support/ClosureCaptureListParsing.swift Support/CommentParsing.swift Support/ConditionArgumentParsing.swift Support/DiagnosticMessage.swift diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 8ae26bf82..326522858 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -420,29 +420,28 @@ extension ExitTestConditionMacro { _ = try Base.expansion(of: macro, in: context) var arguments = argumentList(of: macro, in: context) - let requirementIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("exitsWith") } - guard let requirementIndex else { - fatalError("Could not find the requirement for this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") - } - let observationListIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("observing") } - if observationListIndex == nil { - arguments.insert( - Argument(label: "observing", expression: ArrayExprSyntax(expressions: [])), - at: arguments.index(after: requirementIndex) - ) - } let trailingClosureIndex = arguments.firstIndex { $0.label?.tokenKind == _trailingClosureLabel.tokenKind } guard let trailingClosureIndex else { fatalError("Could not find the body argument to this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } - // Extract the body argument and, if it's a closure with a capture list, - // emit an appropriate diagnostic. var bodyArgumentExpr = arguments[trailingClosureIndex].expression bodyArgumentExpr = removeParentheses(from: bodyArgumentExpr) ?? bodyArgumentExpr - if let closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), - let captureClause = closureExpr.signature?.capture, - !captureClause.items.isEmpty { + + // Find any captured values and extract them from the trailing closure. + var capturedValues = [CapturedValueInfo]() + if ExitTestExpectMacro.isValueCapturingEnabled { + // The source file imports @_spi(Experimental), so allow value capturing. + if var closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), + let captureList = closureExpr.signature?.capture?.items { + closureExpr.signature?.capture = ClosureCaptureClauseSyntax(items: [], trailingTrivia: .space) + capturedValues = captureList.map { CapturedValueInfo($0, in: context) } + bodyArgumentExpr = ExprSyntax(closureExpr) + } + + } else if let closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), + let captureClause = closureExpr.signature?.capture, + !captureClause.items.isEmpty { context.diagnose(.captureClauseUnsupported(captureClause, in: closureExpr, inExitTest: macro)) } @@ -454,10 +453,20 @@ extension ExitTestConditionMacro { // Implement the body of the exit test outside the enum we're declaring so // that `Self` resolves to the type containing the exit test, not the enum. let bodyThunkName = context.makeUniqueName("") + let bodyThunkParameterList = FunctionParameterListSyntax { + for capturedValue in capturedValues { + FunctionParameterSyntax( + firstName: .wildcardToken(trailingTrivia: .space), + secondName: capturedValue.name.trimmed, + colon: .colonToken(trailingTrivia: .space), + type: capturedValue.type.trimmed + ) + } + } decls.append( """ - @Sendable func \(bodyThunkName)() async throws -> Swift.Void { - return \(applyEffectfulKeywords([.try, .await, .unsafe], to: bodyArgumentExpr))() + @Sendable func \(bodyThunkName)(\(bodyThunkParameterList)) async throws { + _ = \(applyEffectfulKeywords([.try, .await, .unsafe], to: bodyArgumentExpr))() } """ ) @@ -521,12 +530,24 @@ extension ExitTestConditionMacro { } ) - // Insert the exit test's ID as the first argument. Note that this will - // invalidate all indices into `arguments`! - arguments.insert( + // Insert additional arguments at the beginning of the argument list. Note + // that this will invalidate all indices into `arguments`! + var leadingArguments = [ Argument(label: "identifiedBy", expression: idExpr), - at: arguments.startIndex - ) + ] + if !capturedValues.isEmpty { + leadingArguments.append( + Argument( + label: "encodingCapturedValues", + expression: TupleExprSyntax { + for capturedValue in capturedValues { + LabeledExprSyntax(expression: capturedValue.expression.trimmed) + } + } + ) + ) + } + arguments = leadingArguments + arguments // Replace the exit test body (as an argument to the macro) with a stub // closure that hosts the type we created above. @@ -582,6 +603,22 @@ extension ExitTestConditionMacro { } } +extension ExitTestExpectMacro { + /// Whether or not experimental value capturing via explicit capture lists is + /// enabled. + /// + /// This member is declared on ``ExitTestExpectMacro`` but also applies to + /// ``ExitTestRequireMacro``. + @TaskLocal + static var isValueCapturingEnabled: Bool = { +#if ExperimentalExitTestValueCapture + return true +#else + return false +#endif + }() +} + /// A type describing the expansion of the `#expect(exitsWith:)` macro. /// /// This type checks for nested invocations of `#expect()` and `#require()` and diff --git a/Sources/TestingMacros/Support/Additions/EditorPlaceholderExprSyntaxAdditions.swift b/Sources/TestingMacros/Support/Additions/EditorPlaceholderExprSyntaxAdditions.swift index a8b5063cc..9a0d31ab3 100644 --- a/Sources/TestingMacros/Support/Additions/EditorPlaceholderExprSyntaxAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/EditorPlaceholderExprSyntaxAdditions.swift @@ -52,3 +52,16 @@ extension EditorPlaceholderExprSyntax { self.init(type, type: type) } } + +extension TypeSyntax { + /// Construct a type syntax node containing a placeholder string. + /// + /// - Parameters: + /// - placeholder: The placeholder string, not including surrounding angle + /// brackets or pound characters. + /// + /// - Returns: A new `TypeSyntax` instance representing a placeholder. + static func placeholder(_ placeholder: String) -> Self { + return Self(IdentifierTypeSyntax(name: .identifier("<#\(placeholder)#" + ">"))) + } +} diff --git a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift index 322a84f3a..d0f296892 100644 --- a/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/MacroExpansionContextAdditions.swift @@ -14,17 +14,18 @@ import SwiftSyntaxMacros import SwiftDiagnostics extension MacroExpansionContext { - /// Get the type of the lexical context enclosing the given node. + /// Get the type of the given lexical context. /// /// - Parameters: - /// - node: The node whose lexical context should be examined. + /// - lexicalContext: The lexical context. /// - /// - Returns: The type of the lexical context enclosing `node`, or `nil` if - /// the lexical context cannot be represented as a type. + /// - Returns: The type represented by `lexicalContext`, or `nil` if one could + /// not be derived (for example, because the lexical context inclues a + /// function, closure, or some other non-type scope.) /// /// If the lexical context includes functions, closures, or some other /// non-type scope, the value of this property is `nil`. - var typeOfLexicalContext: TypeSyntax? { + func type(ofLexicalContext lexicalContext: some RandomAccessCollection) -> TypeSyntax? { var typeNames = [String]() for lexicalContext in lexicalContext.reversed() { guard let decl = lexicalContext.asProtocol((any DeclGroupSyntax).self) else { @@ -38,6 +39,14 @@ extension MacroExpansionContext { return "\(raw: typeNames.joined(separator: "."))" } + + /// The type of the lexical context enclosing the given node. + /// + /// If the lexical context includes functions, closures, or some other + /// non-type scope, the value of this property is `nil`. + var typeOfLexicalContext: TypeSyntax? { + type(ofLexicalContext: lexicalContext) + } } // MARK: - diff --git a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift new file mode 100644 index 000000000..41abe711c --- /dev/null +++ b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift @@ -0,0 +1,88 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023–2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +import SwiftDiagnostics +import SwiftParser +import SwiftSyntax +import SwiftSyntaxBuilder +import SwiftSyntaxMacros + +/// A type representing a value extracted from a closure's capture list. +struct CapturedValueInfo { + /// The original instance of `ClosureCaptureSyntax` used to create this value. + var capture: ClosureCaptureSyntax + + /// The name of the captured value. + var name: TokenSyntax { + let text = capture.name.textWithoutBackticks + if text.isValidSwiftIdentifier(for: .variableName) { + return capture.name + } + return .identifier("`\(text)`") + } + + /// The expression to assign to the captured value. + var expression: ExprSyntax + + /// The type of the captured value. + var type: TypeSyntax + + init(_ capture: ClosureCaptureSyntax, in context: some MacroExpansionContext) { + self.capture = capture + self.expression = "()" + self.type = "Swift.Void" + + // We don't support capture specifiers at this time. + if let specifier = capture.specifier { + context.diagnose(.specifierUnsupported(specifier, on: capture)) + return + } + + // Potentially get the name of the type comprising the current lexical + // context (i.e. whatever `Self` is.) + lazy var typeNameOfLexicalContext = { + let lexicalContext = context.lexicalContext.drop { !$0.isProtocol((any DeclGroupSyntax).self) } + return context.type(ofLexicalContext: lexicalContext) + }() + + if let initializer = capture.initializer { + // Found an initializer clause. Extract the expression it captures. + self.expression = removeParentheses(from: initializer.value) ?? initializer.value + + // Find the 'as' clause so we can determine the type of the captured value. + if let asExpr = self.expression.as(AsExprSyntax.self) { + self.type = if asExpr.questionOrExclamationMark?.tokenKind == .postfixQuestionMark { + // If the caller is using as?, make the type optional. + TypeSyntax(OptionalTypeSyntax(wrappedType: asExpr.type.trimmed)) + } else { + asExpr.type + } + } else if let selfExpr = self.expression.as(DeclReferenceExprSyntax.self), + selfExpr.baseName.tokenKind == .keyword(.self), + selfExpr.argumentNames == nil, + let typeNameOfLexicalContext { + // Copying self. + self.type = typeNameOfLexicalContext + } else { + context.diagnose(.typeOfCaptureIsAmbiguous(capture, initializedWith: initializer)) + } + + } else if capture.name.tokenKind == .keyword(.self), + let typeNameOfLexicalContext { + // Capturing self. + self.expression = "self" + self.type = typeNameOfLexicalContext + + } else { + // Not enough contextual information to derive the type here. + context.diagnose(.typeOfCaptureIsAmbiguous(capture)) + } + } +} diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index dc9defe5d..36186ec4b 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -739,22 +739,6 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage { ) } - /// Create a diagnostic message stating that a condition macro nested inside - /// an exit test will not record any diagnostics. - /// - /// - Parameters: - /// - checkMacro: The inner condition macro invocation. - /// - exitTestMacro: The containing exit test macro invocation. - /// - /// - Returns: A diagnostic message. - static func checkUnsupported(_ checkMacro: some FreestandingMacroExpansionSyntax, inExitTest exitTestMacro: some FreestandingMacroExpansionSyntax) -> Self { - Self( - syntax: Syntax(checkMacro), - message: "Expression \(_macroName(checkMacro)) will not record an issue on failure inside exit test \(_macroName(exitTestMacro))", - severity: .error - ) - } - var syntax: Syntax // MARK: - DiagnosticMessage @@ -768,6 +752,81 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage { // MARK: - Captured values extension DiagnosticMessage { + /// Create a diagnostic message stating that a specifier keyword cannot be + /// used with a given closure capture list item. + /// + /// - Parameters: + /// - specifier: The invalid specifier. + /// - capture: The closure capture list item. + /// + /// - Returns: A diagnostic message. + static func specifierUnsupported(_ specifier: ClosureCaptureSpecifierSyntax, on capture: ClosureCaptureSyntax) -> Self { + Self( + syntax: Syntax(specifier), + message: "Specifier '\(specifier.trimmed)' cannot be used with captured value '\(capture.name.textWithoutBackticks)'", + severity: .error, + fixIts: [ + FixIt( + message: MacroExpansionFixItMessage("Remove '\(specifier.trimmed)'"), + changes: [ + .replace( + oldNode: Syntax(capture), + newNode: Syntax(capture.with(\.specifier, nil)) + ) + ] + ), + ] + ) + } + + /// Create a diagnostic message stating that a closure capture list item's + /// type is ambiguous and must be made explicit. + /// + /// - Parameters: + /// - capture: The closure capture list item. + /// - initializerClause: The existing initializer clause, if any. + /// + /// - Returns: A diagnostic message. + static func typeOfCaptureIsAmbiguous(_ capture: ClosureCaptureSyntax, initializedWith initializerClause: InitializerClauseSyntax? = nil) -> Self { + let castValueExpr: some ExprSyntaxProtocol = if let initializerClause { + ExprSyntax(initializerClause.value.trimmed) + } else { + ExprSyntax(DeclReferenceExprSyntax(baseName: capture.name.trimmed)) + } + let initializerValueExpr = ExprSyntax( + AsExprSyntax( + expression: castValueExpr, + asKeyword: .keyword(.as, leadingTrivia: .space, trailingTrivia: .space), + type: TypeSyntax.placeholder("T") + ) + ) + let placeholderInitializerClause = if let initializerClause { + initializerClause.with(\.value, initializerValueExpr) + } else { + InitializerClauseSyntax( + equal: .equalToken(leadingTrivia: .space, trailingTrivia: .space), + value: initializerValueExpr + ) + } + + return Self( + syntax: Syntax(capture), + message: "Type of captured value '\(capture.name.textWithoutBackticks)' is ambiguous", + severity: .error, + fixIts: [ + FixIt( + message: MacroExpansionFixItMessage("Add '= \(castValueExpr) as T'"), + changes: [ + .replace( + oldNode: Syntax(capture), + newNode: Syntax(capture.with(\.initializer, placeholderInitializerClause)) + ) + ] + ), + ] + ) + } + /// Create a diagnostic message stating that a capture clause cannot be used /// in an exit test. /// diff --git a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift index b8f6d125d..f67ca40ee 100644 --- a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift +++ b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift @@ -107,7 +107,7 @@ private func _makeCallToEffectfulThunk(_ thunkName: TokenSyntax, passing expr: s /// adds the keywords in `effectfulKeywords` to `expr`. func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some ExprSyntaxProtocol) -> ExprSyntax { let originalExpr = expr - var expr = ExprSyntax(expr) + var expr = ExprSyntax(expr.trimmed) let needAwait = effectfulKeywords.contains(.await) && !expr.is(AwaitExprSyntax.self) let needTry = effectfulKeywords.contains(.try) && !expr.is(TryExprSyntax.self) diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index 07d84b0f8..cd1333941 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -383,6 +383,30 @@ struct ConditionMacroTests { #expect(diagnostic.message.contains("is redundant")) } +#if ExperimentalExitTestValueCapture + @Test("#expect(exitsWith:) produces a diagnostic for a bad capture", + arguments: [ + "#expectExitTest(exitsWith: x) { [weak a] in }": + "Specifier 'weak' cannot be used with captured value 'a'", + "#expectExitTest(exitsWith: x) { [a] in }": + "Type of captured value 'a' is ambiguous", + "#expectExitTest(exitsWith: x) { [a = b] in }": + "Type of captured value 'a' is ambiguous", + ] + ) + func exitTestCaptureDiagnostics(input: String, expectedMessage: String) throws { + try ExitTestExpectMacro.$isValueCapturingEnabled.withValue(true) { + let (_, diagnostics) = try parse(input) + + #expect(diagnostics.count > 0) + for diagnostic in diagnostics { + #expect(diagnostic.diagMessage.severity == .error) + #expect(diagnostic.message == expectedMessage) + } + } + } +#endif + @Test( "Capture list on an exit test produces a diagnostic", arguments: [ @@ -391,12 +415,14 @@ struct ConditionMacroTests { ] ) func exitTestCaptureListProducesDiagnostic(input: String, expectedMessage: String) throws { - let (_, diagnostics) = try parse(input) + try ExitTestExpectMacro.$isValueCapturingEnabled.withValue(false) { + let (_, diagnostics) = try parse(input) - #expect(diagnostics.count > 0) - for diagnostic in diagnostics { - #expect(diagnostic.diagMessage.severity == .error) - #expect(diagnostic.message == expectedMessage) + #expect(diagnostics.count > 0) + for diagnostic in diagnostics { + #expect(diagnostic.diagMessage.severity == .error) + #expect(diagnostic.message == expectedMessage) + } } } diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index bc3425e0a..896784f22 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -380,6 +380,83 @@ private import _TestingInternals #expect((ExitTest.current != nil) as Bool) } } + +#if ExperimentalExitTestValueCapture + @Test("Capture list") + func captureList() async { + let i = 123 + let s = "abc" as Any + await #expect(exitsWith: .success) { [i = i as Int, s = s as! String, t = (s as Any) as? String?] in + #expect(i == 123) + #expect(s == "abc") + #expect(t == "abc") + } + } + + @Test("Capture list (very long encoded form)") + func longCaptureList() async { + let count = 1 * 1024 * 1024 + let buffer = Array(repeatElement(0 as UInt8, count: count)) + await #expect(exitsWith: .success) { [count = count as Int, buffer = buffer as [UInt8]] in + #expect(buffer.count == count) + } + } + + struct CapturableSuite: Codable { + var property = 456 + + @Test("self in capture list") + func captureListWithSelf() async { + await #expect(exitsWith: .success) { [self, x = self] in + #expect(self.property == 456) + #expect(x.property == 456) + } + } + } + + class CapturableBaseClass: @unchecked Sendable, Codable { + init() {} + + required init(from decoder: any Decoder) throws {} + func encode(to encoder: any Encoder) throws {} + } + + final class CapturableDerivedClass: CapturableBaseClass, @unchecked Sendable { + let x: Int + + init(x: Int) { + self.x = x + super.init() + } + + required init(from decoder: any Decoder) throws { + let container = try decoder.singleValueContainer() + self.x = try container.decode(Int.self) + super.init() + } + + override func encode(to encoder: any Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(x) + } + } + + @Test("Capturing an instance of a subclass") + func captureSubclass() async { + let instance = CapturableDerivedClass(x: 123) + await #expect(exitsWith: .success) { [instance = instance as CapturableBaseClass] in + #expect((instance as AnyObject) is CapturableBaseClass) + // However, because the static type of `instance` is not Derived, we won't + // be able to cast it to Derived. + #expect(!((instance as AnyObject) is CapturableDerivedClass)) + } + await #expect(exitsWith: .success) { [instance = instance as CapturableDerivedClass] in + #expect((instance as AnyObject) is CapturableBaseClass) + #expect((instance as AnyObject) is CapturableDerivedClass) + #expect(instance.x == 123) + } + } +#endif } // MARK: - Fixtures From 64789e2f215277033c3b974c16ca5c02913a0142 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 15 Apr 2025 13:41:26 -0400 Subject: [PATCH 171/234] Fix typo in #1040 --- Sources/Testing/ExitTests/ExitTest.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 503e143c6..baabb7eaa 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -526,7 +526,7 @@ extension ExitTest { // Erase the environment variable so that it cannot accidentally be opened // twice (nor, in theory, affect the code of the exit test.) - Environment.setVariable(nil, named: "SWT_EXPERIMENTAL_BACKCHANNEL") + Environment.setVariable(nil, named: name) var fd: CInt? #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) From 78506a59e56333241dd8b938da9d2439515ccf5e Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 16 Apr 2025 01:06:30 -0400 Subject: [PATCH 172/234] Make sure ExitTest.CapturedValue is still sufficiently available for the compiler when SWT_NO_EXIT_TESTS is defined. --- .../ExitTests/ExitTest.CapturedValue.swift | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift index aeeb13818..d4c84e446 100644 --- a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift +++ b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift @@ -8,8 +8,10 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -#if !SWT_NO_EXIT_TESTS @_spi(Experimental) @_spi(ForToolsIntegrationOnly) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif extension ExitTest { /// A type representing a value captured by an exit test's body. /// @@ -46,6 +48,7 @@ extension ExitTest { /// encoded in the parent process and then updates the ``wrappedValue`` /// property of each element in the array before calling the exit test's body. public struct CapturedValue: Sendable { +#if !SWT_NO_EXIT_TESTS /// An enumeration of the different states a captured value can have. private enum _Kind: Sendable { /// The runtime value of the captured value is known. @@ -65,6 +68,7 @@ extension ExitTest { init(typeOnly type: (some Codable & Sendable).Type) { _kind = .typeOnly(type) } +#endif /// The underlying value captured by this instance at runtime. /// @@ -72,13 +76,18 @@ extension ExitTest { /// of this property is `nil` until the entry point sets it. public var wrappedValue: (any Codable & Sendable)? { get { +#if !SWT_NO_EXIT_TESTS if case let .wrappedValue(wrappedValue) = _kind { return wrappedValue } return nil +#else + fatalError("Unsupported") +#endif } set { +#if !SWT_NO_EXIT_TESTS let type = typeOfWrappedValue func validate(_ newValue: T, is expectedType: U.Type) { @@ -91,6 +100,9 @@ extension ExitTest { } else { _kind = .typeOnly(type) } +#else + fatalError("Unsupported") +#endif } } @@ -99,16 +111,21 @@ extension ExitTest { /// This type is known at compile time and is always available, even before /// this instance's ``wrappedValue`` property is set. public var typeOfWrappedValue: any (Codable & Sendable).Type { +#if !SWT_NO_EXIT_TESTS switch _kind { case let .wrappedValue(wrappedValue): type(of: wrappedValue) case let .typeOnly(type): type } +#else + fatalError("Unsupported") +#endif } } } +#if !SWT_NO_EXIT_TESTS // MARK: - Collection conveniences extension Array where Element == ExitTest.CapturedValue { @@ -166,3 +183,4 @@ extension Collection where Element == ExitTest.CapturedValue { } } #endif + From d75d0e3c421beae4e1533616647d372032e2b274 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 16 Apr 2025 21:58:34 -0500 Subject: [PATCH 173/234] Ensure the .whenEmbedded() build setting condition evaluates to false when building for non-Embedded without a fallback condition (#1081) This fixes an unintended side effect from #1043 where the `SWT_NO_LEGACY_TEST_DISCOVERY` compilation conditional was being applied when building for non-Embedded. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 80e6ce797..2bd95be6a 100644 --- a/Package.swift +++ b/Package.swift @@ -239,8 +239,9 @@ extension BuildSettingCondition { if let nonEmbeddedCondition = nonEmbeddedCondition() { nonEmbeddedCondition } else { - // The caller did not supply a fallback. - .when(platforms: []) + // The caller did not supply a fallback. Specify a non-existent platform + // to ensure this condition never matches. + .when(platforms: [.custom("DoesNotExist")]) } } else { // Enable unconditionally because the target is Embedded Swift. From 9342542e9cc6b4136d9ef2600c4d6f7205eba23c Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 17 Apr 2025 12:43:05 -0400 Subject: [PATCH 174/234] Relax the alignment requirement for `DiscoverableAsTestContent.Context`. (#1076) This PR allows `DiscoverableAsTestContent.Context` to be less-aligned than `UInt` so long as its stride remains the same. It also removes the sneaky conformance of `ExitTest` to `DiscoverableAsTestContent`, opting instead to use an internal type. Since the conformance to `DiscoverableAsTestContent` and the implementation of `__store()` form a closed system (where all type information is controlled by Swift Testing at runtime), we can do this without breaking any ABI. I've updated ABI/TestContent.md to remove some of the relevant implementation details. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --------- Co-authored-by: Stuart Montgomery --- Documentation/ABI/TestContent.md | 26 +++----- Sources/Testing/Discovery+Macro.swift | 9 --- Sources/Testing/ExitTests/ExitTest.swift | 61 +++++++++++++------ .../_TestDiscovery/TestContentRecord.swift | 2 +- Tests/TestingTests/DiscoveryTests.swift | 2 +- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index be2493530..2d0b9a4b8 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -149,25 +149,12 @@ The fourth argument to this function, `reserved`, is reserved for future use. Accessor functions should assume it is `0` and must not access it. The concrete Swift type of the value written to `outValue`, the type pointed to -by `type`, and the value pointed to by `hint` depend on the kind of record: +by `type`, and the value pointed to by `hint` depend on the kind of record. -- For test or suite declarations (kind `0x74657374`), the accessor produces a - structure of type `Testing.Test.Generator` that the testing library can use - to generate the corresponding test[^notAccessorSignature]. - - [^notAccessorSignature]: This level of indirection is necessary because - loading a test or suite declaration is an asynchronous operation, but C - functions cannot be `async`. - - Test content records of this kind do not specify a type for `hint`. Always - pass `nil`. - -- For exit test declarations (kind `0x65786974`), the accessor produces a - structure describing the exit test (of type `Testing.ExitTest`.) - - Test content records of this kind accept a `hint` of type `Testing.ExitTest.ID`. - They only produce a result if they represent an exit test declared with the - same ID (or if `hint` is `nil`.) +The record kinds defined by Swift Testing (kinds `0x74657374` and `0x65786974`) +make use of the `DiscoverableAsTestContent` protocol in the `_TestDiscovery` +module and do not publicly expose the types of their accessor functions' +arguments. Do not call the accessor functions for these records directly. > [!WARNING] > Calling code should use [`withUnsafeTemporaryAllocation(of:capacity:_:)`](https://developer.apple.com/documentation/swift/withunsafetemporaryallocation(of:capacity:_:)) @@ -274,7 +261,8 @@ extension FoodTruckDiagnostic: DiscoverableAsTestContent { ``` If you customize `TestContentContext`, be aware that the type you specify must -have the same stride and alignment as `UInt`. +have the same stride as `UInt` and must have an alignment less than or equal to +that of `UInt`. When you are done configuring your type's protocol conformance, you can then enumerate all test content records matching it as instances of diff --git a/Sources/Testing/Discovery+Macro.swift b/Sources/Testing/Discovery+Macro.swift index 97b925e55..35b276efe 100644 --- a/Sources/Testing/Discovery+Macro.swift +++ b/Sources/Testing/Discovery+Macro.swift @@ -8,15 +8,6 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) internal import _TestDiscovery - -/// A shadow declaration of `_TestDiscovery.DiscoverableAsTestContent` that -/// allows us to add public conformances to it without causing the -/// `_TestDiscovery` module to appear in `Testing.private.swiftinterface`. -/// -/// This protocol is not part of the public interface of the testing library. -protocol DiscoverableAsTestContent: _TestDiscovery.DiscoverableAsTestContent, ~Copyable {} - /// The type of the accessor function used to access a test content record. /// /// The signature of this function type must match that of the corresponding diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index baabb7eaa..2ad905379 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -66,15 +66,18 @@ public struct ExitTest: Sendable, ~Copyable { @_spi(ForToolsIntegrationOnly) public var id: ID - /// The body closure of the exit test. + /// An exit test body function. /// /// - Parameters: /// - exitTest: The exit test to which this body closure belongs. + fileprivate typealias Body = @Sendable (_ exitTest: inout Self) async throws -> Void + + /// The body closure of the exit test. /// /// Do not invoke this closure directly. Instead, invoke ``callAsFunction()`` /// to run the exit test. Running the exit test will always terminate the /// current process. - fileprivate var body: @Sendable (_ exitTest: inout Self) async throws -> Void = { _ in } + fileprivate var body: Body = { _ in } /// Storage for ``observedValues``. /// @@ -275,12 +278,34 @@ extension ExitTest { // MARK: - Discovery -extension ExitTest: DiscoverableAsTestContent { - fileprivate static var testContentKind: TestContentKind { - "exit" - } +extension ExitTest { + /// A type representing an exit test as a test content record. + fileprivate struct Record: Sendable, DiscoverableAsTestContent { + static var testContentKind: TestContentKind { + "exit" + } + + typealias TestContentAccessorHint = ID - fileprivate typealias TestContentAccessorHint = ID + /// The ID of the represented exit test. + var id: ExitTest.ID + + /// The body of the represented exit test. + var body: ExitTest.Body + + /// The set of values captured in the parent process before the exit test is + /// called. + var capturedValues = [CapturedValue]() + + /// Make the exit test represented by this instance. + /// + /// - Returns: A new exit test as represented by this instance. + func makeExitTest() -> ExitTest { + var exitTest = ExitTest(id: id, body: body) + exitTest.capturedValues = capturedValues + return exitTest + } + } /// Store the exit test into the given memory. /// @@ -305,9 +330,7 @@ extension ExitTest: DiscoverableAsTestContent { ) -> CBool where repeat each T: Codable & Sendable { #if !hasFeature(Embedded) // Check that the type matches. - let callerExpectedType = TypeInfo(describing: typeAddress.load(as: Any.Type.self)) - let selfType = TypeInfo(describing: Self.self) - guard callerExpectedType == selfType else { + guard typeAddress.load(as: Any.Type.self) == Record.self else { return false } #endif @@ -320,15 +343,15 @@ extension ExitTest: DiscoverableAsTestContent { // Wrap the body function in a thunk that decodes any captured state and // passes it along. - let body: @Sendable (inout Self) async throws -> Void = { exitTest in + let body: ExitTest.Body = { exitTest in let values: (repeat each T) = try exitTest.capturedValues.takeCapturedValues() try await body(repeat each values) } - // Construct and return the instance. - var exitTest = Self(id: id, body: body) - exitTest.capturedValues = Array(repeat (each T).self) - outValue.initializeMemory(as: Self.self, to: exitTest) + // Construct and return the record. + var record = Record(id: id, body: body) + record.capturedValues = Array(repeat (each T).self) + outValue.initializeMemory(as: Record.self, to: record) return true } } @@ -343,16 +366,16 @@ extension ExitTest { /// - Returns: The specified exit test function, or `nil` if no such exit test /// could be found. public static func find(identifiedBy id: ExitTest.ID) -> Self? { - for record in Self.allTestContentRecords() { - if let exitTest = record.load(withHint: id) { + for record in Record.allTestContentRecords() { + if let exitTest = record.load(withHint: id)?.makeExitTest() { return exitTest } } #if !SWT_NO_LEGACY_TEST_DISCOVERY // Call the legacy lookup function that discovers tests embedded in types. - for record in Self.allTypeMetadataBasedTestContentRecords() { - if let exitTest = record.load(withHint: id) { + for record in Record.allTypeMetadataBasedTestContentRecords() { + if let exitTest = record.load(withHint: id)?.makeExitTest() { return exitTest } } diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index d893664ee..9224fc2ea 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -52,7 +52,7 @@ extension DiscoverableAsTestContent where Self: ~Copyable { /// ([swift-#79667](https://github.com/swiftlang/swift/issues/79667)) fileprivate static func validateMemoryLayout() { precondition(MemoryLayout.stride == MemoryLayout.stride, "'\(self).TestContentContext' aka '\(TestContentContext.self)' must have the same stride as 'UInt'.") - precondition(MemoryLayout.alignment == MemoryLayout.alignment, "'\(self).TestContentContext' aka '\(TestContentContext.self)' must have the same alignment as 'UInt'.") + precondition(MemoryLayout.alignment <= MemoryLayout.alignment, "'\(self).TestContentContext' aka '\(TestContentContext.self)' must have an alignment less than or equal to that of 'UInt'.") } } diff --git a/Tests/TestingTests/DiscoveryTests.swift b/Tests/TestingTests/DiscoveryTests.swift index 2b53cd467..8ec185813 100644 --- a/Tests/TestingTests/DiscoveryTests.swift +++ b/Tests/TestingTests/DiscoveryTests.swift @@ -59,7 +59,7 @@ struct DiscoveryTests { #endif #if !SWT_NO_DYNAMIC_LINKING && hasFeature(SymbolLinkageMarkers) - struct MyTestContent: Testing.DiscoverableAsTestContent { + struct MyTestContent: DiscoverableAsTestContent { typealias TestContentAccessorHint = UInt32 var value: UInt32 From 390cb1c903a27e1ca95d89b03a5c82ae03a3b381 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 17 Apr 2025 13:08:26 -0500 Subject: [PATCH 175/234] Enable Library Evolution for package-based builds of the _TestDiscovery target (#1082) This enables Library Evolution (LE) for the `_TestDiscovery` target in Swift package-based builds. The CMake-based build is already configured to enable LE for this target, so this is just matching behavior. (Note that we already enable LE for several other targets which vend public API as of #951.) ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 2bd95be6a..3ce814138 100644 --- a/Package.swift +++ b/Package.swift @@ -171,7 +171,7 @@ let package = Package( dependencies: ["_TestingInternals",], exclude: ["CMakeLists.txt"], cxxSettings: .packageSettings, - swiftSettings: .packageSettings + swiftSettings: .packageSettings + .enableLibraryEvolution() ), // Cross-import overlays (not supported by Swift Package Manager) From 3abbb2be70e2ec7fb490b52617712a54e682830b Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 17 Apr 2025 18:16:27 -0500 Subject: [PATCH 176/234] Update README to reference 6.1 CI jobs instead of 6.0 and add Windows job (#1083) --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 01b8a4fd6..9f7eb424f 100644 --- a/README.md +++ b/README.md @@ -93,15 +93,15 @@ very best ideas, from anywhere, can help shape the future of testing in Swift. The table below describes the current level of support that Swift Testing has for various platforms: -| **Platform** | **CI Status (6.0)** | **CI Status (main)** | **Support Status** | +| **Platform** | **CI Status (6.1)** | **CI Status (main)** | **Support Status** | |---|:-:|:-:|---| -| **macOS** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.0-macos)](https://ci.swift.org/job/swift-testing-main-swift-6.0-macos/) | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-macos)](https://ci.swift.org/view/Swift%20Packages/job/swift-testing-main-swift-main-macos/) | Supported | +| **macOS** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-macos)](https://ci.swift.org/job/swift-testing-main-swift-6.1-macos/) | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-macos)](https://ci.swift.org/view/Swift%20Packages/job/swift-testing-main-swift-main-macos/) | Supported | | **iOS** | | | Supported | | **watchOS** | | | Supported | | **tvOS** | | | Supported | | **visionOS** | | | Supported | -| **Ubuntu 22.04** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.0-linux)](https://ci.swift.org/job/swift-testing-main-swift-6.0-linux/) | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-linux)](https://ci.swift.org/view/Swift%20Packages/job/swift-testing-main-swift-main-linux/) | Supported | -| **Windows** | | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-windows)](https://ci-external.swift.org/job/swift-testing-main-swift-main-windows/) | Supported | +| **Ubuntu 22.04** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-linux)](https://ci.swift.org/job/swift-testing-main-swift-6.1-linux/) | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-linux)](https://ci.swift.org/view/Swift%20Packages/job/swift-testing-main-swift-main-linux/) | Supported | +| **Windows** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-windows)](https://ci-external.swift.org/view/all/job/swift-testing-main-swift-6.1-windows/) | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-windows)](https://ci-external.swift.org/job/swift-testing-main-swift-main-windows/) | Supported | | **Wasm** | | | Experimental | ### Works with XCTest From 05ae4240d2d58de4c9ca5e00ab09d19ddc7f7b27 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 18 Apr 2025 19:54:48 -0500 Subject: [PATCH 177/234] Enable experimental 'AllowUnsafeAttribute' feature to continue supporting 6.1 development snapshot toolchains (#1084) This fixes a build failure when attempting to build the `main` branch using a 6.1 development snapshot toolchain. This failure was introduced by #1069, which added usage of the new `@unsafe` attribute, and the failure was revealed when we set up the 6.1 CI jobs in #1083. Here are some relevant related Swift PRs which give context around these changes: - https://github.com/swiftlang/swift/pull/75413 - https://github.com/swiftlang/swift/pull/79645 See the code comment for more details. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Package.swift b/Package.swift index 3ce814138..44116b6d1 100644 --- a/Package.swift +++ b/Package.swift @@ -277,6 +277,17 @@ extension Array where Element == PackageDescription.SwiftSetting { // proposal via Swift Evolution. .enableExperimentalFeature("SymbolLinkageMarkers"), + // This setting is no longer needed when building with a 6.2 or later + // toolchain now that SE-0458 has been accepted and implemented, but it is + // needed in order to preserve support for building with 6.1 development + // snapshot toolchains. (Production 6.1 toolchains can build the testing + // library even without this setting since this experimental feature is + // _suppressible_.) This setting can be removed once the minimum supported + // toolchain for building the testing library is ≥ 6.2. It is not needed + // in the CMake settings since that is expected to build using a + // new-enough toolchain. + .enableExperimentalFeature("AllowUnsafeAttribute"), + // When building as a package, the macro plugin always builds as an // executable rather than a library. .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), From 463a7a7fedba1fa8a2e3fbe93fe26388dadaedc6 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 22 Apr 2025 14:00:27 -0500 Subject: [PATCH 178/234] Reserve the 'play' content kind in _TestDiscovery (#1090) This defines the `'play'` test content kind in the `TestContentKind` enum and corresponding documentation. ### Motivation: This is intended to "reserve" this FourCC code for eventual use by the [swift-play-experimental](https://github.com/apple/swift-play-experimental) project, which has adopted the `_TestDiscovery` library from this package. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes #1089 Fixes rdar://147585572 --- Documentation/ABI/TestContent.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/TestContent.md b/Documentation/ABI/TestContent.md index 2d0b9a4b8..c14eb9ec6 100644 --- a/Documentation/ABI/TestContent.md +++ b/Documentation/ABI/TestContent.md @@ -100,9 +100,10 @@ record's kind is a 32-bit unsigned value. The following kinds are defined: | `0x00000000` | – | Reserved (**do not use**) | | `0x74657374` | `'test'` | Test or suite declaration | | `0x65786974` | `'exit'` | Exit test | +| `0x706c6179` | `'play'` | [Playground](https://github.com/apple/swift-play-experimental) | - + If a test content record's `kind` field equals `0x00000000`, the values of all other fields in that record are undefined. From 534d7812d56c5a78ff833c9123d7427308beba24 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 23 Apr 2025 11:32:09 -0500 Subject: [PATCH 179/234] Update CODEOWNERS file (#1094) This PR updates the `CODEOWNERS` file to remove @SeanROlszewski as a code owner. We sincerely thank him for his valuable contributions and past collaboration! ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 546238eb3..e2590577e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,4 +8,4 @@ # See https://swift.org/CONTRIBUTORS.txt for Swift project authors # -* @stmontgomery @grynspan @briancroom @SeanROlszewski @suzannaratcliff +* @stmontgomery @grynspan @briancroom @suzannaratcliff From 696d9b9d672461c4812a78446a5223df31a5ef41 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 23 Apr 2025 22:23:44 -0500 Subject: [PATCH 180/234] Fix crash in ConsoleOutputRecorder when an issue has a comment with an empty string (#1091) This fixes a crash which can occur when recording an issue with a comment whose string is empty. It also adds some test coverage of this scenario and other "uncommon" comment examples. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes rdar://149482060 --- .../Event.ConsoleOutputRecorder.swift | 2 +- Tests/TestingTests/EventRecorderTests.swift | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index ea48e7ad1..80e68c609 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -323,7 +323,7 @@ extension Event.ConsoleOutputRecorder { // text instead of just the symbol. Details may be multi-line messages, // so split the message on newlines and indent all lines to align them // to the indentation provided by the symbol. - var lines = message.stringValue.split(whereSeparator: \.isNewline) + var lines = message.stringValue.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) lines = CollectionOfOne(lines[0]) + lines.dropFirst().map { line in "\(padding) \(line)" } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 8ac7f6728..690fd416f 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -257,6 +257,30 @@ struct EventRecorderTests { } #endif + @Test( + "Uncommonly-formatted comments", + .bug("rdar://149482060"), + arguments: [ + "", // Empty string + "\n\n\n", // Only newlines + "\nFoo\n\nBar\n\n\nBaz\n", // Newlines interspersed with non-empty strings + ] + ) + func uncommonComments(text: String) async throws { + let stream = Stream() + + var configuration = Configuration() + configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true + let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) + configuration.eventHandler = { event, context in + eventRecorder.record(event, in: context) + } + + await Test { + Issue.record(Comment(rawValue: text) /* empty */) + }.run(configuration: configuration) + } + @available(_regexAPI, *) @Test("Issue counts are omitted on a successful test") func issueCountOmittedForPassingTest() async throws { From cacb295938a1794812ecdf771b78f0cde2177dd1 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 23 Apr 2025 22:29:23 -0500 Subject: [PATCH 181/234] Fix build failure when building with pre-6.2 toolchain due to unrecognized `unsafe` keyword (#1093) This fixes another bit of fallout from #1069 when building this project's test targets with a 6.1 (or any pre-6.2) toolchain. The `unsafe` keyword was introduced in 6.2 as part of [SE-0458: Opt-in Strict Memory Safety Checking](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0458-strict-memory-safety.md). Older toolchains are not aware of it, so the fix is to avoid emitting expressions involving that keyword when the macro plugin has been built using an older toolchain. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Support/EffectfulExpressionHandling.swift | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift index f67ca40ee..a70f4ae0f 100644 --- a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift +++ b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift @@ -111,7 +111,13 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp let needAwait = effectfulKeywords.contains(.await) && !expr.is(AwaitExprSyntax.self) let needTry = effectfulKeywords.contains(.try) && !expr.is(TryExprSyntax.self) + + // The 'unsafe' keyword was introduced in 6.2 as part of SE-0458. Older + // toolchains are not aware of it, so avoid emitting expressions involving + // that keyword when the macro has been built using an older toolchain. +#if compiler(>=6.2) let needUnsafe = effectfulKeywords.contains(.unsafe) && !expr.is(UnsafeExprSyntax.self) +#endif // First, add thunk function calls. if needAwait { @@ -120,9 +126,11 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp if needTry { expr = _makeCallToEffectfulThunk(.identifier("__requiringTry"), passing: expr) } +#if compiler(>=6.2) if needUnsafe { expr = _makeCallToEffectfulThunk(.identifier("__requiringUnsafe"), passing: expr) } +#endif // Then add keyword expressions. (We do this separately so we end up writing // `try await __r(__r(self))` instead of `try __r(await __r(self))` which is @@ -143,6 +151,7 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp ) ) } +#if compiler(>=6.2) if needUnsafe { expr = ExprSyntax( UnsafeExprSyntax( @@ -151,6 +160,7 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp ) ) } +#endif expr.leadingTrivia = originalExpr.leadingTrivia expr.trailingTrivia = originalExpr.trailingTrivia From b5fa554b5af01c87455c92d0e0db88b84667c20f Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 24 Apr 2025 10:46:11 -0500 Subject: [PATCH 182/234] Include code comments before expectations which are preceded by try/await in recorded issues (#1092) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes an issue where code comments placed before an expectation like `#expect()` which has effect introducer keywords like `try` or `await` are ignored, and ensures they are included in the recorded issue if the expectation fails. Consider this example of two failing expectations: ```swift // Uh oh! #expect(try x() == 2) // Uh oh! try #expect(x() == 2) ``` Prior to this PR, if `x()` returned a value other than `2`, there would be two issues recorded, but the second one would not have the comment `“Uh oh!”` because from the macro’s perspective, that code comment was on the `try` keyword and it could only see trivia associated with `#expect()`. Now, with the recent swift-syntax fix from https://github.com/swiftlang/swift-syntax/pull/3037, the `try` keyword and its associated trivia can be included and this bug can be fixed. We recently adopted a new-enough swift-syntax in #1069, so the only fix needed is to adopt `lexicalContext` for this new purpose in our macro. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/TestingMacros/ConditionMacro.swift | 11 +++- .../Support/EffectfulExpressionHandling.swift | 17 +++++- .../ConditionMacroTests.swift | 53 +++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 326522858..e21938041 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -157,8 +157,17 @@ extension ConditionMacro { expandedFunctionName = conditionArgument.expandedFunctionName } - // Capture any comments as well (either in source or as a macro argument.) + // Capture any comments as well -- either in source, preceding the + // expression macro or one of its lexical context nodes, or as an argument + // to the macro. let commentsArrayExpr = ArrayExprSyntax { + // Lexical context is ordered innermost-to-outermost, so reverse it to + // maintain the expected order. + for lexicalSyntaxNode in context.lexicalContext.trailingEffectExpressions.reversed() { + for commentTraitExpr in createCommentTraitExprs(for: lexicalSyntaxNode) { + ArrayElementSyntax(expression: commentTraitExpr) + } + } for commentTraitExpr in createCommentTraitExprs(for: macro) { ArrayElementSyntax(expression: commentTraitExpr) } diff --git a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift index a70f4ae0f..494d2fcfc 100644 --- a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift +++ b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift @@ -12,7 +12,7 @@ import SwiftSyntax import SwiftSyntaxBuilder import SwiftSyntaxMacros -// MARK: - Finding effect keywords +// MARK: - Finding effect keywords and expressions /// A syntax visitor class that looks for effectful keywords in a given /// expression. @@ -69,6 +69,21 @@ func findEffectKeywords(in node: some SyntaxProtocol, context: some MacroExpansi return effectFinder.effectKeywords } +extension BidirectionalCollection { + /// The suffix of syntax nodes in this collection which are effectful + /// expressions, such as those for `try` or `await`. + var trailingEffectExpressions: some Collection { + reversed() + .prefix { node in + // This could be simplified if/when swift-syntax introduces a protocol + // which all effectful expression syntax node types conform to. + // See https://github.com/swiftlang/swift-syntax/issues/3040 + node.is(TryExprSyntax.self) || node.is(AwaitExprSyntax.self) || node.is(UnsafeExprSyntax.self) + } + .reversed() + } +} + // MARK: - Inserting effect keywords/thunks /// Make a function call expression to an effectful thunk function provided by diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index cd1333941..67531dabf 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -240,6 +240,59 @@ struct ConditionMacroTests { // Capture me Testing.__checkValue(try x(), expression: .__fromSyntaxNode("try x()"), comments: [.__line("// Capture me")], isRequired: false, sourceLocation: Testing.SourceLocation.__here()).__expected() """, + + """ + // Capture me + try #expect(x) + """: + """ + // Capture me + try Testing.__checkValue(x, expression: .__fromSyntaxNode("x"), comments: [.__line("// Capture me")], isRequired: false, sourceLocation: Testing.SourceLocation.__here()).__expected() + """, + + """ + // Capture me + await #expect(x) + """: + """ + // Capture me + await Testing.__checkValue(x, expression: .__fromSyntaxNode("x"), comments: [.__line("// Capture me")], isRequired: false, sourceLocation: Testing.SourceLocation.__here()).__expected() + """, + + """ + // Ignore me + + // Comment for try + try + // Comment for await + await + // Comment for expect + #expect(x) + """: + """ + // Comment for try + try + // Comment for await + await + // Comment for expect + Testing.__checkValue(x, expression: .__fromSyntaxNode("x"), comments: [.__line("// Comment for try"), .__line("// Comment for await"), .__line("// Comment for expect")], isRequired: false, sourceLocation: Testing.SourceLocation.__here()).__expected() + """, + + """ + // Ignore me + func example() { + // Capture me + #expect(x()) + } + """: + """ + func example() { + // Capture me + Testing.__checkFunctionCall((), calling: { _ in + x() + }, expression: .__fromFunctionCall(nil, "x"), comments: [.__line("// Capture me")], isRequired: false, sourceLocation: Testing.SourceLocation.__here()).__expected() + } + """, ] ) func commentCapture(input: String, expectedOutput: String) throws { From 32e231cf58f97a3d1af05a448dc67d4d835164be Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 24 Apr 2025 12:43:43 -0500 Subject: [PATCH 183/234] Fix image URL for Swift 6.1 Windows CI badge in README (#1096) Fix a small oversight in the URL of the CI status badge for Swift 6.1 Windows, which I added in #1083. The URL points to the wrong CI server domain, so it says "Not run" instead of "Passing". ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f7eb424f..8df465217 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ for various platforms: | **tvOS** | | | Supported | | **visionOS** | | | Supported | | **Ubuntu 22.04** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-linux)](https://ci.swift.org/job/swift-testing-main-swift-6.1-linux/) | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-linux)](https://ci.swift.org/view/Swift%20Packages/job/swift-testing-main-swift-main-linux/) | Supported | -| **Windows** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-windows)](https://ci-external.swift.org/view/all/job/swift-testing-main-swift-6.1-windows/) | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-windows)](https://ci-external.swift.org/job/swift-testing-main-swift-main-windows/) | Supported | +| **Windows** | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-windows)](https://ci-external.swift.org/view/all/job/swift-testing-main-swift-6.1-windows/) | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-windows)](https://ci-external.swift.org/job/swift-testing-main-swift-main-windows/) | Supported | | **Wasm** | | | Experimental | ### Works with XCTest From 2f456650dc91aae9b5545b1028efcf22c0116ae5 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Thu, 24 Apr 2025 15:42:46 -0400 Subject: [PATCH 184/234] Add withKnownIssue comments to known Issues (#1014) Add withKnownIssue comments to known Issues ### Motivation: The Comment passed to withKnownIssue() is currently only used when a known issue does not occur. When a known issue does occur, it is marked isKnown but does not contain any record of the withKnownIssue() comment, making it harder to understand why the issue was considered to be known, especially if there are multiple nested withKnownIssue() calls. ### Modifications: When an issue is marked "known" by a `withKnownIssue()` call, the recorded issue will now have a new `knownIssueContext` property containing the comment passed to `withKnownIssue()` (if any). This comment will be included in the `messages` array of the `ABI.EncodedEvent` that represents the issue. If the issue is recorded within multiple nested `withKnownIssue()` calls, `knownIssueContext` corresponds to the innermost matching call. The `Issue.isKnown` setter is now deprecated and a no-op. When an error is thrown within a `withKnownIssue()` call, the `Issue` passed to the issue matcher used to have the known issue comment in `Issue.comments`. That has now been removed; Issues for thrown errors now have no comments at all when passed to the issue matcher. This matches Issues for thrown errors that occur outside of `withKnownIssue()` calls. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --------- Co-authored-by: Jonathan Grynspan Co-authored-by: Stuart Montgomery --- .../Event.HumanReadableOutputRecorder.swift | 19 +- Sources/Testing/ExitTests/ExitTest.swift | 6 +- Sources/Testing/Issues/Issue+Recording.swift | 12 +- Sources/Testing/Issues/Issue.swift | 22 +- Sources/Testing/Issues/KnownIssue.swift | 100 +++++--- Tests/TestingTests/EventRecorderTests.swift | 61 +++++ Tests/TestingTests/KnownIssueTests.swift | 232 +++++++++++++++++- 7 files changed, 402 insertions(+), 50 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index f585495a9..6abb71442 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -93,10 +93,20 @@ extension Event.HumanReadableOutputRecorder { /// - Parameters: /// - comments: The comments that should be formatted. /// - /// - Returns: A formatted string representing `comments`, or `nil` if there - /// are none. + /// - Returns: An array of formatted messages representing `comments`, or an + /// empty array if there are none. private func _formattedComments(_ comments: [Comment]) -> [Message] { - comments.map { Message(symbol: .details, stringValue: $0.rawValue) } + comments.map(_formattedComment) + } + + /// Get a string representing a single comment, formatted for output. + /// + /// - Parameters: + /// - comment: The comment that should be formatted. + /// + /// - Returns: A formatted message representing `comment`. + private func _formattedComment(_ comment: Comment) -> Message { + Message(symbol: .details, stringValue: comment.rawValue) } /// Get a string representing the comments attached to a test, formatted for @@ -449,6 +459,9 @@ extension Event.HumanReadableOutputRecorder { additionalMessages.append(Message(symbol: .difference, stringValue: differenceDescription)) } additionalMessages += _formattedComments(issue.comments) + if let knownIssueComment = issue.knownIssueContext?.comment { + additionalMessages.append(_formattedComment(knownIssueComment)) + } if verbosity > 0, case let .expectationFailed(expectation) = issue.kind { let expression = expectation.evaluatedExpression diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 2ad905379..14d10a04b 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -930,7 +930,11 @@ extension ExitTest { sourceLocation: issue.sourceLocation ) var issueCopy = Issue(kind: issueKind, comments: comments, sourceContext: sourceContext) - issueCopy.isKnown = issue.isKnown + if issue.isKnown { + // The known issue comment, if there was one, is already included in + // the `comments` array above. + issueCopy.knownIssueContext = Issue.KnownIssueContext() + } issueCopy.record() } } diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index bd3e9a3bb..b79e94269 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -9,14 +9,6 @@ // extension Issue { - /// The known issue matcher, as set by `withKnownIssue()`, associated with the - /// current task. - /// - /// If there is no call to `withKnownIssue()` executing on the current task, - /// the value of this property is `nil`. - @TaskLocal - static var currentKnownIssueMatcher: KnownIssueMatcher? - /// Record this issue by wrapping it in an ``Event`` and passing it to the /// current event handler. /// @@ -38,9 +30,9 @@ extension Issue { // If this issue matches via the known issue matcher, set a copy of it to be // known and record the copy instead. - if !isKnown, let issueMatcher = Self.currentKnownIssueMatcher, issueMatcher(self) { + if !isKnown, let context = KnownIssueScope.current?.matcher(self) { var selfCopy = self - selfCopy.isKnown = true + selfCopy.knownIssueContext = context return selfCopy.record(configuration: configuration) } diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 53364c151..4a17cb945 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -129,9 +129,29 @@ public struct Issue: Sendable { @_spi(ForToolsIntegrationOnly) public var sourceContext: SourceContext + /// A type representing a + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` call + /// that matched an issue. + @_spi(ForToolsIntegrationOnly) + public struct KnownIssueContext: Sendable { + /// The comment that was passed to + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)``. + public var comment: Comment? + } + + /// A ``KnownIssueContext-swift.struct`` representing the + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` call + /// that matched this issue, if any. + @_spi(ForToolsIntegrationOnly) + public var knownIssueContext: KnownIssueContext? = nil + /// Whether or not this issue is known to occur. @_spi(ForToolsIntegrationOnly) - public var isKnown: Bool = false + public var isKnown: Bool { + get { knownIssueContext != nil } + @available(*, deprecated, message: "Setting this property has no effect.") + set {} + } /// Initialize an issue instance with the specified details. /// diff --git a/Sources/Testing/Issues/KnownIssue.swift b/Sources/Testing/Issues/KnownIssue.swift index f59e9d422..f59185388 100644 --- a/Sources/Testing/Issues/KnownIssue.swift +++ b/Sources/Testing/Issues/KnownIssue.swift @@ -8,25 +8,62 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// Combine an instance of ``KnownIssueMatcher`` with any previously-set one. -/// -/// - Parameters: -/// - issueMatcher: A function to invoke when an issue occurs that is used to -/// determine if the issue is known to occur. -/// - matchCounter: The counter responsible for tracking the number of matches -/// found with `issueMatcher`. -/// -/// - Returns: A new instance of ``Configuration`` or `nil` if there was no -/// current configuration set. -private func _combineIssueMatcher(_ issueMatcher: @escaping KnownIssueMatcher, matchesCountedBy matchCounter: Locked) -> KnownIssueMatcher { - let oldIssueMatcher = Issue.currentKnownIssueMatcher - return { issue in - if issueMatcher(issue) || true == oldIssueMatcher?(issue) { - matchCounter.increment() - return true +/// A type that represents an active +/// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` +/// call and any parent calls. +/// +/// A stack of these is stored in `KnownIssueScope.current`. +struct KnownIssueScope: Sendable { + /// A function which determines if an issue matches a known issue scope or + /// any of its ancestor scopes. + /// + /// - Parameters: + /// - issue: The issue being matched. + /// + /// - Returns: A known issue context containing information about the known + /// issue, if the issue is considered "known" by this known issue scope or any + /// ancestor scope, or `nil` otherwise. + typealias Matcher = @Sendable (_ issue: Issue) -> Issue.KnownIssueContext? + + /// The matcher function for this known issue scope. + var matcher: Matcher + + /// The number of issues this scope and its ancestors have matched. + let matchCounter: Locked + + /// Create a new ``KnownIssueScope`` by combining a new issue matcher with + /// any already-active scope. + /// + /// - Parameters: + /// - parent: The context that should be checked next if `issueMatcher` + /// fails to match an issue. Defaults to ``KnownIssueScope.current``. + /// - issueMatcher: A function to invoke when an issue occurs that is used + /// to determine if the issue is known to occur. + /// - context: The context to be associated with issues matched by + /// `issueMatcher`. + init(parent: KnownIssueScope? = .current, issueMatcher: @escaping KnownIssueMatcher, context: Issue.KnownIssueContext) { + let matchCounter = Locked(rawValue: 0) + self.matchCounter = matchCounter + matcher = { issue in + let matchedContext = if issueMatcher(issue) { + context + } else { + parent?.matcher(issue) + } + if matchedContext != nil { + matchCounter.increment() + } + return matchedContext } - return false } + + /// The active known issue scope for the current task, if any. + /// + /// If there is no call to + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` + /// executing on the current task, the value of this property is `nil`. + @TaskLocal + static var current: KnownIssueScope? } /// Check if an error matches using an issue-matching function, and throw it if @@ -34,18 +71,17 @@ private func _combineIssueMatcher(_ issueMatcher: @escaping KnownIssueMatcher, m /// /// - Parameters: /// - error: The error to test. -/// - issueMatcher: A function to which `error` is passed (after boxing it in -/// an instance of ``Issue``) to determine if it is known to occur. +/// - scope: The known issue scope that is processing the error. /// - comment: An optional comment to apply to any issues generated by this /// function. /// - sourceLocation: The source location to which the issue should be /// attributed. -private func _matchError(_ error: any Error, using issueMatcher: KnownIssueMatcher, comment: Comment?, sourceLocation: SourceLocation) throws { +private func _matchError(_ error: any Error, in scope: KnownIssueScope, comment: Comment?, sourceLocation: SourceLocation) throws { let sourceContext = SourceContext(backtrace: Backtrace(forFirstThrowOf: error), sourceLocation: sourceLocation) - var issue = Issue(kind: .errorCaught(error), comments: Array(comment), sourceContext: sourceContext) - if issueMatcher(issue) { + var issue = Issue(kind: .errorCaught(error), comments: [], sourceContext: sourceContext) + if let context = scope.matcher(issue) { // It's a known issue, so mark it as such before recording it. - issue.isKnown = true + issue.knownIssueContext = context issue.record() } else { // Rethrow the error, allowing the caller to catch it or for it to propagate @@ -184,18 +220,17 @@ public func withKnownIssue( guard precondition() else { return try body() } - let matchCounter = Locked(rawValue: 0) - let issueMatcher = _combineIssueMatcher(issueMatcher, matchesCountedBy: matchCounter) + let scope = KnownIssueScope(issueMatcher: issueMatcher, context: Issue.KnownIssueContext(comment: comment)) defer { if !isIntermittent { - _handleMiscount(by: matchCounter, comment: comment, sourceLocation: sourceLocation) + _handleMiscount(by: scope.matchCounter, comment: comment, sourceLocation: sourceLocation) } } - try Issue.$currentKnownIssueMatcher.withValue(issueMatcher) { + try KnownIssueScope.$current.withValue(scope) { do { try body() } catch { - try _matchError(error, using: issueMatcher, comment: comment, sourceLocation: sourceLocation) + try _matchError(error, in: scope, comment: comment, sourceLocation: sourceLocation) } } } @@ -304,18 +339,17 @@ public func withKnownIssue( guard await precondition() else { return try await body() } - let matchCounter = Locked(rawValue: 0) - let issueMatcher = _combineIssueMatcher(issueMatcher, matchesCountedBy: matchCounter) + let scope = KnownIssueScope(issueMatcher: issueMatcher, context: Issue.KnownIssueContext(comment: comment)) defer { if !isIntermittent { - _handleMiscount(by: matchCounter, comment: comment, sourceLocation: sourceLocation) + _handleMiscount(by: scope.matchCounter, comment: comment, sourceLocation: sourceLocation) } } - try await Issue.$currentKnownIssueMatcher.withValue(issueMatcher) { + try await KnownIssueScope.$current.withValue(scope) { do { try await body() } catch { - try _matchError(error, using: issueMatcher, comment: comment, sourceLocation: sourceLocation) + try _matchError(error, in: scope, comment: comment, sourceLocation: sourceLocation) } } } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 690fd416f..18f70186a 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -521,6 +521,35 @@ struct EventRecorderTests { recorder.record(Event(.runEnded, testID: nil, testCaseID: nil), in: context) } } + + @Test( + "HumanReadableOutputRecorder includes known issue comment in messages array", + arguments: [ + ("recordWithoutKnownIssueComment()", ["#expect comment"]), + ("recordWithKnownIssueComment()", ["#expect comment", "withKnownIssue comment"]), + ("throwWithoutKnownIssueComment()", []), + ("throwWithKnownIssueComment()", ["withKnownIssue comment"]), + ] + ) + func knownIssueComments(testName: String, expectedComments: [String]) async throws { + var configuration = Configuration() + let recorder = Event.HumanReadableOutputRecorder() + let messages = Locked<[Event.HumanReadableOutputRecorder.Message]>(rawValue: []) + configuration.eventHandler = { event, context in + guard case .issueRecorded = event.kind else { return } + messages.withLock { + $0.append(contentsOf: recorder.record(event, in: context)) + } + } + + await runTestFunction(named: testName, in: PredictablyFailingKnownIssueTests.self, configuration: configuration) + + // The first message is something along the lines of "Test foo recorded a + // known issue" and includes a source location, so is inconvenient to + // include in our expectation here. + let actualComments = messages.rawValue.dropFirst().map(\.stringValue) + #expect(actualComments == expectedComments) + } } // MARK: - Fixtures @@ -663,3 +692,35 @@ struct EventRecorderTests { #expect(arg > 0) } } + +@Suite(.hidden) struct PredictablyFailingKnownIssueTests { + @Test(.hidden) + func recordWithoutKnownIssueComment() { + withKnownIssue { + #expect(Bool(false), "#expect comment") + } + } + + @Test(.hidden) + func recordWithKnownIssueComment() { + withKnownIssue("withKnownIssue comment") { + #expect(Bool(false), "#expect comment") + } + } + + @Test(.hidden) + func throwWithoutKnownIssueComment() { + withKnownIssue { + struct TheError: Error {} + throw TheError() + } + } + + @Test(.hidden) + func throwWithKnownIssueComment() { + withKnownIssue("withKnownIssue comment") { + struct TheError: Error {} + throw TheError() + } + } +} diff --git a/Tests/TestingTests/KnownIssueTests.swift b/Tests/TestingTests/KnownIssueTests.swift index 733fbbf01..9174bfdd8 100644 --- a/Tests/TestingTests/KnownIssueTests.swift +++ b/Tests/TestingTests/KnownIssueTests.swift @@ -38,7 +38,7 @@ final class KnownIssueTests: XCTestCase { await fulfillment(of: [issueRecorded], timeout: 0.0) } - func testKnownIssueWithComment() async { + func testKnownIssueNotRecordedWithComment() async { let issueRecorded = expectation(description: "Issue recorded") var configuration = Configuration() @@ -52,8 +52,8 @@ final class KnownIssueTests: XCTestCase { return } - XCTAssertEqual(issue.comments.first, "With Known Issue Comment") XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.comments, ["With Known Issue Comment"]) } await Test { @@ -63,6 +63,234 @@ final class KnownIssueTests: XCTestCase { await fulfillment(of: [issueRecorded], timeout: 0.0) } + func testKnownIssueRecordedWithComment() async { + let issueMatched = expectation(description: "Issue matched") + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue Comment"]) + XCTAssertEqual(issue.knownIssueContext?.comment, "With Known Issue Comment") + } + + await Test { + withKnownIssue("With Known Issue Comment") { + Issue.record("Issue Comment") + } matching: { issue in + // The issue isn't yet considered known since we haven't matched it yet. + XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue Comment"]) + XCTAssertNil(issue.knownIssueContext) + issueMatched.fulfill() + return true + } + }.run(configuration: configuration) + + await fulfillment(of: [issueMatched, issueRecorded], timeout: 0.0) + } + + func testThrownKnownIssueRecordedWithComment() async { + let issueMatched = expectation(description: "Issue matched") + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["With Known Issue Comment"]) + } + + struct E: Error {} + + await Test { + try withKnownIssue("With Known Issue Comment") { + throw E() + } matching: { issue in + // The issue isn't yet considered known since we haven't matched it yet. + XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.comments, []) + XCTAssertNil(issue.knownIssueContext) + issueMatched.fulfill() + return true + } + }.run(configuration: configuration) + + await fulfillment(of: [issueMatched, issueRecorded], timeout: 0.0) + } + + func testKnownIssueRecordedWithNoComment() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue Comment"]) + } + + await Test { + withKnownIssue { + Issue.record("Issue Comment") + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + + func testKnownIssueRecordedWithInnermostMatchingComment() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue B"]) + XCTAssertEqual(issue.knownIssueContext?.comment, "Inner Contains B") + } + + await Test { + withKnownIssue("Contains A", isIntermittent: true) { + withKnownIssue("Outer Contains B", isIntermittent: true) { + withKnownIssue("Inner Contains B") { + withKnownIssue("Contains C", isIntermittent: true) { + Issue.record("Issue B") + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("C") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("A") } + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + + func testThrownKnownIssueRecordedWithInnermostMatchingComment() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Inner Is B", "B"]) + } + + struct A: Error {} + struct B: Error {} + struct C: Error {} + + await Test { + try withKnownIssue("Is A", isIntermittent: true) { + try withKnownIssue("Outer Is B", isIntermittent: true) { + try withKnownIssue("Inner Is B") { + try withKnownIssue("Is C", isIntermittent: true) { + throw B() + } matching: { issue in + issue.error is C + } + } matching: { issue in + issue.error is B + } + } matching: { issue in + issue.error is B + } + } matching: { issue in + issue.error is A + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + + func testKnownIssueRecordedWithNoCommentOnInnermostMatch() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue B"]) + } + + await Test { + withKnownIssue("Contains A", isIntermittent: true) { + withKnownIssue("Outer Contains B", isIntermittent: true) { + withKnownIssue { // No comment here on the withKnownIssue that will actually match. + withKnownIssue("Contains C", isIntermittent: true) { + Issue.record("Issue B") + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("C") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("A") } + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + func testIssueIsKnownPropertyIsSetCorrectlyWithCustomIssueMatcher() async { struct MyError: Error {} From 6023a5fb9a6ff4f67790b7a9065ddf449ffd7f54 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 25 Apr 2025 15:39:38 -0500 Subject: [PATCH 185/234] Introduce issue handling trait (as SPI) (#1080) This introduces a new trait type named `IssueHandlingTrait` as SPI. It allows observing, transforming, or filtering the issue(s) recorded during the test it's applied to. Here's a contrived example: ```swift @Test(.transformIssues { issue in var issue = issue issue.comments.append("A comparison of two literals") return issue // Or, return `nil` to suppress the issue }) func example() { #expect(1 == 2) } ``` ### Motivation: Sometimes it can be useful to customize an issue recorded during a test. For example, you might wish to add supplemental information to it, such by adding comments. Another example of this could be adding an attachment to an issue, which was a capability mentioned as a [future direction](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0009-attachments.md#future-directions) in [ST-0009 Attachments](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0009-attachments.md). Other times, you might wish to suppress an issue which is later determined to be irrelevant or cannot be marked as a [known issue](https://swiftpackageindex.com/swiftlang/swift-testing/main/documentation/testing/known-issues). Or, you may simply want to be notified that an issue was recorded, to react to it in some other way while still recording the issue normally. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Resolves rdar://140144041 --- Sources/Testing/CMakeLists.txt | 1 + Sources/Testing/Running/Runner.swift | 58 +++++- Sources/Testing/Testing.docc/Traits.md | 8 + .../Testing/Traits/IssueHandlingTrait.swift | 167 +++++++++++++++ .../Traits/IssueHandlingTraitTests.swift | 197 ++++++++++++++++++ 5 files changed, 426 insertions(+), 5 deletions(-) create mode 100644 Sources/Testing/Traits/IssueHandlingTrait.swift create mode 100644 Tests/TestingTests/Traits/IssueHandlingTraitTests.swift diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index b4e865427..6eca45dde 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -97,6 +97,7 @@ add_library(Testing Traits/ConditionTrait.swift Traits/ConditionTrait+Macro.swift Traits/HiddenTrait.swift + Traits/IssueHandlingTrait.swift Traits/ParallelizationTrait.swift Traits/Tags/Tag.Color.swift Traits/Tags/Tag.Color+Loading.swift diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 16eff103f..8520d1aaf 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -91,10 +91,11 @@ extension Runner { return try await body() } - // Construct a recursive function that invokes each trait's ``execute(_:for:testCase:)`` - // function. The order of the sequence is reversed so that the last trait is - // the one that invokes body, then the second-to-last invokes the last, etc. - // and ultimately the first trait is the first one to be invoked. + // Construct a recursive function that invokes each scope provider's + // `provideScope(for:testCase:performing:)` function. The order of the + // sequence is reversed so that the last trait is the one that invokes body, + // then the second-to-last invokes the last, etc. and ultimately the first + // trait is the first one to be invoked. let executeAllTraits = test.traits.lazy .reversed() .compactMap { $0.scopeProvider(for: test, testCase: testCase) } @@ -108,6 +109,41 @@ extension Runner { try await executeAllTraits() } + /// Apply the custom scope from any issue handling traits for the specified + /// test. + /// + /// - Parameters: + /// - test: The test being run, for which to apply its issue handling traits. + /// - body: A function to execute within the scope provided by the test's + /// issue handling traits. + /// + /// - Throws: Whatever is thrown by `body` or by any of the traits' provide + /// scope function calls. + private static func _applyIssueHandlingTraits(for test: Test, _ body: @escaping @Sendable () async throws -> Void) async throws { + // If the test does not have any traits, exit early to avoid unnecessary + // heap allocations below. + if test.traits.isEmpty { + return try await body() + } + + // Construct a recursive function that invokes each issue handling trait's + // `provideScope(performing:)` function. The order of the sequence is + // reversed so that the last trait is the one that invokes body, then the + // second-to-last invokes the last, etc. and ultimately the first trait is + // the first one to be invoked. + let executeAllTraits = test.traits.lazy + .compactMap { $0 as? IssueHandlingTrait } + .reversed() + .map { $0.provideScope(performing:) } + .reduce(body) { executeAllTraits, provideScope in + { + try await provideScope(executeAllTraits) + } + } + + try await executeAllTraits() + } + /// Enumerate the elements of a sequence, parallelizing enumeration in a task /// group if a given plan step has parallelization enabled. /// @@ -177,7 +213,19 @@ extension Runner { Event.post(.testSkipped(skipInfo), for: (step.test, nil), configuration: configuration) shouldSendTestEnded = false case let .recordIssue(issue): - Event.post(.issueRecorded(issue), for: (step.test, nil), configuration: configuration) + // Scope posting the issue recorded event such that issue handling + // traits have the opportunity to handle it. This ensures that if a test + // has an issue handling trait _and_ some other trait which caused an + // issue to be recorded, the issue handling trait can process the issue + // even though it wasn't recorded by the test function. + try await Test.withCurrent(step.test) { + try await _applyIssueHandlingTraits(for: step.test) { + // Don't specify `configuration` when posting this issue so that + // traits can provide scope and potentially customize the + // configuration. + Event.post(.issueRecorded(issue), for: (step.test, nil)) + } + } shouldSendTestEnded = false } } else { diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index 413b4327c..e6d19c1b9 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -48,6 +48,13 @@ types that customize the behavior of your tests. - ``Trait/bug(_:id:_:)-10yf5`` - ``Trait/bug(_:id:_:)-3vtpl`` + + ### Creating custom traits - ``Trait`` @@ -64,3 +71,4 @@ types that customize the behavior of your tests. - ``Tag`` - ``Tag/List`` - ``TimeLimitTrait`` + diff --git a/Sources/Testing/Traits/IssueHandlingTrait.swift b/Sources/Testing/Traits/IssueHandlingTrait.swift new file mode 100644 index 000000000..d70d68e93 --- /dev/null +++ b/Sources/Testing/Traits/IssueHandlingTrait.swift @@ -0,0 +1,167 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type that allows transforming or filtering the issues recorded by a test. +/// +/// Use this type to observe or customize the issue(s) recorded by the test this +/// trait is applied to. You can transform a recorded issue by copying it, +/// modifying one or more of its properties, and returning the copy. You can +/// observe recorded issues by returning them unmodified. Or you can suppress an +/// issue by either filtering it using ``Trait/filterIssues(_:)`` or returning +/// `nil` from the closure passed to ``Trait/transformIssues(_:)``. +/// +/// When an instance of this trait is applied to a suite, it is recursively +/// inherited by all child suites and tests. +/// +/// To add this trait to a test, use one of the following functions: +/// +/// - ``Trait/transformIssues(_:)`` +/// - ``Trait/filterIssues(_:)`` +@_spi(Experimental) +public struct IssueHandlingTrait: TestTrait, SuiteTrait { + /// A function which transforms an issue and returns an optional replacement. + /// + /// - Parameters: + /// - issue: The issue to transform. + /// + /// - Returns: An issue to replace `issue`, or else `nil` if the issue should + /// not be recorded. + fileprivate typealias Transformer = @Sendable (_ issue: Issue) -> Issue? + + /// This trait's transformer function. + private var _transformer: Transformer + + fileprivate init(transformer: @escaping Transformer) { + _transformer = transformer + } + + public var isRecursive: Bool { + true + } +} + +extension IssueHandlingTrait: TestScoping { + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + // Provide scope for tests at both the suite and test case levels, but not + // for the test function level. This avoids redundantly invoking the closure + // twice, and potentially double-processing, issues recorded by test + // functions. + test.isSuite || testCase != nil ? self : nil + } + + public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + try await provideScope(performing: function) + } + + /// Provide scope for a specified function. + /// + /// - Parameters: + /// - function: The function to perform. + /// + /// This is a simplified version of ``provideScope(for:testCase:performing:)`` + /// which doesn't accept test or test case parameters. It's included so that + /// a runner can invoke this trait's closure even when there is no test case, + /// such as if a trait on a test function threw an error during `prepare(for:)` + /// and caused an issue to be recorded for the test function. In that scenario, + /// this trait still needs to be invoked, but its `scopeProvider(for:testCase:)` + /// intentionally returns `nil` (see the comment in that method), so this + /// function can be called instead to ensure this trait can still handle that + /// issue. + func provideScope(performing function: @Sendable () async throws -> Void) async throws { + guard var configuration = Configuration.current else { + preconditionFailure("Configuration.current is nil when calling \(#function). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + } + + configuration.eventHandler = { [oldConfiguration = configuration] event, context in + guard case let .issueRecorded(issue) = event.kind else { + oldConfiguration.eventHandler(event, context) + return + } + + // Use the original configuration's event handler when invoking the + // transformer to avoid infinite recursion if the transformer itself + // records new issues. This means only issue handling traits whose scope + // is outside this one will be allowed to handle such issues. + let newIssue = Configuration.withCurrent(oldConfiguration) { + _transformer(issue) + } + + if let newIssue { + var event = event + event.kind = .issueRecorded(newIssue) + oldConfiguration.eventHandler(event, context) + } + } + + try await Configuration.withCurrent(configuration, perform: function) + } +} + +@_spi(Experimental) +extension Trait where Self == IssueHandlingTrait { + /// Constructs an trait that transforms issues recorded by a test. + /// + /// - Parameters: + /// - transformer: The closure called for each issue recorded by the test + /// this trait is applied to. It is passed a recorded issue, and returns + /// an optional issue to replace the passed-in one. + /// + /// The `transformer` closure is called synchronously each time an issue is + /// recorded by the test this trait is applied to. The closure is passed the + /// recorded issue, and if it returns a non-`nil` value, that will be recorded + /// instead of the original. Otherwise, if the closure returns `nil`, the + /// issue is suppressed and will not be included in the results. + /// + /// The `transformer` closure may be called more than once if the test records + /// multiple issues. If more than one instance of this trait is applied to a + /// test (including via inheritance from a containing suite), the `transformer` + /// closure for each instance will be called in right-to-left, innermost-to- + /// outermost order, unless `nil` is returned, which will skip invoking the + /// remaining traits' closures. + /// + /// Within `transformer`, you may access the current test or test case (if any) + /// using ``Test/current`` ``Test/Case/current``, respectively. You may also + /// record new issues, although they will only be handled by issue handling + /// traits which precede this trait or were inherited from a containing suite. + public static func transformIssues(_ transformer: @escaping @Sendable (Issue) -> Issue?) -> Self { + Self(transformer: transformer) + } + + /// Constructs a trait that filters issues recorded by a test. + /// + /// - Parameters: + /// - isIncluded: The predicate with which to filter issues recorded by the + /// test this trait is applied to. It is passed a recorded issue, and + /// should return `true` if the issue should be included, or `false` if it + /// should be suppressed. + /// + /// The `isIncluded` closure is called synchronously each time an issue is + /// recorded by the test this trait is applied to. The closure is passed the + /// recorded issue, and if it returns `true`, the issue will be preserved in + /// the test results. Otherwise, if the closure returns `false`, the issue + /// will not be included in the test results. + /// + /// The `isIncluded` closure may be called more than once if the test records + /// multiple issues. If more than one instance of this trait is applied to a + /// test (including via inheritance from a containing suite), the `isIncluded` + /// closure for each instance will be called in right-to-left, innermost-to- + /// outermost order, unless `false` is returned, which will skip invoking the + /// remaining traits' closures. + /// + /// Within `isIncluded`, you may access the current test or test case (if any) + /// using ``Test/current`` ``Test/Case/current``, respectively. You may also + /// record new issues, although they will only be handled by issue handling + /// traits which precede this trait or were inherited from a containing suite. + public static func filterIssues(_ isIncluded: @escaping @Sendable (Issue) -> Bool) -> Self { + Self { issue in + isIncluded(issue) ? issue : nil + } + } +} diff --git a/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift new file mode 100644 index 000000000..4d749b07f --- /dev/null +++ b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift @@ -0,0 +1,197 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("IssueHandlingTrait Tests") +struct IssueHandlingTraitTests { + @Test("Transforming an issue by appending a comment") + func addComment() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind else { + return + } + + #expect(issue.comments == ["Foo", "Bar"]) + } + + let handler = IssueHandlingTrait.transformIssues { issue in + var issue = issue + issue.comments.append("Bar") + return issue + } + + await Test(handler) { + Issue.record("Foo") + }.run(configuration: configuration) + } + + @Test("Suppressing an issue by returning `nil` from the transform closure") + func suppressIssueUsingTransformer() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + if case .issueRecorded = event.kind { + Issue.record("Unexpected issue recorded event: \(event)") + } + } + + let handler = IssueHandlingTrait.transformIssues { _ in + // Return nil to suppress the issue. + nil + } + + await Test(handler) { + Issue.record("Foo") + }.run(configuration: configuration) + } + + @Test("Suppressing an issue by returning `false` from the filter closure") + func filterIssue() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + if case .issueRecorded = event.kind { + Issue.record("Unexpected issue recorded event: \(event)") + } + } + + await Test(.filterIssues { _ in false }) { + Issue.record("Foo") + }.run(configuration: configuration) + } + +#if !SWT_NO_UNSTRUCTURED_TASKS + @Test("Transforming an issue recorded from another trait on the test") + func skipIssue() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .errorCaught = issue.kind else { + return + } + + #expect(issue.comments == ["Transformed!"]) + } + + struct MyError: Error {} + + try await confirmation("Transformer closure is called") { transformerCalled in + let transformer: @Sendable (Issue) -> Issue? = { issue in + defer { + transformerCalled() + } + + #expect(Test.Case.current == nil) + + var issue = issue + issue.comments = ["Transformed!"] + return issue + } + + let test = Test( + .enabled(if: try { throw MyError() }()), + .transformIssues(transformer) + ) {} + + // Use a detached task to intentionally clear task local values for the + // current test and test case, since this test validates their value. + await Task.detached { [configuration] in + await test.run(configuration: configuration) + }.value + } + } +#endif + + @Test("Accessing the current Test and Test.Case from a transformer closure") + func currentTestAndCase() async throws { + await confirmation("Transformer closure is called") { transformerCalled in + let handler = IssueHandlingTrait.transformIssues { issue in + defer { + transformerCalled() + } + #expect(Test.current?.name == "fixture()") + #expect(Test.Case.current != nil) + return issue + } + + var test = Test(handler) { + Issue.record("Foo") + } + test.name = "fixture()" + await test.run() + } + } + + @Test("Validate the relative execution order of multiple issue handling traits") + func traitOrder() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind else { + return + } + + // Ordering is intentional + #expect(issue.comments == ["Foo", "Bar", "Baz"]) + } + + let outerHandler = IssueHandlingTrait.transformIssues { issue in + var issue = issue + issue.comments.append("Baz") + return issue + } + let innerHandler = IssueHandlingTrait.transformIssues { issue in + var issue = issue + issue.comments.append("Bar") + return issue + } + + await Test(outerHandler, innerHandler) { + Issue.record("Foo") + }.run(configuration: configuration) + } + + @Test("Secondary issue recorded from a transformer closure") + func issueRecordedFromClosure() async throws { + await confirmation("Original issue recorded") { originalIssueRecorded in + await confirmation("Secondary issue recorded") { secondaryIssueRecorded in + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind else { + return + } + + if issue.comments.contains("Foo") { + originalIssueRecorded() + } else if issue.comments.contains("Something else") { + secondaryIssueRecorded() + } else { + Issue.record("Unexpected issue recorded: \(issue)") + } + } + + let handler1 = IssueHandlingTrait.transformIssues { issue in + return issue + } + let handler2 = IssueHandlingTrait.transformIssues { issue in + Issue.record("Something else") + return issue + } + let handler3 = IssueHandlingTrait.transformIssues { issue in + // The "Something else" issue should not be passed to this closure. + #expect(issue.comments.contains("Foo")) + return issue + } + + await Test(handler1, handler2, handler3) { + Issue.record("Foo") + }.run(configuration: configuration) + } + } + } +} From 7f0016fd27b1cf80b13454ff7f9a4fbc57dad9ac Mon Sep 17 00:00:00 2001 From: David Catmull Date: Wed, 30 Apr 2025 16:00:58 -0600 Subject: [PATCH 186/234] Remove experimental status for evaluate() and add metadata (#1097) Remove experimental status for ConditionTrait.evaluate() and add metadata ### Motivation: Now that the proposal has been accepted, the experimental tag can be removed. ### Modifications: * Removed the `@_spi(Experimental)` attribute from `ConditionTrait.evaluate()` * Added metadata to the doc comment marking it as part of Swift 6.2 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Traits/ConditionTrait.swift | 5 ++++- Tests/TestingTests/Traits/ConditionTraitTests.swift | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index a3b98c99d..eb3d1bd11 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -69,7 +69,10 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// /// The evaluation is performed each time this function is called, and is not /// cached. - @_spi(Experimental) + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public func evaluate() async throws -> Bool { switch kind { case let .conditional(condition): diff --git a/Tests/TestingTests/Traits/ConditionTraitTests.swift b/Tests/TestingTests/Traits/ConditionTraitTests.swift index 6b5311202..d957e425b 100644 --- a/Tests/TestingTests/Traits/ConditionTraitTests.swift +++ b/Tests/TestingTests/Traits/ConditionTraitTests.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@testable @_spi(Experimental) import Testing +@testable import Testing @Suite("Condition Trait Tests", .tags(.traitRelated)) struct ConditionTraitTests { From 099d177a8cae4c0df186d50653c10728a3ae9873 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 1 May 2025 11:56:37 -0400 Subject: [PATCH 187/234] Promote exit tests to API (#324) This PR promotes exit tests to API, pending approval of the proposal at https://github.com/swiftlang/swift-evolution/pull/2718. View the full proposal [here](https://github.com/grynspan/swift-evolution/blob/jgrynspan/swift-testing-exit-tests/proposals/testing/NNNN-exit-tests.md). ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --------- Co-authored-by: Stuart Montgomery --- Sources/Testing/CMakeLists.txt | 2 +- .../{StatusAtExit.swift => ExitStatus.swift} | 60 ++++--- .../ExitTests/ExitTest.CapturedValue.swift | 2 +- .../ExitTests/ExitTest.Condition.swift | 113 +++++++++---- .../Testing/ExitTests/ExitTest.Result.swift | 40 +++-- Sources/Testing/ExitTests/ExitTest.swift | 101 ++++++------ Sources/Testing/ExitTests/WaitFor.swift | 12 +- .../Expectations/Expectation+Macro.swift | 156 ++---------------- .../ExpectationChecking+Macro.swift | 17 +- Sources/Testing/Running/Configuration.swift | 1 - .../SourceAttribution/Expression.swift | 5 +- Sources/Testing/Testing.docc/Expectations.md | 8 + .../Testing/Testing.docc/OrganizingTests.md | 2 +- Sources/Testing/Testing.docc/exit-testing.md | 155 +++++++++++++++++ Sources/TestingMacros/ConditionMacro.swift | 6 +- .../ConditionMacroTests.swift | 12 +- Tests/TestingTests/AttachmentTests.swift | 2 +- Tests/TestingTests/ConfirmationTests.swift | 6 +- Tests/TestingTests/DiscoveryTests.swift | 4 +- Tests/TestingTests/ExitTestTests.swift | 126 +++++++------- Tests/TestingTests/PlanIterationTests.swift | 4 +- Tests/TestingTests/SourceLocationTests.swift | 16 +- .../Support/FileHandleTests.swift | 2 +- 23 files changed, 478 insertions(+), 374 deletions(-) rename Sources/Testing/ExitTests/{StatusAtExit.swift => ExitStatus.swift} (60%) create mode 100644 Sources/Testing/Testing.docc/exit-testing.md diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index 6eca45dde..5b84aeaf3 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -36,7 +36,7 @@ add_library(Testing ExitTests/ExitTest.Condition.swift ExitTests/ExitTest.Result.swift ExitTests/SpawnProcess.swift - ExitTests/StatusAtExit.swift + ExitTests/ExitStatus.swift ExitTests/WaitFor.swift Expectations/Expectation.swift Expectations/Expectation+Macro.swift diff --git a/Sources/Testing/ExitTests/StatusAtExit.swift b/Sources/Testing/ExitTests/ExitStatus.swift similarity index 60% rename from Sources/Testing/ExitTests/StatusAtExit.swift rename to Sources/Testing/ExitTests/ExitStatus.swift index ea5e287c7..0dd6d86ab 100644 --- a/Sources/Testing/ExitTests/StatusAtExit.swift +++ b/Sources/Testing/ExitTests/ExitStatus.swift @@ -10,74 +10,94 @@ private import _TestingInternals -/// An enumeration describing possible status a process will yield on exit. +/// An enumeration describing possible status a process will report on exit. /// /// You can convert an instance of this type to an instance of /// ``ExitTest/Condition`` using ``ExitTest/Condition/init(_:)``. That value /// can then be used to describe the condition under which an exit test is /// expected to pass or fail by passing it to -/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. -@_spi(Experimental) +/// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or +/// ``require(processExitsWith:observing:_:sourceLocation:performing:)``. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -public enum StatusAtExit: Sendable { - /// The process terminated with the given exit code. +public enum ExitStatus: Sendable { + /// The process exited with the given exit code. /// /// - Parameters: - /// - exitCode: The exit code yielded by the process. + /// - exitCode: The exit code reported by the process. /// - /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), - /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their - /// own non-standard exit codes: + /// The C programming language defines two standard exit codes, `EXIT_SUCCESS` + /// and `EXIT_FAILURE`. Platforms may additionally define their own + /// non-standard exit codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man3/exit.3.html), [``](https://www.kernel.org/doc/man-pages/online/pages/man3/sysexits.h.3head.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/EXIT_status for more + /// information about exit codes defined by the C standard. + /// } + /// /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by - /// the process is yielded to the parent process. Linux and other POSIX-like + /// the process is reported to the parent process. Linux and other POSIX-like /// systems may only reliably report the low unsigned 8 bits (0–255) of /// the exit code. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } case exitCode(_ exitCode: CInt) - /// The process terminated with the given signal. + /// The process exited with the given signal. /// /// - Parameters: - /// - signal: The signal that terminated the process. + /// - signal: The signal that caused the process to exit. /// - /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). - /// Platforms may additionally define their own non-standard signal codes: + /// The C programming language defines a number of standard signals. Platforms + /// may additionally define their own non-standard signal codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man7/signal.7.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/SIG_types for more + /// information about signals defined by the C standard. + /// } + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } case signal(_ signal: CInt) } // MARK: - Equatable -@_spi(Experimental) #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -extension StatusAtExit: Equatable {} +extension ExitStatus: Equatable {} // MARK: - CustomStringConvertible @_spi(Experimental) #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -extension StatusAtExit: CustomStringConvertible { +extension ExitStatus: CustomStringConvertible { public var description: String { switch self { case let .exitCode(exitCode): diff --git a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift index d4c84e446..1d5c9b18a 100644 --- a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift +++ b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift @@ -26,7 +26,7 @@ extension ExitTest { /// exit test: /// /// ```swift - /// await #expect(exitsWith: .failure) { [a = a as T, b = b as U, c = c as V] in + /// await #expect(processExitsWith: .failure) { [a = a as T, b = b as U, c = c as V] in /// ... /// } /// ``` diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift index d2c637d79..f737d8cf6 100644 --- a/Sources/Testing/ExitTests/ExitTest.Condition.swift +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -10,7 +10,6 @@ private import _TestingInternals -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @@ -19,13 +18,29 @@ extension ExitTest { /// /// Values of this type are used to describe the conditions under which an /// exit test is expected to pass or fail by passing them to - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(processExitsWith:observing:_:sourceLocation:performing:)``. + /// + /// ## Topics + /// + /// ### Successful exit conditions + /// + /// - ``success`` + /// + /// ### Failing exit conditions + /// + /// - ``failure`` + /// - ``exitCode(_:)`` + /// - ``signal(_:)`` + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public struct Condition: Sendable { /// An enumeration describing the possible conditions for an exit test. private enum _Kind: Sendable, Equatable { /// The exit test must exit with a particular exit status. - case statusAtExit(StatusAtExit) + case exitStatus(ExitStatus) /// The exit test must exit successfully. case success @@ -41,49 +56,77 @@ extension ExitTest { // MARK: - -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif extension ExitTest.Condition { - /// A condition that matches when a process terminates successfully with exit - /// code `EXIT_SUCCESS`. + /// A condition that matches when a process exits normally. + /// + /// This condition matches the exit code `EXIT_SUCCESS`. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static var success: Self { Self(_kind: .success) } - /// A condition that matches when a process terminates abnormally with any - /// exit code other than `EXIT_SUCCESS` or with any signal. + /// A condition that matches when a process exits abnormally + /// + /// This condition matches any exit code other than `EXIT_SUCCESS` or any + /// signal that causes the process to exit. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static var failure: Self { Self(_kind: .failure) } - public init(_ statusAtExit: StatusAtExit) { - self.init(_kind: .statusAtExit(statusAtExit)) + /// Initialize an instance of this type that matches the specified exit + /// status. + /// + /// - Parameters: + /// - exitStatus: The particular exit status this condition should match. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } + public init(_ exitStatus: ExitStatus) { + self.init(_kind: .exitStatus(exitStatus)) } /// Creates a condition that matches when a process terminates with a given /// exit code. /// /// - Parameters: - /// - exitCode: The exit code yielded by the process. + /// - exitCode: The exit code reported by the process. /// - /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), - /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their - /// own non-standard exit codes: + /// The C programming language defines two standard exit codes, `EXIT_SUCCESS` + /// and `EXIT_FAILURE`. Platforms may additionally define their own + /// non-standard exit codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man3/exit.3.html), [``](https://www.kernel.org/doc/man-pages/online/pages/man3/sysexits.h.3head.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/EXIT_status for more + /// information about exit codes defined by the C standard. + /// } + /// /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by - /// the process is yielded to the parent process. Linux and other POSIX-like + /// the process is reported to the parent process. Linux and other POSIX-like /// systems may only reliably report the low unsigned 8 bits (0–255) of /// the exit code. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func exitCode(_ exitCode: CInt) -> Self { #if !SWT_NO_EXIT_TESTS Self(.exitCode(exitCode)) @@ -92,22 +135,30 @@ extension ExitTest.Condition { #endif } - /// Creates a condition that matches when a process terminates with a given - /// signal. + /// Creates a condition that matches when a process exits with a given signal. /// /// - Parameters: - /// - signal: The signal that terminated the process. + /// - signal: The signal that caused the process to exit. /// - /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). - /// Platforms may additionally define their own non-standard signal codes: + /// The C programming language defines a number of standard signals. Platforms + /// may additionally define their own non-standard signal codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man7/signal.7.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/SIG_types for more + /// information about signals defined by the C standard. + /// } + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func signal(_ signal: CInt) -> Self { #if !SWT_NO_EXIT_TESTS Self(.signal(signal)) @@ -131,8 +182,8 @@ extension ExitTest.Condition: CustomStringConvertible { ".failure" case .success: ".success" - case let .statusAtExit(statusAtExit): - String(describing: statusAtExit) + case let .exitStatus(exitStatus): + String(describing: exitStatus) } #else fatalError("Unsupported") @@ -149,19 +200,19 @@ extension ExitTest.Condition { /// Check whether or not an exit test condition matches a given exit status. /// /// - Parameters: - /// - statusAtExit: An exit status to compare against. + /// - exitStatus: An exit status to compare against. /// - /// - Returns: Whether or not `self` and `statusAtExit` represent the same - /// exit condition. + /// - Returns: Whether or not `self` and `exitStatus` represent the same exit + /// condition. /// /// Two exit test conditions can be compared; if either instance is equal to /// ``failure``, it will compare equal to any instance except ``success``. - func isApproximatelyEqual(to statusAtExit: StatusAtExit) -> Bool { + func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool { // Strictly speaking, the C standard treats 0 as a successful exit code and // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern // operating system defines EXIT_SUCCESS to any value other than 0, so the // distinction is academic. - return switch (self._kind, statusAtExit) { + return switch (self._kind, exitStatus) { case let (.success, .exitCode(exitCode)): exitCode == EXIT_SUCCESS case let (.failure, .exitCode(exitCode)): @@ -170,7 +221,7 @@ extension ExitTest.Condition { // All terminating signals are considered failures. true default: - self._kind == .statusAtExit(statusAtExit) + self._kind == .exitStatus(exitStatus) } } } diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift index beb2d56fc..ef70a3789 100644 --- a/Sources/Testing/ExitTests/ExitTest.Result.swift +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -8,7 +8,6 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @@ -16,15 +15,20 @@ extension ExitTest { /// A type representing the result of an exit test after it has exited and /// returned control to the calling test function. /// - /// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and - /// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return - /// instances of this type. + /// Both ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` + /// and ``require(processExitsWith:observing:_:sourceLocation:performing:)`` + /// return instances of this type. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public struct Result: Sendable { - /// The exit condition the exit test exited with. + /// The exit status reported by the process hosting the exit test. /// - /// When the exit test passes, the value of this property is equal to the - /// exit status reported by the process that hosted the exit test. - public var statusAtExit: StatusAtExit + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } + public var exitStatus: ExitStatus /// All bytes written to the standard output stream of the exit test before /// it exited. @@ -45,11 +49,15 @@ extension ExitTest { /// /// To enable gathering output from the standard output stream during an /// exit test, pass `\.standardOutputContent` in the `observedValues` - /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)`` - /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// argument of ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` + /// or ``require(processExitsWith:observing:_:sourceLocation:performing:)``. /// /// If you did not request standard output content when running an exit /// test, the value of this property is the empty array. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public var standardOutputContent: [UInt8] = [] /// All bytes written to the standard error stream of the exit test before @@ -71,16 +79,20 @@ extension ExitTest { /// /// To enable gathering output from the standard error stream during an exit /// test, pass `\.standardErrorContent` in the `observedValues` argument of - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(processExitsWith:observing:_:sourceLocation:performing:)``. /// /// If you did not request standard error content when running an exit test, /// the value of this property is the empty array. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public var standardErrorContent: [UInt8] = [] @_spi(ForToolsIntegrationOnly) - public init(statusAtExit: StatusAtExit) { - self.statusAtExit = statusAtExit + public init(exitStatus: ExitStatus) { + self.exitStatus = exitStatus } } } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 14d10a04b..1e9c29c15 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -26,10 +26,13 @@ private import _TestingInternals /// A type describing an exit test. /// /// Instances of this type describe exit tests you create using the -/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` -/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` macro. You -/// don't usually need to interact directly with an instance of this type. -@_spi(Experimental) +/// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or +/// ``require(processExitsWith:observing:_:sourceLocation:performing:)`` macro. +/// You don't usually need to interact directly with an instance of this type. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @@ -90,12 +93,12 @@ public struct ExitTest: Sendable, ~Copyable { /// observed and returned to the caller. /// /// The testing library sets this property to match what was passed by the - /// developer to the `#expect(exitsWith:)` or `#require(exitsWith:)` macro. - /// If you are implementing an exit test handler, you can check the value of - /// this property to determine what information you need to preserve from your - /// child process. + /// developer to the `#expect(processExitsWith:)` or `#require(processExitsWith:)` + /// macro. If you are implementing an exit test handler, you can check the + /// value of this property to determine what information you need to preserve + /// from your child process. /// - /// The value of this property always includes ``ExitTest/Result/statusAtExit`` + /// The value of this property always includes ``ExitTest/Result/exitStatus`` /// even if the test author does not specify it. /// /// Within a child process running an exit test, the value of this property is @@ -104,8 +107,8 @@ public struct ExitTest: Sendable, ~Copyable { public var observedValues: [any PartialKeyPath & Sendable] { get { var result = _observedValues - if !result.contains(\.statusAtExit) { // O(n), but n <= 3 (no Set needed) - result.append(\.statusAtExit) + if !result.contains(\.exitStatus) { // O(n), but n <= 3 (no Set needed) + result.append(\.exitStatus) } return result } @@ -144,7 +147,6 @@ public struct ExitTest: Sendable, ~Copyable { #if !SWT_NO_EXIT_TESTS // MARK: - Current -@_spi(Experimental) extension ExitTest { /// Storage for ``current``. /// @@ -165,6 +167,10 @@ extension ExitTest { /// /// The value of this property is constant across all tasks in the current /// process. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static var current: ExitTest? { _read { // NOTE: Even though this accessor is `_read` and has borrowing semantics, @@ -180,7 +186,7 @@ extension ExitTest { // MARK: - Invocation -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(ForToolsIntegrationOnly) extension ExitTest { /// Disable crash reporting, crash logging, or core dumps for the current /// process. @@ -228,9 +234,9 @@ extension ExitTest { /// Call the exit test in the current process. /// /// This function invokes the closure originally passed to - /// `#expect(exitsWith:)` _in the current process_. That closure is expected - /// to terminate the process; if it does not, the testing library will - /// terminate the process as if its `main()` function returned naturally. + /// `#expect(processExitsWith:)` _in the current process_. That closure is + /// expected to terminate the process; if it does not, the testing library + /// will terminate the process as if its `main()` function returned naturally. public consuming func callAsFunction() async -> Never { Self._disableCrashReporting() @@ -319,8 +325,8 @@ extension ExitTest { /// /// - Returns: Whether or not an exit test was stored into `outValue`. /// - /// - Warning: This function is used to implement the `#expect(exitsWith:)` - /// macro. Do not use it directly. + /// - Warning: This function is used to implement the + /// `#expect(processExitsWith:)` macro. Do not use it directly. public static func __store( _ id: (UInt64, UInt64, UInt64, UInt64), _ body: @escaping @Sendable (repeat each T) async throws -> Void, @@ -356,7 +362,7 @@ extension ExitTest { } } -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(ForToolsIntegrationOnly) extension ExitTest { /// Find the exit test function at the given source location. /// @@ -396,7 +402,7 @@ extension ExitTest { /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTest/Result/statusAtExit`` property is always returned. +/// ``ExitTest/Result/exitStatus`` property is always returned. /// - expression: The expression, corresponding to `condition`, that is being /// evaluated (if available at compile time.) /// - comments: An array of comments describing the expectation. This array @@ -408,12 +414,12 @@ extension ExitTest { /// - sourceLocation: The source location of the expectation. /// /// This function contains the common implementation for all -/// `await #expect(exitsWith:) { }` invocations regardless of calling +/// `await #expect(processExitsWith:) { }` invocations regardless of calling /// convention. func callExitTest( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), encodingCapturedValues capturedValues: [ExitTest.CapturedValue], - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, comments: @autoclosure () -> [Comment], @@ -422,7 +428,7 @@ func callExitTest( sourceLocation: SourceLocation ) async -> Result { guard let configuration = Configuration.current ?? Configuration.all.first else { - preconditionFailure("A test must be running on the current task to use #expect(exitsWith:).") + preconditionFailure("A test must be running on the current task to use #expect(processExitsWith:).") } var result: ExitTest.Result @@ -438,8 +444,8 @@ func callExitTest( #if os(Windows) // For an explanation of this magic, see the corresponding logic in // ExitTest.callAsFunction(). - if case let .exitCode(exitCode) = result.statusAtExit, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { - result.statusAtExit = .signal(exitCode & STATUS_CODE_MASK) + if case let .exitCode(exitCode) = result.exitStatus, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { + result.exitStatus = .signal(exitCode & STATUS_CODE_MASK) } #endif } catch { @@ -464,22 +470,19 @@ func callExitTest( // For lack of a better way to handle an exit test failing in this way, // we record the system issue above, then let the expectation fail below by // reporting an exit condition that's the inverse of the expected one. - let statusAtExit: StatusAtExit = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) { + let exitStatus: ExitStatus = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) { .exitCode(EXIT_SUCCESS) } else { .exitCode(EXIT_FAILURE) } - result = ExitTest.Result(statusAtExit: statusAtExit) + result = ExitTest.Result(exitStatus: exitStatus) } - // How did the exit test actually exit? - let actualStatusAtExit = result.statusAtExit - // Plumb the exit test's result through the general expectation machinery. return __checkValue( - expectedExitCondition.isApproximatelyEqual(to: actualStatusAtExit), + expectedExitCondition.isApproximatelyEqual(to: result.exitStatus), expression: expression, - expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualStatusAtExit), + expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(result.exitStatus), mismatchedExitConditionDescription: String(describingForTest: expectedExitCondition), comments: comments(), isRequired: isRequired, @@ -499,7 +502,7 @@ extension ABI { fileprivate typealias BackChannelVersion = v1 } -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(ForToolsIntegrationOnly) extension ExitTest { /// A handler that is invoked when an exit test starts. /// @@ -513,11 +516,10 @@ extension ExitTest { /// the exit test. /// /// This handler is invoked when an exit test (i.e. a call to either - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``) is started. - /// The handler is responsible for initializing a new child environment - /// (typically a child process) and running the exit test identified by - /// `sourceLocation` there. + /// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(processExitsWith:observing:_:sourceLocation:performing:)``) is + /// started. The handler is responsible for initializing a new child + /// environment (typically a child process) and running `exitTest` there. /// /// In the child environment, you can find the exit test again by calling /// ``ExitTest/find(at:)`` and can run it by calling @@ -615,28 +617,23 @@ extension ExitTest { static func findInEnvironmentForEntryPoint() -> Self? { // Find the ID of the exit test to run, if any, in the environment block. var id: ExitTest.ID? - if var idString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") { + if var idString = Environment.variable(named: "SWT_EXIT_TEST_ID") { // Clear the environment variable. It's an implementation detail and exit // test code shouldn't be dependent on it. Use ExitTest.current if needed! - Environment.setVariable(nil, named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") + Environment.setVariable(nil, named: "SWT_EXIT_TEST_ID") id = try? idString.withUTF8 { idBuffer in try JSON.decode(ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) } } - guard let id else { + guard let id, var result = find(identifiedBy: id) else { return nil } // If an exit test was found, inject back channel handling into its body. // External tools authors should set up their own back channel mechanisms // and ensure they're installed before calling ExitTest.callAsFunction(). - guard var result = find(identifiedBy: id) else { - return nil - } - - // We can't say guard let here because it counts as a consume. - guard let backChannel = _makeFileHandle(forEnvironmentVariableNamed: "SWT_EXPERIMENTAL_BACKCHANNEL", mode: "wb") else { + guard let backChannel = _makeFileHandle(forEnvironmentVariableNamed: "SWT_BACKCHANNEL", mode: "wb") else { return result } @@ -755,7 +752,7 @@ extension ExitTest { // Insert a specific variable that tells the child process which exit test // to run. try JSON.withEncoding(of: exitTest.id) { json in - childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) + childEnvironment["SWT_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) } typealias ResultUpdater = @Sendable (inout ExitTest.Result) -> Void @@ -795,7 +792,7 @@ extension ExitTest { // captured values channel by setting a known environment variable to // the corresponding file descriptor (HANDLE on Windows) for each. if let backChannelEnvironmentVariable = _makeEnvironmentVariable(for: backChannelWriteEnd) { - childEnvironment["SWT_EXPERIMENTAL_BACKCHANNEL"] = backChannelEnvironmentVariable + childEnvironment["SWT_BACKCHANNEL"] = backChannelEnvironmentVariable } if let capturedValuesEnvironmentVariable = _makeEnvironmentVariable(for: capturedValuesReadEnd) { childEnvironment["SWT_EXPERIMENTAL_CAPTURED_VALUES"] = capturedValuesEnvironmentVariable @@ -831,8 +828,8 @@ extension ExitTest { // Await termination of the child process. taskGroup.addTask { - let statusAtExit = try await wait(for: processID) - return { $0.statusAtExit = statusAtExit } + let exitStatus = try await wait(for: processID) + return { $0.exitStatus = exitStatus } } // Read back the stdout and stderr streams. @@ -862,7 +859,7 @@ extension ExitTest { // Collate the various bits of the result. The exit condition used here // is just a placeholder and will be replaced by the result of one of // the tasks above. - var result = ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) + var result = ExitTest.Result(exitStatus: .exitCode(EXIT_FAILURE)) for try await update in taskGroup { update?(&result) } diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index cc611158f..238ed835a 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -20,7 +20,7 @@ internal import _TestingInternals /// /// - Throws: If the exit status of the process with ID `pid` cannot be /// determined (i.e. it does not represent an exit condition.) -private func _blockAndWait(for pid: consuming pid_t) throws -> StatusAtExit { +private func _blockAndWait(for pid: consuming pid_t) throws -> ExitStatus { let pid = consume pid // Get the exit status of the process or throw an error (other than EINTR.) @@ -61,7 +61,7 @@ private func _blockAndWait(for pid: consuming pid_t) throws -> StatusAtExit { /// - Note: The open-source implementation of libdispatch available on Linux /// and other platforms does not support `DispatchSourceProcess`. Those /// platforms use an alternate implementation below. -func wait(for pid: consuming pid_t) async throws -> StatusAtExit { +func wait(for pid: consuming pid_t) async throws -> ExitStatus { let pid = consume pid let source = DispatchSource.makeProcessSource(identifier: pid, eventMask: .exit) @@ -80,7 +80,7 @@ func wait(for pid: consuming pid_t) async throws -> StatusAtExit { } #elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) /// A mapping of awaited child PIDs to their corresponding Swift continuations. -private let _childProcessContinuations = LockedWith]>() +private let _childProcessContinuations = LockedWith]>() /// A condition variable used to suspend the waiter thread created by /// `_createWaitThread()` when there are no child processes to await. @@ -202,7 +202,7 @@ private let _createWaitThread: Void = { /// /// On Apple platforms, the libdispatch-based implementation above is more /// efficient because it does not need to permanently reserve a thread. -func wait(for pid: consuming pid_t) async throws -> StatusAtExit { +func wait(for pid: consuming pid_t) async throws -> ExitStatus { let pid = consume pid // Ensure the waiter thread is running. @@ -239,7 +239,7 @@ func wait(for pid: consuming pid_t) async throws -> StatusAtExit { /// This implementation of `wait(for:)` calls `RegisterWaitForSingleObject()` to /// wait for `processHandle`, suspends the calling task until the waiter's /// callback is called, then calls `GetExitCodeProcess()`. -func wait(for processHandle: consuming HANDLE) async throws -> StatusAtExit { +func wait(for processHandle: consuming HANDLE) async throws -> ExitStatus { let processHandle = consume processHandle defer { _ = CloseHandle(processHandle) @@ -283,6 +283,6 @@ func wait(for processHandle: consuming HANDLE) async throws -> StatusAtExit { } #else #warning("Platform-specific implementation missing: cannot wait for child processes to exit") -func wait(for processID: consuming Never) async throws -> StatusAtExit {} +func wait(for processID: consuming Never) async throws -> ExitStatus {} #endif #endif diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 5111fbddd..d14920547 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -490,7 +490,7 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTest/Result/statusAtExit`` property is always returned. +/// ``ExitTest/Result/exitStatus`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. @@ -506,89 +506,20 @@ public macro require( /// causes a process to terminate: /// /// ```swift -/// await #expect(exitsWith: .failure) { +/// await #expect(processExitsWith: .failure) { /// fatalError() /// } /// ``` /// -/// - Note: A call to this expectation macro is called an "exit test." -/// -/// ## How exit tests are run -/// -/// When an exit test is performed at runtime, the testing library starts a new -/// process with the same executable as the current process. The current task is -/// then suspended (as with `await`) and waits for the child process to -/// terminate. `expression` is not called in the parent process. -/// -/// Meanwhile, in the child process, `expression` is called directly. To ensure -/// a clean environment for execution, it is not called within the context of -/// the original test. If `expression` does not terminate the child process, the -/// process is terminated automatically as if the main function of the child -/// process were allowed to return naturally. If an error is thrown from -/// `expression`, it is handed as if the error were thrown from `main()` and the -/// process is terminated. -/// -/// Once the child process terminates, the parent process resumes and compares -/// its exit status against `expectedExitCondition`. If they match, the exit -/// test has passed; otherwise, it has failed and an issue is recorded. -/// -/// ## Child process output -/// -/// By default, the child process is configured without a standard output or -/// standard error stream. If your test needs to review the content of either of -/// these streams, you can pass its key path in the `observedValues` argument: -/// -/// ```swift -/// let result = await #expect( -/// exitsWith: .failure, -/// observing: [\.standardOutputContent] -/// ) { -/// print("Goodbye, world!") -/// fatalError() -/// } -/// if let result { -/// #expect(result.standardOutputContent.contains(UInt8(ascii: "G"))) -/// } -/// ``` -/// -/// - Note: The content of the standard output and standard error streams may -/// contain any arbitrary sequence of bytes, including sequences that are not -/// valid UTF-8 and cannot be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). -/// These streams are globally accessible within the child process, and any -/// code running in an exit test may write to it including the operating -/// system and any third-party dependencies you have declared in your package. -/// -/// The actual exit condition of the child process is always reported by the -/// testing library even if you do not specify it in `observedValues`. -/// -/// ## Runtime constraints -/// -/// Exit tests cannot capture any state originating in the parent process or -/// from the enclosing lexical context. For example, the following exit test -/// will fail to compile because it captures an argument to the enclosing -/// parameterized test: -/// -/// ```swift -/// @Test(arguments: 100 ..< 200) -/// func sellIceCreamCones(count: Int) async { -/// await #expect(exitsWith: .failure) { -/// precondition( -/// count < 10, // ERROR: A C function pointer cannot be formed from a -/// // closure that captures context -/// "Too many ice cream cones" -/// ) -/// } +/// @Metadata { +/// @Available(Swift, introduced: 6.2) /// } -/// ``` -/// -/// An exit test cannot run within another exit test. -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @discardableResult @freestanding(expression) public macro expect( - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, @@ -602,7 +533,7 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTest/Result/statusAtExit`` property is always returned. +/// ``ExitTest/Result/exitStatus`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. @@ -620,87 +551,20 @@ public macro require( /// causes a process to terminate: /// /// ```swift -/// try await #require(exitsWith: .failure) { +/// try await #require(processExitsWith: .failure) { /// fatalError() /// } /// ``` /// -/// - Note: A call to this expectation macro is called an "exit test." -/// -/// ## How exit tests are run -/// -/// When an exit test is performed at runtime, the testing library starts a new -/// process with the same executable as the current process. The current task is -/// then suspended (as with `await`) and waits for the child process to -/// terminate. `expression` is not called in the parent process. -/// -/// Meanwhile, in the child process, `expression` is called directly. To ensure -/// a clean environment for execution, it is not called within the context of -/// the original test. If `expression` does not terminate the child process, the -/// process is terminated automatically as if the main function of the child -/// process were allowed to return naturally. If an error is thrown from -/// `expression`, it is handed as if the error were thrown from `main()` and the -/// process is terminated. -/// -/// Once the child process terminates, the parent process resumes and compares -/// its exit status against `expectedExitCondition`. If they match, the exit -/// test has passed; otherwise, it has failed and an issue is recorded. -/// -/// ## Child process output -/// -/// By default, the child process is configured without a standard output or -/// standard error stream. If your test needs to review the content of either of -/// these streams, you can pass its key path in the `observedValues` argument: -/// -/// ```swift -/// let result = try await #require( -/// exitsWith: .failure, -/// observing: [\.standardOutputContent] -/// ) { -/// print("Goodbye, world!") -/// fatalError() -/// } -/// #expect(result.standardOutputContent.contains(UInt8(ascii: "G"))) -/// ``` -/// -/// - Note: The content of the standard output and standard error streams may -/// contain any arbitrary sequence of bytes, including sequences that are not -/// valid UTF-8 and cannot be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). -/// These streams are globally accessible within the child process, and any -/// code running in an exit test may write to it including the operating -/// system and any third-party dependencies you have declared in your package. -/// -/// The actual exit condition of the child process is always reported by the -/// testing library even if you do not specify it in `observedValues`. -/// -/// ## Runtime constraints -/// -/// Exit tests cannot capture any state originating in the parent process or -/// from the enclosing lexical context. For example, the following exit test -/// will fail to compile because it captures an argument to the enclosing -/// parameterized test: -/// -/// ```swift -/// @Test(arguments: 100 ..< 200) -/// func sellIceCreamCones(count: Int) async throws { -/// try await #require(exitsWith: .failure) { -/// precondition( -/// count < 10, // ERROR: A C function pointer cannot be formed from a -/// // closure that captures context -/// "Too many ice cream cones" -/// ) -/// } +/// @Metadata { +/// @Available(Swift, introduced: 6.2) /// } -/// ``` -/// -/// An exit test cannot run within another exit test. -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @discardableResult @freestanding(expression) public macro require( - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index e8767d01f..6d3093f2a 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1139,15 +1139,14 @@ public func __checkClosureCall( /// Check that an expression always exits (terminates the current process) with /// a given status. /// -/// This overload is used for `await #expect(exitsWith:) { }` invocations that -/// do not capture any state. +/// This overload is used for `await #expect(processExitsWith:) { }` invocations +/// that do not capture any state. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. -@_spi(Experimental) public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], performing _: @convention(thin) () -> Void, expression: __Expression, @@ -1159,7 +1158,7 @@ public func __checkClosureCall( await callExitTest( identifiedBy: exitTestID, encodingCapturedValues: [], - exitsWith: expectedExitCondition, + processExitsWith: expectedExitCondition, observing: observedValues, expression: expression, comments: comments(), @@ -1171,8 +1170,8 @@ public func __checkClosureCall( /// Check that an expression always exits (terminates the current process) with /// a given status. /// -/// This overload is used for `await #expect(exitsWith:) { }` invocations that -/// capture some values with an explicit capture list. +/// This overload is used for `await #expect(processExitsWith:) { }` invocations +/// that capture some values with an explicit capture list. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. @@ -1180,7 +1179,7 @@ public func __checkClosureCall( public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), encodingCapturedValues capturedValues: (repeat each T), - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], performing _: @convention(thin) () -> Void, expression: __Expression, @@ -1192,7 +1191,7 @@ public func __checkClosureCall( await callExitTest( identifiedBy: exitTestID, encodingCapturedValues: Array(repeat each capturedValues), - exitsWith: expectedExitCondition, + processExitsWith: expectedExitCondition, observing: observedValues, expression: expression, comments: comments(), diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index b8c48aa79..bca788ec7 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -217,7 +217,6 @@ public struct Configuration: Sendable { /// When using the `swift test` command from Swift Package Manager, this /// property is pre-configured. Otherwise, the default value of this property /// records an issue indicating that it has not been configured. - @_spi(Experimental) public var exitTestHandler: ExitTest.Handler = { exitTest in throw SystemError(description: "Exit test support has not been implemented by the current testing infrastructure.") } diff --git a/Sources/Testing/SourceAttribution/Expression.swift b/Sources/Testing/SourceAttribution/Expression.swift index dce4ed2a2..a294a81e0 100644 --- a/Sources/Testing/SourceAttribution/Expression.swift +++ b/Sources/Testing/SourceAttribution/Expression.swift @@ -22,9 +22,8 @@ /// let swiftSyntaxExpr: ExprSyntax = "\(testExpr)" /// ``` /// -/// - Warning: This type is used to implement the `#expect(exitsWith:)` -/// macro. Do not use it directly. Tools can use the SPI ``Expression`` -/// typealias if needed. +/// - Warning: This type is used to implement the `#expect()` macro. Do not use +/// it directly. Tools can use the SPI ``Expression`` typealias if needed. public struct __Expression: Sendable { /// An enumeration describing the various kinds of expression that can be /// captured. diff --git a/Sources/Testing/Testing.docc/Expectations.md b/Sources/Testing/Testing.docc/Expectations.md index fd3b0070d..e55ff3a1e 100644 --- a/Sources/Testing/Testing.docc/Expectations.md +++ b/Sources/Testing/Testing.docc/Expectations.md @@ -72,6 +72,14 @@ the test when the code doesn't satisfy a requirement, use - ``require(throws:_:sourceLocation:performing:)-4djuw`` - ``require(_:sourceLocation:performing:throws:)`` +### Checking how processes exit + +- +- ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` +- ``require(processExitsWith:observing:_:sourceLocation:performing:)`` +- ``ExitStatus`` +- ``ExitTest`` + ### Confirming that asynchronous events occur - diff --git a/Sources/Testing/Testing.docc/OrganizingTests.md b/Sources/Testing/Testing.docc/OrganizingTests.md index f2b577eb4..3464db4ae 100644 --- a/Sources/Testing/Testing.docc/OrganizingTests.md +++ b/Sources/Testing/Testing.docc/OrganizingTests.md @@ -124,7 +124,7 @@ struct MenuTests { The compiler emits an error when presented with a test suite that doesn't meet this requirement. -### Test suite types must always be available +#### Test suite types must always be available Although `@available` can be applied to a test function to limit its availability at runtime, a test suite type (and any types that contain it) must diff --git a/Sources/Testing/Testing.docc/exit-testing.md b/Sources/Testing/Testing.docc/exit-testing.md new file mode 100644 index 000000000..06ab53dc9 --- /dev/null +++ b/Sources/Testing/Testing.docc/exit-testing.md @@ -0,0 +1,155 @@ +# Exit testing + + + +@Metadata { + @Available(Swift, introduced: 6.2) +} + +Use exit tests to test functionality that might cause a test process to exit. + +## Overview + +Your code might contain calls to [`precondition()`](https://developer.apple.com/documentation/swift/precondition(_:_:file:line:)), +[`fatalError()`](https://developer.apple.com/documentation/swift/fatalerror(_:file:line:)), +or other functions that can cause the current process to exit. For example: + +```swift +extension Customer { + func eat(_ food: consuming some Food) { + precondition(food.isDelicious, "Tasty food only!") + precondition(food.isNutritious, "Healthy food only!") + ... + } +} +``` + +In this function, if `food.isDelicious` or `food.isNutritious` is `false`, the +precondition fails and Swift forces the process to exit. You can write an exit +test to validate preconditions like the ones above and to make sure that your +functions correctly catch invalid inputs. + +- Note: Exit tests are available on macOS, Linux, FreeBSD, OpenBSD, and Windows. + +### Create an exit test + +To create an exit test, call either the ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` +or the ``require(processExitsWith:observing:_:sourceLocation:performing:)`` +macro: + +```swift +@Test func `Customer won't eat food unless it's delicious`() async { + let result = await #expect(processExitsWith: .failure) { + var food = ... + food.isDelicious = false + Customer.current.eat(food) + } +} +``` + +The closure or function reference you pass to the macro is the _body_ of the +exit test. When an exit test is performed at runtime, the testing library starts +a new process with the same executable as the current process. The current task +is then suspended (as with `await`) and waits for the child process to exit. + +- Note: An exit test cannot run within another exit test. + +The parent process doesn't call the body of the exit test. Instead, the child +process treats the body of the exit test as its `main()` function and calls it +directly. + +- Note: Because the body acts as the `main()` function of a new process, it + can't capture any state originating in the parent process or from its lexical + context. For example, the following exit test will fail to compile because it + captures a variable declared outside the exit test itself: + + ```swift + @Test func `Customer won't eat food unless it's nutritious`() async { + let isNutritious = false + await #expect(processExitsWith: .failure) { + var food = ... + food.isNutritious = isNutritious // ❌ ERROR: trying to capture state here + Customer.current.eat(food) + } + } + ``` + +If the body returns before the child process exits, the process exits as if +`main()` returned normally. If the body throws an error, Swift handles it as if +it were thrown from `main()` and forces the process to exit abnormally. + +### Specify an exit condition + +When you create an exit test, specify how you expect the child process exits by +passing an instance of ``ExitTest/Condition``: + +- If you expect the exit test's body to run to completion or exit normally (for + example, by calling [`exit(EXIT_SUCCESS)`](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/exit.3.html) + from the C standard library), pass ``ExitTest/Condition/success``. +- If you expect the body to cause the child process to exit abnormally, but the + exact status reported by the system is not important, pass + ``ExitTest/Condition/failure``. +- If you need to check for a specific exit code or signal, pass + ``ExitTest/Condition/exitCode(_:)`` or ``ExitTest/Condition/signal(_:)``. + +When the child process exits, the parent process resumes and compares the exit +status of the child process against the expected exit condition you passed. If +they match, the exit test passes; otherwise, it fails and the testing library +records an issue. + +### Gather output from the child process + +The ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` and +``require(processExitsWith:observing:_:sourceLocation:performing:)`` macros +return an instance of ``ExitTest/Result`` that contains information about the +state of the child process. + +By default, the child process is configured without a standard output or +standard error stream. If your test needs to review the content of either of +these streams, pass the key path to the corresponding ``ExitTest/Result`` +property to the macro: + +```swift +extension Customer { + func eat(_ food: consuming some Food) { + print("Let's see if I want to eat \(food)...") + precondition(food.isDelicious, "Tasty food only!") + precondition(food.isNutritious, "Healthy food only!") + ... + } +} + +@Test func `Customer won't eat food unless it's delicious`() async { + let result = await #expect( + processExitsWith: .failure, + observing: [\.standardOutputContent] + ) { + var food = ... + food.isDelicious = false + Customer.current.eat(food) + } + if let result { + #expect(result.standardOutputContent.contains(UInt8(ascii: "L"))) + } +} +``` + +- Note: The content of the standard output and standard error streams can + contain any arbitrary sequence of bytes, including sequences that aren't valid + UTF-8 and can't be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + These streams are globally accessible within the child process, and any code + running in an exit test may write to it including the operating system and any + third-party dependencies you declare in your package description or Xcode + project. + +The testing library always sets ``ExitTest/Result/exitStatus`` to the actual +exit status of the child process (as reported by the system) even if you do not +pass it. diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index e21938041..49630cfc9 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -14,7 +14,7 @@ import SwiftSyntaxBuilder public import SwiftSyntaxMacros #if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY -#error("Platform-specific misconfiguration: either SymbolLinkageMarkers or legacy test discovery is required to expand #expect(exitsWith:)") +#error("Platform-specific misconfiguration: either SymbolLinkageMarkers or legacy test discovery is required to expand #expect(processExitsWith:)") #endif /// A protocol containing the common implementation for the expansions of the @@ -628,7 +628,7 @@ extension ExitTestExpectMacro { }() } -/// A type describing the expansion of the `#expect(exitsWith:)` macro. +/// A type describing the expansion of the `#expect(processExitsWith:)` macro. /// /// This type checks for nested invocations of `#expect()` and `#require()` and /// diagnoses them as unsupported. It is otherwise exactly equivalent to @@ -637,7 +637,7 @@ public struct ExitTestExpectMacro: ExitTestConditionMacro { public typealias Base = ExpectMacro } -/// A type describing the expansion of the `#require(exitsWith:)` macro. +/// A type describing the expansion of the `#require(processExitsWith:)` macro. /// /// This type checks for nested invocations of `#expect()` and `#require()` and /// diagnoses them as unsupported. It is otherwise exactly equivalent to diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index 67531dabf..dc36af7cd 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -437,13 +437,13 @@ struct ConditionMacroTests { } #if ExperimentalExitTestValueCapture - @Test("#expect(exitsWith:) produces a diagnostic for a bad capture", + @Test("#expect(processExitsWith:) produces a diagnostic for a bad capture", arguments: [ - "#expectExitTest(exitsWith: x) { [weak a] in }": + "#expectExitTest(processExitsWith: x) { [weak a] in }": "Specifier 'weak' cannot be used with captured value 'a'", - "#expectExitTest(exitsWith: x) { [a] in }": + "#expectExitTest(processExitsWith: x) { [a] in }": "Type of captured value 'a' is ambiguous", - "#expectExitTest(exitsWith: x) { [a = b] in }": + "#expectExitTest(processExitsWith: x) { [a = b] in }": "Type of captured value 'a' is ambiguous", ] ) @@ -463,8 +463,8 @@ struct ConditionMacroTests { @Test( "Capture list on an exit test produces a diagnostic", arguments: [ - "#expectExitTest(exitsWith: x) { [a] in }": - "Cannot specify a capture clause in closure passed to '#expectExitTest(exitsWith:_:)'" + "#expectExitTest(processExitsWith: x) { [a] in }": + "Cannot specify a capture clause in closure passed to '#expectExitTest(processExitsWith:_:)'" ] ) func exitTestCaptureListProducesDiagnostic(input: String, expectedMessage: String) throws { diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 0281b4091..be940371e 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -554,7 +554,7 @@ extension AttachmentTests { #if !SWT_NO_EXIT_TESTS @available(_uttypesAPI, *) @Test func cannotAttachCGImageWithNonImageType() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { let attachment = Attachment(try Self.cgImage.get(), named: "diamond", as: .mp3) try attachment.attachableValue.withUnsafeBytes(for: attachment) { _ in } } diff --git a/Tests/TestingTests/ConfirmationTests.swift b/Tests/TestingTests/ConfirmationTests.swift index 5502cb8d2..c4f076268 100644 --- a/Tests/TestingTests/ConfirmationTests.swift +++ b/Tests/TestingTests/ConfirmationTests.swift @@ -76,7 +76,7 @@ struct ConfirmationTests { await confirmation(expectedCount: Int.max...Int.max) { _ in } #if !SWT_NO_EXIT_TESTS await withKnownIssue("Crashes in Swift standard library (rdar://139568287)") { - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { await confirmation(expectedCount: Int.max...) { _ in } } } @@ -87,10 +87,10 @@ struct ConfirmationTests { #if !SWT_NO_EXIT_TESTS @Test("Confirmation requires positive count") func positiveCount() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { await confirmation { $0.confirm(count: 0) } } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { await confirmation { $0.confirm(count: -1) } } } diff --git a/Tests/TestingTests/DiscoveryTests.swift b/Tests/TestingTests/DiscoveryTests.swift index 8ec185813..24d2eecfa 100644 --- a/Tests/TestingTests/DiscoveryTests.swift +++ b/Tests/TestingTests/DiscoveryTests.swift @@ -49,10 +49,10 @@ struct DiscoveryTests { #if !SWT_NO_EXIT_TESTS @Test("TestContentKind rejects bad string literals") func badTestContentKindLiteral() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { _ = "abc" as TestContentKind } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { _ = "abcde" as TestContentKind } } diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 896784f22..02be1a140 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -14,26 +14,26 @@ private import _TestingInternals #if !SWT_NO_EXIT_TESTS @Suite("Exit test tests") struct ExitTestTests { @Test("Exit tests (passing)") func passing() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { exit(EXIT_FAILURE) } if EXIT_SUCCESS != EXIT_FAILURE + 1 { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { exit(EXIT_FAILURE + 1) } } - await #expect(exitsWith: .success) {} - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) {} + await #expect(processExitsWith: .success) { exit(EXIT_SUCCESS) } - await #expect(exitsWith: .exitCode(123)) { + await #expect(processExitsWith: .exitCode(123)) { exit(123) } - await #expect(exitsWith: .exitCode(123)) { + await #expect(processExitsWith: .exitCode(123)) { await Task.yield() exit(123) } - await #expect(exitsWith: .signal(SIGSEGV)) { + await #expect(processExitsWith: .signal(SIGSEGV)) { _ = raise(SIGSEGV) // Allow up to 1s for the signal to be delivered. On some platforms, // raise() delivers signals fully asynchronously and may not terminate the @@ -44,7 +44,7 @@ private import _TestingInternals try await Task.sleep(nanoseconds: 1_000_000_000) } } - await #expect(exitsWith: .signal(SIGABRT)) { + await #expect(processExitsWith: .signal(SIGABRT)) { abort() } #if !SWT_NO_UNSTRUCTURED_TASKS @@ -55,7 +55,7 @@ private import _TestingInternals #expect(Test.current != nil) await Task.detached { #expect(Test.current == nil) - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { fatalError() } }.value @@ -88,29 +88,29 @@ private import _TestingInternals // Mock an exit test where the process exits successfully. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(exitStatus: .exitCode(EXIT_SUCCESS)) } await Test { - await #expect(exitsWith: .success) {} + await #expect(processExitsWith: .success) {} }.run(configuration: configuration) // Mock an exit test where the process exits with a particular error code. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .exitCode(123)) + return ExitTest.Result(exitStatus: .exitCode(123)) } await Test { - await #expect(exitsWith: .failure) {} + await #expect(processExitsWith: .failure) {} }.run(configuration: configuration) // Mock an exit test where the process exits with a signal. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .signal(SIGABRT)) + return ExitTest.Result(exitStatus: .signal(SIGABRT)) } await Test { - await #expect(exitsWith: .signal(SIGABRT)) {} + await #expect(processExitsWith: .signal(SIGABRT)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .failure) {} + await #expect(processExitsWith: .failure) {} }.run(configuration: configuration) } } @@ -126,30 +126,30 @@ private import _TestingInternals // Mock exit tests that were expected to fail but passed. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(exitStatus: .exitCode(EXIT_SUCCESS)) } await Test { - await #expect(exitsWith: .failure) {} + await #expect(processExitsWith: .failure) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .exitCode(EXIT_FAILURE)) {} + await #expect(processExitsWith: .exitCode(EXIT_FAILURE)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .signal(SIGABRT)) {} + await #expect(processExitsWith: .signal(SIGABRT)) {} }.run(configuration: configuration) // Mock exit tests that unexpectedly signalled. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .signal(SIGABRT)) + return ExitTest.Result(exitStatus: .signal(SIGABRT)) } await Test { - await #expect(exitsWith: .exitCode(EXIT_SUCCESS)) {} + await #expect(processExitsWith: .exitCode(EXIT_SUCCESS)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .exitCode(EXIT_FAILURE)) {} + await #expect(processExitsWith: .exitCode(EXIT_FAILURE)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .success) {} + await #expect(processExitsWith: .success) {} }.run(configuration: configuration) } } @@ -164,7 +164,7 @@ private import _TestingInternals } await Test { - await #expect(exitsWith: .success) {} + await #expect(processExitsWith: .success) {} }.run(configuration: configuration) } } @@ -186,11 +186,11 @@ private import _TestingInternals configuration.exitTestHandler = ExitTest.handlerForEntryPoint() await Test { - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { #expect(Bool(false), "Something went wrong!") exit(0) } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { Issue.record(MyError()) } }.run(configuration: configuration) @@ -219,7 +219,7 @@ private import _TestingInternals // // Windows does not have the 8-bit exit code restriction and always reports // the full CInt value back to the testing library. - await #expect(exitsWith: .exitCode(512)) { + await #expect(processExitsWith: .exitCode(512)) { exit(512) } } @@ -232,7 +232,7 @@ private import _TestingInternals @Test("Exit test can be main-actor-isolated") @MainActor func mainActorIsolation() async { - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { await Self.someMainActorFunction() _ = 0 exit(EXIT_SUCCESS) @@ -242,24 +242,24 @@ private import _TestingInternals @Test("Result is set correctly on success") func successfulArtifacts() async throws { // Test that basic passing exit tests produce the correct results (#expect) - var result = await #expect(exitsWith: .success) { + var result = await #expect(processExitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) - result = await #expect(exitsWith: .exitCode(123)) { + #expect(result?.exitStatus == .exitCode(EXIT_SUCCESS)) + result = await #expect(processExitsWith: .exitCode(123)) { exit(123) } - #expect(result?.statusAtExit == .exitCode(123)) + #expect(result?.exitStatus == .exitCode(123)) // Test that basic passing exit tests produce the correct results (#require) - result = try await #require(exitsWith: .success) { + result = try await #require(processExitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) - result = try await #require(exitsWith: .exitCode(123)) { + #expect(result?.exitStatus == .exitCode(EXIT_SUCCESS)) + result = try await #require(processExitsWith: .exitCode(123)) { exit(123) } - #expect(result?.statusAtExit == .exitCode(123)) + #expect(result?.exitStatus == .exitCode(123)) } @Test("Result is nil on failure") @@ -278,11 +278,11 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTest.Result(statusAtExit: .exitCode(123)) + ExitTest.Result(exitStatus: .exitCode(123)) } await Test { - let result = await #expect(exitsWith: .success) {} + let result = await #expect(processExitsWith: .success) {} #expect(result == nil) }.run(configuration: configuration) } @@ -301,11 +301,11 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) + ExitTest.Result(exitStatus: .exitCode(EXIT_FAILURE)) } await Test { - try await #require(exitsWith: .success) {} + try await #require(processExitsWith: .success) {} fatalError("Unreachable") }.run(configuration: configuration) } @@ -334,7 +334,7 @@ private import _TestingInternals } await Test { - let result = await #expect(exitsWith: .success) {} + let result = await #expect(processExitsWith: .success) {} #expect(result == nil) }.run(configuration: configuration) } @@ -343,21 +343,21 @@ private import _TestingInternals @Test("Result contains stdout/stderr") func exitTestResultContainsStandardStreams() async throws { - var result = try await #require(exitsWith: .success, observing: [\.standardOutputContent]) { + var result = try await #require(processExitsWith: .success, observing: [\.standardOutputContent]) { try FileHandle.stdout.write("STANDARD OUTPUT") try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) + #expect(result.exitStatus == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.contains("STANDARD OUTPUT".utf8)) #expect(result.standardErrorContent.isEmpty) - result = try await #require(exitsWith: .success, observing: [\.standardErrorContent]) { + result = try await #require(processExitsWith: .success, observing: [\.standardErrorContent]) { try FileHandle.stdout.write("STANDARD OUTPUT") try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) + #expect(result.exitStatus == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.isEmpty) #expect(result.standardErrorContent.contains("STANDARD ERROR".utf8.reversed())) } @@ -368,7 +368,7 @@ private import _TestingInternals func nonConstExitCondition() async throws -> ExitTest.Condition { .failure } - await #expect(exitsWith: try await nonConstExitCondition(), sourceLocation: unrelatedSourceLocation) { + await #expect(processExitsWith: try await nonConstExitCondition(), sourceLocation: unrelatedSourceLocation) { fatalError() } } @@ -376,7 +376,7 @@ private import _TestingInternals @Test("ExitTest.current property") func currentProperty() async { #expect((ExitTest.current == nil) as Bool) - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { #expect((ExitTest.current != nil) as Bool) } } @@ -386,7 +386,7 @@ private import _TestingInternals func captureList() async { let i = 123 let s = "abc" as Any - await #expect(exitsWith: .success) { [i = i as Int, s = s as! String, t = (s as Any) as? String?] in + await #expect(processExitsWith: .success) { [i = i as Int, s = s as! String, t = (s as Any) as? String?] in #expect(i == 123) #expect(s == "abc") #expect(t == "abc") @@ -397,7 +397,7 @@ private import _TestingInternals func longCaptureList() async { let count = 1 * 1024 * 1024 let buffer = Array(repeatElement(0 as UInt8, count: count)) - await #expect(exitsWith: .success) { [count = count as Int, buffer = buffer as [UInt8]] in + await #expect(processExitsWith: .success) { [count = count as Int, buffer = buffer as [UInt8]] in #expect(buffer.count == count) } } @@ -407,7 +407,7 @@ private import _TestingInternals @Test("self in capture list") func captureListWithSelf() async { - await #expect(exitsWith: .success) { [self, x = self] in + await #expect(processExitsWith: .success) { [self, x = self] in #expect(self.property == 456) #expect(x.property == 456) } @@ -444,13 +444,13 @@ private import _TestingInternals @Test("Capturing an instance of a subclass") func captureSubclass() async { let instance = CapturableDerivedClass(x: 123) - await #expect(exitsWith: .success) { [instance = instance as CapturableBaseClass] in + await #expect(processExitsWith: .success) { [instance = instance as CapturableBaseClass] in #expect((instance as AnyObject) is CapturableBaseClass) // However, because the static type of `instance` is not Derived, we won't // be able to cast it to Derived. #expect(!((instance as AnyObject) is CapturableDerivedClass)) } - await #expect(exitsWith: .success) { [instance = instance as CapturableDerivedClass] in + await #expect(processExitsWith: .success) { [instance = instance as CapturableDerivedClass] in #expect((instance as AnyObject) is CapturableBaseClass) #expect((instance as AnyObject) is CapturableDerivedClass) #expect(instance.x == 123) @@ -463,28 +463,28 @@ private import _TestingInternals @Suite(.hidden) struct FailingExitTests { @Test(.hidden) func failingExitTests() async { - await #expect(exitsWith: .failure) {} - await #expect(exitsWith: .exitCode(123)) {} - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) {} + await #expect(processExitsWith: .exitCode(123)) {} + await #expect(processExitsWith: .failure) { exit(EXIT_SUCCESS) } - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { exit(EXIT_FAILURE) } - await #expect(exitsWith: .exitCode(123)) { + await #expect(processExitsWith: .exitCode(123)) { exit(0) } - await #expect(exitsWith: .exitCode(SIGABRT)) { + await #expect(processExitsWith: .exitCode(SIGABRT)) { // abort() raises on Windows, but we don't handle that yet and it is // reported as .failure (which will fuzzy-match with SIGABRT.) abort() } - await #expect(exitsWith: .signal(123)) {} - await #expect(exitsWith: .signal(123)) { + await #expect(processExitsWith: .signal(123)) {} + await #expect(processExitsWith: .signal(123)) { exit(123) } - await #expect(exitsWith: .signal(SIGSEGV)) { + await #expect(processExitsWith: .signal(SIGSEGV)) { abort() // sends SIGABRT, not SIGSEGV } } @@ -493,7 +493,7 @@ private import _TestingInternals #if false // intentionally fails to compile @Test(.hidden, arguments: 100 ..< 200) func sellIceCreamCones(count: Int) async throws { - try await #require(exitsWith: .failure) { + try await #require(processExitsWith: .failure) { precondition(count < 10, "Too many ice cream cones") } } diff --git a/Tests/TestingTests/PlanIterationTests.swift b/Tests/TestingTests/PlanIterationTests.swift index 3892b2fb8..21d71f894 100644 --- a/Tests/TestingTests/PlanIterationTests.swift +++ b/Tests/TestingTests/PlanIterationTests.swift @@ -117,11 +117,11 @@ struct PlanIterationTests { #if !SWT_NO_EXIT_TESTS @Test("Iteration count must be positive") func positiveIterationCount() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var configuration = Configuration() configuration.repetitionPolicy.maximumIterationCount = 0 } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var configuration = Configuration() configuration.repetitionPolicy.maximumIterationCount = -1 } diff --git a/Tests/TestingTests/SourceLocationTests.swift b/Tests/TestingTests/SourceLocationTests.swift index 75a8791db..4145687b8 100644 --- a/Tests/TestingTests/SourceLocationTests.swift +++ b/Tests/TestingTests/SourceLocationTests.swift @@ -82,27 +82,27 @@ struct SourceLocationTests { #if !SWT_NO_EXIT_TESTS @Test("SourceLocation.init requires well-formed arguments") func sourceLocationInitPreconditions() async { - await #expect(exitsWith: .failure, "Empty fileID") { + await #expect(processExitsWith: .failure, "Empty fileID") { _ = SourceLocation(fileID: "", filePath: "", line: 1, column: 1) } - await #expect(exitsWith: .failure, "Invalid fileID") { + await #expect(processExitsWith: .failure, "Invalid fileID") { _ = SourceLocation(fileID: "B.swift", filePath: "", line: 1, column: 1) } - await #expect(exitsWith: .failure, "Zero line") { + await #expect(processExitsWith: .failure, "Zero line") { _ = SourceLocation(fileID: "A/B.swift", filePath: "", line: 0, column: 1) } - await #expect(exitsWith: .failure, "Zero column") { + await #expect(processExitsWith: .failure, "Zero column") { _ = SourceLocation(fileID: "A/B.swift", filePath: "", line: 1, column: 0) } } @Test("SourceLocation.fileID property must be well-formed") func sourceLocationFileIDWellFormed() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.fileID = "" } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.fileID = "ABC" } @@ -110,11 +110,11 @@ struct SourceLocationTests { @Test("SourceLocation.line and column properties must be positive") func sourceLocationLineAndColumnPositive() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.line = -1 } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.column = -1 } diff --git a/Tests/TestingTests/Support/FileHandleTests.swift b/Tests/TestingTests/Support/FileHandleTests.swift index c837ac7cf..4be633ad6 100644 --- a/Tests/TestingTests/Support/FileHandleTests.swift +++ b/Tests/TestingTests/Support/FileHandleTests.swift @@ -85,7 +85,7 @@ struct FileHandleTests { #if !SWT_NO_EXIT_TESTS @Test("Writing requires contiguous storage") func writeIsContiguous() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { let fileHandle = try FileHandle.null(mode: "wb") try fileHandle.write([1, 2, 3, 4, 5].lazy.filter { $0 == 1 }) } From a82d0a89dd2eb86d18056f03396055528ce56508 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 1 May 2025 12:34:19 -0500 Subject: [PATCH 188/234] Work around a macOS CI failure (#1100) This works around a macOS CI failure. A revert in the Swift compiler (in https://github.com/swiftlang/swift/pull/80830) is expected to resolve it, but the release of a newer `main` development snapshot toolchain is blocked for unrelated reasons. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Tests/TestingTests/IssueTests.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 6ea1a5827..cc0a7acf5 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -491,6 +491,7 @@ final class IssueTests: XCTestCase { }.run(configuration: .init()) } +#if !SWT_TARGET_OS_APPLE || SWT_FIXED_149299786 func testErrorCheckingWithExpect() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.isInverted = true @@ -610,6 +611,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } +#endif func testErrorCheckingWithExpect_mismatchedErrorDescription() async throws { let expectationFailed = expectation(description: "Expectation failed") From 6e4fe1f913fd6b287c247cb2a8e164fd2435920e Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Sat, 3 May 2025 08:07:12 -0500 Subject: [PATCH 189/234] Restore DocC `@Comment` blocks to documentation line comments (#1103) --- Sources/Testing/Attachments/Attachment.swift | 10 ++-- Sources/Testing/Issues/Issue.swift | 24 ++++----- .../Test.Case.Generator.swift | 30 +++++------ .../Testing/Support/CartesianProduct.swift | 20 ++++---- Sources/Testing/Test+Macro.swift | 50 +++++++++---------- Sources/Testing/Traits/ConditionTrait.swift | 24 ++++----- .../TestSupport/TestingAdditions.swift | 20 ++++---- 7 files changed, 89 insertions(+), 89 deletions(-) diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index 7468834bf..366e288d1 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -131,11 +131,11 @@ extension Attachment where AttachableValue == AnyAttachable { /// events of kind ``Event/Kind/valueAttached(_:)``. Test tools authors who use /// `@_spi(ForToolsIntegrationOnly)` will see instances of this type when /// handling those events. -// -// @Comment { -// Swift's type system requires that this type be at least as visible as -// `Event.Kind.valueAttached(_:)`, otherwise it would be declared private. -// } +/// +/// @Comment { +/// Swift's type system requires that this type be at least as visible as +/// `Event.Kind.valueAttached(_:)`, otherwise it would be declared private. +/// } @_spi(ForToolsIntegrationOnly) public struct AnyAttachable: AttachableWrapper, Copyable, Sendable { #if !SWT_NO_LAZY_ATTACHMENTS diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 4a17cb945..9a2555177 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -49,12 +49,12 @@ public struct Issue: Sendable { /// /// - Parameters: /// - timeLimitComponents: The time limit reached by the test. - // - // @Comment { - // - Bug: The associated value of this enumeration case should be an - // instance of `Duration`, but the testing library's deployment target - // predates the introduction of that type. - // } + /// + /// @Comment { + /// - Bug: The associated value of this enumeration case should be an + /// instance of `Duration`, but the testing library's deployment target + /// predates the introduction of that type. + /// } indirect case timeLimitExceeded(timeLimitComponents: (seconds: Int64, attoseconds: Int64)) /// A known issue was expected, but was not recorded. @@ -434,12 +434,12 @@ extension Issue.Kind { /// /// - Parameters: /// - timeLimitComponents: The time limit reached by the test. - // - // @Comment { - // - Bug: The associated value of this enumeration case should be an - // instance of `Duration`, but the testing library's deployment target - // predates the introduction of that type. - // } + /// + /// @Comment { + /// - Bug: The associated value of this enumeration case should be an + /// instance of `Duration`, but the testing library's deployment target + /// predates the introduction of that type. + /// } indirect case timeLimitExceeded(timeLimitComponents: (seconds: Int64, attoseconds: Int64)) /// A known issue was expected, but was not recorded. diff --git a/Sources/Testing/Parameterization/Test.Case.Generator.swift b/Sources/Testing/Parameterization/Test.Case.Generator.swift index d30e3a7d3..05467d9bd 100644 --- a/Sources/Testing/Parameterization/Test.Case.Generator.swift +++ b/Sources/Testing/Parameterization/Test.Case.Generator.swift @@ -13,11 +13,11 @@ extension Test.Case { /// a known collection of argument values. /// /// Instances of this type can be iterated over multiple times. - // - // @Comment { - // - Bug: The testing library should support variadic generics. - // ([103416861](rdar://103416861)) - // } + /// + /// @Comment { + /// - Bug: The testing library should support variadic generics. + /// ([103416861](rdar://103416861)) + /// } struct Generator: Sendable where S: Sequence & Sendable, S.Element: Sendable { /// The underlying sequence of argument values. /// @@ -146,11 +146,11 @@ extension Test.Case { /// /// This initializer overload is specialized for sequences of 2-tuples to /// efficiently de-structure their elements when appropriate. - // - // @Comment { - // - Bug: The testing library should support variadic generics. - // ([103416861](rdar://103416861)) - // } + /// + /// @Comment { + /// - Bug: The testing library should support variadic generics. + /// ([103416861](rdar://103416861)) + /// } private init( sequence: S, parameters: [Test.Parameter], @@ -184,11 +184,11 @@ extension Test.Case { /// /// This initializer overload is specialized for collections of 2-tuples to /// efficiently de-structure their elements when appropriate. - // - // @Comment { - // - Bug: The testing library should support variadic generics. - // ([103416861](rdar://103416861)) - // } + /// + /// @Comment { + /// - Bug: The testing library should support variadic generics. + /// ([103416861](rdar://103416861)) + /// } init( arguments collection: S, parameters: [Test.Parameter], diff --git a/Sources/Testing/Support/CartesianProduct.swift b/Sources/Testing/Support/CartesianProduct.swift index 07b164eb5..43d92e462 100644 --- a/Sources/Testing/Support/CartesianProduct.swift +++ b/Sources/Testing/Support/CartesianProduct.swift @@ -17,11 +17,11 @@ /// `[(1, "a"), (1, "b"), (1, "c"), (2, "a"), (2, "b"), ... (3, "c")]`. /// /// This type is not part of the public interface of the testing library. -// -// @Comment { -// - Bug: The testing library should support variadic generics. -// ([103416861](rdar://103416861)) -// } +/// +/// @Comment { +/// - Bug: The testing library should support variadic generics. +/// ([103416861](rdar://103416861)) +/// } struct CartesianProduct: LazySequenceProtocol where C1: Collection, C2: Collection { fileprivate var collection1: C1 fileprivate var collection2: C2 @@ -63,11 +63,11 @@ extension CartesianProduct: Sendable where C1: Sendable, C2: Sendable {} /// while `collection2` is iterated `collection1.count` times. /// /// For more information on Cartesian products, see ``CartesianProduct``. -// -// @Comment { -// - Bug: The testing library should support variadic generics. -// ([103416861](rdar://103416861)) -// } +/// +/// @Comment { +/// - Bug: The testing library should support variadic generics. +/// ([103416861](rdar://103416861)) +/// } func cartesianProduct(_ collection1: C1, _ collection2: C2) -> CartesianProduct where C1: Collection, C2: Collection { CartesianProduct(collection1: collection1, collection2: collection2) } diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index d1ad6623b..be0b5a91b 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -220,14 +220,14 @@ public macro Test( /// During testing, the associated test function is called once for each element /// in `collection`. /// +/// @Comment { +/// - Bug: The testing library should support variadic generics. +/// ([103416861](rdar://103416861)) +/// } +/// /// ## See Also /// /// - -// -// @Comment { -// - Bug: The testing library should support variadic generics. -// ([103416861](rdar://103416861)) -// } @attached(peer) public macro Test( _ displayName: _const String? = nil, _ traits: any TestTrait..., @@ -273,14 +273,14 @@ extension Test { /// During testing, the associated test function is called once for each pair of /// elements in `collection1` and `collection2`. /// +/// @Comment { +/// - Bug: The testing library should support variadic generics. +/// ([103416861](rdar://103416861)) +/// } +/// /// ## See Also /// /// - -// -// @Comment { -// - Bug: The testing library should support variadic generics. -// ([103416861](rdar://103416861)) -// } @attached(peer) @_documentation(visibility: private) public macro Test( @@ -301,14 +301,14 @@ public macro Test( /// During testing, the associated test function is called once for each pair of /// elements in `collection1` and `collection2`. /// +/// @Comment { +/// - Bug: The testing library should support variadic generics. +/// ([103416861](rdar://103416861)) +/// } +/// /// ## See Also /// /// - -// -// @Comment { -// - Bug: The testing library should support variadic generics. -// ([103416861](rdar://103416861)) -// } @attached(peer) public macro Test( _ displayName: _const String? = nil, _ traits: any TestTrait..., @@ -327,14 +327,14 @@ public macro Test( /// During testing, the associated test function is called once for each element /// in `zippedCollections`. /// +/// @Comment { +/// - Bug: The testing library should support variadic generics. +/// ([103416861](rdar://103416861)) +/// } +/// /// ## See Also /// /// - -// -// @Comment { -// - Bug: The testing library should support variadic generics. -// ([103416861](rdar://103416861)) -// } @attached(peer) @_documentation(visibility: private) public macro Test( @@ -355,14 +355,14 @@ public macro Test( /// During testing, the associated test function is called once for each element /// in `zippedCollections`. /// +/// @Comment { +/// - Bug: The testing library should support variadic generics. +/// ([103416861](rdar://103416861)) +/// } +/// /// ## See Also /// /// - -// -// @Comment { -// - Bug: The testing library should support variadic generics. -// ([103416861](rdar://103416861)) -// } @attached(peer) public macro Test( _ displayName: _const String? = nil, _ traits: any TestTrait..., diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index eb3d1bd11..079b64d8e 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -115,12 +115,12 @@ extension Trait where Self == ConditionTrait { /// /// - Returns: An instance of ``ConditionTrait`` that evaluates the /// closure you provide. - // - // @Comment { - // - Bug: `condition` cannot be `async` without making this function - // `async` even though `condition` is not evaluated locally. - // ([103037177](rdar://103037177)) - // } + /// + /// @Comment { + /// - Bug: `condition` cannot be `async` without making this function + /// `async` even though `condition` is not evaluated locally. + /// ([103037177](rdar://103037177)) + /// } public static func enabled( if condition: @autoclosure @escaping @Sendable () throws -> Bool, _ comment: Comment? = nil, @@ -174,12 +174,12 @@ extension Trait where Self == ConditionTrait { /// /// - Returns: An instance of ``ConditionTrait`` that evaluates the /// closure you provide. - // - // @Comment { - // - Bug: `condition` cannot be `async` without making this function - // `async` even though `condition` is not evaluated locally. - // ([103037177](rdar://103037177)) - // } + /// + /// @Comment { + /// - Bug: `condition` cannot be `async` without making this function + /// `async` even though `condition` is not evaluated locally. + /// ([103037177](rdar://103037177)) + /// } public static func disabled( if condition: @autoclosure @escaping @Sendable () throws -> Bool, _ comment: Comment? = nil, diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 6807fd62a..4648f96af 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -162,11 +162,11 @@ extension Test { /// - testFunction: The function to call when running this test. During /// testing, this function is called once for each element in /// `collection`. - // - // @Comment { - // - Bug: The testing library should support variadic generics. - // ([103416861](rdar://103416861)) - // } + /// + /// @Comment { + /// - Bug: The testing library should support variadic generics. + /// ([103416861](rdar://103416861)) + /// } init( _ traits: any TestTrait..., arguments collection: C, @@ -191,11 +191,11 @@ extension Test { /// - testFunction: The function to call when running this test. During /// testing, this function is called once for each pair of elements in /// `collection1` and `collection2`. - // - // @Comment { - // - Bug: The testing library should support variadic generics. - // ([103416861](rdar://103416861)) - // } + /// + /// @Comment { + /// - Bug: The testing library should support variadic generics. + /// ([103416861](rdar://103416861)) + /// } init( _ traits: any TestTrait..., arguments collection1: C1, _ collection2: C2, From 91f7889dcb4f8215832d9af813f71f48d73ee4b7 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 6 May 2025 19:55:42 -0700 Subject: [PATCH 190/234] Work around compiler bug affecting macro decls with #if-guarded availability when building w/legacy driver (#1106) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This works around a Swift compiler bug which causes a failure validating the generated .swiftinterface of the `Testing` module due to it having macro declarations with `#if`-conditionalized `@available(...)` attributes _before_ any other `@`-attributes. The PR which recently landed to enable the Exit Tests feature (#324) revealed this compiler bug — specifically, that PR removed `@_spi` attributes which until then _preceded_ `#if SWT_NO_EXIT_TESTS`. The workaround is to move other attributes on the affected macro declarations up before the `#if`. The compiler bug is being fixed in https://github.com/swiftlang/swift/pull/81346. It only appears to happen when building with the legacy driver, and Android uses that driver still. An example CI failure log can be found here: > https://github.com/thebrowsercompany/swift-build/actions/runs/14823859186/job/41615678071#step:32:72 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Expectations/Expectation+Macro.swift | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index d14920547..f85c7042b 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -514,11 +514,12 @@ public macro require( /// @Metadata { /// @Available(Swift, introduced: 6.2) /// } +@freestanding(expression) +@discardableResult #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -@discardableResult -@freestanding(expression) public macro expect( +public macro expect( processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, @@ -559,11 +560,12 @@ public macro require( /// @Metadata { /// @Available(Swift, introduced: 6.2) /// } +@freestanding(expression) +@discardableResult #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -@discardableResult -@freestanding(expression) public macro require( +public macro require( processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, From 3ad851e7a4ea978ed1010b05d8e3c7dc0f5349ae Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 7 May 2025 12:53:21 -0700 Subject: [PATCH 191/234] Simplify usages of withTaskGroup to infer ChildTaskResult type where possible (#1102) This adjusts usages of `withTaskGroup` and `withThrowingTaskGroup` to take advantage of [SE-0442: Allow TaskGroup's ChildTaskResult Type To Be Inferred](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0442-allow-taskgroup-childtaskresult-type-to-be-inferred.md) by inferring the child task result type. I successfully built this PR using a Swift 6.1 toolchain. A couple usages I _did_ need to leave explicitly specified, but most I was able to simplify. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Running/Runner.swift | 4 ++-- Sources/Testing/Test+Discovery.swift | 4 ++-- Sources/Testing/Traits/TimeLimitTrait.swift | 2 +- Tests/TestingTests/Support/CartesianProductTests.swift | 2 +- Tests/TestingTests/Support/LockTests.swift | 2 +- Tests/TestingTests/Traits/TimeLimitTraitTests.swift | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 8520d1aaf..bd1167b8e 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -156,7 +156,7 @@ extension Runner { in sequence: some Sequence, _ body: @Sendable @escaping (E) async throws -> Void ) async throws where E: Sendable { - try await withThrowingTaskGroup(of: Void.self) { taskGroup in + try await withThrowingTaskGroup { taskGroup in for element in sequence { // Each element gets its own subtask to run in. _ = taskGroup.addTaskUnlessCancelled { @@ -430,7 +430,7 @@ extension Runner { Event.post(.iterationEnded(iterationIndex), for: (nil, nil), configuration: runner.configuration) } - await withTaskGroup(of: Void.self) { [runner] taskGroup in + await withTaskGroup { [runner] taskGroup in _ = taskGroup.addTaskUnlessCancelled { try? await _runStep(atRootOf: runner.plan.stepGraph) } diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 35f716525..5e9632d70 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -84,7 +84,7 @@ extension Test { // a task group and collate their results. if useNewMode { let generators = Generator.allTestContentRecords().lazy.compactMap { $0.load() } - await withTaskGroup(of: Self.self) { taskGroup in + await withTaskGroup { taskGroup in for generator in generators { taskGroup.addTask { await generator.rawValue() } } @@ -96,7 +96,7 @@ extension Test { // Perform legacy test discovery if needed. if useLegacyMode && result.isEmpty { let generators = Generator.allTypeMetadataBasedTestContentRecords().lazy.compactMap { $0.load() } - await withTaskGroup(of: Self.self) { taskGroup in + await withTaskGroup { taskGroup in for generator in generators { taskGroup.addTask { await generator.rawValue() } } diff --git a/Sources/Testing/Traits/TimeLimitTrait.swift b/Sources/Testing/Traits/TimeLimitTrait.swift index 4e84a1f92..54a200fb4 100644 --- a/Sources/Testing/Traits/TimeLimitTrait.swift +++ b/Sources/Testing/Traits/TimeLimitTrait.swift @@ -264,7 +264,7 @@ func withTimeLimit( _ body: @escaping @Sendable () async throws -> Void, timeoutHandler: @escaping @Sendable () -> Void ) async throws { - try await withThrowingTaskGroup(of: Void.self) { group in + try await withThrowingTaskGroup { group in group.addTask { // If sleep() returns instead of throwing a CancellationError, that means // the timeout was reached before this task could be cancelled, so call diff --git a/Tests/TestingTests/Support/CartesianProductTests.swift b/Tests/TestingTests/Support/CartesianProductTests.swift index b817b37f6..3cb4f6daf 100644 --- a/Tests/TestingTests/Support/CartesianProductTests.swift +++ b/Tests/TestingTests/Support/CartesianProductTests.swift @@ -96,7 +96,7 @@ struct CartesianProductTests { // Test that the product can be iterated multiple times concurrently. let (_, _, product) = computeCartesianProduct() let expectedSum = product.reduce(into: 0) { $0 &+= $1.1 } - await withTaskGroup(of: Int.self) { taskGroup in + await withTaskGroup { taskGroup in for _ in 0 ..< 10 { taskGroup.addTask { product.reduce(into: 0) { $0 &+= $1.1 } diff --git a/Tests/TestingTests/Support/LockTests.swift b/Tests/TestingTests/Support/LockTests.swift index 0113745e9..2a41e4c1d 100644 --- a/Tests/TestingTests/Support/LockTests.swift +++ b/Tests/TestingTests/Support/LockTests.swift @@ -36,7 +36,7 @@ struct LockTests { @Test("No lock") func noLock() async { let lock = LockedWith(rawValue: 0) - await withTaskGroup(of: Void.self) { taskGroup in + await withTaskGroup { taskGroup in for _ in 0 ..< 100_000 { taskGroup.addTask { lock.increment() diff --git a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift index b29ccb93c..49412b6af 100644 --- a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift +++ b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift @@ -181,7 +181,7 @@ struct TimeLimitTraitTests { @Test("Cancelled tests can exit early (cancellation checking works)") func cancelledTestExitsEarly() async throws { let timeAwaited = await Test.Clock().measure { - await withTaskGroup(of: Void.self) { taskGroup in + await withTaskGroup { taskGroup in taskGroup.addTask { await Test { try await Test.Clock.sleep(for: .seconds(60) * 60) From 1932a1b4c0899987ea85a80c01b446de39f54737 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 7 May 2025 15:37:57 -0700 Subject: [PATCH 192/234] Add missing Foundation imports to fix test build errors on iOS with MemberImportVisibility enabled (#1108) This fixes several instances of a build error when attempting to build this package for iOS, or any non-macOS Apple platform. Here's one example ``` error: instance method 'contains' is not available due to missing import of defining module 'Foundation' Tests/TestingTests/SwiftPMTests.swift:370:5: note: in expansion of macro 'expect' here #expect(testIDs.allSatisfy { $0.contains(".swift:") }) ``` This general kind of build error is being emitted because we adopted [SE-0444: Member import visibility](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0444-member-import-visibility.md) by enabling the `MemberImportVisibility` experimental feature in #1020. In that PR, I fixed several instances of missing imports, including some for `Foundation` in test files. But these errors are from usages of `String.contains()`, and it turns out there are multiple overloads of that function, with an older one in `Foundation` and a newer one directly in the stdlib `Swift` module. The latter has newer, iOS 13.0-aligned API availability, and when building our tests for macOS this issue was not noticed previously because SwiftPM artificially raises the deployment target of macOS test targets to match the testing frameworks included in Xcode (when the testing libraries are being used from the installed copy of Xcode). ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Tests/TestingTests/ConfirmationTests.swift | 4 ++++ Tests/TestingTests/MiscellaneousTests.swift | 4 ++++ Tests/TestingTests/SwiftPMTests.swift | 4 ++++ Tests/TestingTests/Traits/TagListTests.swift | 4 ++++ Tests/TestingTests/Traits/TimeLimitTraitTests.swift | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/Tests/TestingTests/ConfirmationTests.swift b/Tests/TestingTests/ConfirmationTests.swift index c4f076268..2551513eb 100644 --- a/Tests/TestingTests/ConfirmationTests.swift +++ b/Tests/TestingTests/ConfirmationTests.swift @@ -10,6 +10,10 @@ @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +#if canImport(Foundation) +private import Foundation +#endif + @Suite("Confirmation Tests") struct ConfirmationTests { @Test("Successful confirmations") diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index b4b12a217..9ae326afe 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -12,6 +12,10 @@ @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import _TestDiscovery private import _TestingInternals +#if canImport(Foundation) +private import Foundation +#endif + @Test(/* name unspecified */ .hidden) @Sendable func freeSyncFunction() {} @Sendable func freeAsyncFunction() async {} diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 6e7be0f15..eadde29a7 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -11,6 +11,10 @@ @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing private import _TestingInternals +#if canImport(Foundation) +private import Foundation +#endif + private func configurationForEntryPoint(withArguments args: [String]) throws -> Configuration { let args = try parseCommandLineArguments(from: args) return try configurationForEntryPoint(from: args) diff --git a/Tests/TestingTests/Traits/TagListTests.swift b/Tests/TestingTests/Traits/TagListTests.swift index 1ec8d1248..81cba285c 100644 --- a/Tests/TestingTests/Traits/TagListTests.swift +++ b/Tests/TestingTests/Traits/TagListTests.swift @@ -11,6 +11,10 @@ @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing private import _TestingInternals +#if canImport(Foundation) +private import Foundation +#endif + @Suite("Tag/Tag List Tests", .tags(.traitRelated)) struct TagListTests { @Test(".tags() factory method with one tag") diff --git a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift index 49412b6af..7d427f259 100644 --- a/Tests/TestingTests/Traits/TimeLimitTraitTests.swift +++ b/Tests/TestingTests/Traits/TimeLimitTraitTests.swift @@ -10,6 +10,10 @@ @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +#if canImport(Foundation) +private import Foundation +#endif + @Suite("TimeLimitTrait Tests", .tags(.traitRelated)) struct TimeLimitTraitTests { @available(_clockAPI, *) From 72afbb418542654781a6b7853479c7e70a862b6f Mon Sep 17 00:00:00 2001 From: Andrew Kaster Date: Tue, 13 May 2025 10:20:51 -0600 Subject: [PATCH 193/234] Documentation: Update CMake.md to use the ABI entry point (#828) The SwiftPM entry point is unstable and the new ABIv0 entry point has already been added to the library. ### Motivation: Using the SwiftPM entry point when building tests from a CMake project as recommended in the documentation is outdated and unwise. ### Modifications: Replace the example with one using the new ABIv0 entry point. ### Result: CMake projects should stop relying on the SwiftPM entry point. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/CMake.md | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/Documentation/CMake.md b/Documentation/CMake.md index e846f5641..37b9e1d4e 100644 --- a/Documentation/CMake.md +++ b/Documentation/CMake.md @@ -59,21 +59,38 @@ endif() ## Add an entry point You must include a source file in your test executable target with a -`@main` entry point. The following example uses the SwiftPM entry point: +`@main` entry point. The example main below requires the experimental +`Extern` feature. The function `swt_abiv0_getEntryPoint` is exported +from the swift-testing dylib. As such, its declaration could instead +be written in a C header file with its own `module.modulemap`, or +the runtime address could be obtained via +[`dlsym()`](https://pubs.opengroup.org/onlinepubs/9799919799/functions/dlsym.html) or +[`GetProcAddress()`](https://learn.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-getprocaddress). ```swift -import Testing +typealias EntryPoint = @convention(thin) @Sendable (_ configurationJSON: UnsafeRawBufferPointer?, _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void) async throws -> Bool + +@_extern(c, "swt_abiv0_getEntryPoint") +func swt_abiv0_getEntryPoint() -> UnsafeRawPointer @main struct Runner { - static func main() async { - await Testing.__swiftPMEntryPoint() as Never + static func main() async throws { + nonisolated(unsafe) let configurationJSON: UnsafeRawBufferPointer? = nil + let recordHandler: @Sendable (UnsafeRawBufferPointer) -> Void = { _ in } + + let entryPoint = unsafeBitCast(swt_abiv0_getEntryPoint(), to: EntryPoint.self) + + if try await entryPoint(configurationJSON, recordHandler) { + exit(EXIT_SUCCESS) + } else { + exit(EXIT_FAILURE) + } } } ``` -> [!WARNING] -> The entry point is expected to change to an entry point designed for other -> build systems prior to the initial stable release of Swift Testing. +For more information on the input configuration and output records of the ABI entry +point, refer to the [ABI documentation](ABI/JSON.md). ## Integrate with CTest From 464c01aa0b61383921b11a4bce82e05835a34a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Troiti=C3=B1o?= Date: Fri, 16 May 2025 08:43:56 -0700 Subject: [PATCH 194/234] [CMake] Fix usage of lowercase or in condition (#1117) CMake is a strange animal and some things are case sensitive, while others aren't. It seems that `if()` logical operator `OR` is case sensitive, and `or` was not cutting it. At least for CMake 3.26.4. If some CMake version allows `or`, it should also allow `OR`, so this change should not be a problem for those versions. Most of the time these pieces are not actually hit, because the build system sets `SwiftTesting_MACROS` to `NO`, so this is skipped, which might explain why it has not been a problem. --- Sources/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 8106eb2a3..c6575a310 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -66,7 +66,7 @@ if(SwiftTesting_MACRO STREQUAL "") if(NOT SwiftTesting_BuildMacrosAsExecutables) if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(SwiftTesting_MACRO_PATH "${SwiftTesting_MACRO_INSTALL_PREFIX}/lib/swift/host/plugins/testing/libTestingMacros.dylib") - elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux" or CMAKE_HOST_SYSTEM_NAME STREQUAL "FreeBSD") + elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_HOST_SYSTEM_NAME STREQUAL "FreeBSD") set(SwiftTesting_MACRO_PATH "${SwiftTesting_MACRO_INSTALL_PREFIX}/lib/swift/host/plugins/libTestingMacros.so") elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") set(SwiftTesting_MACRO_PATH "${SwiftTesting_MACRO_INSTALL_PREFIX}/bin/TestingMacros.dll") From 17c00b09b7d7a25233c379932b2a296a9f2514d9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 16 May 2025 11:47:31 -0500 Subject: [PATCH 195/234] Include total number of suites in "run ended" console message (#1116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This enhances the console output message shown when a test run finishes by including the total number of suites which ran or skipped, after the total number of test functions. Example: ``` ✔ Test run with 456 tests in 62 suites passed after 3.389 seconds. ``` The data was already being collected to support this, in a property named `suiteCount`, but it was not being used anywhere. So this PR adopts that property to augment the current "test run ended" message. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Event.HumanReadableOutputRecorder.swift | 5 ++- Tests/TestingTests/EventRecorderTests.swift | 39 +++++++++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 6abb71442..06d12de6e 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -542,6 +542,7 @@ extension Event.HumanReadableOutputRecorder { case .runEnded: let testCount = context.testCount + let suiteCount = context.suiteCount let issues = _issueCounts(in: context.testData) let runStartInstant = context.runStartInstant ?? instant let duration = runStartInstant.descriptionOfDuration(to: instant) @@ -550,14 +551,14 @@ extension Event.HumanReadableOutputRecorder { [ Message( symbol: .fail, - stringValue: "Test run with \(testCount.counting("test")) failed after \(duration)\(issues.description)." + stringValue: "Test run with \(testCount.counting("test")) in \(suiteCount.counting("suite")) failed after \(duration)\(issues.description)." ) ] } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "Test run with \(testCount.counting("test")) passed after \(duration)\(issues.description)." + stringValue: "Test run with \(testCount.counting("test")) in \(suiteCount.counting("suite")) passed after \(duration)\(issues.description)." ) ] } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 18f70186a..d06e12c7c 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -322,20 +322,29 @@ struct EventRecorderTests { print(buffer, terminator: "") } + let testCount = Reference() + let suiteCount = Reference() + let issueCount = Reference() + let knownIssueCount = Reference() + let runFailureRegex = Regex { One(.anyGraphemeCluster) " Test run with " - OneOrMore(.digit) + Capture(as: testCount) { OneOrMore(.digit) } transform: { Int($0) } " test" Optionally("s") + " in " + Capture(as: suiteCount) { OneOrMore(.digit) } transform: { Int($0) } + " suite" + Optionally("s") " failed " ZeroOrMore(.any) " with " - Capture { OneOrMore(.digit) } transform: { Int($0) } + Capture(as: issueCount) { OneOrMore(.digit) } transform: { Int($0) } " issue" Optionally("s") " (including " - Capture { OneOrMore(.digit) } transform: { Int($0) } + Capture(as: knownIssueCount) { OneOrMore(.digit) } transform: { Int($0) } " known issue" Optionally("s") ")." @@ -346,8 +355,10 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 12) - #expect(match.output.2 == 5) + #expect(match[testCount] == 9) + #expect(match[suiteCount] == 2) + #expect(match[issueCount] == 12) + #expect(match[knownIssueCount] == 5) } @Test("Issue counts are summed correctly on run end for a test with only warning issues") @@ -369,16 +380,24 @@ struct EventRecorderTests { print(buffer, terminator: "") } + let testCount = Reference() + let suiteCount = Reference() + let warningCount = Reference() + let runFailureRegex = Regex { One(.anyGraphemeCluster) " Test run with " - OneOrMore(.digit) + Capture(as: testCount) { OneOrMore(.digit) } transform: { Int($0) } " test" Optionally("s") + " in " + Capture(as: suiteCount) { OneOrMore(.digit) } transform: { Int($0) } + " suite" + Optionally("s") " passed " ZeroOrMore(.any) " with " - Capture { OneOrMore(.digit) } transform: { Int($0) } + Capture(as: warningCount) { OneOrMore(.digit) } transform: { Int($0) } " warning" Optionally("s") "." @@ -390,7 +409,9 @@ struct EventRecorderTests { .first, "buffer: \(buffer)" ) - #expect(match.output.1 == 1) + #expect(match[testCount] == 1) + #expect(match[suiteCount] == 1) + #expect(match[warningCount] == 1) } #endif @@ -691,6 +712,8 @@ struct EventRecorderTests { func n(_ arg: Int) { #expect(arg > 0) } + + @Suite struct PredictableSubsuite {} } @Suite(.hidden) struct PredictablyFailingKnownIssueTests { From 40edfec8d306dc23a6f11ee96a92b7656b9a2ff9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 20 May 2025 15:16:07 -0500 Subject: [PATCH 196/234] Refinements to IssueHandlingTrait SPI (#1121) A few refinements to `IssueHandlingTrait`, which is still SPI. ### Motivation: Polish this SPI in anticipation of posting a pitch to promote it to public API soon. ### Modifications: - Expose a `handleIssue(_:)` instance method to allow more easily composing multiple issue handling traits and calling their underlying handler closure. This is conceptually similar to what was done for [ST-0010: Public API to evaluate ConditionTrait](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0010-evaluate-condition.md). - Refine the names of private decls. - Add `- Returns:` in DocC for places it's missing. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Testing/Traits/IssueHandlingTrait.swift | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/Traits/IssueHandlingTrait.swift b/Sources/Testing/Traits/IssueHandlingTrait.swift index d70d68e93..6a14132f1 100644 --- a/Sources/Testing/Traits/IssueHandlingTrait.swift +++ b/Sources/Testing/Traits/IssueHandlingTrait.swift @@ -26,20 +26,31 @@ /// - ``Trait/filterIssues(_:)`` @_spi(Experimental) public struct IssueHandlingTrait: TestTrait, SuiteTrait { - /// A function which transforms an issue and returns an optional replacement. + /// A function which handles an issue and returns an optional replacement. /// /// - Parameters: - /// - issue: The issue to transform. + /// - issue: The issue to handle. /// /// - Returns: An issue to replace `issue`, or else `nil` if the issue should /// not be recorded. - fileprivate typealias Transformer = @Sendable (_ issue: Issue) -> Issue? + fileprivate typealias Handler = @Sendable (_ issue: Issue) -> Issue? - /// This trait's transformer function. - private var _transformer: Transformer + /// This trait's handler function. + private var _handler: Handler - fileprivate init(transformer: @escaping Transformer) { - _transformer = transformer + fileprivate init(handler: @escaping Handler) { + _handler = handler + } + + /// Handle a specified issue. + /// + /// - Parameters: + /// - issue: The issue to handle. + /// + /// - Returns: An issue to replace `issue`, or else `nil` if the issue should + /// not be recorded. + public func handleIssue(_ issue: Issue) -> Issue? { + _handler(issue) } public var isRecursive: Bool { @@ -90,7 +101,7 @@ extension IssueHandlingTrait: TestScoping { // records new issues. This means only issue handling traits whose scope // is outside this one will be allowed to handle such issues. let newIssue = Configuration.withCurrent(oldConfiguration) { - _transformer(issue) + handleIssue(issue) } if let newIssue { @@ -113,6 +124,8 @@ extension Trait where Self == IssueHandlingTrait { /// this trait is applied to. It is passed a recorded issue, and returns /// an optional issue to replace the passed-in one. /// + /// - Returns: An instance of ``IssueHandlingTrait`` that transforms issues. + /// /// The `transformer` closure is called synchronously each time an issue is /// recorded by the test this trait is applied to. The closure is passed the /// recorded issue, and if it returns a non-`nil` value, that will be recorded @@ -131,7 +144,7 @@ extension Trait where Self == IssueHandlingTrait { /// record new issues, although they will only be handled by issue handling /// traits which precede this trait or were inherited from a containing suite. public static func transformIssues(_ transformer: @escaping @Sendable (Issue) -> Issue?) -> Self { - Self(transformer: transformer) + Self(handler: transformer) } /// Constructs a trait that filters issues recorded by a test. @@ -142,6 +155,8 @@ extension Trait where Self == IssueHandlingTrait { /// should return `true` if the issue should be included, or `false` if it /// should be suppressed. /// + /// - Returns: An instance of ``IssueHandlingTrait`` that filters issues. + /// /// The `isIncluded` closure is called synchronously each time an issue is /// recorded by the test this trait is applied to. The closure is passed the /// recorded issue, and if it returns `true`, the issue will be preserved in From 981aa1c64215e71b22c64fae0a8ec879e2d13501 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 22 May 2025 15:14:19 -0500 Subject: [PATCH 197/234] Remove the 'triage-needed' label from New Issue templates (#1124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This removes the `triage-needed` label from the GitHub New Issue templates in this repository which have it. I added this while revamping these templates in #962, but more recently we added a `triaged` label which is works the opposite way—it's applied after an issue has been triaged—so we don't need the older label anymore. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .github/ISSUE_TEMPLATE/01-bug-report.yml | 2 +- .github/ISSUE_TEMPLATE/02-change-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/01-bug-report.yml b/.github/ISSUE_TEMPLATE/01-bug-report.yml index ec821ae82..9e7cc171f 100644 --- a/.github/ISSUE_TEMPLATE/01-bug-report.yml +++ b/.github/ISSUE_TEMPLATE/01-bug-report.yml @@ -9,7 +9,7 @@ name: 🪲 Report a bug description: > Report a deviation from expected or documented behavior. -labels: [bug, triage-needed] +labels: [bug] body: - type: markdown attributes: diff --git a/.github/ISSUE_TEMPLATE/02-change-request.yml b/.github/ISSUE_TEMPLATE/02-change-request.yml index 4647a4560..fc29f3c44 100644 --- a/.github/ISSUE_TEMPLATE/02-change-request.yml +++ b/.github/ISSUE_TEMPLATE/02-change-request.yml @@ -9,7 +9,7 @@ name: 🌟 Request a change description: > Request a feature, API, improvement, or other change. -labels: [enhancement, triage-needed] +labels: [enhancement] body: - type: markdown attributes: From ae713136615888862c8318d9bb8234cf554855c8 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 23 May 2025 15:09:19 -0500 Subject: [PATCH 198/234] Output console message for test case ended events in verbose mode, with status and issue counts (#1125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This outputs a console message when each test case of a parameterized test function ends, including its pass/fail status and a count of the number of issues which were recorded, when running in verbose mode (`swift test --verbose`). Here's an example: ``` ◇ Test contrivedExample(x:) started. ◇ Test case passing 1 argument x → 1 (Swift.Int) to contrivedExample(x:) started. ◇ Test case passing 1 argument x → 2 (Swift.Int) to contrivedExample(x:) started. ✔ Test case passing 1 argument x → 1 (Swift.Int) to contrivedExample(x:) passed after 0.001 seconds. ✘ Test contrivedExample(x:) recorded an issue with 1 argument x → 2 at EventRecorderTests.swift:759:3: Expectation failed: (x → 2) == 1 ↳ x: Swift.Int → 2 ↳ 1: Swift.Int → 1 ✘ Test case passing 1 argument x → 2 (Swift.Int) to contrivedExample(x:) failed after 0.001 seconds with 1 issue. ✘ Test contrivedExample(x:) with 2 test cases failed after 0.001 seconds with 1 issue. ``` > Note: This leverages #1000 which added more robust identification of test cases. Fixes #1021 Fixes rdar://146863942 ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Event.HumanReadableOutputRecorder.swift | 97 ++++++++++++++----- .../CustomTestStringConvertible.swift | 12 +-- Tests/TestingTests/EventRecorderTests.swift | 19 ++-- 3 files changed, 93 insertions(+), 35 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 06d12de6e..a3d121e09 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -36,7 +36,7 @@ extension Event { /// A type that contains mutable context for /// ``Event/ConsoleOutputRecorder``. - private struct _Context { + fileprivate struct Context { /// The instant at which the run started. var runStartInstant: Test.Clock.Instant? @@ -51,6 +51,17 @@ extension Event { /// The number of test suites started or skipped during the run. var suiteCount = 0 + /// An enumeration describing the various keys which can be used in a test + /// data graph for an output recorder. + enum TestDataKey: Hashable { + /// A string key, typically containing one key from the key path + /// representation of a ``Test/ID`` instance. + case string(String) + + /// A test case ID. + case testCaseID(Test.Case.ID) + } + /// A type describing data tracked on a per-test basis. struct TestData { /// The instant at which the test started. @@ -62,18 +73,15 @@ extension Event { /// The number of known issues recorded for the test. var knownIssueCount = 0 - - /// The number of test cases for the test. - var testCasesCount = 0 } /// Data tracked on a per-test basis. - var testData = Graph() + var testData = Graph() } /// This event recorder's mutable context about events it has received, /// which may be used to inform how subsequent events are written. - private var _context = Locked(rawValue: _Context()) + private var _context = Locked(rawValue: Context()) /// Initialize a new human-readable event recorder. /// @@ -128,7 +136,9 @@ extension Event.HumanReadableOutputRecorder { /// - graph: The graph to walk while counting issues. /// /// - Returns: A tuple containing the number of issues recorded in `graph`. - private func _issueCounts(in graph: Graph?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { + private func _issueCounts( + in graph: Graph? + ) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { guard let graph else { return (0, 0, 0, 0, "") } @@ -241,6 +251,7 @@ extension Event.HumanReadableOutputRecorder { 0 } let test = eventContext.test + let keyPath = eventContext.keyPath let testName = if let test { if let displayName = test.displayName { if verbosity > 0 { @@ -271,7 +282,7 @@ extension Event.HumanReadableOutputRecorder { case .testStarted: let test = test! - context.testData[test.id.keyPathRepresentation] = .init(startInstant: instant) + context.testData[keyPath] = .init(startInstant: instant) if test.isSuite { context.suiteCount += 1 } else { @@ -287,23 +298,17 @@ extension Event.HumanReadableOutputRecorder { } case let .issueRecorded(issue): - let id: [String] = if let test { - test.id.keyPathRepresentation - } else { - [] - } - var testData = context.testData[id] ?? .init(startInstant: instant) + var testData = context.testData[keyPath] ?? .init(startInstant: instant) if issue.isKnown { testData.knownIssueCount += 1 } else { let issueCount = testData.issueCount[issue.severity] ?? 0 testData.issueCount[issue.severity] = issueCount + 1 } - context.testData[id] = testData + context.testData[keyPath] = testData case .testCaseStarted: - let test = test! - context.testData[test.id.keyPathRepresentation]?.testCasesCount += 1 + context.testData[keyPath] = .init(startInstant: instant) default: // These events do not manipulate the context structure. @@ -384,13 +389,12 @@ extension Event.HumanReadableOutputRecorder { case .testEnded: let test = test! - let id = test.id - let testDataGraph = context.testData.subgraph(at: id.keyPathRepresentation) + let testDataGraph = context.testData.subgraph(at: keyPath) let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - let testCasesCount = if test.isParameterized { - " with \(testData.testCasesCount.counting("test case"))" + let testCasesCount = if test.isParameterized, let testDataGraph { + " with \(testDataGraph.children.count.counting("test case"))" } else { "" } @@ -517,15 +521,37 @@ extension Event.HumanReadableOutputRecorder { break } + let status = verbosity > 0 ? " started" : "" + return [ Message( symbol: .default, - stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)" + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)\(status) started." ) ] case .testCaseEnded: - break + guard verbosity > 0, let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { + break + } + + let testDataGraph = context.testData.subgraph(at: keyPath) + let testData = testDataGraph?.value ?? .init(startInstant: instant) + let issues = _issueCounts(in: testDataGraph) + let duration = testData.startInstant.descriptionOfDuration(to: instant) + + let message = if issues.errorIssueCount > 0 { + Message( + symbol: .fail, + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)." + ) + } else { + Message( + symbol: .pass(knownIssueCount: issues.knownIssueCount), + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)." + ) + } + return [message] case let .iterationEnded(index): guard let iterationStartInstant = context.iterationStartInstant else { @@ -568,6 +594,31 @@ extension Event.HumanReadableOutputRecorder { } } +extension Test.ID { + /// The key path in a test data graph representing this test ID. + fileprivate var keyPath: some Collection { + keyPathRepresentation.map { .string($0) } + } +} + +extension Event.Context { + /// The key path in a test data graph representing this event this context is + /// associated with, including its test and/or test case IDs. + fileprivate var keyPath: some Collection { + var keyPath = [Event.HumanReadableOutputRecorder.Context.TestDataKey]() + + if let test { + keyPath.append(contentsOf: test.id.keyPath) + + if let testCase { + keyPath.append(.testCaseID(testCase.id)) + } + } + + return keyPath + } +} + // MARK: - Codable extension Event.HumanReadableOutputRecorder.Message: Codable {} diff --git a/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift b/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift index 192dde5ad..6f0517468 100644 --- a/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift +++ b/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift @@ -42,9 +42,9 @@ /// the default description of a value may not be adequately descriptive: /// /// ``` -/// ◇ Passing argument food → .paella to isDelicious(_:) -/// ◇ Passing argument food → .oden to isDelicious(_:) -/// ◇ Passing argument food → .ragu to isDelicious(_:) +/// ◇ Test case passing 1 argument food → .paella to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → .oden to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → .ragu to isDelicious(_:) started. /// ``` /// /// By adopting ``CustomTestStringConvertible``, customized descriptions can be @@ -69,9 +69,9 @@ /// ``testDescription`` property: /// /// ``` -/// ◇ Passing argument food → paella valenciana to isDelicious(_:) -/// ◇ Passing argument food → おでん to isDelicious(_:) -/// ◇ Passing argument food → ragù alla bolognese to isDelicious(_:) +/// ◇ Test case passing 1 argument food → paella valenciana to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → おでん to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → ragù alla bolognese to isDelicious(_:) started. /// ``` /// /// ## See Also diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index d06e12c7c..ed7d765a0 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -94,6 +94,7 @@ struct EventRecorderTests { } @Test("Verbose output") + @available(_regexAPI, *) func verboseOutput() async throws { let stream = Stream() @@ -112,6 +113,14 @@ struct EventRecorderTests { #expect(buffer.contains(#"\#(Event.Symbol.details.unicodeCharacter) lhs: Swift.String → "987""#)) #expect(buffer.contains(#""Animal Crackers" (aka 'WrittenTests')"#)) #expect(buffer.contains(#""Not A Lobster" (aka 'actuallyCrab()')"#)) + do { + let regex = try Regex(".* Test case passing 1 argument i → 0 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) passed after .*.") + #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) + } + do { + let regex = try Regex(".* Test case passing 1 argument i → 3 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) failed after .* with 1 issue.") + #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) + } if testsWithSignificantIOAreEnabled { print(buffer, terminator: "") @@ -203,17 +212,15 @@ struct EventRecorderTests { await runTest(for: PredictablyFailingTests.self, configuration: configuration) let buffer = stream.buffer.rawValue - if testsWithSignificantIOAreEnabled { - print(buffer, terminator: "") - } - let aurgmentRegex = try Regex(expectedPattern) + let argumentRegex = try Regex(expectedPattern) #expect( (try buffer .split(whereSeparator: \.isNewline) - .compactMap(aurgmentRegex.wholeMatch(in:)) - .first) != nil + .compactMap(argumentRegex.wholeMatch(in:)) + .first) != nil, + "buffer: \(buffer)" ) } From 8a6ed780e591c242708e51476bdce0e021730888 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 23 May 2025 15:12:31 -0500 Subject: [PATCH 199/234] Enable upcoming feature 'InferIsolatedConformances' and fix issues it reveals (#1126) This enables the `InferIsolatedConformances` upcoming Swift feature from [SE-0470: Global-actor isolated conformances](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0470-isolated-conformances.md) and fixes a couple of pre-existing issues it revealed. I confirmed these code changes still build successfully using a Swift 6.1 toolchain. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 2 ++ Sources/Testing/Running/Configuration.TestFilter.swift | 2 +- Tests/TestingMacrosTests/TestSupport/Parse.swift | 2 +- cmake/modules/shared/CompilerSettings.cmake | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Package.swift b/Package.swift index 44116b6d1..3df51ca48 100644 --- a/Package.swift +++ b/Package.swift @@ -288,6 +288,8 @@ extension Array where Element == PackageDescription.SwiftSetting { // new-enough toolchain. .enableExperimentalFeature("AllowUnsafeAttribute"), + .enableUpcomingFeature("InferIsolatedConformances"), + // When building as a package, the macro plugin always builds as an // executable rather than a library. .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), diff --git a/Sources/Testing/Running/Configuration.TestFilter.swift b/Sources/Testing/Running/Configuration.TestFilter.swift index 7ef1fb08c..92d811ba1 100644 --- a/Sources/Testing/Running/Configuration.TestFilter.swift +++ b/Sources/Testing/Running/Configuration.TestFilter.swift @@ -514,7 +514,7 @@ extension Configuration.TestFilter.Kind { /// A protocol representing a value which can be filtered using /// ``Configuration/TestFilter-swift.struct``. -private protocol _FilterableItem { +private protocol _FilterableItem: Sendable { /// The test this item represents. var test: Test { get } diff --git a/Tests/TestingMacrosTests/TestSupport/Parse.swift b/Tests/TestingMacrosTests/TestSupport/Parse.swift index 2b30df42e..453a631b2 100644 --- a/Tests/TestingMacrosTests/TestSupport/Parse.swift +++ b/Tests/TestingMacrosTests/TestSupport/Parse.swift @@ -19,7 +19,7 @@ import SwiftSyntaxBuilder import SwiftSyntaxMacros import SwiftSyntaxMacroExpansion -fileprivate let allMacros: [String: any Macro.Type] = [ +fileprivate let allMacros: [String: any (Macro & Sendable).Type] = [ "expect": ExpectMacro.self, "require": RequireMacro.self, "requireAmbiguous": AmbiguousRequireMacro.self, // different name needed only for unit testing diff --git a/cmake/modules/shared/CompilerSettings.cmake b/cmake/modules/shared/CompilerSettings.cmake index 0da4216c5..af8b56dfd 100644 --- a/cmake/modules/shared/CompilerSettings.cmake +++ b/cmake/modules/shared/CompilerSettings.cmake @@ -17,7 +17,8 @@ add_compile_options( add_compile_options( "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend ExistentialAny>" "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend InternalImportsByDefault>" - "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend MemberImportVisibility>") + "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend MemberImportVisibility>" + "SHELL:$<$:-Xfrontend -enable-upcoming-feature -Xfrontend InferIsolatedConformances>") # Platform-specific definitions. if(APPLE) From d5b5c59670720524cd8a86bf5c8b3f01887efed2 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 26 May 2025 20:24:13 -0500 Subject: [PATCH 200/234] Revert "Work around a macOS CI failure (#1100)" (#1129) Revert the workaround added in #1100, since the Swift compiler issue has been resolved and newer `main` development snapshot toolchains have become available with the fix. This reverts commit a82d0a89dd2eb86d18056f03396055528ce56508. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Tests/TestingTests/IssueTests.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index cc0a7acf5..6ea1a5827 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -491,7 +491,6 @@ final class IssueTests: XCTestCase { }.run(configuration: .init()) } -#if !SWT_TARGET_OS_APPLE || SWT_FIXED_149299786 func testErrorCheckingWithExpect() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.isInverted = true @@ -611,7 +610,6 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } -#endif func testErrorCheckingWithExpect_mismatchedErrorDescription() async throws { let expectationFailed = expectation(description: "Expectation failed") From d6111205ce07f173dbc2519ddaf4eec800a76b94 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 30 May 2025 14:33:37 -0500 Subject: [PATCH 201/234] Expand parameterized testing documentation to mention try/await support and showcase helper pattern (#1133) This expands a few places where we document parameterized testing APIs to mention `try`/`await` support and showcase a common pattern for sharing arguments between multiple tests. ### Modifications: - For each `@Test` macro which accepts arguments, mention that `try` and `await` are supported and that arguments are lazily evaluated. - In the "Implementing parameterized tests" article, add a new section titled "Pass the same arguments to multiple test functions" showcasing the pattern of extracting common arguments to a separate property. - Add a [`> Tip:` callout](https://www.swift.org/documentation/docc/other-formatting-options#Add-Notes-and-Other-Asides) within that new article section mentioning `try`/`await` support. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes rdar://130929060 --- Sources/Testing/Test+Macro.swift | 30 +++++++++++++----- .../Testing.docc/ParameterizedTesting.md | 31 +++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index be0b5a91b..44e9d3d72 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -217,8 +217,10 @@ public macro Test( /// - collection: A collection of values to pass to the associated test /// function. /// -/// During testing, the associated test function is called once for each element -/// in `collection`. +/// You can prefix the expression you pass to `collection` with `try` or `await`. +/// The testing library evaluates the expression lazily only if it determines +/// that the associated test will run. During testing, the testing library calls +/// the associated test function once for each element in `collection`. /// /// @Comment { /// - Bug: The testing library should support variadic generics. @@ -270,7 +272,10 @@ extension Test { /// - collection1: A collection of values to pass to `testFunction`. /// - collection2: A second collection of values to pass to `testFunction`. /// -/// During testing, the associated test function is called once for each pair of +/// You can prefix the expressions you pass to `collection1` or `collection2` +/// with `try` or `await`. The testing library evaluates the expressions lazily +/// only if it determines that the associated test will run. During testing, the +/// testing library calls the associated test function once for each pair of /// elements in `collection1` and `collection2`. /// /// @Comment { @@ -298,7 +303,10 @@ public macro Test( /// - collection1: A collection of values to pass to `testFunction`. /// - collection2: A second collection of values to pass to `testFunction`. /// -/// During testing, the associated test function is called once for each pair of +/// You can prefix the expressions you pass to `collection1` or `collection2` +/// with `try` or `await`. The testing library evaluates the expressions lazily +/// only if it determines that the associated test will run. During testing, the +/// testing library calls the associated test function once for each pair of /// elements in `collection1` and `collection2`. /// /// @Comment { @@ -324,8 +332,11 @@ public macro Test( /// - zippedCollections: Two zipped collections of values to pass to /// `testFunction`. /// -/// During testing, the associated test function is called once for each element -/// in `zippedCollections`. +/// You can prefix the expression you pass to `zippedCollections` with `try` or +/// `await`. The testing library evaluates the expression lazily only if it +/// determines that the associated test will run. During testing, the testing +/// library calls the associated test function once for each element in +/// `zippedCollections`. /// /// @Comment { /// - Bug: The testing library should support variadic generics. @@ -352,8 +363,11 @@ public macro Test( /// - zippedCollections: Two zipped collections of values to pass to /// `testFunction`. /// -/// During testing, the associated test function is called once for each element -/// in `zippedCollections`. +/// You can prefix the expression you pass to `zippedCollections` with `try` or +/// `await`. The testing library evaluates the expression lazily only if it +/// determines that the associated test will run. During testing, the testing +/// library calls the associated test function once for each element in +/// `zippedCollections`. /// /// @Comment { /// - Bug: The testing library should support variadic generics. diff --git a/Sources/Testing/Testing.docc/ParameterizedTesting.md b/Sources/Testing/Testing.docc/ParameterizedTesting.md index c4310e8e6..2dada707e 100644 --- a/Sources/Testing/Testing.docc/ParameterizedTesting.md +++ b/Sources/Testing/Testing.docc/ParameterizedTesting.md @@ -101,6 +101,37 @@ func makeLargeOrder(count: Int) async throws { - Note: Very large ranges such as `0 ..< .max` may take an excessive amount of time to test, or may never complete due to resource constraints. +### Pass the same arguments to multiple test functions + +If you want to pass the same collection of arguments to two or more +parameterized test functions, you can extract the arguments to a separate +function or property and pass it to each `@Test` attribute. For example: + +```swift +extension Food { + static var bestSelling: [Food] { + get async throws { /* ... */ } + } +} + +@Test(arguments: try await Food.bestSelling) +func `Order entree`(food: Food) { + let foodTruck = FoodTruck() + #expect(foodTruck.order(food)) +} + +@Test(arguments: try await Food.bestSelling) +func `Package leftovers`(food: Food) throws { + let foodTruck = FoodTruck() + let container = try #require(foodTruck.container(fitting: food)) + try container.add(food) +} +``` + +> Tip: You can prefix expressions passed to `arguments:` with `try` or `await`. +> The testing library evaluates them lazily only if it determines that the +> associated test will run. + ### Test with more than one collection It's possible to test more than one collection. Consider the following test From ca81dffea6bd63dc7e26290fb595b7562ac730b9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 2 Jun 2025 14:41:30 -0500 Subject: [PATCH 202/234] Acknowledge unsafe API usages in code expanded from testing library macros (#1134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acknowledge unsafe API usages from various testing library macros such as `@Test`, `@Suite`, and `#expect(processExitsWith:)` which are revealed in modules which enable the new opt-in strict memory safety feature in Swift 6.2. ### Motivation: This fix allows clients of the testing library to enable [SE-0458: Opt-in Strict Memory Safety Checking](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0458-strict-memory-safety.md) if they wish and avoid diagnostics from the testing library macros in their modules. These warnings generally looked like this, before this fix: ``` ⚠️ @__swiftmacro_22MemorySafeTestingTests19exampleTestFunction33_F2EA1AA3013574E5644E5A4339F05086LL0F0fMp_.swift:23:14: warning: expression uses unsafe constructs but is not marked with 'unsafe' Testing.Test.__store($s22MemorySafeTestingTests19exampleTestFunction33_F2EA1AA3013574E5644E5A4339F05086LL0F0fMp_25generator1e3470c498e8fe35fMu_, into: outValue, asTypeAt: type) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``` ### Modifications: - Add test file to reproduce the new diagnostics. It's in a new module which opts-in to strict memory safety, and marked as a dependency of `TestingTests`. - Add `unsafe` keyword in the appropriate places in our macros to acknowledge the existing unsafe usages. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes: rdar://151238560 --- Package.swift | 27 ++++++++++++++++ Sources/Testing/ExitTests/ExitTest.swift | 3 ++ Sources/Testing/Test+Discovery.swift | 3 ++ Sources/TestingMacros/ConditionMacro.swift | 3 +- .../TestingMacros/SuiteDeclarationMacro.swift | 3 +- .../Support/EffectfulExpressionHandling.swift | 28 +++++++++-------- .../Support/TestContentGeneration.swift | 3 +- .../TestingMacros/TestDeclarationMacro.swift | 3 +- .../MemorySafeTestDecls.swift | 31 +++++++++++++++++++ 9 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 Tests/_MemorySafeTestingTests/MemorySafeTestDecls.swift diff --git a/Package.swift b/Package.swift index 3df51ca48..13dbf61cf 100644 --- a/Package.swift +++ b/Package.swift @@ -127,10 +127,25 @@ let package = Package( "Testing", "_Testing_CoreGraphics", "_Testing_Foundation", + "MemorySafeTestingTests", ], swiftSettings: .packageSettings ), + // Use a plain `.target` instead of a `.testTarget` to avoid the unnecessary + // overhead of having a separate test target for this module. Conceptually, + // the content in this module is no different than content which would + // typically be placed in the `TestingTests` target, except this content + // needs the (module-wide) strict memory safety feature to be enabled. + .target( + name: "MemorySafeTestingTests", + dependencies: [ + "Testing", + ], + path: "Tests/_MemorySafeTestingTests", + swiftSettings: .packageSettings + .strictMemorySafety + ), + .macro( name: "TestingMacros", dependencies: [ @@ -355,6 +370,18 @@ extension Array where Element == PackageDescription.SwiftSetting { return result } + + /// Settings necessary to enable Strict Memory Safety, introduced in + /// [SE-0458: Opt-in Strict Memory Safety Checking](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0458-strict-memory-safety.md#swiftpm-integration). + static var strictMemorySafety: Self { +#if compiler(>=6.2) + // FIXME: Adopt official `.strictMemorySafety()` condition once the minimum + // supported toolchain is 6.2. + [.unsafeFlags(["-strict-memory-safety"])] +#else + [] +#endif + } } extension Array where Element == PackageDescription.CXXSetting { diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 1e9c29c15..beda3eb4e 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -327,6 +327,9 @@ extension ExitTest { /// /// - Warning: This function is used to implement the /// `#expect(processExitsWith:)` macro. Do not use it directly. +#if compiler(>=6.2) + @safe +#endif public static func __store( _ id: (UInt64, UInt64, UInt64, UInt64), _ body: @escaping @Sendable (repeat each T) async throws -> Void, diff --git a/Sources/Testing/Test+Discovery.swift b/Sources/Testing/Test+Discovery.swift index 5e9632d70..71862943d 100644 --- a/Sources/Testing/Test+Discovery.swift +++ b/Sources/Testing/Test+Discovery.swift @@ -39,6 +39,9 @@ extension Test { /// /// - Warning: This function is used to implement the `@Test` macro. Do not /// use it directly. +#if compiler(>=6.2) + @safe +#endif public static func __store( _ generator: @escaping @Sendable () async -> Test, into outValue: UnsafeMutableRawPointer, diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 49630cfc9..9f87dfbd3 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -496,10 +496,11 @@ extension ExitTestConditionMacro { var recordDecl: DeclSyntax? #if !SWT_NO_LEGACY_TEST_DISCOVERY let legacyEnumName = context.makeUniqueName("__🟡$") + let unsafeKeyword: TokenSyntax? = isUnsafeKeywordSupported ? .keyword(.unsafe, trailingTrivia: .space) : nil recordDecl = """ enum \(legacyEnumName): Testing.__TestContentRecordContainer { nonisolated static var __testContentRecord: Testing.__TestContentRecord { - \(enumName).testContentRecord + \(unsafeKeyword)\(enumName).testContentRecord } } """ diff --git a/Sources/TestingMacros/SuiteDeclarationMacro.swift b/Sources/TestingMacros/SuiteDeclarationMacro.swift index 60a276689..e44b0460a 100644 --- a/Sources/TestingMacros/SuiteDeclarationMacro.swift +++ b/Sources/TestingMacros/SuiteDeclarationMacro.swift @@ -169,12 +169,13 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable { #if !SWT_NO_LEGACY_TEST_DISCOVERY // Emit a type that contains a reference to the test content record. let enumName = context.makeUniqueName("__🟡$") + let unsafeKeyword: TokenSyntax? = isUnsafeKeywordSupported ? .keyword(.unsafe, trailingTrivia: .space) : nil result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") enum \(enumName): Testing.__TestContentRecordContainer { nonisolated static var __testContentRecord: Testing.__TestContentRecord { - \(testContentRecordName) + \(unsafeKeyword)\(testContentRecordName) } } """ diff --git a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift index 494d2fcfc..a0b84e737 100644 --- a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift +++ b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift @@ -86,6 +86,17 @@ extension BidirectionalCollection { // MARK: - Inserting effect keywords/thunks +/// Whether or not the `unsafe` expression keyword is supported. +var isUnsafeKeywordSupported: Bool { + // The 'unsafe' keyword was introduced in 6.2 as part of SE-0458. Older + // toolchains are not aware of it. +#if compiler(>=6.2) + true +#else + false +#endif +} + /// Make a function call expression to an effectful thunk function provided by /// the testing library. /// @@ -127,12 +138,7 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp let needAwait = effectfulKeywords.contains(.await) && !expr.is(AwaitExprSyntax.self) let needTry = effectfulKeywords.contains(.try) && !expr.is(TryExprSyntax.self) - // The 'unsafe' keyword was introduced in 6.2 as part of SE-0458. Older - // toolchains are not aware of it, so avoid emitting expressions involving - // that keyword when the macro has been built using an older toolchain. -#if compiler(>=6.2) - let needUnsafe = effectfulKeywords.contains(.unsafe) && !expr.is(UnsafeExprSyntax.self) -#endif + let needUnsafe = isUnsafeKeywordSupported && effectfulKeywords.contains(.unsafe) && !expr.is(UnsafeExprSyntax.self) // First, add thunk function calls. if needAwait { @@ -141,11 +147,9 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp if needTry { expr = _makeCallToEffectfulThunk(.identifier("__requiringTry"), passing: expr) } -#if compiler(>=6.2) if needUnsafe { expr = _makeCallToEffectfulThunk(.identifier("__requiringUnsafe"), passing: expr) } -#endif // Then add keyword expressions. (We do this separately so we end up writing // `try await __r(__r(self))` instead of `try __r(await __r(self))` which is @@ -153,7 +157,7 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp if needAwait { expr = ExprSyntax( AwaitExprSyntax( - awaitKeyword: .keyword(.await).with(\.trailingTrivia, .space), + awaitKeyword: .keyword(.await, trailingTrivia: .space), expression: expr ) ) @@ -161,21 +165,19 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp if needTry { expr = ExprSyntax( TryExprSyntax( - tryKeyword: .keyword(.try).with(\.trailingTrivia, .space), + tryKeyword: .keyword(.try, trailingTrivia: .space), expression: expr ) ) } -#if compiler(>=6.2) if needUnsafe { expr = ExprSyntax( UnsafeExprSyntax( - unsafeKeyword: .keyword(.unsafe).with(\.trailingTrivia, .space), + unsafeKeyword: .keyword(.unsafe, trailingTrivia: .space), expression: expr ) ) } -#endif expr.leadingTrivia = originalExpr.leadingTrivia expr.trailingTrivia = originalExpr.trailingTrivia diff --git a/Sources/TestingMacros/Support/TestContentGeneration.swift b/Sources/TestingMacros/Support/TestContentGeneration.swift index 9a2529cee..2999478de 100644 --- a/Sources/TestingMacros/Support/TestContentGeneration.swift +++ b/Sources/TestingMacros/Support/TestContentGeneration.swift @@ -63,12 +63,13 @@ func makeTestContentRecordDecl(named name: TokenSyntax, in typeName: TypeSyntax? IntegerLiteralExprSyntax(context, radix: .binary) } + let unsafeKeyword: TokenSyntax? = isUnsafeKeywordSupported ? .keyword(.unsafe, trailingTrivia: .space) : nil var result: DeclSyntax = """ @available(*, deprecated, message: "This property is an implementation detail of the testing library. Do not use it directly.") private nonisolated \(staticKeyword(for: typeName)) let \(name): Testing.__TestContentRecord = ( \(kindExpr), \(kind.commentRepresentation) 0, - \(accessorName), + \(unsafeKeyword)\(accessorName), \(contextExpr), 0 ) diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 0b2d43f1e..58e8259ec 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -494,12 +494,13 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { #if !SWT_NO_LEGACY_TEST_DISCOVERY // Emit a type that contains a reference to the test content record. let enumName = context.makeUniqueName(thunking: functionDecl, withPrefix: "__🟡$") + let unsafeKeyword: TokenSyntax? = isUnsafeKeywordSupported ? .keyword(.unsafe, trailingTrivia: .space) : nil result.append( """ @available(*, deprecated, message: "This type is an implementation detail of the testing library. Do not use it directly.") enum \(enumName): Testing.__TestContentRecordContainer { nonisolated static var __testContentRecord: Testing.__TestContentRecord { - \(testContentRecordName) + \(unsafeKeyword)\(testContentRecordName) } } """ diff --git a/Tests/_MemorySafeTestingTests/MemorySafeTestDecls.swift b/Tests/_MemorySafeTestingTests/MemorySafeTestDecls.swift new file mode 100644 index 000000000..baf02c026 --- /dev/null +++ b/Tests/_MemorySafeTestingTests/MemorySafeTestDecls.swift @@ -0,0 +1,31 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if compiler(>=6.2) + +@testable import Testing + +#if !hasFeature(StrictMemorySafety) +#error("This file requires strict memory safety to be enabled") +#endif + +@Test(.hidden) +func exampleTestFunction() {} + +@Suite(.hidden) +struct ExampleSuite { + @Test func example() {} +} + +func exampleExitTest() async { + await #expect(processExitsWith: .success) {} +} + +#endif From 603a568c376066365e573e5a6aa043c651c7edc8 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 6 Jun 2025 15:29:21 -0500 Subject: [PATCH 203/234] Work around macOS build failures affecting test targets (#1139) This applies a workaround to fix macOS builds (including CI) which began failing due to a Swift compiler regression which is expected to be resolved by https://github.com/swiftlang/swift/pull/82034. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Package.swift | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Package.swift b/Package.swift index 13dbf61cf..cce384d7a 100644 --- a/Package.swift +++ b/Package.swift @@ -129,7 +129,7 @@ let package = Package( "_Testing_Foundation", "MemorySafeTestingTests", ], - swiftSettings: .packageSettings + swiftSettings: .packageSettings + .disableMandatoryOptimizationsSettings ), // Use a plain `.target` instead of a `.testTarget` to avoid the unnecessary @@ -234,7 +234,7 @@ package.targets.append(contentsOf: [ "Testing", "TestingMacros", ], - swiftSettings: .packageSettings + swiftSettings: .packageSettings + .disableMandatoryOptimizationsSettings ) ]) #endif @@ -290,7 +290,10 @@ extension Array where Element == PackageDescription.SwiftSetting { // This setting is enabled in the package, but not in the toolchain build // (via CMake). Enabling it is dependent on acceptance of the @section // proposal via Swift Evolution. - .enableExperimentalFeature("SymbolLinkageMarkers"), + // + // FIXME: Re-enable this once a CI blocker is resolved: + // https://github.com/swiftlang/swift-testing/issues/1138. +// .enableExperimentalFeature("SymbolLinkageMarkers"), // This setting is no longer needed when building with a 6.2 or later // toolchain now that SE-0458 has been accepted and implemented, but it is @@ -382,6 +385,20 @@ extension Array where Element == PackageDescription.SwiftSetting { [] #endif } + + /// Settings which disable Swift's mandatory optimizations pass. + /// + /// This is intended only to work around a build failure caused by a Swift + /// compiler regression which is expected to be resolved in + /// [swiftlang/swift#82034](https://github.com/swiftlang/swift/pull/82034). + /// + /// @Comment { + /// - Bug: This should be removed once the CI issue is resolved. + /// [swiftlang/swift-testin#1138](https://github.com/swiftlang/swift-testing/issues/1138). + /// } + static var disableMandatoryOptimizationsSettings: Self { + [.unsafeFlags(["-Xllvm", "-sil-disable-pass=mandatory-performance-optimizations"])] + } } extension Array where Element == PackageDescription.CXXSetting { From edb9a6a4ffe80d0517bb95fb04c2b87c69c3c3c6 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Sat, 7 Jun 2025 06:46:17 -0500 Subject: [PATCH 204/234] IssueHandlingTrait refinements: Rename to 'compactMapIssues' and special-case system issues (#1136) Refinements to `IssueHandlingTrait` based on [pitch](https://forums.swift.org/t/pitch-issue-handling-traits/80019) feedback. ### Modifications: - Rename the `transformIssues(_:)` function to `compactMapIssues(_:)` to align better with `filterIssues(_:)`. - Ignore issues for which the value of `kind` is `.system`. - Prohibit returning an issue from the closure passed to `compactMapIssues(_:)` whose `kind` is `.system`. - Adjust documentation and tests. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Testing.docc/Traits.md | 2 +- .../Testing/Traits/IssueHandlingTrait.swift | 38 +++++++--- .../Traits/IssueHandlingTraitTests.swift | 74 ++++++++++++++----- 3 files changed, 84 insertions(+), 30 deletions(-) diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index e6d19c1b9..46fa82b4d 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -51,7 +51,7 @@ types that customize the behavior of your tests. diff --git a/Sources/Testing/Traits/IssueHandlingTrait.swift b/Sources/Testing/Traits/IssueHandlingTrait.swift index 6a14132f1..e8142ff5a 100644 --- a/Sources/Testing/Traits/IssueHandlingTrait.swift +++ b/Sources/Testing/Traits/IssueHandlingTrait.swift @@ -15,14 +15,14 @@ /// modifying one or more of its properties, and returning the copy. You can /// observe recorded issues by returning them unmodified. Or you can suppress an /// issue by either filtering it using ``Trait/filterIssues(_:)`` or returning -/// `nil` from the closure passed to ``Trait/transformIssues(_:)``. +/// `nil` from the closure passed to ``Trait/compactMapIssues(_:)``. /// /// When an instance of this trait is applied to a suite, it is recursively /// inherited by all child suites and tests. /// /// To add this trait to a test, use one of the following functions: /// -/// - ``Trait/transformIssues(_:)`` +/// - ``Trait/compactMapIssues(_:)`` /// - ``Trait/filterIssues(_:)`` @_spi(Experimental) public struct IssueHandlingTrait: TestTrait, SuiteTrait { @@ -96,8 +96,14 @@ extension IssueHandlingTrait: TestScoping { return } + // Ignore system issues, as they are not expected to be caused by users. + if case .system = issue.kind { + oldConfiguration.eventHandler(event, context) + return + } + // Use the original configuration's event handler when invoking the - // transformer to avoid infinite recursion if the transformer itself + // handler closure to avoid infinite recursion if the handler itself // records new issues. This means only issue handling traits whose scope // is outside this one will be allowed to handle such issues. let newIssue = Configuration.withCurrent(oldConfiguration) { @@ -105,6 +111,11 @@ extension IssueHandlingTrait: TestScoping { } if let newIssue { + // Prohibit assigning the issue's kind to system. + if case .system = newIssue.kind { + preconditionFailure("Issue returned by issue handling closure cannot have kind 'system': \(newIssue)") + } + var event = event event.kind = .issueRecorded(newIssue) oldConfiguration.eventHandler(event, context) @@ -120,31 +131,35 @@ extension Trait where Self == IssueHandlingTrait { /// Constructs an trait that transforms issues recorded by a test. /// /// - Parameters: - /// - transformer: The closure called for each issue recorded by the test + /// - transform: A closure called for each issue recorded by the test /// this trait is applied to. It is passed a recorded issue, and returns /// an optional issue to replace the passed-in one. /// /// - Returns: An instance of ``IssueHandlingTrait`` that transforms issues. /// - /// The `transformer` closure is called synchronously each time an issue is + /// The `transform` closure is called synchronously each time an issue is /// recorded by the test this trait is applied to. The closure is passed the /// recorded issue, and if it returns a non-`nil` value, that will be recorded /// instead of the original. Otherwise, if the closure returns `nil`, the /// issue is suppressed and will not be included in the results. /// - /// The `transformer` closure may be called more than once if the test records + /// The `transform` closure may be called more than once if the test records /// multiple issues. If more than one instance of this trait is applied to a - /// test (including via inheritance from a containing suite), the `transformer` + /// test (including via inheritance from a containing suite), the `transform` /// closure for each instance will be called in right-to-left, innermost-to- /// outermost order, unless `nil` is returned, which will skip invoking the /// remaining traits' closures. /// - /// Within `transformer`, you may access the current test or test case (if any) + /// Within `transform`, you may access the current test or test case (if any) /// using ``Test/current`` ``Test/Case/current``, respectively. You may also /// record new issues, although they will only be handled by issue handling /// traits which precede this trait or were inherited from a containing suite. - public static func transformIssues(_ transformer: @escaping @Sendable (Issue) -> Issue?) -> Self { - Self(handler: transformer) + /// + /// - Note: `transform` will never be passed an issue for which the value of + /// ``Issue/kind`` is ``Issue/Kind/system``, and may not return such an + /// issue. + public static func compactMapIssues(_ transform: @escaping @Sendable (Issue) -> Issue?) -> Self { + Self(handler: transform) } /// Constructs a trait that filters issues recorded by a test. @@ -174,6 +189,9 @@ extension Trait where Self == IssueHandlingTrait { /// using ``Test/current`` ``Test/Case/current``, respectively. You may also /// record new issues, although they will only be handled by issue handling /// traits which precede this trait or were inherited from a containing suite. + /// + /// - Note: `isIncluded` will never be passed an issue for which the value of + /// ``Issue/kind`` is ``Issue/Kind/system``. public static func filterIssues(_ isIncluded: @escaping @Sendable (Issue) -> Bool) -> Self { Self { issue in isIncluded(issue) ? issue : nil diff --git a/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift index 4d749b07f..eb1aa1233 100644 --- a/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift +++ b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing @Suite("IssueHandlingTrait Tests") struct IssueHandlingTraitTests { @@ -23,7 +23,7 @@ struct IssueHandlingTraitTests { #expect(issue.comments == ["Foo", "Bar"]) } - let handler = IssueHandlingTrait.transformIssues { issue in + let handler = IssueHandlingTrait.compactMapIssues { issue in var issue = issue issue.comments.append("Bar") return issue @@ -34,8 +34,8 @@ struct IssueHandlingTraitTests { }.run(configuration: configuration) } - @Test("Suppressing an issue by returning `nil` from the transform closure") - func suppressIssueUsingTransformer() async throws { + @Test("Suppressing an issue by returning `nil` from the closure passed to compactMapIssues()") + func suppressIssueUsingCompactMapIssues() async throws { var configuration = Configuration() configuration.eventHandler = { event, context in if case .issueRecorded = event.kind { @@ -43,7 +43,7 @@ struct IssueHandlingTraitTests { } } - let handler = IssueHandlingTrait.transformIssues { _ in + let handler = IssueHandlingTrait.compactMapIssues { _ in // Return nil to suppress the issue. nil } @@ -81,10 +81,10 @@ struct IssueHandlingTraitTests { struct MyError: Error {} - try await confirmation("Transformer closure is called") { transformerCalled in - let transformer: @Sendable (Issue) -> Issue? = { issue in + try await confirmation("Issue handler closure is called") { issueHandlerCalled in + let transform: @Sendable (Issue) -> Issue? = { issue in defer { - transformerCalled() + issueHandlerCalled() } #expect(Test.Case.current == nil) @@ -96,7 +96,7 @@ struct IssueHandlingTraitTests { let test = Test( .enabled(if: try { throw MyError() }()), - .transformIssues(transformer) + .compactMapIssues(transform) ) {} // Use a detached task to intentionally clear task local values for the @@ -108,12 +108,12 @@ struct IssueHandlingTraitTests { } #endif - @Test("Accessing the current Test and Test.Case from a transformer closure") + @Test("Accessing the current Test and Test.Case from an issue handler closure") func currentTestAndCase() async throws { - await confirmation("Transformer closure is called") { transformerCalled in - let handler = IssueHandlingTrait.transformIssues { issue in + await confirmation("Issue handler closure is called") { issueHandlerCalled in + let handler = IssueHandlingTrait.compactMapIssues { issue in defer { - transformerCalled() + issueHandlerCalled() } #expect(Test.current?.name == "fixture()") #expect(Test.Case.current != nil) @@ -140,12 +140,12 @@ struct IssueHandlingTraitTests { #expect(issue.comments == ["Foo", "Bar", "Baz"]) } - let outerHandler = IssueHandlingTrait.transformIssues { issue in + let outerHandler = IssueHandlingTrait.compactMapIssues { issue in var issue = issue issue.comments.append("Baz") return issue } - let innerHandler = IssueHandlingTrait.transformIssues { issue in + let innerHandler = IssueHandlingTrait.compactMapIssues { issue in var issue = issue issue.comments.append("Bar") return issue @@ -156,7 +156,7 @@ struct IssueHandlingTraitTests { }.run(configuration: configuration) } - @Test("Secondary issue recorded from a transformer closure") + @Test("Secondary issue recorded from an issue handler closure") func issueRecordedFromClosure() async throws { await confirmation("Original issue recorded") { originalIssueRecorded in await confirmation("Secondary issue recorded") { secondaryIssueRecorded in @@ -175,14 +175,14 @@ struct IssueHandlingTraitTests { } } - let handler1 = IssueHandlingTrait.transformIssues { issue in + let handler1 = IssueHandlingTrait.compactMapIssues { issue in return issue } - let handler2 = IssueHandlingTrait.transformIssues { issue in + let handler2 = IssueHandlingTrait.compactMapIssues { issue in Issue.record("Something else") return issue } - let handler3 = IssueHandlingTrait.transformIssues { issue in + let handler3 = IssueHandlingTrait.compactMapIssues { issue in // The "Something else" issue should not be passed to this closure. #expect(issue.comments.contains("Foo")) return issue @@ -194,4 +194,40 @@ struct IssueHandlingTraitTests { } } } + + @Test("System issues are not passed to issue handler closures") + func ignoresSystemIssues() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + if case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind { + issue.record() + } + } + + let handler = IssueHandlingTrait.compactMapIssues { issue in + if case .system = issue.kind { + Issue.record("Unexpectedly received a system issue") + } + return nil + } + + await Test(handler) { + Issue(kind: .system).record() + }.run(configuration: configuration) + } + +#if !SWT_NO_EXIT_TESTS + @Test("Disallow assigning kind to .system") + func disallowAssigningSystemKind() async throws { + await #expect(processExitsWith: .failure) { + await Test(.compactMapIssues { issue in + var issue = issue + issue.kind = .system + return issue + }) { + Issue.record("A non-system issue") + }.run() + } + } +#endif } From 25b61ef4667a6d1dda30cb3420336621886529c4 Mon Sep 17 00:00:00 2001 From: Wouter Hennen <62355975+Wouter01@users.noreply.github.com> Date: Sun, 8 Jun 2025 03:47:01 +0200 Subject: [PATCH 205/234] Update outdated Suite macro documentation (#1142) --- Sources/Testing/Test+Macro.swift | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index 44e9d3d72..f2fc415d6 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -53,9 +53,8 @@ public typealias __XCTestCompatibleSelector = Never /// - Parameters: /// - traits: Zero or more traits to apply to this test suite. /// -/// A test suite is a type that contains one or more test functions. Any -/// copyable type (that is, any type that is not marked `~Copyable`) may be a -/// test suite. +/// A test suite is a type that contains one or more test functions. +/// Any type may be a test suite. /// /// The use of the `@Suite` attribute is optional; types are recognized as test /// suites even if they do not have the `@Suite` attribute applied to them. @@ -81,9 +80,8 @@ public macro Suite( /// from the associated type's name. /// - traits: Zero or more traits to apply to this test suite. /// -/// A test suite is a type that contains one or more test functions. Any -/// copyable type (that is, any type that is not marked `~Copyable`) may be a -/// test suite. +/// A test suite is a type that contains one or more test functions. +/// Any type may be a test suite. /// /// The use of the `@Suite` attribute is optional; types are recognized as test /// suites even if they do not have the `@Suite` attribute applied to them. From b0a1efd5e7e9cbb33785cf7e7c0178cac3c1a791 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 9 Jun 2025 19:08:16 -0400 Subject: [PATCH 206/234] Disable inheritance of file descriptors created by Swift Testing by default. (#1145) This PR makes file descriptors created by Swift Testing `FD_CLOEXEC` by default (on Windows, `~HANDLE_FLAG_INHERIT`.) We then clear `FD_CLOEXEC` for specific file descriptors that should be inherited. On Darwin, this is effectively ignored because we use `POSIX_SPAWN_CLOEXEC_DEFAULT`, and on Windows we use `PROC_THREAD_ATTRIBUTE_HANDLE_LIST` which has [much the same effect](https://devblogs.microsoft.com/oldnewthing/20111216-00/?p=8873). (On non-Darwin POSIX platforms, there's no reliable way to ensure only one child process inherits a particular file descriptor.) This change is speculative. No additional unit tests are added because existing test coverage _should be_ sufficient; the reported issue is on a platform (Ubuntu 20.04) where we don't have any CI jobs. Resolves #1140. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Attachments/Attachment.swift | 4 +- Sources/Testing/ExitTests/ExitTest.swift | 9 ++ Sources/Testing/Support/FileHandle.swift | 135 ++++++++++++++++++- Sources/_TestingInternals/include/Stubs.h | 20 +++ 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index 366e288d1..e9b98fb7b 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -462,7 +462,7 @@ extension Attachment where AttachableValue: ~Copyable { // file exists at this path (note "x" in the mode string), an error will // be thrown and we'll try again by adding a suffix. let preferredPath = appendPathComponent(preferredName, to: directoryPath) - file = try FileHandle(atPath: preferredPath, mode: "wxb") + file = try FileHandle(atPath: preferredPath, mode: "wxeb") result = preferredPath } catch { // Split the extension(s) off the preferred name. The first component in @@ -478,7 +478,7 @@ extension Attachment where AttachableValue: ~Copyable { // Propagate any error *except* EEXIST, which would indicate that the // name was already in use (so we should try again with a new suffix.) do { - file = try FileHandle(atPath: preferredPath, mode: "wxb") + file = try FileHandle(atPath: preferredPath, mode: "wxeb") result = preferredPath break } catch let error as CError where error.rawValue == swt_EEXIST() { diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index beda3eb4e..9284310db 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -801,6 +801,15 @@ extension ExitTest { childEnvironment["SWT_EXPERIMENTAL_CAPTURED_VALUES"] = capturedValuesEnvironmentVariable } +#if !SWT_TARGET_OS_APPLE + // Set inherited those file handles that the child process needs. On + // Darwin, this is a no-op because we use POSIX_SPAWN_CLOEXEC_DEFAULT. + try stdoutWriteEnd?.setInherited(true) + try stderrWriteEnd?.setInherited(true) + try backChannelWriteEnd.setInherited(true) + try capturedValuesReadEnd.setInherited(true) +#endif + // Spawn the child process. let processID = try withUnsafePointer(to: backChannelWriteEnd) { backChannelWriteEnd in try withUnsafePointer(to: capturedValuesReadEnd) { capturedValuesReadEnd in diff --git a/Sources/Testing/Support/FileHandle.swift b/Sources/Testing/Support/FileHandle.swift index 2a2bfe967..4e3c17372 100644 --- a/Sources/Testing/Support/FileHandle.swift +++ b/Sources/Testing/Support/FileHandle.swift @@ -73,6 +73,13 @@ struct FileHandle: ~Copyable, Sendable { return } + // On Windows, "N" is used rather than "e" to signify that a file handle is + // not inherited. + var mode = mode + if let eIndex = mode.firstIndex(of: "e") { + mode.replaceSubrange(eIndex ... eIndex, with: "N") + } + // Windows deprecates fopen() as insecure, so call _wfopen_s() instead. let fileHandle = try path.withCString(encodedAs: UTF16.self) { path in try mode.withCString(encodedAs: UTF16.self) { mode in @@ -98,8 +105,13 @@ struct FileHandle: ~Copyable, Sendable { /// - path: The path to read from. /// /// - Throws: Any error preventing the stream from being opened. + /// + /// By default, the resulting file handle is not inherited by any child + /// processes (that is, `FD_CLOEXEC` is set on POSIX-like systems and + /// `HANDLE_FLAG_INHERIT` is cleared on Windows.) To make it inheritable, call + /// ``setInherited()``. init(forReadingAtPath path: String) throws { - try self.init(atPath: path, mode: "rb") + try self.init(atPath: path, mode: "reb") } /// Initialize an instance of this type to write to the given path. @@ -108,8 +120,13 @@ struct FileHandle: ~Copyable, Sendable { /// - path: The path to write to. /// /// - Throws: Any error preventing the stream from being opened. + /// + /// By default, the resulting file handle is not inherited by any child + /// processes (that is, `FD_CLOEXEC` is set on POSIX-like systems and + /// `HANDLE_FLAG_INHERIT` is cleared on Windows.) To make it inheritable, call + /// ``setInherited()``. init(forWritingAtPath path: String) throws { - try self.init(atPath: path, mode: "wb") + try self.init(atPath: path, mode: "web") } /// Initialize an instance of this type with an existing C file handle. @@ -445,6 +462,17 @@ extension FileHandle { #if !SWT_NO_PIPES // MARK: - Pipes +#if !SWT_TARGET_OS_APPLE && !os(Windows) && !SWT_NO_DYNAMIC_LINKING +/// Create a pipe with flags. +/// +/// This function declaration is provided because `pipe2()` is only declared if +/// `_GNU_SOURCE` is set, but setting it causes build errors due to conflicts +/// with Swift's Glibc module. +private let _pipe2 = symbol(named: "pipe2").map { + castCFunction(at: $0, to: (@convention(c) (UnsafeMutablePointer, CInt) -> CInt).self) +} +#endif + extension FileHandle { /// Make a pipe connecting two new file handles. /// @@ -461,15 +489,37 @@ extension FileHandle { /// - Bug: This function should return a tuple containing the file handles /// instead of returning them via `inout` arguments. Swift does not support /// tuples with move-only elements. ([104669935](rdar://104669935)) + /// + /// By default, the resulting file handles are not inherited by any child + /// processes (that is, `FD_CLOEXEC` is set on POSIX-like systems and + /// `HANDLE_FLAG_INHERIT` is cleared on Windows.) To make them inheritable, + /// call ``setInherited()``. static func makePipe(readEnd: inout FileHandle?, writeEnd: inout FileHandle?) throws { +#if !os(Windows) + var pipe2Called = false +#endif + var (fdReadEnd, fdWriteEnd) = try withUnsafeTemporaryAllocation(of: CInt.self, capacity: 2) { fds in #if os(Windows) - guard 0 == _pipe(fds.baseAddress, 0, _O_BINARY) else { + guard 0 == _pipe(fds.baseAddress, 0, _O_BINARY | _O_NOINHERIT) else { throw CError(rawValue: swt_errno()) } #else - guard 0 == pipe(fds.baseAddress!) else { - throw CError(rawValue: swt_errno()) +#if !SWT_TARGET_OS_APPLE && !os(Windows) && !SWT_NO_DYNAMIC_LINKING + if let _pipe2 { + guard 0 == _pipe2(fds.baseAddress!, O_CLOEXEC) else { + throw CError(rawValue: swt_errno()) + } + pipe2Called = true + } +#endif + + if !pipe2Called { + // pipe2() is not available. Use pipe() instead and simulate O_CLOEXEC + // to the best of our ability. + guard 0 == pipe(fds.baseAddress!) else { + throw CError(rawValue: swt_errno()) + } } #endif return (fds[0], fds[1]) @@ -479,6 +529,15 @@ extension FileHandle { Self._close(fdWriteEnd) } +#if !os(Windows) + if !pipe2Called { + // pipe2() is not available. Use pipe() instead and simulate O_CLOEXEC + // to the best of our ability. + try _setFileDescriptorInherited(fdReadEnd, false) + try _setFileDescriptorInherited(fdWriteEnd, false) + } +#endif + do { defer { fdReadEnd = -1 @@ -553,6 +612,72 @@ extension FileHandle { #endif } #endif + +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) + /// Set whether or not the given file descriptor is inherited by child processes. + /// + /// - Parameters: + /// - fd: The file descriptor. + /// - inherited: Whether or not `fd` is inherited by child processes + /// (ignoring overriding functionality such as Apple's + /// `POSIX_SPAWN_CLOEXEC_DEFAULT` flag.) + /// + /// - Throws: Any error that occurred while setting the flag. + private static func _setFileDescriptorInherited(_ fd: CInt, _ inherited: Bool) throws { + switch swt_getfdflags(fd) { + case -1: + // An error occurred reading the flags for this file descriptor. + throw CError(rawValue: swt_errno()) + case let oldValue: + let newValue = if inherited { + oldValue & ~FD_CLOEXEC + } else { + oldValue | FD_CLOEXEC + } + if oldValue == newValue { + // No need to make a second syscall as nothing has changed. + return + } + if -1 == swt_setfdflags(fd, newValue) { + // An error occurred setting the flags for this file descriptor. + throw CError(rawValue: swt_errno()) + } + } + } +#endif + + /// Set whether or not this file handle is inherited by child processes. + /// + /// - Parameters: + /// - inherited: Whether or not this file handle is inherited by child + /// processes (ignoring overriding functionality such as Apple's + /// `POSIX_SPAWN_CLOEXEC_DEFAULT` flag.) + /// + /// - Throws: Any error that occurred while setting the flag. + func setInherited(_ inherited: Bool) throws { +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) + try withUnsafePOSIXFileDescriptor { fd in + guard let fd else { + throw SystemError(description: "Cannot set whether a file handle is inherited unless it is backed by a file descriptor. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + } + try withLock { + try Self._setFileDescriptorInherited(fd, inherited) + } + } +#elseif os(Windows) + return try withUnsafeWindowsHANDLE { handle in + guard let handle else { + throw SystemError(description: "Cannot set whether a file handle is inherited unless it is backed by a Windows file handle. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + } + let newValue = inherited ? DWORD(HANDLE_FLAG_INHERIT) : 0 + guard SetHandleInformation(handle, DWORD(HANDLE_FLAG_INHERIT), newValue) else { + throw Win32Error(rawValue: GetLastError()) + } + } +#else +#warning("Platform-specific implementation missing: cannot set whether a file handle is inherited") +#endif + } } // MARK: - General path utilities diff --git a/Sources/_TestingInternals/include/Stubs.h b/Sources/_TestingInternals/include/Stubs.h index 8093a3722..171cca6a5 100644 --- a/Sources/_TestingInternals/include/Stubs.h +++ b/Sources/_TestingInternals/include/Stubs.h @@ -151,6 +151,26 @@ static int swt_EEXIST(void) { return EEXIST; } +#if defined(F_GETFD) +/// Call `fcntl(F_GETFD)`. +/// +/// This function is provided because `fcntl()` is a variadic function and +/// cannot be imported directly into Swift. +static int swt_getfdflags(int fd) { + return fcntl(fd, F_GETFD); +} +#endif + +#if defined(F_SETFD) +/// Call `fcntl(F_SETFD)`. +/// +/// This function is provided because `fcntl()` is a variadic function and +/// cannot be imported directly into Swift. +static int swt_setfdflags(int fd, int flags) { + return fcntl(fd, F_SETFD, flags); +} +#endif + SWT_ASSUME_NONNULL_END #endif From c0108ad3a3ba304448dc6162a4febd5a4630c0ca Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 10 Jun 2025 13:14:58 -0500 Subject: [PATCH 207/234] Declare Xcode 26 availability for all Swift 6.2 APIs (#1148) This declares Xcode 26.0 API availability for all Swift 6.2 declarations. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. Fixes rdar://151629765 --- .../Attachable+Encodable+NSSecureCoding.swift | 1 + .../Attachments/Attachable+Encodable.swift | 2 ++ .../Attachments/Attachable+NSSecureCoding.swift | 2 ++ .../Attachments/Attachment+URL.swift | 1 + .../Attachments/Data+Attachable.swift | 2 ++ Sources/Testing/Attachments/Attachable.swift | 4 ++++ Sources/Testing/Attachments/AttachableWrapper.swift | 3 +++ Sources/Testing/Attachments/Attachment.swift | 11 +++++++++++ Sources/Testing/ExitTests/ExitStatus.swift | 3 +++ Sources/Testing/ExitTests/ExitTest.Condition.swift | 6 ++++++ Sources/Testing/ExitTests/ExitTest.Result.swift | 4 ++++ Sources/Testing/ExitTests/ExitTest.swift | 2 ++ Sources/Testing/Expectations/Expectation+Macro.swift | 2 ++ Sources/Testing/Testing.docc/exit-testing.md | 1 + Sources/Testing/Traits/ConditionTrait.swift | 1 + 15 files changed, 45 insertions(+) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift index 46a1e11e6..d82c6a6c6 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable+NSSecureCoding.swift @@ -20,6 +20,7 @@ public import Foundation /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } extension Attachable where Self: Encodable & NSSecureCoding { @_documentation(visibility: private) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift index 683888801..747024de3 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+Encodable.swift @@ -56,6 +56,7 @@ func withUnsafeBytes(encoding attachableValue: borrowing E, for attachment /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } extension Attachable where Self: Encodable { /// Encode this value into a buffer using either [`PropertyListEncoder`](https://developer.apple.com/documentation/foundation/propertylistencoder) @@ -92,6 +93,7 @@ extension Attachable where Self: Encodable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try _Testing_Foundation.withUnsafeBytes(encoding: self, for: attachment, body) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift index 4acbf4960..95002be0a 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachable+NSSecureCoding.swift @@ -18,6 +18,7 @@ public import Foundation /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } extension Attachable where Self: NSSecureCoding { /// Encode this object using [`NSKeyedArchiver`](https://developer.apple.com/documentation/foundation/nskeyedarchiver) @@ -52,6 +53,7 @@ extension Attachable where Self: NSSecureCoding { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { let format = try EncodingFormat(for: attachment) diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index 83c3909be..a018363fc 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -53,6 +53,7 @@ extension Attachment where AttachableValue == _AttachableURLWrapper { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public init( contentsOf url: URL, diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift index ce7b719a9..56f058da3 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Data+Attachable.swift @@ -14,10 +14,12 @@ public import Foundation /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } extension Data: Attachable { /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try withUnsafeBytes(body) diff --git a/Sources/Testing/Attachments/Attachable.swift b/Sources/Testing/Attachments/Attachable.swift index be466940b..9ec3ce8ad 100644 --- a/Sources/Testing/Attachments/Attachable.swift +++ b/Sources/Testing/Attachments/Attachable.swift @@ -29,6 +29,7 @@ /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } public protocol Attachable: ~Copyable { /// An estimate of the number of bytes of memory needed to store this value as @@ -48,6 +49,7 @@ public protocol Attachable: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } var estimatedAttachmentByteCount: Int? { get } @@ -74,6 +76,7 @@ public protocol Attachable: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } borrowing func withUnsafeBytes(for attachment: borrowing Attachment, _ body: (UnsafeRawBufferPointer) throws -> R) throws -> R @@ -94,6 +97,7 @@ public protocol Attachable: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } borrowing func preferredName(for attachment: borrowing Attachment, basedOn suggestedName: String) -> String } diff --git a/Sources/Testing/Attachments/AttachableWrapper.swift b/Sources/Testing/Attachments/AttachableWrapper.swift index 81df52d4d..d4b1cbe05 100644 --- a/Sources/Testing/Attachments/AttachableWrapper.swift +++ b/Sources/Testing/Attachments/AttachableWrapper.swift @@ -24,12 +24,14 @@ /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } public protocol AttachableWrapper: Attachable, ~Copyable { /// The type of the underlying value represented by this type. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } associatedtype Wrapped @@ -37,6 +39,7 @@ public protocol AttachableWrapper: Attachable, ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } var wrappedValue: Wrapped { get } } diff --git a/Sources/Testing/Attachments/Attachment.swift b/Sources/Testing/Attachments/Attachment.swift index e9b98fb7b..f8d242c8c 100644 --- a/Sources/Testing/Attachments/Attachment.swift +++ b/Sources/Testing/Attachments/Attachment.swift @@ -21,6 +21,7 @@ private import _TestingInternals /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } public struct Attachment: ~Copyable where AttachableValue: Attachable & ~Copyable { /// Storage for ``attachableValue-7dyjv``. @@ -57,6 +58,7 @@ public struct Attachment: ~Copyable where AttachableValue: Atta /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public var preferredName: String { let suggestedName = if let _preferredName, !_preferredName.isEmpty { @@ -100,6 +102,7 @@ extension Attachment where AttachableValue: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public init(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { self._attachableValue = attachableValue @@ -194,6 +197,7 @@ extension Attachment where AttachableValue: ~Copyable { extension Attachment: CustomStringConvertible { /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public var description: String { #""\#(preferredName)": \#(String(describingForTest: attachableValue))"# @@ -207,6 +211,7 @@ extension Attachment where AttachableValue: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } @_disfavoredOverload public var attachableValue: AttachableValue { _read { @@ -229,6 +234,7 @@ extension Attachment where AttachableValue: AttachableWrapper & ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public var attachableValue: AttachableValue.Wrapped { _read { @@ -259,6 +265,7 @@ extension Attachment where AttachableValue: Sendable & Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } @_documentation(visibility: private) public static func record(_ attachment: consuming Self, sourceLocation: SourceLocation = #_sourceLocation) { @@ -291,6 +298,7 @@ extension Attachment where AttachableValue: Sendable & Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } @_documentation(visibility: private) public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { @@ -318,6 +326,7 @@ extension Attachment where AttachableValue: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static func record(_ attachment: consuming Self, sourceLocation: SourceLocation = #_sourceLocation) { do { @@ -361,6 +370,7 @@ extension Attachment where AttachableValue: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static func record(_ attachableValue: consuming AttachableValue, named preferredName: String? = nil, sourceLocation: SourceLocation = #_sourceLocation) { record(Self(attachableValue, named: preferredName, sourceLocation: sourceLocation), sourceLocation: sourceLocation) @@ -389,6 +399,7 @@ extension Attachment where AttachableValue: ~Copyable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } @inlinable public borrowing func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) throws -> R { try attachableValue.withUnsafeBytes(for: self, body) diff --git a/Sources/Testing/ExitTests/ExitStatus.swift b/Sources/Testing/ExitTests/ExitStatus.swift index 0dd6d86ab..69c583543 100644 --- a/Sources/Testing/ExitTests/ExitStatus.swift +++ b/Sources/Testing/ExitTests/ExitStatus.swift @@ -21,6 +21,7 @@ private import _TestingInternals /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") @@ -55,6 +56,7 @@ public enum ExitStatus: Sendable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } case exitCode(_ exitCode: CInt) @@ -81,6 +83,7 @@ public enum ExitStatus: Sendable { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } case signal(_ signal: CInt) } diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift index f737d8cf6..42d929077 100644 --- a/Sources/Testing/ExitTests/ExitTest.Condition.swift +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -35,6 +35,7 @@ extension ExitTest { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public struct Condition: Sendable { /// An enumeration describing the possible conditions for an exit test. @@ -66,6 +67,7 @@ extension ExitTest.Condition { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static var success: Self { Self(_kind: .success) @@ -78,6 +80,7 @@ extension ExitTest.Condition { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static var failure: Self { Self(_kind: .failure) @@ -91,6 +94,7 @@ extension ExitTest.Condition { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public init(_ exitStatus: ExitStatus) { self.init(_kind: .exitStatus(exitStatus)) @@ -126,6 +130,7 @@ extension ExitTest.Condition { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static func exitCode(_ exitCode: CInt) -> Self { #if !SWT_NO_EXIT_TESTS @@ -158,6 +163,7 @@ extension ExitTest.Condition { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static func signal(_ signal: CInt) -> Self { #if !SWT_NO_EXIT_TESTS diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift index ef70a3789..f2c57e205 100644 --- a/Sources/Testing/ExitTests/ExitTest.Result.swift +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -21,12 +21,14 @@ extension ExitTest { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public struct Result: Sendable { /// The exit status reported by the process hosting the exit test. /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public var exitStatus: ExitStatus @@ -57,6 +59,7 @@ extension ExitTest { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public var standardOutputContent: [UInt8] = [] @@ -87,6 +90,7 @@ extension ExitTest { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public var standardErrorContent: [UInt8] = [] diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 9284310db..65945b13b 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -32,6 +32,7 @@ private import _TestingInternals /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") @@ -170,6 +171,7 @@ extension ExitTest { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static var current: ExitTest? { _read { diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index f85c7042b..fcad6e377 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -513,6 +513,7 @@ public macro require( /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } @freestanding(expression) @discardableResult @@ -559,6 +560,7 @@ public macro expect( /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } @freestanding(expression) @discardableResult diff --git a/Sources/Testing/Testing.docc/exit-testing.md b/Sources/Testing/Testing.docc/exit-testing.md index 06ab53dc9..bb4fccafd 100644 --- a/Sources/Testing/Testing.docc/exit-testing.md +++ b/Sources/Testing/Testing.docc/exit-testing.md @@ -12,6 +12,7 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors @Metadata { @Available(Swift, introduced: 6.2) + @Available(Xcode, introduced: 26.0) } Use exit tests to test functionality that might cause a test process to exit. diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index 079b64d8e..d60cf7cce 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -72,6 +72,7 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public func evaluate() async throws -> Bool { switch kind { From 32782178a6a210e1eb399ed9cac3c6efc581cba3 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 10 Jun 2025 16:18:24 -0400 Subject: [PATCH 208/234] Use `posix_spawn_file_actions_adddup2()` to clear `FD_CLOEXEC`. (#1147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On [FreeBSD](https://man.freebsd.org/cgi/man.cgi?query=posix_spawn_file_actions_adddup2), [OpenBSD](https://github.com/openbsd/src/blob/master/lib/libc/gen/posix_spawn.c#L155), [Android](https://android.googlesource.com/platform/bionic/+/master/libc/bionic/spawn.cpp#103), and [Glibc ≥ 2.29](https://sourceware.org/bugzilla/show_bug.cgi?id=23640), `posix_spawn_file_actions_adddup2()` automatically clears `FD_CLOEXEC` if the file descriptors passed to it are equal. Relying on this behaviour eliminates a race condition when spawning child processes. This functionality is standardized in [POSIX.1-2024](https://pubs.opengroup.org/onlinepubs/9799919799/) thanks to [Austin Group Defect #411](https://www.austingroupbugs.net/view.php?id=411). Some older Linuxes (Amazon Linux 2 in particular) don't have this functionality, so we do a runtime check of the Glibc version. This PR is a follow-up to #1145. Resolves #1140. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 9 -- Sources/Testing/ExitTests/SpawnProcess.swift | 50 ++++++--- Sources/Testing/Support/FileHandle.swift | 110 ++++++------------- Sources/Testing/Support/Versions.swift | 24 ++++ Sources/_TestingInternals/include/Includes.h | 4 + 5 files changed, 99 insertions(+), 98 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 65945b13b..6272cea93 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -803,15 +803,6 @@ extension ExitTest { childEnvironment["SWT_EXPERIMENTAL_CAPTURED_VALUES"] = capturedValuesEnvironmentVariable } -#if !SWT_TARGET_OS_APPLE - // Set inherited those file handles that the child process needs. On - // Darwin, this is a no-op because we use POSIX_SPAWN_CLOEXEC_DEFAULT. - try stdoutWriteEnd?.setInherited(true) - try stderrWriteEnd?.setInherited(true) - try backChannelWriteEnd.setInherited(true) - try capturedValuesReadEnd.setInherited(true) -#endif - // Spawn the child process. let processID = try withUnsafePointer(to: backChannelWriteEnd) { backChannelWriteEnd in try withUnsafePointer(to: capturedValuesReadEnd) { capturedValuesReadEnd in diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 8f8d95db6..647e62dd9 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -123,11 +123,27 @@ func spawnExecutable( guard let fd else { throw SystemError(description: "A child process cannot inherit a file handle without an associated file descriptor. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } - if let standardFD { + if let standardFD, standardFD != fd { _ = posix_spawn_file_actions_adddup2(fileActions, fd, standardFD) } else { #if SWT_TARGET_OS_APPLE _ = posix_spawn_file_actions_addinherit_np(fileActions, fd) +#else + // posix_spawn_file_actions_adddup2() will automatically clear + // FD_CLOEXEC after forking but before execing even if the old and + // new file descriptors are equal. This behavior is supported by + // Glibc ≥ 2.29, FreeBSD, OpenBSD, and Android (Bionic) and is + // standardized in POSIX.1-2024 (see https://pubs.opengroup.org/onlinepubs/9799919799/functions/posix_spawn_file_actions_adddup2.html + // and https://www.austingroupbugs.net/view.php?id=411). + _ = posix_spawn_file_actions_adddup2(fileActions, fd, fd) +#if canImport(Glibc) + if _slowPath(glibcVersion.major < 2 || (glibcVersion.major == 2 && glibcVersion.minor < 29)) { + // This system is using an older version of glibc that does not + // implement FD_CLOEXEC clearing in posix_spawn_file_actions_adddup2(), + // so we must clear it here in the parent process. + try setFD_CLOEXEC(false, onFileDescriptor: fd) + } +#endif #endif highestFD = max(highestFD, fd) } @@ -156,8 +172,6 @@ func spawnExecutable( #if !SWT_NO_DYNAMIC_LINKING // This platform doesn't have POSIX_SPAWN_CLOEXEC_DEFAULT, but we can at // least close all file descriptors higher than the highest inherited one. - // We are assuming here that the caller didn't set FD_CLOEXEC on any of - // these file descriptors. _ = _posix_spawn_file_actions_addclosefrom_np?(fileActions, highestFD + 1) #endif #elseif os(FreeBSD) @@ -216,36 +230,42 @@ func spawnExecutable( } #elseif os(Windows) return try _withStartupInfoEx(attributeCount: 1) { startupInfo in - func inherit(_ fileHandle: borrowing FileHandle, as outWindowsHANDLE: inout HANDLE?) throws { + func inherit(_ fileHandle: borrowing FileHandle) throws -> HANDLE? { try fileHandle.withUnsafeWindowsHANDLE { windowsHANDLE in guard let windowsHANDLE else { throw SystemError(description: "A child process cannot inherit a file handle without an associated Windows handle. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } - outWindowsHANDLE = windowsHANDLE + + // Ensure the file handle can be inherited by the child process. + guard SetHandleInformation(windowsHANDLE, DWORD(HANDLE_FLAG_INHERIT), DWORD(HANDLE_FLAG_INHERIT)) else { + throw Win32Error(rawValue: GetLastError()) + } + + return windowsHANDLE } } - func inherit(_ fileHandle: borrowing FileHandle?, as outWindowsHANDLE: inout HANDLE?) throws { + func inherit(_ fileHandle: borrowing FileHandle?) throws -> HANDLE? { if fileHandle != nil { - try inherit(fileHandle!, as: &outWindowsHANDLE) + return try inherit(fileHandle!) } else { - outWindowsHANDLE = nil + return nil } } // Forward standard I/O streams. - try inherit(standardInput, as: &startupInfo.pointee.StartupInfo.hStdInput) - try inherit(standardOutput, as: &startupInfo.pointee.StartupInfo.hStdOutput) - try inherit(standardError, as: &startupInfo.pointee.StartupInfo.hStdError) + startupInfo.pointee.StartupInfo.hStdInput = try inherit(standardInput) + startupInfo.pointee.StartupInfo.hStdOutput = try inherit(standardOutput) + startupInfo.pointee.StartupInfo.hStdError = try inherit(standardError) startupInfo.pointee.StartupInfo.dwFlags |= STARTF_USESTDHANDLES // Ensure standard I/O streams and any explicitly added file handles are // inherited by the child process. var inheritedHandles = [HANDLE?](repeating: nil, count: additionalFileHandles.count + 3) - try inherit(standardInput, as: &inheritedHandles[0]) - try inherit(standardOutput, as: &inheritedHandles[1]) - try inherit(standardError, as: &inheritedHandles[2]) + inheritedHandles[0] = startupInfo.pointee.StartupInfo.hStdInput + inheritedHandles[1] = startupInfo.pointee.StartupInfo.hStdOutput + inheritedHandles[2] = startupInfo.pointee.StartupInfo.hStdError for i in 0 ..< additionalFileHandles.count { - try inherit(additionalFileHandles[i].pointee, as: &inheritedHandles[i + 3]) + inheritedHandles[i + 3] = try inherit(additionalFileHandles[i].pointee) } inheritedHandles = inheritedHandles.compactMap(\.self) diff --git a/Sources/Testing/Support/FileHandle.swift b/Sources/Testing/Support/FileHandle.swift index 4e3c17372..1c5447460 100644 --- a/Sources/Testing/Support/FileHandle.swift +++ b/Sources/Testing/Support/FileHandle.swift @@ -108,8 +108,7 @@ struct FileHandle: ~Copyable, Sendable { /// /// By default, the resulting file handle is not inherited by any child /// processes (that is, `FD_CLOEXEC` is set on POSIX-like systems and - /// `HANDLE_FLAG_INHERIT` is cleared on Windows.) To make it inheritable, call - /// ``setInherited()``. + /// `HANDLE_FLAG_INHERIT` is cleared on Windows.). init(forReadingAtPath path: String) throws { try self.init(atPath: path, mode: "reb") } @@ -123,8 +122,7 @@ struct FileHandle: ~Copyable, Sendable { /// /// By default, the resulting file handle is not inherited by any child /// processes (that is, `FD_CLOEXEC` is set on POSIX-like systems and - /// `HANDLE_FLAG_INHERIT` is cleared on Windows.) To make it inheritable, call - /// ``setInherited()``. + /// `HANDLE_FLAG_INHERIT` is cleared on Windows.). init(forWritingAtPath path: String) throws { try self.init(atPath: path, mode: "web") } @@ -492,8 +490,7 @@ extension FileHandle { /// /// By default, the resulting file handles are not inherited by any child /// processes (that is, `FD_CLOEXEC` is set on POSIX-like systems and - /// `HANDLE_FLAG_INHERIT` is cleared on Windows.) To make them inheritable, - /// call ``setInherited()``. + /// `HANDLE_FLAG_INHERIT` is cleared on Windows.). static func makePipe(readEnd: inout FileHandle?, writeEnd: inout FileHandle?) throws { #if !os(Windows) var pipe2Called = false @@ -533,8 +530,8 @@ extension FileHandle { if !pipe2Called { // pipe2() is not available. Use pipe() instead and simulate O_CLOEXEC // to the best of our ability. - try _setFileDescriptorInherited(fdReadEnd, false) - try _setFileDescriptorInherited(fdWriteEnd, false) + try setFD_CLOEXEC(true, onFileDescriptor: fdReadEnd) + try setFD_CLOEXEC(true, onFileDescriptor: fdWriteEnd) } #endif @@ -612,72 +609,6 @@ extension FileHandle { #endif } #endif - -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) - /// Set whether or not the given file descriptor is inherited by child processes. - /// - /// - Parameters: - /// - fd: The file descriptor. - /// - inherited: Whether or not `fd` is inherited by child processes - /// (ignoring overriding functionality such as Apple's - /// `POSIX_SPAWN_CLOEXEC_DEFAULT` flag.) - /// - /// - Throws: Any error that occurred while setting the flag. - private static func _setFileDescriptorInherited(_ fd: CInt, _ inherited: Bool) throws { - switch swt_getfdflags(fd) { - case -1: - // An error occurred reading the flags for this file descriptor. - throw CError(rawValue: swt_errno()) - case let oldValue: - let newValue = if inherited { - oldValue & ~FD_CLOEXEC - } else { - oldValue | FD_CLOEXEC - } - if oldValue == newValue { - // No need to make a second syscall as nothing has changed. - return - } - if -1 == swt_setfdflags(fd, newValue) { - // An error occurred setting the flags for this file descriptor. - throw CError(rawValue: swt_errno()) - } - } - } -#endif - - /// Set whether or not this file handle is inherited by child processes. - /// - /// - Parameters: - /// - inherited: Whether or not this file handle is inherited by child - /// processes (ignoring overriding functionality such as Apple's - /// `POSIX_SPAWN_CLOEXEC_DEFAULT` flag.) - /// - /// - Throws: Any error that occurred while setting the flag. - func setInherited(_ inherited: Bool) throws { -#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) - try withUnsafePOSIXFileDescriptor { fd in - guard let fd else { - throw SystemError(description: "Cannot set whether a file handle is inherited unless it is backed by a file descriptor. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") - } - try withLock { - try Self._setFileDescriptorInherited(fd, inherited) - } - } -#elseif os(Windows) - return try withUnsafeWindowsHANDLE { handle in - guard let handle else { - throw SystemError(description: "Cannot set whether a file handle is inherited unless it is backed by a Windows file handle. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") - } - let newValue = inherited ? DWORD(HANDLE_FLAG_INHERIT) : 0 - guard SetHandleInformation(handle, DWORD(HANDLE_FLAG_INHERIT), newValue) else { - throw Win32Error(rawValue: GetLastError()) - } - } -#else -#warning("Platform-specific implementation missing: cannot set whether a file handle is inherited") -#endif - } } // MARK: - General path utilities @@ -757,4 +688,35 @@ func canonicalizePath(_ path: String) -> String? { return nil #endif } + +#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) +/// Set the given file descriptor's `FD_CLOEXEC` flag. +/// +/// - Parameters: +/// - flag: The new value of `fd`'s `FD_CLOEXEC` flag. +/// - fd: The file descriptor. +/// +/// - Throws: Any error that occurred while setting the flag. +func setFD_CLOEXEC(_ flag: Bool, onFileDescriptor fd: CInt) throws { + switch swt_getfdflags(fd) { + case -1: + // An error occurred reading the flags for this file descriptor. + throw CError(rawValue: swt_errno()) + case let oldValue: + let newValue = if flag { + oldValue & ~FD_CLOEXEC + } else { + oldValue | FD_CLOEXEC + } + if oldValue == newValue { + // No need to make a second syscall as nothing has changed. + return + } + if -1 == swt_setfdflags(fd, newValue) { + // An error occurred setting the flags for this file descriptor. + throw CError(rawValue: swt_errno()) + } + } +} +#endif #endif diff --git a/Sources/Testing/Support/Versions.swift b/Sources/Testing/Support/Versions.swift index 1eb7f4e48..1229e80b0 100644 --- a/Sources/Testing/Support/Versions.swift +++ b/Sources/Testing/Support/Versions.swift @@ -153,6 +153,30 @@ let swiftStandardLibraryVersion: String = { return "unknown" }() +#if canImport(Glibc) +/// The (runtime, not compile-time) version of glibc in use on this system. +/// +/// This value is not part of the public interface of the testing library. +let glibcVersion: (major: Int, minor: Int) = { + // Default to the statically available version number if the function call + // fails for some reason. + var major = Int(clamping: __GLIBC__) + var minor = Int(clamping: __GLIBC_MINOR__) + + if let strVersion = gnu_get_libc_version() { + withUnsafeMutablePointer(to: &major) { major in + withUnsafeMutablePointer(to: &minor) { minor in + withVaList([major, minor]) { args in + _ = vsscanf(strVersion, "%zd.%zd", args) + } + } + } + } + + return (major, minor) +}() +#endif + // MARK: - sysctlbyname() Wrapper #if !SWT_NO_SYSCTL && SWT_TARGET_OS_APPLE diff --git a/Sources/_TestingInternals/include/Includes.h b/Sources/_TestingInternals/include/Includes.h index bfc87b001..1b95151cb 100644 --- a/Sources/_TestingInternals/include/Includes.h +++ b/Sources/_TestingInternals/include/Includes.h @@ -53,6 +53,10 @@ #include #endif +#if __has_include() +#include +#endif + #if __has_include() && !defined(__wasi__) #include #endif From 6e462ad31397980cfdaf25220f973d64565ce844 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 12 Jun 2025 17:50:51 -0400 Subject: [PATCH 209/234] Synthesize display names for de facto suites with raw identifiers. (#1105) This PR ensures that suite types that don't have the `@Suite` attribute but which _do_ have raw identifiers for names are correctly given display names the same way those with `@Suite` would be. This PR also ensures that we transform spaces in raw identifiers after they are demangled by the runtime--namely, the runtime replaces ASCII spaces (as typed by the user) with Unicode non-breaking spaces (which aren't otherwise valid in raw identifers) in order to avoid issues with existing uses of spaces in demangled names. We want to make sure that identifiers as presented to the user match what the user has typed, so we need to transform these spaces back. No changes in this area are needed for display names derived during macro expansion because we do the relevant work based on the source text which still has the original ASCII spaces. This PR also deletes the "`raw$`" hack that I put in place when originally implementing raw identifier support as the entire toolchain supports them now. Resolves #1104. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Testing/Parameterization/TypeInfo.swift | 59 ++++++++++++++++--- Sources/Testing/Test.swift | 9 ++- .../Additions/TokenSyntaxAdditions.swift | 6 -- .../TestDeclarationMacroTests.swift | 12 ++-- Tests/TestingTests/MiscellaneousTests.swift | 19 ++++-- 5 files changed, 79 insertions(+), 26 deletions(-) diff --git a/Sources/Testing/Parameterization/TypeInfo.swift b/Sources/Testing/Parameterization/TypeInfo.swift index b0ed814b9..300004e16 100644 --- a/Sources/Testing/Parameterization/TypeInfo.swift +++ b/Sources/Testing/Parameterization/TypeInfo.swift @@ -142,6 +142,35 @@ func rawIdentifierAwareSplit(_ string: S, separator: Character, maxSplits: In } extension TypeInfo { + /// Replace any non-breaking spaces in the given string with normal spaces. + /// + /// - Parameters: + /// - rawIdentifier: The string to rewrite. + /// + /// - Returns: A copy of `rawIdentifier` with non-breaking spaces (`U+00A0`) + /// replaced with normal spaces (`U+0020`). + /// + /// When the Swift runtime demangles a raw identifier, it [replaces](https://github.com/swiftlang/swift/blob/d033eec1aa427f40dcc38679d43b83d9dbc06ae7/lib/Basic/Mangler.cpp#L250) + /// normal ASCII spaces with non-breaking spaces to maintain compatibility + /// with historical usages of spaces in mangled name forms. Non-breaking + /// spaces are not otherwise valid in raw identifiers, so this transformation + /// is reversible. + private static func _rewriteNonBreakingSpacesAsASCIISpaces(in rawIdentifier: some StringProtocol) -> String? { + let nbsp = "\u{00A0}" as UnicodeScalar + + // If there are no non-breaking spaces in the string, exit early to avoid + // any further allocations. + let unicodeScalars = rawIdentifier.unicodeScalars + guard unicodeScalars.contains(nbsp) else { + return nil + } + + // Replace non-breaking spaces, then construct a new string from the + // resulting sequence. + let result = unicodeScalars.lazy.map { $0 == nbsp ? " " : $0 } + return String(String.UnicodeScalarView(result)) + } + /// An in-memory cache of fully-qualified type name components. private static let _fullyQualifiedNameComponentsCache = Locked<[ObjectIdentifier: [String]]>() @@ -166,12 +195,21 @@ extension TypeInfo { components[0] = moduleName } - // If a type is private or embedded in a function, its fully qualified - // name may include "(unknown context at $xxxxxxxx)" as a component. Strip - // those out as they're uninteresting to us. - components = components.filter { !$0.starts(with: "(unknown context at") } - - return components.map(String.init) + return components.lazy + .filter { component in + // If a type is private or embedded in a function, its fully qualified + // name may include "(unknown context at $xxxxxxxx)" as a component. + // Strip those out as they're uninteresting to us. + !component.starts(with: "(unknown context at") + }.map { component in + // Replace non-breaking spaces with spaces. See the helper function's + // documentation for more information. + if let component = _rewriteNonBreakingSpacesAsASCIISpaces(in: component) { + component[...] + } else { + component + } + }.map(String.init) } /// The complete name of this type, with the names of all referenced types @@ -242,9 +280,14 @@ extension TypeInfo { public var unqualifiedName: String { switch _kind { case let .type(type): - String(describing: type) + // Replace non-breaking spaces with spaces. See the helper function's + // documentation for more information. + var result = String(describing: type) + result = Self._rewriteNonBreakingSpacesAsASCIISpaces(in: result) ?? result + + return result case let .nameOnly(_, unqualifiedName, _): - unqualifiedName + return unqualifiedName } } diff --git a/Sources/Testing/Test.swift b/Sources/Testing/Test.swift index 738daf72d..5f2ac2406 100644 --- a/Sources/Testing/Test.swift +++ b/Sources/Testing/Test.swift @@ -209,8 +209,13 @@ public struct Test: Sendable { containingTypeInfo: TypeInfo, isSynthesized: Bool = false ) { - self.name = containingTypeInfo.unqualifiedName - self.displayName = displayName + let name = containingTypeInfo.unqualifiedName + self.name = name + if let displayName { + self.displayName = displayName + } else if isSynthesized && name.count > 2 && name.first == "`" && name.last == "`" { + self.displayName = String(name.dropFirst().dropLast()) + } self.traits = traits self.sourceLocation = sourceLocation self.containingTypeInfo = containingTypeInfo diff --git a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift index 447a18dee..26b9d1923 100644 --- a/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/TokenSyntaxAdditions.swift @@ -39,12 +39,6 @@ extension TokenSyntax { return textWithoutBackticks } - // TODO: remove this mock path once the toolchain fully supports raw IDs. - let mockPrefix = "__raw__$" - if backticksRemoved, textWithoutBackticks.starts(with: mockPrefix) { - return String(textWithoutBackticks.dropFirst(mockPrefix.count)) - } - return nil } } diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index 13ae3d180..6c04eb9eb 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -217,17 +217,17 @@ struct TestDeclarationMacroTests { ] ), - #"@Test("Goodbye world") func `__raw__$helloWorld`()"#: + #"@Test("Goodbye world") func `hello world`()"#: ( - message: "Attribute 'Test' specifies display name 'Goodbye world' for function with implicit display name 'helloWorld'", + message: "Attribute 'Test' specifies display name 'Goodbye world' for function with implicit display name 'hello world'", fixIts: [ ExpectedFixIt( message: "Remove 'Goodbye world'", changes: [.replace(oldSourceCode: #""Goodbye world""#, newSourceCode: "")] ), ExpectedFixIt( - message: "Rename '__raw__$helloWorld'", - changes: [.replace(oldSourceCode: "`__raw__$helloWorld`", newSourceCode: "\(EditorPlaceholderExprSyntax("name"))")] + message: "Rename 'hello world'", + changes: [.replace(oldSourceCode: "`hello world`", newSourceCode: "\(EditorPlaceholderExprSyntax("name"))")] ), ] ), @@ -281,10 +281,10 @@ struct TestDeclarationMacroTests { @Test("Raw function name components") func rawFunctionNameComponents() throws { let decl = """ - func `__raw__$hello`(`__raw__$world`: T, etc: U, `blah`: V) {} + func `hello there`(`world of mine`: T, etc: U, `blah`: V) {} """ as DeclSyntax let functionDecl = try #require(decl.as(FunctionDeclSyntax.self)) - #expect(functionDecl.completeName.trimmedDescription == "`hello`(`world`:etc:blah:)") + #expect(functionDecl.completeName.trimmedDescription == "`hello there`(`world of mine`:etc:blah:)") } @Test("Warning diagnostics emitted on API misuse", diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index 9ae326afe..b895f6c1b 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -297,15 +297,26 @@ struct MiscellaneousTests { #expect(testType.displayName == "Named Sendable test type") } - @Test func `__raw__$raw_identifier_provides_a_display_name`() throws { +#if compiler(>=6.2) && hasFeature(RawIdentifiers) + @Test func `Test with raw identifier gets a display name`() throws { let test = try #require(Test.current) - #expect(test.displayName == "raw_identifier_provides_a_display_name") - #expect(test.name == "`raw_identifier_provides_a_display_name`()") + #expect(test.displayName == "Test with raw identifier gets a display name") + #expect(test.name == "`Test with raw identifier gets a display name`()") let id = test.id #expect(id.moduleName == "TestingTests") - #expect(id.nameComponents == ["MiscellaneousTests", "`raw_identifier_provides_a_display_name`()"]) + #expect(id.nameComponents == ["MiscellaneousTests", "`Test with raw identifier gets a display name`()"]) } + @Test func `Suite type with raw identifier gets a display name`() throws { + struct `Suite With De Facto Display Name` {} + let typeInfo = TypeInfo(describing: `Suite With De Facto Display Name`.self) + let suite = Test(traits: [], sourceLocation: #_sourceLocation, containingTypeInfo: typeInfo, isSynthesized: true) + #expect(suite.name == "`Suite With De Facto Display Name`") + let displayName = try #require(suite.displayName) + #expect(displayName == "Suite With De Facto Display Name") + } +#endif + @Test("Free functions are runnable") func freeFunction() async throws { await Test(testFunction: freeSyncFunction).run() From 61a01cb30c7e862d4ff1689a470f61d88e9e680a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 12 Jun 2025 19:34:37 -0400 Subject: [PATCH 210/234] Have `DiscoverableAsTestContent` enumeration produce `some Sequence` instead of `AnySequence`. (#1122) This PR changes how `DiscoverableAsTestContent` enumeration works so that we can return `some Sequence` instead of `AnySequence`. We do so by removing the `~Copyable` constraint on the protocol, which subsequently causes the compiler to get confused and crash trying to represent `some Sequence>` where `T: DiscoverableAsTestContent`. The only supported/allowed consumers of the `DiscoverableAsTestContent` protocol are Swift Testing and the experimental Playgrounds package, neither of which uses (or needs to use) a move-only type here. Earlier, `ExitTest` conformed to `DiscoverableAsTestContent`, but this was changed and is no longer necessary. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Test+Discovery+Legacy.swift | 4 +-- .../DiscoverableAsTestContent.swift | 4 +-- .../_TestDiscovery/TestContentRecord.swift | 28 ++++++------------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/Sources/Testing/Test+Discovery+Legacy.swift b/Sources/Testing/Test+Discovery+Legacy.swift index 8ff878338..cafc55a4e 100644 --- a/Sources/Testing/Test+Discovery+Legacy.swift +++ b/Sources/Testing/Test+Discovery+Legacy.swift @@ -24,14 +24,14 @@ public protocol __TestContentRecordContainer { nonisolated static var __testContentRecord: __TestContentRecord { get } } -extension DiscoverableAsTestContent where Self: ~Copyable { +extension DiscoverableAsTestContent { /// Get all test content of this type known to Swift and found in the current /// process using the legacy discovery mechanism. /// /// - Returns: A sequence of instances of ``TestContentRecord``. Only test /// content records matching this ``TestContent`` type's requirements are /// included in the sequence. - static func allTypeMetadataBasedTestContentRecords() -> AnySequence> { + static func allTypeMetadataBasedTestContentRecords() -> some Sequence> { return allTypeMetadataBasedTestContentRecords { type, buffer in guard let type = type as? any __TestContentRecordContainer.Type else { return false diff --git a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift index d4b15f8db..16369f82a 100644 --- a/Sources/_TestDiscovery/DiscoverableAsTestContent.swift +++ b/Sources/_TestDiscovery/DiscoverableAsTestContent.swift @@ -16,7 +16,7 @@ /// because they may be discovered within any isolation context or within /// multiple isolation contexts running concurrently. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) -public protocol DiscoverableAsTestContent: Sendable, ~Copyable { +public protocol DiscoverableAsTestContent: Sendable { /// The value of the `kind` field in test content records associated with this /// type. /// @@ -49,7 +49,7 @@ public protocol DiscoverableAsTestContent: Sendable, ~Copyable { } #if !SWT_NO_LEGACY_TEST_DISCOVERY -extension DiscoverableAsTestContent where Self: ~Copyable { +extension DiscoverableAsTestContent { public static var _testContentTypeNameHint: String { "__🟡$" } diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index 9224fc2ea..df868c975 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -44,7 +44,7 @@ private typealias _TestContentRecord = ( reserved2: UInt ) -extension DiscoverableAsTestContent where Self: ~Copyable { +extension DiscoverableAsTestContent { /// Check that the layout of this structure in memory matches its expected /// layout in the test content section. /// @@ -64,7 +64,7 @@ extension DiscoverableAsTestContent where Self: ~Copyable { /// ``DiscoverableAsTestContent/allTestContentRecords()`` on a type that /// conforms to ``DiscoverableAsTestContent``. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) -public struct TestContentRecord where T: DiscoverableAsTestContent & ~Copyable { +public struct TestContentRecord where T: DiscoverableAsTestContent { /// The base address of the image containing this instance, if known. /// /// The type of this pointer is platform-dependent: @@ -229,24 +229,19 @@ extension TestContentRecord: CustomStringConvertible { // MARK: - Enumeration of test content records -extension DiscoverableAsTestContent where Self: ~Copyable { +extension DiscoverableAsTestContent { /// Get all test content of this type known to Swift and found in the current /// process. /// /// - Returns: A sequence of instances of ``TestContentRecord``. Only test /// content records matching this ``TestContent`` type's requirements are /// included in the sequence. - /// - /// @Comment { - /// - Bug: This function returns an instance of `AnySequence` instead of an - /// opaque type due to a compiler crash. ([143080508](rdar://143080508)) - /// } - public static func allTestContentRecords() -> AnySequence> { + public static func allTestContentRecords() -> some Sequence> { validateMemoryLayout() let kind = testContentKind.rawValue - let result = SectionBounds.all(.testContent).lazy.flatMap { sb in + return SectionBounds.all(.testContent).lazy.flatMap { sb in sb.buffer.withMemoryRebound(to: _TestContentRecord.self) { records in (0 ..< records.count).lazy .map { (records.baseAddress! + $0) as UnsafePointer<_TestContentRecord> } @@ -254,7 +249,6 @@ extension DiscoverableAsTestContent where Self: ~Copyable { .map { TestContentRecord(imageAddress: sb.imageAddress, recordAddress: $0) } } } - return AnySequence(result) } } @@ -263,7 +257,7 @@ extension DiscoverableAsTestContent where Self: ~Copyable { private import _TestingInternals -extension DiscoverableAsTestContent where Self: ~Copyable { +extension DiscoverableAsTestContent { /// Get all test content of this type known to Swift and found in the current /// process using the legacy discovery mechanism. /// @@ -277,15 +271,10 @@ extension DiscoverableAsTestContent where Self: ~Copyable { /// - Returns: A sequence of instances of ``TestContentRecord``. Only test /// content records matching this ``TestContent`` type's requirements are /// included in the sequence. - /// - /// @Comment { - /// - Bug: This function returns an instance of `AnySequence` instead of an - /// opaque type due to a compiler crash. ([143080508](rdar://143080508)) - /// } @available(swift, deprecated: 100000.0, message: "Do not adopt this functionality in new code. It will be removed in a future release.") public static func allTypeMetadataBasedTestContentRecords( loadingWith loader: @escaping @Sendable (Any.Type, UnsafeMutableRawBufferPointer) -> Bool - ) -> AnySequence> { + ) -> some Sequence> { validateMemoryLayout() let typeNameHint = _testContentTypeNameHint @@ -300,7 +289,7 @@ extension DiscoverableAsTestContent where Self: ~Copyable { } } - let result = SectionBounds.all(.typeMetadata).lazy.flatMap { sb in + return SectionBounds.all(.typeMetadata).lazy.flatMap { sb in stride(from: 0, to: sb.buffer.count, by: SWTTypeMetadataRecordByteCount).lazy .map { sb.buffer.baseAddress! + $0 } .compactMap { swt_getType(fromTypeMetadataRecord: $0, ifNameContains: typeNameHint) } @@ -309,7 +298,6 @@ extension DiscoverableAsTestContent where Self: ~Copyable { .filter { $0.kind == kind } .map { TestContentRecord(imageAddress: sb.imageAddress, record: $0) } } - return AnySequence(result) } } #endif From eac4833c11eba5ba7fd926cae5acebd0643f53d9 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 12 Jun 2025 19:34:59 -0400 Subject: [PATCH 211/234] Improve the diagnostics for a bad exit test capture. (#1146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR improves the diagnostics presented at compile time when an exit test captures an unsupported value. For example, given the following (bad) exit test: ```swift struct NonCodableValue {} let x = NonCodableValue() await #expect(processExitsWith: .success) { [x = x as NonCodableValue] in _ = x } ``` We currently get diagnostics of the form: > 🛑 Global function '__checkClosureCall(identifiedBy:encodingCapturedValues:processExitsWith:observing:performing:expression:comments:isRequired:isolation:sourceLocation:)' requires that 'NonCodableValue' conform to 'Decodable' > 🛑 Global function '__checkClosureCall(identifiedBy:encodingCapturedValues:processExitsWith:observing:performing:expression:comments:isRequired:isolation:sourceLocation:)' requires that 'NonCodableValue' conform to 'Encodable' > ⚠️ No 'async' operations occur within 'await' expression None of which actually tell the developer (clearly) what's wrong. With this PR, we instead get: > 🛑 Type of captured value 'x' must conform to 'Sendable' and 'Codable' (from macro '__capturedValue') Much better! The diagnostic is attributed to the temporary file containing the expansion of `#expect()` rather than to the original source file, but I've opened [an issue](https://github.com/swiftlang/swift-syntax/issues/3085) against swift-syntax with a fix in mind. Even with the misattribution, this diagnostic is still an improvement, yeah? > [!NOTE] > Exit test value capture remains an experimental feature. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.swift | 21 ++++++++ .../Expectations/Expectation+Macro.swift | 34 ++++++++++++ Sources/TestingMacros/CMakeLists.txt | 1 + Sources/TestingMacros/ConditionMacro.swift | 2 +- .../ExitTestCapturedValueMacro.swift | 54 +++++++++++++++++++ .../Support/ClosureCaptureListParsing.swift | 9 +++- .../Support/DiagnosticMessage.swift | 18 +++++++ Sources/TestingMacros/TestingMacrosMain.swift | 2 + Tests/TestingTests/ExitTestTests.swift | 24 +++++++++ 9 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 Sources/TestingMacros/ExitTestCapturedValueMacro.swift diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 6272cea93..c5579981e 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -365,6 +365,27 @@ extension ExitTest { outValue.initializeMemory(as: Record.self, to: record) return true } + + /// Attempt to store an invalid exit test into the given memory. + /// + /// This overload of `__store()` is provided to suppress diagnostics when a + /// value of an unsupported type is captured as an argument of `body`. It + /// always terminates the current process. + /// + /// - Warning: This function is used to implement the + /// `#expect(processExitsWith:)` macro. Do not use it directly. +#if compiler(>=6.2) + @safe +#endif + public static func __store( + _ id: (UInt64, UInt64, UInt64, UInt64), + _ body: T, + into outValue: UnsafeMutableRawPointer, + asTypeAt typeAddress: UnsafeRawPointer, + withHintAt hintAddress: UnsafeRawPointer? = nil + ) -> CBool { + fatalError("Unimplemented") + } } @_spi(ForToolsIntegrationOnly) diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index fcad6e377..973efde53 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -574,3 +574,37 @@ public macro require( sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void ) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") + +/// Capture a sendable and codable value to pass to an exit test. +/// +/// - Parameters: +/// - value: The captured value. +/// - name: The name of the capture list item corresponding to `value`. +/// +/// - Returns: `value` verbatim. +/// +/// - Warning: This macro is used to implement the `#expect(processExitsWith:)` +/// macro. Do not use it directly. +@freestanding(expression) +public macro __capturedValue( + _ value: T, + _ name: String +) -> T = #externalMacro(module: "TestingMacros", type: "ExitTestCapturedValueMacro") where T: Sendable & Codable + +/// Emit a compile-time diagnostic when an unsupported value is captured by an +/// exit test. +/// +/// - Parameters: +/// - value: The captured value. +/// - name: The name of the capture list item corresponding to `value`. +/// +/// - Returns: The result of a call to `fatalError()`. `value` is discarded at +/// compile time. +/// +/// - Warning: This macro is used to implement the `#expect(processExitsWith:)` +/// macro. Do not use it directly. +@freestanding(expression) +public macro __capturedValue( + _ value: borrowing T, + _ name: String +) -> Never = #externalMacro(module: "TestingMacros", type: "ExitTestBadCapturedValueMacro") where T: ~Copyable & ~Escapable diff --git a/Sources/TestingMacros/CMakeLists.txt b/Sources/TestingMacros/CMakeLists.txt index c9a579eaf..effa782a9 100644 --- a/Sources/TestingMacros/CMakeLists.txt +++ b/Sources/TestingMacros/CMakeLists.txt @@ -81,6 +81,7 @@ endif() target_sources(TestingMacros PRIVATE ConditionMacro.swift + ExitTestCapturedValueMacro.swift PragmaMacro.swift SourceLocationMacro.swift SuiteDeclarationMacro.swift diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 9f87dfbd3..37cbc7339 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -551,7 +551,7 @@ extension ExitTestConditionMacro { label: "encodingCapturedValues", expression: TupleExprSyntax { for capturedValue in capturedValues { - LabeledExprSyntax(expression: capturedValue.expression.trimmed) + LabeledExprSyntax(expression: capturedValue.typeCheckedExpression) } } ) diff --git a/Sources/TestingMacros/ExitTestCapturedValueMacro.swift b/Sources/TestingMacros/ExitTestCapturedValueMacro.swift new file mode 100644 index 000000000..0038dac7c --- /dev/null +++ b/Sources/TestingMacros/ExitTestCapturedValueMacro.swift @@ -0,0 +1,54 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +public import SwiftSyntax +import SwiftSyntaxBuilder +public import SwiftSyntaxMacros + +/// The implementation of the `#__capturedValue()` macro when the value conforms +/// to the necessary protocols. +/// +/// This type is used to implement the `#__capturedValue()` macro. Do not use it +/// directly. +public struct ExitTestCapturedValueMacro: ExpressionMacro, Sendable { + public static func expansion( + of macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) throws -> ExprSyntax { + let arguments = Array(macro.arguments) + let expr = arguments[0].expression + + // No additional processing is required as this expression's type meets our + // requirements. + + return expr + } +} + +/// The implementation of the `#__capturedValue()` macro when the value does +/// _not_ conform to the necessary protocols. +/// +/// This type is used to implement the `#__capturedValue()` macro. Do not use it +/// directly. +public struct ExitTestBadCapturedValueMacro: ExpressionMacro, Sendable { + public static func expansion( + of macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) throws -> ExprSyntax { + let arguments = Array(macro.arguments) + let expr = arguments[0].expression + let nameExpr = arguments[1].expression.cast(StringLiteralExprSyntax.self) + + // Diagnose that the type of 'expr' is invalid. + context.diagnose(.capturedValueMustBeSendableAndCodable(expr, name: nameExpr)) + + return #"Swift.fatalError("Unsupported")"# + } +} diff --git a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift index 41abe711c..a7dca88af 100644 --- a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift +++ b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift @@ -34,10 +34,15 @@ struct CapturedValueInfo { /// The type of the captured value. var type: TypeSyntax + /// The expression to assign to the captured value with type-checking applied. + var typeCheckedExpression: ExprSyntax { + #"#__capturedValue(\#(expression.trimmed), \#(literal: name.trimmedDescription))"# + } + init(_ capture: ClosureCaptureSyntax, in context: some MacroExpansionContext) { self.capture = capture - self.expression = "()" - self.type = "Swift.Void" + self.expression = #"Swift.fatalError("Unsupported")"# + self.type = "Swift.Never" // We don't support capture specifiers at this time. if let specifier = capture.specifier { diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index 36186ec4b..3a8957207 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -827,6 +827,24 @@ extension DiagnosticMessage { ) } + /// Create a diagnostic message stating that a captured value must conform to + /// `Sendable` and `Codable`. + /// + /// - Parameters: + /// - valueExpr: The captured value. + /// - nameExpr: The name of the capture list item corresponding to + /// `valueExpr`. + /// + /// - Returns: A diagnostic message. + static func capturedValueMustBeSendableAndCodable(_ valueExpr: ExprSyntax, name nameExpr: StringLiteralExprSyntax) -> Self { + let name = nameExpr.representedLiteralValue ?? valueExpr.trimmedDescription + return Self( + syntax: Syntax(valueExpr), + message: "Type of captured value '\(name)' must conform to 'Sendable' and 'Codable'", + severity: .error + ) + } + /// Create a diagnostic message stating that a capture clause cannot be used /// in an exit test. /// diff --git a/Sources/TestingMacros/TestingMacrosMain.swift b/Sources/TestingMacros/TestingMacrosMain.swift index 1894f4282..4e98115d0 100644 --- a/Sources/TestingMacros/TestingMacrosMain.swift +++ b/Sources/TestingMacros/TestingMacrosMain.swift @@ -28,6 +28,8 @@ struct TestingMacrosMain: CompilerPlugin { RequireThrowsNeverMacro.self, ExitTestExpectMacro.self, ExitTestRequireMacro.self, + ExitTestCapturedValueMacro.self, + ExitTestBadCapturedValueMacro.self, TagMacro.self, SourceLocationMacro.self, PragmaMacro.self, diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 02be1a140..fb2d47f03 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -456,6 +456,30 @@ private import _TestingInternals #expect(instance.x == 123) } } + + @Test("Capturing #_sourceLocation") + func captureListPreservesSourceLocationMacro() async { + func sl(_ sl: SourceLocation = #_sourceLocation) -> SourceLocation { + sl + } + await #expect(processExitsWith: .success) { [sl = sl() as SourceLocation] in + #expect(sl.fileID == #fileID) + } + } + +#if false // intentionally fails to compile + struct NonCodableValue {} + + // We can't capture a value that isn't Codable. A unit test is not possible + // for this case as the type checker needs to get involved. + @Test("Capturing a move-only value") + func captureListWithMoveOnlyValue() async { + let x = NonCodableValue() + await #expect(processExitsWith: .success) { [x = x as NonCodableValue] in + _ = x + } + } +#endif #endif } From e7851c5f3ff9994b0e8d8680ffaa7e8e4712d5a8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 12 Jun 2025 19:35:25 -0400 Subject: [PATCH 212/234] Use our `posix_spawn()` wrapper in the Foundation CIO rather than `Process`. (#1114) We already have custom `async`-friendly code for spawning child processes, so let's use it instead of relying on `Foundation.Process` when we need to call the platform's `zip` or `tar` tool. Currently we do this in the Foundation cross-import overlay (hence why we were using `Foundation.Process` at all). ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Documentation/Porting.md | 3 + Package.swift | 47 ++++++++----- .../Attachments/Attachment+URL.swift | 70 ++++++++----------- Sources/Testing/ExitTests/SpawnProcess.swift | 40 +++++++++-- cmake/modules/shared/CompilerSettings.cmake | 1 + 5 files changed, 95 insertions(+), 66 deletions(-) diff --git a/Documentation/Porting.md b/Documentation/Porting.md index 6e83e0eb0..f7aecf97e 100644 --- a/Documentation/Porting.md +++ b/Documentation/Porting.md @@ -67,6 +67,9 @@ platform-specific attention. > conflicting requirements (for example, attempting to enable support for pipes > without also enabling support for file I/O.) You should be able to resolve > these issues by updating `Package.swift` and/or `CompilerSettings.cmake`. +> +> Don't forget to add your platform to the `BuildSettingCondition/whenApple(_:)` +> function in `Package.swift`. Most platform dependencies can be resolved through the use of platform-specific API. For example, Swift Testing uses the C11 standard [`timespec`](https://en.cppreference.com/w/c/chrono/timespec) diff --git a/Package.swift b/Package.swift index cce384d7a..4360aabdc 100644 --- a/Package.swift +++ b/Package.swift @@ -27,7 +27,7 @@ let buildingForDevelopment = (git?.currentTag == nil) /// to change in the future. /// /// - Bug: There is currently no way for us to tell if we are being asked to -/// build for an Embedded Swift target at the package manifest level. +/// build for an Embedded Swift target at the package manifest level. /// ([swift-syntax-#8431](https://github.com/swiftlang/swift-package-manager/issues/8431)) let buildingForEmbedded: Bool = { guard let envvar = Context.environment["SWT_EMBEDDED"] else { @@ -208,7 +208,7 @@ let package = Package( // The Foundation module only has Library Evolution enabled on Apple // platforms, and since this target's module publicly imports Foundation, // it can only enable Library Evolution itself on those platforms. - swiftSettings: .packageSettings + .enableLibraryEvolution(applePlatformsOnly: true) + swiftSettings: .packageSettings + .enableLibraryEvolution(.whenApple()) ), // Utility targets: These are utilities intended for use when developing @@ -244,11 +244,11 @@ extension BuildSettingCondition { /// Swift. /// /// - Parameters: - /// - nonEmbeddedCondition: The value to return if the target is not - /// Embedded Swift. If `nil`, the build condition evaluates to `false`. + /// - nonEmbeddedCondition: The value to return if the target is not + /// Embedded Swift. If `nil`, the build condition evaluates to `false`. /// /// - Returns: A build setting condition that evaluates to `true` for Embedded - /// Swift or is equal to `nonEmbeddedCondition` for non-Embedded Swift. + /// Swift or is equal to `nonEmbeddedCondition` for non-Embedded Swift. static func whenEmbedded(or nonEmbeddedCondition: @autoclosure () -> Self? = nil) -> Self? { if !buildingForEmbedded { if let nonEmbeddedCondition = nonEmbeddedCondition() { @@ -263,6 +263,21 @@ extension BuildSettingCondition { nil } } + + /// A build setting condition representing all Apple or non-Apple platforms. + /// + /// - Parameters: + /// - isApple: Whether or not the result represents Apple platforms. + /// + /// - Returns: A build setting condition that evaluates to `isApple` for Apple + /// platforms. + static func whenApple(_ isApple: Bool = true) -> Self { + if isApple { + .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS]) + } else { + .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android]) + } + } } extension Array where Element == PackageDescription.SwiftSetting { @@ -312,13 +327,14 @@ extension Array where Element == PackageDescription.SwiftSetting { // executable rather than a library. .define("SWT_NO_LIBRARY_MACRO_PLUGINS"), - .define("SWT_TARGET_OS_APPLE", .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS])), + .define("SWT_TARGET_OS_APPLE", .whenApple()), .define("SWT_NO_EXIT_TESTS", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), .define("SWT_NO_PROCESS_SPAWNING", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), - .define("SWT_NO_SNAPSHOT_TYPES", .whenEmbedded(or: .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android]))), + .define("SWT_NO_SNAPSHOT_TYPES", .whenEmbedded(or: .whenApple(false))), .define("SWT_NO_DYNAMIC_LINKING", .whenEmbedded(or: .when(platforms: [.wasi]))), .define("SWT_NO_PIPES", .whenEmbedded(or: .when(platforms: [.wasi]))), + .define("SWT_NO_FOUNDATION_FILE_COORDINATION", .whenEmbedded(or: .whenApple(false))), .define("SWT_NO_LEGACY_TEST_DISCOVERY", .whenEmbedded()), .define("SWT_NO_LIBDISPATCH", .whenEmbedded()), @@ -354,20 +370,16 @@ extension Array where Element == PackageDescription.SwiftSetting { ] } - /// Create a Swift setting which enables Library Evolution, optionally - /// constraining it to only Apple platforms. + /// Create a Swift setting which enables Library Evolution. /// /// - Parameters: - /// - applePlatformsOnly: Whether to constrain this setting to only Apple - /// platforms. - static func enableLibraryEvolution(applePlatformsOnly: Bool = false) -> Self { + /// - condition: A build setting condition to apply to this setting. + /// + /// - Returns: A Swift setting that enables Library Evolution. + static func enableLibraryEvolution(_ condition: BuildSettingCondition? = nil) -> Self { var result = [PackageDescription.SwiftSetting]() if buildingForDevelopment { - var condition: BuildSettingCondition? - if applePlatformsOnly { - condition = .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS]) - } result.append(.unsafeFlags(["-enable-library-evolution"], condition)) } @@ -410,9 +422,10 @@ extension Array where Element == PackageDescription.CXXSetting { result += [ .define("SWT_NO_EXIT_TESTS", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), .define("SWT_NO_PROCESS_SPAWNING", .whenEmbedded(or: .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android]))), - .define("SWT_NO_SNAPSHOT_TYPES", .whenEmbedded(or: .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android]))), + .define("SWT_NO_SNAPSHOT_TYPES", .whenEmbedded(or: .whenApple(false))), .define("SWT_NO_DYNAMIC_LINKING", .whenEmbedded(or: .when(platforms: [.wasi]))), .define("SWT_NO_PIPES", .whenEmbedded(or: .when(platforms: [.wasi]))), + .define("SWT_NO_FOUNDATION_FILE_COORDINATION", .whenEmbedded(or: .whenApple(false))), .define("SWT_NO_LEGACY_TEST_DISCOVERY", .whenEmbedded()), .define("SWT_NO_LIBDISPATCH", .whenEmbedded()), diff --git a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift index a018363fc..bb3668180 100644 --- a/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift +++ b/Sources/Overlays/_Testing_Foundation/Attachments/Attachment+URL.swift @@ -71,7 +71,7 @@ extension Attachment where AttachableValue == _AttachableURLWrapper { let url = url.resolvingSymlinksInPath() let isDirectory = try url.resourceValues(forKeys: [.isDirectoryKey]).isDirectory! -#if SWT_TARGET_OS_APPLE +#if SWT_TARGET_OS_APPLE && !SWT_NO_FOUNDATION_FILE_COORDINATION let data: Data = try await withCheckedThrowingContinuation { continuation in let fileCoordinator = NSFileCoordinator() let fileAccessIntent = NSFileAccessIntent.readingIntent(with: url, options: [.forUploading]) @@ -166,25 +166,31 @@ private func _compressContentsOfDirectory(at directoryURL: URL) async throws -> // knows how to write PKZIP archives, while Windows inherited FreeBSD's tar // tool in Windows 10 Build 17063 (per https://techcommunity.microsoft.com/blog/containers/tar-and-curl-come-to-windows/382409). // - // On Linux (which does not have FreeBSD's version of tar(1)), we can use - // zip(1) instead. + // On Linux and OpenBSD (which do not have FreeBSD's version of tar(1)), we + // can use zip(1) instead. This tool compresses paths relative to the current + // working directory, and posix_spawn_file_actions_addchdir_np() is not always + // available for us to call (not present on OpenBSD, requires glibc ≥ 2.28 on + // Linux), so we'll spawn a shell that calls cd before calling zip(1). // // OpenBSD's tar(1) does not support writing PKZIP archives, and /usr/bin/zip // tool is an optional install, so we check if it's present before trying to // execute it. +#if os(Linux) || os(OpenBSD) + let archiverPath = "/bin/sh" #if os(Linux) - let archiverPath = "/usr/bin/zip" -#elseif SWT_TARGET_OS_APPLE || os(FreeBSD) - let archiverPath = "/usr/bin/tar" -#elseif os(OpenBSD) - let archiverPath = "/usr/local/bin/zip" + let trueArchiverPath = "/usr/bin/zip" +#else + let trueArchiverPath = "/usr/local/bin/zip" var isDirectory = false - if !FileManager.default.fileExists(atPath: archiverPath, isDirectory: &isDirectory) || isDirectory { + if !FileManager.default.fileExists(atPath: trueArchiverPath, isDirectory: &isDirectory) || isDirectory { throw CocoaError(.fileNoSuchFile, userInfo: [ NSLocalizedDescriptionKey: "The 'zip' package is not installed.", - NSFilePathErrorKey: archiverPath + NSFilePathErrorKey: trueArchiverPath ]) } +#endif +#elseif SWT_TARGET_OS_APPLE || os(FreeBSD) + let archiverPath = "/usr/bin/tar" #elseif os(Windows) guard let archiverPath = _archiverPath else { throw CocoaError(.fileWriteUnknown, userInfo: [ @@ -197,20 +203,15 @@ private func _compressContentsOfDirectory(at directoryURL: URL) async throws -> throw CocoaError(.featureUnsupported, userInfo: [NSLocalizedDescriptionKey: "This platform does not support attaching directories to tests."]) #endif - try await withCheckedThrowingContinuation { continuation in - let process = Process() - - process.executableURL = URL(fileURLWithPath: archiverPath, isDirectory: false) - - let sourcePath = directoryURL.fileSystemPath - let destinationPath = temporaryURL.fileSystemPath + let sourcePath = directoryURL.fileSystemPath + let destinationPath = temporaryURL.fileSystemPath + let arguments = { #if os(Linux) || os(OpenBSD) // The zip command constructs relative paths from the current working // directory rather than from command-line arguments. - process.arguments = [destinationPath, "--recurse-paths", "."] - process.currentDirectoryURL = directoryURL + ["-c", #"cd "$0" && "$1" "$2" --recurse-paths ."#, sourcePath, trueArchiverPath, destinationPath] #elseif SWT_TARGET_OS_APPLE || os(FreeBSD) - process.arguments = ["--create", "--auto-compress", "--directory", sourcePath, "--file", destinationPath, "."] + ["--create", "--auto-compress", "--directory", sourcePath, "--file", destinationPath, "."] #elseif os(Windows) // The Windows version of bsdtar can handle relative paths for other archive // formats, but produces empty archives when inferring the zip format with @@ -219,30 +220,15 @@ private func _compressContentsOfDirectory(at directoryURL: URL) async throws -> // An alternative may be to use PowerShell's Compress-Archive command, // however that comes with a security risk as we'd be responsible for two // levels of command-line argument escaping. - process.arguments = ["--create", "--auto-compress", "--file", destinationPath, sourcePath] + ["--create", "--auto-compress", "--file", destinationPath, sourcePath] #endif + }() - process.standardOutput = nil - process.standardError = nil - - process.terminationHandler = { process in - let terminationReason = process.terminationReason - let terminationStatus = process.terminationStatus - if terminationReason == .exit && terminationStatus == EXIT_SUCCESS { - continuation.resume() - } else { - let error = CocoaError(.fileWriteUnknown, userInfo: [ - NSLocalizedDescriptionKey: "The directory at '\(sourcePath)' could not be compressed (\(terminationStatus)).", - ]) - continuation.resume(throwing: error) - } - } - - do { - try process.run() - } catch { - continuation.resume(throwing: error) - } + let exitStatus = try await spawnExecutableAtPathAndWait(archiverPath, arguments: arguments) + guard case .exitCode(EXIT_SUCCESS) = exitStatus else { + throw CocoaError(.fileWriteUnknown, userInfo: [ + NSLocalizedDescriptionKey: "The directory at '\(sourcePath)' could not be compressed (\(exitStatus)).", + ]) } return try Data(contentsOf: temporaryURL, options: [.mappedIfSafe]) diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 647e62dd9..66143a7e0 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -38,7 +38,7 @@ private let _posix_spawn_file_actions_addclosefrom_np = symbol(named: "posix_spa } #endif -/// Spawn a process and wait for it to terminate. +/// Spawn a child process. /// /// - Parameters: /// - executablePath: The path to the executable to spawn. @@ -61,8 +61,7 @@ private let _posix_spawn_file_actions_addclosefrom_np = symbol(named: "posix_spa /// eventually pass this value to ``wait(for:)`` to avoid leaking system /// resources. /// -/// - Throws: Any error that prevented the process from spawning or its exit -/// condition from being read. +/// - Throws: Any error that prevented the process from spawning. func spawnExecutable( atPath executablePath: String, arguments: [String], @@ -83,8 +82,9 @@ func spawnExecutable( #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) return try withUnsafeTemporaryAllocation(of: P.self, capacity: 1) { fileActions in let fileActions = fileActions.baseAddress! - guard 0 == posix_spawn_file_actions_init(fileActions) else { - throw CError(rawValue: swt_errno()) + let fileActionsInitialized = posix_spawn_file_actions_init(fileActions) + guard 0 == fileActionsInitialized else { + throw CError(rawValue: fileActionsInitialized) } defer { _ = posix_spawn_file_actions_destroy(fileActions) @@ -92,8 +92,9 @@ func spawnExecutable( return try withUnsafeTemporaryAllocation(of: P.self, capacity: 1) { attrs in let attrs = attrs.baseAddress! - guard 0 == posix_spawnattr_init(attrs) else { - throw CError(rawValue: swt_errno()) + let attrsInitialized = posix_spawnattr_init(attrs) + guard 0 == attrsInitialized else { + throw CError(rawValue: attrsInitialized) } defer { _ = posix_spawnattr_destroy(attrs) @@ -416,4 +417,29 @@ private func _escapeCommandLine(_ arguments: [String]) -> String { }.joined(separator: " ") } #endif + +/// Spawn a child process and wait for it to terminate. +/// +/// - Parameters: +/// - executablePath: The path to the executable to spawn. +/// - arguments: The arguments to pass to the executable, not including the +/// executable path. +/// - environment: The environment block to pass to the executable. +/// +/// - Returns: The exit status of the spawned process. +/// +/// - Throws: Any error that prevented the process from spawning or its exit +/// condition from being read. +/// +/// This function is a convenience that spawns the given process and waits for +/// it to terminate. It is primarily for use by other targets in this package +/// such as its cross-import overlays. +package func spawnExecutableAtPathAndWait( + _ executablePath: String, + arguments: [String] = [], + environment: [String: String] = [:] +) async throws -> ExitStatus { + let processID = try spawnExecutable(atPath: executablePath, arguments: arguments, environment: environment) + return try await wait(for: processID) +} #endif diff --git a/cmake/modules/shared/CompilerSettings.cmake b/cmake/modules/shared/CompilerSettings.cmake index af8b56dfd..a667f5ba1 100644 --- a/cmake/modules/shared/CompilerSettings.cmake +++ b/cmake/modules/shared/CompilerSettings.cmake @@ -34,6 +34,7 @@ if(CMAKE_SYSTEM_NAME IN_LIST SWT_NO_PROCESS_SPAWNING_LIST) endif() if(NOT APPLE) add_compile_definitions("SWT_NO_SNAPSHOT_TYPES") + add_compile_definitions("SWT_NO_FOUNDATION_FILE_COORDINATION") endif() if(CMAKE_SYSTEM_NAME STREQUAL "WASI") add_compile_definitions("SWT_NO_DYNAMIC_LINKING") From f66ef03052a1e40d93af408ecc0795ff91c06421 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 12 Jun 2025 19:35:45 -0400 Subject: [PATCH 213/234] Add an unreachable marker function. (#1150) This PR adds a wrapper around `__builtin_unreachable()` (`Builtin.unreachable()` when building the Swift standard library) that we can use in place of `fatalError()`. The benefit is that the generated code size for unreachable paths is significantly reduced. For example, given the following function compiled with `-O`: ```swift @available(*, unavailable) func f() { fatalError("Unreachable") } ``` The compiler currently produces: ```asm sub sp, sp, #0x20 stp x29, x30, [sp, #0x10] add x29, sp, #0x10 mov w8, #0x1 ; =1 str w8, [sp, #0x8] mov w8, #0xc ; =12 str x8, [sp] adrp x0, 0 add x0, x0, #0x6a8 ; "Fatal error" adrp x5, 0 add x5, x5, #0x690 ; "UnreachableTest/S.swift" mov x3, #0x6e55 ; =28245 movk x3, #0x6572, lsl #16 movk x3, #0x6361, lsl #32 movk x3, #0x6168, lsl #48 mov x4, #0x6c62 ; =27746 movk x4, #0x65, lsl #16 movk x4, #0xeb00, lsl #48 mov w1, #0xb ; =11 mov w2, #0x2 ; =2 mov w6, #0x17 ; =23 mov w7, #0x2 ; =2 bl 0x100000680 ; symbol stub for: Swift._assertionFailure(_: Swift.StaticString, _: Swift.String, file: Swift.StaticString, line: Swift.UInt, flags: Swift.UInt32) -> Swift.Never brk #0x1 ``` But with this change: ```swift @available(*, unavailable) func f() { swt_unreachable() } ``` It instead compiles to simply: ```asm brk #0x1 ``` ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../ExitTests/ExitTest.CapturedValue.swift | 8 +++++--- .../Testing/ExitTests/ExitTest.Condition.swift | 6 +++--- Sources/Testing/Issues/Confirmation.swift | 8 +++++--- Sources/Testing/Traits/TimeLimitTrait.swift | 16 +++++++++------- Sources/_TestingInternals/include/Stubs.h | 9 +++++++++ Tests/TestingTests/ExitTestTests.swift | 2 +- 6 files changed, 32 insertions(+), 17 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift index 1d5c9b18a..867fdecc5 100644 --- a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift +++ b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift @@ -8,6 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +private import _TestingInternals + @_spi(Experimental) @_spi(ForToolsIntegrationOnly) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") @@ -82,7 +84,7 @@ extension ExitTest { } return nil #else - fatalError("Unsupported") + swt_unreachable() #endif } @@ -101,7 +103,7 @@ extension ExitTest { _kind = .typeOnly(type) } #else - fatalError("Unsupported") + swt_unreachable() #endif } } @@ -119,7 +121,7 @@ extension ExitTest { type } #else - fatalError("Unsupported") + swt_unreachable() #endif } } diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift index 42d929077..0a23bf47c 100644 --- a/Sources/Testing/ExitTests/ExitTest.Condition.swift +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -136,7 +136,7 @@ extension ExitTest.Condition { #if !SWT_NO_EXIT_TESTS Self(.exitCode(exitCode)) #else - fatalError("Unsupported") + swt_unreachable() #endif } @@ -169,7 +169,7 @@ extension ExitTest.Condition { #if !SWT_NO_EXIT_TESTS Self(.signal(signal)) #else - fatalError("Unsupported") + swt_unreachable() #endif } } @@ -192,7 +192,7 @@ extension ExitTest.Condition: CustomStringConvertible { String(describing: exitStatus) } #else - fatalError("Unsupported") + swt_unreachable() #endif } } diff --git a/Sources/Testing/Issues/Confirmation.swift b/Sources/Testing/Issues/Confirmation.swift index 21baec505..33b0ec25a 100644 --- a/Sources/Testing/Issues/Confirmation.swift +++ b/Sources/Testing/Issues/Confirmation.swift @@ -8,6 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +private import _TestingInternals + /// A type that can be used to confirm that an event occurs zero or more times. public struct Confirmation: Sendable { /// The number of times ``confirm(count:)`` has been called. @@ -202,7 +204,7 @@ public func confirmation( sourceLocation: SourceLocation = #_sourceLocation, _ body: (Confirmation) async throws -> R ) async rethrows -> R { - fatalError("Unsupported") + swt_unreachable() } /// An overload of ``confirmation(_:expectedCount:isolation:sourceLocation:_:)-l3il`` @@ -218,7 +220,7 @@ public func confirmation( sourceLocation: SourceLocation = #_sourceLocation, _ body: (Confirmation) async throws -> R ) async rethrows -> R { - fatalError("Unsupported") + swt_unreachable() } /// An overload of ``confirmation(_:expectedCount:isolation:sourceLocation:_:)-l3il`` @@ -234,5 +236,5 @@ public func confirmation( sourceLocation: SourceLocation = #_sourceLocation, _ body: (Confirmation) async throws -> R ) async rethrows -> R { - fatalError("Unsupported") + swt_unreachable() } diff --git a/Sources/Testing/Traits/TimeLimitTrait.swift b/Sources/Testing/Traits/TimeLimitTrait.swift index 54a200fb4..7b21209b6 100644 --- a/Sources/Testing/Traits/TimeLimitTrait.swift +++ b/Sources/Testing/Traits/TimeLimitTrait.swift @@ -8,6 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +private import _TestingInternals + /// A type that defines a time limit to apply to a test. /// /// To add this trait to a test, use ``Trait/timeLimit(_:)-4kzjp``. @@ -126,7 +128,7 @@ extension TimeLimitTrait.Duration { /// This function is unavailable and is provided for diagnostic purposes only. @available(*, unavailable, message: "Time limit must be specified in minutes") public static func seconds(_ seconds: some BinaryInteger) -> Self { - fatalError("Unsupported") + swt_unreachable() } /// Construct a time limit duration given a number of seconds. @@ -134,7 +136,7 @@ extension TimeLimitTrait.Duration { /// This function is unavailable and is provided for diagnostic purposes only. @available(*, unavailable, message: "Time limit must be specified in minutes") public static func seconds(_ seconds: Double) -> Self { - fatalError("Unsupported") + swt_unreachable() } /// Construct a time limit duration given a number of milliseconds. @@ -142,7 +144,7 @@ extension TimeLimitTrait.Duration { /// This function is unavailable and is provided for diagnostic purposes only. @available(*, unavailable, message: "Time limit must be specified in minutes") public static func milliseconds(_ milliseconds: some BinaryInteger) -> Self { - fatalError("Unsupported") + swt_unreachable() } /// Construct a time limit duration given a number of milliseconds. @@ -150,7 +152,7 @@ extension TimeLimitTrait.Duration { /// This function is unavailable and is provided for diagnostic purposes only. @available(*, unavailable, message: "Time limit must be specified in minutes") public static func milliseconds(_ milliseconds: Double) -> Self { - fatalError("Unsupported") + swt_unreachable() } /// Construct a time limit duration given a number of microseconds. @@ -158,7 +160,7 @@ extension TimeLimitTrait.Duration { /// This function is unavailable and is provided for diagnostic purposes only. @available(*, unavailable, message: "Time limit must be specified in minutes") public static func microseconds(_ microseconds: some BinaryInteger) -> Self { - fatalError("Unsupported") + swt_unreachable() } /// Construct a time limit duration given a number of microseconds. @@ -166,7 +168,7 @@ extension TimeLimitTrait.Duration { /// This function is unavailable and is provided for diagnostic purposes only. @available(*, unavailable, message: "Time limit must be specified in minutes") public static func microseconds(_ microseconds: Double) -> Self { - fatalError("Unsupported") + swt_unreachable() } /// Construct a time limit duration given a number of nanoseconds. @@ -174,7 +176,7 @@ extension TimeLimitTrait.Duration { /// This function is unavailable and is provided for diagnostic purposes only. @available(*, unavailable, message: "Time limit must be specified in minutes") public static func nanoseconds(_ nanoseconds: some BinaryInteger) -> Self { - fatalError("Unsupported") + swt_unreachable() } } diff --git a/Sources/_TestingInternals/include/Stubs.h b/Sources/_TestingInternals/include/Stubs.h index 171cca6a5..636ea9aff 100644 --- a/Sources/_TestingInternals/include/Stubs.h +++ b/Sources/_TestingInternals/include/Stubs.h @@ -16,6 +16,15 @@ SWT_ASSUME_NONNULL_BEGIN +/// Mark a code path as unreachable. +/// +/// This function is necessary because Swift does not have an equivalent of +/// `__builtin_unreachable()`. +__attribute__((always_inline, noreturn)) +static inline void swt_unreachable(void) { + __builtin_unreachable(); +} + #if !SWT_NO_FILE_IO /// The C file handle type. /// diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index fb2d47f03..265a5cfaf 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -306,7 +306,7 @@ private import _TestingInternals await Test { try await #require(processExitsWith: .success) {} - fatalError("Unreachable") + Issue.record("#require(processExitsWith:) should have thrown an error") }.run(configuration: configuration) } } From e6061e66eb6557835272a48fabff3febd9803e88 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 13 Jun 2025 11:50:17 -0400 Subject: [PATCH 214/234] Remove `@convention(thin)` from exit test macro declarations. (#1153) `@convention(thin)` is not fully plumbed through the compiler and should not be in our API surface, but also because of the experimental value capturing feature the closures may be thick anyway. See also https://github.com/swiftlang/swift-evolution/pull/2884. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/Expectations/Expectation+Macro.swift | 4 ++-- Sources/Testing/Expectations/ExpectationChecking+Macro.swift | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 973efde53..df9d80058 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -525,7 +525,7 @@ public macro expect( observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, - performing expression: @escaping @Sendable @convention(thin) () async throws -> Void + performing expression: @escaping @Sendable () async throws -> Void ) -> ExitTest.Result? = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") /// Check that an expression causes the process to terminate in a given fashion @@ -572,7 +572,7 @@ public macro require( observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, - performing expression: @escaping @Sendable @convention(thin) () async throws -> Void + performing expression: @escaping @Sendable () async throws -> Void ) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") /// Capture a sendable and codable value to pass to an exit test. diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 6d3093f2a..3a190e679 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1148,7 +1148,7 @@ public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], - performing _: @convention(thin) () -> Void, + performing _: @convention(c) () -> Void, expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, @@ -1181,7 +1181,7 @@ public func __checkClosureCall( encodingCapturedValues capturedValues: (repeat each T), processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], - performing _: @convention(thin) () -> Void, + performing _: @convention(c) () -> Void, expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, From 567285e5c39f98a36cb7ea35f08cfbef6753374f Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 13 Jun 2025 16:38:14 -0400 Subject: [PATCH 215/234] Infer the types of function/closure arguments when captured by an exit test. (#1130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the ability to infer the type of a parameter of a function or closure that encloses an exit test. For example, `x` here: ```swift func f(x: Int) async { await #expect(processExitsWith: .failure) { [x] in ... } } ``` This inference still fails if a parameter is shadowed by a variable with an incompatible type; we still need something like `decltype()` to solve for such cases. We emit a custom diagnostic of the form "🛑 Type of captured value 'x' is ambiguous" if the inferred type of the captured value doesn't match what the compiler thinks it is (see #1146). Still, being able to capture `@Test` function arguments with minimal ceremony is helpful: ```swift @Test(arguments: 0 ..< 100) func f(i: Int) async { await #expect(exitsWith: .failure) { [i] in ... } } ``` Also type inference for literals because "why not?" > [!NOTE] > Exit test value capture remains an experimental feature. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/TestingMacros/ConditionMacro.swift | 77 +++++++++++++++---- .../Support/ClosureCaptureListParsing.swift | 60 ++++++++++++++- .../Support/DiagnosticMessage.swift | 32 ++++++++ .../ConditionMacroTests.swift | 18 +++++ Tests/TestingTests/ExitTestTests.swift | 39 ++++++++++ 5 files changed, 210 insertions(+), 16 deletions(-) diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 37cbc7339..0ef0970e9 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -437,21 +437,23 @@ extension ExitTestConditionMacro { var bodyArgumentExpr = arguments[trailingClosureIndex].expression bodyArgumentExpr = removeParentheses(from: bodyArgumentExpr) ?? bodyArgumentExpr - // Find any captured values and extract them from the trailing closure. - var capturedValues = [CapturedValueInfo]() - if ExitTestExpectMacro.isValueCapturingEnabled { - // The source file imports @_spi(Experimental), so allow value capturing. - if var closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), - let captureList = closureExpr.signature?.capture?.items { - closureExpr.signature?.capture = ClosureCaptureClauseSyntax(items: [], trailingTrivia: .space) - capturedValues = captureList.map { CapturedValueInfo($0, in: context) } - bodyArgumentExpr = ExprSyntax(closureExpr) + // Before building the macro expansion, look for any problems and return + // early if found. + guard _diagnoseIssues(with: macro, body: bodyArgumentExpr, in: context) else { + if Self.isThrowing { + return #"{ () async throws -> Testing.ExitTest.Result in Swift.fatalError("Unreachable") }()"# + } else { + return #"{ () async -> Testing.ExitTest.Result in Swift.fatalError("Unreachable") }()"# } + } - } else if let closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), - let captureClause = closureExpr.signature?.capture, - !captureClause.items.isEmpty { - context.diagnose(.captureClauseUnsupported(captureClause, in: closureExpr, inExitTest: macro)) + // Find any captured values and extract them from the trailing closure. + var capturedValues = [CapturedValueInfo]() + if var closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), + let captureList = closureExpr.signature?.capture?.items { + closureExpr.signature?.capture = ClosureCaptureClauseSyntax(items: [], trailingTrivia: .space) + capturedValues = captureList.map { CapturedValueInfo($0, in: context) } + bodyArgumentExpr = ExprSyntax(closureExpr) } // Generate a unique identifier for this exit test. @@ -611,6 +613,55 @@ extension ExitTestConditionMacro { return ExprSyntax(tupleExpr) } } + + /// Diagnose issues with an exit test macro call. + /// + /// - Parameters: + /// - macro: The exit test macro call. + /// - bodyArgumentExpr: The exit test's body. + /// - context: The macro context in which the expression is being parsed. + /// + /// - Returns: Whether or not macro expansion should continue (i.e. stopping + /// if a fatal error was diagnosed.) + private static func _diagnoseIssues( + with macro: some FreestandingMacroExpansionSyntax, + body bodyArgumentExpr: ExprSyntax, + in context: some MacroExpansionContext + ) -> Bool { + var diagnostics = [DiagnosticMessage]() + + if let closureExpr = bodyArgumentExpr.as(ClosureExprSyntax.self), + let captureClause = closureExpr.signature?.capture, + !captureClause.items.isEmpty { + // Disallow capture lists if the experimental feature is not enabled. + if !ExitTestExpectMacro.isValueCapturingEnabled { + diagnostics.append(.captureClauseUnsupported(captureClause, in: closureExpr, inExitTest: macro)) + } + } + + // Disallow exit tests in generic types and functions as they cannot be + // correctly expanded due to the use of a nested type with static members. + for lexicalContext in context.lexicalContext { + if let lexicalContext = lexicalContext.asProtocol((any WithGenericParametersSyntax).self) { + if let genericClause = lexicalContext.genericParameterClause { + diagnostics.append(.expressionMacroUnsupported(macro, inGenericContextBecauseOf: genericClause, on: lexicalContext)) + } else if let whereClause = lexicalContext.genericWhereClause { + diagnostics.append(.expressionMacroUnsupported(macro, inGenericContextBecauseOf: whereClause, on: lexicalContext)) + } else if let functionDecl = lexicalContext.as(FunctionDeclSyntax.self) { + for parameter in functionDecl.signature.parameterClause.parameters { + if parameter.type.isSome { + diagnostics.append(.expressionMacroUnsupported(macro, inGenericContextBecauseOf: parameter, on: functionDecl)) + } + } + } + } + } + + for diagnostic in diagnostics { + context.diagnose(diagnostic) + } + return diagnostics.isEmpty + } } extension ExitTestExpectMacro { diff --git a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift index a7dca88af..37e696865 100644 --- a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift +++ b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift @@ -52,8 +52,9 @@ struct CapturedValueInfo { // Potentially get the name of the type comprising the current lexical // context (i.e. whatever `Self` is.) + lazy var lexicalContext = context.lexicalContext lazy var typeNameOfLexicalContext = { - let lexicalContext = context.lexicalContext.drop { !$0.isProtocol((any DeclGroupSyntax).self) } + let lexicalContext = lexicalContext.drop { !$0.isProtocol((any DeclGroupSyntax).self) } return context.type(ofLexicalContext: lexicalContext) }() @@ -76,7 +77,19 @@ struct CapturedValueInfo { // Copying self. self.type = typeNameOfLexicalContext } else { - context.diagnose(.typeOfCaptureIsAmbiguous(capture, initializedWith: initializer)) + // Handle literals. Any other types are ambiguous. + switch self.expression.kind { + case .integerLiteralExpr: + self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("IntegerLiteralType"))) + case .floatLiteralExpr: + self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("FloatLiteralType"))) + case .booleanLiteralExpr: + self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("BooleanLiteralType"))) + case .stringLiteralExpr, .simpleStringLiteralExpr: + self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("StringLiteralType"))) + default: + context.diagnose(.typeOfCaptureIsAmbiguous(capture, initializedWith: initializer)) + } } } else if capture.name.tokenKind == .keyword(.self), @@ -84,10 +97,51 @@ struct CapturedValueInfo { // Capturing self. self.expression = "self" self.type = typeNameOfLexicalContext - + } else if let parameterType = Self._findTypeOfParameter(named: capture.name, in: lexicalContext) { + self.expression = ExprSyntax(DeclReferenceExprSyntax(baseName: capture.name.trimmed)) + self.type = parameterType } else { // Not enough contextual information to derive the type here. context.diagnose(.typeOfCaptureIsAmbiguous(capture)) } } + + /// Find a function or closure parameter in the given lexical context with a + /// given name and return its type. + /// + /// - Parameters: + /// - parameterName: The name of the parameter of interest. + /// - lexicalContext: The lexical context to examine. + /// + /// - Returns: The Swift type of first parameter found whose name matches, or + /// `nil` if none was found. The lexical context is searched in the order + /// provided which, by default, starts with the innermost scope. + private static func _findTypeOfParameter(named parameterName: TokenSyntax, in lexicalContext: [Syntax]) -> TypeSyntax? { + for lexicalContext in lexicalContext { + var parameterType: TypeSyntax? + if let functionDecl = lexicalContext.as(FunctionDeclSyntax.self) { + parameterType = functionDecl.signature.parameterClause.parameters + .first { ($0.secondName ?? $0.firstName).tokenKind == parameterName.tokenKind } + .map(\.type) + } else if let closureExpr = lexicalContext.as(ClosureExprSyntax.self) { + if case let .parameterClause(parameterClause) = closureExpr.signature?.parameterClause { + parameterType = parameterClause.parameters + .first { ($0.secondName ?? $0.firstName).tokenKind == parameterName.tokenKind } + .flatMap(\.type) + } + } else if lexicalContext.is(DeclSyntax.self) { + // If we've reached any other enclosing declaration, then any parameters + // beyond it won't be capturable and thus it isn't possible to infer + // types from them (any capture of `x`, for instance, must refer to some + // more-local variable with that name, not to a parameter named `x`.) + return nil + } + + if let parameterType { + return parameterType + } + } + + return nil + } } diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index 3a8957207..b7103bcc6 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -888,4 +888,36 @@ extension DiagnosticMessage { ] ) } + + /// Create a diagnostic message stating that an expression macro is not + /// supported in a generic context. + /// + /// - Parameters: + /// - macro: The invalid macro. + /// - genericClause: The child node on `genericDecl` that makes it generic. + /// - genericDecl: The generic declaration to which `genericClause` is + /// attached, possibly equal to `decl`. + /// + /// - Returns: A diagnostic message. + static func expressionMacroUnsupported(_ macro: some FreestandingMacroExpansionSyntax, inGenericContextBecauseOf genericClause: some SyntaxProtocol, on genericDecl: some SyntaxProtocol) -> Self { + if let functionDecl = genericDecl.as(FunctionDeclSyntax.self) { + return Self( + syntax: Syntax(macro), + message: "Cannot call macro '\(_macroName(macro))' within generic function '\(functionDecl.completeName)'", + severity: .error + ) + } else if let namedDecl = genericDecl.asProtocol((any NamedDeclSyntax).self) { + return Self( + syntax: Syntax(macro), + message: "Cannot call macro '\(_macroName(macro))' within generic \(_kindString(for: genericDecl)) '\(namedDecl.name.trimmed)'", + severity: .error + ) + } else { + return Self( + syntax: Syntax(macro), + message: "Cannot call macro '\(_macroName(macro))' within a generic \(_kindString(for: genericDecl))", + severity: .error + ) + } + } } diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index dc36af7cd..e5d8f05cb 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -436,6 +436,22 @@ struct ConditionMacroTests { #expect(diagnostic.message.contains("is redundant")) } + @Test("#expect(processExitsWith:) diagnostics", + arguments: [ + "func f() { #expectExitTest(processExitsWith: x) {} }": + "Cannot call macro ''#expectExitTest(processExitsWith:_:)'' within generic function 'f()'", + ] + ) + func exitTestDiagnostics(input: String, expectedMessage: String) throws { + let (_, diagnostics) = try parse(input) + + #expect(diagnostics.count > 0) + for diagnostic in diagnostics { + #expect(diagnostic.diagMessage.severity == .error) + #expect(diagnostic.message == expectedMessage) + } + } + #if ExperimentalExitTestValueCapture @Test("#expect(processExitsWith:) produces a diagnostic for a bad capture", arguments: [ @@ -445,6 +461,8 @@ struct ConditionMacroTests { "Type of captured value 'a' is ambiguous", "#expectExitTest(processExitsWith: x) { [a = b] in }": "Type of captured value 'a' is ambiguous", + "struct S { func f() { #expectExitTest(processExitsWith: x) { [a] in } } }": + "Cannot call macro ''#expectExitTest(processExitsWith:_:)'' within generic structure 'S'", ] ) func exitTestCaptureDiagnostics(input: String, expectedMessage: String) throws { diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 265a5cfaf..16f7b0fec 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -457,6 +457,45 @@ private import _TestingInternals } } + @Test("Capturing a parameter to the test function") + func captureListWithParameter() async { + let i = Int.random(in: 0 ..< 1000) + + func f(j: Int) async { + await #expect(processExitsWith: .success) { [i = i as Int, j] in + #expect(i == j) + #expect(j >= 0) + #expect(j < 1000) + } + } + await f(j: i) + + await { (j: Int) in + _ = await #expect(processExitsWith: .success) { [i = i as Int, j] in + #expect(i == j) + #expect(j >= 0) + #expect(j < 1000) + } + }(i) + + // FAILS TO COMPILE: shadowing `i` with a variable of a different type will + // prevent correct expansion (we need an equivalent of decltype() for that.) +// let i = String(i) +// await #expect(processExitsWith: .success) { [i] in +// #expect(!i.isEmpty) +// } + } + + @Test("Capturing a literal expression") + func captureListWithLiterals() async { + await #expect(processExitsWith: .success) { [i = 0, f = 1.0, s = "", b = true] in + #expect(i == 0) + #expect(f == 1.0) + #expect(s == "") + #expect(b == true) + } + } + @Test("Capturing #_sourceLocation") func captureListPreservesSourceLocationMacro() async { func sl(_ sl: SourceLocation = #_sourceLocation) -> SourceLocation { From 8471b1f578bd0c5ca642d194e6422f850dc3c892 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 16 Jun 2025 12:42:29 -0400 Subject: [PATCH 216/234] Fix some typos in documentation. (#1156) This PR fixes some typos/minor errors in our documentation. Resolves #1118. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- Sources/Testing/ExitTests/ExitTest.Result.swift | 14 +++++++------- Sources/Testing/Testing.docc/exit-testing.md | 2 +- Sources/_TestDiscovery/TestContentRecord.swift | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift index f2c57e205..a427d4005 100644 --- a/Sources/Testing/ExitTests/ExitTest.Result.swift +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -43,9 +43,9 @@ extension ExitTest { /// /// When checking the value of this property, keep in mind that the standard /// output stream is globally accessible, and any code running in an exit - /// test may write to it including including the operating system and any - /// third-party dependencies you have declared in your package. Rather than - /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// test may write to it including the operating system and any third-party + /// dependencies you have declared in your package. Rather than comparing + /// the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) /// to check if expected output is present. /// @@ -73,10 +73,10 @@ extension ExitTest { /// instead. /// /// When checking the value of this property, keep in mind that the standard - /// error stream is globally accessible, and any code running in an exit - /// test may write to it including including the operating system and any - /// third-party dependencies you have declared in your package. Rather than - /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// output stream is globally accessible, and any code running in an exit + /// test may write to it including the operating system and any third-party + /// dependencies you have declared in your package. Rather than comparing + /// the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) /// to check if expected output is present. /// diff --git a/Sources/Testing/Testing.docc/exit-testing.md b/Sources/Testing/Testing.docc/exit-testing.md index bb4fccafd..6ae81b980 100644 --- a/Sources/Testing/Testing.docc/exit-testing.md +++ b/Sources/Testing/Testing.docc/exit-testing.md @@ -153,4 +153,4 @@ extension Customer { The testing library always sets ``ExitTest/Result/exitStatus`` to the actual exit status of the child process (as reported by the system) even if you do not -pass it. +observe `\.exitStatus`. diff --git a/Sources/_TestDiscovery/TestContentRecord.swift b/Sources/_TestDiscovery/TestContentRecord.swift index df868c975..384113c1b 100644 --- a/Sources/_TestDiscovery/TestContentRecord.swift +++ b/Sources/_TestDiscovery/TestContentRecord.swift @@ -71,8 +71,8 @@ public struct TestContentRecord where T: DiscoverableAsTestContent { /// /// | Platform | Pointer Type | /// |-|-| - /// | macOS, iOS, watchOS, tvOS, visionOS | `UnsafePointer` | - /// | Linux, FreeBSD, Android | `UnsafePointer` | + /// | macOS, iOS, watchOS, tvOS, visionOS | `UnsafePointer` | + /// | Linux, FreeBSD, Android | `UnsafePointer` | /// | OpenBSD | `UnsafePointer` | /// | Windows | `HMODULE` | /// From 9015405745aa106179e5f8472691ae0e3b71b206 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 16 Jun 2025 15:35:05 -0400 Subject: [PATCH 217/234] Revert "Update outdated Suite macro documentation (#1142)" This reverts commit 25b61ef4667a6d1dda30cb3420336621886529c4. --- Sources/Testing/Test+Macro.swift | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index f2fc415d6..44e9d3d72 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -53,8 +53,9 @@ public typealias __XCTestCompatibleSelector = Never /// - Parameters: /// - traits: Zero or more traits to apply to this test suite. /// -/// A test suite is a type that contains one or more test functions. -/// Any type may be a test suite. +/// A test suite is a type that contains one or more test functions. Any +/// copyable type (that is, any type that is not marked `~Copyable`) may be a +/// test suite. /// /// The use of the `@Suite` attribute is optional; types are recognized as test /// suites even if they do not have the `@Suite` attribute applied to them. @@ -80,8 +81,9 @@ public macro Suite( /// from the associated type's name. /// - traits: Zero or more traits to apply to this test suite. /// -/// A test suite is a type that contains one or more test functions. -/// Any type may be a test suite. +/// A test suite is a type that contains one or more test functions. Any +/// copyable type (that is, any type that is not marked `~Copyable`) may be a +/// test suite. /// /// The use of the `@Suite` attribute is optional; types are recognized as test /// suites even if they do not have the `@Suite` attribute applied to them. From 7c672abd0580d4fa4cb9a90acf5a95e10a703bb8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 16 Jun 2025 15:37:26 -0400 Subject: [PATCH 218/234] Replace ~Copyable with ~Escapable in the relevant documentation about Suite types. --- Sources/Testing/Test+Macro.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Test+Macro.swift b/Sources/Testing/Test+Macro.swift index 44e9d3d72..a8d9eaf51 100644 --- a/Sources/Testing/Test+Macro.swift +++ b/Sources/Testing/Test+Macro.swift @@ -54,7 +54,7 @@ public typealias __XCTestCompatibleSelector = Never /// - traits: Zero or more traits to apply to this test suite. /// /// A test suite is a type that contains one or more test functions. Any -/// copyable type (that is, any type that is not marked `~Copyable`) may be a +/// escapable type (that is, any type that is not marked `~Escapable`) may be a /// test suite. /// /// The use of the `@Suite` attribute is optional; types are recognized as test @@ -82,7 +82,7 @@ public macro Suite( /// - traits: Zero or more traits to apply to this test suite. /// /// A test suite is a type that contains one or more test functions. Any -/// copyable type (that is, any type that is not marked `~Copyable`) may be a +/// escapable type (that is, any type that is not marked `~Escapable`) may be a /// test suite. /// /// The use of the `@Suite` attribute is optional; types are recognized as test From eeeffd49695b44aff6c908c9e463f48cc7739250 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 18 Jun 2025 13:54:51 -0400 Subject: [PATCH 219/234] Suppress the complex expansion of expectations when we see effects in the lexical context. (#1161) This PR changes the behaviour of the testing library for expressions such as: ```swift try #expect(a == b) ``` Currently, we don't expand that expression correctly because we can't tell where the `try` keyword should be applied. It sometimes expands and sometimes doesn't. This PR detects the presence of those keywords (with a recent-enough toolchain) and, if found, disables the fancy expansion in favour of a simpler one that is less likely to fail to compile. More thorough support for effectful expressions in expectations is tracked by #840 which involves fully refactoring the implementation of the `#expect()` macro. See also #162 for some more context. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Support/ConditionArgumentParsing.swift | 6 +- .../Support/EffectfulExpressionHandling.swift | 121 ++++++++++++------ 2 files changed, 87 insertions(+), 40 deletions(-) diff --git a/Sources/TestingMacros/Support/ConditionArgumentParsing.swift b/Sources/TestingMacros/Support/ConditionArgumentParsing.swift index e0ccda9a7..254f3d0aa 100644 --- a/Sources/TestingMacros/Support/ConditionArgumentParsing.swift +++ b/Sources/TestingMacros/Support/ConditionArgumentParsing.swift @@ -517,10 +517,8 @@ private func _parseCondition(from expr: ExprSyntax, for macro: some Freestanding /// - Returns: An instance of ``Condition`` describing `expr`. func parseCondition(from expr: ExprSyntax, for macro: some FreestandingMacroExpansionSyntax, in context: some MacroExpansionContext) -> Condition { // If the condition involves the `unsafe`, `try`, or `await` keywords, assume - // we cannot expand it. This check cannot handle expressions like - // `try #expect(a.b(c))` where `b()` is throwing because the `try` keyword is - // outside the macro expansion. SEE: rdar://109470248 - let effectKeywordsToApply = findEffectKeywords(in: expr, context: context) + // we cannot expand it. + let effectKeywordsToApply = findEffectKeywords(in: expr).union(findEffectKeywords(in: context)) guard effectKeywordsToApply.intersection([.unsafe, .try, .await]).isEmpty else { return Condition(expression: expr) } diff --git a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift index a0b84e737..b093d1e77 100644 --- a/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift +++ b/Sources/TestingMacros/Support/EffectfulExpressionHandling.swift @@ -14,6 +14,58 @@ import SwiftSyntaxMacros // MARK: - Finding effect keywords and expressions +/// Get the effect keyword corresponding to a given syntax node, if any. +/// +/// - Parameters: +/// - expr: The syntax node that may represent an effectful expression. +/// +/// - Returns: The effect keyword corresponding to `expr`, if any. +private func _effectKeyword(for expr: ExprSyntax) -> Keyword? { + switch expr.kind { + case .tryExpr: + return .try + case .awaitExpr: + return .await + case .consumeExpr: + return .consume + case .borrowExpr: + return .borrow + case .unsafeExpr: + return .unsafe + default: + return nil + } +} + +/// Determine how to descend further into a syntax node tree from a given node. +/// +/// - Parameters: +/// - node: The syntax node currently being walked. +/// +/// - Returns: Whether or not to descend into `node` and visit its children. +private func _continueKind(for node: Syntax) -> SyntaxVisitorContinueKind { + switch node.kind { + case .tryExpr, .awaitExpr, .consumeExpr, .borrowExpr, .unsafeExpr: + // If this node represents an effectful expression, look inside it for + // additional such expressions. + return .visitChildren + case .closureExpr, .functionDecl: + // Do not delve into closures or function declarations. + return .skipChildren + case .variableDecl: + // Delve into variable declarations. + return .visitChildren + default: + // Do not delve into declarations other than variables. + if node.isProtocol((any DeclSyntaxProtocol).self) { + return .skipChildren + } + } + + // Recurse into everything else. + return .visitChildren +} + /// A syntax visitor class that looks for effectful keywords in a given /// expression. private final class _EffectFinder: SyntaxAnyVisitor { @@ -21,32 +73,11 @@ private final class _EffectFinder: SyntaxAnyVisitor { var effectKeywords: Set = [] override func visitAny(_ node: Syntax) -> SyntaxVisitorContinueKind { - switch node.kind { - case .tryExpr: - effectKeywords.insert(.try) - case .awaitExpr: - effectKeywords.insert(.await) - case .consumeExpr: - effectKeywords.insert(.consume) - case .borrowExpr: - effectKeywords.insert(.borrow) - case .unsafeExpr: - effectKeywords.insert(.unsafe) - case .closureExpr, .functionDecl: - // Do not delve into closures or function declarations. - return .skipChildren - case .variableDecl: - // Delve into variable declarations. - return .visitChildren - default: - // Do not delve into declarations other than variables. - if node.isProtocol((any DeclSyntaxProtocol).self) { - return .skipChildren - } + if let expr = node.as(ExprSyntax.self), let keyword = _effectKeyword(for: expr) { + effectKeywords.insert(keyword) } - // Recurse into everything else. - return .visitChildren + return _continueKind(for: node) } } @@ -54,7 +85,6 @@ private final class _EffectFinder: SyntaxAnyVisitor { /// /// - Parameters: /// - node: The node to inspect. -/// - context: The macro context in which the expression is being parsed. /// /// - Returns: A set of effectful keywords such as `await` that are present in /// `node`. @@ -62,13 +92,27 @@ private final class _EffectFinder: SyntaxAnyVisitor { /// This function does not descend into function declarations or closure /// expressions because they represent distinct lexical contexts and their /// effects are uninteresting in the context of `node` unless they are called. -func findEffectKeywords(in node: some SyntaxProtocol, context: some MacroExpansionContext) -> Set { - // TODO: gather any effects from the lexical context once swift-syntax-#3037 and related PRs land +func findEffectKeywords(in node: some SyntaxProtocol) -> Set { let effectFinder = _EffectFinder(viewMode: .sourceAccurate) effectFinder.walk(node) return effectFinder.effectKeywords } +/// Find effectful keywords in a macro's lexical context. +/// +/// - Parameters: +/// - context: The macro context in which the expression is being parsed. +/// +/// - Returns: A set of effectful keywords such as `await` that are present in +/// `context` and would apply to an expression macro during its expansion. +func findEffectKeywords(in context: some MacroExpansionContext) -> Set { + let result = context.lexicalContext.reversed().lazy + .prefix { _continueKind(for: $0) == .visitChildren } + .compactMap { $0.as(ExprSyntax.self) } + .compactMap(_effectKeyword(for:)) + return Set(result) +} + extension BidirectionalCollection { /// The suffix of syntax nodes in this collection which are effectful /// expressions, such as those for `try` or `await`. @@ -128,10 +172,13 @@ private func _makeCallToEffectfulThunk(_ thunkName: TokenSyntax, passing expr: s /// - Parameters: /// - effectfulKeywords: The effectful keywords to apply. /// - expr: The expression to apply the keywords and thunk functions to. +/// - insertThunkCalls: Whether or not to also insert calls to thunks to +/// ensure the inserted keywords do not generate warnings. If you aren't +/// sure whether thunk calls are needed, pass `true`. /// /// - Returns: A copy of `expr` if no changes are needed, or an expression that /// adds the keywords in `effectfulKeywords` to `expr`. -func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some ExprSyntaxProtocol) -> ExprSyntax { +func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some ExprSyntaxProtocol, insertThunkCalls: Bool = true) -> ExprSyntax { let originalExpr = expr var expr = ExprSyntax(expr.trimmed) @@ -141,14 +188,16 @@ func applyEffectfulKeywords(_ effectfulKeywords: Set, to expr: some Exp let needUnsafe = isUnsafeKeywordSupported && effectfulKeywords.contains(.unsafe) && !expr.is(UnsafeExprSyntax.self) // First, add thunk function calls. - if needAwait { - expr = _makeCallToEffectfulThunk(.identifier("__requiringAwait"), passing: expr) - } - if needTry { - expr = _makeCallToEffectfulThunk(.identifier("__requiringTry"), passing: expr) - } - if needUnsafe { - expr = _makeCallToEffectfulThunk(.identifier("__requiringUnsafe"), passing: expr) + if insertThunkCalls { + if needAwait { + expr = _makeCallToEffectfulThunk(.identifier("__requiringAwait"), passing: expr) + } + if needTry { + expr = _makeCallToEffectfulThunk(.identifier("__requiringTry"), passing: expr) + } + if needUnsafe { + expr = _makeCallToEffectfulThunk(.identifier("__requiringUnsafe"), passing: expr) + } } // Then add keyword expressions. (We do this separately so we end up writing From 66701fb9370a7ee5554b3a8f21067869616c2099 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 18 Jun 2025 14:00:39 -0400 Subject: [PATCH 220/234] Improve type inference for exit test value captures. (#1163) This PR refactors the (new) type inference logic for exit test capture lists to use a syntax visitor, which allows for the types of more complex expressions to be inferred. For example, previously the type of this capture would not be inferred: ```swift [x = try await f() as Int] ``` Even though the type (`Int`) is clearly present, because the `AsExprSyntax` is nested in an `AwaitExprSyntax` and then a `TryExprSyntax`. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../TypeSyntaxProtocolAdditions.swift | 7 + .../Support/ClosureCaptureListParsing.swift | 203 ++++++++++++++---- .../ConditionMacroTests.swift | 4 + Tests/TestingTests/ExitTestTests.swift | 38 +++- 4 files changed, 207 insertions(+), 45 deletions(-) diff --git a/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift b/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift index e1bd346ed..e9bf03a02 100644 --- a/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/TypeSyntaxProtocolAdditions.swift @@ -36,6 +36,13 @@ extension TypeSyntaxProtocol { .contains(.keyword(.some)) } + /// Whether or not this type is `any T` or a type derived from such a type. + var isAny: Bool { + tokens(viewMode: .fixedUp).lazy + .map(\.tokenKind) + .contains(.keyword(.any)) + } + /// Check whether or not this type is named with the specified name and /// module. /// diff --git a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift index 37e696865..0c36fbbf7 100644 --- a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift +++ b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift @@ -50,62 +50,177 @@ struct CapturedValueInfo { return } - // Potentially get the name of the type comprising the current lexical - // context (i.e. whatever `Self` is.) - lazy var lexicalContext = context.lexicalContext - lazy var typeNameOfLexicalContext = { - let lexicalContext = lexicalContext.drop { !$0.isProtocol((any DeclGroupSyntax).self) } - return context.type(ofLexicalContext: lexicalContext) - }() + if let (expr, type) = Self._inferExpressionAndType(of: capture, in: context) { + self.expression = expr + self.type = type + } else { + // Not enough contextual information to derive the type here. + context.diagnose(.typeOfCaptureIsAmbiguous(capture)) + } + } + /// Infer the captured expression and the type of a closure capture list item. + /// + /// - Parameters: + /// - capture: The closure capture list item to inspect. + /// - context: The macro context in which the expression is being parsed. + /// + /// - Returns: A tuple containing the expression and type of `capture`, or + /// `nil` if they could not be inferred. + private static func _inferExpressionAndType(of capture: ClosureCaptureSyntax, in context: some MacroExpansionContext) -> (ExprSyntax, TypeSyntax)? { if let initializer = capture.initializer { // Found an initializer clause. Extract the expression it captures. - self.expression = removeParentheses(from: initializer.value) ?? initializer.value + let finder = _ExprTypeFinder(in: context) + finder.walk(initializer.value) + if let inferredType = finder.inferredType { + return (initializer.value, inferredType) + } + } else if capture.name.tokenKind == .keyword(.self), + let typeNameOfLexicalContext = Self._inferSelf(from: context) { + // Capturing self. + return (ExprSyntax(DeclReferenceExprSyntax(baseName: .keyword(.self))), typeNameOfLexicalContext) + } else if let parameterType = Self._findTypeOfParameter(named: capture.name, in: context.lexicalContext) { + return (ExprSyntax(DeclReferenceExprSyntax(baseName: capture.name.trimmed)), parameterType) + } + + return nil + } + + private final class _ExprTypeFinder: SyntaxAnyVisitor where C: MacroExpansionContext { + var context: C + + /// The type that was inferred from the visited syntax tree, if any. + /// + /// This type has not been fixed up yet. Use ``inferredType`` for the final + /// derived type. + private var _inferredType: TypeSyntax? + + /// Whether or not the inferred type has been made optional by e.g. `try?`. + private var _needsOptionalApplied = false + + /// The type that was inferred from the visited syntax tree, if any. + var inferredType: TypeSyntax? { + _inferredType.flatMap { inferredType in + if inferredType.isSome || inferredType.isAny { + // `some` and `any` types are not concrete and cannot be inferred. + nil + } else if _needsOptionalApplied { + TypeSyntax(OptionalTypeSyntax(wrappedType: inferredType.trimmed)) + } else { + inferredType + } + } + } + + init(in context: C) { + self.context = context + super.init(viewMode: .sourceAccurate) + } + + override func visitAny(_ node: Syntax) -> SyntaxVisitorContinueKind { + if inferredType != nil { + // Another part of the syntax tree has already provided a type. Stop. + return .skipChildren + } - // Find the 'as' clause so we can determine the type of the captured value. - if let asExpr = self.expression.as(AsExprSyntax.self) { - self.type = if asExpr.questionOrExclamationMark?.tokenKind == .postfixQuestionMark { + switch node.kind { + case .asExpr: + let asExpr = node.cast(AsExprSyntax.self) + if let type = asExpr.type.as(IdentifierTypeSyntax.self), type.name.tokenKind == .keyword(.Self) { + // `Self` should resolve to the lexical context's type. + _inferredType = CapturedValueInfo._inferSelf(from: context) + } else if asExpr.questionOrExclamationMark?.tokenKind == .postfixQuestionMark { // If the caller is using as?, make the type optional. - TypeSyntax(OptionalTypeSyntax(wrappedType: asExpr.type.trimmed)) + _inferredType = TypeSyntax(OptionalTypeSyntax(wrappedType: asExpr.type.trimmed)) } else { - asExpr.type + _inferredType = asExpr.type } - } else if let selfExpr = self.expression.as(DeclReferenceExprSyntax.self), - selfExpr.baseName.tokenKind == .keyword(.self), - selfExpr.argumentNames == nil, - let typeNameOfLexicalContext { - // Copying self. - self.type = typeNameOfLexicalContext - } else { - // Handle literals. Any other types are ambiguous. - switch self.expression.kind { - case .integerLiteralExpr: - self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("IntegerLiteralType"))) - case .floatLiteralExpr: - self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("FloatLiteralType"))) - case .booleanLiteralExpr: - self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("BooleanLiteralType"))) - case .stringLiteralExpr, .simpleStringLiteralExpr: - self.type = TypeSyntax(IdentifierTypeSyntax(name: .identifier("StringLiteralType"))) - default: - context.diagnose(.typeOfCaptureIsAmbiguous(capture, initializedWith: initializer)) + return .skipChildren + + case .awaitExpr, .unsafeExpr: + // These effect keywords do not affect the type of the expression. + return .visitChildren + + case .tryExpr: + let tryExpr = node.cast(TryExprSyntax.self) + if tryExpr.questionOrExclamationMark?.tokenKind == .postfixQuestionMark { + // The resulting type from the inner expression will be optionalized. + _needsOptionalApplied = true } - } + return .visitChildren - } else if capture.name.tokenKind == .keyword(.self), - let typeNameOfLexicalContext { - // Capturing self. - self.expression = "self" - self.type = typeNameOfLexicalContext - } else if let parameterType = Self._findTypeOfParameter(named: capture.name, in: lexicalContext) { - self.expression = ExprSyntax(DeclReferenceExprSyntax(baseName: capture.name.trimmed)) - self.type = parameterType - } else { - // Not enough contextual information to derive the type here. - context.diagnose(.typeOfCaptureIsAmbiguous(capture)) + case .tupleExpr: + // If the tuple contains exactly one element, it's just parentheses + // around that expression. + let tupleExpr = node.cast(TupleExprSyntax.self) + if tupleExpr.elements.count == 1 { + return .visitChildren + } + + // Otherwise, we need to try to compose the type as a tuple type from + // the types of all elements in the tuple expression. Note that tuples + // do not conform to Sendable or Codable, so our current use of this + // code in exit tests will still diagnose an error, but the error ("must + // conform") will be more useful than "couldn't infer". + let elements = tupleExpr.elements.compactMap { element in + let finder = Self(in: context) + finder.walk(element.expression) + return finder.inferredType.map { type in + TupleTypeElementSyntax(firstName: element.label?.trimmed, type: type.trimmed) + } + } + if elements.count == tupleExpr.elements.count { + _inferredType = TypeSyntax( + TupleTypeSyntax(elements: TupleTypeElementListSyntax { elements }) + ) + } + return .skipChildren + + case .declReferenceExpr: + // If the reference is to `self` without any arguments, its type can be + // inferred from the lexical context. + let expr = node.cast(DeclReferenceExprSyntax.self) + if expr.baseName.tokenKind == .keyword(.self), expr.argumentNames == nil { + _inferredType = CapturedValueInfo._inferSelf(from: context) + } + return .skipChildren + + case .integerLiteralExpr: + _inferredType = TypeSyntax(IdentifierTypeSyntax(name: .identifier("IntegerLiteralType"))) + return .skipChildren + + case .floatLiteralExpr: + _inferredType = TypeSyntax(IdentifierTypeSyntax(name: .identifier("FloatLiteralType"))) + return .skipChildren + + case .booleanLiteralExpr: + _inferredType = TypeSyntax(IdentifierTypeSyntax(name: .identifier("BooleanLiteralType"))) + return .skipChildren + + case .stringLiteralExpr, .simpleStringLiteralExpr: + _inferredType = TypeSyntax(IdentifierTypeSyntax(name: .identifier("StringLiteralType"))) + return .skipChildren + + default: + // We don't know how to infer a type from this syntax node, so do not + // proceed further. + return .skipChildren + } } } + /// Get the type of `self` inferred from the given context. + /// + /// - Parameters: + /// - context: The macro context in which the expression is being parsed. + /// + /// - Returns: The type in `lexicalContext` corresponding to `Self`, or `nil` + /// if it could not be determined. + private static func _inferSelf(from context: some MacroExpansionContext) -> TypeSyntax? { + let lexicalContext = context.lexicalContext.drop { !$0.isProtocol((any DeclGroupSyntax).self) } + return context.type(ofLexicalContext: lexicalContext) + } + /// Find a function or closure parameter in the given lexical context with a /// given name and return its type. /// diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index e5d8f05cb..2c3cc38d8 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -461,6 +461,10 @@ struct ConditionMacroTests { "Type of captured value 'a' is ambiguous", "#expectExitTest(processExitsWith: x) { [a = b] in }": "Type of captured value 'a' is ambiguous", + "#expectExitTest(processExitsWith: x) { [a = b as any T] in }": + "Type of captured value 'a' is ambiguous", + "#expectExitTest(processExitsWith: x) { [a = b as some T] in }": + "Type of captured value 'a' is ambiguous", "struct S { func f() { #expectExitTest(processExitsWith: x) { [a] in } } }": "Cannot call macro ''#expectExitTest(processExitsWith:_:)'' within generic structure 'S'", ] diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 16f7b0fec..89aae3894 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -407,9 +407,10 @@ private import _TestingInternals @Test("self in capture list") func captureListWithSelf() async { - await #expect(processExitsWith: .success) { [self, x = self] in + await #expect(processExitsWith: .success) { [self, x = self, y = self as Self] in #expect(self.property == 456) #expect(x.property == 456) + #expect(y.property == 456) } } } @@ -506,6 +507,41 @@ private import _TestingInternals } } + @Test("Capturing an optional value") + func captureListWithOptionalValue() async throws { + await #expect(processExitsWith: .success) { [x = nil as Int?] in + #expect(x != 1) + } + await #expect(processExitsWith: .success) { [x = (0 as Any) as? String] in + #expect(x == nil) + } + } + + @Test("Capturing an effectful expression") + func captureListWithEffectfulExpression() async throws { + func f() async throws -> Int { 0 } + try await #require(processExitsWith: .success) { [f = try await f() as Int] in + #expect(f == 0) + } + try await #expect(processExitsWith: .success) { [f = f() as Int] in + #expect(f == 0) + } + } + +#if false // intentionally fails to compile + @Test("Capturing a tuple") + func captureListWithTuple() async throws { + // A tuple whose elements conform to Codable does not itself conform to + // Codable, so we cannot actually express this capture list in a way that + // works with #expect(). + await #expect(processExitsWith: .success) { [x = (0 as Int, 1 as Double, "2" as String)] in + #expect(x.0 == 0) + #expect(x.1 == 1) + #expect(x.2 == "2") + } + } +#endif + #if false // intentionally fails to compile struct NonCodableValue {} From e63d542c824859916234e2a1307697056e1712d1 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 18 Jun 2025 14:55:22 -0400 Subject: [PATCH 221/234] Diagnose when we incorrectly infer the type of a capture list item in an exit test. (#1152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #1130, split out for clarity. This PR adds a custom diagnostic at compile time if we incorrectly infer the type of a captured function argument or `self` in an exit test. For example: ```swift func f(_ x: Int) async { let x = String(x) // local type of 'x' is String, not Int await #expect(processExitsWith: ...) { [x] in ... } } ``` This improves our feedback to the developer when we encounter a pattern like that. The developer will now see: > 🛑 Type of captured value 'x' is ambiguous ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. --- .../Expectations/Expectation+Macro.swift | 29 +++++++++++++++++-- .../ExitTestCapturedValueMacro.swift | 24 +++++++++++++++ .../Support/ClosureCaptureListParsing.swift | 2 +- Sources/TestingMacros/TestingMacrosMain.swift | 1 + Tests/TestingTests/ExitTestTests.swift | 12 +++++--- 5 files changed, 61 insertions(+), 7 deletions(-) diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index df9d80058..8a6749a31 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -580,6 +580,7 @@ public macro require( /// - Parameters: /// - value: The captured value. /// - name: The name of the capture list item corresponding to `value`. +/// - expectedType: The type of `value`. /// /// - Returns: `value` verbatim. /// @@ -588,7 +589,8 @@ public macro require( @freestanding(expression) public macro __capturedValue( _ value: T, - _ name: String + _ name: String, + _ expectedType: T.Type ) -> T = #externalMacro(module: "TestingMacros", type: "ExitTestCapturedValueMacro") where T: Sendable & Codable /// Emit a compile-time diagnostic when an unsupported value is captured by an @@ -597,6 +599,7 @@ public macro __capturedValue( /// - Parameters: /// - value: The captured value. /// - name: The name of the capture list item corresponding to `value`. +/// - expectedType: The type of `value`. /// /// - Returns: The result of a call to `fatalError()`. `value` is discarded at /// compile time. @@ -606,5 +609,27 @@ public macro __capturedValue( @freestanding(expression) public macro __capturedValue( _ value: borrowing T, - _ name: String + _ name: String, + _ expectedType: T.Type ) -> Never = #externalMacro(module: "TestingMacros", type: "ExitTestBadCapturedValueMacro") where T: ~Copyable & ~Escapable + +/// Emit a compile-time diagnostic when a value is captured by an exit test but +/// we inferred the wrong type. +/// +/// - Parameters: +/// - value: The captured value. +/// - name: The name of the capture list item corresponding to `value`. +/// - expectedType: The _expected_ type of `value`, which will differ from the +/// _actual_ type of `value`. +/// +/// - Returns: The result of a call to `fatalError()`. `value` is discarded at +/// compile time. +/// +/// - Warning: This macro is used to implement the `#expect(processExitsWith:)` +/// macro. Do not use it directly. +@freestanding(expression) +public macro __capturedValue( + _ value: borrowing T, + _ name: String, + _ expectedType: U.Type +) -> T = #externalMacro(module: "TestingMacros", type: "ExitTestIncorrectlyCapturedValueMacro") where T: ~Copyable & ~Escapable, U: ~Copyable & ~Escapable diff --git a/Sources/TestingMacros/ExitTestCapturedValueMacro.swift b/Sources/TestingMacros/ExitTestCapturedValueMacro.swift index 0038dac7c..4bba47c79 100644 --- a/Sources/TestingMacros/ExitTestCapturedValueMacro.swift +++ b/Sources/TestingMacros/ExitTestCapturedValueMacro.swift @@ -8,6 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +import SwiftParser public import SwiftSyntax import SwiftSyntaxBuilder public import SwiftSyntaxMacros @@ -52,3 +53,26 @@ public struct ExitTestBadCapturedValueMacro: ExpressionMacro, Sendable { return #"Swift.fatalError("Unsupported")"# } } + +/// The implementation of the `#__capturedValue()` macro when the type we +/// inferred for the value was incorrect. +/// +/// This type is used to implement the `#__capturedValue()` macro. Do not use it +/// directly. +public struct ExitTestIncorrectlyCapturedValueMacro: ExpressionMacro, Sendable { + public static func expansion( + of macro: some FreestandingMacroExpansionSyntax, + in context: some MacroExpansionContext + ) throws -> ExprSyntax { + let arguments = Array(macro.arguments) + let expr = arguments[0].expression + let nameExpr = arguments[1].expression.cast(StringLiteralExprSyntax.self) + + // Diagnose that the type of 'expr' is invalid. + let name = nameExpr.representedLiteralValue ?? expr.trimmedDescription + let capture = ClosureCaptureSyntax(name: .identifier(name)) + context.diagnose(.typeOfCaptureIsAmbiguous(capture)) + + return expr + } +} diff --git a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift index 0c36fbbf7..80c216854 100644 --- a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift +++ b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift @@ -36,7 +36,7 @@ struct CapturedValueInfo { /// The expression to assign to the captured value with type-checking applied. var typeCheckedExpression: ExprSyntax { - #"#__capturedValue(\#(expression.trimmed), \#(literal: name.trimmedDescription))"# + #"#__capturedValue(\#(expression.trimmed), \#(literal: name.trimmedDescription), (\#(type.trimmed)).self)"# } init(_ capture: ClosureCaptureSyntax, in context: some MacroExpansionContext) { diff --git a/Sources/TestingMacros/TestingMacrosMain.swift b/Sources/TestingMacros/TestingMacrosMain.swift index 4e98115d0..074aeb86b 100644 --- a/Sources/TestingMacros/TestingMacrosMain.swift +++ b/Sources/TestingMacros/TestingMacrosMain.swift @@ -30,6 +30,7 @@ struct TestingMacrosMain: CompilerPlugin { ExitTestRequireMacro.self, ExitTestCapturedValueMacro.self, ExitTestBadCapturedValueMacro.self, + ExitTestIncorrectlyCapturedValueMacro.self, TagMacro.self, SourceLocationMacro.self, PragmaMacro.self, diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 89aae3894..89d78cdde 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -479,12 +479,16 @@ private import _TestingInternals } }(i) +#if false // intentionally fails to compile // FAILS TO COMPILE: shadowing `i` with a variable of a different type will // prevent correct expansion (we need an equivalent of decltype() for that.) -// let i = String(i) -// await #expect(processExitsWith: .success) { [i] in -// #expect(!i.isEmpty) -// } + func g(i: Int) async { + let i = String(i) + await #expect(processExitsWith: .success) { [i] in + #expect(!i.isEmpty) + } + } +#endif } @Test("Capturing a literal expression") From 77de51d550fe406c15af4eb5ac3834580afa3eaf Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 24 Jun 2025 21:29:55 -0400 Subject: [PATCH 222/234] Work around a compiler regression affecting exit test value capturing. (#1171) --- .../ExpectationChecking+Macro.swift | 27 +++++++++++++++++++ Sources/TestingMacros/ConditionMacro.swift | 27 +++++++++++++++++++ .../Support/ClosureCaptureListParsing.swift | 2 +- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 3a190e679..9dfc5ace7 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1175,6 +1175,7 @@ public func __checkClosureCall( /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. +#if SWT_FIXED_154221449 @_spi(Experimental) public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), @@ -1199,6 +1200,32 @@ public func __checkClosureCall( sourceLocation: sourceLocation ) } +#else +@_spi(Experimental) +public func __checkClosureCall( + identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), + encodingCapturedValues capturedValues: repeat each T, + processExitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], + performing _: @convention(c) () -> Void, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation +) async -> Result where repeat each T: Codable & Sendable { + await callExitTest( + identifiedBy: exitTestID, + encodingCapturedValues: Array(repeat each capturedValues), + processExitsWith: expectedExitCondition, + observing: observedValues, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} +#endif #endif // MARK: - diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 0ef0970e9..9d0c4f15f 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -98,7 +98,20 @@ extension ConditionMacro { if let trailingClosureIndex { // Assume that the comment, if present is the last argument in the // argument list prior to the trailing closure that has no label. +#if SWT_FIXED_154221449 commentIndex = macroArguments[.. 1 { // If there is no trailing closure argument and there is more than one // argument, then the comment is the last argument with no label (and also @@ -547,6 +560,7 @@ extension ExitTestConditionMacro { var leadingArguments = [ Argument(label: "identifiedBy", expression: idExpr), ] +#if SWT_FIXED_154221449 if !capturedValues.isEmpty { leadingArguments.append( Argument( @@ -559,6 +573,19 @@ extension ExitTestConditionMacro { ) ) } +#else + if let firstCapturedValue = capturedValues.first { + leadingArguments.append( + Argument( + label: "encodingCapturedValues", + expression: firstCapturedValue.typeCheckedExpression + ) + ) + leadingArguments += capturedValues.dropFirst() + .map(\.typeCheckedExpression) + .map { Argument(expression: $0) } + } +#endif arguments = leadingArguments + arguments // Replace the exit test body (as an argument to the macro) with a stub diff --git a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift index 80c216854..08536aa69 100644 --- a/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift +++ b/Sources/TestingMacros/Support/ClosureCaptureListParsing.swift @@ -36,7 +36,7 @@ struct CapturedValueInfo { /// The expression to assign to the captured value with type-checking applied. var typeCheckedExpression: ExprSyntax { - #"#__capturedValue(\#(expression.trimmed), \#(literal: name.trimmedDescription), (\#(type.trimmed)).self)"# + #"#__capturedValue(\#(expression.trimmed), \#(literal: name.trimmedDescription), \#(type.trimmed).self)"# } init(_ capture: ClosureCaptureSyntax, in context: some MacroExpansionContext) { From f2fb70df0ae5dae2df1378c12d4f323eda469139 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 25 Jun 2025 14:12:45 -0400 Subject: [PATCH 223/234] [6.2] Fix a miscompile when a test function has a raw identifier parameter label. (#1173) - **Explanation**: Fixes parsing of function names to preserve backticks for raw parameter labels. - **Scope**: `@Test` macro and parameterized tests. - **Issues**: #1167 - **Original PRs**: #1168 - **Risk**: No obvious risk. - **Testing**: Standard CI jobs should verify the fix. - **Reviewers**: @briancroom @stmontgomery @hamishknight --- Sources/TestingMacros/TestDeclarationMacro.swift | 2 ++ Tests/TestingTests/MiscellaneousTests.swift | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 58e8259ec..50ac690d2 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -160,6 +160,8 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { for (label, parameter) in parametersWithLabels { if parameter.firstName.tokenKind == .wildcard { LabeledExprSyntax(expression: label) + } else if let rawIdentifier = parameter.firstName.rawIdentifier { + LabeledExprSyntax(label: "`\(rawIdentifier)`", expression: label) } else { LabeledExprSyntax(label: parameter.firstName.textWithoutBackticks, expression: label) } diff --git a/Tests/TestingTests/MiscellaneousTests.swift b/Tests/TestingTests/MiscellaneousTests.swift index b895f6c1b..6a65fb658 100644 --- a/Tests/TestingTests/MiscellaneousTests.swift +++ b/Tests/TestingTests/MiscellaneousTests.swift @@ -315,6 +315,11 @@ struct MiscellaneousTests { let displayName = try #require(suite.displayName) #expect(displayName == "Suite With De Facto Display Name") } + + @Test(arguments: [0]) + func `Test with raw identifier and raw identifier parameter labels can compile`(`argument name` i: Int) { + #expect(i == 0) + } #endif @Test("Free functions are runnable") From d72cb0a1a350a29c759afd12c8d25b1fdc060fc3 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 26 Jun 2025 13:54:09 -0400 Subject: [PATCH 224/234] [6.2] Add (hidden) synchronous overloads of `#expect(throws:)`. (#1179) - **Explanation**: Suppress some new/spurious warnings from the compiler in Swift 6.2 when using `#expect(throws:)` with a synchronous closure that calls a `noasync` function. - **Scope**: Testing errors thrown from synchronous code. - **Issues**: #1177, rdar://149299786 - **Original PRs**: #1178 - **Risk**: No obvious risk. - **Testing**: Existing test functions show the issue and that it's been resolved. - **Reviewers**: @stmontgomery @briancroom --- .../Expectations/Expectation+Macro.swift | 373 ++++++++++++++++++ 1 file changed, 373 insertions(+) diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 8a6749a31..2f64aff3a 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -131,6 +131,74 @@ public macro require( // MARK: - Matching errors by type +#if compiler(>=6.2) +/// Check that an expression always throws an error of a given type. +/// +/// - Parameters: +/// - errorType: The type of error that is expected to be thrown. If +/// `expression` could throw _any_ error, or the specific type of thrown +/// error is unimportant, pass `(any Error).self`. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which recorded expectations and +/// issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// - Returns: If the expectation passes, the instance of `errorType` that was +/// thrown by `expression`. If the expectation fails, the result is `nil`. +/// +/// Use this overload of `#expect()` when the expression `expression` _should_ +/// throw an error of a given type: +/// +/// ```swift +/// #expect(throws: EngineFailureError.self) { +/// FoodTruck.shared.engine.batteryLevel = 0 +/// try FoodTruck.shared.engine.start() +/// } +/// ``` +/// +/// If `expression` does not throw an error, or if it throws an error that is +/// not an instance of `errorType`, an ``Issue`` is recorded for the test that +/// is running in the current task. Any value returned by `expression` is +/// discarded. +/// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// +/// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), +/// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. +/// +/// ## Expressions that should never throw +/// +/// If the expression `expression` should _never_ throw any error, you can pass +/// [`Never.self`](https://developer.apple.com/documentation/swift/never): +/// +/// ```swift +/// #expect(throws: Never.self) { +/// FoodTruck.shared.engine.batteryLevel = 100 +/// try FoodTruck.shared.engine.start() +/// } +/// ``` +/// +/// If `expression` throws an error, an ``Issue`` is recorded for the test that +/// is running in the current task. Any value returned by `expression` is +/// discarded. +/// +/// Test functions can be annotated with `throws` and can throw errors which are +/// then recorded as issues when the test runs. If the intent is for a test to +/// fail when an error is thrown by `expression`, rather than to explicitly +/// check that an error is _not_ thrown by it, do not use this macro. Instead, +/// simply call the code in question and allow it to throw an error naturally. +@discardableResult +@freestanding(expression) +@_documentation(visibility: private) +public macro expect( + throws errorType: E.Type, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () throws -> R +) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error +#endif + /// Check that an expression always throws an error of a given type. /// /// - Parameters: @@ -195,6 +263,58 @@ public macro require( performing expression: () async throws -> R ) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error +#if compiler(>=6.2) +/// Check that an expression always throws an error of a given type, and throw +/// an error if it does not. +/// +/// - Parameters: +/// - errorType: The type of error that is expected to be thrown. If +/// `expression` could throw _any_ error, or the specific type of thrown +/// error is unimportant, pass `(any Error).self`. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which recorded expectations and +/// issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// - Returns: The instance of `errorType` that was thrown by `expression`. +/// +/// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not +/// throw a matching error. The error thrown by `expression` is not rethrown. +/// +/// Use this overload of `#require()` when the expression `expression` _should_ +/// throw an error of a given type: +/// +/// ```swift +/// try #require(throws: EngineFailureError.self) { +/// FoodTruck.shared.engine.batteryLevel = 0 +/// try FoodTruck.shared.engine.start() +/// } +/// ``` +/// +/// If `expression` does not throw an error, or if it throws an error that is +/// not an instance of `errorType`, an ``Issue`` is recorded for the test that +/// is running in the current task and an instance of ``ExpectationFailedError`` +/// is thrown. Any value returned by `expression` is discarded. +/// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// +/// If the thrown error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), +/// use ``require(throws:_:sourceLocation:performing:)-4djuw`` instead. +/// +/// If `expression` should _never_ throw, simply invoke the code without using +/// this macro. The test will then fail if an error is thrown. +@discardableResult +@freestanding(expression) +@_documentation(visibility: private) +public macro require( + throws errorType: E.Type, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () throws -> R +) -> E = #externalMacro(module: "TestingMacros", type: "RequireThrowsMacro") where E: Error +#endif + /// Check that an expression always throws an error of a given type, and throw /// an error if it does not. /// @@ -243,6 +363,28 @@ public macro require( performing expression: () async throws -> R ) -> E = #externalMacro(module: "TestingMacros", type: "RequireThrowsMacro") where E: Error +#if compiler(>=6.2) +/// Check that an expression never throws an error, and throw an error if it +/// does. +/// +/// - Parameters: +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which recorded expectations and +/// issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// - Throws: An instance of ``ExpectationFailedError`` if `expression` throws +/// any error. The error thrown by `expression` is not rethrown. +@freestanding(expression) +@_documentation(visibility: private) +public macro require( + throws _: Never.Type, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () throws -> R +) = #externalMacro(module: "TestingMacros", type: "RequireThrowsNeverMacro") +#endif + /// Check that an expression never throws an error, and throw an error if it /// does. /// @@ -265,6 +407,50 @@ public macro require( // MARK: - Matching instances of equatable errors +#if compiler(>=6.2) +/// Check that an expression always throws a specific error. +/// +/// - Parameters: +/// - error: The error that is expected to be thrown. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which recorded expectations and +/// issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// - Returns: If the expectation passes, the instance of `E` that was thrown by +/// `expression` and is equal to `error`. If the expectation fails, the result +/// is `nil`. +/// +/// Use this overload of `#expect()` when the expression `expression` _should_ +/// throw a specific error: +/// +/// ```swift +/// #expect(throws: EngineFailureError.batteryDied) { +/// FoodTruck.shared.engine.batteryLevel = 0 +/// try FoodTruck.shared.engine.start() +/// } +/// ``` +/// +/// If `expression` does not throw an error, or if it throws an error that is +/// not equal to `error`, an ``Issue`` is recorded for the test that is running +/// in the current task. Any value returned by `expression` is discarded. +/// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// +/// If the thrown error need only be an instance of a particular type, use +/// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. +@discardableResult +@freestanding(expression) +@_documentation(visibility: private) +public macro expect( + throws error: E, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () throws -> R +) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error & Equatable +#endif + /// Check that an expression always throws a specific error. /// /// - Parameters: @@ -305,6 +491,54 @@ public macro require( performing expression: () async throws -> R ) -> E? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") where E: Error & Equatable +#if compiler(>=6.2) +/// Check that an expression always throws a specific error, and throw an error +/// if it does not. +/// +/// - Parameters: +/// - error: The error that is expected to be thrown. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which recorded expectations and +/// issues should be attributed. +/// - expression: The expression to be evaluated. + +/// - Returns: The instance of `E` that was thrown by `expression` and is equal +/// to `error`. +/// +/// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not +/// throw a matching error. The error thrown by `expression` is not rethrown. +/// +/// Use this overload of `#require()` when the expression `expression` _should_ +/// throw a specific error: +/// +/// ```swift +/// try #require(throws: EngineFailureError.batteryDied) { +/// FoodTruck.shared.engine.batteryLevel = 0 +/// try FoodTruck.shared.engine.start() +/// } +/// ``` +/// +/// If `expression` does not throw an error, or if it throws an error that is +/// not equal to `error`, an ``Issue`` is recorded for the test that is running +/// in the current task and an instance of ``ExpectationFailedError`` is thrown. +/// Any value returned by `expression` is discarded. +/// +/// - Note: If you use this macro with a Swift compiler version lower than 6.1, +/// it doesn't return a value. +/// +/// If the thrown error need only be an instance of a particular type, use +/// ``require(throws:_:sourceLocation:performing:)-7n34r`` instead. +@discardableResult +@freestanding(expression) +@_documentation(visibility: private) +public macro require( + throws error: E, + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () throws -> R +) -> E = #externalMacro(module: "TestingMacros", type: "RequireMacro") where E: Error & Equatable +#endif + /// Check that an expression always throws a specific error, and throw an error /// if it does not. /// @@ -351,6 +585,72 @@ public macro require( // MARK: - Arbitrary error matching +#if compiler(>=6.2) +/// Check that an expression always throws an error matching some condition. +/// +/// - Parameters: +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which recorded expectations and +/// issues should be attributed. +/// - expression: The expression to be evaluated. +/// - errorMatcher: A closure to invoke when `expression` throws an error that +/// indicates if it matched or not. +/// +/// - Returns: If the expectation passes, the error that was thrown by +/// `expression`. If the expectation fails, the result is `nil`. +/// +/// Use this overload of `#expect()` when the expression `expression` _should_ +/// throw an error, but the logic to determine if the error matches is complex: +/// +/// ```swift +/// #expect { +/// FoodTruck.shared.engine.batteryLevel = 0 +/// try FoodTruck.shared.engine.start() +/// } throws: { error in +/// return error == EngineFailureError.batteryDied +/// || error == EngineFailureError.stillCharging +/// } +/// ``` +/// +/// If `expression` does not throw an error, if it throws an error that is +/// not matched by `errorMatcher`, or if `errorMatcher` throws an error +/// (including the error passed to it), an ``Issue`` is recorded for the test +/// that is running in the current task. Any value returned by `expression` is +/// discarded. +/// +/// If the thrown error need only be an instance of a particular type, use +/// ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead. If the thrown +/// error need only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), +/// use ``expect(throws:_:sourceLocation:performing:)-7du1h`` instead. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.0) +/// @Available(Xcode, introduced: 16.0) +/// } +/// +/// @DeprecationSummary { +/// Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` +/// or ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: +/// +/// ```swift +/// let error = #expect(throws: FoodTruckError.self) { +/// ... +/// } +/// #expect(error?.napkinCount == 0) +/// ``` +/// } +@available(swift, deprecated: 100000.0, message: "Examine the result of '#expect(throws:)' instead.") +@discardableResult +@freestanding(expression) +@_documentation(visibility: private) +public macro expect( + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () throws -> R, + throws errorMatcher: (any Error) throws -> Bool +) -> (any Error)? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") +#endif + /// Check that an expression always throws an error matching some condition. /// /// - Parameters: @@ -413,6 +713,79 @@ public macro require( throws errorMatcher: (any Error) async throws -> Bool ) -> (any Error)? = #externalMacro(module: "TestingMacros", type: "ExpectMacro") +#if compiler(>=6.2) +/// Check that an expression always throws an error matching some condition, and +/// throw an error if it does not. +/// +/// - Parameters: +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which recorded expectations and +/// issues should be attributed. +/// - expression: The expression to be evaluated. +/// - errorMatcher: A closure to invoke when `expression` throws an error that +/// indicates if it matched or not. +/// +/// - Returns: The error that was thrown by `expression`. +/// +/// - Throws: An instance of ``ExpectationFailedError`` if `expression` does not +/// throw a matching error. The error thrown by `expression` is not rethrown. +/// +/// Use this overload of `#require()` when the expression `expression` _should_ +/// throw an error, but the logic to determine if the error matches is complex: +/// +/// ```swift +/// #expect { +/// FoodTruck.shared.engine.batteryLevel = 0 +/// try FoodTruck.shared.engine.start() +/// } throws: { error in +/// return error == EngineFailureError.batteryDied +/// || error == EngineFailureError.stillCharging +/// } +/// ``` +/// +/// If `expression` does not throw an error, if it throws an error that is +/// not matched by `errorMatcher`, or if `errorMatcher` throws an error +/// (including the error passed to it), an ``Issue`` is recorded for the test +/// that is running in the current task and an instance of +/// ``ExpectationFailedError`` is thrown. Any value returned by `expression` is +/// discarded. +/// +/// If the thrown error need only be an instance of a particular type, use +/// ``require(throws:_:sourceLocation:performing:)-7n34r`` instead. If the thrown error need +/// only equal another instance of [`Error`](https://developer.apple.com/documentation/swift/error), +/// use ``require(throws:_:sourceLocation:performing:)-4djuw`` instead. +/// +/// If `expression` should _never_ throw, simply invoke the code without using +/// this macro. The test will then fail if an error is thrown. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.0) +/// @Available(Xcode, introduced: 16.0) +/// } +/// +/// @DeprecationSummary { +/// Examine the result of ``expect(throws:_:sourceLocation:performing:)-7du1h`` +/// or ``expect(throws:_:sourceLocation:performing:)-1hfms`` instead: +/// +/// ```swift +/// let error = try #require(throws: FoodTruckError.self) { +/// ... +/// } +/// #expect(error.napkinCount == 0) +/// ``` +/// } +@available(swift, deprecated: 100000.0, message: "Examine the result of '#require(throws:)' instead.") +@discardableResult +@freestanding(expression) +@_documentation(visibility: private) +public macro require( + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing expression: () throws -> R, + throws errorMatcher: (any Error) throws -> Bool +) -> any Error = #externalMacro(module: "TestingMacros", type: "RequireMacro") +#endif + /// Check that an expression always throws an error matching some condition, and /// throw an error if it does not. /// From 9ebfc4ebbb2840da60d37d3786e379a6a5833dbc Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 26 Jun 2025 16:13:11 -0500 Subject: [PATCH 225/234] [6.2] Remove redundant word from console output for test case started events in verbose mode (#1181) - **Explanation**: This is a small fix for an oversight: the word "started" is printed twice at the end of the console message for `.testCaseStarted` events in verbose mode. - **Scope**: Affects console output in verbose mode. - **Issues**: n/a - **Original PRs**: https://github.com/swiftlang/swift-testing/pull/1180 - **Risk**: Low - **Testing**: New unit test added - **Reviewers**: @grynspan --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 4 +--- Tests/TestingTests/EventRecorderTests.swift | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index a3d121e09..248bb4aec 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -521,12 +521,10 @@ extension Event.HumanReadableOutputRecorder { break } - let status = verbosity > 0 ? " started" : "" - return [ Message( symbol: .default, - stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)\(status) started." + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) started." ) ] diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index ed7d765a0..e3dc9ca99 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -113,6 +113,10 @@ struct EventRecorderTests { #expect(buffer.contains(#"\#(Event.Symbol.details.unicodeCharacter) lhs: Swift.String → "987""#)) #expect(buffer.contains(#""Animal Crackers" (aka 'WrittenTests')"#)) #expect(buffer.contains(#""Not A Lobster" (aka 'actuallyCrab()')"#)) + do { + let regex = try Regex(".* Test case passing 1 argument i → 0 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) started.") + #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) + } do { let regex = try Regex(".* Test case passing 1 argument i → 0 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) passed after .*.") #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) From 7751ca50d7fa87d729bf2e9bb600b13395c3e56b Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 30 Jun 2025 17:06:54 -0500 Subject: [PATCH 226/234] [6.2] Ensure that when .serialized is applied to a parameterized @Test func, its test cases are serialized (#1190) - **Explanation**: This fixes a regression when `.serialized` is applied directly to a parameterized `@Test` function, not to a containing suite, its test cases are no longer serialized. - **Scope**: Affects the `.serialized` trait when applied to `@Test` functions, specifically, not to suite types. - **Issues**: rdar://154529146 - **Original PRs**: https://github.com/swiftlang/swift-testing/pull/1188 - **Risk**: - **Testing**: New regression test added - **Reviewers**: @suzannaratcliff, @briancroom, @grynspan --- .../Testing/Traits/ParallelizationTrait.swift | 8 +++++++ .../TestSupport/TestingAdditions.swift | 21 +++++++++++++++++-- .../Traits/ParallelizationTraitTests.swift | 13 +++++++++--- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/Sources/Testing/Traits/ParallelizationTrait.swift b/Sources/Testing/Traits/ParallelizationTrait.swift index 9eb1bd2a5..c91e01761 100644 --- a/Sources/Testing/Traits/ParallelizationTrait.swift +++ b/Sources/Testing/Traits/ParallelizationTrait.swift @@ -31,6 +31,14 @@ public struct ParallelizationTrait: TestTrait, SuiteTrait {} // MARK: - TestScoping extension ParallelizationTrait: TestScoping { + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + // When applied to a test function, this trait should provide scope to the + // test function itself, not its individual test cases, since that allows + // Runner to correctly interpret the configuration setting to disable + // parallelization. + test.isSuite || testCase == nil ? self : nil + } + public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { guard var configuration = Configuration.current else { throw SystemError(description: "There is no current Configuration when attempting to provide scope for test '\(test.name)'. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 4648f96af..5c596785e 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -103,6 +103,25 @@ extension Runner { /// - fileID: The `#fileID` string whose module should be used to locate /// the test function to run. /// - configuration: The configuration to use for running. + init( + selecting testName: String, + inModuleOf fileID: String = #fileID, + configuration: Configuration = .init() + ) async { + let plan = await Runner.Plan(selecting: testName, inModuleOf: fileID, configuration: configuration) + self.init(plan: plan, configuration: configuration) + } +} + +extension Runner.Plan { + /// Initialize an instance of this type that selects the free test function + /// named `testName` in the module specified in `fileID`. + /// + /// - Parameters: + /// - testName: The name of the test function this instance should run. + /// - fileID: The `#fileID` string whose module should be used to locate + /// the test function to run. + /// - configuration: The configuration to use for running. init( selecting testName: String, inModuleOf fileID: String = #fileID, @@ -116,9 +135,7 @@ extension Runner { await self.init(configuration: configuration) } -} -extension Runner.Plan { /// Initialize an instance of this type with the specified suite type. /// /// - Parameters: diff --git a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift index 776e5c320..6c4963dc5 100644 --- a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift +++ b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift @@ -12,8 +12,11 @@ @Suite("Parallelization Trait Tests", .tags(.traitRelated)) struct ParallelizationTraitTests { - @Test(".serialized trait serializes parameterized test") - func serializesParameterizedTestFunction() async { + @Test(".serialized trait serializes parameterized test", arguments: await [ + Runner.Plan(selecting: OuterSuite.self), + Runner.Plan(selecting: "globalParameterized(i:)"), + ]) + func serializesParameterizedTestFunction(plan: Runner.Plan) async { var configuration = Configuration() configuration.isParallelizationEnabled = true @@ -33,7 +36,6 @@ struct ParallelizationTraitTests { } } - let plan = await Runner.Plan(selecting: OuterSuite.self, configuration: configuration) let runner = Runner(plan: plan, configuration: configuration) await runner.run() @@ -59,3 +61,8 @@ private struct OuterSuite { } } } + +@Test(.hidden, .serialized, arguments: 0 ..< 10_000) +private func globalParameterized(i: Int) { + Issue.record("PARAMETERIZED\(i)") +} From 754c808ee666154c4237b0c370381c52d962b2d0 Mon Sep 17 00:00:00 2001 From: Evan Wilde Date: Wed, 2 Jul 2025 16:20:49 -0700 Subject: [PATCH 227/234] [6.2] FreeBSD: Gate GNU-only API (#1194) The FreeBSD builds are currently using the GlibC modulemap to import the C runtimes. FreeBSD does not have `gnu_get_libc_version` resulting in build failures. The use of this API was introduced in https://github.com/swiftlang/swift-testing/pull/1147 (cherry picked from commit 79c22ad7b9c372499c305ca0f8fef78af2a907c5) - **Explanation**: The FreeBSD builds are currently using the GlibC modulemap to import the C runtimes. FreeBSD does not have `gnu_get_libc_version` resulting in build failures. - **Scope**: Build failure on platforms using Glibc modulemap that don't have GNU extensions. (FreeBSD, OpenBSD) - **Issues**: https://github.com/swiftlang/swift-testing/issues/1193 - **Original PRs**: https://github.com/swiftlang/swift-testing/pull/1183 - **Risk**: Low risk. Removes use of unavailable API. - **Testing**: Built swift-testing on FreeBSD and Linux. - **Reviewers**: @grynspan @3405691582 Fixes: #1193 --- Sources/Testing/ExitTests/SpawnProcess.swift | 2 +- Sources/Testing/Support/Versions.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 66143a7e0..fe51a7086 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -137,7 +137,7 @@ func spawnExecutable( // standardized in POSIX.1-2024 (see https://pubs.opengroup.org/onlinepubs/9799919799/functions/posix_spawn_file_actions_adddup2.html // and https://www.austingroupbugs.net/view.php?id=411). _ = posix_spawn_file_actions_adddup2(fileActions, fd, fd) -#if canImport(Glibc) +#if canImport(Glibc) && !os(FreeBSD) && !os(OpenBSD) if _slowPath(glibcVersion.major < 2 || (glibcVersion.major == 2 && glibcVersion.minor < 29)) { // This system is using an older version of glibc that does not // implement FD_CLOEXEC clearing in posix_spawn_file_actions_adddup2(), diff --git a/Sources/Testing/Support/Versions.swift b/Sources/Testing/Support/Versions.swift index 1229e80b0..7f190ebb2 100644 --- a/Sources/Testing/Support/Versions.swift +++ b/Sources/Testing/Support/Versions.swift @@ -153,7 +153,7 @@ let swiftStandardLibraryVersion: String = { return "unknown" }() -#if canImport(Glibc) +#if canImport(Glibc) && !os(FreeBSD) && !os(OpenBSD) /// The (runtime, not compile-time) version of glibc in use on this system. /// /// This value is not part of the public interface of the testing library. From 93fd96796de5ddd772f1e697d677c8ba856d1c19 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 8 Jul 2025 15:14:18 -0400 Subject: [PATCH 228/234] [6.2] Disallow the `@Test` attribute on operator declarations. (#1207) - **Explanation**: Block `@Test` on operator declarations, which doesn't work correctly and was never supported. - **Scope**: Attempting to declare `@Test func +()`. - **Issues**: #1204 - **Original PRs**: #1205 - **Risk**: Low (this was never supported) - **Testing**: New test case added. - **Reviewers**: @stmontgomery @briancroom --- .../Additions/FunctionDeclSyntaxAdditions.swift | 10 ++++++++++ Sources/TestingMacros/Support/DiagnosticMessage.swift | 6 +++++- Sources/TestingMacros/TestDeclarationMacro.swift | 2 +- .../TestingMacrosTests/TestDeclarationMacroTests.swift | 2 ++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Sources/TestingMacros/Support/Additions/FunctionDeclSyntaxAdditions.swift b/Sources/TestingMacros/Support/Additions/FunctionDeclSyntaxAdditions.swift index 9b9378283..9b55bc157 100644 --- a/Sources/TestingMacros/Support/Additions/FunctionDeclSyntaxAdditions.swift +++ b/Sources/TestingMacros/Support/Additions/FunctionDeclSyntaxAdditions.swift @@ -34,6 +34,16 @@ extension FunctionDeclSyntax { .contains(.keyword(.nonisolated)) } + /// Whether or not this function declares an operator. + var isOperator: Bool { + switch name.tokenKind { + case .binaryOperator, .prefixOperator, .postfixOperator: + true + default: + false + } + } + /// The name of this function including parentheses, parameter labels, and /// colons. var completeName: DeclReferenceExprSyntax { diff --git a/Sources/TestingMacros/Support/DiagnosticMessage.swift b/Sources/TestingMacros/Support/DiagnosticMessage.swift index b7103bcc6..dd35b118d 100644 --- a/Sources/TestingMacros/Support/DiagnosticMessage.swift +++ b/Sources/TestingMacros/Support/DiagnosticMessage.swift @@ -93,7 +93,11 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage { let result: (value: String, article: String) switch node.kind { case .functionDecl: - result = ("function", "a") + if node.cast(FunctionDeclSyntax.self).isOperator { + result = ("operator", "an") + } else { + result = ("function", "a") + } case .classDecl: result = ("class", "a") case .structDecl: diff --git a/Sources/TestingMacros/TestDeclarationMacro.swift b/Sources/TestingMacros/TestDeclarationMacro.swift index 50ac690d2..ef156edd6 100644 --- a/Sources/TestingMacros/TestDeclarationMacro.swift +++ b/Sources/TestingMacros/TestDeclarationMacro.swift @@ -61,7 +61,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable { } // The @Test attribute is only supported on function declarations. - guard let function = declaration.as(FunctionDeclSyntax.self) else { + guard let function = declaration.as(FunctionDeclSyntax.self), !function.isOperator else { diagnostics.append(.attributeNotSupported(testAttribute, on: declaration)) return false } diff --git a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift index 6c04eb9eb..c53783b6a 100644 --- a/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift +++ b/Tests/TestingMacrosTests/TestDeclarationMacroTests.swift @@ -67,6 +67,8 @@ struct TestDeclarationMacroTests { "Attribute 'Test' cannot be applied to a structure", "@Test enum E {}": "Attribute 'Test' cannot be applied to an enumeration", + "@Test func +() {}": + "Attribute 'Test' cannot be applied to an operator", // Availability "@available(*, unavailable) @Suite struct S {}": From 1c9c9af6a29b6488c9d25352bb4c9c23a7a893dd Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 10 Jul 2025 22:17:56 -0400 Subject: [PATCH 229/234] [6.2] Emit "barriers" into the stdout/stderr streams of an exit test. (#1220) --- Sources/Testing/ExitTests/ExitTest.swift | 78 +++++++++++++++++++++++- Tests/TestingTests/ExitTestTests.swift | 2 + 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index c5579981e..41f3b14df 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -530,6 +530,73 @@ extension ABI { @_spi(ForToolsIntegrationOnly) extension ExitTest { + /// A barrier value to insert into the standard output and standard error + /// streams immediately before and after the body of an exit test runs in + /// order to distinguish output produced by the host process. + /// + /// The value of this property was randomly generated. It could conceivably + /// show up in actual output from an exit test, but the statistical likelihood + /// of that happening is negligible. + static var barrierValue: [UInt8] { + [ + 0x39, 0x74, 0x87, 0x6d, 0x96, 0xdd, 0xf6, 0x17, + 0x7f, 0x05, 0x61, 0x5d, 0x46, 0xeb, 0x37, 0x0c, + 0x90, 0x07, 0xca, 0xe5, 0xed, 0x0b, 0xc4, 0xc4, + 0x46, 0x36, 0xc5, 0xb8, 0x9c, 0xc7, 0x86, 0x57, + ] + } + + /// Remove the leading and trailing barrier values from the given array of + /// bytes along. + /// + /// - Parameters: + /// - buffer: The buffer to trim. + /// + /// - Returns: A copy of `buffer`. If a barrier value (equal to + /// ``barrierValue``) is present in `buffer`, it and everything before it + /// are trimmed from the beginning of the copy. If there is more than one + /// barrier value present, the last one and everything after it are trimmed + /// from the end of the copy. If no barrier value is present, `buffer` is + /// returned verbatim. + private static func _trimToBarrierValues(_ buffer: [UInt8]) -> [UInt8] { + let barrierValue = barrierValue + let firstBarrierByte = barrierValue[0] + + // If the buffer is too small to contain the barrier value, exit early. + guard buffer.count > barrierValue.count else { + return buffer + } + + // Find all the indices where the first byte of the barrier is present. + let splits = buffer.indices.filter { buffer[$0] == firstBarrierByte } + + // Trim off the leading barrier value. If we didn't find any barrier values, + // we do nothing. + let leadingIndex = splits.first { buffer[$0...].starts(with: barrierValue) } + guard let leadingIndex else { + return buffer + } + var trimmedBuffer = buffer[leadingIndex...].dropFirst(barrierValue.count) + + // If there's a trailing barrier value, trim it too. If it's at the same + // index as the leading barrier value, that means only one barrier value + // was present and we should assume it's the leading one. + let trailingIndex = splits.last { buffer[$0...].starts(with: barrierValue) } + if let trailingIndex, trailingIndex > leadingIndex { + trimmedBuffer = trimmedBuffer[.. Date: Thu, 10 Jul 2025 22:18:08 -0400 Subject: [PATCH 230/234] [6.2] Reset the working directory of child processes we spawn on Windows. (#1219) --- Sources/Testing/ExitTests/SpawnProcess.swift | 30 +++++++++++++++--- Sources/Testing/Support/FileHandle.swift | 31 +++++++++++++++++++ .../Support/FileHandleTests.swift | 11 +++++++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index fe51a7086..9f01a1d11 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -284,25 +284,47 @@ func spawnExecutable( let commandLine = _escapeCommandLine(CollectionOfOne(executablePath) + arguments) let environ = environment.map { "\($0.key)=\($0.value)" }.joined(separator: "\0") + "\0\0" + // CreateProcessW() may modify the command line argument, so we must make + // a mutable copy of it. (environ is also passed as a mutable raw pointer, + // but it is not documented as actually being mutated.) + let commandLineCopy = commandLine.withCString(encodedAs: UTF16.self) { _wcsdup($0) } + defer { + free(commandLineCopy) + } + + // On Windows, a process holds a reference to its current working + // directory, which prevents other processes from deleting it. This causes + // code to fail if it tries to set the working directory to a temporary + // path. SEE: https://github.com/swiftlang/swift-testing/issues/1209 + // + // This problem manifests for us when we spawn a child process without + // setting its working directory, which causes it to default to that of + // the parent process. To avoid this problem, we set the working directory + // of the new process to the root directory of the boot volume (which is + // unlikely to be deleted, one hopes). + // + // SEE: https://devblogs.microsoft.com/oldnewthing/20101109-00/?p=12323 + let workingDirectoryPath = rootDirectoryPath + var flags = DWORD(CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT | EXTENDED_STARTUPINFO_PRESENT) #if DEBUG // Start the process suspended so we can attach a debugger if needed. flags |= DWORD(CREATE_SUSPENDED) #endif - return try commandLine.withCString(encodedAs: UTF16.self) { commandLine in - try environ.withCString(encodedAs: UTF16.self) { environ in + return try environ.withCString(encodedAs: UTF16.self) { environ in + try workingDirectoryPath.withCString(encodedAs: UTF16.self) { workingDirectoryPath in var processInfo = PROCESS_INFORMATION() guard CreateProcessW( nil, - .init(mutating: commandLine), + commandLineCopy, nil, nil, true, // bInheritHandles flags, .init(mutating: environ), - nil, + workingDirectoryPath, startupInfo.pointer(to: \.StartupInfo)!, &processInfo ) else { diff --git a/Sources/Testing/Support/FileHandle.swift b/Sources/Testing/Support/FileHandle.swift index 1c5447460..37774b91a 100644 --- a/Sources/Testing/Support/FileHandle.swift +++ b/Sources/Testing/Support/FileHandle.swift @@ -719,4 +719,35 @@ func setFD_CLOEXEC(_ flag: Bool, onFileDescriptor fd: CInt) throws { } } #endif + +/// The path to the root directory of the boot volume. +/// +/// On Windows, this string is usually of the form `"C:\"`. On UNIX-like +/// platforms, it is always equal to `"/"`. +let rootDirectoryPath: String = { +#if os(Windows) + var result: String? + + // The boot volume is, except in some legacy scenarios, the volume that + // contains the system Windows directory. For an explanation of the difference + // between the Windows directory and the _system_ Windows directory, see + // https://devblogs.microsoft.com/oldnewthing/20140723-00/?p=423 . + let count = GetSystemWindowsDirectoryW(nil, 0) + if count > 0 { + withUnsafeTemporaryAllocation(of: wchar_t.self, capacity: Int(count) + 1) { buffer in + _ = GetSystemWindowsDirectoryW(buffer.baseAddress!, UINT(buffer.count)) + let rStrip = PathCchStripToRoot(buffer.baseAddress!, buffer.count) + if rStrip == S_OK || rStrip == S_FALSE { + result = String.decodeCString(buffer.baseAddress!, as: UTF16.self)?.result + } + } + } + + // If we weren't able to get a path, fall back to "C:\" on the assumption that + // it's the common case and most likely correct. + return result ?? #"C:\"# +#else + return "/" +#endif +}() #endif diff --git a/Tests/TestingTests/Support/FileHandleTests.swift b/Tests/TestingTests/Support/FileHandleTests.swift index 4be633ad6..acca1dbea 100644 --- a/Tests/TestingTests/Support/FileHandleTests.swift +++ b/Tests/TestingTests/Support/FileHandleTests.swift @@ -201,6 +201,17 @@ struct FileHandleTests { #endif } #endif + + @Test("Root directory path is correct") + func rootDirectoryPathIsCorrect() throws { +#if os(Windows) + if let systemDrive = Environment.variable(named: "SYSTEMDRIVE") { + #expect(rootDirectoryPath.starts(with: systemDrive)) + } +#else + #expect(rootDirectoryPath == "/") +#endif + } } // MARK: - Fixtures From 695200452c0f7b0ca5f36d44efe22fb9fe84dbce Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 16 Jul 2025 18:55:20 -0500 Subject: [PATCH 231/234] [6.2] Only clear exit test ID env var upon successful lookup (#1227) --- Sources/Testing/ExitTests/ExitTest.swift | 35 +++++++++++++++--------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 41f3b14df..9b7fa7c21 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -233,6 +233,10 @@ extension ExitTest { #endif } + /// The name of the environment variable used to identify the exit test to + /// call in a spawned exit test process. + private static let _idEnvironmentVariableName = "SWT_EXIT_TEST_ID" + /// Call the exit test in the current process. /// /// This function invokes the closure originally passed to @@ -698,6 +702,17 @@ extension ExitTest { #endif } + /// The ID of the exit test to run, if any, specified in the environment. + static var environmentIDForEntryPoint: ID? { + guard var idString = Environment.variable(named: Self._idEnvironmentVariableName) else { + return nil + } + + return try? idString.withUTF8 { idBuffer in + try JSON.decode(ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) + } + } + /// Find the exit test function specified in the environment of the current /// process, if any. /// @@ -708,21 +723,15 @@ extension ExitTest { /// `__swiftPMEntryPoint()` function. The effect of using it under other /// configurations is undefined. static func findInEnvironmentForEntryPoint() -> Self? { - // Find the ID of the exit test to run, if any, in the environment block. - var id: ExitTest.ID? - if var idString = Environment.variable(named: "SWT_EXIT_TEST_ID") { - // Clear the environment variable. It's an implementation detail and exit - // test code shouldn't be dependent on it. Use ExitTest.current if needed! - Environment.setVariable(nil, named: "SWT_EXIT_TEST_ID") - - id = try? idString.withUTF8 { idBuffer in - try JSON.decode(ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) - } - } - guard let id, var result = find(identifiedBy: id) else { + guard let id = environmentIDForEntryPoint, var result = find(identifiedBy: id) else { return nil } + // Since an exit test was found, clear the environment variable. It's an + // implementation detail and exit test code shouldn't be dependent on it. + // Use ExitTest.current if needed! + Environment.setVariable(nil, named: Self._idEnvironmentVariableName) + // If an exit test was found, inject back channel handling into its body. // External tools authors should set up their own back channel mechanisms // and ensure they're installed before calling ExitTest.callAsFunction(). @@ -852,7 +861,7 @@ extension ExitTest { // Insert a specific variable that tells the child process which exit test // to run. try JSON.withEncoding(of: exitTest.id) { json in - childEnvironment["SWT_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) + childEnvironment[Self._idEnvironmentVariableName] = String(decoding: json, as: UTF8.self) } typealias ResultUpdater = @Sendable (inout ExitTest.Result) -> Void From 9d0323e30b1c3cd0511f173a06beb4b18793447f Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 17 Jul 2025 07:09:31 -0500 Subject: [PATCH 232/234] [6.2] Promote Issue Handling Traits to public API (#1228) --- Sources/Testing/Testing.docc/Traits.md | 4 +-- .../Testing/Traits/IssueHandlingTrait.swift | 36 ++++++++++++++++--- .../Traits/IssueHandlingTraitTests.swift | 36 ++++++++++++++++++- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index 46fa82b4d..d14eda999 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -48,12 +48,10 @@ types that customize the behavior of your tests. - ``Trait/bug(_:id:_:)-10yf5`` - ``Trait/bug(_:id:_:)-3vtpl`` - ### Creating custom traits @@ -67,8 +65,8 @@ types that customize the behavior of your tests. - ``Bug`` - ``Comment`` - ``ConditionTrait`` +- ``IssueHandlingTrait`` - ``ParallelizationTrait`` - ``Tag`` - ``Tag/List`` - ``TimeLimitTrait`` - diff --git a/Sources/Testing/Traits/IssueHandlingTrait.swift b/Sources/Testing/Traits/IssueHandlingTrait.swift index e8142ff5a..4d7d408d6 100644 --- a/Sources/Testing/Traits/IssueHandlingTrait.swift +++ b/Sources/Testing/Traits/IssueHandlingTrait.swift @@ -24,7 +24,10 @@ /// /// - ``Trait/compactMapIssues(_:)`` /// - ``Trait/filterIssues(_:)`` -@_spi(Experimental) +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } public struct IssueHandlingTrait: TestTrait, SuiteTrait { /// A function which handles an issue and returns an optional replacement. /// @@ -49,6 +52,10 @@ public struct IssueHandlingTrait: TestTrait, SuiteTrait { /// /// - Returns: An issue to replace `issue`, or else `nil` if the issue should /// not be recorded. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public func handleIssue(_ issue: Issue) -> Issue? { _handler(issue) } @@ -58,6 +65,9 @@ public struct IssueHandlingTrait: TestTrait, SuiteTrait { } } +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } extension IssueHandlingTrait: TestScoping { public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { // Provide scope for tests at both the suite and test case levels, but not @@ -111,9 +121,20 @@ extension IssueHandlingTrait: TestScoping { } if let newIssue { - // Prohibit assigning the issue's kind to system. - if case .system = newIssue.kind { + // Validate the value of the returned issue's 'kind' property. + switch (issue.kind, newIssue.kind) { + case (_, .system): + // Prohibited by ST-0011. preconditionFailure("Issue returned by issue handling closure cannot have kind 'system': \(newIssue)") + case (.apiMisused, .apiMisused): + // This is permitted, but must be listed explicitly before the + // wildcard case below. + break + case (_, .apiMisused): + // Prohibited by ST-0011. + preconditionFailure("Issue returned by issue handling closure cannot have kind 'apiMisused' when the passed-in issue had a different kind: \(newIssue)") + default: + break } var event = event @@ -126,7 +147,6 @@ extension IssueHandlingTrait: TestScoping { } } -@_spi(Experimental) extension Trait where Self == IssueHandlingTrait { /// Constructs an trait that transforms issues recorded by a test. /// @@ -158,6 +178,10 @@ extension Trait where Self == IssueHandlingTrait { /// - Note: `transform` will never be passed an issue for which the value of /// ``Issue/kind`` is ``Issue/Kind/system``, and may not return such an /// issue. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func compactMapIssues(_ transform: @escaping @Sendable (Issue) -> Issue?) -> Self { Self(handler: transform) } @@ -192,6 +216,10 @@ extension Trait where Self == IssueHandlingTrait { /// /// - Note: `isIncluded` will never be passed an issue for which the value of /// ``Issue/kind`` is ``Issue/Kind/system``. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func filterIssues(_ isIncluded: @escaping @Sendable (Issue) -> Bool) -> Self { Self { issue in isIncluded(issue) ? issue : nil diff --git a/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift index eb1aa1233..6ca9f02c0 100644 --- a/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift +++ b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +@testable @_spi(ForToolsIntegrationOnly) import Testing @Suite("IssueHandlingTrait Tests") struct IssueHandlingTraitTests { @@ -216,6 +216,27 @@ struct IssueHandlingTraitTests { }.run(configuration: configuration) } + @Test("An API misused issue can be returned by issue handler closure when the original issue had that kind") + func returningAPIMisusedIssue() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + if case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind { + issue.record() + } + } + + let handler = IssueHandlingTrait.compactMapIssues { issue in + guard case .apiMisused = issue.kind else { + return Issue.record("Expected an issue of kind 'apiMisused': \(issue)") + } + return issue + } + + await Test(handler) { + Issue(kind: .apiMisused).record() + }.run(configuration: configuration) + } + #if !SWT_NO_EXIT_TESTS @Test("Disallow assigning kind to .system") func disallowAssigningSystemKind() async throws { @@ -229,5 +250,18 @@ struct IssueHandlingTraitTests { }.run() } } + + @Test("Disallow assigning kind to .apiMisused") + func disallowAssigningAPIMisusedKind() async throws { + await #expect(processExitsWith: .failure) { + await Test(.compactMapIssues { issue in + var issue = issue + issue.kind = .apiMisused + return issue + }) { + Issue.record("A non-system issue") + }.run() + } + } #endif } From fb80eb9ecaccaae5cbe42ae0266360505a7b52a5 Mon Sep 17 00:00:00 2001 From: Graham Lee Date: Thu, 17 Jul 2025 16:22:35 +0100 Subject: [PATCH 233/234] [6.2] Clarify the outcome of applying a timeLimit trait to a suite. (#1225) (#1229) - **Explanation**: Update the documentation for the `timeLimit` trait to clarify that applying the trait to a suite means that each test in that suite gets the time limit. - **Scope**: Documentation changes only. - **Issues**: - **Original PRs**: #1225 - **Risk**: Documentation changes only, so very low risk. - **Testing**: N/A - **Reviewers**: @stmontgomery --- .../Testing.docc/LimitingExecutionTime.md | 25 ++++++++++++------- Sources/Testing/Traits/TimeLimitTrait.swift | 14 +++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/Testing.docc/LimitingExecutionTime.md b/Sources/Testing/Testing.docc/LimitingExecutionTime.md index 151b52028..9ea346230 100644 --- a/Sources/Testing/Testing.docc/LimitingExecutionTime.md +++ b/Sources/Testing/Testing.docc/LimitingExecutionTime.md @@ -40,8 +40,8 @@ hour (60 x 60 seconds) to execute, the task in which it's running is and the test fails with an issue of kind ``Issue/Kind-swift.enum/timeLimitExceeded(timeLimitComponents:)``. -- Note: If multiple time limit traits apply to a test, the shortest time limit - is used. +- Note: If multiple time limit traits apply to a test, the testing library uses + the shortest time limit. The testing library may adjust the specified time limit for performance reasons or to ensure tests have enough time to run. In particular, a granularity of (by @@ -49,13 +49,20 @@ default) one minute is applied to tests. The testing library can also be configured with a maximum time limit per test that overrides any applied time limit traits. -### Time limits applied to test suites +### Apply time limits to test suites -When a time limit is applied to a test suite, it's recursively applied to all -test functions and child test suites within that suite. +When you apply a time limit to a test suite, the testing library recursively +applies it to all test functions and child test suites within that suite. +The time limit applies to each test in the test suite and any child test suites, +or each test case for parameterized tests. -### Time limits applied to parameterized tests +For example, if a suite contains five tests and you apply a time limit trait +with a duration of one minute, then each test in the suite may run for up to +one minute. -When a time limit is applied to a parameterized test function, it's applied to -each invocation _separately_ so that if only some arguments cause failures, then -successful arguments aren't incorrectly marked as failing too. +### Apply time limits to parameterized tests + +When you apply a time limit to a parameterized test function, the testing +library applies it to each invocation _separately_ so that if only some +cases cause failures due to timeouts, then the testing library doesn't +incorrectly mark successful cases as failing. diff --git a/Sources/Testing/Traits/TimeLimitTrait.swift b/Sources/Testing/Traits/TimeLimitTrait.swift index 7b21209b6..e9216b242 100644 --- a/Sources/Testing/Traits/TimeLimitTrait.swift +++ b/Sources/Testing/Traits/TimeLimitTrait.swift @@ -83,6 +83,13 @@ extension Trait where Self == TimeLimitTrait { /// test cases individually. If a test has more than one time limit associated /// with it, the shortest one is used. A test run may also be configured with /// a maximum time limit per test case. + /// + /// If you apply this trait to a test suite, then it sets the time limit for + /// each test in the suite, or each test case in parameterized tests in the + /// suite. + /// For example, if a suite contains five tests and you apply a time limit trait + /// with a duration of one minute, then each test in the suite may run for up to + /// one minute. @_spi(Experimental) public static func timeLimit(_ timeLimit: Duration) -> Self { return Self(timeLimit: timeLimit) @@ -116,6 +123,13 @@ extension Trait where Self == TimeLimitTrait { /// If a test is parameterized, this time limit is applied to each of its /// test cases individually. If a test has more than one time limit associated /// with it, the testing library uses the shortest time limit. + /// + /// If you apply this trait to a test suite, then it sets the time limit for + /// each test in the suite, or each test case in parameterized tests in the + /// suite. + /// For example, if a suite contains five tests and you apply a time limit trait + /// with a duration of one minute, then each test in the suite may run for up to + /// one minute. public static func timeLimit(_ timeLimit: Self.Duration) -> Self { return Self(timeLimit: timeLimit.underlyingDuration) } From 3fdabe5392108d874abae1c1e58e1328ab46f681 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 6 Aug 2025 10:24:52 -0500 Subject: [PATCH 234/234] [6.2] Declare Xcode 26 availability for IssueHandlingTrait (#1252) - **Explanation**: This declares Xcode 26 availability for `IssueHandlingTrait`, which was proposed in [ST-0011: Issue Handling Traits](https://github.com/swiftlang/swift-evolution/blob/main/proposals/testing/0011-issue-handling-traits.md) and included in Swift 6.2 in #1228. - **Scope**: The `IssueHandlingTrait` API, and affects documentation only. - **Issues**: n/a - **Original PRs**: #1251 - **Risk**: Low, documentation only change - **Testing**: n/a - **Reviewers**: @briancroom --- Sources/Testing/Traits/IssueHandlingTrait.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Sources/Testing/Traits/IssueHandlingTrait.swift b/Sources/Testing/Traits/IssueHandlingTrait.swift index 4d7d408d6..21a9adaac 100644 --- a/Sources/Testing/Traits/IssueHandlingTrait.swift +++ b/Sources/Testing/Traits/IssueHandlingTrait.swift @@ -27,6 +27,7 @@ /// /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } public struct IssueHandlingTrait: TestTrait, SuiteTrait { /// A function which handles an issue and returns an optional replacement. @@ -55,6 +56,7 @@ public struct IssueHandlingTrait: TestTrait, SuiteTrait { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public func handleIssue(_ issue: Issue) -> Issue? { _handler(issue) @@ -67,6 +69,7 @@ public struct IssueHandlingTrait: TestTrait, SuiteTrait { /// @Metadata { /// @Available(Swift, introduced: 6.2) +/// @Available(Xcode, introduced: 26.0) /// } extension IssueHandlingTrait: TestScoping { public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { @@ -181,6 +184,7 @@ extension Trait where Self == IssueHandlingTrait { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static func compactMapIssues(_ transform: @escaping @Sendable (Issue) -> Issue?) -> Self { Self(handler: transform) @@ -219,6 +223,7 @@ extension Trait where Self == IssueHandlingTrait { /// /// @Metadata { /// @Available(Swift, introduced: 6.2) + /// @Available(Xcode, introduced: 26.0) /// } public static func filterIssues(_ isIncluded: @escaping @Sendable (Issue) -> Bool) -> Self { Self { issue in