diff --git a/README.md b/README.md index 9f7eb424f..8df465217 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ for various platforms: | **tvOS** | | | Supported | | **visionOS** | | | Supported | | **Ubuntu 22.04** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-linux)](https://ci.swift.org/job/swift-testing-main-swift-6.1-linux/) | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-linux)](https://ci.swift.org/view/Swift%20Packages/job/swift-testing-main-swift-main-linux/) | Supported | -| **Windows** | [![Build Status](https://ci.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-windows)](https://ci-external.swift.org/view/all/job/swift-testing-main-swift-6.1-windows/) | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-windows)](https://ci-external.swift.org/job/swift-testing-main-swift-main-windows/) | Supported | +| **Windows** | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-6.1-windows)](https://ci-external.swift.org/view/all/job/swift-testing-main-swift-6.1-windows/) | [![Build Status](https://ci-external.swift.org/buildStatus/icon?job=swift-testing-main-swift-main-windows)](https://ci-external.swift.org/job/swift-testing-main-swift-main-windows/) | Supported | | **Wasm** | | | Experimental | ### Works with XCTest diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index b4e865427..5b84aeaf3 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -36,7 +36,7 @@ add_library(Testing ExitTests/ExitTest.Condition.swift ExitTests/ExitTest.Result.swift ExitTests/SpawnProcess.swift - ExitTests/StatusAtExit.swift + ExitTests/ExitStatus.swift ExitTests/WaitFor.swift Expectations/Expectation.swift Expectations/Expectation+Macro.swift @@ -97,6 +97,7 @@ add_library(Testing Traits/ConditionTrait.swift Traits/ConditionTrait+Macro.swift Traits/HiddenTrait.swift + Traits/IssueHandlingTrait.swift Traits/ParallelizationTrait.swift Traits/Tags/Tag.Color.swift Traits/Tags/Tag.Color+Loading.swift diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index f585495a9..6abb71442 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -93,10 +93,20 @@ extension Event.HumanReadableOutputRecorder { /// - Parameters: /// - comments: The comments that should be formatted. /// - /// - Returns: A formatted string representing `comments`, or `nil` if there - /// are none. + /// - Returns: An array of formatted messages representing `comments`, or an + /// empty array if there are none. private func _formattedComments(_ comments: [Comment]) -> [Message] { - comments.map { Message(symbol: .details, stringValue: $0.rawValue) } + comments.map(_formattedComment) + } + + /// Get a string representing a single comment, formatted for output. + /// + /// - Parameters: + /// - comment: The comment that should be formatted. + /// + /// - Returns: A formatted message representing `comment`. + private func _formattedComment(_ comment: Comment) -> Message { + Message(symbol: .details, stringValue: comment.rawValue) } /// Get a string representing the comments attached to a test, formatted for @@ -449,6 +459,9 @@ extension Event.HumanReadableOutputRecorder { additionalMessages.append(Message(symbol: .difference, stringValue: differenceDescription)) } additionalMessages += _formattedComments(issue.comments) + if let knownIssueComment = issue.knownIssueContext?.comment { + additionalMessages.append(_formattedComment(knownIssueComment)) + } if verbosity > 0, case let .expectationFailed(expectation) = issue.kind { let expression = expectation.evaluatedExpression diff --git a/Sources/Testing/ExitTests/StatusAtExit.swift b/Sources/Testing/ExitTests/ExitStatus.swift similarity index 60% rename from Sources/Testing/ExitTests/StatusAtExit.swift rename to Sources/Testing/ExitTests/ExitStatus.swift index ea5e287c7..0dd6d86ab 100644 --- a/Sources/Testing/ExitTests/StatusAtExit.swift +++ b/Sources/Testing/ExitTests/ExitStatus.swift @@ -10,74 +10,94 @@ private import _TestingInternals -/// An enumeration describing possible status a process will yield on exit. +/// An enumeration describing possible status a process will report on exit. /// /// You can convert an instance of this type to an instance of /// ``ExitTest/Condition`` using ``ExitTest/Condition/init(_:)``. That value /// can then be used to describe the condition under which an exit test is /// expected to pass or fail by passing it to -/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. -@_spi(Experimental) +/// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or +/// ``require(processExitsWith:observing:_:sourceLocation:performing:)``. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -public enum StatusAtExit: Sendable { - /// The process terminated with the given exit code. +public enum ExitStatus: Sendable { + /// The process exited with the given exit code. /// /// - Parameters: - /// - exitCode: The exit code yielded by the process. + /// - exitCode: The exit code reported by the process. /// - /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), - /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their - /// own non-standard exit codes: + /// The C programming language defines two standard exit codes, `EXIT_SUCCESS` + /// and `EXIT_FAILURE`. Platforms may additionally define their own + /// non-standard exit codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man3/exit.3.html), [``](https://www.kernel.org/doc/man-pages/online/pages/man3/sysexits.h.3head.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/EXIT_status for more + /// information about exit codes defined by the C standard. + /// } + /// /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by - /// the process is yielded to the parent process. Linux and other POSIX-like + /// the process is reported to the parent process. Linux and other POSIX-like /// systems may only reliably report the low unsigned 8 bits (0–255) of /// the exit code. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } case exitCode(_ exitCode: CInt) - /// The process terminated with the given signal. + /// The process exited with the given signal. /// /// - Parameters: - /// - signal: The signal that terminated the process. + /// - signal: The signal that caused the process to exit. /// - /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). - /// Platforms may additionally define their own non-standard signal codes: + /// The C programming language defines a number of standard signals. Platforms + /// may additionally define their own non-standard signal codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man7/signal.7.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/SIG_types for more + /// information about signals defined by the C standard. + /// } + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } case signal(_ signal: CInt) } // MARK: - Equatable -@_spi(Experimental) #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -extension StatusAtExit: Equatable {} +extension ExitStatus: Equatable {} // MARK: - CustomStringConvertible @_spi(Experimental) #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif -extension StatusAtExit: CustomStringConvertible { +extension ExitStatus: CustomStringConvertible { public var description: String { switch self { case let .exitCode(exitCode): diff --git a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift index d4c84e446..1d5c9b18a 100644 --- a/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift +++ b/Sources/Testing/ExitTests/ExitTest.CapturedValue.swift @@ -26,7 +26,7 @@ extension ExitTest { /// exit test: /// /// ```swift - /// await #expect(exitsWith: .failure) { [a = a as T, b = b as U, c = c as V] in + /// await #expect(processExitsWith: .failure) { [a = a as T, b = b as U, c = c as V] in /// ... /// } /// ``` diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift index d2c637d79..f737d8cf6 100644 --- a/Sources/Testing/ExitTests/ExitTest.Condition.swift +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -10,7 +10,6 @@ private import _TestingInternals -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @@ -19,13 +18,29 @@ extension ExitTest { /// /// Values of this type are used to describe the conditions under which an /// exit test is expected to pass or fail by passing them to - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(processExitsWith:observing:_:sourceLocation:performing:)``. + /// + /// ## Topics + /// + /// ### Successful exit conditions + /// + /// - ``success`` + /// + /// ### Failing exit conditions + /// + /// - ``failure`` + /// - ``exitCode(_:)`` + /// - ``signal(_:)`` + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public struct Condition: Sendable { /// An enumeration describing the possible conditions for an exit test. private enum _Kind: Sendable, Equatable { /// The exit test must exit with a particular exit status. - case statusAtExit(StatusAtExit) + case exitStatus(ExitStatus) /// The exit test must exit successfully. case success @@ -41,49 +56,77 @@ extension ExitTest { // MARK: - -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif extension ExitTest.Condition { - /// A condition that matches when a process terminates successfully with exit - /// code `EXIT_SUCCESS`. + /// A condition that matches when a process exits normally. + /// + /// This condition matches the exit code `EXIT_SUCCESS`. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static var success: Self { Self(_kind: .success) } - /// A condition that matches when a process terminates abnormally with any - /// exit code other than `EXIT_SUCCESS` or with any signal. + /// A condition that matches when a process exits abnormally + /// + /// This condition matches any exit code other than `EXIT_SUCCESS` or any + /// signal that causes the process to exit. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static var failure: Self { Self(_kind: .failure) } - public init(_ statusAtExit: StatusAtExit) { - self.init(_kind: .statusAtExit(statusAtExit)) + /// Initialize an instance of this type that matches the specified exit + /// status. + /// + /// - Parameters: + /// - exitStatus: The particular exit status this condition should match. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } + public init(_ exitStatus: ExitStatus) { + self.init(_kind: .exitStatus(exitStatus)) } /// Creates a condition that matches when a process terminates with a given /// exit code. /// /// - Parameters: - /// - exitCode: The exit code yielded by the process. + /// - exitCode: The exit code reported by the process. /// - /// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status), - /// `EXIT_SUCCESS` and `EXIT_FAILURE`. Platforms may additionally define their - /// own non-standard exit codes: + /// The C programming language defines two standard exit codes, `EXIT_SUCCESS` + /// and `EXIT_FAILURE`. Platforms may additionally define their own + /// non-standard exit codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/_Exit.3.html), [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysexits.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `` | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man3/exit.3.html), [``](https://www.kernel.org/doc/man-pages/online/pages/man3/sysexits.h.3head.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?exit(3)), [``](https://man.freebsd.org/cgi/man.cgi?sysexits(3)) | /// | OpenBSD | [``](https://man.openbsd.org/exit.3), [``](https://man.openbsd.org/sysexits.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) | /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/EXIT_status for more + /// information about exit codes defined by the C standard. + /// } + /// /// On macOS, FreeBSD, OpenBSD, and Windows, the full exit code reported by - /// the process is yielded to the parent process. Linux and other POSIX-like + /// the process is reported to the parent process. Linux and other POSIX-like /// systems may only reliably report the low unsigned 8 bits (0–255) of /// the exit code. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func exitCode(_ exitCode: CInt) -> Self { #if !SWT_NO_EXIT_TESTS Self(.exitCode(exitCode)) @@ -92,22 +135,30 @@ extension ExitTest.Condition { #endif } - /// Creates a condition that matches when a process terminates with a given - /// signal. + /// Creates a condition that matches when a process exits with a given signal. /// /// - Parameters: - /// - signal: The signal that terminated the process. + /// - signal: The signal that caused the process to exit. /// - /// The C programming language defines a number of [standard signals](https://en.cppreference.com/w/c/program/SIG_types). - /// Platforms may additionally define their own non-standard signal codes: + /// The C programming language defines a number of standard signals. Platforms + /// may additionally define their own non-standard signal codes: /// /// | Platform | Header | /// |-|-| /// | macOS | [``](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) | - /// | Linux | [``](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) | + /// | Linux | [``](https://www.kernel.org/doc/man-pages/online/pages/man7/signal.7.html) | /// | FreeBSD | [``](https://man.freebsd.org/cgi/man.cgi?signal(3)) | /// | OpenBSD | [``](https://man.openbsd.org/signal.3) | /// | Windows | [``](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) | + /// + /// @Comment { + /// See https://en.cppreference.com/w/c/program/SIG_types for more + /// information about signals defined by the C standard. + /// } + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static func signal(_ signal: CInt) -> Self { #if !SWT_NO_EXIT_TESTS Self(.signal(signal)) @@ -131,8 +182,8 @@ extension ExitTest.Condition: CustomStringConvertible { ".failure" case .success: ".success" - case let .statusAtExit(statusAtExit): - String(describing: statusAtExit) + case let .exitStatus(exitStatus): + String(describing: exitStatus) } #else fatalError("Unsupported") @@ -149,19 +200,19 @@ extension ExitTest.Condition { /// Check whether or not an exit test condition matches a given exit status. /// /// - Parameters: - /// - statusAtExit: An exit status to compare against. + /// - exitStatus: An exit status to compare against. /// - /// - Returns: Whether or not `self` and `statusAtExit` represent the same - /// exit condition. + /// - Returns: Whether or not `self` and `exitStatus` represent the same exit + /// condition. /// /// Two exit test conditions can be compared; if either instance is equal to /// ``failure``, it will compare equal to any instance except ``success``. - func isApproximatelyEqual(to statusAtExit: StatusAtExit) -> Bool { + func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool { // Strictly speaking, the C standard treats 0 as a successful exit code and // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern // operating system defines EXIT_SUCCESS to any value other than 0, so the // distinction is academic. - return switch (self._kind, statusAtExit) { + return switch (self._kind, exitStatus) { case let (.success, .exitCode(exitCode)): exitCode == EXIT_SUCCESS case let (.failure, .exitCode(exitCode)): @@ -170,7 +221,7 @@ extension ExitTest.Condition { // All terminating signals are considered failures. true default: - self._kind == .statusAtExit(statusAtExit) + self._kind == .exitStatus(exitStatus) } } } diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift index beb2d56fc..ef70a3789 100644 --- a/Sources/Testing/ExitTests/ExitTest.Result.swift +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -8,7 +8,6 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @@ -16,15 +15,20 @@ extension ExitTest { /// A type representing the result of an exit test after it has exited and /// returned control to the calling test function. /// - /// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and - /// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return - /// instances of this type. + /// Both ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` + /// and ``require(processExitsWith:observing:_:sourceLocation:performing:)`` + /// return instances of this type. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public struct Result: Sendable { - /// The exit condition the exit test exited with. + /// The exit status reported by the process hosting the exit test. /// - /// When the exit test passes, the value of this property is equal to the - /// exit status reported by the process that hosted the exit test. - public var statusAtExit: StatusAtExit + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } + public var exitStatus: ExitStatus /// All bytes written to the standard output stream of the exit test before /// it exited. @@ -45,11 +49,15 @@ extension ExitTest { /// /// To enable gathering output from the standard output stream during an /// exit test, pass `\.standardOutputContent` in the `observedValues` - /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)`` - /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// argument of ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` + /// or ``require(processExitsWith:observing:_:sourceLocation:performing:)``. /// /// If you did not request standard output content when running an exit /// test, the value of this property is the empty array. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public var standardOutputContent: [UInt8] = [] /// All bytes written to the standard error stream of the exit test before @@ -71,16 +79,20 @@ extension ExitTest { /// /// To enable gathering output from the standard error stream during an exit /// test, pass `\.standardErrorContent` in the `observedValues` argument of - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(processExitsWith:observing:_:sourceLocation:performing:)``. /// /// If you did not request standard error content when running an exit test, /// the value of this property is the empty array. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public var standardErrorContent: [UInt8] = [] @_spi(ForToolsIntegrationOnly) - public init(statusAtExit: StatusAtExit) { - self.statusAtExit = statusAtExit + public init(exitStatus: ExitStatus) { + self.exitStatus = exitStatus } } } diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 2ad905379..1e9c29c15 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -26,10 +26,13 @@ private import _TestingInternals /// A type describing an exit test. /// /// Instances of this type describe exit tests you create using the -/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` -/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` macro. You -/// don't usually need to interact directly with an instance of this type. -@_spi(Experimental) +/// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or +/// ``require(processExitsWith:observing:_:sourceLocation:performing:)`` macro. +/// You don't usually need to interact directly with an instance of this type. +/// +/// @Metadata { +/// @Available(Swift, introduced: 6.2) +/// } #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @@ -90,12 +93,12 @@ public struct ExitTest: Sendable, ~Copyable { /// observed and returned to the caller. /// /// The testing library sets this property to match what was passed by the - /// developer to the `#expect(exitsWith:)` or `#require(exitsWith:)` macro. - /// If you are implementing an exit test handler, you can check the value of - /// this property to determine what information you need to preserve from your - /// child process. + /// developer to the `#expect(processExitsWith:)` or `#require(processExitsWith:)` + /// macro. If you are implementing an exit test handler, you can check the + /// value of this property to determine what information you need to preserve + /// from your child process. /// - /// The value of this property always includes ``ExitTest/Result/statusAtExit`` + /// The value of this property always includes ``ExitTest/Result/exitStatus`` /// even if the test author does not specify it. /// /// Within a child process running an exit test, the value of this property is @@ -104,8 +107,8 @@ public struct ExitTest: Sendable, ~Copyable { public var observedValues: [any PartialKeyPath & Sendable] { get { var result = _observedValues - if !result.contains(\.statusAtExit) { // O(n), but n <= 3 (no Set needed) - result.append(\.statusAtExit) + if !result.contains(\.exitStatus) { // O(n), but n <= 3 (no Set needed) + result.append(\.exitStatus) } return result } @@ -144,7 +147,6 @@ public struct ExitTest: Sendable, ~Copyable { #if !SWT_NO_EXIT_TESTS // MARK: - Current -@_spi(Experimental) extension ExitTest { /// Storage for ``current``. /// @@ -165,6 +167,10 @@ extension ExitTest { /// /// The value of this property is constant across all tasks in the current /// process. + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public static var current: ExitTest? { _read { // NOTE: Even though this accessor is `_read` and has borrowing semantics, @@ -180,7 +186,7 @@ extension ExitTest { // MARK: - Invocation -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(ForToolsIntegrationOnly) extension ExitTest { /// Disable crash reporting, crash logging, or core dumps for the current /// process. @@ -228,9 +234,9 @@ extension ExitTest { /// Call the exit test in the current process. /// /// This function invokes the closure originally passed to - /// `#expect(exitsWith:)` _in the current process_. That closure is expected - /// to terminate the process; if it does not, the testing library will - /// terminate the process as if its `main()` function returned naturally. + /// `#expect(processExitsWith:)` _in the current process_. That closure is + /// expected to terminate the process; if it does not, the testing library + /// will terminate the process as if its `main()` function returned naturally. public consuming func callAsFunction() async -> Never { Self._disableCrashReporting() @@ -319,8 +325,8 @@ extension ExitTest { /// /// - Returns: Whether or not an exit test was stored into `outValue`. /// - /// - Warning: This function is used to implement the `#expect(exitsWith:)` - /// macro. Do not use it directly. + /// - Warning: This function is used to implement the + /// `#expect(processExitsWith:)` macro. Do not use it directly. public static func __store( _ id: (UInt64, UInt64, UInt64, UInt64), _ body: @escaping @Sendable (repeat each T) async throws -> Void, @@ -356,7 +362,7 @@ extension ExitTest { } } -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(ForToolsIntegrationOnly) extension ExitTest { /// Find the exit test function at the given source location. /// @@ -396,7 +402,7 @@ extension ExitTest { /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTest/Result/statusAtExit`` property is always returned. +/// ``ExitTest/Result/exitStatus`` property is always returned. /// - expression: The expression, corresponding to `condition`, that is being /// evaluated (if available at compile time.) /// - comments: An array of comments describing the expectation. This array @@ -408,12 +414,12 @@ extension ExitTest { /// - sourceLocation: The source location of the expectation. /// /// This function contains the common implementation for all -/// `await #expect(exitsWith:) { }` invocations regardless of calling +/// `await #expect(processExitsWith:) { }` invocations regardless of calling /// convention. func callExitTest( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), encodingCapturedValues capturedValues: [ExitTest.CapturedValue], - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable], expression: __Expression, comments: @autoclosure () -> [Comment], @@ -422,7 +428,7 @@ func callExitTest( sourceLocation: SourceLocation ) async -> Result { guard let configuration = Configuration.current ?? Configuration.all.first else { - preconditionFailure("A test must be running on the current task to use #expect(exitsWith:).") + preconditionFailure("A test must be running on the current task to use #expect(processExitsWith:).") } var result: ExitTest.Result @@ -438,8 +444,8 @@ func callExitTest( #if os(Windows) // For an explanation of this magic, see the corresponding logic in // ExitTest.callAsFunction(). - if case let .exitCode(exitCode) = result.statusAtExit, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { - result.statusAtExit = .signal(exitCode & STATUS_CODE_MASK) + if case let .exitCode(exitCode) = result.exitStatus, (exitCode & ~STATUS_CODE_MASK) == STATUS_SIGNAL_CAUGHT_BITS { + result.exitStatus = .signal(exitCode & STATUS_CODE_MASK) } #endif } catch { @@ -464,22 +470,19 @@ func callExitTest( // For lack of a better way to handle an exit test failing in this way, // we record the system issue above, then let the expectation fail below by // reporting an exit condition that's the inverse of the expected one. - let statusAtExit: StatusAtExit = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) { + let exitStatus: ExitStatus = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) { .exitCode(EXIT_SUCCESS) } else { .exitCode(EXIT_FAILURE) } - result = ExitTest.Result(statusAtExit: statusAtExit) + result = ExitTest.Result(exitStatus: exitStatus) } - // How did the exit test actually exit? - let actualStatusAtExit = result.statusAtExit - // Plumb the exit test's result through the general expectation machinery. return __checkValue( - expectedExitCondition.isApproximatelyEqual(to: actualStatusAtExit), + expectedExitCondition.isApproximatelyEqual(to: result.exitStatus), expression: expression, - expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualStatusAtExit), + expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(result.exitStatus), mismatchedExitConditionDescription: String(describingForTest: expectedExitCondition), comments: comments(), isRequired: isRequired, @@ -499,7 +502,7 @@ extension ABI { fileprivate typealias BackChannelVersion = v1 } -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(ForToolsIntegrationOnly) extension ExitTest { /// A handler that is invoked when an exit test starts. /// @@ -513,11 +516,10 @@ extension ExitTest { /// the exit test. /// /// This handler is invoked when an exit test (i.e. a call to either - /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:observing:_:sourceLocation:performing:)``) is started. - /// The handler is responsible for initializing a new child environment - /// (typically a child process) and running the exit test identified by - /// `sourceLocation` there. + /// ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(processExitsWith:observing:_:sourceLocation:performing:)``) is + /// started. The handler is responsible for initializing a new child + /// environment (typically a child process) and running `exitTest` there. /// /// In the child environment, you can find the exit test again by calling /// ``ExitTest/find(at:)`` and can run it by calling @@ -615,28 +617,23 @@ extension ExitTest { static func findInEnvironmentForEntryPoint() -> Self? { // Find the ID of the exit test to run, if any, in the environment block. var id: ExitTest.ID? - if var idString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") { + if var idString = Environment.variable(named: "SWT_EXIT_TEST_ID") { // Clear the environment variable. It's an implementation detail and exit // test code shouldn't be dependent on it. Use ExitTest.current if needed! - Environment.setVariable(nil, named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") + Environment.setVariable(nil, named: "SWT_EXIT_TEST_ID") id = try? idString.withUTF8 { idBuffer in try JSON.decode(ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer)) } } - guard let id else { + guard let id, var result = find(identifiedBy: id) else { return nil } // If an exit test was found, inject back channel handling into its body. // External tools authors should set up their own back channel mechanisms // and ensure they're installed before calling ExitTest.callAsFunction(). - guard var result = find(identifiedBy: id) else { - return nil - } - - // We can't say guard let here because it counts as a consume. - guard let backChannel = _makeFileHandle(forEnvironmentVariableNamed: "SWT_EXPERIMENTAL_BACKCHANNEL", mode: "wb") else { + guard let backChannel = _makeFileHandle(forEnvironmentVariableNamed: "SWT_BACKCHANNEL", mode: "wb") else { return result } @@ -755,7 +752,7 @@ extension ExitTest { // Insert a specific variable that tells the child process which exit test // to run. try JSON.withEncoding(of: exitTest.id) { json in - childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) + childEnvironment["SWT_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self) } typealias ResultUpdater = @Sendable (inout ExitTest.Result) -> Void @@ -795,7 +792,7 @@ extension ExitTest { // captured values channel by setting a known environment variable to // the corresponding file descriptor (HANDLE on Windows) for each. if let backChannelEnvironmentVariable = _makeEnvironmentVariable(for: backChannelWriteEnd) { - childEnvironment["SWT_EXPERIMENTAL_BACKCHANNEL"] = backChannelEnvironmentVariable + childEnvironment["SWT_BACKCHANNEL"] = backChannelEnvironmentVariable } if let capturedValuesEnvironmentVariable = _makeEnvironmentVariable(for: capturedValuesReadEnd) { childEnvironment["SWT_EXPERIMENTAL_CAPTURED_VALUES"] = capturedValuesEnvironmentVariable @@ -831,8 +828,8 @@ extension ExitTest { // Await termination of the child process. taskGroup.addTask { - let statusAtExit = try await wait(for: processID) - return { $0.statusAtExit = statusAtExit } + let exitStatus = try await wait(for: processID) + return { $0.exitStatus = exitStatus } } // Read back the stdout and stderr streams. @@ -862,7 +859,7 @@ extension ExitTest { // Collate the various bits of the result. The exit condition used here // is just a placeholder and will be replaced by the result of one of // the tasks above. - var result = ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) + var result = ExitTest.Result(exitStatus: .exitCode(EXIT_FAILURE)) for try await update in taskGroup { update?(&result) } @@ -930,7 +927,11 @@ extension ExitTest { sourceLocation: issue.sourceLocation ) var issueCopy = Issue(kind: issueKind, comments: comments, sourceContext: sourceContext) - issueCopy.isKnown = issue.isKnown + if issue.isKnown { + // The known issue comment, if there was one, is already included in + // the `comments` array above. + issueCopy.knownIssueContext = Issue.KnownIssueContext() + } issueCopy.record() } } diff --git a/Sources/Testing/ExitTests/WaitFor.swift b/Sources/Testing/ExitTests/WaitFor.swift index cc611158f..238ed835a 100644 --- a/Sources/Testing/ExitTests/WaitFor.swift +++ b/Sources/Testing/ExitTests/WaitFor.swift @@ -20,7 +20,7 @@ internal import _TestingInternals /// /// - Throws: If the exit status of the process with ID `pid` cannot be /// determined (i.e. it does not represent an exit condition.) -private func _blockAndWait(for pid: consuming pid_t) throws -> StatusAtExit { +private func _blockAndWait(for pid: consuming pid_t) throws -> ExitStatus { let pid = consume pid // Get the exit status of the process or throw an error (other than EINTR.) @@ -61,7 +61,7 @@ private func _blockAndWait(for pid: consuming pid_t) throws -> StatusAtExit { /// - Note: The open-source implementation of libdispatch available on Linux /// and other platforms does not support `DispatchSourceProcess`. Those /// platforms use an alternate implementation below. -func wait(for pid: consuming pid_t) async throws -> StatusAtExit { +func wait(for pid: consuming pid_t) async throws -> ExitStatus { let pid = consume pid let source = DispatchSource.makeProcessSource(identifier: pid, eventMask: .exit) @@ -80,7 +80,7 @@ func wait(for pid: consuming pid_t) async throws -> StatusAtExit { } #elseif SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) /// A mapping of awaited child PIDs to their corresponding Swift continuations. -private let _childProcessContinuations = LockedWith]>() +private let _childProcessContinuations = LockedWith]>() /// A condition variable used to suspend the waiter thread created by /// `_createWaitThread()` when there are no child processes to await. @@ -202,7 +202,7 @@ private let _createWaitThread: Void = { /// /// On Apple platforms, the libdispatch-based implementation above is more /// efficient because it does not need to permanently reserve a thread. -func wait(for pid: consuming pid_t) async throws -> StatusAtExit { +func wait(for pid: consuming pid_t) async throws -> ExitStatus { let pid = consume pid // Ensure the waiter thread is running. @@ -239,7 +239,7 @@ func wait(for pid: consuming pid_t) async throws -> StatusAtExit { /// This implementation of `wait(for:)` calls `RegisterWaitForSingleObject()` to /// wait for `processHandle`, suspends the calling task until the waiter's /// callback is called, then calls `GetExitCodeProcess()`. -func wait(for processHandle: consuming HANDLE) async throws -> StatusAtExit { +func wait(for processHandle: consuming HANDLE) async throws -> ExitStatus { let processHandle = consume processHandle defer { _ = CloseHandle(processHandle) @@ -283,6 +283,6 @@ func wait(for processHandle: consuming HANDLE) async throws -> StatusAtExit { } #else #warning("Platform-specific implementation missing: cannot wait for child processes to exit") -func wait(for processID: consuming Never) async throws -> StatusAtExit {} +func wait(for processID: consuming Never) async throws -> ExitStatus {} #endif #endif diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 5111fbddd..d14920547 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -490,7 +490,7 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTest/Result/statusAtExit`` property is always returned. +/// ``ExitTest/Result/exitStatus`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. @@ -506,89 +506,20 @@ public macro require( /// causes a process to terminate: /// /// ```swift -/// await #expect(exitsWith: .failure) { +/// await #expect(processExitsWith: .failure) { /// fatalError() /// } /// ``` /// -/// - Note: A call to this expectation macro is called an "exit test." -/// -/// ## How exit tests are run -/// -/// When an exit test is performed at runtime, the testing library starts a new -/// process with the same executable as the current process. The current task is -/// then suspended (as with `await`) and waits for the child process to -/// terminate. `expression` is not called in the parent process. -/// -/// Meanwhile, in the child process, `expression` is called directly. To ensure -/// a clean environment for execution, it is not called within the context of -/// the original test. If `expression` does not terminate the child process, the -/// process is terminated automatically as if the main function of the child -/// process were allowed to return naturally. If an error is thrown from -/// `expression`, it is handed as if the error were thrown from `main()` and the -/// process is terminated. -/// -/// Once the child process terminates, the parent process resumes and compares -/// its exit status against `expectedExitCondition`. If they match, the exit -/// test has passed; otherwise, it has failed and an issue is recorded. -/// -/// ## Child process output -/// -/// By default, the child process is configured without a standard output or -/// standard error stream. If your test needs to review the content of either of -/// these streams, you can pass its key path in the `observedValues` argument: -/// -/// ```swift -/// let result = await #expect( -/// exitsWith: .failure, -/// observing: [\.standardOutputContent] -/// ) { -/// print("Goodbye, world!") -/// fatalError() -/// } -/// if let result { -/// #expect(result.standardOutputContent.contains(UInt8(ascii: "G"))) -/// } -/// ``` -/// -/// - Note: The content of the standard output and standard error streams may -/// contain any arbitrary sequence of bytes, including sequences that are not -/// valid UTF-8 and cannot be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). -/// These streams are globally accessible within the child process, and any -/// code running in an exit test may write to it including the operating -/// system and any third-party dependencies you have declared in your package. -/// -/// The actual exit condition of the child process is always reported by the -/// testing library even if you do not specify it in `observedValues`. -/// -/// ## Runtime constraints -/// -/// Exit tests cannot capture any state originating in the parent process or -/// from the enclosing lexical context. For example, the following exit test -/// will fail to compile because it captures an argument to the enclosing -/// parameterized test: -/// -/// ```swift -/// @Test(arguments: 100 ..< 200) -/// func sellIceCreamCones(count: Int) async { -/// await #expect(exitsWith: .failure) { -/// precondition( -/// count < 10, // ERROR: A C function pointer cannot be formed from a -/// // closure that captures context -/// "Too many ice cream cones" -/// ) -/// } +/// @Metadata { +/// @Available(Swift, introduced: 6.2) /// } -/// ``` -/// -/// An exit test cannot run within another exit test. -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @discardableResult @freestanding(expression) public macro expect( - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, @@ -602,7 +533,7 @@ public macro require( /// - expectedExitCondition: The expected exit condition. /// - observedValues: An array of key paths representing results from within /// the exit test that should be observed and returned by this macro. The -/// ``ExitTest/Result/statusAtExit`` property is always returned. +/// ``ExitTest/Result/exitStatus`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. @@ -620,87 +551,20 @@ public macro require( /// causes a process to terminate: /// /// ```swift -/// try await #require(exitsWith: .failure) { +/// try await #require(processExitsWith: .failure) { /// fatalError() /// } /// ``` /// -/// - Note: A call to this expectation macro is called an "exit test." -/// -/// ## How exit tests are run -/// -/// When an exit test is performed at runtime, the testing library starts a new -/// process with the same executable as the current process. The current task is -/// then suspended (as with `await`) and waits for the child process to -/// terminate. `expression` is not called in the parent process. -/// -/// Meanwhile, in the child process, `expression` is called directly. To ensure -/// a clean environment for execution, it is not called within the context of -/// the original test. If `expression` does not terminate the child process, the -/// process is terminated automatically as if the main function of the child -/// process were allowed to return naturally. If an error is thrown from -/// `expression`, it is handed as if the error were thrown from `main()` and the -/// process is terminated. -/// -/// Once the child process terminates, the parent process resumes and compares -/// its exit status against `expectedExitCondition`. If they match, the exit -/// test has passed; otherwise, it has failed and an issue is recorded. -/// -/// ## Child process output -/// -/// By default, the child process is configured without a standard output or -/// standard error stream. If your test needs to review the content of either of -/// these streams, you can pass its key path in the `observedValues` argument: -/// -/// ```swift -/// let result = try await #require( -/// exitsWith: .failure, -/// observing: [\.standardOutputContent] -/// ) { -/// print("Goodbye, world!") -/// fatalError() -/// } -/// #expect(result.standardOutputContent.contains(UInt8(ascii: "G"))) -/// ``` -/// -/// - Note: The content of the standard output and standard error streams may -/// contain any arbitrary sequence of bytes, including sequences that are not -/// valid UTF-8 and cannot be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). -/// These streams are globally accessible within the child process, and any -/// code running in an exit test may write to it including the operating -/// system and any third-party dependencies you have declared in your package. -/// -/// The actual exit condition of the child process is always reported by the -/// testing library even if you do not specify it in `observedValues`. -/// -/// ## Runtime constraints -/// -/// Exit tests cannot capture any state originating in the parent process or -/// from the enclosing lexical context. For example, the following exit test -/// will fail to compile because it captures an argument to the enclosing -/// parameterized test: -/// -/// ```swift -/// @Test(arguments: 100 ..< 200) -/// func sellIceCreamCones(count: Int) async throws { -/// try await #require(exitsWith: .failure) { -/// precondition( -/// count < 10, // ERROR: A C function pointer cannot be formed from a -/// // closure that captures context -/// "Too many ice cream cones" -/// ) -/// } +/// @Metadata { +/// @Available(Swift, introduced: 6.2) /// } -/// ``` -/// -/// An exit test cannot run within another exit test. -@_spi(Experimental) #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif @discardableResult @freestanding(expression) public macro require( - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index e8767d01f..6d3093f2a 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1139,15 +1139,14 @@ public func __checkClosureCall( /// Check that an expression always exits (terminates the current process) with /// a given status. /// -/// This overload is used for `await #expect(exitsWith:) { }` invocations that -/// do not capture any state. +/// This overload is used for `await #expect(processExitsWith:) { }` invocations +/// that do not capture any state. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. -@_spi(Experimental) public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], performing _: @convention(thin) () -> Void, expression: __Expression, @@ -1159,7 +1158,7 @@ public func __checkClosureCall( await callExitTest( identifiedBy: exitTestID, encodingCapturedValues: [], - exitsWith: expectedExitCondition, + processExitsWith: expectedExitCondition, observing: observedValues, expression: expression, comments: comments(), @@ -1171,8 +1170,8 @@ public func __checkClosureCall( /// Check that an expression always exits (terminates the current process) with /// a given status. /// -/// This overload is used for `await #expect(exitsWith:) { }` invocations that -/// capture some values with an explicit capture list. +/// This overload is used for `await #expect(processExitsWith:) { }` invocations +/// that capture some values with an explicit capture list. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. @@ -1180,7 +1179,7 @@ public func __checkClosureCall( public func __checkClosureCall( identifiedBy exitTestID: (UInt64, UInt64, UInt64, UInt64), encodingCapturedValues capturedValues: (repeat each T), - exitsWith expectedExitCondition: ExitTest.Condition, + processExitsWith expectedExitCondition: ExitTest.Condition, observing observedValues: [any PartialKeyPath & Sendable] = [], performing _: @convention(thin) () -> Void, expression: __Expression, @@ -1192,7 +1191,7 @@ public func __checkClosureCall( await callExitTest( identifiedBy: exitTestID, encodingCapturedValues: Array(repeat each capturedValues), - exitsWith: expectedExitCondition, + processExitsWith: expectedExitCondition, observing: observedValues, expression: expression, comments: comments(), diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index bd3e9a3bb..b79e94269 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -9,14 +9,6 @@ // extension Issue { - /// The known issue matcher, as set by `withKnownIssue()`, associated with the - /// current task. - /// - /// If there is no call to `withKnownIssue()` executing on the current task, - /// the value of this property is `nil`. - @TaskLocal - static var currentKnownIssueMatcher: KnownIssueMatcher? - /// Record this issue by wrapping it in an ``Event`` and passing it to the /// current event handler. /// @@ -38,9 +30,9 @@ extension Issue { // If this issue matches via the known issue matcher, set a copy of it to be // known and record the copy instead. - if !isKnown, let issueMatcher = Self.currentKnownIssueMatcher, issueMatcher(self) { + if !isKnown, let context = KnownIssueScope.current?.matcher(self) { var selfCopy = self - selfCopy.isKnown = true + selfCopy.knownIssueContext = context return selfCopy.record(configuration: configuration) } diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 53364c151..4a17cb945 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -129,9 +129,29 @@ public struct Issue: Sendable { @_spi(ForToolsIntegrationOnly) public var sourceContext: SourceContext + /// A type representing a + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` call + /// that matched an issue. + @_spi(ForToolsIntegrationOnly) + public struct KnownIssueContext: Sendable { + /// The comment that was passed to + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)``. + public var comment: Comment? + } + + /// A ``KnownIssueContext-swift.struct`` representing the + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` call + /// that matched this issue, if any. + @_spi(ForToolsIntegrationOnly) + public var knownIssueContext: KnownIssueContext? = nil + /// Whether or not this issue is known to occur. @_spi(ForToolsIntegrationOnly) - public var isKnown: Bool = false + public var isKnown: Bool { + get { knownIssueContext != nil } + @available(*, deprecated, message: "Setting this property has no effect.") + set {} + } /// Initialize an issue instance with the specified details. /// diff --git a/Sources/Testing/Issues/KnownIssue.swift b/Sources/Testing/Issues/KnownIssue.swift index f59e9d422..f59185388 100644 --- a/Sources/Testing/Issues/KnownIssue.swift +++ b/Sources/Testing/Issues/KnownIssue.swift @@ -8,25 +8,62 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// Combine an instance of ``KnownIssueMatcher`` with any previously-set one. -/// -/// - Parameters: -/// - issueMatcher: A function to invoke when an issue occurs that is used to -/// determine if the issue is known to occur. -/// - matchCounter: The counter responsible for tracking the number of matches -/// found with `issueMatcher`. -/// -/// - Returns: A new instance of ``Configuration`` or `nil` if there was no -/// current configuration set. -private func _combineIssueMatcher(_ issueMatcher: @escaping KnownIssueMatcher, matchesCountedBy matchCounter: Locked) -> KnownIssueMatcher { - let oldIssueMatcher = Issue.currentKnownIssueMatcher - return { issue in - if issueMatcher(issue) || true == oldIssueMatcher?(issue) { - matchCounter.increment() - return true +/// A type that represents an active +/// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` +/// call and any parent calls. +/// +/// A stack of these is stored in `KnownIssueScope.current`. +struct KnownIssueScope: Sendable { + /// A function which determines if an issue matches a known issue scope or + /// any of its ancestor scopes. + /// + /// - Parameters: + /// - issue: The issue being matched. + /// + /// - Returns: A known issue context containing information about the known + /// issue, if the issue is considered "known" by this known issue scope or any + /// ancestor scope, or `nil` otherwise. + typealias Matcher = @Sendable (_ issue: Issue) -> Issue.KnownIssueContext? + + /// The matcher function for this known issue scope. + var matcher: Matcher + + /// The number of issues this scope and its ancestors have matched. + let matchCounter: Locked + + /// Create a new ``KnownIssueScope`` by combining a new issue matcher with + /// any already-active scope. + /// + /// - Parameters: + /// - parent: The context that should be checked next if `issueMatcher` + /// fails to match an issue. Defaults to ``KnownIssueScope.current``. + /// - issueMatcher: A function to invoke when an issue occurs that is used + /// to determine if the issue is known to occur. + /// - context: The context to be associated with issues matched by + /// `issueMatcher`. + init(parent: KnownIssueScope? = .current, issueMatcher: @escaping KnownIssueMatcher, context: Issue.KnownIssueContext) { + let matchCounter = Locked(rawValue: 0) + self.matchCounter = matchCounter + matcher = { issue in + let matchedContext = if issueMatcher(issue) { + context + } else { + parent?.matcher(issue) + } + if matchedContext != nil { + matchCounter.increment() + } + return matchedContext } - return false } + + /// The active known issue scope for the current task, if any. + /// + /// If there is no call to + /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` + /// executing on the current task, the value of this property is `nil`. + @TaskLocal + static var current: KnownIssueScope? } /// Check if an error matches using an issue-matching function, and throw it if @@ -34,18 +71,17 @@ private func _combineIssueMatcher(_ issueMatcher: @escaping KnownIssueMatcher, m /// /// - Parameters: /// - error: The error to test. -/// - issueMatcher: A function to which `error` is passed (after boxing it in -/// an instance of ``Issue``) to determine if it is known to occur. +/// - scope: The known issue scope that is processing the error. /// - comment: An optional comment to apply to any issues generated by this /// function. /// - sourceLocation: The source location to which the issue should be /// attributed. -private func _matchError(_ error: any Error, using issueMatcher: KnownIssueMatcher, comment: Comment?, sourceLocation: SourceLocation) throws { +private func _matchError(_ error: any Error, in scope: KnownIssueScope, comment: Comment?, sourceLocation: SourceLocation) throws { let sourceContext = SourceContext(backtrace: Backtrace(forFirstThrowOf: error), sourceLocation: sourceLocation) - var issue = Issue(kind: .errorCaught(error), comments: Array(comment), sourceContext: sourceContext) - if issueMatcher(issue) { + var issue = Issue(kind: .errorCaught(error), comments: [], sourceContext: sourceContext) + if let context = scope.matcher(issue) { // It's a known issue, so mark it as such before recording it. - issue.isKnown = true + issue.knownIssueContext = context issue.record() } else { // Rethrow the error, allowing the caller to catch it or for it to propagate @@ -184,18 +220,17 @@ public func withKnownIssue( guard precondition() else { return try body() } - let matchCounter = Locked(rawValue: 0) - let issueMatcher = _combineIssueMatcher(issueMatcher, matchesCountedBy: matchCounter) + let scope = KnownIssueScope(issueMatcher: issueMatcher, context: Issue.KnownIssueContext(comment: comment)) defer { if !isIntermittent { - _handleMiscount(by: matchCounter, comment: comment, sourceLocation: sourceLocation) + _handleMiscount(by: scope.matchCounter, comment: comment, sourceLocation: sourceLocation) } } - try Issue.$currentKnownIssueMatcher.withValue(issueMatcher) { + try KnownIssueScope.$current.withValue(scope) { do { try body() } catch { - try _matchError(error, using: issueMatcher, comment: comment, sourceLocation: sourceLocation) + try _matchError(error, in: scope, comment: comment, sourceLocation: sourceLocation) } } } @@ -304,18 +339,17 @@ public func withKnownIssue( guard await precondition() else { return try await body() } - let matchCounter = Locked(rawValue: 0) - let issueMatcher = _combineIssueMatcher(issueMatcher, matchesCountedBy: matchCounter) + let scope = KnownIssueScope(issueMatcher: issueMatcher, context: Issue.KnownIssueContext(comment: comment)) defer { if !isIntermittent { - _handleMiscount(by: matchCounter, comment: comment, sourceLocation: sourceLocation) + _handleMiscount(by: scope.matchCounter, comment: comment, sourceLocation: sourceLocation) } } - try await Issue.$currentKnownIssueMatcher.withValue(issueMatcher) { + try await KnownIssueScope.$current.withValue(scope) { do { try await body() } catch { - try _matchError(error, using: issueMatcher, comment: comment, sourceLocation: sourceLocation) + try _matchError(error, in: scope, comment: comment, sourceLocation: sourceLocation) } } } diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index b8c48aa79..bca788ec7 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -217,7 +217,6 @@ public struct Configuration: Sendable { /// When using the `swift test` command from Swift Package Manager, this /// property is pre-configured. Otherwise, the default value of this property /// records an issue indicating that it has not been configured. - @_spi(Experimental) public var exitTestHandler: ExitTest.Handler = { exitTest in throw SystemError(description: "Exit test support has not been implemented by the current testing infrastructure.") } diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 16eff103f..8520d1aaf 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -91,10 +91,11 @@ extension Runner { return try await body() } - // Construct a recursive function that invokes each trait's ``execute(_:for:testCase:)`` - // function. The order of the sequence is reversed so that the last trait is - // the one that invokes body, then the second-to-last invokes the last, etc. - // and ultimately the first trait is the first one to be invoked. + // Construct a recursive function that invokes each scope provider's + // `provideScope(for:testCase:performing:)` function. The order of the + // sequence is reversed so that the last trait is the one that invokes body, + // then the second-to-last invokes the last, etc. and ultimately the first + // trait is the first one to be invoked. let executeAllTraits = test.traits.lazy .reversed() .compactMap { $0.scopeProvider(for: test, testCase: testCase) } @@ -108,6 +109,41 @@ extension Runner { try await executeAllTraits() } + /// Apply the custom scope from any issue handling traits for the specified + /// test. + /// + /// - Parameters: + /// - test: The test being run, for which to apply its issue handling traits. + /// - body: A function to execute within the scope provided by the test's + /// issue handling traits. + /// + /// - Throws: Whatever is thrown by `body` or by any of the traits' provide + /// scope function calls. + private static func _applyIssueHandlingTraits(for test: Test, _ body: @escaping @Sendable () async throws -> Void) async throws { + // If the test does not have any traits, exit early to avoid unnecessary + // heap allocations below. + if test.traits.isEmpty { + return try await body() + } + + // Construct a recursive function that invokes each issue handling trait's + // `provideScope(performing:)` function. The order of the sequence is + // reversed so that the last trait is the one that invokes body, then the + // second-to-last invokes the last, etc. and ultimately the first trait is + // the first one to be invoked. + let executeAllTraits = test.traits.lazy + .compactMap { $0 as? IssueHandlingTrait } + .reversed() + .map { $0.provideScope(performing:) } + .reduce(body) { executeAllTraits, provideScope in + { + try await provideScope(executeAllTraits) + } + } + + try await executeAllTraits() + } + /// Enumerate the elements of a sequence, parallelizing enumeration in a task /// group if a given plan step has parallelization enabled. /// @@ -177,7 +213,19 @@ extension Runner { Event.post(.testSkipped(skipInfo), for: (step.test, nil), configuration: configuration) shouldSendTestEnded = false case let .recordIssue(issue): - Event.post(.issueRecorded(issue), for: (step.test, nil), configuration: configuration) + // Scope posting the issue recorded event such that issue handling + // traits have the opportunity to handle it. This ensures that if a test + // has an issue handling trait _and_ some other trait which caused an + // issue to be recorded, the issue handling trait can process the issue + // even though it wasn't recorded by the test function. + try await Test.withCurrent(step.test) { + try await _applyIssueHandlingTraits(for: step.test) { + // Don't specify `configuration` when posting this issue so that + // traits can provide scope and potentially customize the + // configuration. + Event.post(.issueRecorded(issue), for: (step.test, nil)) + } + } shouldSendTestEnded = false } } else { diff --git a/Sources/Testing/SourceAttribution/Expression.swift b/Sources/Testing/SourceAttribution/Expression.swift index dce4ed2a2..a294a81e0 100644 --- a/Sources/Testing/SourceAttribution/Expression.swift +++ b/Sources/Testing/SourceAttribution/Expression.swift @@ -22,9 +22,8 @@ /// let swiftSyntaxExpr: ExprSyntax = "\(testExpr)" /// ``` /// -/// - Warning: This type is used to implement the `#expect(exitsWith:)` -/// macro. Do not use it directly. Tools can use the SPI ``Expression`` -/// typealias if needed. +/// - Warning: This type is used to implement the `#expect()` macro. Do not use +/// it directly. Tools can use the SPI ``Expression`` typealias if needed. public struct __Expression: Sendable { /// An enumeration describing the various kinds of expression that can be /// captured. diff --git a/Sources/Testing/Testing.docc/Expectations.md b/Sources/Testing/Testing.docc/Expectations.md index fd3b0070d..e55ff3a1e 100644 --- a/Sources/Testing/Testing.docc/Expectations.md +++ b/Sources/Testing/Testing.docc/Expectations.md @@ -72,6 +72,14 @@ the test when the code doesn't satisfy a requirement, use - ``require(throws:_:sourceLocation:performing:)-4djuw`` - ``require(_:sourceLocation:performing:throws:)`` +### Checking how processes exit + +- +- ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` +- ``require(processExitsWith:observing:_:sourceLocation:performing:)`` +- ``ExitStatus`` +- ``ExitTest`` + ### Confirming that asynchronous events occur - diff --git a/Sources/Testing/Testing.docc/OrganizingTests.md b/Sources/Testing/Testing.docc/OrganizingTests.md index f2b577eb4..3464db4ae 100644 --- a/Sources/Testing/Testing.docc/OrganizingTests.md +++ b/Sources/Testing/Testing.docc/OrganizingTests.md @@ -124,7 +124,7 @@ struct MenuTests { The compiler emits an error when presented with a test suite that doesn't meet this requirement. -### Test suite types must always be available +#### Test suite types must always be available Although `@available` can be applied to a test function to limit its availability at runtime, a test suite type (and any types that contain it) must diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index 413b4327c..e6d19c1b9 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -48,6 +48,13 @@ types that customize the behavior of your tests. - ``Trait/bug(_:id:_:)-10yf5`` - ``Trait/bug(_:id:_:)-3vtpl`` + + ### Creating custom traits - ``Trait`` @@ -64,3 +71,4 @@ types that customize the behavior of your tests. - ``Tag`` - ``Tag/List`` - ``TimeLimitTrait`` + diff --git a/Sources/Testing/Testing.docc/exit-testing.md b/Sources/Testing/Testing.docc/exit-testing.md new file mode 100644 index 000000000..06ab53dc9 --- /dev/null +++ b/Sources/Testing/Testing.docc/exit-testing.md @@ -0,0 +1,155 @@ +# Exit testing + + + +@Metadata { + @Available(Swift, introduced: 6.2) +} + +Use exit tests to test functionality that might cause a test process to exit. + +## Overview + +Your code might contain calls to [`precondition()`](https://developer.apple.com/documentation/swift/precondition(_:_:file:line:)), +[`fatalError()`](https://developer.apple.com/documentation/swift/fatalerror(_:file:line:)), +or other functions that can cause the current process to exit. For example: + +```swift +extension Customer { + func eat(_ food: consuming some Food) { + precondition(food.isDelicious, "Tasty food only!") + precondition(food.isNutritious, "Healthy food only!") + ... + } +} +``` + +In this function, if `food.isDelicious` or `food.isNutritious` is `false`, the +precondition fails and Swift forces the process to exit. You can write an exit +test to validate preconditions like the ones above and to make sure that your +functions correctly catch invalid inputs. + +- Note: Exit tests are available on macOS, Linux, FreeBSD, OpenBSD, and Windows. + +### Create an exit test + +To create an exit test, call either the ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` +or the ``require(processExitsWith:observing:_:sourceLocation:performing:)`` +macro: + +```swift +@Test func `Customer won't eat food unless it's delicious`() async { + let result = await #expect(processExitsWith: .failure) { + var food = ... + food.isDelicious = false + Customer.current.eat(food) + } +} +``` + +The closure or function reference you pass to the macro is the _body_ of the +exit test. When an exit test is performed at runtime, the testing library starts +a new process with the same executable as the current process. The current task +is then suspended (as with `await`) and waits for the child process to exit. + +- Note: An exit test cannot run within another exit test. + +The parent process doesn't call the body of the exit test. Instead, the child +process treats the body of the exit test as its `main()` function and calls it +directly. + +- Note: Because the body acts as the `main()` function of a new process, it + can't capture any state originating in the parent process or from its lexical + context. For example, the following exit test will fail to compile because it + captures a variable declared outside the exit test itself: + + ```swift + @Test func `Customer won't eat food unless it's nutritious`() async { + let isNutritious = false + await #expect(processExitsWith: .failure) { + var food = ... + food.isNutritious = isNutritious // ❌ ERROR: trying to capture state here + Customer.current.eat(food) + } + } + ``` + +If the body returns before the child process exits, the process exits as if +`main()` returned normally. If the body throws an error, Swift handles it as if +it were thrown from `main()` and forces the process to exit abnormally. + +### Specify an exit condition + +When you create an exit test, specify how you expect the child process exits by +passing an instance of ``ExitTest/Condition``: + +- If you expect the exit test's body to run to completion or exit normally (for + example, by calling [`exit(EXIT_SUCCESS)`](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/exit.3.html) + from the C standard library), pass ``ExitTest/Condition/success``. +- If you expect the body to cause the child process to exit abnormally, but the + exact status reported by the system is not important, pass + ``ExitTest/Condition/failure``. +- If you need to check for a specific exit code or signal, pass + ``ExitTest/Condition/exitCode(_:)`` or ``ExitTest/Condition/signal(_:)``. + +When the child process exits, the parent process resumes and compares the exit +status of the child process against the expected exit condition you passed. If +they match, the exit test passes; otherwise, it fails and the testing library +records an issue. + +### Gather output from the child process + +The ``expect(processExitsWith:observing:_:sourceLocation:performing:)`` and +``require(processExitsWith:observing:_:sourceLocation:performing:)`` macros +return an instance of ``ExitTest/Result`` that contains information about the +state of the child process. + +By default, the child process is configured without a standard output or +standard error stream. If your test needs to review the content of either of +these streams, pass the key path to the corresponding ``ExitTest/Result`` +property to the macro: + +```swift +extension Customer { + func eat(_ food: consuming some Food) { + print("Let's see if I want to eat \(food)...") + precondition(food.isDelicious, "Tasty food only!") + precondition(food.isNutritious, "Healthy food only!") + ... + } +} + +@Test func `Customer won't eat food unless it's delicious`() async { + let result = await #expect( + processExitsWith: .failure, + observing: [\.standardOutputContent] + ) { + var food = ... + food.isDelicious = false + Customer.current.eat(food) + } + if let result { + #expect(result.standardOutputContent.contains(UInt8(ascii: "L"))) + } +} +``` + +- Note: The content of the standard output and standard error streams can + contain any arbitrary sequence of bytes, including sequences that aren't valid + UTF-8 and can't be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + These streams are globally accessible within the child process, and any code + running in an exit test may write to it including the operating system and any + third-party dependencies you declare in your package description or Xcode + project. + +The testing library always sets ``ExitTest/Result/exitStatus`` to the actual +exit status of the child process (as reported by the system) even if you do not +pass it. diff --git a/Sources/Testing/Traits/ConditionTrait.swift b/Sources/Testing/Traits/ConditionTrait.swift index a3b98c99d..eb3d1bd11 100644 --- a/Sources/Testing/Traits/ConditionTrait.swift +++ b/Sources/Testing/Traits/ConditionTrait.swift @@ -69,7 +69,10 @@ public struct ConditionTrait: TestTrait, SuiteTrait { /// /// The evaluation is performed each time this function is called, and is not /// cached. - @_spi(Experimental) + /// + /// @Metadata { + /// @Available(Swift, introduced: 6.2) + /// } public func evaluate() async throws -> Bool { switch kind { case let .conditional(condition): diff --git a/Sources/Testing/Traits/IssueHandlingTrait.swift b/Sources/Testing/Traits/IssueHandlingTrait.swift new file mode 100644 index 000000000..d70d68e93 --- /dev/null +++ b/Sources/Testing/Traits/IssueHandlingTrait.swift @@ -0,0 +1,167 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type that allows transforming or filtering the issues recorded by a test. +/// +/// Use this type to observe or customize the issue(s) recorded by the test this +/// trait is applied to. You can transform a recorded issue by copying it, +/// modifying one or more of its properties, and returning the copy. You can +/// observe recorded issues by returning them unmodified. Or you can suppress an +/// issue by either filtering it using ``Trait/filterIssues(_:)`` or returning +/// `nil` from the closure passed to ``Trait/transformIssues(_:)``. +/// +/// When an instance of this trait is applied to a suite, it is recursively +/// inherited by all child suites and tests. +/// +/// To add this trait to a test, use one of the following functions: +/// +/// - ``Trait/transformIssues(_:)`` +/// - ``Trait/filterIssues(_:)`` +@_spi(Experimental) +public struct IssueHandlingTrait: TestTrait, SuiteTrait { + /// A function which transforms an issue and returns an optional replacement. + /// + /// - Parameters: + /// - issue: The issue to transform. + /// + /// - Returns: An issue to replace `issue`, or else `nil` if the issue should + /// not be recorded. + fileprivate typealias Transformer = @Sendable (_ issue: Issue) -> Issue? + + /// This trait's transformer function. + private var _transformer: Transformer + + fileprivate init(transformer: @escaping Transformer) { + _transformer = transformer + } + + public var isRecursive: Bool { + true + } +} + +extension IssueHandlingTrait: TestScoping { + public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? { + // Provide scope for tests at both the suite and test case levels, but not + // for the test function level. This avoids redundantly invoking the closure + // twice, and potentially double-processing, issues recorded by test + // functions. + test.isSuite || testCase != nil ? self : nil + } + + public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws { + try await provideScope(performing: function) + } + + /// Provide scope for a specified function. + /// + /// - Parameters: + /// - function: The function to perform. + /// + /// This is a simplified version of ``provideScope(for:testCase:performing:)`` + /// which doesn't accept test or test case parameters. It's included so that + /// a runner can invoke this trait's closure even when there is no test case, + /// such as if a trait on a test function threw an error during `prepare(for:)` + /// and caused an issue to be recorded for the test function. In that scenario, + /// this trait still needs to be invoked, but its `scopeProvider(for:testCase:)` + /// intentionally returns `nil` (see the comment in that method), so this + /// function can be called instead to ensure this trait can still handle that + /// issue. + func provideScope(performing function: @Sendable () async throws -> Void) async throws { + guard var configuration = Configuration.current else { + preconditionFailure("Configuration.current is nil when calling \(#function). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + } + + configuration.eventHandler = { [oldConfiguration = configuration] event, context in + guard case let .issueRecorded(issue) = event.kind else { + oldConfiguration.eventHandler(event, context) + return + } + + // Use the original configuration's event handler when invoking the + // transformer to avoid infinite recursion if the transformer itself + // records new issues. This means only issue handling traits whose scope + // is outside this one will be allowed to handle such issues. + let newIssue = Configuration.withCurrent(oldConfiguration) { + _transformer(issue) + } + + if let newIssue { + var event = event + event.kind = .issueRecorded(newIssue) + oldConfiguration.eventHandler(event, context) + } + } + + try await Configuration.withCurrent(configuration, perform: function) + } +} + +@_spi(Experimental) +extension Trait where Self == IssueHandlingTrait { + /// Constructs an trait that transforms issues recorded by a test. + /// + /// - Parameters: + /// - transformer: The closure called for each issue recorded by the test + /// this trait is applied to. It is passed a recorded issue, and returns + /// an optional issue to replace the passed-in one. + /// + /// The `transformer` closure is called synchronously each time an issue is + /// recorded by the test this trait is applied to. The closure is passed the + /// recorded issue, and if it returns a non-`nil` value, that will be recorded + /// instead of the original. Otherwise, if the closure returns `nil`, the + /// issue is suppressed and will not be included in the results. + /// + /// The `transformer` closure may be called more than once if the test records + /// multiple issues. If more than one instance of this trait is applied to a + /// test (including via inheritance from a containing suite), the `transformer` + /// closure for each instance will be called in right-to-left, innermost-to- + /// outermost order, unless `nil` is returned, which will skip invoking the + /// remaining traits' closures. + /// + /// Within `transformer`, you may access the current test or test case (if any) + /// using ``Test/current`` ``Test/Case/current``, respectively. You may also + /// record new issues, although they will only be handled by issue handling + /// traits which precede this trait or were inherited from a containing suite. + public static func transformIssues(_ transformer: @escaping @Sendable (Issue) -> Issue?) -> Self { + Self(transformer: transformer) + } + + /// Constructs a trait that filters issues recorded by a test. + /// + /// - Parameters: + /// - isIncluded: The predicate with which to filter issues recorded by the + /// test this trait is applied to. It is passed a recorded issue, and + /// should return `true` if the issue should be included, or `false` if it + /// should be suppressed. + /// + /// The `isIncluded` closure is called synchronously each time an issue is + /// recorded by the test this trait is applied to. The closure is passed the + /// recorded issue, and if it returns `true`, the issue will be preserved in + /// the test results. Otherwise, if the closure returns `false`, the issue + /// will not be included in the test results. + /// + /// The `isIncluded` closure may be called more than once if the test records + /// multiple issues. If more than one instance of this trait is applied to a + /// test (including via inheritance from a containing suite), the `isIncluded` + /// closure for each instance will be called in right-to-left, innermost-to- + /// outermost order, unless `false` is returned, which will skip invoking the + /// remaining traits' closures. + /// + /// Within `isIncluded`, you may access the current test or test case (if any) + /// using ``Test/current`` ``Test/Case/current``, respectively. You may also + /// record new issues, although they will only be handled by issue handling + /// traits which precede this trait or were inherited from a containing suite. + public static func filterIssues(_ isIncluded: @escaping @Sendable (Issue) -> Bool) -> Self { + Self { issue in + isIncluded(issue) ? issue : nil + } + } +} diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index e21938041..49630cfc9 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -14,7 +14,7 @@ import SwiftSyntaxBuilder public import SwiftSyntaxMacros #if !hasFeature(SymbolLinkageMarkers) && SWT_NO_LEGACY_TEST_DISCOVERY -#error("Platform-specific misconfiguration: either SymbolLinkageMarkers or legacy test discovery is required to expand #expect(exitsWith:)") +#error("Platform-specific misconfiguration: either SymbolLinkageMarkers or legacy test discovery is required to expand #expect(processExitsWith:)") #endif /// A protocol containing the common implementation for the expansions of the @@ -628,7 +628,7 @@ extension ExitTestExpectMacro { }() } -/// A type describing the expansion of the `#expect(exitsWith:)` macro. +/// A type describing the expansion of the `#expect(processExitsWith:)` macro. /// /// This type checks for nested invocations of `#expect()` and `#require()` and /// diagnoses them as unsupported. It is otherwise exactly equivalent to @@ -637,7 +637,7 @@ public struct ExitTestExpectMacro: ExitTestConditionMacro { public typealias Base = ExpectMacro } -/// A type describing the expansion of the `#require(exitsWith:)` macro. +/// A type describing the expansion of the `#require(processExitsWith:)` macro. /// /// This type checks for nested invocations of `#expect()` and `#require()` and /// diagnoses them as unsupported. It is otherwise exactly equivalent to diff --git a/Tests/TestingMacrosTests/ConditionMacroTests.swift b/Tests/TestingMacrosTests/ConditionMacroTests.swift index 67531dabf..dc36af7cd 100644 --- a/Tests/TestingMacrosTests/ConditionMacroTests.swift +++ b/Tests/TestingMacrosTests/ConditionMacroTests.swift @@ -437,13 +437,13 @@ struct ConditionMacroTests { } #if ExperimentalExitTestValueCapture - @Test("#expect(exitsWith:) produces a diagnostic for a bad capture", + @Test("#expect(processExitsWith:) produces a diagnostic for a bad capture", arguments: [ - "#expectExitTest(exitsWith: x) { [weak a] in }": + "#expectExitTest(processExitsWith: x) { [weak a] in }": "Specifier 'weak' cannot be used with captured value 'a'", - "#expectExitTest(exitsWith: x) { [a] in }": + "#expectExitTest(processExitsWith: x) { [a] in }": "Type of captured value 'a' is ambiguous", - "#expectExitTest(exitsWith: x) { [a = b] in }": + "#expectExitTest(processExitsWith: x) { [a = b] in }": "Type of captured value 'a' is ambiguous", ] ) @@ -463,8 +463,8 @@ struct ConditionMacroTests { @Test( "Capture list on an exit test produces a diagnostic", arguments: [ - "#expectExitTest(exitsWith: x) { [a] in }": - "Cannot specify a capture clause in closure passed to '#expectExitTest(exitsWith:_:)'" + "#expectExitTest(processExitsWith: x) { [a] in }": + "Cannot specify a capture clause in closure passed to '#expectExitTest(processExitsWith:_:)'" ] ) func exitTestCaptureListProducesDiagnostic(input: String, expectedMessage: String) throws { diff --git a/Tests/TestingTests/AttachmentTests.swift b/Tests/TestingTests/AttachmentTests.swift index 0281b4091..be940371e 100644 --- a/Tests/TestingTests/AttachmentTests.swift +++ b/Tests/TestingTests/AttachmentTests.swift @@ -554,7 +554,7 @@ extension AttachmentTests { #if !SWT_NO_EXIT_TESTS @available(_uttypesAPI, *) @Test func cannotAttachCGImageWithNonImageType() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { let attachment = Attachment(try Self.cgImage.get(), named: "diamond", as: .mp3) try attachment.attachableValue.withUnsafeBytes(for: attachment) { _ in } } diff --git a/Tests/TestingTests/ConfirmationTests.swift b/Tests/TestingTests/ConfirmationTests.swift index 5502cb8d2..c4f076268 100644 --- a/Tests/TestingTests/ConfirmationTests.swift +++ b/Tests/TestingTests/ConfirmationTests.swift @@ -76,7 +76,7 @@ struct ConfirmationTests { await confirmation(expectedCount: Int.max...Int.max) { _ in } #if !SWT_NO_EXIT_TESTS await withKnownIssue("Crashes in Swift standard library (rdar://139568287)") { - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { await confirmation(expectedCount: Int.max...) { _ in } } } @@ -87,10 +87,10 @@ struct ConfirmationTests { #if !SWT_NO_EXIT_TESTS @Test("Confirmation requires positive count") func positiveCount() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { await confirmation { $0.confirm(count: 0) } } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { await confirmation { $0.confirm(count: -1) } } } diff --git a/Tests/TestingTests/DiscoveryTests.swift b/Tests/TestingTests/DiscoveryTests.swift index 8ec185813..24d2eecfa 100644 --- a/Tests/TestingTests/DiscoveryTests.swift +++ b/Tests/TestingTests/DiscoveryTests.swift @@ -49,10 +49,10 @@ struct DiscoveryTests { #if !SWT_NO_EXIT_TESTS @Test("TestContentKind rejects bad string literals") func badTestContentKindLiteral() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { _ = "abc" as TestContentKind } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { _ = "abcde" as TestContentKind } } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 690fd416f..18f70186a 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -521,6 +521,35 @@ struct EventRecorderTests { recorder.record(Event(.runEnded, testID: nil, testCaseID: nil), in: context) } } + + @Test( + "HumanReadableOutputRecorder includes known issue comment in messages array", + arguments: [ + ("recordWithoutKnownIssueComment()", ["#expect comment"]), + ("recordWithKnownIssueComment()", ["#expect comment", "withKnownIssue comment"]), + ("throwWithoutKnownIssueComment()", []), + ("throwWithKnownIssueComment()", ["withKnownIssue comment"]), + ] + ) + func knownIssueComments(testName: String, expectedComments: [String]) async throws { + var configuration = Configuration() + let recorder = Event.HumanReadableOutputRecorder() + let messages = Locked<[Event.HumanReadableOutputRecorder.Message]>(rawValue: []) + configuration.eventHandler = { event, context in + guard case .issueRecorded = event.kind else { return } + messages.withLock { + $0.append(contentsOf: recorder.record(event, in: context)) + } + } + + await runTestFunction(named: testName, in: PredictablyFailingKnownIssueTests.self, configuration: configuration) + + // The first message is something along the lines of "Test foo recorded a + // known issue" and includes a source location, so is inconvenient to + // include in our expectation here. + let actualComments = messages.rawValue.dropFirst().map(\.stringValue) + #expect(actualComments == expectedComments) + } } // MARK: - Fixtures @@ -663,3 +692,35 @@ struct EventRecorderTests { #expect(arg > 0) } } + +@Suite(.hidden) struct PredictablyFailingKnownIssueTests { + @Test(.hidden) + func recordWithoutKnownIssueComment() { + withKnownIssue { + #expect(Bool(false), "#expect comment") + } + } + + @Test(.hidden) + func recordWithKnownIssueComment() { + withKnownIssue("withKnownIssue comment") { + #expect(Bool(false), "#expect comment") + } + } + + @Test(.hidden) + func throwWithoutKnownIssueComment() { + withKnownIssue { + struct TheError: Error {} + throw TheError() + } + } + + @Test(.hidden) + func throwWithKnownIssueComment() { + withKnownIssue("withKnownIssue comment") { + struct TheError: Error {} + throw TheError() + } + } +} diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 896784f22..02be1a140 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -14,26 +14,26 @@ private import _TestingInternals #if !SWT_NO_EXIT_TESTS @Suite("Exit test tests") struct ExitTestTests { @Test("Exit tests (passing)") func passing() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { exit(EXIT_FAILURE) } if EXIT_SUCCESS != EXIT_FAILURE + 1 { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { exit(EXIT_FAILURE + 1) } } - await #expect(exitsWith: .success) {} - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) {} + await #expect(processExitsWith: .success) { exit(EXIT_SUCCESS) } - await #expect(exitsWith: .exitCode(123)) { + await #expect(processExitsWith: .exitCode(123)) { exit(123) } - await #expect(exitsWith: .exitCode(123)) { + await #expect(processExitsWith: .exitCode(123)) { await Task.yield() exit(123) } - await #expect(exitsWith: .signal(SIGSEGV)) { + await #expect(processExitsWith: .signal(SIGSEGV)) { _ = raise(SIGSEGV) // Allow up to 1s for the signal to be delivered. On some platforms, // raise() delivers signals fully asynchronously and may not terminate the @@ -44,7 +44,7 @@ private import _TestingInternals try await Task.sleep(nanoseconds: 1_000_000_000) } } - await #expect(exitsWith: .signal(SIGABRT)) { + await #expect(processExitsWith: .signal(SIGABRT)) { abort() } #if !SWT_NO_UNSTRUCTURED_TASKS @@ -55,7 +55,7 @@ private import _TestingInternals #expect(Test.current != nil) await Task.detached { #expect(Test.current == nil) - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { fatalError() } }.value @@ -88,29 +88,29 @@ private import _TestingInternals // Mock an exit test where the process exits successfully. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(exitStatus: .exitCode(EXIT_SUCCESS)) } await Test { - await #expect(exitsWith: .success) {} + await #expect(processExitsWith: .success) {} }.run(configuration: configuration) // Mock an exit test where the process exits with a particular error code. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .exitCode(123)) + return ExitTest.Result(exitStatus: .exitCode(123)) } await Test { - await #expect(exitsWith: .failure) {} + await #expect(processExitsWith: .failure) {} }.run(configuration: configuration) // Mock an exit test where the process exits with a signal. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .signal(SIGABRT)) + return ExitTest.Result(exitStatus: .signal(SIGABRT)) } await Test { - await #expect(exitsWith: .signal(SIGABRT)) {} + await #expect(processExitsWith: .signal(SIGABRT)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .failure) {} + await #expect(processExitsWith: .failure) {} }.run(configuration: configuration) } } @@ -126,30 +126,30 @@ private import _TestingInternals // Mock exit tests that were expected to fail but passed. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .exitCode(EXIT_SUCCESS)) + return ExitTest.Result(exitStatus: .exitCode(EXIT_SUCCESS)) } await Test { - await #expect(exitsWith: .failure) {} + await #expect(processExitsWith: .failure) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .exitCode(EXIT_FAILURE)) {} + await #expect(processExitsWith: .exitCode(EXIT_FAILURE)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .signal(SIGABRT)) {} + await #expect(processExitsWith: .signal(SIGABRT)) {} }.run(configuration: configuration) // Mock exit tests that unexpectedly signalled. configuration.exitTestHandler = { _ in - return ExitTest.Result(statusAtExit: .signal(SIGABRT)) + return ExitTest.Result(exitStatus: .signal(SIGABRT)) } await Test { - await #expect(exitsWith: .exitCode(EXIT_SUCCESS)) {} + await #expect(processExitsWith: .exitCode(EXIT_SUCCESS)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .exitCode(EXIT_FAILURE)) {} + await #expect(processExitsWith: .exitCode(EXIT_FAILURE)) {} }.run(configuration: configuration) await Test { - await #expect(exitsWith: .success) {} + await #expect(processExitsWith: .success) {} }.run(configuration: configuration) } } @@ -164,7 +164,7 @@ private import _TestingInternals } await Test { - await #expect(exitsWith: .success) {} + await #expect(processExitsWith: .success) {} }.run(configuration: configuration) } } @@ -186,11 +186,11 @@ private import _TestingInternals configuration.exitTestHandler = ExitTest.handlerForEntryPoint() await Test { - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { #expect(Bool(false), "Something went wrong!") exit(0) } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { Issue.record(MyError()) } }.run(configuration: configuration) @@ -219,7 +219,7 @@ private import _TestingInternals // // Windows does not have the 8-bit exit code restriction and always reports // the full CInt value back to the testing library. - await #expect(exitsWith: .exitCode(512)) { + await #expect(processExitsWith: .exitCode(512)) { exit(512) } } @@ -232,7 +232,7 @@ private import _TestingInternals @Test("Exit test can be main-actor-isolated") @MainActor func mainActorIsolation() async { - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { await Self.someMainActorFunction() _ = 0 exit(EXIT_SUCCESS) @@ -242,24 +242,24 @@ private import _TestingInternals @Test("Result is set correctly on success") func successfulArtifacts() async throws { // Test that basic passing exit tests produce the correct results (#expect) - var result = await #expect(exitsWith: .success) { + var result = await #expect(processExitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) - result = await #expect(exitsWith: .exitCode(123)) { + #expect(result?.exitStatus == .exitCode(EXIT_SUCCESS)) + result = await #expect(processExitsWith: .exitCode(123)) { exit(123) } - #expect(result?.statusAtExit == .exitCode(123)) + #expect(result?.exitStatus == .exitCode(123)) // Test that basic passing exit tests produce the correct results (#require) - result = try await #require(exitsWith: .success) { + result = try await #require(processExitsWith: .success) { exit(EXIT_SUCCESS) } - #expect(result?.statusAtExit == .exitCode(EXIT_SUCCESS)) - result = try await #require(exitsWith: .exitCode(123)) { + #expect(result?.exitStatus == .exitCode(EXIT_SUCCESS)) + result = try await #require(processExitsWith: .exitCode(123)) { exit(123) } - #expect(result?.statusAtExit == .exitCode(123)) + #expect(result?.exitStatus == .exitCode(123)) } @Test("Result is nil on failure") @@ -278,11 +278,11 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTest.Result(statusAtExit: .exitCode(123)) + ExitTest.Result(exitStatus: .exitCode(123)) } await Test { - let result = await #expect(exitsWith: .success) {} + let result = await #expect(processExitsWith: .success) {} #expect(result == nil) }.run(configuration: configuration) } @@ -301,11 +301,11 @@ private import _TestingInternals } } configuration.exitTestHandler = { _ in - ExitTest.Result(statusAtExit: .exitCode(EXIT_FAILURE)) + ExitTest.Result(exitStatus: .exitCode(EXIT_FAILURE)) } await Test { - try await #require(exitsWith: .success) {} + try await #require(processExitsWith: .success) {} fatalError("Unreachable") }.run(configuration: configuration) } @@ -334,7 +334,7 @@ private import _TestingInternals } await Test { - let result = await #expect(exitsWith: .success) {} + let result = await #expect(processExitsWith: .success) {} #expect(result == nil) }.run(configuration: configuration) } @@ -343,21 +343,21 @@ private import _TestingInternals @Test("Result contains stdout/stderr") func exitTestResultContainsStandardStreams() async throws { - var result = try await #require(exitsWith: .success, observing: [\.standardOutputContent]) { + var result = try await #require(processExitsWith: .success, observing: [\.standardOutputContent]) { try FileHandle.stdout.write("STANDARD OUTPUT") try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) + #expect(result.exitStatus == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.contains("STANDARD OUTPUT".utf8)) #expect(result.standardErrorContent.isEmpty) - result = try await #require(exitsWith: .success, observing: [\.standardErrorContent]) { + result = try await #require(processExitsWith: .success, observing: [\.standardErrorContent]) { try FileHandle.stdout.write("STANDARD OUTPUT") try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) exit(EXIT_SUCCESS) } - #expect(result.statusAtExit == .exitCode(EXIT_SUCCESS)) + #expect(result.exitStatus == .exitCode(EXIT_SUCCESS)) #expect(result.standardOutputContent.isEmpty) #expect(result.standardErrorContent.contains("STANDARD ERROR".utf8.reversed())) } @@ -368,7 +368,7 @@ private import _TestingInternals func nonConstExitCondition() async throws -> ExitTest.Condition { .failure } - await #expect(exitsWith: try await nonConstExitCondition(), sourceLocation: unrelatedSourceLocation) { + await #expect(processExitsWith: try await nonConstExitCondition(), sourceLocation: unrelatedSourceLocation) { fatalError() } } @@ -376,7 +376,7 @@ private import _TestingInternals @Test("ExitTest.current property") func currentProperty() async { #expect((ExitTest.current == nil) as Bool) - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { #expect((ExitTest.current != nil) as Bool) } } @@ -386,7 +386,7 @@ private import _TestingInternals func captureList() async { let i = 123 let s = "abc" as Any - await #expect(exitsWith: .success) { [i = i as Int, s = s as! String, t = (s as Any) as? String?] in + await #expect(processExitsWith: .success) { [i = i as Int, s = s as! String, t = (s as Any) as? String?] in #expect(i == 123) #expect(s == "abc") #expect(t == "abc") @@ -397,7 +397,7 @@ private import _TestingInternals func longCaptureList() async { let count = 1 * 1024 * 1024 let buffer = Array(repeatElement(0 as UInt8, count: count)) - await #expect(exitsWith: .success) { [count = count as Int, buffer = buffer as [UInt8]] in + await #expect(processExitsWith: .success) { [count = count as Int, buffer = buffer as [UInt8]] in #expect(buffer.count == count) } } @@ -407,7 +407,7 @@ private import _TestingInternals @Test("self in capture list") func captureListWithSelf() async { - await #expect(exitsWith: .success) { [self, x = self] in + await #expect(processExitsWith: .success) { [self, x = self] in #expect(self.property == 456) #expect(x.property == 456) } @@ -444,13 +444,13 @@ private import _TestingInternals @Test("Capturing an instance of a subclass") func captureSubclass() async { let instance = CapturableDerivedClass(x: 123) - await #expect(exitsWith: .success) { [instance = instance as CapturableBaseClass] in + await #expect(processExitsWith: .success) { [instance = instance as CapturableBaseClass] in #expect((instance as AnyObject) is CapturableBaseClass) // However, because the static type of `instance` is not Derived, we won't // be able to cast it to Derived. #expect(!((instance as AnyObject) is CapturableDerivedClass)) } - await #expect(exitsWith: .success) { [instance = instance as CapturableDerivedClass] in + await #expect(processExitsWith: .success) { [instance = instance as CapturableDerivedClass] in #expect((instance as AnyObject) is CapturableBaseClass) #expect((instance as AnyObject) is CapturableDerivedClass) #expect(instance.x == 123) @@ -463,28 +463,28 @@ private import _TestingInternals @Suite(.hidden) struct FailingExitTests { @Test(.hidden) func failingExitTests() async { - await #expect(exitsWith: .failure) {} - await #expect(exitsWith: .exitCode(123)) {} - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) {} + await #expect(processExitsWith: .exitCode(123)) {} + await #expect(processExitsWith: .failure) { exit(EXIT_SUCCESS) } - await #expect(exitsWith: .success) { + await #expect(processExitsWith: .success) { exit(EXIT_FAILURE) } - await #expect(exitsWith: .exitCode(123)) { + await #expect(processExitsWith: .exitCode(123)) { exit(0) } - await #expect(exitsWith: .exitCode(SIGABRT)) { + await #expect(processExitsWith: .exitCode(SIGABRT)) { // abort() raises on Windows, but we don't handle that yet and it is // reported as .failure (which will fuzzy-match with SIGABRT.) abort() } - await #expect(exitsWith: .signal(123)) {} - await #expect(exitsWith: .signal(123)) { + await #expect(processExitsWith: .signal(123)) {} + await #expect(processExitsWith: .signal(123)) { exit(123) } - await #expect(exitsWith: .signal(SIGSEGV)) { + await #expect(processExitsWith: .signal(SIGSEGV)) { abort() // sends SIGABRT, not SIGSEGV } } @@ -493,7 +493,7 @@ private import _TestingInternals #if false // intentionally fails to compile @Test(.hidden, arguments: 100 ..< 200) func sellIceCreamCones(count: Int) async throws { - try await #require(exitsWith: .failure) { + try await #require(processExitsWith: .failure) { precondition(count < 10, "Too many ice cream cones") } } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index 6ea1a5827..cc0a7acf5 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -491,6 +491,7 @@ final class IssueTests: XCTestCase { }.run(configuration: .init()) } +#if !SWT_TARGET_OS_APPLE || SWT_FIXED_149299786 func testErrorCheckingWithExpect() async throws { let expectationFailed = expectation(description: "Expectation failed") expectationFailed.isInverted = true @@ -610,6 +611,7 @@ final class IssueTests: XCTestCase { await fulfillment(of: [expectationFailed], timeout: 0.0) } +#endif func testErrorCheckingWithExpect_mismatchedErrorDescription() async throws { let expectationFailed = expectation(description: "Expectation failed") diff --git a/Tests/TestingTests/KnownIssueTests.swift b/Tests/TestingTests/KnownIssueTests.swift index 733fbbf01..9174bfdd8 100644 --- a/Tests/TestingTests/KnownIssueTests.swift +++ b/Tests/TestingTests/KnownIssueTests.swift @@ -38,7 +38,7 @@ final class KnownIssueTests: XCTestCase { await fulfillment(of: [issueRecorded], timeout: 0.0) } - func testKnownIssueWithComment() async { + func testKnownIssueNotRecordedWithComment() async { let issueRecorded = expectation(description: "Issue recorded") var configuration = Configuration() @@ -52,8 +52,8 @@ final class KnownIssueTests: XCTestCase { return } - XCTAssertEqual(issue.comments.first, "With Known Issue Comment") XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.comments, ["With Known Issue Comment"]) } await Test { @@ -63,6 +63,234 @@ final class KnownIssueTests: XCTestCase { await fulfillment(of: [issueRecorded], timeout: 0.0) } + func testKnownIssueRecordedWithComment() async { + let issueMatched = expectation(description: "Issue matched") + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue Comment"]) + XCTAssertEqual(issue.knownIssueContext?.comment, "With Known Issue Comment") + } + + await Test { + withKnownIssue("With Known Issue Comment") { + Issue.record("Issue Comment") + } matching: { issue in + // The issue isn't yet considered known since we haven't matched it yet. + XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue Comment"]) + XCTAssertNil(issue.knownIssueContext) + issueMatched.fulfill() + return true + } + }.run(configuration: configuration) + + await fulfillment(of: [issueMatched, issueRecorded], timeout: 0.0) + } + + func testThrownKnownIssueRecordedWithComment() async { + let issueMatched = expectation(description: "Issue matched") + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["With Known Issue Comment"]) + } + + struct E: Error {} + + await Test { + try withKnownIssue("With Known Issue Comment") { + throw E() + } matching: { issue in + // The issue isn't yet considered known since we haven't matched it yet. + XCTAssertFalse(issue.isKnown) + XCTAssertEqual(issue.comments, []) + XCTAssertNil(issue.knownIssueContext) + issueMatched.fulfill() + return true + } + }.run(configuration: configuration) + + await fulfillment(of: [issueMatched, issueRecorded], timeout: 0.0) + } + + func testKnownIssueRecordedWithNoComment() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue Comment"]) + } + + await Test { + withKnownIssue { + Issue.record("Issue Comment") + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + + func testKnownIssueRecordedWithInnermostMatchingComment() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue B"]) + XCTAssertEqual(issue.knownIssueContext?.comment, "Inner Contains B") + } + + await Test { + withKnownIssue("Contains A", isIntermittent: true) { + withKnownIssue("Outer Contains B", isIntermittent: true) { + withKnownIssue("Inner Contains B") { + withKnownIssue("Contains C", isIntermittent: true) { + Issue.record("Issue B") + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("C") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("A") } + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + + func testThrownKnownIssueRecordedWithInnermostMatchingComment() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Inner Is B", "B"]) + } + + struct A: Error {} + struct B: Error {} + struct C: Error {} + + await Test { + try withKnownIssue("Is A", isIntermittent: true) { + try withKnownIssue("Outer Is B", isIntermittent: true) { + try withKnownIssue("Inner Is B") { + try withKnownIssue("Is C", isIntermittent: true) { + throw B() + } matching: { issue in + issue.error is C + } + } matching: { issue in + issue.error is B + } + } matching: { issue in + issue.error is B + } + } matching: { issue in + issue.error is A + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + + func testKnownIssueRecordedWithNoCommentOnInnermostMatch() async { + let issueRecorded = expectation(description: "Issue recorded") + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + issueRecorded.fulfill() + + guard case .unconditional = issue.kind else { + return + } + + XCTAssertTrue(issue.isKnown) + XCTAssertEqual(issue.comments, ["Issue B"]) + } + + await Test { + withKnownIssue("Contains A", isIntermittent: true) { + withKnownIssue("Outer Contains B", isIntermittent: true) { + withKnownIssue { // No comment here on the withKnownIssue that will actually match. + withKnownIssue("Contains C", isIntermittent: true) { + Issue.record("Issue B") + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("C") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("B") } + } + } matching: { issue in + issue.comments.contains { $0.rawValue.contains("A") } + } + }.run(configuration: configuration) + + await fulfillment(of: [issueRecorded], timeout: 0.0) + } + func testIssueIsKnownPropertyIsSetCorrectlyWithCustomIssueMatcher() async { struct MyError: Error {} diff --git a/Tests/TestingTests/PlanIterationTests.swift b/Tests/TestingTests/PlanIterationTests.swift index 3892b2fb8..21d71f894 100644 --- a/Tests/TestingTests/PlanIterationTests.swift +++ b/Tests/TestingTests/PlanIterationTests.swift @@ -117,11 +117,11 @@ struct PlanIterationTests { #if !SWT_NO_EXIT_TESTS @Test("Iteration count must be positive") func positiveIterationCount() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var configuration = Configuration() configuration.repetitionPolicy.maximumIterationCount = 0 } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var configuration = Configuration() configuration.repetitionPolicy.maximumIterationCount = -1 } diff --git a/Tests/TestingTests/SourceLocationTests.swift b/Tests/TestingTests/SourceLocationTests.swift index 75a8791db..4145687b8 100644 --- a/Tests/TestingTests/SourceLocationTests.swift +++ b/Tests/TestingTests/SourceLocationTests.swift @@ -82,27 +82,27 @@ struct SourceLocationTests { #if !SWT_NO_EXIT_TESTS @Test("SourceLocation.init requires well-formed arguments") func sourceLocationInitPreconditions() async { - await #expect(exitsWith: .failure, "Empty fileID") { + await #expect(processExitsWith: .failure, "Empty fileID") { _ = SourceLocation(fileID: "", filePath: "", line: 1, column: 1) } - await #expect(exitsWith: .failure, "Invalid fileID") { + await #expect(processExitsWith: .failure, "Invalid fileID") { _ = SourceLocation(fileID: "B.swift", filePath: "", line: 1, column: 1) } - await #expect(exitsWith: .failure, "Zero line") { + await #expect(processExitsWith: .failure, "Zero line") { _ = SourceLocation(fileID: "A/B.swift", filePath: "", line: 0, column: 1) } - await #expect(exitsWith: .failure, "Zero column") { + await #expect(processExitsWith: .failure, "Zero column") { _ = SourceLocation(fileID: "A/B.swift", filePath: "", line: 1, column: 0) } } @Test("SourceLocation.fileID property must be well-formed") func sourceLocationFileIDWellFormed() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.fileID = "" } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.fileID = "ABC" } @@ -110,11 +110,11 @@ struct SourceLocationTests { @Test("SourceLocation.line and column properties must be positive") func sourceLocationLineAndColumnPositive() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.line = -1 } - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { var sourceLocation = #_sourceLocation sourceLocation.column = -1 } diff --git a/Tests/TestingTests/Support/FileHandleTests.swift b/Tests/TestingTests/Support/FileHandleTests.swift index c837ac7cf..4be633ad6 100644 --- a/Tests/TestingTests/Support/FileHandleTests.swift +++ b/Tests/TestingTests/Support/FileHandleTests.swift @@ -85,7 +85,7 @@ struct FileHandleTests { #if !SWT_NO_EXIT_TESTS @Test("Writing requires contiguous storage") func writeIsContiguous() async { - await #expect(exitsWith: .failure) { + await #expect(processExitsWith: .failure) { let fileHandle = try FileHandle.null(mode: "wb") try fileHandle.write([1, 2, 3, 4, 5].lazy.filter { $0 == 1 }) } diff --git a/Tests/TestingTests/Traits/ConditionTraitTests.swift b/Tests/TestingTests/Traits/ConditionTraitTests.swift index 6b5311202..d957e425b 100644 --- a/Tests/TestingTests/Traits/ConditionTraitTests.swift +++ b/Tests/TestingTests/Traits/ConditionTraitTests.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@testable @_spi(Experimental) import Testing +@testable import Testing @Suite("Condition Trait Tests", .tags(.traitRelated)) struct ConditionTraitTests { diff --git a/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift new file mode 100644 index 000000000..4d749b07f --- /dev/null +++ b/Tests/TestingTests/Traits/IssueHandlingTraitTests.swift @@ -0,0 +1,197 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("IssueHandlingTrait Tests") +struct IssueHandlingTraitTests { + @Test("Transforming an issue by appending a comment") + func addComment() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind else { + return + } + + #expect(issue.comments == ["Foo", "Bar"]) + } + + let handler = IssueHandlingTrait.transformIssues { issue in + var issue = issue + issue.comments.append("Bar") + return issue + } + + await Test(handler) { + Issue.record("Foo") + }.run(configuration: configuration) + } + + @Test("Suppressing an issue by returning `nil` from the transform closure") + func suppressIssueUsingTransformer() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + if case .issueRecorded = event.kind { + Issue.record("Unexpected issue recorded event: \(event)") + } + } + + let handler = IssueHandlingTrait.transformIssues { _ in + // Return nil to suppress the issue. + nil + } + + await Test(handler) { + Issue.record("Foo") + }.run(configuration: configuration) + } + + @Test("Suppressing an issue by returning `false` from the filter closure") + func filterIssue() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + if case .issueRecorded = event.kind { + Issue.record("Unexpected issue recorded event: \(event)") + } + } + + await Test(.filterIssues { _ in false }) { + Issue.record("Foo") + }.run(configuration: configuration) + } + +#if !SWT_NO_UNSTRUCTURED_TASKS + @Test("Transforming an issue recorded from another trait on the test") + func skipIssue() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .errorCaught = issue.kind else { + return + } + + #expect(issue.comments == ["Transformed!"]) + } + + struct MyError: Error {} + + try await confirmation("Transformer closure is called") { transformerCalled in + let transformer: @Sendable (Issue) -> Issue? = { issue in + defer { + transformerCalled() + } + + #expect(Test.Case.current == nil) + + var issue = issue + issue.comments = ["Transformed!"] + return issue + } + + let test = Test( + .enabled(if: try { throw MyError() }()), + .transformIssues(transformer) + ) {} + + // Use a detached task to intentionally clear task local values for the + // current test and test case, since this test validates their value. + await Task.detached { [configuration] in + await test.run(configuration: configuration) + }.value + } + } +#endif + + @Test("Accessing the current Test and Test.Case from a transformer closure") + func currentTestAndCase() async throws { + await confirmation("Transformer closure is called") { transformerCalled in + let handler = IssueHandlingTrait.transformIssues { issue in + defer { + transformerCalled() + } + #expect(Test.current?.name == "fixture()") + #expect(Test.Case.current != nil) + return issue + } + + var test = Test(handler) { + Issue.record("Foo") + } + test.name = "fixture()" + await test.run() + } + } + + @Test("Validate the relative execution order of multiple issue handling traits") + func traitOrder() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind else { + return + } + + // Ordering is intentional + #expect(issue.comments == ["Foo", "Bar", "Baz"]) + } + + let outerHandler = IssueHandlingTrait.transformIssues { issue in + var issue = issue + issue.comments.append("Baz") + return issue + } + let innerHandler = IssueHandlingTrait.transformIssues { issue in + var issue = issue + issue.comments.append("Bar") + return issue + } + + await Test(outerHandler, innerHandler) { + Issue.record("Foo") + }.run(configuration: configuration) + } + + @Test("Secondary issue recorded from a transformer closure") + func issueRecordedFromClosure() async throws { + await confirmation("Original issue recorded") { originalIssueRecorded in + await confirmation("Secondary issue recorded") { secondaryIssueRecorded in + var configuration = Configuration() + configuration.eventHandler = { event, context in + guard case let .issueRecorded(issue) = event.kind, case .unconditional = issue.kind else { + return + } + + if issue.comments.contains("Foo") { + originalIssueRecorded() + } else if issue.comments.contains("Something else") { + secondaryIssueRecorded() + } else { + Issue.record("Unexpected issue recorded: \(issue)") + } + } + + let handler1 = IssueHandlingTrait.transformIssues { issue in + return issue + } + let handler2 = IssueHandlingTrait.transformIssues { issue in + Issue.record("Something else") + return issue + } + let handler3 = IssueHandlingTrait.transformIssues { issue in + // The "Something else" issue should not be passed to this closure. + #expect(issue.comments.contains("Foo")) + return issue + } + + await Test(handler1, handler2, handler3) { + Issue.record("Foo") + }.run(configuration: configuration) + } + } + } +}