From b0c3b2191e076af96f8b9028ffc3502cede77405 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 24 Oct 2023 09:47:30 -0400 Subject: [PATCH] Add the ability to filter tests by tag using an envvar. This PR is a follow-on to #77. It adds the ability to filter tests to just the ones that contain a given tag using an environment variable. As before, `swift test --filter` is unaware of swift-testing, so we need a temporary solution until such time as `swift test` is taught about us. To try it out: ``` SWT_SELECTED_TAGS='trait' swift test ``` Will run only those tests with the `"trait"` tag. If both `"SWT_SELECTED_TAGS""` and `"SWT_SELECTED_TEST_IDS"` are specified, a test must have (at least one of) the given tags _and_ be selected by ID. Whether or not this is the right way to combine these environment variables is, to be polite, probably uninteresting since this is not meant to be a permanent solution. This change also implements `==` and `hash(into:)` on `Tag` as they were previously synthesized which would cause them to consider the internal `sourceCode` property, which was not intentional. --- Sources/Testing/Running/Runner.Plan.swift | 16 ++++- Sources/Testing/Running/XCTestScaffold.swift | 63 +++++++++++++++----- Sources/Testing/Traits/Tag.swift | 8 +++ 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index d7ccab791..9e3c108f3 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -159,17 +159,27 @@ extension Runner.Plan { // them, in which case it will be .recordIssue(). var testGraph = Graph() var actionGraph = Graph(value: .run) - for test in tests where _isTestIncluded(test, using: configuration.testFilter) { + for test in tests { let idComponents = test.id.keyPathRepresentation testGraph.insertValue(test, at: idComponents) actionGraph.insertValue(.run, at: idComponents, intermediateValue: .run) } // Ensure the trait lists are complete for all nested tests. (Make sure to - // do this before we start calling prepare(for:) or we'll miss the - // recursively-added ones.) + // do this before we start calling configuration.testFilter or prepare(for:) + // or we'll miss the recursively-added traits.) _recursivelyApplyTraits(to: &testGraph) + // Remove any tests that should be filtered out per the runner's + // configuration. The action graph is not modified here: actions that lose + // their corresponding tests are effectively filtered out by the call to + // zip() near the end of the function. + testGraph = testGraph.mapValues { test in + test.flatMap { test in + _isTestIncluded(test, using: configuration.testFilter) ? test : nil + } + } + // For each test value, determine the appropriate action for it. await testGraph.forEach { keyPath, test in // Skip any nil test, which implies this node is just a placeholder and diff --git a/Sources/Testing/Running/XCTestScaffold.swift b/Sources/Testing/Running/XCTestScaffold.swift index f71413950..750ca6c5a 100644 --- a/Sources/Testing/Running/XCTestScaffold.swift +++ b/Sources/Testing/Running/XCTestScaffold.swift @@ -113,30 +113,44 @@ public enum XCTestScaffold: Sendable { /// ### Filtering tests /// /// This function does not support the `--filter` argument passed to - /// `swift test`. Instead, set the `SWT_SELECTED_TEST_IDS` environment - /// variable to the ``Test/ID`` of the test that should run (or, if multiple - /// tests should be run, their IDs separated by `";"`.) + /// `swift test`. Instead, use one of several environment variables to control + /// which tests run. + /// + /// #### Filtering by ID + /// + /// To run a specific test, set the `SWT_SELECTED_TEST_IDS` environment + /// variable to the ``Test/ID`` of that test (or, if multiple tests should be + /// run, their IDs separated by `";"`.) /// /// A test ID is composed of its module name, containing type name, and (if /// the test is a function rather than a suite), the name of the function /// including parentheses and any parameter labels. For example, given the - /// following test functions in a module named `"MyTests"`: + /// following test functions in a module named `"FoodTruckTests"`: /// /// ```swift - /// struct MySuite { - /// @Test func hello() { ... } - /// @Test(arguments: 0 ..< 10) func world(i: Int) { ... } + /// struct CashRegisterTests { + /// @Test func hasCash() { ... } + /// @Test(arguments: Card.allCases) func acceptsCard(card: Card) { ... } /// } /// ``` /// - /// Their IDs are the strings `"MyTests/MySuite/hello()"` and - /// `"MyTests/MySuite/world(i:)"` respectively, and they can be passed as the - /// environment variable value - /// `"MyTests/MySuite/hello();MyTests/MySuite/world(i:)"`. + /// Their IDs are the strings `"FoodTruckTests/CashRegisterTests/hasCash()"` + /// and `"FoodTruckTests/CashRegisterTests/acceptsCard(card:)"` respectively, + /// and they can be passed as the environment variable value + /// `"FoodTruckTests/CashRegisterTests/hasCash();FoodTruckTests/CashRegisterTests/acceptsCard(card:)"`. /// /// - Note: The module name of a test target in a Swift package is typically /// the name of the test target. /// + /// #### Filtering by tag + /// + /// To run only those tests with a given ``Tag``, set the `SWT_SELECTED_TAGS` + /// environment variable to the string value of that tag. Separate multiple + /// tags with `";"`; tests with _any_ of the specified tags will be run. For + /// example, to run all tests tagged `"critical"` _or_ ``Tag/red`` (or both), + /// set the value of the `SWT_SELECTED_TAGS` environment variable to + /// `"critical;red"`. + /// /// ### Configuring output /// /// By default, this function uses @@ -200,15 +214,34 @@ public enum XCTestScaffold: Sendable { // the configuration's test filter to match it. // // This environment variable stands in for `swift test --filter`. - let testIDs: [Test.ID]? = Environment.variable(named: "SWT_SELECTED_TEST_IDS").map { testIDs in - testIDs.split(separator: ";", omittingEmptySubsequences: true).map { testID in - Test.ID(testID.split(separator: "/", omittingEmptySubsequences: true).map(String.init)) + let testIDs: [Test.ID]? = Environment.variable(named: "SWT_SELECTED_TEST_IDS") + .map { testIDs in + testIDs.split(separator: ";", omittingEmptySubsequences: true).map { testID in + Test.ID(testID.split(separator: "/", omittingEmptySubsequences: true).map(String.init)) + } } - } if let testIDs { configuration.setTestFilter(toMatch: Set(testIDs)) } + // If the SWT_SELECTED_TAGS environment variable is set, split it by ";" + // (similar to test IDs above) and check if tests' tags overlap. + let tags: Set? = Environment.variable(named: "SWT_SELECTED_TAGS") + .map { tags in + tags + .split(separator: ";", omittingEmptySubsequences: true) + .map(String.init) + .map(Tag.init(rawValue:)) + }.map(Set.init) + if let tags { + // Check if the test's tags intersect the set of selected tags. If there + // was a previous filter function, it must also pass. + let oldTestFilter = configuration.testFilter ?? { _ in true } + configuration.testFilter = { test in + !tags.isDisjoint(with: test.tags) && oldTestFilter(test) + } + } + let runner = await Runner(configuration: configuration) await runner.run() } diff --git a/Sources/Testing/Traits/Tag.swift b/Sources/Testing/Traits/Tag.swift index 8e0d91c08..c0a079cfd 100644 --- a/Sources/Testing/Traits/Tag.swift +++ b/Sources/Testing/Traits/Tag.swift @@ -83,6 +83,14 @@ extension Tag: ExpressibleByStringLiteral, CustomStringConvertible { // MARK: - Equatable, Hashable, Comparable extension Tag: Equatable, Hashable, Comparable { + public static func ==(lhs: Self, rhs: Self) -> Bool { + lhs.rawValue == rhs.rawValue + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(rawValue) + } + /// The index of this color, relative to other colors. /// /// The value of this property can be used for sorting color tags distinctly