Fail tests if they crash (#1505)

If a test crashes during a test run the extension never marks the test
as failed, since it doesn't see the output that marks the test as
completed.

If we finish a test run and we still have a test in the 'pending' state
where no pass/fail/skip has been seen, mark the test as failed with a
message stating the test never finished.

Issue: #1504

Author: plemarquand

assets/test/defaultPackage/Tests/PackageTests/PackageTests.swift

@@ -106,16 +106,25 @@ struct MixedSwiftTestingSuite {
   }
   print(string)
 }
-#endif
+
+@Test func testCrashing() throws {
+  print([1,2][3])
+}
 
 // Disabled until Attachments are formalized and released.
 // #if swift(>=6.1)
 // @Test func testAttachment() throws {
 //   Attachment("Hello, world!", named: "hello.txt").attach()
 // }
-// #endif
+#endif
 
 final class DuplicateSuffixTests: XCTestCase {
   func testPassing() throws {}
   func testPassingSuffix() throws {}
 }
+
+final class CrashingXCTests: XCTestCase {
+  func testCrashing() throws {
+    print([1,2][3])
+  }
+}

src/TestExplorer/TestRunner.ts

@@ -58,6 +58,7 @@ export enum TestLibrary {
 }
 
 export interface TestRunState {
+    pending: vscode.TestItem[];
     failed: {
         test: vscode.TestItem;
         message: vscode.TestMessage | readonly vscode.TestMessage[];
@@ -88,6 +89,7 @@ export class TestRunProxy {
 
     public static initialTestRunState(): TestRunState {
         return {
+            pending: [],
             failed: [],
             passed: [],
             skipped: [],
@@ -220,18 +222,25 @@ export class TestRunProxy {
     }
 
     public started(test: vscode.TestItem) {
+        this.runState.pending.push(test);
         this.testRun?.started(test);
     }
 
+    private clearPendingTest(test: vscode.TestItem) {
+        this.runState.pending = this.runState.pending.filter(t => t !== test);
+    }
+
     public skipped(test: vscode.TestItem) {
         test.tags = [...test.tags, new vscode.TestTag(TestRunProxy.Tags.SKIPPED)];
 
         this.runState.skipped.push(test);
+        this.clearPendingTest(test);
         this.testRun?.skipped(test);
     }
 
     public passed(test: vscode.TestItem, duration?: number) {
         this.runState.passed.push(test);
+        this.clearPendingTest(test);
         this.testRun?.passed(test, duration);
     }
 
@@ -241,6 +250,7 @@ export class TestRunProxy {
         duration?: number
     ) {
         this.runState.failed.push({ test, message });
+        this.clearPendingTest(test);
         this.testRun?.failed(test, message, duration);
     }
 
@@ -250,6 +260,7 @@ export class TestRunProxy {
         duration?: number
     ) {
         this.runState.errored.push(test);
+        this.clearPendingTest(test);
         this.testRun?.errored(test, message, duration);
     }
 
@@ -260,6 +271,12 @@ export class TestRunProxy {
             this.testRunStarted();
         }
 
+        // Any tests still in the pending state are considered failed. This
+        // can happen if a test causes a crash and aborts the test run.
+        this.runState.pending.forEach(test => {
+            this.failed(test, new vscode.TestMessage("Test did not complete."));
+        });
+
         this.reportAttachments();
         this.testRun?.end();
         this.testRunCompleteEmitter.fire();

test/integration-tests/testexplorer/TestExplorerIntegration.test.ts

@@ -198,8 +198,11 @@ suite("Test Explorer Suite", function () {
                         "testWithKnownIssue()",
                         "testWithKnownIssueAndUnknownIssue()",
                         "testLotsOfOutput()",
+                        "testCrashing()",
                         "DuplicateSuffixTests",
                         ["testPassing()", "testPassingSuffix()"],
+                        "CrashingXCTests",
+                        ["testCrashing()"],
                     ],
                 ]);
             } else if (workspaceContext.swiftVersion.isLessThan(new Version(6, 0, 0))) {
@@ -208,6 +211,8 @@ suite("Test Explorer Suite", function () {
                 assertTestControllerHierarchy(testExplorer.controller, [
                     "PackageTests",
                     [
+                        "CrashingXCTests",
+                        ["testCrashing"],
                         "DebugReleaseTestSuite",
                         ["testDebug", "testRelease"],
                         "DuplicateSuffixTests",
@@ -342,6 +347,23 @@ suite("Test Explorer Suite", function () {
                 });
             });
 
+            test("crashing", async () => {
+                const testRun = await runTest(
+                    testExplorer,
+                    TestKind.standard,
+                    "PackageTests.testCrashing()"
+                );
+
+                assertTestResults(testRun, {
+                    failed: [
+                        {
+                            test: "PackageTests.testCrashing()",
+                            issues: ["Test did not complete."],
+                        },
+                    ],
+                });
+            });
+
             test("tests run in debug mode @slow", async function () {
                 const testRun = await runTest(
                     testExplorer,
@@ -441,6 +463,23 @@ suite("Test Explorer Suite", function () {
                 });
             });
 
+            test("Crashing XCTest", async function () {
+                const crashingRun = await runTest(
+                    testExplorer,
+                    TestKind.standard,
+                    "PackageTests.CrashingXCTests/testCrashing"
+                );
+
+                assertTestResults(crashingRun, {
+                    failed: [
+                        {
+                            test: "PackageTests.CrashingXCTests/testCrashing",
+                            issues: ["Test did not complete."],
+                        },
+                    ],
+                });
+            });
+
             test("Cancellation", async function () {
                 const targetProfile = testExplorer.testRunProfiles.find(
                     profile => profile.label === TestKind.standard