Add integration tests for TestExplorer (#806)

* Add integration tests for TestExplorer

Add some integration tests that exercise finding and running tests,
ensuring they complete with the expected outcome. It currently tests
both normal test execution as well as debugging the tests.

This patch adds a new profile used when running the tests locally via
the Extension Tests task. This ensures that the workspace is always in a
clean and consistent state.

It adds some caps to the docker containers so we can run the debugger
in tests.

It also cleans up the test logs, silencing some harmless warnings.

Author: plemarquand

.gitignore

@@ -1,6 +1,7 @@
 out
 dist
 node_modules
+default.profraw
 *.vsix
 .vscode-test
 .build

.vscode/launch.json

@@ -25,14 +25,15 @@
 			"type": "extensionHost",
 			"request": "launch",
 			"args": [
+				"--profile=testing-debug",
 				"--extensionDevelopmentPath=${workspaceFolder}",
 				"--extensionTestsPath=${workspaceFolder}/out/test/suite/index",
 				"${workspaceFolder}/assets/test"
 			],
 			"outFiles": [
 				"${workspaceFolder}/out/test/**/*.js"
 			],
-			"preLaunchTask": "compile"
+			"preLaunchTask": "compile-tests"
 		}
 	]
 }

.vscode/tasks.json

@@ -18,9 +18,9 @@
 			}
 		},
 		{
-			"label": "compile",
+			"label": "compile-tests",
 			"type": "npm",
-			"script": "compile",
+			"script": "compile-tests",
 			"problemMatcher": "$tsc",
 			"presentation": {
 				"reveal": "never"

.vscode/testing-debug.code-profile

@@ -0,0 +1,4 @@
+{
+    "name": "testing-debug",
+    "extensions": "[{\"identifier\":{\"id\":\"ms-vscode-remote.remote-containers\",\"uuid\":\"93ce222b-5f6f-49b7-9ab1-a0463c6238df\"},\"displayName\":\"Dev Containers\"},{\"identifier\":{\"id\":\"vadimcn.vscode-lldb\",\"uuid\":\"bee31e34-a44b-4a76-9ec2-e9fd1439a0f6\"},\"displayName\":\"CodeLLDB\"}]"
+}

CONTRIBUTING.md

@@ -38,7 +38,12 @@ Please keep your PRs to a minimal number of changes. If a PR is large, try to sp
 
 ### Testing
 
-Where possible any new feature should have tests that go along with it, to ensure it works and will continue to work in the future. When a PR is submitted one of the prerequisites for it to be merged is that all tests pass. You can run tests locally using either of the following methods:
+Where possible any new feature should have tests that go along with it, to ensure it works and will continue to work in the future. When a PR is submitted one of the prerequisites for it to be merged is that all tests pass.
+
+To get started running tests first import the `testing-debug.code-profile` VSCode profile used by the tests. Run the `> Profiles: Import Profile...` command then `Select File` and pick `./.vscode/testing-debug.code-profile`.
+
+Now you can run tests locally using either of the following methods:
+
 - From VSCode, by selecting "Extension Tests" in the Run and Debug activity.
 - Using `npm run test` from the command line (when VS Code is not running, or you'll get an error)
 

assets/test/.vscode/tasks.json

@@ -6,7 +6,7 @@
 			"args": [
 				"build",
 				"--build-tests",
-                "--verbose"
+				"--verbose"
 			],
 			"cwd": "defaultPackage",
 			"problemMatcher": [

assets/test/defaultPackage/Package@swift-5.10.swift

@@ -0,0 +1,33 @@
+// swift-tools-version:5.10
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "defaultPackage",
+    platforms: [
+        .macOS(.v13)
+    ],
+    dependencies: [
+        .package(url: "https://github.com/apple/swift-testing.git", branch: "main")
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+        // Targets can depend on other targets in this package, and on products in packages this package depends on.
+        .executableTarget(
+            name: "PackageExe",
+            dependencies: ["PackageLib"]
+        ),
+        .target(
+            name: "PackageLib",
+            dependencies: []
+        ),
+        .testTarget(
+            name: "PackageTests",
+            dependencies: [
+                "PackageLib",
+                .product(name: "Testing", package: "swift-testing")
+            ]
+        ),
+    ]
+)

assets/test/defaultPackage/Tests/PackageTests/PackageTests.swift

@@ -1,11 +1,42 @@
-import XCTest
 import PackageLib
+import XCTest
+
+final class PassingXCTestSuite: XCTestCase {
+  func testPassing() throws {}
+}
+
+final class FailingXCTestSuite: XCTestCase {
+  func testFailing() throws {
+    XCTFail("oh no")
+  }
+}
+
+final class MixedXCTestSuite: XCTestCase {
+  func testPassing() throws {}
+
+  func testFailing() throws {
+    XCTFail("oh no")
+  }
+}
+
+#if swift(>=5.10)
+import Testing
+
+@Test func topLevelTestPassing() {}
+@Test func topLevelTestFailing() {
+  #expect(1 == 2)
+}
+
+@Suite
+struct MixedSwiftTestingSuite {
+  @Test
+  func testPassing() throws {}
+
+  @Test
+  func testFailing() throws {
+    #expect(1 == 2)
+  }
 
-final class DefaultPackageTests: XCTestCase {
-    func testExample() throws {
-        // This is an example of a functional test case.
-        // Use XCTAssert and related functions to verify your tests produce the correct
-        // results.
-        XCTAssertEqual(a, "B")
-    }
+  @Test(.disabled()) func testDisabled() {}
 }
+#endif

docker/docker-compose.2004.56.yaml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
 
   runtime-setup:

docker/docker-compose.2004.57.yaml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
 
   runtime-setup: