[skip-changelog] Run integration tests concurrently (#1953)

* Run integration tests concurrently

This enhancement reduces the amount of time needed to execute integration tests, since each test package no longer has to wait for the previous ones to be completed. In order to do this, a regex is used
to match each test file. It is also specified the path to the directory that contains the file.

* Merge all test files that share a package into one

Having different test files that shared the same package was inefficient, because the whole package test was run one time for each file. This enhancement avoids repeating the same tests more than once.

Author: MatteoPologruto

.github/workflows/test-go-task.yml

@@ -58,13 +58,61 @@ jobs:
 
           echo "result=$RESULT" >> $GITHUB_OUTPUT
 
-  test:
+  tests-collector:
+    runs-on: ubuntu-latest
     needs: run-determination
     if: needs.run-determination.outputs.result == 'true'
+    outputs:
+      tests-data: ${{ steps.collection.outputs.tests-data }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Collect tests
+        id: collection
+        run: |
+          echo "tests-data=$(go list ./internal/integrationtest/... | grep integrationtest/ | tr "/" " " | cut -d " " -f 6 | jq -cR '[inputs]')" >> $GITHUB_OUTPUT
 
+  test-integration:
+    needs: tests-collector
     strategy:
-      fail-fast: false
+      matrix:
+        operating-system:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+        tests: ${{ fromJSON(needs.tests-collector.outputs.tests-data) }}
+
+    runs-on: ${{ matrix.operating-system }}
+
+    steps:
+      # By default, actions/checkout converts the repo's LF line endings to CRLF on the Windows runner.
+      - name: Disable EOL conversions
+        run: git config --global core.autocrlf false
 
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          version: 3.x
+
+      - name: Run tests
+        shell: bash
+        run: |
+          export GO_TEST_PACKAGE="github.com/arduino/arduino-cli/internal/integrationtest/${{ matrix.tests }}"
+          task go:integration-test
+
+  test:
+    needs: run-determination
+    strategy:
       matrix:
         operating-system:
           - ubuntu-latest

Taskfile.yml

@@ -105,6 +105,23 @@ tasks:
           {{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}} \
           {{.TEST_LDFLAGS}}
 
+  go:integration-test:
+    desc: Run the Go-based integration tests
+    deps:
+      - task: go:build
+    dir: '{{default "./" .GO_MODULE_PATH}}'
+    cmds:
+      - |
+        go test \
+          -v \
+          -short \
+          {{ .GO_TEST_PACKAGE }} \
+          -run '{{default ".*" .GO_TEST_REGEX}}' \
+          {{default "-timeout 20m -coverpkg=./... -covermode=atomic" .GO_TEST_FLAGS}} \
+          -coverprofile=coverage_unit.txt \
+          {{default .DEFAULT_INTEGRATIONTEST_GO_PACKAGES .GO_PACKAGES}} \
+          {{.TEST_LDFLAGS}}
+
   # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/test-go-integration-task/Taskfile.yml
   go:test-integration:
     desc: Run integration tests
@@ -342,10 +359,13 @@ tasks:
 vars:
   PROJECT_NAME: "arduino-cli"
   DIST_DIR: "dist"
-  # all modules of this project except for "legacy/..." module
+  # all modules of this project except for "legacy/..." module and integration test
   DEFAULT_GO_PACKAGES:
     sh: |
-      echo $(cd {{default "./" .GO_MODULE_PATH}} && go list ./... | grep -v legacy | tr '\n' ' ' || echo '"ERROR: Unable to discover Go packages"')
+      echo $(cd {{default "./" .GO_MODULE_PATH}} && go list ./... | grep -v internal/integrationtest | grep -v legacy | tr '\n' ' ' || echo '"ERROR: Unable to discover Go packages"')
+  DEFAULT_INTEGRATIONTEST_GO_PACKAGES:
+    sh: |
+      echo $(cd {{default "./" .GO_MODULE_PATH}} && go list ./... | grep internal/integrationtest | tr '\n' ' ' || echo '"ERROR: Unable to discover Go packages"')
   # build vars
   COMMIT:
     sh: echo "$(git log --no-show-signature -n 1 --format=%h)"

internal/integrationtest/board/board_list_test.go

@@ -29,6 +29,45 @@ import (
 	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
+func TestCorrectBoardListOrdering(t *testing.T) {
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	defer env.CleanUp()
+
+	_, _, err := cli.Run("core", "install", "arduino:avr")
+	require.NoError(t, err)
+	jsonOut, _, err := cli.Run("board", "listall", "--format", "json")
+	require.NoError(t, err)
+	requirejson.Query(t, jsonOut, "[.boards[] | .fqbn]", `[
+		"arduino:avr:yun",
+		"arduino:avr:uno",
+		"arduino:avr:unomini",
+		"arduino:avr:diecimila",
+		"arduino:avr:nano",
+		"arduino:avr:mega",
+		"arduino:avr:megaADK",
+		"arduino:avr:leonardo",
+		"arduino:avr:leonardoeth",
+		"arduino:avr:micro",
+		"arduino:avr:esplora",
+		"arduino:avr:mini",
+		"arduino:avr:ethernet",
+		"arduino:avr:fio",
+		"arduino:avr:bt",
+		"arduino:avr:LilyPadUSB",
+		"arduino:avr:lilypad",
+		"arduino:avr:pro",
+		"arduino:avr:atmegang",
+		"arduino:avr:robotControl",
+		"arduino:avr:robotMotor",
+		"arduino:avr:gemma",
+		"arduino:avr:circuitplay32u4cat",
+		"arduino:avr:yunmini",
+		"arduino:avr:chiwawa",
+		"arduino:avr:one",
+		"arduino:avr:unowifi"
+	]`)
+}
+
 func TestBoardList(t *testing.T) {
 	if os.Getenv("CI") != "" {
 		t.Skip("VMs have no serial ports")