[skip-changelog] Use a matrix to create parallel builds for each OS (#1883)

MatteoPologruto · web-flow · commit 9c784d809f16 · 2022-09-29T15:48:56.000+02:00
* Use a matrix create parallel builds for each os

Using a matrix to run each build task greatly improves performances,
since they can all start concurrently. The finishing time of the job
will be equal to the one of the longer task to build, instead of being
the sum of each individual task's finishing time.

* Calculate checksums during release creation

Checksums of the output files where previously calculated during the initial creation of the artifacts, during the notarization process and, finally, at the release creation's step.
The whole process has been simplified and checksums are now computed only during the creation of the release.

* Set condition to create changelog once

The changelog is the same for each OS. It does not make sense to generate it more than once.

* Disable s3 push for testing

* Upload nightly artifacts for testing

* Fix linux_arm_6 typo

* Stop uploading nigthly artifacts

* Enable s3 pushing

* Upload build artifacts separately

Previously, the different builds were firstly uploaded using a single artifact, which was then downloaded to create different ones and eventually deleted.
Now, since builds are created concurrently, the same matrix can be used to directly upload an artifact for each build. It's necessary to use a second job to calculate the checksum related to
each build and save them all in a single .txt file.
diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml
@@ -22,6 +22,19 @@ jobs:
   create-nightly-artifacts:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        os:
+          - Windows_32bit
+          - Windows_64bit
+          - Linux_32bit
+          - Linux_64bit
+          - Linux_ARMv6
+          - Linux_ARMv7
+          - Linux_ARM64
+          - macOS_64bit
+          - macOS_ARM64
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -35,7 +48,7 @@ jobs:
       - name: Build
         env:
           NIGHTLY: true
-        run: task dist:all
+        run: task dist:${{ matrix.os }}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -127,14 +140,10 @@ jobs:
         run: |
           gon "${{ env.GON_CONFIG_PATH }}"
 
-      - name: Re-package binary and output checksum
+      - name: Re-package binary
         id: re-package
         working-directory: ${{ env.DIST_DIR }}
-        # This step performs the following:
-        # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
-        # 2. Recalculate package checksum
-        # 3. Output the new checksum to include in the nnnnnn-checksums.txt file
-        #    (it cannot be done there because of workflow job parallelization)
+        # Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
         run: |
           # GitHub's upload/download-artifact@v2 actions don't preserve file permissions,
           # so we need to add execution permission back until the action is made to do this.
@@ -144,11 +153,9 @@ jobs:
           tar -czvf "$PACKAGE_FILENAME" \
           -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
           -C ../../ LICENSE.txt
-          CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
           echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
-          echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
 
-      - name: Upload artifacts
+      - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
@@ -166,15 +173,11 @@ jobs:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.DIST_DIR }}
 
-      - name: Update checksum
+      - name: Output checksum
+        working-directory: ${{ env.DIST_DIR}}
         run: |
-          declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
-          for checksum_line in "${checksum_lines[@]}"
-          do
-            CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
-            PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
-            perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM}  ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
-          done
+          TAG="nightly-$(date -u +"%Y%m%d")"
+          sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt
 
       - name: Upload release files on Arduino downloads servers
         uses: docker://plugins/s3
diff --git a/.github/workflows/publish-go-tester-task.yml b/.github/workflows/publish-go-tester-task.yml
@@ -24,6 +24,8 @@ on:
   repository_dispatch:
 
 env:
+  # As defined by the Taskfile's PROJECT_NAME variable
+  PROJECT_NAME: arduino-cli
   # As defined by the Taskfile's DIST_DIR variable
   DIST_DIR: dist
   BUILDS_ARTIFACT: build-artifacts
@@ -53,10 +55,42 @@ jobs:
           echo "::set-output name=result::$RESULT"
 
   build:
+    name: Build ${{ matrix.os.name }}
     needs: run-determination
     if: needs.run-determination.outputs.result == 'true'
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        os:
+          - dist: Windows_32bit
+            path: "*Windows_32bit.zip"
+            name: Windows_X86-32
+          - dist: Windows_64bit
+            path: "*Windows_64bit.zip"
+            name: Windows_X86-64
+          - dist: Linux_32bit
+            path: "*Linux_32bit.tar.gz"
+            name: Linux_X86-32
+          - dist: Linux_64bit
+            path: "*Linux_64bit.tar.gz"
+            name: Linux_X86-64
+          - dist: Linux_ARMv6
+            path: "*Linux_ARMv6.tar.gz"
+            name: Linux_ARMv6
+          - dist: Linux_ARMv7
+            path: "*Linux_ARMv7.tar.gz"
+            name: Linux_ARMv7
+          - dist: Linux_ARM64
+            path: "*Linux_ARM64.tar.gz"
+            name: Linux_ARM64
+          - dist: macOS_64bit
+            path: "*macOS_64bit.tar.gz"
+            name: macOS_64
+          - dist: macOS_ARM64
+            path: "*macOS_ARM64.tar.gz"
+            name: macOS_ARM64
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -77,63 +111,44 @@ jobs:
           fi
           PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-"
           export PACKAGE_NAME_PREFIX
-          task dist:all
+          task dist:${{ matrix.os.dist }}
 
-      # Transfer builds to artifacts job
-      - name: Upload combined builds artifact
+      - name: Upload build artifact
         uses: actions/upload-artifact@v3
         with:
-          path: ${{ env.DIST_DIR }}
-          name: ${{ env.BUILDS_ARTIFACT }}
+          path: ${{ env.DIST_DIR }}/${{ matrix.os.path }}
+          name: ${{ matrix.os.name }}
 
-  artifacts:
-    name: ${{ matrix.artifact.name }} artifact
+  checksums:
     needs: build
     runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        artifact:
-          - path: "*checksums.txt"
-            name: checksums
-          - path: "*Linux_32bit.tar.gz"
-            name: Linux_X86-32
-          - path: "*Linux_64bit.tar.gz"
-            name: Linux_X86-64
-          - path: "*Linux_ARM64.tar.gz"
-            name: Linux_ARM64
-          - path: "*Linux_ARMv6.tar.gz"
-            name: Linux_ARMv6
-          - path: "*Linux_ARMv7.tar.gz"
-            name: Linux_ARMv7
-          - path: "*macOS_64bit.tar.gz"
-            name: macOS_64
-          - path: "*macOS_ARM64.tar.gz"
-            name: macOS_ARM64
-          - path: "*Windows_32bit.zip"
-            name: Windows_X86-32
-          - path: "*Windows_64bit.zip"
-            name: Windows_X86-64
-
     steps:
-      - name: Download combined builds artifact
+      - name: Download build artifacts
         uses: actions/download-artifact@v3
         with:
-          name: ${{ env.BUILDS_ARTIFACT }}
           path: ${{ env.BUILDS_ARTIFACT }}
 
-      - name: Upload individual build artifact
-        uses: actions/upload-artifact@v3
-        with:
-          path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }}
-          name: ${{ matrix.artifact.name }}
-
-  clean:
-    needs: artifacts
-    runs-on: ubuntu-latest
+      - name: Output checksum
+        working-directory: ${{ env.BUILDS_ARTIFACT}}
+        run: |
+          PACKAGE_NAME_PREFIX="test"
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}"
+          fi
+          PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-"
+          TAG="${PACKAGE_NAME_PREFIX}git-snapshot"
+          declare -a artifacts=($(ls -d */))
+          for artifact in ${artifacts[@]}
+          do
+            cd $artifact
+            checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*)
+            cd ..
+            echo $checksum >> ${TAG}-checksums.txt
+          done
 
-    steps:
-      - name: Remove unneeded combined builds artifact
-        uses: geekyeggo/delete-artifact@v1
+      - name: Upload checksum artifact
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.BUILDS_ARTIFACT }}
+          path: ${{ env.BUILDS_ARTIFACT }}/*checksums.txt
+          name: checksums
diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml
@@ -19,13 +19,28 @@ jobs:
   create-release-artifacts:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        os:
+          - Windows_32bit
+          - Windows_64bit
+          - Linux_32bit
+          - Linux_64bit
+          - Linux_ARMv6
+          - Linux_ARMv7
+          - Linux_ARM64
+          - macOS_64bit
+          - macOS_ARM64
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Create changelog
+        # Avoid creating the same changelog for each os
+        if: matrix.os == 'Windows_32bit'
         uses: arduino/create-changelog@v1
         with:
           tag-regex: '^[0-9]+\.[0-9]+\.[0-9]+.*$'
@@ -40,7 +55,7 @@ jobs:
           version: 3.x
 
       - name: Build
-        run: task dist:all
+        run: task dist:${{ matrix.os }}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -132,14 +147,10 @@ jobs:
         run: |
           gon "${{ env.GON_CONFIG_PATH }}"
 
-      - name: Re-package binary and output checksum
+      - name: Re-package binary
         id: re-package
         working-directory: ${{ env.DIST_DIR }}
-        # This step performs the following:
-        # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
-        # 2. Recalculate package checksum
-        # 3. Output the new checksum to include in the nnnnnn-checksums.txt file
-        #    (it cannot be done there because of workflow job parallelization)
+        # Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
         run: |
           # GitHub's upload/download-artifact@v2 actions don't preserve file permissions,
           # so we need to add execution permission back until the action is made to do this.
@@ -149,11 +160,9 @@ jobs:
           tar -czvf "$PACKAGE_FILENAME" \
           -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
           -C ../../ LICENSE.txt
-          CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
           echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
-          echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
 
-      - name: Upload artifacts
+      - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
@@ -171,15 +180,11 @@ jobs:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.DIST_DIR }}
 
-      - name: Update checksum
+      - name: Output checksum
+        working-directory: ${{ env.DIST_DIR}}
         run: |
-          declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
-          for checksum_line in "${checksum_lines[@]}"
-          do
-            CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
-            PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
-            perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM}  ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
-          done
+          TAG="${GITHUB_REF/refs\/tags\//}"
+          sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt
 
       - name: Identify Prerelease
         # This is a workaround while waiting for create-release action
diff --git a/DistTasks.yml b/DistTasks.yml
