ci: run Zephyr builds in parallel

.github/workflows/package_core.yml

@@ -6,11 +6,9 @@ on:
 
 jobs:
 
-  package-core:
-    name: Build and package cores
+  build-env:
+    name: Prepare build environment
     runs-on: ubuntu-latest
-    env:
-      CCACHE_IGNOREOPTIONS: -specs=*
     outputs:
       CORE_TAG: ${{ env.CORE_TAG }}
       CORE_HASH: ${{ env.CORE_HASH }}
@@ -46,64 +44,156 @@ jobs:
           echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
           echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
 
+          (cd && tar cphf - .cmake work zephyr-sdk-* | zstd > build-env.tar.zstd)
+          tar cphf - cores/arduino/api | zstd > arduino-api.tar.zstd
+
+      - name: Archive build environment
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-env
+          path: ~/build-env.tar.zstd
+
+      - name: Archive API snapshot
+        uses: actions/upload-artifact@v4
+        with:
+          name: arduino-api
+          path: arduino-api.tar.zstd
+
+  build-board:
+    name: Build loader for ${{ matrix.board }}
+    runs-on: ubuntu-latest
+    needs:
+      - build-env
+    env:
+      CCACHE_IGNOREOPTIONS: -specs=*
+      OUTPUT_ARTIFACT: binaries-${{ matrix.board }}-${{ needs.build-env.outputs.CORE_HASH }}
+    strategy:
+      matrix:
+        include:
+          ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }}
+      fail-fast: false
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: /home/runner
+          name: build-env
+
+      - name: Restore build environment
+        run: |
+          (cd ~ && tar --use-compress-program=unzstd -xpf build-env.tar.zstd && rm build-env.tar.zstd)
+
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2
         with:
           verbose: 1
 
-      - name: Build variants
+      - name: Build loader
         shell: bash
         run: |
-          ./extra/build_all.sh -f
+          if ! ./extra/build.sh ${{ matrix.board }} 2> >(tee error.log) ; then
+            echo "### :x: ${{ matrix.board }} (\`${{ matrix.variant }}\`) build errors" > $GITHUB_STEP_SUMMARY
+            echo >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            cat error.log >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            exit 1
+          fi
 
-      - name: Package cores
+      - name: Package board binaries
+        if: ${{ !cancelled() }}
         run: |
-          jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do
-            ARTIFACT_NAME=ArduinoCore-${artifact}-${CORE_HASH}
-            ./extra/package_core.sh ${artifact} ${CORE_TAG} distrib/${ARTIFACT_NAME}.tar.bz2
-          done
+          tar chf - \
+            firmwares/*${{ matrix.variant }}* \
+            variants/${{ matrix.variant }}/ \
+            ${{ (job.status == 'failure') && format('build/{0}/', matrix.variant) }} \
+          | zstd > ${OUTPUT_ARTIFACT}.tar.zstd
 
-      - name: Archive cores
+      - name: Archive board binaries
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: ArduinoCore-archives-${{ env.CORE_HASH }}
-          path: distrib/*.tar.bz2
+          name: ${{ format('{0}{1}', (job.status == 'failure') && 'failed-' || '', env.OUTPUT_ARTIFACT) }}
+          path: ${{ env.OUTPUT_ARTIFACT }}.tar.zstd
 
-  split-core:
-    name: Split off ${{ matrix.artifact }}
+  package-core:
+    name: Package ${{ matrix.artifact }}
     runs-on: ubuntu-latest
-    needs: package-core
+    needs:
+      - build-env
+      - build-board
     env:
-      ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
-      CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }}
+      CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }}
+      CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }}
     strategy:
       matrix:
-        artifact: ${{ fromJSON( needs.package-core.outputs.ARTIFACTS ) }}
+        artifact: ${{ fromJSON( needs.build-env.outputs.ARTIFACTS ) }}
+      fail-fast: false
+    if: ${{ !cancelled() && needs.build-env.result == 'success' }}
     steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0
+          persist-credentials: false
+          fetch-tags: true
+
       - uses: actions/download-artifact@v4
         with:
-          name: ${{ env.ALL_CORES_ARTIFACT }}
+          path: .
+          name: arduino-api
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: .
+          pattern: binaries-*
+          merge-multiple: true
+
+      - name: Package core
+        run: |
+          rm -f cores/arduino/api # remove broken symlink
+          tar --use-compress-program=unzstd -xpf arduino-api.tar.zstd
+          for f in binaries-*.tar.zstd ; do
+            tar --use-compress-program=unzstd -xpf $f
+          done
+          ./extra/package_core.sh ${{ matrix.artifact }} ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2
 
       - uses: actions/upload-artifact@v4
+        if: ${{ success() || failure() }}
         with:
           name: ${{ env.CORE_ARTIFACT }}
-          path: ${{ env.CORE_ARTIFACT }}.tar.bz2
+          path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2
+
+  cleanup-build:
+    name: Clean up intermediates
+    runs-on: ubuntu-latest
+    needs:
+      - package-core
+    if: always()
+    steps:
+      - uses: geekyeggo/delete-artifact@v5.1.0
+        with:
+          name: |
+            arduino-api
+            binaries-*
+            build-env
+          failOnError: false
 
   test-core:
     name: Test ${{ matrix.subarch }}:${{ matrix.board }}
     runs-on: ubuntu-latest
     needs:
+      - build-env
       - package-core
-      - split-core
     strategy:
       matrix:
         include:
-          ${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }}
+          ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }}
       fail-fast: false
     env:
       PLAT: arduino:${{ matrix.subarch }}
       FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }}
-      CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }}
+      CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }}
+    if: ${{ !cancelled() && needs.build-env.result == 'success' }}
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -163,18 +253,19 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: ${{ success() || failure() }}
         with:
-          name: test-report-${{ needs.package-core.outputs.CORE_TAG }}-${{ matrix.board }}
+          name: test-report-${{ needs.build-env.outputs.CORE_TAG }}-${{ matrix.board }}
           path: sketches-reports/*
 
   collect-logs:
     name: Collect logs
     runs-on: ubuntu-latest
     needs:
+      - build-env
       - package-core
       - test-core
-    if: ${{ !cancelled() && needs.package-core.result == 'success' }}
+    if: ${{ !cancelled() && needs.build-env.result == 'success' }}
     env:
-      ALL_BOARD_DATA: ${{ needs.package-core.outputs.ALL_BOARD_DATA }}
+      ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }}
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -230,9 +321,10 @@ jobs:
     runs-on: ubuntu-latest
     if: cancelled() || contains(needs.*.result, 'failure')
     needs:
+      - build-env
       - package-core
       - test-core
-    steps: 
+    steps:
       - name: Notify failure
         run: exit 1
 
@@ -241,18 +333,19 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }}
     needs:
+      - build-env
       - package-core
       - test-core
     environment: production
     permissions:
       id-token: write
       contents: read
-    env:
-      ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: ${{ env.ALL_CORES_ARTIFACT }}
+          path: .
+          pattern: ArduinoCore-*
+          merge-multiple: true
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -266,18 +359,17 @@ jobs:
             aws s3 cp $f s3://${{ secrets.S3_BUCKET }}/
           done
 
-  publish-json:
-    name: Publish jsons
+  prepare-json:
+    name: Prepare jsons
     runs-on: ubuntu-latest
-    if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }}
     needs:
+      - build-env
       - package-core
       - test-core
     env:
-      ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
-      CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }}
-      CORE_HASH: ${{ needs.package-core.outputs.CORE_HASH }}
-      ARTIFACTS: ${{ needs.package-core.outputs.ARTIFACTS }}
+      CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }}
+      CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }}
+      ARTIFACTS: ${{ needs.build-env.outputs.ARTIFACTS }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -287,7 +379,9 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          name: ${{ env.ALL_CORES_ARTIFACT }}
+          path: .
+          pattern: ArduinoCore-*
+          merge-multiple: true
 
       - name: Prepare package index snippets
         run: |
@@ -302,17 +396,3 @@ jobs:
         with:
           name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons
           path: ArduinoCore-*-${{ env.CORE_TAG }}.json
-
-  cleanup-artifacts:
-    runs-on: ubuntu-latest
-    needs:
-      - package-core
-      - publish-core
-      - publish-json
-    if: ${{ !cancelled() }}
-    steps:
-      - name: Clean up intermediate artifacts
-        uses: geekyeggo/delete-artifact@v5.1.0
-        with:
-          name: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
-          failOnError: false

extra/get_variant_name.sh

@@ -4,8 +4,5 @@ set -e
 source venv/bin/activate
 
 # Get the variant name (NORMALIZED_BOARD_TARGET in Zephyr)
-tmpdir=$(mktemp -d)
 variant=$(cmake "-DBOARD=$1" -P extra/get_variant_name.cmake 2>/dev/null | grep 'VARIANT=' | cut -d '=' -f 2)
-rm -rf ${tmpdir}
-
 echo $variant

extra/package_core.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -e
+RET=0
 
 if [ -z "$1" ]; then
 	echo "Usage: $0 ARTIFACT VERSION [OUTPUT_FILE]"
@@ -16,6 +17,14 @@ ARTIFACT=$1
 VERSION=$2
 OUTPUT_FILE=${3:-distrib/${ARTIFACT}-${VERSION}.tar.bz2}
 
+log_msg() {
+	if [ -n $GITHUB_WORKSPACE ] ; then
+		echo "::$1::$2"
+	else
+		echo "$2"
+	fi
+}
+
 # we use variants for include because we filter on file paths
 # and boards for exclude because we want to remove matching lines in boards.txt
 BOARD_DETAILS=$(extra/get_board_details.sh)
@@ -31,7 +40,7 @@ else
 	EXCLUDED_BOARDS=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.artifact != \"$ARTIFACT\")) | .[].board")
 fi
 
-[ -n $GITHUB_WORKSPACE ] && echo "::group::Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))"
+log_msg group "Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))"
 
 # create a temporary boards.txt file with the correct list of boards
 TEMP_BOARDS=$(mktemp -p . | sed 's/\.\///')
@@ -50,6 +59,7 @@ cat platform.txt > ${TEMP_PLATFORM}
 sed -ie "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM}
 
 declutter_file() {
+	# remove comments and empty lines
 	[ -f "$1" ] || return 0
 	cat "$1" | sed -e 's/\s*#.*//' | grep -v '^\s*$'
 }
@@ -61,9 +71,13 @@ echo ${TEMP_PLATFORM} >> ${TEMP_INC}
 declutter_file extra/artifacts/_common.inc >> ${TEMP_INC}
 declutter_file extra/artifacts/$ARTIFACT.inc >> ${TEMP_INC}
 for variant in $INCLUDED_VARIANTS ; do
-	echo "::info::\`${variant}\`"
+	echo "- ${variant}"
 	echo "variants/${variant}/" >> ${TEMP_INC}
-	ls firmwares/zephyr-${variant}.* >> ${TEMP_INC}
+	# add the firmwares, if some are missing notify at end
+	if ! ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} ; then
+		log_msg error "No firmware for '${variant}' found."
+		RET=3
+	fi
 done
 
 # create the list of files and directories to exclude
@@ -78,4 +92,6 @@ tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \
 	--transform "s,^,ArduinoCore-zephyr/,"
 rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_BOARDS} ${TEMP_PLATFORM}
 
-[ -n $GITHUB_WORKSPACE ] && echo "::endgroup::"
+log_msg endgroup
+
+exit $RET