From bd88735456d4aafc6e72f2c4e77d8cc673c1d00b Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Wed, 19 Nov 2025 15:14:38 +0100 Subject: [PATCH 1/5] get_variant_name: remove redundant temp dir No longer useful for any purpose. Signed-off-by: Luca Burelli --- extra/get_variant_name.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/extra/get_variant_name.sh b/extra/get_variant_name.sh index a1eaba458..cc8399fe9 100755 --- a/extra/get_variant_name.sh +++ b/extra/get_variant_name.sh @@ -4,8 +4,5 @@ set -e source venv/bin/activate # Get the variant name (NORMALIZED_BOARD_TARGET in Zephyr) -tmpdir=$(mktemp -d) variant=$(cmake "-DBOARD=$1" -P extra/get_variant_name.cmake 2>/dev/null | grep 'VARIANT=' | cut -d '=' -f 2) -rm -rf ${tmpdir} - echo $variant From 8fc5c56e8c847a0798bb8428492a8f812ed7b0f3 Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Fri, 21 Nov 2025 12:06:23 +0100 Subject: [PATCH 2/5] ci/package_core: save and reload build environment Save the prepared build environment as an artifact after the initial setup, to allow for multiple jobs to reuse it without repeating the initial Zephyr configuration steps. Signed-off-by: Luca Burelli --- .github/workflows/package_core.yml | 70 ++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 05589f6a0..f79d6c538 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -6,11 +6,9 @@ on: jobs: - package-core: - name: Build and package cores + build-env: + name: Prepare build environment runs-on: ubuntu-latest - env: - CCACHE_IGNOREOPTIONS: -specs=* outputs: CORE_TAG: ${{ env.CORE_TAG }} CORE_HASH: ${{ env.CORE_HASH }} @@ -46,6 +44,32 @@ jobs: echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" + (cd && tar cphf - .cmake work zephyr-sdk-* | zstd > build-env.tar.zstd) + + - name: Archive build environment + uses: actions/upload-artifact@v4 + with: + name: build-env + path: ~/build-env.tar.zstd + + package-core: + name: Build and package cores + runs-on: ubuntu-latest + needs: + - build-env + env: + CCACHE_IGNOREOPTIONS: -specs=* + CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} + steps: + - uses: actions/download-artifact@v4 + with: + path: /home/runner + name: build-env + + - name: Restore build environment + run: | + (cd ~ && tar --use-compress-program=unzstd -xpf build-env.tar.zstd && rm build-env.tar.zstd) + - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: @@ -72,13 +96,15 @@ jobs: split-core: name: Split off ${{ matrix.artifact }} runs-on: ubuntu-latest - needs: package-core + needs: + - build-env + - package-core env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} - CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} strategy: matrix: - artifact: ${{ fromJSON( needs.package-core.outputs.ARTIFACTS ) }} + artifact: ${{ fromJSON( needs.build-env.outputs.ARTIFACTS ) }} steps: - uses: actions/download-artifact@v4 with: @@ -93,17 +119,18 @@ jobs: name: Test ${{ matrix.subarch }}:${{ matrix.board }} runs-on: ubuntu-latest needs: + - build-env - package-core - split-core strategy: matrix: include: - ${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }} + ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }} fail-fast: false env: PLAT: arduino:${{ matrix.subarch }} FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }} - CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: @@ -163,18 +190,19 @@ jobs: - uses: actions/upload-artifact@v4 if: ${{ success() || failure() }} with: - name: test-report-${{ needs.package-core.outputs.CORE_TAG }}-${{ matrix.board }} + name: test-report-${{ needs.build-env.outputs.CORE_TAG }}-${{ matrix.board }} path: sketches-reports/* collect-logs: name: Collect logs runs-on: ubuntu-latest needs: + - build-env - package-core - test-core if: ${{ !cancelled() && needs.package-core.result == 'success' }} env: - ALL_BOARD_DATA: ${{ needs.package-core.outputs.ALL_BOARD_DATA }} + ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }} steps: - uses: actions/download-artifact@v4 with: @@ -230,9 +258,10 @@ jobs: runs-on: ubuntu-latest if: cancelled() || contains(needs.*.result, 'failure') needs: + - build-env - package-core - test-core - steps: + steps: - name: Notify failure run: exit 1 @@ -241,6 +270,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} needs: + - build-env - package-core - test-core environment: production @@ -248,7 +278,7 @@ jobs: id-token: write contents: read env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: @@ -271,13 +301,14 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }} needs: + - build-env - package-core - test-core env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} - CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }} - CORE_HASH: ${{ needs.package-core.outputs.CORE_HASH }} - ARTIFACTS: ${{ needs.package-core.outputs.ARTIFACTS }} + ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} + CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} + CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }} + ARTIFACTS: ${{ needs.build-env.outputs.ARTIFACTS }} steps: - uses: actions/checkout@v4 with: @@ -306,6 +337,7 @@ jobs: cleanup-artifacts: runs-on: ubuntu-latest needs: + - build-env - package-core - publish-core - publish-json @@ -314,5 +346,5 @@ jobs: - name: Clean up intermediate artifacts uses: geekyeggo/delete-artifact@v5.1.0 with: - name: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} + name: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} failOnError: false From 1fc608befb2a52543a9c2d137f1b723f421b53aa Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Mon, 24 Nov 2025 18:45:42 +0100 Subject: [PATCH 3/5] ci/package_core: use matrix for building board binaries Convert the sequential building and packaging of all board binaries into a matrix job, allowing parallel execution. This makes the workflow more efficient and independent of the number of supported boards. Signed-off-by: Luca Burelli --- .github/workflows/package_core.yml | 116 ++++++++++++++++++----------- 1 file changed, 73 insertions(+), 43 deletions(-) diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index f79d6c538..225692fe6 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -45,6 +45,7 @@ jobs: echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" (cd && tar cphf - .cmake work zephyr-sdk-* | zstd > build-env.tar.zstd) + tar cphf - cores/arduino/api | zstd > arduino-api.tar.zstd - name: Archive build environment uses: actions/upload-artifact@v4 @@ -52,14 +53,25 @@ jobs: name: build-env path: ~/build-env.tar.zstd - package-core: - name: Build and package cores + - name: Archive API snapshot + uses: actions/upload-artifact@v4 + with: + name: arduino-api + path: arduino-api.tar.zstd + + build-board: + name: Build loader for ${{ matrix.board }} runs-on: ubuntu-latest needs: - build-env env: CCACHE_IGNOREOPTIONS: -specs=* - CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} + OUTPUT_ARTIFACT: binaries-${{ matrix.board }}-${{ needs.build-env.outputs.CORE_HASH }} + strategy: + matrix: + include: + ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }} + fail-fast: false steps: - uses: actions/download-artifact@v4 with: @@ -75,45 +87,79 @@ jobs: with: verbose: 1 - - name: Build variants + - name: Build loader shell: bash run: | - ./extra/build_all.sh -f + ./extra/build.sh ${{ matrix.board }} - - name: Package cores + - name: Package board binaries run: | - jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do - ARTIFACT_NAME=ArduinoCore-${artifact}-${CORE_HASH} - ./extra/package_core.sh ${artifact} ${CORE_TAG} distrib/${ARTIFACT_NAME}.tar.bz2 - done + tar chf - firmwares/*${{ matrix.variant }}* variants/${{ matrix.variant }}/ | zstd > ${OUTPUT_ARTIFACT}.tar.zstd - - name: Archive cores + - name: Archive board binaries uses: actions/upload-artifact@v4 with: - name: ArduinoCore-archives-${{ env.CORE_HASH }} - path: distrib/*.tar.bz2 + name: ${{ env.OUTPUT_ARTIFACT }} + path: ${{ env.OUTPUT_ARTIFACT }}.tar.zstd - split-core: - name: Split off ${{ matrix.artifact }} + package-core: + name: Package ${{ matrix.artifact }} runs-on: ubuntu-latest needs: - build-env - - package-core + - build-board env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} + CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} strategy: matrix: artifact: ${{ fromJSON( needs.build-env.outputs.ARTIFACTS ) }} steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 + persist-credentials: false + fetch-tags: true + - uses: actions/download-artifact@v4 with: - name: ${{ env.ALL_CORES_ARTIFACT }} + path: . + name: arduino-api + + - uses: actions/download-artifact@v4 + with: + path: . + pattern: binaries-* + merge-multiple: true + + - name: Package core + run: | + rm -f cores/arduino/api # remove broken symlink + tar --use-compress-program=unzstd -xpf arduino-api.tar.zstd + for f in binaries-*.tar.zstd ; do + tar --use-compress-program=unzstd -xpf $f + done + ./extra/package_core.sh ${{ matrix.artifact }} ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2 - uses: actions/upload-artifact@v4 with: name: ${{ env.CORE_ARTIFACT }} - path: ${{ env.CORE_ARTIFACT }}.tar.bz2 + path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 + + cleanup-build: + name: Clean up intermediates + runs-on: ubuntu-latest + needs: + - package-core + steps: + - uses: geekyeggo/delete-artifact@v5.1.0 + with: + name: | + arduino-api + binaries-* + build-env + failOnError: false test-core: name: Test ${{ matrix.subarch }}:${{ matrix.board }} @@ -121,7 +167,6 @@ jobs: needs: - build-env - package-core - - split-core strategy: matrix: include: @@ -277,12 +322,12 @@ jobs: permissions: id-token: write contents: read - env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: - name: ${{ env.ALL_CORES_ARTIFACT }} + path: . + pattern: ArduinoCore-* + merge-multiple: true - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -296,16 +341,14 @@ jobs: aws s3 cp $f s3://${{ secrets.S3_BUCKET }}/ done - publish-json: - name: Publish jsons + prepare-json: + name: Prepare jsons runs-on: ubuntu-latest - if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }} needs: - build-env - package-core - test-core env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }} ARTIFACTS: ${{ needs.build-env.outputs.ARTIFACTS }} @@ -318,7 +361,9 @@ jobs: - uses: actions/download-artifact@v4 with: - name: ${{ env.ALL_CORES_ARTIFACT }} + path: . + pattern: ArduinoCore-* + merge-multiple: true - name: Prepare package index snippets run: | @@ -333,18 +378,3 @@ jobs: with: name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons path: ArduinoCore-*-${{ env.CORE_TAG }}.json - - cleanup-artifacts: - runs-on: ubuntu-latest - needs: - - build-env - - package-core - - publish-core - - publish-json - if: ${{ !cancelled() }} - steps: - - name: Clean up intermediate artifacts - uses: geekyeggo/delete-artifact@v5.1.0 - with: - name: ArduinoCore-archives-${{ needs.build-env.outputs.CORE_HASH }} - failOnError: false From 0ccd7654927ea2ec6d72847b5f620096f6bf5cad Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Mon, 24 Nov 2025 18:53:37 +0100 Subject: [PATCH 4/5] package_core: fix GitHub Actions logging syntax Use a common function to handle logging messages for GitHub Actions and for the console, for a cleaner output. Signed-off-by: Luca Burelli :1 --- extra/package_core.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/extra/package_core.sh b/extra/package_core.sh index bc95aec1b..47cb98b6d 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -16,6 +16,14 @@ ARTIFACT=$1 VERSION=$2 OUTPUT_FILE=${3:-distrib/${ARTIFACT}-${VERSION}.tar.bz2} +log_msg() { + if [ -n $GITHUB_WORKSPACE ] ; then + echo "::$1::$2" + else + echo "$2" + fi +} + # we use variants for include because we filter on file paths # and boards for exclude because we want to remove matching lines in boards.txt BOARD_DETAILS=$(extra/get_board_details.sh) @@ -31,7 +39,7 @@ else EXCLUDED_BOARDS=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.artifact != \"$ARTIFACT\")) | .[].board") fi -[ -n $GITHUB_WORKSPACE ] && echo "::group::Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))" +log_msg group "Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))" # create a temporary boards.txt file with the correct list of boards TEMP_BOARDS=$(mktemp -p . | sed 's/\.\///') @@ -50,6 +58,7 @@ cat platform.txt > ${TEMP_PLATFORM} sed -ie "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM} declutter_file() { + # remove comments and empty lines [ -f "$1" ] || return 0 cat "$1" | sed -e 's/\s*#.*//' | grep -v '^\s*$' } @@ -61,7 +70,7 @@ echo ${TEMP_PLATFORM} >> ${TEMP_INC} declutter_file extra/artifacts/_common.inc >> ${TEMP_INC} declutter_file extra/artifacts/$ARTIFACT.inc >> ${TEMP_INC} for variant in $INCLUDED_VARIANTS ; do - echo "::info::\`${variant}\`" + echo "- ${variant}" echo "variants/${variant}/" >> ${TEMP_INC} ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} done @@ -78,4 +87,4 @@ tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \ --transform "s,^,ArduinoCore-zephyr/," rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_BOARDS} ${TEMP_PLATFORM} -[ -n $GITHUB_WORKSPACE ] && echo "::endgroup::" +log_msg endgroup From 5a55b1192859912e8258af5f8d6bcfb066f97f87 Mon Sep 17 00:00:00 2001 From: Luca Burelli Date: Mon, 24 Nov 2025 19:28:12 +0100 Subject: [PATCH 5/5] ci/package_core: proceed with testing as much as possible These changes allow the packaging and testing jobs to start even if some of the build jobs have resulted in errors, thus maximizing the amount of testing that can be performed in parallel CI workflows even if one board is causing issues. The build step artifacts will be saved with a "failed-" prefix in case of errors and can be inspected for further information. Signed-off-by: Luca Burelli --- .github/workflows/package_core.yml | 26 ++++++++++++++++++++++---- extra/package_core.sh | 9 ++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 225692fe6..cc88a5f89 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -90,16 +90,29 @@ jobs: - name: Build loader shell: bash run: | - ./extra/build.sh ${{ matrix.board }} + if ! ./extra/build.sh ${{ matrix.board }} 2> >(tee error.log) ; then + echo "### :x: ${{ matrix.board }} (\`${{ matrix.variant }}\`) build errors" > $GITHUB_STEP_SUMMARY + echo >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + cat error.log >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + exit 1 + fi - name: Package board binaries + if: ${{ !cancelled() }} run: | - tar chf - firmwares/*${{ matrix.variant }}* variants/${{ matrix.variant }}/ | zstd > ${OUTPUT_ARTIFACT}.tar.zstd + tar chf - \ + firmwares/*${{ matrix.variant }}* \ + variants/${{ matrix.variant }}/ \ + ${{ (job.status == 'failure') && format('build/{0}/', matrix.variant) }} \ + | zstd > ${OUTPUT_ARTIFACT}.tar.zstd - name: Archive board binaries + if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: - name: ${{ env.OUTPUT_ARTIFACT }} + name: ${{ format('{0}{1}', (job.status == 'failure') && 'failed-' || '', env.OUTPUT_ARTIFACT) }} path: ${{ env.OUTPUT_ARTIFACT }}.tar.zstd package-core: @@ -114,6 +127,8 @@ jobs: strategy: matrix: artifact: ${{ fromJSON( needs.build-env.outputs.ARTIFACTS ) }} + fail-fast: false + if: ${{ !cancelled() && needs.build-env.result == 'success' }} steps: - uses: actions/checkout@v4 with: @@ -143,6 +158,7 @@ jobs: ./extra/package_core.sh ${{ matrix.artifact }} ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2 - uses: actions/upload-artifact@v4 + if: ${{ success() || failure() }} with: name: ${{ env.CORE_ARTIFACT }} path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 @@ -152,6 +168,7 @@ jobs: runs-on: ubuntu-latest needs: - package-core + if: always() steps: - uses: geekyeggo/delete-artifact@v5.1.0 with: @@ -176,6 +193,7 @@ jobs: PLAT: arduino:${{ matrix.subarch }} FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }} CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} + if: ${{ !cancelled() && needs.build-env.result == 'success' }} steps: - uses: actions/download-artifact@v4 with: @@ -245,7 +263,7 @@ jobs: - build-env - package-core - test-core - if: ${{ !cancelled() && needs.package-core.result == 'success' }} + if: ${{ !cancelled() && needs.build-env.result == 'success' }} env: ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }} steps: diff --git a/extra/package_core.sh b/extra/package_core.sh index 47cb98b6d..191678bc3 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +RET=0 if [ -z "$1" ]; then echo "Usage: $0 ARTIFACT VERSION [OUTPUT_FILE]" @@ -72,7 +73,11 @@ declutter_file extra/artifacts/$ARTIFACT.inc >> ${TEMP_INC} for variant in $INCLUDED_VARIANTS ; do echo "- ${variant}" echo "variants/${variant}/" >> ${TEMP_INC} - ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} + # add the firmwares, if some are missing notify at end + if ! ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} ; then + log_msg error "No firmware for '${variant}' found." + RET=3 + fi done # create the list of files and directories to exclude @@ -88,3 +93,5 @@ tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \ rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_BOARDS} ${TEMP_PLATFORM} log_msg endgroup + +exit $RET