diff --git a/.github/workflows/package_core.yml b/.github/workflows/package_core.yml index 05589f6a..cc88a5f8 100644 --- a/.github/workflows/package_core.yml +++ b/.github/workflows/package_core.yml @@ -6,11 +6,9 @@ on: jobs: - package-core: - name: Build and package cores + build-env: + name: Prepare build environment runs-on: ubuntu-latest - env: - CCACHE_IGNOREOPTIONS: -specs=* outputs: CORE_TAG: ${{ env.CORE_TAG }} CORE_HASH: ${{ env.CORE_HASH }} @@ -46,64 +44,156 @@ jobs: echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" + (cd && tar cphf - .cmake work zephyr-sdk-* | zstd > build-env.tar.zstd) + tar cphf - cores/arduino/api | zstd > arduino-api.tar.zstd + + - name: Archive build environment + uses: actions/upload-artifact@v4 + with: + name: build-env + path: ~/build-env.tar.zstd + + - name: Archive API snapshot + uses: actions/upload-artifact@v4 + with: + name: arduino-api + path: arduino-api.tar.zstd + + build-board: + name: Build loader for ${{ matrix.board }} + runs-on: ubuntu-latest + needs: + - build-env + env: + CCACHE_IGNOREOPTIONS: -specs=* + OUTPUT_ARTIFACT: binaries-${{ matrix.board }}-${{ needs.build-env.outputs.CORE_HASH }} + strategy: + matrix: + include: + ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }} + fail-fast: false + steps: + - uses: actions/download-artifact@v4 + with: + path: /home/runner + name: build-env + + - name: Restore build environment + run: | + (cd ~ && tar --use-compress-program=unzstd -xpf build-env.tar.zstd && rm build-env.tar.zstd) + - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: verbose: 1 - - name: Build variants + - name: Build loader shell: bash run: | - ./extra/build_all.sh -f + if ! ./extra/build.sh ${{ matrix.board }} 2> >(tee error.log) ; then + echo "### :x: ${{ matrix.board }} (\`${{ matrix.variant }}\`) build errors" > $GITHUB_STEP_SUMMARY + echo >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + cat error.log >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + exit 1 + fi - - name: Package cores + - name: Package board binaries + if: ${{ !cancelled() }} run: | - jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do - ARTIFACT_NAME=ArduinoCore-${artifact}-${CORE_HASH} - ./extra/package_core.sh ${artifact} ${CORE_TAG} distrib/${ARTIFACT_NAME}.tar.bz2 - done + tar chf - \ + firmwares/*${{ matrix.variant }}* \ + variants/${{ matrix.variant }}/ \ + ${{ (job.status == 'failure') && format('build/{0}/', matrix.variant) }} \ + | zstd > ${OUTPUT_ARTIFACT}.tar.zstd - - name: Archive cores + - name: Archive board binaries + if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: - name: ArduinoCore-archives-${{ env.CORE_HASH }} - path: distrib/*.tar.bz2 + name: ${{ format('{0}{1}', (job.status == 'failure') && 'failed-' || '', env.OUTPUT_ARTIFACT) }} + path: ${{ env.OUTPUT_ARTIFACT }}.tar.zstd - split-core: - name: Split off ${{ matrix.artifact }} + package-core: + name: Package ${{ matrix.artifact }} runs-on: ubuntu-latest - needs: package-core + needs: + - build-env + - build-board env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} - CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} + CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} strategy: matrix: - artifact: ${{ fromJSON( needs.package-core.outputs.ARTIFACTS ) }} + artifact: ${{ fromJSON( needs.build-env.outputs.ARTIFACTS ) }} + fail-fast: false + if: ${{ !cancelled() && needs.build-env.result == 'success' }} steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 + persist-credentials: false + fetch-tags: true + - uses: actions/download-artifact@v4 with: - name: ${{ env.ALL_CORES_ARTIFACT }} + path: . + name: arduino-api + + - uses: actions/download-artifact@v4 + with: + path: . + pattern: binaries-* + merge-multiple: true + + - name: Package core + run: | + rm -f cores/arduino/api # remove broken symlink + tar --use-compress-program=unzstd -xpf arduino-api.tar.zstd + for f in binaries-*.tar.zstd ; do + tar --use-compress-program=unzstd -xpf $f + done + ./extra/package_core.sh ${{ matrix.artifact }} ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2 - uses: actions/upload-artifact@v4 + if: ${{ success() || failure() }} with: name: ${{ env.CORE_ARTIFACT }} - path: ${{ env.CORE_ARTIFACT }}.tar.bz2 + path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 + + cleanup-build: + name: Clean up intermediates + runs-on: ubuntu-latest + needs: + - package-core + if: always() + steps: + - uses: geekyeggo/delete-artifact@v5.1.0 + with: + name: | + arduino-api + binaries-* + build-env + failOnError: false test-core: name: Test ${{ matrix.subarch }}:${{ matrix.board }} runs-on: ubuntu-latest needs: + - build-env - package-core - - split-core strategy: matrix: include: - ${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }} + ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }} fail-fast: false env: PLAT: arduino:${{ matrix.subarch }} FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }} - CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }} + CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} + if: ${{ !cancelled() && needs.build-env.result == 'success' }} steps: - uses: actions/download-artifact@v4 with: @@ -163,18 +253,19 @@ jobs: - uses: actions/upload-artifact@v4 if: ${{ success() || failure() }} with: - name: test-report-${{ needs.package-core.outputs.CORE_TAG }}-${{ matrix.board }} + name: test-report-${{ needs.build-env.outputs.CORE_TAG }}-${{ matrix.board }} path: sketches-reports/* collect-logs: name: Collect logs runs-on: ubuntu-latest needs: + - build-env - package-core - test-core - if: ${{ !cancelled() && needs.package-core.result == 'success' }} + if: ${{ !cancelled() && needs.build-env.result == 'success' }} env: - ALL_BOARD_DATA: ${{ needs.package-core.outputs.ALL_BOARD_DATA }} + ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }} steps: - uses: actions/download-artifact@v4 with: @@ -230,9 +321,10 @@ jobs: runs-on: ubuntu-latest if: cancelled() || contains(needs.*.result, 'failure') needs: + - build-env - package-core - test-core - steps: + steps: - name: Notify failure run: exit 1 @@ -241,18 +333,19 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} needs: + - build-env - package-core - test-core environment: production permissions: id-token: write contents: read - env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} steps: - uses: actions/download-artifact@v4 with: - name: ${{ env.ALL_CORES_ARTIFACT }} + path: . + pattern: ArduinoCore-* + merge-multiple: true - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -266,18 +359,17 @@ jobs: aws s3 cp $f s3://${{ secrets.S3_BUCKET }}/ done - publish-json: - name: Publish jsons + prepare-json: + name: Prepare jsons runs-on: ubuntu-latest - if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }} needs: + - build-env - package-core - test-core env: - ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} - CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }} - CORE_HASH: ${{ needs.package-core.outputs.CORE_HASH }} - ARTIFACTS: ${{ needs.package-core.outputs.ARTIFACTS }} + CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} + CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }} + ARTIFACTS: ${{ needs.build-env.outputs.ARTIFACTS }} steps: - uses: actions/checkout@v4 with: @@ -287,7 +379,9 @@ jobs: - uses: actions/download-artifact@v4 with: - name: ${{ env.ALL_CORES_ARTIFACT }} + path: . + pattern: ArduinoCore-* + merge-multiple: true - name: Prepare package index snippets run: | @@ -302,17 +396,3 @@ jobs: with: name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons path: ArduinoCore-*-${{ env.CORE_TAG }}.json - - cleanup-artifacts: - runs-on: ubuntu-latest - needs: - - package-core - - publish-core - - publish-json - if: ${{ !cancelled() }} - steps: - - name: Clean up intermediate artifacts - uses: geekyeggo/delete-artifact@v5.1.0 - with: - name: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }} - failOnError: false diff --git a/extra/get_variant_name.sh b/extra/get_variant_name.sh index a1eaba45..cc8399fe 100755 --- a/extra/get_variant_name.sh +++ b/extra/get_variant_name.sh @@ -4,8 +4,5 @@ set -e source venv/bin/activate # Get the variant name (NORMALIZED_BOARD_TARGET in Zephyr) -tmpdir=$(mktemp -d) variant=$(cmake "-DBOARD=$1" -P extra/get_variant_name.cmake 2>/dev/null | grep 'VARIANT=' | cut -d '=' -f 2) -rm -rf ${tmpdir} - echo $variant diff --git a/extra/package_core.sh b/extra/package_core.sh index bc95aec1..191678bc 100755 --- a/extra/package_core.sh +++ b/extra/package_core.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +RET=0 if [ -z "$1" ]; then echo "Usage: $0 ARTIFACT VERSION [OUTPUT_FILE]" @@ -16,6 +17,14 @@ ARTIFACT=$1 VERSION=$2 OUTPUT_FILE=${3:-distrib/${ARTIFACT}-${VERSION}.tar.bz2} +log_msg() { + if [ -n $GITHUB_WORKSPACE ] ; then + echo "::$1::$2" + else + echo "$2" + fi +} + # we use variants for include because we filter on file paths # and boards for exclude because we want to remove matching lines in boards.txt BOARD_DETAILS=$(extra/get_board_details.sh) @@ -31,7 +40,7 @@ else EXCLUDED_BOARDS=$(echo ${BOARD_DETAILS} | jq -cr "map(select(.artifact != \"$ARTIFACT\")) | .[].board") fi -[ -n $GITHUB_WORKSPACE ] && echo "::group::Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))" +log_msg group "Packaging ${ARTIFACT_NAME:-all variants} ($(basename $OUTPUT_FILE))" # create a temporary boards.txt file with the correct list of boards TEMP_BOARDS=$(mktemp -p . | sed 's/\.\///') @@ -50,6 +59,7 @@ cat platform.txt > ${TEMP_PLATFORM} sed -ie "s/^version=.*/version=$(extra/get_core_version.sh)/" ${TEMP_PLATFORM} declutter_file() { + # remove comments and empty lines [ -f "$1" ] || return 0 cat "$1" | sed -e 's/\s*#.*//' | grep -v '^\s*$' } @@ -61,9 +71,13 @@ echo ${TEMP_PLATFORM} >> ${TEMP_INC} declutter_file extra/artifacts/_common.inc >> ${TEMP_INC} declutter_file extra/artifacts/$ARTIFACT.inc >> ${TEMP_INC} for variant in $INCLUDED_VARIANTS ; do - echo "::info::\`${variant}\`" + echo "- ${variant}" echo "variants/${variant}/" >> ${TEMP_INC} - ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} + # add the firmwares, if some are missing notify at end + if ! ls firmwares/zephyr-${variant}.* >> ${TEMP_INC} ; then + log_msg error "No firmware for '${variant}' found." + RET=3 + fi done # create the list of files and directories to exclude @@ -78,4 +92,6 @@ tar -cjhf ${OUTPUT_FILE} -X ${TEMP_EXC} -T ${TEMP_INC} \ --transform "s,^,ArduinoCore-zephyr/," rm -f ${TEMP_INC} ${TEMP_EXC} ${TEMP_BOARDS} ${TEMP_PLATFORM} -[ -n $GITHUB_WORKSPACE ] && echo "::endgroup::" +log_msg endgroup + +exit $RET