diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml new file mode 100644 index 000000000..d158f8816 --- /dev/null +++ b/.github/workflows/cron.yml @@ -0,0 +1,39 @@ +name: Cron Build + +on: + schedule: +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) +# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) +# │ │ │ │ │ +# │ │ │ │ │ +# │ │ │ │ │ +# * * * * * + - cron: '0 */6 * * *' + +jobs: + run: + name: Build with IDF ${{ matrix.idf_branch }} + runs-on: ubuntu-latest + + strategy: + matrix: + idf_branch: [release/v3.2] + steps: + - uses: actions/checkout@v1 + - name: Install dependencies + run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache + - name: Install Python Wheel + run: pip install wheel + - name: Build + env: + GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }} + IDF_BRANCH: ${{ matrix.idf_branch }} + run: bash ./tools/cron.sh + - name: Upload archive + uses: actions/upload-artifact@v1 + with: + name: artifacts + path: dist diff --git a/.github/workflows/main.yml b/.github/workflows/push.yml similarity index 78% rename from .github/workflows/main.yml rename to .github/workflows/push.yml index 896ee706f..299c78948 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/push.yml @@ -6,7 +6,6 @@ on: - master - release/* pull_request: - repository_dispatch: jobs: @@ -20,11 +19,9 @@ jobs: - name: Install Python Wheel run: pip install wheel - name: Build Arduino Libs - env: - TRAVIS_BUILD_DIR: ${{ github.workspace }} - run: bash $TRAVIS_BUILD_DIR/build.sh - - name: Archive libs + run: bash ./build.sh + - name: Upload archive uses: actions/upload-artifact@v1 with: - name: arduino-libs + name: artifacts path: dist diff --git a/.github/workflows/repository_dispatch.yml b/.github/workflows/repository_dispatch.yml new file mode 100644 index 000000000..5a8b072cb --- /dev/null +++ b/.github/workflows/repository_dispatch.yml @@ -0,0 +1,23 @@ +name: Remote Trigger + +on: repository_dispatch + +jobs: + run: + name: Dispatch Event + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Install dependencies + run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache + - name: Install Python Wheel + run: pip install wheel + - name: Handle Event + env: + GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }} + run: bash ./tools/repository_dispatch.sh + - name: Upload archive + uses: actions/upload-artifact@v1 + with: + name: artifacts + path: dist diff --git a/build.sh b/build.sh index 553384329..aea6f7b23 100755 --- a/build.sh +++ b/build.sh @@ -35,14 +35,16 @@ if ! [ -x "$(command -v stat)" ]; then exit 1 fi -# install esp-idf and gcc toolchain -source ./tools/install-esp-idf.sh -if [ $? -ne 0 ]; then exit 1; fi +mkdir -p dist # update components from git ./tools/update-components.sh if [ $? -ne 0 ]; then exit 1; fi +# install esp-idf and gcc toolchain +source ./tools/install-esp-idf.sh +if [ $? -ne 0 ]; then exit 1; fi + # build and prepare libs ./tools/build-libs.sh if [ $? -ne 0 ]; then exit 1; fi diff --git a/tools/archive-build.sh b/tools/archive-build.sh index 05129026a..d6682ff97 100755 --- a/tools/archive-build.sh +++ b/tools/archive-build.sh @@ -1,10 +1,16 @@ #!/bin/bash +IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD) +IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD) + idf_version_string=${IDF_BRANCH//\//_}"-$IDF_COMMIT" archive_path="dist/arduino-esp32-libs-$idf_version_string.tar.gz" +build_archive_path="dist/arduino-esp32-build-$idf_version_string.tar.gz" -mkdir -p dist && \ -rm -rf $archive_path && \ -cd out && \ -tar zcf ../$archive_path * \ -&& cd .. +mkdir -p dist && rm -rf "$archive_path" "$build_archive_path" +if [ -d "out" ]; then + cd out && tar zcf "../$archive_path" * && cd .. +fi +if [ -d "build" ]; then + cd build && tar zcf "../$build_archive_path" * && cd .. +fi diff --git a/tools/config.sh b/tools/config.sh index 89c80e0bd..02c79320f 100755 --- a/tools/config.sh +++ b/tools/config.sh @@ -1,6 +1,5 @@ #!/bin/bash -IDF_REPO="https://github.com/espressif/esp-idf.git" -IDF_BRANCH="release/v3.2" + IDF_COMPS="$IDF_PATH/components" IDF_TOOLCHAIN="xtensa-esp32-elf" IDF_TOOLCHAIN_LINUX_ARMEL="https://dl.espressif.com/dl/xtensa-esp32-elf-linux-armel-1.22.0-87-gb57bad3-5.2.0.tar.gz" @@ -9,10 +8,25 @@ IDF_TOOLCHAIN_LINUX64="https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22 IDF_TOOLCHAIN_WIN32="https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip" IDF_TOOLCHAIN_MACOS="https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz" -CAMERA_REPO="https://github.com/espressif/esp32-camera.git" -FACE_REPO="https://github.com/espressif/esp-face.git" +if [ -z $IDF_BRANCH ]; then + IDF_BRANCH="release/v3.2" +fi + +# Owner of the target ESP32 Arduino repository +AR_USER="espressif" + +# The full name of the repository +AR_REPO="$AR_USER/arduino-esp32" + +IDF_REPO_URL="https://github.com/espressif/esp-idf.git" +CAMERA_REPO_URL="https://github.com/espressif/esp32-camera.git" +FACE_REPO_URL="https://github.com/espressif/esp-face.git" +AR_REPO_URL="https://github.com/$AR_REPO.git" + +if [ -n $GITHUB_TOKEN ]; then + AR_REPO_URL="https://$GITHUB_TOKEN@github.com/$AR_REPO.git" +fi -AR_REPO="https://github.com/espressif/arduino-esp32.git" AR_ROOT="$PWD" AR_COMPS="$AR_ROOT/components" AR_OUT="$AR_ROOT/out" @@ -56,3 +70,38 @@ if [[ "$AR_OS" == "macos" ]]; then export SED="gsed" export SSTAT="stat -f %z" fi + +function git_commit_exists(){ #git_commit_exists + local repo_path="$1" + local commit_message="$2" + local commits_found=`git -C "$repo_path" log --all --grep="$commit_message" | grep commit` + if [ -n "$commits_found" ]; then echo 1; else echo 0; fi +} + +function git_branch_exists(){ # git_branch_exists + local repo_path="$1" + local branch_name="$2" + local branch_found=`git -C "$repo_path" ls-remote --heads origin "$branch_name"` + if [ -n "$branch_found" ]; then echo 1; else echo 0; fi +} + +function git_pr_exists(){ # git_pr_exists + local pr_num=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$AR_REPO/pulls?head=$AR_USER:$1&state=open" | jq -r '.[].number'` + if [ ! "$pr_num" == "" ] && [ ! "$pr_num" == "null" ]; then echo 1; else echo 0; fi +} + +function git_create_pr(){ # git_create_pr + local pr_branch="$1" + local pr_title="$2" + local pr_body="" + for component in `ls "$AR_COMPS"`; do + if [ ! $component == "arduino" ] && [ -d "$AR_COMPS/$component/.git" ]; then + pr_body+="$component: "$(git -C "$AR_COMPS/$component" symbolic-ref --short HEAD)" "$(git -C "$AR_COMPS/$component" rev-parse --short HEAD)"\r\n" + fi + done + local pr_data="{\"title\": \"$pr_title\", \"body\": \"$pr_body\", \"head\": \"$AR_USER:$pr_branch\", \"base\": \"master\"}" + git_create_pr_res=`echo "$pr_data" | curl -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" --data @- "https://api.github.com/repos/$AR_REPO/pulls"` + local done_pr=`echo "$git_create_pr_res" | jq -r '.title'` + if [ ! "$done_pr" == "" ] && [ ! "$done_pr" == "null" ]; then echo 1; else echo 0; fi +} + diff --git a/tools/cron.sh b/tools/cron.sh new file mode 100644 index 000000000..6122b1eed --- /dev/null +++ b/tools/cron.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then + echo "Wrong event '$GITHUB_EVENT_NAME'!" + exit 1 +fi + +git checkout "$IDF_BRANCH" #local branches should match what the matrix wants to build +source ./build.sh +bash ./tools/push-to-arduino.sh diff --git a/tools/install-esp-idf.sh b/tools/install-esp-idf.sh index e825c3621..ad62e7155 100755 --- a/tools/install-esp-idf.sh +++ b/tools/install-esp-idf.sh @@ -7,26 +7,102 @@ if ! [ -x "$(command -v $SED)" ]; then exit 1 fi +# +# CLONE ESP-IDF +# + if [ -z "$IDF_PATH" ]; then echo "ESP-IDF is not installed! Installing local copy" + idf_was_installed="1" if ! [ -d esp-idf ]; then - git clone $IDF_REPO -b $IDF_BRANCH + git clone $IDF_REPO_URL -b $IDF_BRANCH fi export IDF_PATH="$AR_ROOT/esp-idf" - cd $IDF_PATH - git fetch origin && git pull origin $IDF_BRANCH - git submodule update --init --recursive - python -m pip install -r requirements.txt - cd "$AR_ROOT" fi if [ "$IDF_COMMIT" ]; then - git -C $IDF_PATH checkout $IDF_COMMIT - git -C $IDF_PATH submodule update + git -C "$IDF_PATH" checkout "$IDF_COMMIT" + commit_predefined="1" +fi + +export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD) +export IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD) + +# +# SETUP ARDUINO DEPLOY +# + +if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then + # format new branch name and pr title + if [ -x $commit_predefined ]; then #commit was not specified at build time + AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH" + AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT" + AR_NEW_PR_TITLE="IDF $IDF_BRANCH" + else + AR_NEW_BRANCH_NAME="idf-$IDF_COMMIT" + AR_NEW_COMMIT_MESSAGE="IDF $IDF_COMMIT" + AR_NEW_PR_TITLE="$AR_NEW_COMMIT_MESSAGE" + fi + + AR_HAS_COMMIT=`git_commit_exists "$AR_COMPS/arduino" "$AR_NEW_COMMIT_MESSAGE"` + AR_HAS_BRANCH=`git_branch_exists "$AR_COMPS/arduino" "$AR_NEW_BRANCH_NAME"` + AR_HAS_PR=`git_pr_exists "$AR_NEW_BRANCH_NAME"` + + if [ "$AR_HAS_COMMIT" == "1" ]; then + echo "Commit '$AR_NEW_COMMIT_MESSAGE' Already Exists" + exit 0 + fi + + if [ "$AR_HAS_BRANCH" == "1" ]; then + echo "Branch '$AR_NEW_BRANCH_NAME' Already Exists" + fi + + if [ "$AR_HAS_PR" == "1" ]; then + echo "PR '$AR_NEW_PR_TITLE' Already Exists" + fi + + # setup git for pushing + git config --global github.user "$GITHUB_ACTOR" + git config --global user.name "$GITHUB_ACTOR" + git config --global user.email "$GITHUB_ACTOR@github.com" + + # create or checkout the branch + if [ ! $AR_HAS_BRANCH == "0" ]; then + echo "Switching to arduino branch '$AR_NEW_BRANCH_NAME'..." + git -C "$AR_COMPS/arduino" checkout $AR_NEW_BRANCH_NAME + else + echo "Creating arduino branch '$AR_NEW_BRANCH_NAME'..." + git -C "$AR_COMPS/arduino" checkout -b $AR_NEW_BRANCH_NAME + fi + if [ $? -ne 0 ]; then + echo "ERROR: Checkout of branch '$AR_NEW_BRANCH_NAME' failed" + exit 1 + fi + + export AR_NEW_BRANCH_NAME + export AR_NEW_COMMIT_MESSAGE + export AR_NEW_PR_TITLE + + export AR_HAS_COMMIT + export AR_HAS_BRANCH + export AR_HAS_PR +fi + +# +# UPDATE IDF MODULES +# + +if [ -x $idf_was_installed ]; then + git -C $IDF_PATH fetch origin && git -C $IDF_PATH pull origin $IDF_BRANCH + git -C $IDF_PATH submodule update --init --recursive +else + git -C $IDF_PATH submodule update --init --recursive + cd $IDF_PATH && python -m pip install -r requirements.txt && cd "$AR_ROOT" fi -export IDF_COMMIT=$(git -C $IDF_PATH rev-parse --short HEAD) -export IDF_BRANCH=$(git -C $IDF_PATH symbolic-ref --short HEAD) +# +# INSTALL TOOLCHAIN +# if ! [ -x "$(command -v $IDF_TOOLCHAIN-gcc)" ]; then echo "GCC toolchain is not installed! Installing local copy" diff --git a/tools/prepare-libs.sh b/tools/prepare-libs.sh index 064e48eaf..65390ec2e 100755 --- a/tools/prepare-libs.sh +++ b/tools/prepare-libs.sh @@ -16,7 +16,7 @@ cat pio_start.txt > "$AR_PLATFORMIO_PY" rm pio_end.txt 1pio_start.txt 2pio_start.txt pio_start.txt # include dirs -AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H \"-I{compiler.sdk.path}/include/config\"" +AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H -DGCC_NOT_5_2_0=0 -DWITH_POSIX \"-I{compiler.sdk.path}/include/config\"" echo " CPPPATH=[" >> "$AR_PLATFORMIO_PY" && echo " join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"include\", \"config\")," >> "$AR_PLATFORMIO_PY" while [ "$1" != "" ]; do cpath=$1 diff --git a/tools/push-to-arduino.sh b/tools/push-to-arduino.sh new file mode 100755 index 000000000..7ebb077f0 --- /dev/null +++ b/tools/push-to-arduino.sh @@ -0,0 +1,59 @@ +#!/bin/bash +source ./tools/config.sh + +if [ -x $GITHUB_TOKEN ]; then + echo "ERROR: GITHUB_TOKEN was not defined" + exit 1 +fi + +if ! [ -d "$AR_COMPS/arduino" ]; then + echo "ERROR: Target arduino folder does not exist!" + exit 1 +fi + +# +# UPDATE FILES +# + +if [ $AR_HAS_COMMIT == "0" ]; then + cd $AR_COMPS/arduino + + # make changes to the files + echo "Patching files in branch '$AR_NEW_BRANCH_NAME'..." + rm -rf $AR_COMPS/arduino/tools/sdk + cp -Rf $AR_SDK $AR_COMPS/arduino/tools/sdk + cp -f $AR_ESPTOOL_PY $AR_COMPS/arduino/tools/esptool.py + cp -f $AR_GEN_PART_PY $AR_COMPS/arduino/tools/gen_esp32part.py + cp -f $AR_PLATFORMIO_PY $AR_COMPS/arduino/tools/platformio-build.py + cp -f $AR_PLATFORM_TXT $AR_COMPS/arduino/platform.txt + + # did any of the files change? + if [ -n "$(git status --porcelain)" ]; then + echo "Pushing changes to branch '$AR_NEW_BRANCH_NAME'..." + git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME + if [ $? -ne 0 ]; then + echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed" + exit 1 + fi + else + echo "No changes in branch '$AR_NEW_BRANCH_NAME'" + if [ $AR_HAS_BRANCH == "0" ]; then + echo "Delete created branch '$AR_NEW_BRANCH_NAME'" + git branch -d $AR_NEW_BRANCH_NAME + fi + exit 0 + fi +fi + +# +# CREATE PULL REQUEST +# + +if [ "$AR_HAS_PR" == "0" ]; then + pr_created=`git_create_pr "$AR_NEW_BRANCH_NAME" "$AR_NEW_PR_TITLE"` + if [ $pr_created == "0" ]; then + echo "ERROR: Failed to create PR '$AR_NEW_PR_TITLE': "`echo "$git_create_pr_res" | jq -r '.message'`": "`echo "$git_create_pr_res" | jq -r '.errors[].message'` + exit 1 + fi +fi +exit 0 diff --git a/tools/repository_dispatch.sh b/tools/repository_dispatch.sh new file mode 100644 index 000000000..2fdce4ae2 --- /dev/null +++ b/tools/repository_dispatch.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +if [ ! "$GITHUB_EVENT_NAME" == "repository_dispatch" ]; then + echo "Wrong event '$GITHUB_EVENT_NAME'!" + exit 1 +fi + +EVENT_JSON=`cat "$GITHUB_EVENT_PATH"` +action=`echo "$EVENT_JSON" | jq -r '.action'` +payload=`echo "$EVENT_JSON" | jq -r '.client_payload'` +branch=`echo "$payload" | jq -r '.branch'` +commit=`echo "$payload" | jq -r '.commit'` +builder=`echo "$payload" | jq -r '.builder'` + +echo "Action: $action, Branch: $branch, Commit: $commit, Builder: $builder" + +if [ ! "$action" == "deploy" ] && [ ! "$action" == "build" ]; then + echo "Bad Action $action" + exit 1 +fi + +export GITHUB_EVENT_ACTION="$action" + +if [ ! "$commit" == "" ] && [ ! "$commit" == "null" ]; then + export IDF_COMMIT="$commit" +else + commit="" + if [ ! "$branch" == "" ] && [ ! "$branch" == "null" ]; then + export IDF_BRANCH="$branch" + fi +fi + +if [ ! "$builder" == "" ] && [ ! "$builder" == "null" ]; then + git checkout "$builder" +fi + +source ./build.sh + +if [ "$action" == "deploy" ]; then + bash ./tools/push-to-arduino.sh +fi diff --git a/tools/update-components.sh b/tools/update-components.sh index db0b2cfa9..7cae8c15e 100755 --- a/tools/update-components.sh +++ b/tools/update-components.sh @@ -2,29 +2,39 @@ source ./tools/config.sh -cd "$AR_COMPS" - -if [ ! -d "arduino" ]; then - git clone $AR_REPO arduino -fi - -if [ ! -d "esp32-camera" ]; then - git clone $CAMERA_REPO +# +# CLONE/UPDATE ARDUINO +# + +if [ ! -d "$AR_COMPS/arduino" ]; then + git clone $AR_REPO_URL "$AR_COMPS/arduino" +else + git -C "$AR_COMPS/arduino" fetch origin && \ + git -C "$AR_COMPS/arduino" pull origin master fi - -if [ ! -d "esp-face" ]; then - git clone $FACE_REPO +if [ $? -ne 0 ]; then exit 1; fi +git -C "$AR_COMPS/arduino" submodule update --init --recursive + +# +# CLONE/UPDATE ESP32-CAMERA +# + +if [ ! -d "$AR_COMPS/esp32-camera" ]; then + git clone $CAMERA_REPO_URL "$AR_COMPS/esp32-camera" +else + git -C "$AR_COMPS/esp32-camera" fetch origin && \ + git -C "$AR_COMPS/esp32-camera" pull origin master fi +if [ $? -ne 0 ]; then exit 1; fi -cd "$AR_ROOT" - -for component in `ls components`; do - cd "$AR_COMPS/$component" - if [ -d ".git" ]; then - git fetch origin && git pull origin master - fi -done +# +# CLONE/UPDATE ESP-FACE +# -cd "$AR_COMPS/arduino" -git submodule update --init --recursive -cd "$AR_ROOT" +if [ ! -d "$AR_COMPS/esp-face" ]; then + git clone $FACE_REPO_URL "$AR_COMPS/esp-face" +else + git -C "$AR_COMPS/esp-face" fetch origin && \ + git -C "$AR_COMPS/esp-face" pull origin master +fi +if [ $? -ne 0 ]; then exit 1; fi