Generate automatically module_firmware_index.json (#82)

* add shell script to easily copy dir structure to s3

* add generated content to .gitignore

* remove code duplication

* add python3 shebang and chmod +x

* add first prototype of the generate-index workflow (still to test)

* various fixes, should work now

* apply formatting

* trigger workflow only on master

Author: umbynos

.github/workflows/generate-index.yml

@@ -0,0 +1,83 @@
+name: Generate Index
+
+# See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "generator/**"
+      - "firmwares/**"
+      - "poetry.lock"
+      - "pyproject.toml"
+      - ".github/workflows/generate-index.yml"
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  generate-index:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: generator
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Taskfile
+        uses: arduino/setup-task@v1
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Install Arduino CLI
+        uses: arduino/setup-arduino-cli@v1
+
+      - name: Install platforms
+        run: |
+          arduino-cli core update-index -v
+          arduino-cli version
+          arduino-cli core install arduino:samd@${{ env.SAMD_V }} -v
+          arduino-cli core install arduino:megaavr@${{ env.MEGAAVR_V }} -v
+          arduino-cli core install arduino:mbed_nano@${{ env.MBED_NANO_V }} -v
+        env:
+          SAMD_V: 1.8.11
+          MEGAAVR_V: 1.8.7
+          MBED_NANO_V: 2.2.0
+
+      - name: Install dependencies
+        run: |
+          cd $GITHUB_WORKSPACE
+          task poetry:install-deps
+
+      - name: Generate index
+        run: poetry run ./generator.py -a $(which arduino-cli)
+
+        # fix `gpg: signing failed: Inappropriate ioctl for device`
+        # https://github.com/keybase/keybase-issues/issues/2798
+      - name: Import GPG key
+        run: |
+          echo "${{ secrets.GPG_PRIVATE_KEY }}" | base64 -di > private.key
+          gpg --batch --import --passphrase "${{ secrets.PASSPHRASE }}" private.key
+          echo "GPG_TTY=$(tty)" >> $GITHUB_ENV
+
+        # disable gpg pass prompt
+        # https://stackoverflow.com/questions/49072403/suppress-the-passphrase-prompt-in-gpg-command
+      - name: sign the json
+        run: gpg --pinentry-mode=loopback --passphrase "${{ secrets.PASSPHRASE }}" --output boards/module_firmware_index.json.sig --detach-sign boards/module_firmware_index.json
+
+      - name: create the gzip
+        run: gzip --keep boards/module_firmware_index.json
+
+      - name: s3 sync
+        run: ./s3Copy.sh . s3://arduino-downloads-prod-beagle/arduino-fwuploader
+        env:
+          AWS_REGION: "us-east-1" # or https://github.com/aws/aws-cli/issues/5623
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.gitignore

@@ -5,6 +5,8 @@
 coverage_*.txt
 /dist
 __pycache__/
+/generator/boards/
+/generator/firmwares/
 
 # Misc.
 .DS_Store

Taskfile.yml

@@ -93,7 +93,7 @@ tasks:
     desc: Run integration tests
     cmds:
       - task: build
-      - poetry install --no-root
+      - task: poetry:install-deps
       - poetry run pytest test
 
   check:
@@ -123,13 +123,13 @@ tasks:
   python:lint:
     desc: Lint Python code
     cmds:
-      - poetry install --no-root
+      - task: poetry:install-deps
       - poetry run flake8
 
   python:format:
     desc: Automatically formats Python files
     cmds:
-      - poetry install --no-root
+      - task: poetry:install-deps
       - poetry run black .
 
 vars:

generator/generator.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import argparse
 import subprocess
 import sys

generator/s3Copy.sh

@@ -0,0 +1,11 @@
+path=$1 # the path of the directory where the files and directories that need to be copied are located
+s3Dir=$2 # the s3 bucket path
+
+for entry in "$path"/*; do
+    name=`echo $entry | sed 's/.*\///'`  # getting the name of the file or directory
+    if [[ -d  $entry ]]; then  # if it is a directory
+        aws s3 cp  --recursive "$name" "$s3Dir/$name/"
+    else  # if it is a file
+        aws s3 cp "$name" "$s3Dir/" --exclude "generator.py" --exclude "raw_boards.json" --exclude "s3Copy.sh"
+    fi
+done