More efficient CI builds (espressif#3303)

* More efficient CI builds

* Update main.yml

Author: me-no-dev

Diff for: .github/workflows/main.yml

@@ -8,43 +8,34 @@ on:
   pull_request:
 
 jobs:
-
+  
+  # Ubuntu
   build-arduino-linux:
     name: Arduino ${{ matrix.chunk }} on ubuntu-latest
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        chunk: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+        chunk: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
 
     steps:
     - uses: actions/checkout@v1
     - name: Build Sketches
-      run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 12
+      run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 15
 
-  build-arduino-mac:
-    name: Arduino ${{ matrix.chunk }} on macOS-latest
-    runs-on: macOS-latest
-    strategy:
-      matrix:
-        chunk: [3, 6]
-    
-    steps:
-    - uses: actions/checkout@v1
-    - name: Build Sketches
-      run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 12
-
-  build-arduino-win:
-    name: Arduino ${{ matrix.chunk }} on windows-latest
-    runs-on: windows-latest
+  # Windows and MacOS
+  build-arduino-win-mac:
+    name: Arduino on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        chunk: [5, 15, 25]
+        os: [windows-latest, macOS-latest]
 
     steps:
     - uses: actions/checkout@v1
     - name: Build Sketches
-      run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 30
+      run: bash ./tools/ci/on-push.sh
 
+  # PlatformIO on Windows, Ubuntu and Mac
   build-platformio:
     name: PlatformIO on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -55,4 +46,4 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Build Sketches
-      run: bash ./tools/ci/on-push.sh 1 1
+      run: bash ./tools/ci/on-push.sh 1 1 #equal and non-zero to trigger PIO

Diff for: tools/ci/install-arduino-core-esp32.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-if [ ! -d "$ARDUINO_USR_PATH/hardware/espressif/esp32" ]; then
-	echo "Installing ESP32 Arduino Core in '$ARDUINO_USR_PATH/hardware/espressif/esp32'..."
+export ARDUINO_ESP32_PATH="$ARDUINO_USR_PATH/hardware/espressif/esp32"
+if [ ! -d "$ARDUINO_ESP32_PATH" ]; then
+	echo "Installing ESP32 Arduino Core in '$ARDUINO_ESP32_PATH'..."
 	script_init_path="$PWD"
 	mkdir -p "$ARDUINO_USR_PATH/hardware/espressif" && \
 	cd "$ARDUINO_USR_PATH/hardware/espressif"
@@ -36,6 +37,6 @@ if [ ! -d "$ARDUINO_USR_PATH/hardware/espressif/esp32" ]; then
 	if [ $? -ne 0 ]; then echo "ERROR: Download failed"; exit 1; fi
 	cd $script_init_path
 
-	echo "ESP32 Arduino has been installed in '$ARDUINO_USR_PATH/hardware/espressif/esp32'"
+	echo "ESP32 Arduino has been installed in '$ARDUINO_ESP32_PATH'"
 	echo ""
 fi

Diff for: tools/ci/install-arduino-ide.sh

@@ -45,9 +45,8 @@ else
 	export ARDUINO_USR_PATH="$HOME/Arduino"
 fi
 
-echo "Installing Arduino IDE on $OS_NAME..."
-
 if [ ! -d "$ARDUINO_IDE_PATH" ]; then
+	echo "Installing Arduino IDE on $OS_NAME..."
 	echo "Downloading 'arduino-nightly-$OS_NAME.$ARCHIVE_FORMAT' to 'arduino.$ARCHIVE_FORMAT'..."
 	if [ "$OS_IS_LINUX" == "1" ]; then
 		wget -O "arduino.$ARCHIVE_FORMAT" "https://www.arduino.cc/download.php?f=/arduino-nightly-$OS_NAME.$ARCHIVE_FORMAT" > /dev/null 2>&1
@@ -70,12 +69,21 @@ if [ ! -d "$ARDUINO_IDE_PATH" ]; then
 	fi
 	if [ $? -ne 0 ]; then exit 1; fi
 	rm -rf "arduino.$ARCHIVE_FORMAT"
-fi
 
-mkdir -p "$ARDUINO_USR_PATH/libraries"
-mkdir -p "$ARDUINO_USR_PATH/hardware"
+	mkdir -p "$ARDUINO_USR_PATH/libraries"
+	mkdir -p "$ARDUINO_USR_PATH/hardware"
+
+	echo "Arduino IDE Installed in '$ARDUINO_IDE_PATH'"
+	echo ""
+fi
 
 function build_sketch(){ # build_sketch <fqbn> <path-to-ino> [extra-options]
+    if [ "$#" -lt 2 ]; then
+		echo "ERROR: Illegal number of parameters"
+		echo "USAGE: build_sketch <fqbn> <path-to-ino> [extra-options]"
+		return 1
+	fi
+
 	local fqbn="$1"
 	local sketch="$2"
 	local xtra_opts="$3"
@@ -126,14 +134,26 @@ function count_sketches() # count_sketches <examples-path>
     return $sketchnum
 }
 
-function build_sketches() # build_sketches <examples-path> <fqbn> <chunk> <total-chunks> [extra-options]
+function build_sketches() # build_sketches <fqbn> <examples-path> <chunk> <total-chunks> [extra-options]
 {
-    local examples=$1
-    local fqbn=$2
+    local fqbn=$1
+    local examples=$2
     local chunk_idex=$3
     local chunks_num=$4
     local xtra_opts=$5
 
+    if [ "$#" -lt 2 ]; then
+		echo "ERROR: Illegal number of parameters"
+		echo "USAGE: build_sketches <fqbn> <examples-path> [<chunk> <total-chunks>] [extra-options]"
+		return 1
+	fi
+
+    if [ "$#" -lt 4 ]; then
+		chunk_idex="0"
+		chunks_num="1"
+		xtra_opts=$3
+	fi
+
 	if [ "$chunks_num" -le 0 ]; then
 		echo "ERROR: Chunks count must be positive number"
 		return 1
@@ -195,9 +215,3 @@ function build_sketches() # build_sketches <examples-path> <fqbn> <chunk> <total
     return 0
 }
 
-echo "Arduino IDE Installed in '$ARDUINO_IDE_PATH'"
-# echo "You can install boards in '$ARDUINO_IDE_PATH/hardware' or in '$ARDUINO_USR_PATH/hardware'"
-# echo "User libraries should be installed in '$ARDUINO_USR_PATH/libraries'"
-# echo "Then you can call 'build_sketch <fqbn> <path-to-ino> [extra-options]' to build your sketches"
-echo ""
-

Diff for: tools/ci/install-platformio-esp32.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+export PLATFORMIO_ESP32_PATH="$HOME/.platformio/packages/framework-arduinoespressif32"
+
 echo "Installing Python Wheel..."
 pip install wheel > /dev/null 2>&1
 if [ $? -ne 0 ]; then echo "ERROR: Install failed"; exit 1; fi
@@ -23,18 +25,24 @@ if [ $? -ne 0 ]; then echo "ERROR: Replace failed"; exit 1; fi
 
 if [ "$GITHUB_REPOSITORY" == "espressif/arduino-esp32" ];  then
 	echo "Linking Core..." && \
-	ln -s $GITHUB_WORKSPACE "$HOME/.platformio/packages/framework-arduinoespressif32"
+	ln -s $GITHUB_WORKSPACE "$PLATFORMIO_ESP32_PATH"
 else
 	echo "Cloning Core Repository..." && \
-	git clone https://github.com/espressif/arduino-esp32.git "$HOME/.platformio/packages/framework-arduinoespressif32" > /dev/null 2>&1
+	git clone https://github.com/espressif/arduino-esp32.git "$PLATFORMIO_ESP32_PATH" > /dev/null 2>&1
 	if [ $? -ne 0 ]; then echo "ERROR: GIT clone failed"; exit 1; fi
 fi
 
 echo "PlatformIO for ESP32 has been installed"
 echo ""
 
 
-function build_pio_sketch(){ # build_pio_sketch <board> <path-to-ino> [extra-options]
+function build_pio_sketch(){ # build_pio_sketch <board> <path-to-ino>
+    if [ "$#" -lt 2 ]; then
+        echo "ERROR: Illegal number of parameters"
+        echo "USAGE: build_pio_sketch <board> <path-to-ino>"
+        return 1
+    fi
+
 	local board="$1"
 	local sketch="$2"
 	local sketch_dir=$(dirname "$sketch")
@@ -65,13 +73,24 @@ function count_sketches() # count_sketches <examples-path>
     return $sketchnum
 }
 
-function build_pio_sketches() # build_pio_sketches <examples-path> <board> <chunk> <total-chunks>
+function build_pio_sketches() # build_pio_sketches <board> <examples-path> <chunk> <total-chunks>
 {
-    local examples=$1
-    local board=$2
+    if [ "$#" -lt 2 ]; then
+        echo "ERROR: Illegal number of parameters"
+        echo "USAGE: build_pio_sketches <board> <examples-path> [<chunk> <total-chunks>]"
+        return 1
+    fi
+
+    local board=$1
+    local examples=$2
     local chunk_idex=$3
     local chunks_num=$4
 
+    if [ "$#" -lt 4 ]; then
+        chunk_idex="0"
+        chunks_num="1"
+    fi
+
 	if [ "$chunks_num" -le 0 ]; then
 		echo "ERROR: Chunks count must be positive number"
 		return 1

Diff for: tools/ci/on-push.sh

@@ -19,38 +19,50 @@ fi
 CHUNK_INDEX=$1
 CHUNKS_CNT=$2
 BUILD_PIO=0
-if [ "$#" -lt 2 ] ||  [ "$CHUNKS_CNT" -le 0 ]; then
-	echo "Building all sketches"
+if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
 	CHUNK_INDEX=0
 	CHUNKS_CNT=1
-fi
-if [ "$CHUNK_INDEX" -gt "$CHUNKS_CNT" ]; then
+elif [ "$CHUNK_INDEX" -gt "$CHUNKS_CNT" ]; then
 	CHUNK_INDEX=$CHUNKS_CNT
-fi
-if [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then
+elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then
 	BUILD_PIO=1
 fi
 
-# CMake Test
-if [ "$CHUNK_INDEX" -eq 0 ]; then
-	bash ./tools/ci/check-cmakelists.sh
-	if [ $? -ne 0 ]; then exit 1; fi
-fi
-
 if [ "$BUILD_PIO" -eq 0 ]; then
 	# ArduinoIDE Test
+	FQBN="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
 	source ./tools/ci/install-arduino-ide.sh
 	source ./tools/ci/install-arduino-core-esp32.sh
-	build_sketches "$GITHUB_WORKSPACE/libraries" "espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app" "$CHUNK_INDEX" "$CHUNKS_CNT"
+	if [ "$OS_IS_WINDOWS" == "1" ]; then
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
+	elif [ "$OS_IS_MACOS" == "1" ]; then
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \
+		build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
+	else
+		# CMake Test
+		if [ "$CHUNK_INDEX" -eq 0 ]; then
+			bash "$ARDUINO_ESP32_PATH/tools/ci/check-cmakelists.sh"
+			if [ $? -ne 0 ]; then exit 1; fi
+		fi
+		build_sketches "$FQBN" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
+	fi
 else
 	# PlatformIO Test
 	source ./tools/ci/install-platformio-esp32.sh
-	python -m platformio ci  --board esp32dev libraries/WiFi/examples/WiFiClient && \
-	python -m platformio ci  --board esp32dev libraries/WiFiClientSecure/examples/WiFiClientSecure && \
-	python -m platformio ci  --board esp32dev libraries/BluetoothSerial/examples/SerialToSerialBT && \
-	python -m platformio ci  --board esp32dev libraries/BLE/examples/BLE_server && \
-	python -m platformio ci  --board esp32dev libraries/AzureIoT/examples/GetStarted && \
-	python -m platformio ci  --board esp32dev libraries/ESP32/examples/Camera/CameraWebServer --project-option="board_build.partitions = huge_app.csv"
-	#build_pio_sketches libraries esp32dev $CHUNK_INDEX $CHUNKS_CNT
-	if [ $? -ne 0 ]; then exit 1; fi
-fi
+	BOARD="esp32dev"
+	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
+	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
+	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \
+	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
+	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \
+	build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
+	#build_pio_sketches esp32dev "$PLATFORMIO_ESP32_PATH/libraries"
+fi
+if [ $? -ne 0 ]; then exit 1; fi