ci(refactor): Refactor workflows and skip files (#9696)

* ci(refactor): Refactor workflows and skip files

* ci(refactor): Refactor workflows and skip files

* ci(refactor): Improvements and compilation of only related files

* ci(refactor): Delete duplicated steps

* ci(refactor): General improvements

* ci(refactor): Delete duplicated lines

* ci(refactor): Rename jobs

Author: lucasssvaz

.github/scripts/install-platformio-esp32.sh

@@ -89,7 +89,9 @@ function count_sketches(){ # count_sketches <examples-path>
         if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
             continue
         fi
-        if [[ -f "$sketchdir/.test.skip" ]]; then
+        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+        # If the target is listed as false, skip the sketch. Otherwise, include it.
+        if [[ "$is_target" == "false" ]]; then
             continue
         fi
         echo $sketch >> sketches.txt
@@ -161,8 +163,10 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
+        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+        # If the target is listed as false, skip the sketch. Otherwise, include it.
         if [ "${sketchdirname}.ino" != "$sketchname" ] \
-        || [ -f "$sketchdir/.test.skip" ]; then
+        || [[ "$is_target" == "false" ]]; then
             continue
         fi
         sketchnum=$(($sketchnum + 1))

.github/scripts/on-push.sh

@@ -10,7 +10,8 @@ function build(){
     local chunk_index=$3
     local chunks_cnt=$4
     local build_log=$5
-    shift; shift; shift; shift; shift;
+    local sketches_file=$6
+    shift; shift; shift; shift; shift; shift;
     local sketches=$*
 
     local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
@@ -23,6 +24,9 @@ function build(){
     if [ "$OS_IS_LINUX" == "1" ]; then
         args+=" -p $ARDUINO_ESP32_PATH/libraries"
         args+=" -i $chunk_index -m $chunks_cnt"
+        if [ -n "$sketches_file" ]; then
+          args+=" -f $sketches_file"
+        fi
         if [ $build_log -eq 1 ]; then
             args+=" -l $build_log"
         fi
@@ -50,6 +54,7 @@ fi
 CHUNK_INDEX=$1
 CHUNKS_CNT=$2
 BUILD_LOG=$3
+SKETCHES_FILE=$4
 BUILD_PIO=0
 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
     CHUNK_INDEX=0
@@ -69,7 +74,6 @@ fi
 
 SCRIPTS_DIR="./.github/scripts"
 if [ "$BUILD_PIO" -eq 0 ]; then
-    #source ${SCRIPTS_DIR}/install-arduino-ide.sh
     source ${SCRIPTS_DIR}/install-arduino-cli.sh
     source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
@@ -95,12 +99,12 @@ if [ "$BUILD_PIO" -eq 0 ]; then
     fi
 
     #build sketches for different targets
-    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c6" $FQBN_ESP32C6 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32h2" $FQBN_ESP32H2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
+    build "esp32s3" $FQBN_ESP32S3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32s2" $FQBN_ESP32S2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c3" $FQBN_ESP32C3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c6" $FQBN_ESP32C6 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32h2" $FQBN_ESP32H2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32"   $FQBN_ESP32   "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
 
     if [ "$BUILD_LOG" -eq 1 ]; then
         #remove last comma from the last JSON object

.github/scripts/sketch_utils.sh

@@ -43,6 +43,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
     done
 
     xtra_opts=$*
+    len=0
 
     if [ -z $sketchdir ]; then
         echo "ERROR: Sketch directory not provided"
@@ -64,26 +65,30 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
         # precedence.  Note that the following logic also falls to the default
         # parameters if no arguments were passed and no file was found.
 
-        if [ -z $options ] && [ -f $sketchdir/cfg.json ]; then
+        if [ -z $options ] && [ -f $sketchdir/ci.json ]; then
             # The config file could contain multiple FQBNs for one chip.  If
             # that's the case we build one time for every FQBN.
 
-            len=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn | length' $sketchdir/cfg.json`
-            fqbn=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn' $sketchdir/cfg.json`
-        else
+            len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
+            if [ $len -gt 0 ]; then
+                fqbn=`jq -r --arg target $target '.fqbn[$target] | sort' $sketchdir/ci.json`
+            fi
+        fi
+
+        if [ ! -z $options ] || [ $len -eq 0 ]; then
             # Since we are passing options, we will end up with only one FQBN to
             # build.
 
             len=1
 
             # Default FQBN options if none were passed in the command line.
 
-            esp32_opts="FlashMode=dio,PSRAM=enabled,PartitionScheme=huge_app"
-            esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app"
-            esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app"
-            esp32c3_opts="FlashMode=dio,PartitionScheme=huge_app"
-            esp32c6_opts="PartitionScheme=huge_app"
-            esp32h2_opts="PartitionScheme=huge_app"
+            esp32_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
+            esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
+            esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app,FlashMode=dio"
+            esp32c3_opts="PartitionScheme=huge_app,FlashMode=dio"
+            esp32c6_opts="PartitionScheme=huge_app,FlashMode=dio"
+            esp32h2_opts="PartitionScheme=huge_app,FlashMode=dio"
 
             # Select the common part of the FQBN based on the target.  The rest will be
             # appended depending on the passed options.
@@ -135,7 +140,14 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
 
     sketchname=$(basename $sketchdir)
 
-    if [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
+    # If the target is listed as false, skip the sketch. Otherwise, include it.
+    if [ -f $sketchdir/ci.json ]; then
+        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+    else
+        is_target="true"
+    fi
+
+    if [[ "$is_target" == "false" ]]; then
         echo "Skipping $sketchname for target $target"
         exit 0
     fi
@@ -247,35 +259,48 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
     unset options
 }
 
-function count_sketches(){ # count_sketches <path> [target]
+function count_sketches(){ # count_sketches <path> [target] [file]
     local path=$1
     local target=$2
+    local file=$3
 
     if [ $# -lt 1 ]; then
       echo "ERROR: Illegal number of parameters"
       echo "USAGE: ${0} count <path> [target]"
     fi
 
     rm -rf sketches.txt
+    touch sketches.txt
     if [ ! -d "$path" ]; then
-        touch sketches.txt
         return 0
     fi
 
-    local sketches=$(find $path -name *.ino | sort)
+    if [ -n "$file" ]; then
+        local sketches=$(cat $file)
+    else
+        local sketches=$(find $path -name *.ino | sort)
+    fi
+
     local sketchnum=0
     for sketch in $sketches; do
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
         if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
             continue
-        elif [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
-            continue
-        else
-            echo $sketch >> sketches.txt
-            sketchnum=$(($sketchnum + 1))
+        elif [[ -n $target ]]; then
+            # If the target is listed as false, skip the sketch. Otherwise, include it.
+            if [ -f $sketchdir/ci.json ]; then
+                is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+            else
+                is_target="true"
+            fi
+            if [[ "$is_target" == "false" ]]; then
+                continue
+            fi
         fi
+        echo $sketch >> sketches.txt
+        sketchnum=$(($sketchnum + 1))
     done
     return $sketchnum
 }
@@ -319,6 +344,10 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
             shift
             log_compilation=$1
             ;;
+        -f )
+            shift
+            sketches_file=$1
+            ;;
         * )
             break
             ;;
@@ -328,7 +357,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
 
     local xtra_opts=$*
 
-    if [ -z $chunk_index ] || [ -z $chunk_max ]; then
+    if [ -z "$chunk_index" ] || [ -z "$chunk_max" ]; then
         echo "ERROR: Invalid chunk paramters"
         echo "$USAGE"
         exit 1
@@ -339,13 +368,18 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
         return 1
     fi
 
-    if [ "$chunk_index" -gt "$chunk_max" ] &&  [ "$chunk_max" -ge 2 ]; then
+    if [ "$chunk_index" -gt "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
         chunk_index=$chunk_max
     fi
 
     set +e
-    count_sketches "$path" "$target"
-    local sketchcount=$?
+    if [ -n "$sketches_file" ]; then
+        count_sketches "$path" "$target" "$sketches_file"
+        local sketchcount=$?
+    else
+        count_sketches "$path" "$target"
+        local sketchcount=$?
+    fi
     set -e
     local sketches=$(cat sketches.txt)
     rm -rf sketches.txt
@@ -364,8 +398,6 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     else
         start_index=$(( $chunk_index * $chunk_size ))
         if [ "$sketchcount" -le "$start_index" ]; then
-            echo "Skipping job"
-            touch ~/.build_skipped
             return 0
         fi
 

.github/scripts/tests_build.sh

@@ -51,7 +51,6 @@ while [ ! -z "$1" ]; do
     shift
 done
 
-#source ${SCRIPTS_DIR}/install-arduino-ide.sh
 source ${SCRIPTS_DIR}/install-arduino-cli.sh
 source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
@@ -72,7 +71,7 @@ fi
 
 if [ $chunk_build -eq 1 ]; then
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
-    args+=" -p $test_folder"
+    args+=" -p $test_folder -i 0 -m 1"
 else
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
     args+=" -s $test_folder/$sketch"