Skip to content

Commit 8ce5f77

Browse files
lucasssvazme-no-devP-R-O-C-H-Y
authored
ci(compilation): Use default partition and add append to FQBN option (#10392)
* ci(partitions): Use default partition for compilation in CI * fix(ci): Fix paths for sdkconfig * Fix build of camera web server * fix(ci): Fix test requirements check * ci(append): Add option to append to all FQBNs * fix(json): Fix JSON files to compile examples * fix(example): Use requires instead of target in ci.json fix(zigbee): Improve JSON files Co-authored-by: Jan Prochazka <90197375+P-R-O-C-H-Y@users.noreply.github.com> * fix(regex): Trim argument before grep * docs(ci): Add documentation about FQBNs in CI * fix(json): Remove redundant FQBNs * fix(json): Skip requirements if libs are not installed * fix(partitions): Use rainmaker specific partitions --------- Co-authored-by: me-no-dev <hristo@espressif.com> Co-authored-by: Jan Prochazka <90197375+P-R-O-C-H-Y@users.noreply.github.com>
1 parent 5fd7826 commit 8ce5f77

File tree

21 files changed

+168
-147
lines changed

21 files changed

+168
-147
lines changed

‎.github/scripts/install-platformio-esp32.sh

+5-3
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git"
66
TOOLCHAIN_VERSION="12.2.0+20230208"
77
ESPTOOLPY_VERSION="~1.40501.0"
88
ESPRESSIF_ORGANIZATION_NAME="espressif"
9-
LIBS_DIR="tools/esp32-arduino-libs"
9+
SDKCONFIG_DIR="$PLATFORMIO_ESP32_PATH/tools/esp32-arduino-libs"
1010

1111
echo "Installing Python Wheel ..."
1212
pip install wheel > /dev/null 2>&1
@@ -100,7 +100,8 @@ function count_sketches(){ # count_sketches <examples-path>
100100
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
101101
if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
102102
for requirement in $requirements; do
103-
found_line=$(grep -E "^$requirement" $LIBS_DIR/esp32/sdkconfig)
103+
requirement=$(echo $requirement | xargs)
104+
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
104105
if [[ "$found_line" == "" ]]; then
105106
continue 2
106107
fi
@@ -190,7 +191,8 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
190191
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
191192
if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
192193
for requirement in $requirements; do
193-
found_line=$(grep -E "^$requirement" $LIBS_DIR/esp32/sdkconfig)
194+
requirement=$(echo $requirement | xargs)
195+
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
194196
if [[ "$found_line" == "" ]]; then
195197
continue 2
196198
fi

‎.github/scripts/sketch_utils.sh

+39-21
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
#!/bin/bash
22

3-
LIBS_DIR="tools/esp32-arduino-libs"
3+
if [ -d "$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs" ]; then
4+
SDKCONFIG_DIR="$ARDUINO_ESP32_PATH/tools/esp32-arduino-libs"
5+
elif [ -d "$GITHUB_WORKSPACE/tools/esp32-arduino-libs" ]; then
6+
SDKCONFIG_DIR="$GITHUB_WORKSPACE/tools/esp32-arduino-libs"
7+
else
8+
SDKCONFIG_DIR="tools/esp32-arduino-libs"
9+
fi
410

511
function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [extra-options]
612
while [ ! -z "$1" ]; do
@@ -83,14 +89,21 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
8389

8490
len=1
8591

92+
if [ -f $sketchdir/ci.json ]; then
93+
fqbn_append=`jq -r '.fqbn_append' $sketchdir/ci.json`
94+
if [ $fqbn_append == "null" ]; then
95+
fqbn_append=""
96+
fi
97+
fi
98+
8699
# Default FQBN options if none were passed in the command line.
87100

88-
esp32_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
89-
esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
90-
esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app,FlashMode=dio"
91-
esp32c3_opts="PartitionScheme=huge_app,FlashMode=dio"
92-
esp32c6_opts="PartitionScheme=huge_app,FlashMode=dio"
93-
esp32h2_opts="PartitionScheme=huge_app,FlashMode=dio"
101+
esp32_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}"
102+
esp32s2_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}"
103+
esp32s3_opts="PSRAM=opi,USBMode=default,FlashMode=dio${fqbn_append:+,$fqbn_append}"
104+
esp32c3_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
105+
esp32c6_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
106+
esp32h2_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
94107

95108
# Select the common part of the FQBN based on the target. The rest will be
96109
# appended depending on the passed options.
@@ -154,7 +167,8 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
154167
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
155168
if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
156169
for requirement in $requirements; do
157-
found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
170+
requirement=$(echo $requirement | xargs)
171+
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig")
158172
if [[ "$found_line" == "" ]]; then
159173
echo "Target $target does not meet the requirement $requirement for $sketchname. Skipping."
160174
exit 0
@@ -270,10 +284,11 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
270284
unset options
271285
}
272286

273-
function count_sketches(){ # count_sketches <path> [target] [file]
287+
function count_sketches(){ # count_sketches <path> [target] [file] [ignore-requirements]
274288
local path=$1
275289
local target=$2
276-
local file=$3
290+
local ignore_requirements=$3
291+
local file=$4
277292

278293
if [ $# -lt 1 ]; then
279294
echo "ERROR: Illegal number of parameters"
@@ -286,7 +301,7 @@ function count_sketches(){ # count_sketches <path> [target] [file]
286301
return 0
287302
fi
288303

289-
if [ -n "$file" ]; then
304+
if [ -f "$file" ]; then
290305
local sketches=$(cat $file)
291306
else
292307
local sketches=$(find $path -name *.ino | sort)
@@ -306,15 +321,18 @@ function count_sketches(){ # count_sketches <path> [target] [file]
306321
continue
307322
fi
308323

309-
# Check if the sketch requires any configuration options
310-
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
311-
if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
312-
for requirement in $requirements; do
313-
found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
314-
if [[ "$found_line" == "" ]]; then
315-
continue 2
316-
fi
317-
done
324+
if [ "$ignore_requirements" != "1" ]; then
325+
# Check if the sketch requires any configuration options
326+
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
327+
if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
328+
for requirement in $requirements; do
329+
requirement=$(echo $requirement | xargs)
330+
found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig)
331+
if [[ "$found_line" == "" ]]; then
332+
continue 2
333+
fi
334+
done
335+
fi
318336
fi
319337
fi
320338
echo $sketch >> sketches.txt
@@ -392,7 +410,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
392410

393411
set +e
394412
if [ -n "$sketches_file" ]; then
395-
count_sketches "$path" "$target" "$sketches_file"
413+
count_sketches "$path" "$target" "0" "$sketches_file"
396414
local sketchcount=$?
397415
else
398416
count_sketches "$path" "$target"

‎.github/scripts/tests_run.sh

+19-12
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,21 @@ function run_test() {
1010
local result=0
1111
local error=0
1212

13+
if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then
14+
len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
15+
if [ $len -eq 0 ]; then
16+
len=1
17+
fi
18+
else
19+
len=1
20+
fi
21+
22+
if [ $len -eq 1 ]; then
23+
SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig"
24+
else
25+
SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig"
26+
fi
27+
1328
if [ -f $sketchdir/ci.json ]; then
1429
# If the target or platform is listed as false, skip the sketch. Otherwise, include it.
1530
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
@@ -25,7 +40,8 @@ function run_test() {
2540
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
2641
if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
2742
for requirement in $requirements; do
28-
found_line=$(grep -E "^$requirement" $LIBS_DIR/$target/sdkconfig)
43+
requirement=$(echo $requirement | xargs)
44+
found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH")
2945
if [[ "$found_line" == "" ]]; then
3046
printf "\033[93mTarget $target does not meet the requirement $requirement for $sketchname. Skipping.\033[0m\n"
3147
printf "\n\n\n"
@@ -35,15 +51,6 @@ function run_test() {
3551
fi
3652
fi
3753

38-
if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then
39-
len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
40-
if [ $len -eq 0 ]; then
41-
len=1
42-
fi
43-
else
44-
len=1
45-
fi
46-
4754
if [ $len -eq 1 ]; then
4855
# build_dir="$sketchdir/build"
4956
build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
@@ -120,7 +127,6 @@ function run_test() {
120127

121128
SCRIPTS_DIR="./.github/scripts"
122129
COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
123-
LIBS_DIR="tools/esp32-arduino-libs"
124130

125131
platform="hardware"
126132
wokwi_timeout=60000
@@ -223,7 +229,8 @@ else
223229
fi
224230

225231
set +e
226-
${COUNT_SKETCHES} $test_folder $target
232+
# Ignore requirements as we don't have the libs. The requirements will be checked in the run_test function
233+
${COUNT_SKETCHES} "$test_folder" "$target" "1"
227234
sketchcount=$?
228235
set -e
229236
sketches=$(cat sketches.txt)

‎.github/workflows/tests_build.yml

+3
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ jobs:
2929
~/.arduino/tests/**/build*.tmp/*.bin
3030
~/.arduino/tests/**/build*.tmp/*.elf
3131
~/.arduino/tests/**/build*.tmp/*.json
32+
~/.arduino/tests/**/build*.tmp/sdkconfig
3233
3334
- name: Evaluate if tests should be built
3435
id: check-build
@@ -75,6 +76,7 @@ jobs:
7576
~/.arduino/tests/**/build*.tmp/*.bin
7677
~/.arduino/tests/**/build*.tmp/*.elf
7778
~/.arduino/tests/**/build*.tmp/*.json
79+
~/.arduino/tests/**/build*.tmp/sdkconfig
7880
7981
- name: Upload ${{ inputs.chip }} ${{ inputs.type }} binaries as artifacts
8082
uses: actions/upload-artifact@v4
@@ -85,3 +87,4 @@ jobs:
8587
~/.arduino/tests/**/build*.tmp/*.bin
8688
~/.arduino/tests/**/build*.tmp/*.elf
8789
~/.arduino/tests/**/build*.tmp/*.json
90+
~/.arduino/tests/**/build*.tmp/sdkconfig

‎.github/workflows/tests_hw.yml

-4
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,6 @@ jobs:
5959
sparse-checkout: |
6060
*
6161
62-
- name: List files
63-
if: ${{ steps.check-tests.outputs.enabled == 'true' }}
64-
run: ls -la
65-
6662
# setup-python currently only works on ubuntu images
6763
# - uses: actions/setup-python@v5
6864
# if: ${{ steps.check-tests.outputs.enabled == 'true' }}

‎docs/en/contributing.rst

+43-3
Original file line numberDiff line numberDiff line change
@@ -166,8 +166,47 @@ And in the ``README.md`` file:
166166
167167
Currently, this example requires Wi-Fi and supports the following targets.
168168
169-
| Supported Targets | ESP32 | ESP32-H2 | ESP32-S3 | ESP32-C3 | ESP32-C6 |
170-
| ----------------- | ----- | -------- | -------- | -------- | -------- |
169+
| Supported Targets | ESP32 | ESP32-S3 | ESP32-C3 | ESP32-C6 |
170+
| ----------------- | ----- | -------- | -------- | -------- |
171+
172+
By default, the CI system will use the FQBNs specified in the ``.github/scripts/sketch_utils.sh`` file to compile the sketches.
173+
Currently, the default FQBNs are:
174+
175+
* ``espressif:esp32:esp32:PSRAM=enabled,FlashMode=dio``
176+
* ``espressif:esp32:esp32s2:PSRAM=enabled,FlashMode=dio``
177+
* ``espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,FlashMode=dio``
178+
* ``espressif:esp32:esp32c3:FlashMode=dio``
179+
* ``espressif:esp32:esp32c6:FlashMode=dio``
180+
* ``espressif:esp32:esp32h2:FlashMode=dio``
181+
182+
There are two ways to alter the FQBNs used to compile the sketches: by using the ``fqbn`` or ``fqbn_append`` fields in the ``ci.json`` file.
183+
184+
If you just want to append a string to the default FQBNs, you can use the ``fqbn_append`` field. For example, to add the ``DebugLevel=debug`` to the FQBNs, you would use:
185+
186+
.. code-block:: json
187+
188+
{
189+
"fqbn_append": "DebugLevel=debug"
190+
}
191+
192+
If you want to override the default FQBNs, you can use the ``fqbn`` field. It is a dictionary where the key is the target name and the value is a list of FQBNs.
193+
The FQBNs in the list will be used in sequence to compile the sketch. For example, to compile a sketch for ESP32-S2 with and without PSRAM enabled, you would use:
194+
195+
.. code-block:: json
196+
197+
{
198+
"fqbn": {
199+
"esp32s2": [
200+
"espressif:esp32:esp32s2:PSRAM=enabled,FlashMode=dio",
201+
"espressif:esp32:esp32s2:PSRAM=disabled,FlashMode=dio"
202+
]
203+
}
204+
}
205+
206+
.. note::
207+
208+
The FQBNs specified in the ``fqbn`` field will also override the options specified in the ``fqbn_append`` field.
209+
That means that if the ``fqbn`` field is specified, the ``fqbn_append`` field will be ignored and will have no effect.
171210

172211
Example Template
173212
****************
@@ -376,9 +415,10 @@ The ``ci.json`` file is used to specify how the test suite and sketches will han
376415
* ``platforms``: A dictionary that specifies the supported platforms. The key is the platform name and the value is a boolean that specifies if
377416
the platform is supported. By default, all platforms are assumed to be supported.
378417
* ``extra_tags``: A list of extra tags that the runner will require when running the test suite in hardware. By default, no extra tags are required.
418+
* ``fqbn_append``: A string to be appended to the default FQBNs. By default, no string is appended. This has no effect if ``fqbn`` is specified.
379419
* ``fqbn``: A dictionary that specifies the FQBNs that will be used to compile the sketch. The key is the target name and the value is a list
380420
of FQBNs. The `default FQBNs <https://github.com/espressif/arduino-esp32/blob/a31a5fca1739993173caba995f7785b8eed6b30e/.github/scripts/sketch_utils.sh#L86-L91>`_
381-
are used if this field is not specified.
421+
are used if this field is not specified. This overrides the default FQBNs and the ``fqbn_append`` field.
382422

383423
The ``wifi`` test suite is a good example of how to use the ``ci.json`` file:
384424

‎libraries/ESP32/examples/Camera/CameraWebServer/ci.json

+15
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
11
{
2+
"fqbn": {
3+
"esp32": [
4+
"espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=custom,FlashMode=dio",
5+
"espressif:esp32:esp32:PSRAM=disabled,PartitionScheme=custom,FlashMode=dio"
6+
],
7+
"esp32s2": [
8+
"espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=custom,FlashMode=dio",
9+
"espressif:esp32:esp32s2:PSRAM=disabled,PartitionScheme=custom,FlashMode=dio"
10+
],
11+
"esp32s3": [
12+
"espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,PartitionScheme=custom,FlashMode=qio",
13+
"espressif:esp32:esp32s3:PSRAM=enabled,USBMode=default,PartitionScheme=custom,FlashMode=qio",
14+
"espressif:esp32:esp32s3:PSRAM=disabled,USBMode=default,PartitionScheme=custom,FlashMode=qio"
15+
]
16+
},
217
"requires": [
318
"CONFIG_CAMERA_TASK_STACK_SIZE=[0-9]+"
419
]

‎libraries/ESP32/examples/Camera/CameraWebServer/partitions.csv

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Name, Type, SubType, Offset, Size, Flags
22
nvs, data, nvs, 0x9000, 0x5000,
33
otadata, data, ota, 0xe000, 0x2000,
4-
app0, app, ota_0, 0x10000, 0x3d0000,
5-
fr, data, , 0x3e0000, 0x20000,
4+
app0, app, ota_0, 0x10000, 0x3c0000,
5+
fr, data, , 0x3d0000, 0x20000,
6+
coredump, data, coredump,0x3f0000, 0x10000,

‎libraries/RainMaker/examples/RMakerCustom/ci.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2+
"fqbn_append": "PartitionScheme=rainmaker_4MB",
23
"requires": [
34
"CONFIG_SOC_WIFI_SUPPORTED=y",
4-
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
5+
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
56
]
67
}

‎libraries/RainMaker/examples/RMakerCustomAirCooler/ci.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2+
"fqbn_append": "PartitionScheme=rainmaker_4MB",
23
"requires": [
34
"CONFIG_SOC_WIFI_SUPPORTED=y",
4-
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
5+
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
56
]
67
}

‎libraries/RainMaker/examples/RMakerSonoffDualR3/ci.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2+
"fqbn_append": "PartitionScheme=rainmaker_4MB",
23
"requires": [
34
"CONFIG_SOC_WIFI_SUPPORTED=y",
4-
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
5+
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
56
]
67
}

‎libraries/RainMaker/examples/RMakerSwitch/ci.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2+
"fqbn_append": "PartitionScheme=rainmaker_4MB",
23
"requires": [
34
"CONFIG_SOC_WIFI_SUPPORTED=y",
4-
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK"
5+
"CONFIG_ESP_RMAKER_WORK_QUEUE_TASK_STACK=[1-9][0-9]*"
56
]
67
}

‎libraries/WiFiProv/examples/WiFiProv/ci.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"fqbn_append": "PartitionScheme=huge_app",
23
"requires": [
34
"CONFIG_SOC_WIFI_SUPPORTED=y"
45
]

‎libraries/Zigbee/examples/Zigbee_Color_Dimmable_Light/ci.json

+4-14
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,6 @@
11
{
2-
"fqbn": {
3-
"esp32c6": [
4-
"espressif:esp32:esp32c6:PartitionScheme=zigbee,ZigbeeMode=ed"
5-
],
6-
"esp32h2": [
7-
"espressif:esp32:esp32h2:PartitionScheme=zigbee,ZigbeeMode=ed"
8-
]
9-
},
10-
"targets": {
11-
"esp32": false,
12-
"esp32c3": false,
13-
"esp32s2": false,
14-
"esp32s3": false
15-
}
2+
"fqbn_append": "PartitionScheme=zigbee,ZigbeeMode=ed",
3+
"requires": [
4+
"CONFIG_SOC_IEEE802154_SUPPORTED=y"
5+
]
166
}

0 commit comments

Comments
 (0)