From ebafc230030f62c324eb4e57626832baf261c951 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 30 May 2025 16:54:33 +0200 Subject: [PATCH 1/6] Allow directories in profile libraries --- commands/instances.go | 18 +++++++++++++++++ internal/arduino/sketch/profiles.go | 30 +++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index 5b5fe09fcdb..102115072e6 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -363,6 +363,24 @@ func (s *arduinoCoreServerImpl) Init(req *rpc.InitRequest, stream rpc.ArduinoCor } else { // Load libraries required for profile for _, libraryRef := range profile.Libraries { + if libraryRef.InstallDir != nil { + libDir := libraryRef.InstallDir + if !libDir.IsAbs() { + libDir = paths.New(req.GetSketchPath()).JoinPath(libraryRef.InstallDir) + } + if !libDir.IsDir() { + return &cmderrors.InvalidArgumentError{ + Message: i18n.Tr("Invalid library directory in sketch project: %s", libraryRef.InstallDir), + } + } + lmb.AddLibrariesDir(librariesmanager.LibrariesDir{ + Path: libDir, + Location: libraries.Unmanaged, + IsSingleLibrary: true, + }) + continue + } + uid := libraryRef.InternalUniqueIdentifier() libRoot := s.settings.ProfilesCacheDir().Join(uid) libDir := libRoot.Join(libraryRef.Library) diff --git a/internal/arduino/sketch/profiles.go b/internal/arduino/sketch/profiles.go index cd32ccdf681..ea85fb75c1f 100644 --- a/internal/arduino/sketch/profiles.go +++ b/internal/arduino/sketch/profiles.go @@ -28,6 +28,7 @@ import ( "github.com/arduino/arduino-cli/internal/i18n" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" + "go.bug.st/f" semver "go.bug.st/relaxed-semver" "gopkg.in/yaml.v3" ) @@ -268,12 +269,26 @@ func (p *ProfilePlatformReference) UnmarshalYAML(unmarshal func(interface{}) err // ProfileLibraryReference is a reference to a library type ProfileLibraryReference struct { - Library string - Version *semver.Version + Library string + InstallDir *paths.Path + Version *semver.Version } // UnmarshalYAML decodes a ProfileLibraryReference from YAML source. func (l *ProfileLibraryReference) UnmarshalYAML(unmarshal func(interface{}) error) error { + var dataMap map[string]any + if err := unmarshal(&dataMap); err == nil { + if installDir, ok := dataMap["dir"]; !ok { + return errors.New(i18n.Tr("invalid library reference: %s", dataMap)) + } else if installDir, ok := installDir.(string); !ok { + return fmt.Errorf("%s: %s", i18n.Tr("invalid library reference: %s"), dataMap) + } else { + l.InstallDir = paths.New(installDir) + l.Library = l.InstallDir.Base() + return nil + } + } + var data string if err := unmarshal(&data); err != nil { return err @@ -291,16 +306,23 @@ func (l *ProfileLibraryReference) UnmarshalYAML(unmarshal func(interface{}) erro // AsYaml outputs the required library as Yaml func (l *ProfileLibraryReference) AsYaml() string { - res := fmt.Sprintf(" - %s (%s)\n", l.Library, l.Version) - return res + if l.InstallDir != nil { + return fmt.Sprintf(" - dir: %s\n", l.InstallDir) + } + return fmt.Sprintf(" - %s (%s)\n", l.Library, l.Version) } func (l *ProfileLibraryReference) String() string { + if l.InstallDir != nil { + return fmt.Sprintf("%s@dir:%s", l.Library, l.InstallDir) + } return fmt.Sprintf("%s@%s", l.Library, l.Version) } // InternalUniqueIdentifier returns the unique identifier for this object func (l *ProfileLibraryReference) InternalUniqueIdentifier() string { + f.Assert(l.InstallDir == nil, + "InternalUniqueIdentifier should not be called for library references with an install directory") id := l.String() h := sha256.Sum256([]byte(id)) res := fmt.Sprintf("%s_%s", id, hex.EncodeToString(h[:])[:16]) From a9cabb360f7eabeab43bc9ec24d1c27dc3bb0ee4 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 10 Jun 2025 10:59:03 +0200 Subject: [PATCH 2/6] Updated docs --- docs/sketch-project-file.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/sketch-project-file.md b/docs/sketch-project-file.md index 1660bba9f1b..6f66a1f0fa3 100644 --- a/docs/sketch-project-file.md +++ b/docs/sketch-project-file.md @@ -14,7 +14,9 @@ Each profile will define: - The target core platform name and version (with the 3rd party platform index URL if needed) - A possible core platform name and version, that is a dependency of the target core platform (with the 3rd party platform index URL if needed) -- The libraries used in the sketch (including their version) +- A list of libraries used in the sketch. Each library could be: + - a library taken from the Arduino Libraries Index + - a library installed anywhere in the filesystem - The port and protocol to upload the sketch and monitor the board The format of the file is the following: @@ -31,9 +33,8 @@ profiles: - platform: () platform_index_url: <3RD_PARTY_PLATFORM_DEPENDENCY_URL> libraries: - - () - - () - - () + - () + - dir: port: port_config: : @@ -55,7 +56,11 @@ otherwise below). The available fields are: information as ``, ``, and `<3RD_PARTY_PLATFORM_URL>` respectively but for the core platform dependency of the main core platform. These fields are optional. - `libraries:` is a section where the required libraries to build the project are defined. This section is optional. -- `` is the version required for the library, for example, `1.0.0`. + - ` ()` represents a library from the Arduino Libraries Index, for example, + `MyLib (1.0.0)`. + - `dir: ` represents a library installed in the filesystem and `` is the path to the + library. The path could be absolute or relative to the sketch folder. This option is available since Arduino CLI + 1.3.0. - `` is a free text string available to the developer to add comments. This field is optional. - `` is the programmer that will be used. This field is optional. From a37057c8e9db8eb9fa336ceac94c9ee02f114dc3 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 10 Jun 2025 12:52:52 +0200 Subject: [PATCH 3/6] Dump custom libraries in profiles with --dump-profile command --- internal/cli/compile/compile.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/internal/cli/compile/compile.go b/internal/cli/compile/compile.go index ad906fcd919..88a5f7b6dc2 100644 --- a/internal/cli/compile/compile.go +++ b/internal/cli/compile/compile.go @@ -323,22 +323,23 @@ func runCompileCommand(cmd *cobra.Command, args []string, srv rpc.ArduinoCoreSer // Output profile libs := "" - hasVendoredLibs := false for _, lib := range builderRes.GetUsedLibraries() { if lib.GetLocation() != rpc.LibraryLocation_LIBRARY_LOCATION_USER && lib.GetLocation() != rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED { continue } - if lib.GetVersion() == "" { - hasVendoredLibs = true - continue + if lib.GetVersion() == "" || lib.Location == rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED { + libDir := paths.New(lib.GetInstallDir()) + // If the library is installed in the sketch path, we want to output the relative path + // to the sketch path, so that the sketch is portable. + if ok, err := libDir.IsInsideDir(sketchPath); err == nil && ok { + if ref, err := libDir.RelFrom(sketchPath); err == nil { + libDir = ref + } + } + libs += fmt.Sprintln(" - dir: " + libDir.String()) + } else { + libs += fmt.Sprintln(" - " + lib.GetName() + " (" + lib.GetVersion() + ")") } - libs += fmt.Sprintln(" - " + lib.GetName() + " (" + lib.GetVersion() + ")") - } - if hasVendoredLibs { - msg := "\n" - msg += i18n.Tr("WARNING: The sketch is compiled using one or more custom libraries.") + "\n" - msg += i18n.Tr("Currently, Build Profiles only support libraries available through Arduino Library Manager.") - feedback.Warning(msg) } newProfileName := "my_profile_name" From 211a7ea66bf9dc592243407d4c314618222ffdb2 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 10 Jun 2025 16:27:10 +0200 Subject: [PATCH 4/6] Added integration tests --- .../integrationtest/sketch/profiles_test.go | 67 +++++++++++++++++++ .../testdata/MyLibOutside/MyLibOutside.h | 0 .../testdata/MyLibOutside/library.properties | 10 +++ .../SketchWithLibrary/SketchWithLibrary.ino | 6 ++ .../SketchWithLibrary/libraries/MyLib/MyLib.h | 0 .../libraries/MyLib/library.properties | 10 +++ 6 files changed, 93 insertions(+) create mode 100644 internal/integrationtest/sketch/profiles_test.go create mode 100644 internal/integrationtest/sketch/testdata/MyLibOutside/MyLibOutside.h create mode 100644 internal/integrationtest/sketch/testdata/MyLibOutside/library.properties create mode 100644 internal/integrationtest/sketch/testdata/SketchWithLibrary/SketchWithLibrary.ino create mode 100644 internal/integrationtest/sketch/testdata/SketchWithLibrary/libraries/MyLib/MyLib.h create mode 100644 internal/integrationtest/sketch/testdata/SketchWithLibrary/libraries/MyLib/library.properties diff --git a/internal/integrationtest/sketch/profiles_test.go b/internal/integrationtest/sketch/profiles_test.go new file mode 100644 index 00000000000..b40ca69e2fe --- /dev/null +++ b/internal/integrationtest/sketch/profiles_test.go @@ -0,0 +1,67 @@ +// This file is part of arduino-cli. +// +// Copyright 2022-2025 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package sketch_test + +import ( + "strings" + "testing" + + "github.com/arduino/arduino-cli/internal/integrationtest" + "github.com/arduino/go-paths-helper" + "github.com/stretchr/testify/require" +) + +func TestSketchProfileDump(t *testing.T) { + env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) + defer env.CleanUp() + + sketch, err := paths.New("testdata", "SketchWithLibrary").Abs() + require.NoError(t, err) + + _, _, err = cli.Run("core", "install", "arduino:avr@1.8.6") + require.NoError(t, err) + _, _, err = cli.Run("lib", "install", "Adafruit BusIO@1.17.1") + require.NoError(t, err) + _, _, err = cli.Run("lib", "install", "Adafruit GFX Library@1.12.1") + require.NoError(t, err) + _, _, err = cli.Run("lib", "install", "Adafruit SSD1306@2.5.14") + require.NoError(t, err) + + // Check if the profile dump: + // - keeps libraries in the sketch with a relative path + // - keeps libraries outside the sketch with an absolute path + // - keeps libraries installed in the system with just the name and version + libOutside := sketch.Join("..", "MyLibOutside") + out, _, err := cli.Run("compile", "-b", "arduino:avr:uno", + "--library", sketch.Join("libraries", "MyLib").String(), + "--library", libOutside.String(), + "--dump-profile", + sketch.String()) + require.NoError(t, err) + require.Equal(t, strings.TrimSpace(` +profiles: + uno: + fqbn: arduino:avr:uno + platforms: + - platform: arduino:avr (1.8.6) + libraries: + - dir: libraries/MyLib + - dir: `+libOutside.String()+` + - Adafruit SSD1306 (2.5.14) + - Adafruit GFX Library (1.12.1) + - Adafruit BusIO (1.17.1) +`), strings.TrimSpace(string(out))) +} diff --git a/internal/integrationtest/sketch/testdata/MyLibOutside/MyLibOutside.h b/internal/integrationtest/sketch/testdata/MyLibOutside/MyLibOutside.h new file mode 100644 index 00000000000..e69de29bb2d diff --git a/internal/integrationtest/sketch/testdata/MyLibOutside/library.properties b/internal/integrationtest/sketch/testdata/MyLibOutside/library.properties new file mode 100644 index 00000000000..33914d38b35 --- /dev/null +++ b/internal/integrationtest/sketch/testdata/MyLibOutside/library.properties @@ -0,0 +1,10 @@ +name=MyLibOutside +version=1.3.7 +author=Arduino +maintainer=Arduino +sentence= +paragraph= +category=Communication +url= +architectures=* +includes=MyLibOutside.h diff --git a/internal/integrationtest/sketch/testdata/SketchWithLibrary/SketchWithLibrary.ino b/internal/integrationtest/sketch/testdata/SketchWithLibrary/SketchWithLibrary.ino new file mode 100644 index 00000000000..d8bef92dfee --- /dev/null +++ b/internal/integrationtest/sketch/testdata/SketchWithLibrary/SketchWithLibrary.ino @@ -0,0 +1,6 @@ +#include +#include +#include + +void setup() {} +void loop() {} diff --git a/internal/integrationtest/sketch/testdata/SketchWithLibrary/libraries/MyLib/MyLib.h b/internal/integrationtest/sketch/testdata/SketchWithLibrary/libraries/MyLib/MyLib.h new file mode 100644 index 00000000000..e69de29bb2d diff --git a/internal/integrationtest/sketch/testdata/SketchWithLibrary/libraries/MyLib/library.properties b/internal/integrationtest/sketch/testdata/SketchWithLibrary/libraries/MyLib/library.properties new file mode 100644 index 00000000000..cefe1b7ff62 --- /dev/null +++ b/internal/integrationtest/sketch/testdata/SketchWithLibrary/libraries/MyLib/library.properties @@ -0,0 +1,10 @@ +name=MyLib +version=1.3.7 +author=Arduino +maintainer=Arduino +sentence= +paragraph= +category=Communication +url= +architectures=* +includes=MyLib.h From cdd7f4eb3e525749a4a225d438872dec006dc788 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 10 Jun 2025 16:48:30 +0200 Subject: [PATCH 5/6] Fix integration test in Windows --- internal/cli/compile/compile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/cli/compile/compile.go b/internal/cli/compile/compile.go index 88a5f7b6dc2..6a81499a236 100644 --- a/internal/cli/compile/compile.go +++ b/internal/cli/compile/compile.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "github.com/arduino/arduino-cli/commands" @@ -333,7 +334,7 @@ func runCompileCommand(cmd *cobra.Command, args []string, srv rpc.ArduinoCoreSer // to the sketch path, so that the sketch is portable. if ok, err := libDir.IsInsideDir(sketchPath); err == nil && ok { if ref, err := libDir.RelFrom(sketchPath); err == nil { - libDir = ref + libDir = paths.New(filepath.ToSlash(ref.String())) } } libs += fmt.Sprintln(" - dir: " + libDir.String()) From 0bd806e07e5b577f773d0fd38de428575904e12e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 11 Jun 2025 15:41:44 +0200 Subject: [PATCH 6/6] Improved integration tests --- .../integrationtest/sketch/profiles_test.go | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/internal/integrationtest/sketch/profiles_test.go b/internal/integrationtest/sketch/profiles_test.go index b40ca69e2fe..86a32f2e678 100644 --- a/internal/integrationtest/sketch/profiles_test.go +++ b/internal/integrationtest/sketch/profiles_test.go @@ -16,21 +16,39 @@ package sketch_test import ( + "encoding/json" "strings" "testing" "github.com/arduino/arduino-cli/internal/integrationtest" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" + "go.bug.st/testifyjson/requirejson" ) func TestSketchProfileDump(t *testing.T) { env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) - defer env.CleanUp() + t.Cleanup(env.CleanUp) - sketch, err := paths.New("testdata", "SketchWithLibrary").Abs() + // Prepare the sketch with libraries + tmpDir, err := paths.MkTempDir("", "") require.NoError(t, err) + t.Cleanup(func() { _ = tmpDir.RemoveAll }) + sketchTemplate, err := paths.New("testdata", "SketchWithLibrary").Abs() + require.NoError(t, err) + + sketch := tmpDir.Join("SketchWithLibrary") + libInside := sketch.Join("libraries", "MyLib") + err = sketchTemplate.CopyDirTo(sketch) + require.NoError(t, err) + + libOutsideTemplate := sketchTemplate.Join("..", "MyLibOutside") + libOutside := sketch.Join("..", "MyLibOutside") + err = libOutsideTemplate.CopyDirTo(libOutside) + require.NoError(t, err) + + // Install the required core and libraries _, _, err = cli.Run("core", "install", "arduino:avr@1.8.6") require.NoError(t, err) _, _, err = cli.Run("lib", "install", "Adafruit BusIO@1.17.1") @@ -44,9 +62,8 @@ func TestSketchProfileDump(t *testing.T) { // - keeps libraries in the sketch with a relative path // - keeps libraries outside the sketch with an absolute path // - keeps libraries installed in the system with just the name and version - libOutside := sketch.Join("..", "MyLibOutside") out, _, err := cli.Run("compile", "-b", "arduino:avr:uno", - "--library", sketch.Join("libraries", "MyLib").String(), + "--library", libInside.String(), "--library", libOutside.String(), "--dump-profile", sketch.String()) @@ -64,4 +81,19 @@ profiles: - Adafruit GFX Library (1.12.1) - Adafruit BusIO (1.17.1) `), strings.TrimSpace(string(out))) + + // Dump the profile in the sketch directory and compile with it again + err = sketch.Join("sketch.yaml").WriteFile(out) + require.NoError(t, err) + out, _, err = cli.Run("compile", "-m", "uno", "--json", sketch.String()) + require.NoError(t, err) + // Check if local libraries are picked up correctly + libInsideJson, _ := json.Marshal(libInside.String()) + libOutsideJson, _ := json.Marshal(libOutside.String()) + j := requirejson.Parse(t, out).Query(".builder_result.used_libraries") + j.MustContain(` + [ + {"name": "MyLib", "install_dir": ` + string(libInsideJson) + `}, + {"name": "MyLibOutside", "install_dir": ` + string(libOutsideJson) + `} + ]`) }