From 00fcb0620aa2ce08380128a1b1e519b5dd10b052 Mon Sep 17 00:00:00 2001 From: lucarin91 Date: Fri, 28 Nov 2025 16:02:42 +0100 Subject: [PATCH] feat: add secret management --- cmd/arduino-app-cli/app/restart.go | 1 + cmd/arduino-app-cli/app/start.go | 1 + .../internal/servicelocator/servicelocator.go | 1 + internal/api/api.go | 4 + internal/api/handlers/app_start.go | 2 +- internal/api/handlers/secrets.go | 79 +++++++++++++++++++ .../api/models/{properties.go => models.go} | 4 + internal/orchestrator/config/config.go | 7 ++ internal/orchestrator/orchestrator.go | 8 +- internal/orchestrator/provision.go | 36 ++++++++- internal/orchestrator/provision_test.go | 8 +- internal/orchestrator/secrets/secrets.go | 72 +++++++++++++++++ 12 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 internal/api/handlers/secrets.go rename internal/api/models/{properties.go => models.go} (91%) create mode 100644 internal/orchestrator/secrets/secrets.go diff --git a/cmd/arduino-app-cli/app/restart.go b/cmd/arduino-app-cli/app/restart.go index 892f108f..22f289eb 100644 --- a/cmd/arduino-app-cli/app/restart.go +++ b/cmd/arduino-app-cli/app/restart.go @@ -63,6 +63,7 @@ func restartHandler(ctx context.Context, cfg config.Configuration, app app.Ardui app, cfg, servicelocator.GetStaticStore(), + servicelocator.GetAppIDProvider(), ) for message := range stream { switch message.GetType() { diff --git a/cmd/arduino-app-cli/app/start.go b/cmd/arduino-app-cli/app/start.go index 6899f338..5296d6de 100644 --- a/cmd/arduino-app-cli/app/start.go +++ b/cmd/arduino-app-cli/app/start.go @@ -65,6 +65,7 @@ func startHandler(ctx context.Context, cfg config.Configuration, app app.Arduino app, cfg, servicelocator.GetStaticStore(), + servicelocator.GetAppIDProvider(), ) for message := range stream { switch message.GetType() { diff --git a/cmd/arduino-app-cli/internal/servicelocator/servicelocator.go b/cmd/arduino-app-cli/internal/servicelocator/servicelocator.go index 22cbc58b..d386d8a0 100644 --- a/cmd/arduino-app-cli/internal/servicelocator/servicelocator.go +++ b/cmd/arduino-app-cli/internal/servicelocator/servicelocator.go @@ -53,6 +53,7 @@ var ( return f.Must(orchestrator.NewProvision( GetDockerClient(), globalConfig, + GetAppIDProvider(), )) }) diff --git a/internal/api/api.go b/internal/api/api.go index 65c66771..a6b9b84a 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -93,6 +93,10 @@ func NewHTTPRouter( mux.Handle("PATCH /v1/apps/{appID}/bricks/{brickID}", handlers.HandleBrickUpdates(brickService, idProvider)) mux.Handle("DELETE /v1/apps/{appID}/bricks/{brickID}", handlers.HandleBrickDelete(brickService, idProvider)) + mux.Handle("GET /v1/apps/{appID}/secrets", handlers.HandleSecretsList(cfg, idProvider)) + mux.Handle("PUT /v1/apps/{appID}/secrets/{secretName}", handlers.HandleSecretsUpdate(cfg, idProvider)) + mux.Handle("DELETE /v1/apps/{appID}/secrets/{secretName}", handlers.HandleSecretsDelete(cfg, idProvider)) + mux.Handle("GET /v1/docs/", http.StripPrefix("/v1/docs/", handlers.DocsServer(docsFS))) mux.Handle("GET /v1/monitor/ws", handlers.HandleMonitorWS(allowedOrigins)) diff --git a/internal/api/handlers/app_start.go b/internal/api/handlers/app_start.go index bfd8a034..8f5f765b 100644 --- a/internal/api/handlers/app_start.go +++ b/internal/api/handlers/app_start.go @@ -69,7 +69,7 @@ func HandleAppStart( type log struct { Message string `json:"message"` } - for item := range orchestrator.StartApp(r.Context(), dockerCli, provisioner, modelsIndex, bricksIndex, app, cfg, staticStore) { + for item := range orchestrator.StartApp(r.Context(), dockerCli, provisioner, modelsIndex, bricksIndex, app, cfg, staticStore, idProvider) { switch item.GetType() { case orchestrator.ProgressType: sseStream.Send(render.SSEEvent{Type: "progress", Data: progress(*item.GetProgress())}) diff --git a/internal/api/handlers/secrets.go b/internal/api/handlers/secrets.go new file mode 100644 index 00000000..9f508930 --- /dev/null +++ b/internal/api/handlers/secrets.go @@ -0,0 +1,79 @@ +package handlers + +import ( + "io" + "log/slog" + "net/http" + + "github.com/arduino/arduino-app-cli/internal/api/models" + "github.com/arduino/arduino-app-cli/internal/orchestrator/app" + "github.com/arduino/arduino-app-cli/internal/orchestrator/config" + "github.com/arduino/arduino-app-cli/internal/orchestrator/secrets" + "github.com/arduino/arduino-app-cli/internal/render" +) + +func HandleSecretsList(cfg config.Configuration, idProvider *app.IDProvider) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + appID, err := idProvider.IDFromBase64(r.PathValue("appID")) + if err != nil { + render.EncodeResponse(w, http.StatusPreconditionFailed, models.ErrorResponse{Details: "invalid app id"}) + return + } + + secrets, err := secrets.ListSecrets(cfg, appID) + if err != nil { + slog.Error("Unable to list secrets", slog.String("error", err.Error())) + render.EncodeResponse(w, http.StatusInternalServerError, models.ErrorResponse{Details: "unable to list secret"}) + return + } + + render.EncodeResponse(w, http.StatusOK, models.SecretListResponse{Secrets: secrets}) + }) +} + +func HandleSecretsUpdate(cfg config.Configuration, idProvider *app.IDProvider) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + appID, err := idProvider.IDFromBase64(r.PathValue("appID")) + if err != nil { + render.EncodeResponse(w, http.StatusPreconditionFailed, models.ErrorResponse{Details: "invalid app id"}) + return + } + name := r.PathValue("secretName") + + value, err := io.ReadAll(r.Body) + if err != nil { + slog.Warn("Failed to read request body", "error", err.Error()) + render.EncodeResponse(w, http.StatusBadRequest, models.ErrorResponse{Details: "invalid body"}) + return + } + + err = secrets.UpdateSecret(cfg, appID, name, value) + if err != nil { + slog.Error("Unable to update secret", slog.String("error", err.Error())) + render.EncodeResponse(w, http.StatusInternalServerError, models.ErrorResponse{Details: "unable to update secret"}) + return + } + + render.EncodeResponse(w, http.StatusNoContent, nil) + }) +} + +func HandleSecretsDelete(cfg config.Configuration, idProvider *app.IDProvider) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + appID, err := idProvider.IDFromBase64(r.PathValue("appID")) + if err != nil { + render.EncodeResponse(w, http.StatusPreconditionFailed, models.ErrorResponse{Details: "invalid app id"}) + return + } + name := r.PathValue("secretName") + + err = secrets.RemoveSecret(cfg, appID, name) + if err != nil { + slog.Error("Unable to remove secret", slog.String("error", err.Error())) + render.EncodeResponse(w, http.StatusInternalServerError, models.ErrorResponse{Details: "unable to remove secret"}) + return + } + + render.EncodeResponse(w, http.StatusNoContent, nil) + }) +} diff --git a/internal/api/models/properties.go b/internal/api/models/models.go similarity index 91% rename from internal/api/models/properties.go rename to internal/api/models/models.go index 258d4e6a..1bcf2281 100644 --- a/internal/api/models/properties.go +++ b/internal/api/models/models.go @@ -18,3 +18,7 @@ package models type PropertyKeysResponse struct { Keys []string `json:"keys"` } + +type SecretListResponse struct { + Secrets []string `json:"secrets"` +} diff --git a/internal/orchestrator/config/config.go b/internal/orchestrator/config/config.go index 0838f29d..044f3714 100644 --- a/internal/orchestrator/config/config.go +++ b/internal/orchestrator/config/config.go @@ -133,6 +133,9 @@ func (c *Configuration) init() error { if err := c.AssetsDir().MkdirAll(); err != nil { return err } + if err := c.SecretsDir().MkdirAll(); err != nil { + return err + } return nil } @@ -152,6 +155,10 @@ func (c *Configuration) RouterSocketPath() *paths.Path { return c.routerSocketPath } +func (c *Configuration) SecretsDir() *paths.Path { + return c.dataDir.Join("secrets") +} + func (c *Configuration) AssetsDir() *paths.Path { return c.dataDir.Join("assets") } diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index 745ff922..52c24726 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -117,6 +117,7 @@ func StartApp( appToStart app.ArduinoApp, cfg config.Configuration, staticStore *store.StaticStore, + idProvider *app.IDProvider, ) iter.Seq[StreamMessage] { return func(yield func(StreamMessage) bool) { ctx, cancel := context.WithCancel(ctx) @@ -183,7 +184,7 @@ func StartApp( return } - if err := provisioner.App(ctx, bricksIndex, &appToStart, cfg, envs, staticStore); err != nil { + if err := provisioner.App(ctx, bricksIndex, &appToStart, cfg, envs, staticStore, idProvider); err != nil { yield(StreamMessage{error: err}) return } @@ -458,6 +459,7 @@ func RestartApp( appToStart app.ArduinoApp, cfg config.Configuration, staticStore *store.StaticStore, + idProvider *app.IDProvider, ) iter.Seq[StreamMessage] { return func(yield func(StreamMessage) bool) { ctx, cancel := context.WithCancel(ctx) @@ -484,7 +486,7 @@ func RestartApp( } } } - startStream := StartApp(ctx, docker, provisioner, modelsIndex, bricksIndex, appToStart, cfg, staticStore) + startStream := StartApp(ctx, docker, provisioner, modelsIndex, bricksIndex, appToStart, cfg, staticStore, idProvider) startStream(yield) } } @@ -517,7 +519,7 @@ func StartDefaultApp( } // TODO: we need to stop all other running app before starting the default app. - for msg := range StartApp(ctx, docker, provisioner, modelsIndex, bricksIndex, *app, cfg, staticStore) { + for msg := range StartApp(ctx, docker, provisioner, modelsIndex, bricksIndex, *app, cfg, staticStore, idProvider) { if msg.IsError() { return fmt.Errorf("failed to start app: %w", msg.GetError()) } diff --git a/internal/orchestrator/provision.go b/internal/orchestrator/provision.go index c5ab52b6..7c65ec04 100644 --- a/internal/orchestrator/provision.go +++ b/internal/orchestrator/provision.go @@ -36,6 +36,7 @@ import ( "github.com/arduino/arduino-app-cli/internal/orchestrator/app" "github.com/arduino/arduino-app-cli/internal/orchestrator/bricksindex" "github.com/arduino/arduino-app-cli/internal/orchestrator/config" + "github.com/arduino/arduino-app-cli/internal/orchestrator/secrets" "github.com/arduino/arduino-app-cli/internal/store" ) @@ -67,11 +68,13 @@ type service struct { Labels map[string]string `yaml:"labels,omitempty"` Environment map[string]string `yaml:"environment,omitempty"` Logging *logging `yaml:"logging,omitempty"` + Secrets []string `yaml:"secrets,omitempty"` } type Provision struct { docker command.Cli pythonImage string + idProvider *app.IDProvider } func isDevelopmentMode(cfg config.Configuration) bool { @@ -81,10 +84,12 @@ func isDevelopmentMode(cfg config.Configuration) bool { func NewProvision( docker command.Cli, cfg config.Configuration, + idProvider *app.IDProvider, ) (*Provision, error) { provision := &Provision{ docker: docker, pythonImage: cfg.PythonImage, + idProvider: idProvider, } dynamicProvisionDir := cfg.AssetsDir().Join(cfg.UsedPythonImageTag) @@ -119,6 +124,7 @@ func (p *Provision) App( cfg config.Configuration, mapped_env map[string]string, staticStore *store.StaticStore, + idProvider *app.IDProvider, ) error { if arduinoApp == nil { return fmt.Errorf("provisioning failed: arduinoApp is nil") @@ -130,7 +136,7 @@ func (p *Provision) App( } } - return generateMainComposeFile(arduinoApp, bricksIndex, p.pythonImage, cfg, mapped_env, staticStore) + return generateMainComposeFile(arduinoApp, idProvider, bricksIndex, p.pythonImage, cfg, mapped_env, staticStore) } func (p *Provision) init( @@ -207,6 +213,7 @@ const ( func generateMainComposeFile( app *app.ArduinoApp, + idProvider *app.IDProvider, bricksIndex *bricksindex.BricksIndex, pythonImage string, cfg config.Configuration, @@ -288,10 +295,14 @@ func generateMainComposeFile( type mainService struct { Main service `yaml:"main"` } + type secretObj struct { + File string `yaml:"file"` + } var mainAppCompose struct { - Name string `yaml:"name"` - Include []string `yaml:"include,omitempty"` - Services *mainService `yaml:"services,omitempty"` + Name string `yaml:"name"` + Include []string `yaml:"include,omitempty"` + Services *mainService `yaml:"services,omitempty"` + Secrets map[string]secretObj `yaml:"secrets,omitempty"` } // Merge compose composeProjectName, err := getAppComposeProjectNameFromApp(*app, cfg) @@ -356,6 +367,22 @@ func generateMainComposeFile( } } + // Add secrets (if defined) + appID, err := idProvider.IDFromPath(app.FullPath) + if err != nil { + return fmt.Errorf("failed to retrieve app id from path %s: %w", app.FullPath.String(), err) + } + secrets, err := secrets.GetSecrets(cfg, appID) + if err != nil { + slog.Error("Failed to retrieve secrets for app", slog.String("app_path", app.FullPath.String()), slog.String("app_id", appID.String()), slog.Any("error", err)) + } + secretsList := make([]string, 0, len(secrets)) + mainAppCompose.Secrets = make(map[string]secretObj, len(secrets)) + for _, secret := range secrets { + secretsList = append(secretsList, secret.Name) + mainAppCompose.Secrets[secret.Name] = secretObj{File: secret.Path} + } + mainAppCompose.Services = &mainService{ Main: service{ Image: pythonImage, @@ -380,6 +407,7 @@ func generateMainComposeFile( "max-file": "2", }, }, + Secrets: secretsList, }, } diff --git a/internal/orchestrator/provision_test.go b/internal/orchestrator/provision_test.go index 1bd7aa65..d000d29a 100644 --- a/internal/orchestrator/provision_test.go +++ b/internal/orchestrator/provision_test.go @@ -36,6 +36,7 @@ func TestProvisionAppWithOverrides(t *testing.T) { tempDirectory := t.TempDir() staticStore := store.NewStaticStore(cfg.AssetsDir().String()) + idProvider := app.NewAppIDProvider(cfg) // Define a mock app with bricks that require overrides app := app.ArduinoApp{ @@ -116,7 +117,7 @@ bricks: env := map[string]string{ "FOO": "bar", } - err = generateMainComposeFile(&app, bricksIndex, "app-bricks:python-apps-base:dev-latest", cfg, env, staticStore) + err = generateMainComposeFile(&app, idProvider, bricksIndex, "app-bricks:python-apps-base:dev-latest", cfg, env, staticStore) // Validate that the main compose file and overrides are created require.NoError(t, err, "Failed to generate main compose file") @@ -274,6 +275,7 @@ services: func TestProvisionAppWithDependsOn(t *testing.T) { cfg := setTestOrchestratorConfig(t) staticStore := store.NewStaticStore(cfg.AssetsDir().String()) + idProvider := app.NewAppIDProvider(cfg) tempDirectory := t.TempDir() var env = map[string]string{} type services struct { @@ -340,7 +342,7 @@ services: require.NoError(t, err) // Run the provision function to generate the main compose file - err = generateMainComposeFile(&app, bricksIndex, "app-bricks:python-apps-base:dev-latest", cfg, env, staticStore) + err = generateMainComposeFile(&app, idProvider, bricksIndex, "app-bricks:python-apps-base:dev-latest", cfg, env, staticStore) require.NoError(t, err, "Failed to generate main compose file") composeFilePath := paths.New(tempDirectory).Join(".cache").Join("app-compose.yaml") require.True(t, composeFilePath.Exist(), "Main compose file should exist") @@ -390,7 +392,7 @@ services: require.NoError(t, err) // Run the provision function to generate the main compose file - err = generateMainComposeFile(&app, bricksIndex, "app-bricks:python-apps-base:dev-latest", cfg, env, staticStore) + err = generateMainComposeFile(&app, idProvider, bricksIndex, "app-bricks:python-apps-base:dev-latest", cfg, env, staticStore) require.NoError(t, err, "Failed to generate main compose file") composeFilePath := paths.New(tempDirectory).Join(".cache").Join("app-compose.yaml") require.True(t, composeFilePath.Exist(), "Main compose file should exist") diff --git a/internal/orchestrator/secrets/secrets.go b/internal/orchestrator/secrets/secrets.go new file mode 100644 index 00000000..e9586617 --- /dev/null +++ b/internal/orchestrator/secrets/secrets.go @@ -0,0 +1,72 @@ +package secrets + +import ( + "github.com/arduino/go-paths-helper" + "github.com/google/renameio/v2" + + "github.com/arduino/arduino-app-cli/internal/orchestrator/app" + "github.com/arduino/arduino-app-cli/internal/orchestrator/config" +) + +func UpdateSecret(cfg config.Configuration, appID app.ID, name string, value []byte) error { + appPath := cfg.SecretsDir().Join(appID.String()) + _ = appPath.MkdirAll() + + secretPath := appPath.Join(name) + return renameio.WriteFile(secretPath.String(), value, 0o600) +} + +func RemoveSecret(cfg config.Configuration, appID app.ID, name string) error { + secretPath := cfg.SecretsDir().Join(appID.String(), name) + if secretPath.NotExist() { + return nil + } + + return secretPath.Remove() +} + +func ListSecrets(cfg config.Configuration, appID app.ID) ([]string, error) { + appPath := cfg.SecretsDir().Join(appID.String()) + if appPath.NotExist() { + return nil, nil + } + + files, err := appPath.ReadDir(func(p *paths.Path) bool { return !p.IsDir() }) + if err != nil { + return nil, err + } + + secrets := make([]string, 0, len(files)) + for _, file := range files { + secrets = append(secrets, file.Base()) + } + + return secrets, nil +} + +type Secret struct { + Name string + Path string +} + +func GetSecrets(cfg config.Configuration, appID app.ID) ([]Secret, error) { + appPath := cfg.SecretsDir().Join(appID.String()) + if appPath.NotExist() { + return nil, nil + } + + files, err := appPath.ReadDir(func(p *paths.Path) bool { return !p.IsDir() }) + if err != nil { + return nil, err + } + + secrets := make([]Secret, 0, len(files)) + for _, file := range files { + secrets = append(secrets, Secret{ + Name: file.Base(), + Path: file.String(), + }) + } + + return secrets, nil +}