From 89cff8726f180fe0511980ff1ae1ae162b914f64 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 12 Aug 2024 22:13:23 -0700 Subject: [PATCH 1/6] feat: add sys.model.provider.credential --- go.mod | 2 +- go.sum | 4 +- pkg/builtin/builtin.go | 42 +++++++++++--- pkg/credentials/credential.go | 1 + pkg/engine/cmd.go | 2 +- pkg/engine/engine.go | 8 ++- pkg/llm/proxy.go | 104 ++++++++++++++++++++++++++++++++++ pkg/llm/registry.go | 57 ++++++++++++++++++- pkg/openai/client.go | 7 +++ pkg/runner/runner.go | 27 ++++++--- pkg/tests/tester/runner.go | 4 ++ pkg/types/tool.go | 11 +++- pkg/types/toolstring.go | 2 +- 13 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 pkg/llm/proxy.go diff --git a/go.mod b/go.mod index 4cf0ba73..3cfbc98e 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid v1.6.0 github.com/gptscript-ai/broadcaster v0.0.0-20240625175512-c43682019b86 - github.com/gptscript-ai/chat-completion-client v0.0.0-20240531200700-af8e7ecf0379 + github.com/gptscript-ai/chat-completion-client v0.0.0-20240813051153-a440ada7e3c3 github.com/gptscript-ai/cmd v0.0.0-20240802230653-326b7baf6fcb github.com/gptscript-ai/go-gptscript v0.9.4-0.20240801203434-840b14393b17 github.com/gptscript-ai/tui v0.0.0-20240804004233-efc5673dc76e diff --git a/go.sum b/go.sum index 1b518130..85a3f76e 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gptscript-ai/broadcaster v0.0.0-20240625175512-c43682019b86 h1:m9yLtIEd0z1ia8qFjq3u0Ozb6QKwidyL856JLJp6nbA= github.com/gptscript-ai/broadcaster v0.0.0-20240625175512-c43682019b86/go.mod h1:lK3K5EZx4dyT24UG3yCt0wmspkYqrj4D/8kxdN3relk= -github.com/gptscript-ai/chat-completion-client v0.0.0-20240531200700-af8e7ecf0379 h1:vYnXoIyCXzaCEw0sYifQ4bDpsv3/fO/dZ2suEsTwCIo= -github.com/gptscript-ai/chat-completion-client v0.0.0-20240531200700-af8e7ecf0379/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo= +github.com/gptscript-ai/chat-completion-client v0.0.0-20240813051153-a440ada7e3c3 h1:EQiFTZv+BnOWJX2B9XdF09fL2Zj7h19n1l23TpWCafc= +github.com/gptscript-ai/chat-completion-client v0.0.0-20240813051153-a440ada7e3c3/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo= github.com/gptscript-ai/cmd v0.0.0-20240802230653-326b7baf6fcb h1:ky2J2CzBOskC7Jgm2VJAQi2x3p7FVGa+2/PcywkFJuc= github.com/gptscript-ai/cmd v0.0.0-20240802230653-326b7baf6fcb/go.mod h1:DJAo1xTht1LDkNYFNydVjTHd576TC7MlpsVRl3oloVw= github.com/gptscript-ai/go-gptscript v0.9.4-0.20240801203434-840b14393b17 h1:BTfJ6ls31Roq42lznlZnuPzRf0wrT8jT+tWcvq7wDXY= diff --git a/pkg/builtin/builtin.go b/pkg/builtin/builtin.go index f6811549..23db5152 100644 --- a/pkg/builtin/builtin.go +++ b/pkg/builtin/builtin.go @@ -26,14 +26,15 @@ import ( ) var SafeTools = map[string]struct{}{ - "sys.abort": {}, - "sys.chat.finish": {}, - "sys.chat.history": {}, - "sys.chat.current": {}, - "sys.echo": {}, - "sys.prompt": {}, - "sys.time.now": {}, - "sys.context": {}, + "sys.abort": {}, + "sys.chat.finish": {}, + "sys.chat.history": {}, + "sys.chat.current": {}, + "sys.echo": {}, + "sys.prompt": {}, + "sys.time.now": {}, + "sys.context": {}, + "sys.model.provider.credential": {}, } var tools = map[string]types.Tool{ @@ -248,6 +249,15 @@ var tools = map[string]types.Tool{ BuiltinFunc: SysContext, }, }, + "sys.model.provider.credential": { + ToolDef: types.ToolDef{ + Parameters: types.Parameters{ + Description: "A credential tool to set the OPENAI_API_KEY and OPENAI_BASE_URL to give access to the default model provider", + Arguments: types.ObjectSchema(), + }, + BuiltinFunc: SysModelProviderCredential, + }, + }, } func ListTools() (result []types.Tool) { @@ -678,6 +688,22 @@ func invalidArgument(input string, err error) string { return fmt.Sprintf("Failed to parse arguments %s: %v", input, err) } +func SysModelProviderCredential(ctx context.Context, _ []string, _ string, _ chan<- string) (string, error) { + engineContext, _ := engine.FromContext(ctx) + auth, url, err := engineContext.Engine.Model.ProxyInfo() + if err != nil { + return "", err + } + data, err := json.Marshal(map[string]any{ + "env": map[string]string{ + "OPENAI_API_KEY": auth, + "OPENAI_BASE_URL": url, + }, + "ephemeral": true, + }) + return string(data), err +} + func SysContext(ctx context.Context, _ []string, _ string, _ chan<- string) (string, error) { engineContext, _ := engine.FromContext(ctx) diff --git a/pkg/credentials/credential.go b/pkg/credentials/credential.go index 3d1e2192..f589a065 100644 --- a/pkg/credentials/credential.go +++ b/pkg/credentials/credential.go @@ -24,6 +24,7 @@ type Credential struct { ToolName string `json:"toolName"` Type CredentialType `json:"type"` Env map[string]string `json:"env"` + Ephemeral bool `json:"ephemeral,omitempty"` ExpiresAt *time.Time `json:"expiresAt"` RefreshToken string `json:"refreshToken"` } diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index 14b41183..960bcfe8 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -109,7 +109,7 @@ func (e *Engine) runCommand(ctx Context, tool types.Tool, input string, toolCate } }() - return tool.BuiltinFunc(ctx.WrappedContext(), e.Env, input, progress) + return tool.BuiltinFunc(ctx.WrappedContext(e), e.Env, input, progress) } var instructions []string diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index d3daa674..20ca43a9 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -16,6 +16,7 @@ import ( type Model interface { Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) + ProxyInfo() (string, string, error) } type RuntimeManager interface { @@ -79,6 +80,7 @@ type Context struct { Parent *Context LastReturn *Return CurrentReturn *Return + Engine *Engine Program *types.Program // Input is saved only so that we can render display text, don't use otherwise Input string @@ -250,8 +252,10 @@ func FromContext(ctx context.Context) (*Context, bool) { return c, ok } -func (c *Context) WrappedContext() context.Context { - return context.WithValue(c.Ctx, engineContext{}, c) +func (c *Context) WrappedContext(e *Engine) context.Context { + cp := *c + cp.Engine = e + return context.WithValue(c.Ctx, engineContext{}, &cp) } func (e *Engine) Start(ctx Context, input string) (ret *Return, _ error) { diff --git a/pkg/llm/proxy.go b/pkg/llm/proxy.go new file mode 100644 index 00000000..7c3091b3 --- /dev/null +++ b/pkg/llm/proxy.go @@ -0,0 +1,104 @@ +package llm + +import ( + "bytes" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httputil" + "net/url" + "path" + "strings" + + "github.com/gptscript-ai/gptscript/pkg/builtin" + "github.com/gptscript-ai/gptscript/pkg/openai" +) + +func (r *Registry) ProxyInfo() (string, string, error) { + r.proxyLock.Lock() + defer r.proxyLock.Unlock() + + if r.proxyURL != "" { + return r.proxyToken, r.proxyURL, nil + } + + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", "", err + } + + go func() { + _ = http.Serve(l, r) + r.proxyLock.Lock() + defer r.proxyLock.Unlock() + _ = l.Close() + r.proxyURL = "" + }() + + r.proxyURL = "http://" + l.Addr().String() + return r.proxyToken, r.proxyURL, nil +} + +func (r *Registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if r.proxyToken != strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ") { + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + + inBytes, err := io.ReadAll(req.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var ( + model string + data = map[string]any{} + ) + + if json.Unmarshal(inBytes, &data) == nil { + model, _ = data["model"].(string) + } + + if model == "" { + model = builtin.GetDefaultModel() + } + + c, err := r.getClient(req.Context(), model) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + oai, ok := c.(*openai.Client) + if !ok { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + auth, targetURL := oai.ProxyInfo() + if targetURL == "" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + newURL, err := url.Parse(targetURL) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + newURL.Path = path.Join(newURL.Path, req.URL.Path) + + rp := httputil.ReverseProxy{ + Director: func(proxyReq *http.Request) { + proxyReq.Body = io.NopCloser(bytes.NewReader(inBytes)) + proxyReq.URL = newURL + proxyReq.Header.Del("Authorization") + proxyReq.Header.Add("Authorization", "Bearer "+auth) + proxyReq.Host = newURL.Hostname() + }, + } + rp.ServeHTTP(w, req) +} diff --git a/pkg/llm/registry.go b/pkg/llm/registry.go index c568b43c..8129c788 100644 --- a/pkg/llm/registry.go +++ b/pkg/llm/registry.go @@ -5,7 +5,10 @@ import ( "errors" "fmt" "sort" + "sync" + "github.com/google/uuid" + "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/openai" "github.com/gptscript-ai/gptscript/pkg/remote" "github.com/gptscript-ai/gptscript/pkg/types" @@ -18,11 +21,16 @@ type Client interface { } type Registry struct { - clients []Client + proxyToken string + proxyURL string + proxyLock sync.Mutex + clients []Client } func NewRegistry() *Registry { - return &Registry{} + return &Registry{ + proxyToken: env.VarOrDefault("GPTSCRIPT_INTERNAL_PROXY_TOKEN", uuid.New().String()), + } } func (r *Registry) AddClient(client Client) error { @@ -44,6 +52,10 @@ func (r *Registry) ListModels(ctx context.Context, providers ...string) (result func (r *Registry) fastPath(modelName string) Client { // This is optimization hack to avoid doing List Models + if len(r.clients) == 1 { + return r.clients[0] + } + if len(r.clients) != 2 { return nil } @@ -66,6 +78,47 @@ func (r *Registry) fastPath(modelName string) Client { return r.clients[0] } +func (r *Registry) getClient(ctx context.Context, modelName string) (Client, error) { + if c := r.fastPath(modelName); c != nil { + return c, nil + } + + var errs []error + var oaiClient *openai.Client + for _, client := range r.clients { + ok, err := client.Supports(ctx, modelName) + if err != nil { + // If we got an OpenAI invalid auth error back, store the OpenAI client for later. + if errors.Is(err, openai.InvalidAuthError{}) { + oaiClient = client.(*openai.Client) + } + + errs = append(errs, err) + } else if ok { + return client, nil + } + } + + if len(errs) > 0 && oaiClient != nil { + // Prompt the user to enter their OpenAI API key and try again. + if err := oaiClient.RetrieveAPIKey(ctx); err != nil { + return nil, err + } + ok, err := oaiClient.Supports(ctx, modelName) + if err != nil { + return nil, err + } else if ok { + return oaiClient, nil + } + } + + if len(errs) == 0 { + return nil, fmt.Errorf("failed to find a model provider for model [%s]", modelName) + } + + return nil, errors.Join(errs...) +} + func (r *Registry) Call(ctx context.Context, messageRequest types.CompletionRequest, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) { if messageRequest.Model == "" { return nil, fmt.Errorf("model is required") diff --git a/pkg/openai/client.go b/pkg/openai/client.go index 53252895..42a1a39e 100644 --- a/pkg/openai/client.go +++ b/pkg/openai/client.go @@ -130,6 +130,13 @@ func NewClient(ctx context.Context, credStore credentials.CredentialStore, opts }, nil } +func (c *Client) ProxyInfo() (token, urlBase string) { + if c.invalidAuth { + return "", "" + } + return c.c.GetAPIKeyAndBaseURL() +} + func (c *Client) ValidAuth() error { if c.invalidAuth { return InvalidAuthError{} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index a8d88fee..f92b0705 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -872,6 +872,11 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env return nil, fmt.Errorf("failed to parse credential tool %q: %w", ref.Reference, err) } + if callCtx.Program.ToolSet[ref.ToolID].IsNoop() { + // ignore empty tools + continue + } + credName := toolName if credentialAlias != "" { credName = credentialAlias @@ -944,6 +949,10 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env return nil, fmt.Errorf("invalid state: credential tool [%s] can not result in a continuation", ref.Reference) } + if *res.Result == "" { + continue + } + if err := json.Unmarshal([]byte(*res.Result), &c); err != nil { return nil, fmt.Errorf("failed to unmarshal credential tool %s response: %w", ref.Reference, err) } @@ -958,15 +967,17 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env } } - // Only store the credential if the tool is on GitHub or has an alias, and the credential is non-empty. - if (isGitHubTool(toolName) && callCtx.Program.ToolSet[ref.ToolID].Source.Repo != nil) || credentialAlias != "" { - if isEmpty { - log.Warnf("Not saving empty credential for tool %s", toolName) - } else if err := r.credStore.Add(callCtx.Ctx, *c); err != nil { - return nil, fmt.Errorf("failed to add credential for tool %s: %w", toolName, err) + if !c.Ephemeral { + // Only store the credential if the tool is on GitHub or has an alias, and the credential is non-empty. + if (isGitHubTool(toolName) && callCtx.Program.ToolSet[ref.ToolID].Source.Repo != nil) || credentialAlias != "" { + if isEmpty { + log.Warnf("Not saving empty credential for tool %s", toolName) + } else if err := r.credStore.Add(callCtx.Ctx, *c); err != nil { + return nil, fmt.Errorf("failed to add credential for tool %s: %w", toolName, err) + } + } else { + log.Warnf("Not saving credential for tool %s - credentials will only be saved for tools from GitHub, or tools that use aliases.", toolName) } - } else { - log.Warnf("Not saving credential for tool %s - credentials will only be saved for tools from GitHub, or tools that use aliases.", toolName) } } diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index a36c5e91..66337ff5 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -31,6 +31,10 @@ type Result struct { Err error } +func (c *Client) ProxyInfo() (string, string, error) { + return "test-auth", "test-url", nil +} + func (c *Client) Call(_ context.Context, messageRequest types.CompletionRequest, _ chan<- types.CompletionStatus) (resp *types.CompletionMessage, respErr error) { msgData, err := json.MarshalIndent(messageRequest, "", " ") require.NoError(c.t, err) diff --git a/pkg/types/tool.go b/pkg/types/tool.go index b59a1953..57ce3fbf 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -753,7 +753,16 @@ func (t Tool) GetCredentialTools(prg Program, agentGroup []ToolReference) ([]Too result.AddAll(t.getCompletionToolRefs(prg, nil, ToolTypeCredential)) - toolRefs, err := t.getCompletionToolRefs(prg, agentGroup) + toolRefs, err := result.List() + if err != nil { + return nil, err + } + for _, toolRef := range toolRefs { + referencedTool := prg.ToolSet[toolRef.ToolID] + result.AddAll(referencedTool.GetToolRefsFromNames(referencedTool.ExportCredentials)) + } + + toolRefs, err = t.getCompletionToolRefs(prg, agentGroup) if err != nil { return nil, err } diff --git a/pkg/types/toolstring.go b/pkg/types/toolstring.go index 64f53638..2be6d0fc 100644 --- a/pkg/types/toolstring.go +++ b/pkg/types/toolstring.go @@ -74,7 +74,7 @@ func ToSysDisplayString(id string, args map[string]string) (string, error) { return fmt.Sprintf("Removing `%s`", args["location"]), nil case "sys.write": return fmt.Sprintf("Writing `%s`", args["filename"]), nil - case "sys.context", "sys.stat", "sys.getenv", "sys.abort", "sys.chat.current", "sys.chat.finish", "sys.chat.history", "sys.echo", "sys.prompt", "sys.time.now": + case "sys.context", "sys.stat", "sys.getenv", "sys.abort", "sys.chat.current", "sys.chat.finish", "sys.chat.history", "sys.echo", "sys.prompt", "sys.time.now", "sys.model.provider.credential": return "", nil default: return "", fmt.Errorf("unknown tool for display string: %s", id) From a4f3253870cd9d55ca1a3c5f42eca67098d5b56d Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Tue, 13 Aug 2024 09:40:50 -0700 Subject: [PATCH 2/6] bug: allow asterick on go binary checksum files --- pkg/repos/runtimes/golang/golang.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index f82b628a..3514fadb 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -223,10 +223,9 @@ func getChecksum(ctx context.Context, rel *release) string { scan := bufio.NewScanner(resp.Body) for scan.Scan() { fields := strings.Fields(scan.Text()) - if len(fields) != 2 || fields[1] != rel.srcBinName() { - continue + if len(fields) == 2 && (fields[1] == rel.srcBinName() || fields[1] == "*"+rel.srcBinName()) { + return fields[0] } - return fields[0] } return "" From 5c608125e1af9b9a0bba0d34079eebecc576fbac Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 13 Aug 2024 14:43:19 -0400 Subject: [PATCH 3/6] feat: download binaries for cred helpers Signed-off-by: Donnie Adams --- pkg/cli/credential.go | 2 +- pkg/cli/credential_delete.go | 2 +- pkg/cli/credential_show.go | 2 +- pkg/credentials/util.go | 4 +- pkg/engine/engine.go | 2 +- pkg/gptscript/gptscript.go | 2 +- pkg/repos/get.go | 74 ++++++++++++++--------------- pkg/repos/runtimes/golang/golang.go | 55 ++++++++++++--------- pkg/runner/runtimemanager.go | 2 +- 9 files changed, 77 insertions(+), 68 deletions(-) diff --git a/pkg/cli/credential.go b/pkg/cli/credential.go index b0c4a30a..cb000125 100644 --- a/pkg/cli/credential.go +++ b/pkg/cli/credential.go @@ -58,7 +58,7 @@ func (c *Credential) Run(cmd *cobra.Command, _ []string) error { opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) } - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil { + if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } diff --git a/pkg/cli/credential_delete.go b/pkg/cli/credential_delete.go index 9c986c54..4e9919df 100644 --- a/pkg/cli/credential_delete.go +++ b/pkg/cli/credential_delete.go @@ -40,7 +40,7 @@ func (c *Delete) Run(cmd *cobra.Command, args []string) error { opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) } - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil { + if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } diff --git a/pkg/cli/credential_show.go b/pkg/cli/credential_show.go index ccfe3675..fac1b719 100644 --- a/pkg/cli/credential_show.go +++ b/pkg/cli/credential_show.go @@ -42,7 +42,7 @@ func (c *Show) Run(cmd *cobra.Command, args []string) error { opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir) } - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg, opts.Env); err != nil { + if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } diff --git a/pkg/credentials/util.go b/pkg/credentials/util.go index 367b4d1d..70f31e97 100644 --- a/pkg/credentials/util.go +++ b/pkg/credentials/util.go @@ -5,7 +5,7 @@ import ( ) type CredentialHelperDirs struct { - RevisionFile, LastCheckedFile, BinDir, RepoDir, HelperDir string + RevisionFile, LastCheckedFile, BinDir string } func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs { @@ -13,7 +13,5 @@ func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs { RevisionFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "revision"), LastCheckedFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "last-checked"), BinDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "bin"), - RepoDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "repo"), - HelperDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers"), } } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index d3daa674..cb8fe273 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -21,7 +21,7 @@ type Model interface { type RuntimeManager interface { GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) EnsureCredentialHelpers(ctx context.Context) error - SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error + SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error } type Engine struct { diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index 43f429fc..755fe632 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -99,7 +99,7 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) { opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir()) } - if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg, opts.Env); err != nil { + if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg); err != nil { return nil, err } diff --git a/pkg/repos/get.go b/pkg/repos/get.go index 416f4c61..8981d1fa 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "strings" "sync" "time" @@ -15,15 +16,13 @@ import ( "github.com/BurntSushi/locker" "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" - "github.com/gptscript-ai/gptscript/pkg/loader/github" "github.com/gptscript-ai/gptscript/pkg/repos/git" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" "github.com/gptscript-ai/gptscript/pkg/types" ) -const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-helpers" - type Runtime interface { ID() string Supports(tool types.Tool, cmd []string) bool @@ -68,7 +67,6 @@ type credHelperConfig struct { lock sync.Mutex initialized bool cliCfg *config.CLIConfig - env []string } func New(cacheDir string, runtimes ...Runtime) *Manager { @@ -90,7 +88,7 @@ func (m *Manager) EnsureCredentialHelpers(ctx context.Context) error { defer m.credHelperConfig.lock.Unlock() if !m.credHelperConfig.initialized { - if err := m.deferredSetUpCredentialHelpers(ctx, m.credHelperConfig.cliCfg, m.credHelperConfig.env); err != nil { + if err := m.deferredSetUpCredentialHelpers(ctx, m.credHelperConfig.cliCfg); err != nil { return err } m.credHelperConfig.initialized = true @@ -99,27 +97,28 @@ func (m *Manager) EnsureCredentialHelpers(ctx context.Context) error { return nil } -func (m *Manager) SetUpCredentialHelpers(_ context.Context, cliCfg *config.CLIConfig, env []string) error { +func (m *Manager) SetUpCredentialHelpers(_ context.Context, cliCfg *config.CLIConfig) error { m.credHelperConfig = &credHelperConfig{ cliCfg: cliCfg, - env: env, } return nil } -func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig, env []string) error { +func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error { var ( - helperName = cliCfg.CredentialsStore - suffix string + helperName = cliCfg.CredentialsStore + distInfo, suffix string ) - if helperName == "wincred" { - suffix = ".exe" - } - - // The file helper is built-in and does not need to be compiled. + // The file helper is built-in and does not need to be downloaded. if helperName == "file" { return nil } + switch helperName { + case "wincred": + suffix = ".exe" + default: + distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH) + } locker.Lock("gptscript-credential-helpers") defer locker.Unlock("gptscript-credential-helpers") @@ -137,13 +136,7 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co } } - // Load the credential helpers repo information. - _, _, repo, _, err := github.Load(ctx, nil, credentialHelpersRepo) - if err != nil { - return err - } - - if err := os.MkdirAll(m.credHelperDirs.HelperDir, 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(m.credHelperDirs.LastCheckedFile), 0755); err != nil { return err } @@ -152,37 +145,44 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co return err } - var needsBuild bool + tool := types.Tool{ + Source: types.ToolSource{ + Repo: &types.Repo{ + Root: runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"), + }, + }, + } + tag, err := golang.GetLatestTag(tool) + if err != nil { + return err + } + var needsDownloaded bool // Check the last revision shasum and see if it is different from the current one. lastRevision, err := os.ReadFile(m.credHelperDirs.RevisionFile) - if (err == nil && strings.TrimSpace(string(lastRevision)) != repo.Revision) || errors.Is(err, fs.ErrNotExist) { + if (err == nil && strings.TrimSpace(string(lastRevision)) != tool.Source.Repo.Root+tag) || errors.Is(err, fs.ErrNotExist) { // Need to pull the latest version. - needsBuild = true - if err := git.Checkout(ctx, m.gitDir, repo.Root, repo.Revision, filepath.Join(m.credHelperDirs.RepoDir, repo.Revision)); err != nil { - return err - } + needsDownloaded = true // Update the revision file to the new revision. - if err := os.WriteFile(m.credHelperDirs.RevisionFile, []byte(repo.Revision), 0644); err != nil { + if err = os.WriteFile(m.credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil { return err } } else if err != nil { return err } - if !needsBuild { - // Check for the existence of the gptscript-credential-osxkeychain binary. - // If it's there, we have no need to build it and can just return. - if _, err := os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { + if !needsDownloaded { + // Check for the existence of the credential helper binary. + // If it's there, we have no need to download it and can just return. + if _, err = os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { return nil } } // Find the Go runtime and use it to build the credential helper. - for _, runtime := range m.runtimes { - if strings.HasPrefix(runtime.ID(), "go") { - goRuntime := runtime.(*golang.Runtime) - return goRuntime.BuildCredentialHelper(ctx, helperName, m.credHelperDirs, m.runtimeDir, repo.Revision, env) + for _, rt := range m.runtimes { + if strings.HasPrefix(rt.ID(), "go") { + return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, m.credHelperDirs.BinDir) } } diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 3514fadb..52e8fe0b 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -18,7 +18,6 @@ import ( "runtime" "strings" - "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/debugcmd" runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" @@ -97,6 +96,14 @@ type tag struct { } `json:"commit"` } +func GetLatestTag(tool types.Tool) (string, error) { + r, ok := getLatestRelease(tool) + if !ok { + return "", fmt.Errorf("failed to get latest release for %s", tool.Name) + } + return r.label, nil +} + func getLatestRelease(tool types.Tool) (*release, bool) { if tool.Source.Repo == nil || !strings.HasPrefix(tool.Source.Repo.Root, "https://github.com/") { return nil, false @@ -116,11 +123,14 @@ func getLatestRelease(tool types.Tool) (*release, bool) { account, repo := parts[1], parts[2] resp, err := client.Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/tags", account, repo)) - if err != nil || resp.StatusCode != http.StatusOK { + if err != nil { // ignore error return nil, false } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, false + } var tags []tag if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil { @@ -137,11 +147,14 @@ func getLatestRelease(tool types.Tool) (*release, bool) { } resp, err = client.Get(fmt.Sprintf("https://github.com/%s/%s/releases/latest", account, repo)) - if err != nil || resp.StatusCode != http.StatusFound { + if err != nil { // ignore error return nil, false } defer resp.Body.Close() + if resp.StatusCode != http.StatusFound { + return nil, false + } target := resp.Header.Get("Location") if target == "" { @@ -212,7 +225,7 @@ func downloadBin(ctx context.Context, checksum, src, url, bin string) error { return nil } -func getChecksum(ctx context.Context, rel *release) string { +func getChecksum(ctx context.Context, rel *release, artifactName string) string { resp, err := get(ctx, rel.checksumTxt()) if err != nil { // ignore error @@ -223,7 +236,7 @@ func getChecksum(ctx context.Context, rel *release) string { scan := bufio.NewScanner(resp.Body) for scan.Scan() { fields := strings.Fields(scan.Text()) - if len(fields) == 2 && (fields[1] == rel.srcBinName() || fields[1] == "*"+rel.srcBinName()) { + if len(fields) == 2 && (fields[1] == artifactName || fields[1] == "*"+artifactName) { return fields[0] } } @@ -241,7 +254,7 @@ func (r *Runtime) Binary(ctx context.Context, tool types.Tool, _, toolSource str return false, nil, nil } - checksum := getChecksum(ctx, rel) + checksum := getChecksum(ctx, rel, rel.srcBinName()) if checksum == "" { return false, nil, nil } @@ -268,30 +281,28 @@ func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource return newEnv, nil } -func (r *Runtime) BuildCredentialHelper(ctx context.Context, helperName string, credHelperDirs credentials.CredentialHelperDirs, dataRoot, revision string, env []string) error { +func (r *Runtime) DownloadCredentialHelper(ctx context.Context, tool types.Tool, helperName, distInfo, suffix string, binDir string) error { if helperName == "file" { return nil } - var suffix string - if helperName == "wincred" { - suffix = ".exe" + rel, ok := getLatestRelease(tool) + if !ok { + return fmt.Errorf("failed to find %s release", r.ID()) + } + binaryName := "gptscript-credential-" + helperName + checksum := getChecksum(ctx, rel, binaryName+distInfo+suffix) + if checksum == "" { + return fmt.Errorf("failed to find %s release checksum for os=%s arch=%s", r.ID(), runtime.GOOS, runtime.GOARCH) } - binPath, err := r.getRuntime(ctx, dataRoot) - if err != nil { - return err + url, _ := strings.CutSuffix(rel.binURL(), rel.srcBinName()) + url += binaryName + distInfo + suffix + if err := downloadBin(ctx, checksum, strings.TrimSuffix(binDir, "bin"), url, binaryName+suffix); err != nil { + return fmt.Errorf("failed to download %s release for os=%s arch=%s: %w", r.ID(), runtime.GOOS, runtime.GOARCH, err) } - newEnv := runtimeEnv.AppendPath(env, binPath) - log.InfofCtx(ctx, "Building credential helper %s", helperName) - cmd := debugcmd.New(ctx, filepath.Join(binPath, "go"), - "build", "-buildvcs=false", "-o", - filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix), - fmt.Sprintf("./%s/cmd/", helperName)) - cmd.Env = stripGo(append(env, newEnv...)) - cmd.Dir = filepath.Join(credHelperDirs.RepoDir, revision) - return cmd.Run() + return nil } func (r *Runtime) getReleaseAndDigest() (string, string, error) { diff --git a/pkg/runner/runtimemanager.go b/pkg/runner/runtimemanager.go index e1c5a4c6..ed191d15 100644 --- a/pkg/runner/runtimemanager.go +++ b/pkg/runner/runtimemanager.go @@ -45,6 +45,6 @@ func (r runtimeManagerLogger) EnsureCredentialHelpers(ctx context.Context) error return r.rm.EnsureCredentialHelpers(mvl.WithInfo(ctx, r)) } -func (r runtimeManagerLogger) SetUpCredentialHelpers(_ context.Context, _ *config.CLIConfig, _ []string) error { +func (r runtimeManagerLogger) SetUpCredentialHelpers(_ context.Context, _ *config.CLIConfig) error { panic("not implemented") } From 983cd8c9e58342e520c91a7a2c4a1c556ec85fec Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Tue, 13 Aug 2024 15:07:17 -0700 Subject: [PATCH 4/6] chore: add load method to sdk server --- pkg/repos/runtimes/default.go | 2 +- pkg/repos/runtimes/node/SHASUMS256.txt.asc | 165 +++++++-------------- pkg/repos/runtimes/node/node_test.go | 14 -- pkg/sdkserver/routes.go | 38 +++++ pkg/sdkserver/types.go | 9 ++ 5 files changed, 102 insertions(+), 126 deletions(-) diff --git a/pkg/repos/runtimes/default.go b/pkg/repos/runtimes/default.go index 3782e26e..d4eb4db5 100644 --- a/pkg/repos/runtimes/default.go +++ b/pkg/repos/runtimes/default.go @@ -22,7 +22,7 @@ var Runtimes = []repos.Runtime{ Version: "3.10", }, &node.Runtime{ - Version: "21", + Version: "20", Default: true, }, &golang.Runtime{ diff --git a/pkg/repos/runtimes/node/SHASUMS256.txt.asc b/pkg/repos/runtimes/node/SHASUMS256.txt.asc index 093da96b..faaeab97 100644 --- a/pkg/repos/runtimes/node/SHASUMS256.txt.asc +++ b/pkg/repos/runtimes/node/SHASUMS256.txt.asc @@ -1,117 +1,60 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 -43a881788549e1b3425eb5f2b92608f438f146e08213de09c5bd5ff841cae7ae node-v20.11.1-aix-ppc64.tar.gz -3f8e77b775372c0b27d2b85ce899d80339691f480e64dde43d4eb01504a58679 node-v20.11.1-arm64.msi -e0065c61f340e85106a99c4b54746c5cee09d59b08c5712f67f99e92aa44995d node-v20.11.1-darwin-arm64.tar.gz -fd771bf3881733bfc0622128918ae6baf2ed1178146538a53c30ac2f7006af5b node-v20.11.1-darwin-arm64.tar.xz -c52e7fb0709dbe63a4cbe08ac8af3479188692937a7bd8e776e0eedfa33bb848 node-v20.11.1-darwin-x64.tar.gz -ed69f1f300beb75fb4cad45d96aacd141c3ddca03b6d77c76b42cb258202363d node-v20.11.1-darwin-x64.tar.xz -0aa42c91b441e945ff43bd3a837759c58b436de57dcd033d02e5cbcd2fba1f87 node-v20.11.1-headers.tar.gz -edce238817acf5adce3123366b55304aff2a1f0849231d1b49f42370e454b6f8 node-v20.11.1-headers.tar.xz -e34ab2fc2726b4abd896bcbff0250e9b2da737cbd9d24267518a802ed0606f3b node-v20.11.1-linux-arm64.tar.gz -c957f29eb4e341903520caf362534f0acd1db7be79c502ae8e283994eed07fe1 node-v20.11.1-linux-arm64.tar.xz -e42791f76ece283c7a4b97fbf716da72c5128c54a9779f10f03ae74a4bcfb8f6 node-v20.11.1-linux-armv7l.tar.gz -28e0120d2d150a8f41717899d33167b8b32053778665583d49ff971bfd188d1b node-v20.11.1-linux-armv7l.tar.xz -9823305ac3a66925a9b61d8032f6bbb4c3e33c28e7f957ebb27e49732feffb23 node-v20.11.1-linux-ppc64le.tar.gz -51343cacf5cdf5c4b5e93e919d19dd373d6ef43d5f2c666eae299f26e31d08b5 node-v20.11.1-linux-ppc64le.tar.xz -4c66b2f247fdd8720853321526d7cda483018fcb32014b75c30f3a54ecacaea7 node-v20.11.1-linux-s390x.tar.gz -b32616b705cd0ddbb230b95c693e3d7a37becc2ced9bcadea8dc824cceed6be0 node-v20.11.1-linux-s390x.tar.xz -bf3a779bef19452da90fb88358ec2c57e0d2f882839b20dc6afc297b6aafc0d7 node-v20.11.1-linux-x64.tar.gz -d8dab549b09672b03356aa2257699f3de3b58c96e74eb26a8b495fbdc9cf6fbe node-v20.11.1-linux-x64.tar.xz -f1cd449fcbeb1b948e8498cb8edd9655fa319d109a7f4c5bd96a9b122b91538a node-v20.11.1-win-arm64.7z -e85461ec124956a2853c4ee6e13c4f4889d63c88beb3d530c1ee0c4b51dc10e7 node-v20.11.1-win-arm64.zip -fb9b5348259988a562a48eed7349e7e716c0bec78d98ad0a336b2993a8b3bf34 node-v20.11.1-win-x64.7z -bc032628d77d206ffa7f133518a6225a9c5d6d9210ead30d67e294ff37044bda node-v20.11.1-win-x64.zip -c2b1863d8979546804a39fc63d0a9bc9c6e49cb2f6c9d1e52844a24629b24765 node-v20.11.1-win-x86.7z -b98e95f78416d1359b647cfa09ba2a48b76d41b56a776df822bf36ffe8e76a2d node-v20.11.1-win-x86.zip -c54f5f7e2416e826fd84e878f28e3b53363ae9c3f60a140af4434b2453b5ae89 node-v20.11.1-x64.msi -63e2aed4dabb96eed6903a3974e006d3c29c218472aac60ae3c3c7de00df13b1 node-v20.11.1-x86.msi -c46019a095a1549d000e85da13f17972a448e0be5854a51786ecccde7278a012 node-v20.11.1.pkg -4af1ba6ea848cc05908b8a62b02fb27684dd52b2a7988ee82b0cfa72deb90b94 node-v20.11.1.tar.gz -77813edbf3f7f16d2d35d3353443dee4e61d5ee84d9e3138c7538a3c0ca5209e node-v20.11.1.tar.xz -a5a9d30a8f7d56e00ccb27c1a7d24c8d0bc96a2689ebba8eb7527698793496f1 win-arm64/node.exe -93529170cebe57c0f4830a4cc6a261b6cc9bcf0cd8b3e88ac4995a5015031d79 win-arm64/node.lib -c14c6e927406b8683cbfb8a67ca4c8fd5093ca7812b5b1627e3d6a53d3674565 win-arm64/node_pdb.7z -68034cd09d8dfaa755d1b280da13e20388cc486ac57b037b3e11dfe2d6b74284 win-arm64/node_pdb.zip -bc585910690318aaebe3c57669cb83ca9d1e5791efd63195e238f54686e6c2ec win-x64/node.exe -53a982d490cb9fcc4b231a8b95147de423b36186bc6f4ba5697b20117fdcbd5d win-x64/node.lib -ccac9f2f5219ed858aeddb306d6493478ba9675c7cbf009e83742437d6752c4f win-x64/node_pdb.7z -bec5da4035c84580843978a59ef9bcc1c0eaca881cf9e1c94e63a1862cf14421 win-x64/node_pdb.zip -3829137e062b1e2eb9947ef05e4b717ae578a8fce1c5c60fe4f6ae7ef2ec0240 win-x86/node.exe -c5321bb65dcecb3989f9b8f6ec56369c16627ca4bade0c78afb6b88f7dde50e4 win-x86/node.lib -20ca60ced1fc21f15ea952b4406aec6bde39d20eab11cf042040628841b2249e win-x86/node_pdb.7z -bef05cebedce5949ae35e87e7d4789c16fa73caf478483fcf92e5dbb9ba5d774 win-x86/node_pdb.zip +5eb1b7ea405c86be0a21ec3850997c89df238d6e4659a0b990aa793a8cbfd9cf node-v20.16.0-aix-ppc64.tar.gz +f366fe5903dcb3b6cd495c8add77c87a32772085718a672d52ad17d9d91d2018 node-v20.16.0-arm64.msi +fc7355e778b181575153b7dea4879e8021776eeb376c43c50f65893d2ea70aa3 node-v20.16.0-darwin-arm64.tar.gz +5043e98cdf859963b1a0aff54c1f1813a2a8059e4179403171860d664ca090f2 node-v20.16.0-darwin-arm64.tar.xz +e18942cd706e4d69a4845ddacee2f1c17a72e853a229e3d2623d2edeb7efde72 node-v20.16.0-darwin-x64.tar.gz +9df751ac5edbb2181335200060dff14de25f828eaed70d8b48459d2c203aeedc node-v20.16.0-darwin-x64.tar.xz +6cc5690a67b9b1e1fa8cedaeca41f1bdb5e1af1f7948761c798d33d99f789a5c node-v20.16.0-headers.tar.gz +a1464c304980d3ab41922cda7025ebc2ec0dc2a0b89d9b9183c589560810feaa node-v20.16.0-headers.tar.xz +551588f8f5ca05c04efb53f1b2bb7d9834603327bdc82d60a944d385569866e1 node-v20.16.0-linux-arm64.tar.gz +1d9929e72f692179f884cd676b2dfabd879cb77defa7869dc8cfc802619277fb node-v20.16.0-linux-arm64.tar.xz +1c77c52ab507ddee479012f0b4bf523dd8400df4504447d623632353076e2e27 node-v20.16.0-linux-armv7l.tar.gz +a23a49029e8c7788c701eb3ace553260b7676a5a2ea9965ba92e4817008fbefe node-v20.16.0-linux-armv7l.tar.xz +80b515595e46afb9bae77f61083a4ca7c21bbdb627f69ff53fd5dca3a26773fb node-v20.16.0-linux-ppc64le.tar.gz +86cf6e8c93a9e517bfcfdfb4ad2774105312679ad21e03da75ab516ebc10e2dc node-v20.16.0-linux-ppc64le.tar.xz +ae7a9f6e631a0bede76a501d8b1d806f56b97acfa5a1d6833bab5ce90a404e5e node-v20.16.0-linux-s390x.tar.gz +6c38ac5c516a6a36ee6e0426975e6466795db30b9ced04e59f0f33fe6b3d657e node-v20.16.0-linux-s390x.tar.xz +b3f874ea84e440d69ed02ca92429d0eccd17737fde86db69c1c153d16ec654f2 node-v20.16.0-linux-x64.tar.gz +c30af7dfea46de7d8b9b370fa33b8b15440bc93f0a686af8601bbb48b82f16c0 node-v20.16.0-linux-x64.tar.xz +55852a420ca41db9f128f97e0dd8751199c23d63f5a7978432fd7c9e0c74c323 node-v20.16.0.pkg +8f24bf9abe455a09ab30f9ae8edda1e945ed678a4b1c3b07ee0f901fdc0ff4fd node-v20.16.0.tar.gz +cd6c8fc3ff2606aadbc7155db6f7e77247d2d0065ac18e2f7f049095584b8b46 node-v20.16.0.tar.xz +52e5666a379acd8533d9ccab66c2321a6ffc83766248419bfbd41ba8bc071244 node-v20.16.0-win-arm64.7z +af5a85ea299fcebd34c3c726a47a926e73171f9b657a6eaa796c011597241bf8 node-v20.16.0-win-arm64.zip +1b3961054a484476872715d9ca04bc491d797fde6336db514b6e6fcbb71fae9d node-v20.16.0-win-x64.7z +4e88373ac5ae859ad4d50cc3c5fa86eb3178d089b72e64c4dbe6eeac5d7b5979 node-v20.16.0-win-x64.zip +76f1806fde0b09ed4044f29ea140fb2bea9bce745b9892ec4aeb6537344db6f1 node-v20.16.0-win-x86.7z +1adc1f086595ecbc98da40eccb42fa1691b6c6c0658ff875dda19e4e02b1d5f0 node-v20.16.0-win-x86.zip +813306c94e6f5f061a5789f037d48f57d52240284a679e5ace4a0f73f8f2feeb node-v20.16.0-x64.msi +2bb8c3084384c95c47c4191c38098d5ecf55c0f02c1de5e0968730dec957ea15 node-v20.16.0-x86.msi +7e773fba3a19eac5ccbe85c1f87a05d7b112ecf41440076e6b6de1c7bffa0fdf win-arm64/node.exe +a4f01329c1c211082ac3ed387ff6651530040bbf7250ec419ce8f95b10d7804a win-arm64/node.lib +e1bec70ae9529cc637a21de850c070125f8016070451094d72f96408001200a2 win-arm64/node_pdb.7z +bc5b60eecd3b6c92b35755adef2e4aad01e021a3c434d46c2555a49056c5bcf7 win-arm64/node_pdb.zip +ba221658a3b68bd583e3068903eb675b5206d86a883c084ed95502e8f634b82a win-x64/node.exe +87056190b7cd06f40058f8e059efd328cdcc7600b825afa102c0aa5039865af5 win-x64/node.lib +bf2ad1e1f4e7c3853d5209fe9ef24ad7117edafc71f6401ec0121d8b681b8c3c win-x64/node_pdb.7z +5386f3c3af1af1b325b43b574043c5a7e830b3e9e7df0370ae0797ce4f39b375 win-x64/node_pdb.zip +b7b8d6b5fdd1c073b6f5f6d15bc849f4b5f92c4a66f23e77294f4bdf5f51e9f6 win-x86/node.exe +fa02ae7feca7eb6c4a0f1b929126df400719f5d18a2ec4b7d12c52fbe0b13814 win-x86/node.lib +328b2dcc91255c1c75faa8ce7eb687a8960ae09555d3bca0ae8e0dac4238c873 win-x86/node_pdb.7z +71b1e6b75c61227342ba6f1edb9014445dbee857d6cb14dce3d9b8d94c694d55 win-x86/node_pdb.zip -----BEGIN PGP SIGNATURE----- -iQGzBAEBCAAdFiEEiQwI24V5Fi/uDfnbi+q0389VXvQFAmXM+TcACgkQi+q0389V -XvQl3AwAqqm2uBMDzd+BlR1sG7y/eUtUYPVdwmCh0DeFXPHxuaIbFf0PGMEgcV8u -kn3OBF4pnSCPZNbJYJsLO1S+b/5Vk+Vlkq1WkOxqQHUHmM9GcJUuShadl0YaDNen -WXXMoYKWqMRJ6fQ3tRRh+vbMSXtsLqXT8TMVJq+Qb7a7yj4QRjw/Dd+8uKGGIhBY -U04HWsz33RJLu6AUnhF03eO1N8E1V48JptklDx5ZkY8GYa3F6jQsFld+jhmkZ9tg -4q9NDNijVpj56UsUhLAYD0J9IKS18tvQxNrKmBGUSZjFOByVhbUdLXnSMtW1i1U9 -cYhP6Q5wg/fnjqCfQ90TauoJZOblKIL/PHlf6cQGPrrRa1bz3xGyCAIve5KFhLxf -Vfj1ctk2ktzmuNhjAu5G/1VALQUNpiTm4Yz433JpoMMZ3mTHN+fuALOX4TQbdLRz -HKphTz02436348XC9bNz2cvjm74cy9fqwjQ/y84AmxiTJMFPg0XqICg4tu9rd49d -8FJc4TLZ -=r/CD ------END PGP SIGNATURE----- - ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA256 - -c31d5b60c4a284c6855bd468d4aae4436a16b351362b2971d3c0db2a471d3f24 node-v21.7.0-aix-ppc64.tar.gz -7d7dc37aa30b6dbb52d698d01cfed1d99056c82396eadd41a49fc55a873c423d node-v21.7.0-arm64.msi -f48ad51cf3c2814bbf61c8c26efd810e5e22dcac980786fd7ac5b54365233d2c node-v21.7.0-darwin-arm64.tar.gz -0805239d8a7402dae49e0033b7ad8208fed498dbeee9a3194863e52f6f3c6d7f node-v21.7.0-darwin-arm64.tar.xz -3f81adca80b523b413e26f03f855f4a2ae52d9af20f0cda2e259dd26a0608607 node-v21.7.0-darwin-x64.tar.gz -6a755416292854f2be38e74704ccf09edeba247718e9f047f5e1939b0dba17bd node-v21.7.0-darwin-x64.tar.xz -628d9e4f866e3828b77ce812dc99f33d3e7c673d0c499f13eadff6fa6ccb4383 node-v21.7.0-headers.tar.gz -627d6552d2120660a51f74fff0d40573f4a35d8545462250d30592ce0ba4eec7 node-v21.7.0-headers.tar.xz -520a3e5c83a05a782b1f4959f150c2fdc03e2ea056e855ef6bbb74f6ccf7aa7d node-v21.7.0-linux-arm64.tar.gz -73ce1e4e956532e0916fc7014f5b649573bd2b5870fef5cfc26cc42f58358ae7 node-v21.7.0-linux-arm64.tar.xz -723abb32135ad4baa6e9671447a72f5c9a5bfc681fc540b0e4864e965171b6ed node-v21.7.0-linux-armv7l.tar.gz -8a367a3bf667f0bb3abb9e8121326911d47a31886419ad052d5a52d8c6531d9d node-v21.7.0-linux-armv7l.tar.xz -c2290cb35b11ee2b0f0ae34ad3c8372652688ff2dc3d9a89ada46c2b84ea5dda node-v21.7.0-linux-ppc64le.tar.gz -b85348211a4d195de2f850a17cdec77aedc8fc1c402864b2bc3501608e6c9c47 node-v21.7.0-linux-ppc64le.tar.xz -90b8678ed113950613edeae5eaf298cf795c72005fac6ffd9b7fbb90ddd86738 node-v21.7.0-linux-s390x.tar.gz -99a09f4c790f3210a6d26032bf69713ba199cf2e73af43e04b1b1d9bd1c8db76 node-v21.7.0-linux-s390x.tar.xz -0fce039e2b6af00766492127a49f959ae92ed22fede4c49e9a8c2543aadbd6e2 node-v21.7.0-linux-x64.tar.gz -68510c3851133a21c6a6f9940e58c5bc8fed39f1d91a08e34c5070dd0615fef1 node-v21.7.0-linux-x64.tar.xz -d680d5c3d0b2476a97d11b30cbbdaf1d7f92ffd1cc89e5c640782a6b52480666 node-v21.7.0-win-arm64.7z -11b11b9a3f2db7b5076cf16655e05cd63dc3d8843cd4836ecb12e11315f03441 node-v21.7.0-win-arm64.zip -31c8b4721f37e30ca8e2131a4cb848fc7347f67bf87618e82959b58481f17bc4 node-v21.7.0-win-x64.7z -204de88f4073b08ae3dbe4c412b071eee565fc681e163be205d5cc88065f0322 node-v21.7.0-win-x64.zip -b17ef0c5557e61610774cae5beb0f877699ab419c4672e9c6e3bb3da3d571ed1 node-v21.7.0-win-x86.7z -6aba3fe2258d5c0c40a89e81dfe90113a67489f2a67fd05b7f216b63b4c7bb02 node-v21.7.0-win-x86.zip -512945cf8816e1e906143ea2ee6816f8744a3d114ea38f3540c3ebe685fe3e3a node-v21.7.0-x64.msi -4bedb6069c94a71fd6f0b8fbea280468d5ecdcf209eef6da1a45808e8b15cba6 node-v21.7.0-x86.msi -ccac99782e587c6090b6ad82979210fa0c352322636a6cf290d37eb41152d0b5 node-v21.7.0.pkg -26d6b600e1076f132d4175a90ddc1a709263e75d81967300aa1ffbd86103b991 node-v21.7.0.tar.gz -e41eefe1e59624ee7f312c38f8f7dfc11595641acb2293d21176f03d2763e9d4 node-v21.7.0.tar.xz -25511d1e05d7d0b049945c5ef1cf2a4daa5d6ad16692ccd2c1399142a1c57a65 win-arm64/node.exe -7920932f7be355dbf4568492ab7d104fc059f689ba1a46ac0d6568884c8d201a win-arm64/node.lib -40c423a2b126fc5b6858f8617f0f8537fd8f8d2fa73a5c918607f3ccd386f8c9 win-arm64/node_pdb.7z -dec9eaa91a431ea0f3c243605f0556dbe6459df5c04c10df7935d678a6b3fca4 win-arm64/node_pdb.zip -c486fe72a3663379105538e886ef9d2deacad1deaa64b338e570cb086be592d3 win-x64/node.exe -96d09c2055c2f252122c86b65d2aabd5f90b1a075844f24bf8bcdbab05baf53e win-x64/node.lib -08990dd6bcce80710d59ef76cd74ab98b5bed36b0d2584ca3acbc029f92db4fc win-x64/node_pdb.7z -1a27a25c92f6339b3aa77588063cca537af389aee26bfdf1d0ef505d790e63a3 win-x64/node_pdb.zip -4aaa5b3a95ee4ab932a80b9708c31662a9c4a99d19fea7cb1f7b0ff79d8399ed win-x86/node.exe -6e2502e84c3a0e2da643f6399b59381ade5b525f544a5bcabae923188b8f9998 win-x86/node.lib -d0cd5494364039f558c76d4fc7a1db69739149873e10a5200fb9e2a0ab12fe10 win-x86/node_pdb.7z -354031f3f9576733ebeeccbcafcc691c8326427153a48978ff5cd6f2c8ef5d36 win-x86/node_pdb.zip ------BEGIN PGP SIGNATURE----- - -iQGzBAEBCAAdFiEEiQwI24V5Fi/uDfnbi+q0389VXvQFAmXouAIACgkQi+q0389V -XvRp+wv+IPHjBUmVC6YzAxFhRD4GHVUgjckfSbP2jH/acre1mYgm9LJ//7l2GaJy -oEOO85WaHgaKCHCdv9GBc3dDbbt1n9J2IGmBqcdE8e9cRko5qhBoVUvW7p7Ki7ci -nAq5DS3YkpWAocsY/k+LyR0Ky8mW466ARAucTp9kuZmxB2FW53B0bYK57++1qGuo -tr9kJPoGYQB0cUiTSwTaMbOIdl/4CL+a9J7mIrpaDVW5g3PnNy5y1vgDvtuU7Qcn -uEucciBlOn0Ib4mBnky+NX1ThL9WNwLjaivxdioFgc0E4sMwf0CjF3vMUuEvI8qi -PJ5lYndsHI4fdh1SbcgoFNZzTkMZbTr9xcZIGLzLkMX8r+ztLTiFtiLIUSQq0jgm -fqKQghuDN2SVi7WW4KAa7K1285zmV7L27N9mnNWH4ujTqCW73Wdo2XkG/TwM3yEC -5o+YookAV6RHT1X6RPJ8rQaC0BrBgpm/MQH1kvH4vUyF2HRVZ2ZgEYorvKtOwf9D -f7v3IC9J -=/YNz +iQIzBAEBCAAdFiEEzGj1oxBv9EgyLkjtJ/XjjVsKIV8FAmag7tcACgkQJ/XjjVsK +IV/hlRAAjvdBRTWPfjXzTqxQODXLZps1HREXRZAa8C8bbAoagCJ1jfm6d8yVUegH +Bl5FqDAutGfTlEhXtQqBmnbQPv4Ahj156cVYtp3dFrxPF15bP+o9q+Un5+R1zcfX +kH9W26G7IvfrtFJkDClpBlPKYE5HcDrrJBNfvALf4th17bkVHMpr04oJz5IwGV2M +petLMwqFxcqQ+15tzRW42Z6EhWHvNaMveab6SM4JEqBxvqB8K+m4nsw2ER7ycU5b +Isa0bUsxtRICtSX0yzzdzEYrFXZmb9eXZRVfJ4sBpUhw0xtBmHn3c1MZH0qez+Nm +tbc6pcgGv9cUSXauBeD8rrYMzQHcrhihd51i9a3Cen3RDy/dtuNx9jEXnxfkY+n9 +wkwKb4Lask962L+yTHQCfJ+JQxgouADxqzMxhcup1iiHXCd7pSBSoeAvd5Z1AeGX +qBYrLU9mcyIuLrbtADSfnWmXWs2k1hgnP3UXMBhu/GuobQf9kJ2Gwwx5Gp0aB8z9 +4EA+oUXnkM2kJF0MYVMXL+z8VcQpHgyVPujglhNn/a4WdCVTr1jKptNqqnriH9zl +bHMZuiKbAt8RL9rQ3XuFD1sN9k1z/mj8bCHES2WVta+3kCmY9u+eKXNdXJYt+5Xh +bGxwXP5T+Z8Yzc9FVmgPzZzVddCX74Yug0j8BUyE3vPaDq32H6M= +=cZH5 -----END PGP SIGNATURE----- diff --git a/pkg/repos/runtimes/node/node_test.go b/pkg/repos/runtimes/node/node_test.go index 014619c8..ce2fcde1 100644 --- a/pkg/repos/runtimes/node/node_test.go +++ b/pkg/repos/runtimes/node/node_test.go @@ -37,17 +37,3 @@ func TestRuntime(t *testing.T) { } require.NoError(t, err) } - -func TestRuntime21(t *testing.T) { - r := Runtime{ - Version: "21", - } - - s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ()) - require.NoError(t, err) - _, err = os.Stat(filepath.Join(firstPath(s), "node.exe")) - if errors.Is(err, fs.ErrNotExist) { - _, err = os.Stat(filepath.Join(firstPath(s), "node")) - } - require.NoError(t, err) -} diff --git a/pkg/sdkserver/routes.go b/pkg/sdkserver/routes.go index 98957624..e19f5708 100644 --- a/pkg/sdkserver/routes.go +++ b/pkg/sdkserver/routes.go @@ -50,6 +50,8 @@ func (s *server) addRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /run", s.execHandler) mux.HandleFunc("POST /evaluate", s.execHandler) + mux.HandleFunc("POST /load", s.load) + mux.HandleFunc("POST /parse", s.parse) mux.HandleFunc("POST /fmt", s.fmtDocument) @@ -212,6 +214,42 @@ func (s *server) execHandler(w http.ResponseWriter, r *http.Request) { s.execAndStream(ctx, programLoader, logger, w, opts, reqObject.ChatState, reqObject.Input, reqObject.SubTool, def) } +// load will load the file and return the corresponding Program. +func (s *server) load(w http.ResponseWriter, r *http.Request) { + logger := gcontext.GetLogger(r.Context()) + reqObject := new(loadRequest) + if err := json.NewDecoder(r.Body).Decode(reqObject); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("invalid request body: %w", err)) + return + } + + logger.Debugf("parsing file: file=%s, content=%s", reqObject.File, reqObject.Content) + + var ( + prg types.Program + err error + cache = s.client.Cache + ) + + if reqObject.DisableCache { + cache = nil + } + + if reqObject.Content != "" { + prg, err = loader.ProgramFromSource(r.Context(), reqObject.Content, reqObject.SubTool, loader.Options{Cache: cache}) + } else if reqObject.File != "" { + prg, err = loader.Program(r.Context(), reqObject.File, reqObject.SubTool, loader.Options{Cache: cache}) + } else { + prg, err = loader.ProgramFromSource(r.Context(), reqObject.ToolDefs.String(), reqObject.SubTool, loader.Options{Cache: cache}) + } + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to load program: %w", err)) + return + } + + writeResponse(logger, w, map[string]any{"stdout": map[string]any{"program": prg}}) +} + // parse will parse the file and return the corresponding Document. func (s *server) parse(w http.ResponseWriter, r *http.Request) { logger := gcontext.GetLogger(r.Context()) diff --git a/pkg/sdkserver/types.go b/pkg/sdkserver/types.go index ade035b2..b24ca645 100644 --- a/pkg/sdkserver/types.go +++ b/pkg/sdkserver/types.go @@ -82,6 +82,15 @@ func (f *file) String() string { return f.File } +type loadRequest struct { + content `json:",inline"` + + ToolDefs toolDefs `json:"toolDefs,inline"` + DisableCache bool `json:"disableCache"` + SubTool string `json:"subTool,omitempty"` + File string `json:"file"` +} + type parseRequest struct { parser.Options `json:",inline"` content `json:",inline"` From b522585a9f94a37aa17449c64071809d83d9bbc9 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 14 Aug 2024 14:38:27 -0400 Subject: [PATCH 5/6] chore: upgrade Go to 1.23.0 Signed-off-by: Donnie Adams --- .github/workflows/integration.yaml | 2 +- .github/workflows/main.yaml | 4 +--- .github/workflows/release.yaml | 4 +--- .github/workflows/test.yaml | 2 +- .github/workflows/validate-docs.yaml | 2 +- Makefile | 2 +- go.mod | 3 +-- pkg/loader/openapi.go | 2 +- pkg/openapi/getschema.go | 2 +- pkg/repos/runtimes/default.go | 2 +- pkg/repos/runtimes/golang/digests.txt | 20 ++++++++++---------- pkg/repos/runtimes/golang/golang_test.go | 2 +- pkg/repos/runtimes/golang/testdata/go.mod | 2 +- pkg/repos/runtimes/node/node.go | 2 +- pkg/sdkserver/routes.go | 2 +- 15 files changed, 24 insertions(+), 29 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index e0b78be2..22f4a678 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/setup-go@v5 with: cache: false - go-version: "1.22" + go-version: "1.23" - name: Build if: matrix.os == 'ubuntu-22.04' run: make build diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index ae26c52c..7a2114ea 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -26,9 +26,7 @@ jobs: uses: actions/setup-go@v5 with: cache: false - # This can't be upgraded until the issue with sys.daemon on Windows is resolved - # After the issue is resolved, this can be set to 1.22 - go-version: "1.22.4" + go-version: "1.23" - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f710e953..8b5b0eae 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,9 +20,7 @@ jobs: uses: actions/setup-go@v5 with: cache: false - # This can't be upgraded until the issue with sys.daemon on Windows is resolved - # After the issue is resolved, this can be set to 1.22 - go-version: "1.22.4" + go-version: "1.23" - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f3829c35..f4da7e62 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/setup-go@v5 with: cache: false - go-version: "1.22" + go-version: "1.23" - name: Validate if: matrix.os == 'ubuntu-22.04' run: make validate diff --git a/.github/workflows/validate-docs.yaml b/.github/workflows/validate-docs.yaml index 18368355..f7e3a016 100644 --- a/.github/workflows/validate-docs.yaml +++ b/.github/workflows/validate-docs.yaml @@ -17,6 +17,6 @@ jobs: - uses: actions/setup-go@v5 with: cache: false - go-version: "1.22" + go-version: "1.23" - run: make init-docs - run: make validate-docs diff --git a/Makefile b/Makefile index 5b1b6309..4a52694a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ smoke: build smoke: go test -v -tags='smoke' ./pkg/tests/smoke/... -GOLANGCI_LINT_VERSION ?= v1.59.0 +GOLANGCI_LINT_VERSION ?= v1.60.1 lint: if ! command -v golangci-lint &> /dev/null; then \ echo "Could not find golangci-lint, installing version $(GOLANGCI_LINT_VERSION)."; \ diff --git a/go.mod b/go.mod index 4cf0ba73..0d38ca0c 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,6 @@ module github.com/gptscript-ai/gptscript -// This can't be upgraded until the issue with sys.daemon on Windows is resolved -go 1.22.4 +go 1.23.0 require ( github.com/AlecAivazis/survey/v2 v2.3.7 diff --git a/pkg/loader/openapi.go b/pkg/loader/openapi.go index bc469a4e..cf3c3f34 100644 --- a/pkg/loader/openapi.go +++ b/pkg/loader/openapi.go @@ -86,7 +86,7 @@ func getOpenAPITools(t *openapi3.T, defaultHost, source, targetToolName string) pathObj := pathMap[pathString] // Handle path-level server override, if one exists pathServer := defaultServer - if pathObj.Servers != nil && len(pathObj.Servers) > 0 { + if len(pathObj.Servers) > 0 { pathServer, err = parseServer(pathObj.Servers[0]) if err != nil { return nil, err diff --git a/pkg/openapi/getschema.go b/pkg/openapi/getschema.go index 3550afcf..d2966aac 100644 --- a/pkg/openapi/getschema.go +++ b/pkg/openapi/getschema.go @@ -83,7 +83,7 @@ func GetSchema(operationID, defaultHost string, t *openapi3.T) (string, Operatio for path, pathItem := range t.Paths.Map() { // Handle path-level server override, if one exists. pathServer := defaultServer - if pathItem.Servers != nil && len(pathItem.Servers) > 0 { + if len(pathItem.Servers) > 0 { pathServer, err = parseServer(pathItem.Servers[0]) if err != nil { return "", OperationInfo{}, false, err diff --git a/pkg/repos/runtimes/default.go b/pkg/repos/runtimes/default.go index d4eb4db5..a93fb735 100644 --- a/pkg/repos/runtimes/default.go +++ b/pkg/repos/runtimes/default.go @@ -26,7 +26,7 @@ var Runtimes = []repos.Runtime{ Default: true, }, &golang.Runtime{ - Version: "1.22.1", + Version: "1.23.0", }, } diff --git a/pkg/repos/runtimes/golang/digests.txt b/pkg/repos/runtimes/golang/digests.txt index 8a1b82c6..df86facf 100644 --- a/pkg/repos/runtimes/golang/digests.txt +++ b/pkg/repos/runtimes/golang/digests.txt @@ -1,10 +1,10 @@ -3bc971772f4712fec0364f4bc3de06af22a00a12daab10b6f717fdcd13156cc0 go1.22.1.darwin-amd64.tar.gz -943e4f9f038239f9911c44366f52ab9202f6ee13610322a668fe42406fb3deef go1.22.1.darwin-amd64.pkg -f6a9cec6b8a002fcc9c0ee24ec04d67f430a52abc3cfd613836986bcc00d8383 go1.22.1.darwin-arm64.tar.gz -5f10b95e2678618f85ba9d87fbed506b3b87efc9d5a8cafda939055cb97949ba go1.22.1.darwin-arm64.pkg -8484df36d3d40139eaf0fe5e647b006435d826cc12f9ae72973bf7ec265e0ae4 go1.22.1.linux-386.tar.gz -aab8e15785c997ae20f9c88422ee35d962c4562212bb0f879d052a35c8307c7f go1.22.1.linux-amd64.tar.gz -e56685a245b6a0c592fc4a55f0b7803af5b3f827aaa29feab1f40e491acf35b8 go1.22.1.linux-arm64.tar.gz -8cb7a90e48c20daed39a6ac8b8a40760030ba5e93c12274c42191d868687c281 go1.22.1.linux-armv6l.tar.gz -0c5ebb7eb39b7884ec99f92b425d4c03a96a72443562aafbf6e7d15c42a3108a go1.22.1.windows-386.zip -cf9c66a208a106402a527f5b956269ca506cfe535fc388e828d249ea88ed28ba go1.22.1.windows-amd64.zip +ffd070acf59f054e8691b838f274d540572db0bd09654af851e4e76ab88403dc go1.23.0.darwin-amd64.tar.gz +bc91d2573939a01731413fac0884c329606c1c168883692131ce772669caf27b go1.23.0.darwin-amd64.pkg +b770812aef17d7b2ea406588e2b97689e9557aac7e646fe76218b216e2c51406 go1.23.0.darwin-arm64.tar.gz +d73ae741ed449ea842238f76f4b02935277eb867689f84ace0640965b2caf700 go1.23.0.darwin-arm64.pkg +0e8a7340c2632e6fb5088d60f95b52be1f8303143e04cd34e9b2314fafc24edd go1.23.0.linux-386.tar.gz +905a297f19ead44780548933e0ff1a1b86e8327bb459e92f9c0012569f76f5e3 go1.23.0.linux-amd64.tar.gz +62788056693009bcf7020eedc778cdd1781941c6145eab7688bd087bce0f8659 go1.23.0.linux-arm64.tar.gz +0efa1338e644d7f74064fa7f1016b5da7872b2df0070ea3b56e4fef63192e35b go1.23.0.linux-armv6l.tar.gz +09448fedec0cdf98ad12397222e0c8bfc835b1d0894c0015ced653534b8d7427 go1.23.0.windows-386.zip +d4be481ef73079ee0ad46081d278923aa3fd78db1b3cf147172592f73e14c1ac go1.23.0.windows-amd64.zip diff --git a/pkg/repos/runtimes/golang/golang_test.go b/pkg/repos/runtimes/golang/golang_test.go index 56098a51..f3d888fd 100644 --- a/pkg/repos/runtimes/golang/golang_test.go +++ b/pkg/repos/runtimes/golang/golang_test.go @@ -25,7 +25,7 @@ func TestRuntime(t *testing.T) { os.RemoveAll("testdata/bin") }) r := Runtime{ - Version: "1.22.1", + Version: "1.23.0", } s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ()) diff --git a/pkg/repos/runtimes/golang/testdata/go.mod b/pkg/repos/runtimes/golang/testdata/go.mod index 93eac7fe..6725cbfd 100644 --- a/pkg/repos/runtimes/golang/testdata/go.mod +++ b/pkg/repos/runtimes/golang/testdata/go.mod @@ -1,3 +1,3 @@ module example.com -go 1.22.1 +go 1.23.0 diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go index 01a752e6..aa57b059 100644 --- a/pkg/repos/runtimes/node/node.go +++ b/pkg/repos/runtimes/node/node.go @@ -136,7 +136,7 @@ func (r *Runtime) runNPM(ctx context.Context, tool types.Tool, toolSource, binDi if tool.WorkingDir == "" { return nil } - if _, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); errors.Is(fs.ErrNotExist, err) { + if _, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); errors.Is(err, fs.ErrNotExist) { return nil } else if err != nil { return err diff --git a/pkg/sdkserver/routes.go b/pkg/sdkserver/routes.go index e19f5708..4309bc28 100644 --- a/pkg/sdkserver/routes.go +++ b/pkg/sdkserver/routes.go @@ -271,7 +271,7 @@ func (s *server) parse(w http.ResponseWriter, r *http.Request) { } else { content, loadErr := input.FromLocation(reqObject.File, reqObject.DisableCache) if loadErr != nil { - logger.Errorf(loadErr.Error()) + logger.Errorf("failed to load file: %v", loadErr) writeError(logger, w, http.StatusInternalServerError, loadErr) return } From 538323b7e19ca4d5581272591bbd5dfb3469bc15 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 14 Aug 2024 14:33:53 -0700 Subject: [PATCH 6/6] chore: allow wildcard matching in metadata key names --- pkg/parser/parser.go | 14 +++++++++++++- pkg/parser/parser_test.go | 6 ++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index b57cb658..956822dd 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -4,6 +4,8 @@ import ( "bufio" "fmt" "io" + "maps" + "path" "regexp" "slices" "strconv" @@ -16,7 +18,7 @@ import ( var ( sepRegex = regexp.MustCompile(`^\s*---+\s*$`) strictSepRegex = regexp.MustCompile(`^---\n$`) - skipRegex = regexp.MustCompile(`^![-.:\w]+\s*$`) + skipRegex = regexp.MustCompile(`^![-.:*\w]+\s*$`) ) func normalize(key string) string { @@ -390,6 +392,16 @@ func assignMetadata(nodes []Node) (result []Node) { for _, node := range nodes { if node.ToolNode != nil { node.ToolNode.Tool.MetaData = metadata[node.ToolNode.Tool.Name] + for wildcard := range metadata { + if strings.Contains(wildcard, "*") { + if m, err := path.Match(wildcard, node.ToolNode.Tool.Name); m && err == nil { + if node.ToolNode.Tool.MetaData == nil { + node.ToolNode.Tool.MetaData = map[string]string{} + } + maps.Copy(node.ToolNode.Tool.MetaData, metadata[wildcard]) + } + } + } } result = append(result, node) } diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index 3967ebd5..f98b74e2 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -258,6 +258,11 @@ asdf2 --- !metadata:first:requirements.txt asdf + +--- +!metadata:f*r*:other + +foo bar ` tools, err := ParseTools(strings.NewReader(input)) require.NoError(t, err) @@ -266,5 +271,6 @@ asdf autogold.Expect(map[string]string{ "package.json": "foo=base\nf", "requirements.txt": "asdf", + "other": "foo bar", }).Equal(t, tools[0].MetaData) }