diff --git a/examples/gcp-assistant.gpt b/examples/gcp-assistant.gpt new file mode 100644 index 00000000..29b682d0 --- /dev/null +++ b/examples/gcp-assistant.gpt @@ -0,0 +1,28 @@ +Name: GCP Assistant +Description: Agent to help you interact with Google Cloud +Context: learn-gcp, learn-kubectl +Tools: sys.exec, sys.http.html2text?, sys.find, sys.read, sys.write +Chat:true +You are an assistant for Google Cloud Platform (GCP). +Rules +1. Use gcloud CLI to interact with GCP. +2. Assume the user is using Google cloud. + +--- +Name: learn-gcp +Description: A tool to help you learn gcp cli +#!/bin/bash +echo "Current gcloud config:" +gcloud config list || true +--- +Name: learn-kubectl +Description: A tool to help you learn k8s and related commands +#!/bin/bash + +CMDS="kubectl helm" +echo 'The additional CLI commands are available locally, use the `exec` tool to invoke them:' +for i in $CMDS; do + if [ -e "$(command -v $i)" ]; then + echo ' ' $i + fi +done diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index 7970415f..dd358d52 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -52,14 +52,14 @@ func (a *AuthConfig) UnmarshalJSON(data []byte) error { } type CLIConfig struct { - Auths map[string]AuthConfig `json:"auths,omitempty"` - CredentialsStore string `json:"credsStore,omitempty"` - GPTScriptConfigFile string `json:"gptscriptConfig,omitempty"` - GatewayURL string `json:"gatewayURL,omitempty"` - Integrations map[string]string `json:"integrations,omitempty"` + Auths map[string]AuthConfig `json:"auths,omitempty"` + CredentialsStore string `json:"credsStore,omitempty"` + GatewayURL string `json:"gatewayURL,omitempty"` + Integrations map[string]string `json:"integrations,omitempty"` auths map[string]types.AuthConfig authsLock *sync.Mutex + location string } func (c *CLIConfig) Sanitize() *CLIConfig { @@ -93,7 +93,7 @@ func (c *CLIConfig) Save() error { if err != nil { return err } - return os.WriteFile(c.GPTScriptConfigFile, data, 0655) + return os.WriteFile(c.location, data, 0655) } func (c *CLIConfig) GetAuthConfigs() map[string]types.AuthConfig { @@ -113,7 +113,7 @@ func (c *CLIConfig) GetAuthConfigs() map[string]types.AuthConfig { } func (c *CLIConfig) GetFilename() string { - return c.GPTScriptConfigFile + return c.location } func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { @@ -133,11 +133,11 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { return nil, err } result := &CLIConfig{ - authsLock: &sync.Mutex{}, - GPTScriptConfigFile: gptscriptConfigFile, + authsLock: &sync.Mutex{}, + location: gptscriptConfigFile, } if err := json.Unmarshal(data, result); err != nil { - return nil, err + return nil, fmt.Errorf("failed to unmarshal %s: %v", gptscriptConfigFile, err) } if result.CredentialsStore == "" { @@ -158,7 +158,7 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { default: errMsg += " (use 'file')" } - errMsg += fmt.Sprintf("\nPlease edit your config file at %s to fix this.", result.GPTScriptConfigFile) + errMsg += fmt.Sprintf("\nPlease edit your config file at %s to fix this.", result.location) return nil, errors.New(errMsg) } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index f8fd8154..d028d50b 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -204,7 +204,7 @@ func NewContext(ctx context.Context, prg *types.Program, input string) (Context, Input: input, } - agentGroup, err := callCtx.Tool.GetAgents(*prg) + agentGroup, err := callCtx.Tool.GetToolsByType(prg, types.ToolTypeAgent) if err != nil { return callCtx, err } @@ -225,7 +225,7 @@ func (c *Context) SubCallContext(ctx context.Context, input, toolID, callID stri callID = counter.Next() } - agentGroup, err := c.Tool.GetNextAgentGroup(*c.Program, c.AgentGroup, toolID) + agentGroup, err := c.Tool.GetNextAgentGroup(c.Program, c.AgentGroup, toolID) if err != nil { return Context{}, err } @@ -258,6 +258,29 @@ func (c *Context) WrappedContext(e *Engine) context.Context { return context.WithValue(c.Ctx, engineContext{}, &cp) } +func populateMessageParams(ctx Context, completion *types.CompletionRequest, tool types.Tool) error { + completion.Model = tool.Parameters.ModelName + completion.MaxTokens = tool.Parameters.MaxTokens + completion.JSONResponse = tool.Parameters.JSONResponse + completion.Cache = tool.Parameters.Cache + completion.Chat = tool.Parameters.Chat + completion.Temperature = tool.Parameters.Temperature + completion.InternalSystemPrompt = tool.Parameters.InternalPrompt + + if tool.Chat && completion.InternalSystemPrompt == nil { + completion.InternalSystemPrompt = new(bool) + } + + var err error + completion.Tools, err = tool.GetChatCompletionTools(*ctx.Program, ctx.AgentGroup...) + if err != nil { + return err + } + + completion.Messages = addUpdateSystem(ctx, tool, completion.Messages) + return nil +} + func (e *Engine) Start(ctx Context, input string) (ret *Return, _ error) { tool := ctx.Tool @@ -290,28 +313,11 @@ func (e *Engine) Start(ctx Context, input string) (ret *Return, _ error) { return nil, fmt.Errorf("credential tools cannot make calls to the LLM") } - completion := types.CompletionRequest{ - Model: tool.Parameters.ModelName, - MaxTokens: tool.Parameters.MaxTokens, - JSONResponse: tool.Parameters.JSONResponse, - Cache: tool.Parameters.Cache, - Chat: tool.Parameters.Chat, - Temperature: tool.Parameters.Temperature, - InternalSystemPrompt: tool.Parameters.InternalPrompt, - } - - if tool.Chat && completion.InternalSystemPrompt == nil { - completion.InternalSystemPrompt = new(bool) - } - - var err error - completion.Tools, err = tool.GetCompletionTools(*ctx.Program, ctx.AgentGroup...) - if err != nil { + var completion types.CompletionRequest + if err := populateMessageParams(ctx, &completion, tool); err != nil { return nil, err } - completion.Messages = addUpdateSystem(ctx, tool, completion.Messages) - if tool.Chat && input == "{}" { input = "" } @@ -497,6 +503,9 @@ func (e *Engine) Continue(ctx Context, state *State, results ...CallResult) (*Re return nil, fmt.Errorf("invalid continue call, no completion needed") } - state.Completion.Messages = addUpdateSystem(ctx, ctx.Tool, state.Completion.Messages) + if err := populateMessageParams(ctx, &state.Completion, ctx.Tool); err != nil { + return nil, err + } + return e.complete(ctx.Ctx, state) } diff --git a/pkg/openai/client.go b/pkg/openai/client.go index 42a1a39e..61a7ec77 100644 --- a/pkg/openai/client.go +++ b/pkg/openai/client.go @@ -2,6 +2,7 @@ package openai import ( "context" + "errors" "io" "log/slog" "os" @@ -24,6 +25,7 @@ import ( const ( DefaultModel = openai.GPT4o BuiltinCredName = "sys.openai" + TooLongMessage = "Error: tool call output is too long" ) var ( @@ -317,6 +319,14 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques } if messageRequest.Chat { + // Check the last message. If it is from a tool call, and if it takes up more than 80% of the budget on its own, reject it. + lastMessage := msgs[len(msgs)-1] + if lastMessage.Role == string(types.CompletionMessageRoleTypeTool) && countMessage(lastMessage) > int(float64(getBudget(messageRequest.MaxTokens))*0.8) { + // We need to update it in the msgs slice for right now and in the messageRequest for future calls. + msgs[len(msgs)-1].Content = TooLongMessage + messageRequest.Messages[len(messageRequest.Messages)-1].Content = types.Text(TooLongMessage) + } + msgs = dropMessagesOverCount(messageRequest.MaxTokens, msgs) } @@ -383,6 +393,16 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques return nil, err } else if !ok { response, err = c.call(ctx, request, id, status) + + // If we got back a context length exceeded error, keep retrying and shrinking the message history until we pass. + var apiError *openai.APIError + if errors.As(err, &apiError) && apiError.Code == "context_length_exceeded" && messageRequest.Chat { + // Decrease maxTokens by 10% to make garbage collection more aggressive. + // The retry loop will further decrease maxTokens if needed. + maxTokens := decreaseTenPercent(messageRequest.MaxTokens) + response, err = c.contextLimitRetryLoop(ctx, request, id, maxTokens, status) + } + if err != nil { return nil, err } @@ -421,6 +441,32 @@ func (c *Client) Call(ctx context.Context, messageRequest types.CompletionReques return &result, nil } +func (c *Client) contextLimitRetryLoop(ctx context.Context, request openai.ChatCompletionRequest, id string, maxTokens int, status chan<- types.CompletionStatus) ([]openai.ChatCompletionStreamResponse, error) { + var ( + response []openai.ChatCompletionStreamResponse + err error + ) + + for range 10 { // maximum 10 tries + // Try to drop older messages again, with a decreased max tokens. + request.Messages = dropMessagesOverCount(maxTokens, request.Messages) + response, err = c.call(ctx, request, id, status) + if err == nil { + return response, nil + } + + var apiError *openai.APIError + if errors.As(err, &apiError) && apiError.Code == "context_length_exceeded" { + // Decrease maxTokens and try again + maxTokens = decreaseTenPercent(maxTokens) + continue + } + return nil, err + } + + return nil, err +} + func appendMessage(msg types.CompletionMessage, response openai.ChatCompletionStreamResponse) types.CompletionMessage { msg.Usage.CompletionTokens = types.FirstSet(msg.Usage.CompletionTokens, response.Usage.CompletionTokens) msg.Usage.PromptTokens = types.FirstSet(msg.Usage.PromptTokens, response.Usage.PromptTokens) diff --git a/pkg/openai/count.go b/pkg/openai/count.go index 47c5c9bd..ffd902e5 100644 --- a/pkg/openai/count.go +++ b/pkg/openai/count.go @@ -1,20 +1,30 @@ package openai -import openai "github.com/gptscript-ai/chat-completion-client" +import ( + openai "github.com/gptscript-ai/chat-completion-client" +) + +const DefaultMaxTokens = 128_000 + +func decreaseTenPercent(maxTokens int) int { + maxTokens = getBudget(maxTokens) + return int(float64(maxTokens) * 0.9) +} + +func getBudget(maxTokens int) int { + if maxTokens == 0 { + return DefaultMaxTokens + } + return maxTokens +} func dropMessagesOverCount(maxTokens int, msgs []openai.ChatCompletionMessage) (result []openai.ChatCompletionMessage) { var ( lastSystem int withinBudget int - budget = maxTokens + budget = getBudget(maxTokens) ) - if maxTokens == 0 { - budget = 300_000 - } else { - budget *= 3 - } - for i, msg := range msgs { if msg.Role == openai.ChatMessageRoleSystem { budget -= countMessage(msg) @@ -33,6 +43,14 @@ func dropMessagesOverCount(maxTokens int, msgs []openai.ChatCompletionMessage) ( } } + // OpenAI gets upset if there is a tool message without a tool call preceding it. + // Check the oldest message within budget, and if it is a tool message, just drop it. + // We do this in a loop because it is possible for multiple tool messages to be in a row, + // due to parallel tool calls. + for withinBudget < len(msgs) && msgs[withinBudget].Role == openai.ChatMessageRoleTool { + withinBudget++ + } + if withinBudget == len(msgs)-1 { // We are going to drop all non system messages, which seems useless, so just return them // all and let it fail diff --git a/pkg/openapi/run.go b/pkg/openapi/run.go index 6c7e4ca7..fb3b746c 100644 --- a/pkg/openapi/run.go +++ b/pkg/openapi/run.go @@ -107,6 +107,7 @@ func Run(operationID, defaultHost, args string, t *openapi3.T, envs []string) (s return "", false, fmt.Errorf("failed to encode JSON: %w", err) } req.Header.Set("Content-Type", "application/json") + req.ContentLength = int64(body.Len()) case "text/plain": reqBody := "" diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 52e8fe0b..2601f521 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -244,7 +244,7 @@ func getChecksum(ctx context.Context, rel *release, artifactName string) string return "" } -func (r *Runtime) Binary(ctx context.Context, tool types.Tool, _, toolSource string, env []string) (bool, []string, error) { +func (r *Runtime) Binary(ctx context.Context, tool types.Tool, _, toolSource string, _ []string) (bool, []string, error) { if !tool.Source.IsGit() { return false, nil, nil } @@ -264,7 +264,7 @@ func (r *Runtime) Binary(ctx context.Context, tool types.Tool, _, toolSource str return false, nil, nil } - return true, env, nil + return true, nil, nil } func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource string, env []string) ([]string, error) { diff --git a/pkg/repos/runtimes/node/node.go b/pkg/repos/runtimes/node/node.go index aa57b059..4d73c13b 100644 --- a/pkg/repos/runtimes/node/node.go +++ b/pkg/repos/runtimes/node/node.go @@ -26,6 +26,7 @@ var releasesData []byte const ( downloadURL = "https://nodejs.org/dist/%s/" packageJSON = "package.json" + nodeModules = "node_modules" ) type Runtime struct { @@ -64,8 +65,15 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool { func (r *Runtime) GetHash(tool types.Tool) (string, error) { if !tool.Source.IsGit() && tool.WorkingDir != "" { + var prefix string + // This hashes if the node_modules directory was deleted + if s, err := os.Stat(filepath.Join(tool.WorkingDir, nodeModules)); err == nil { + prefix = hash.Digest(tool.WorkingDir + s.ModTime().String())[:7] + } else if s, err := os.Stat(tool.WorkingDir); err == nil { + prefix = hash.Digest(tool.WorkingDir + s.ModTime().String())[:7] + } if s, err := os.Stat(filepath.Join(tool.WorkingDir, packageJSON)); err == nil { - return hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil + return prefix + hash.Digest(tool.WorkingDir + s.ModTime().String())[:7], nil } } return "", nil diff --git a/pkg/runner/input.go b/pkg/runner/input.go index 7d77330e..a211ec9d 100644 --- a/pkg/runner/input.go +++ b/pkg/runner/input.go @@ -5,10 +5,11 @@ import ( "fmt" "github.com/gptscript-ai/gptscript/pkg/engine" + "github.com/gptscript-ai/gptscript/pkg/types" ) func (r *Runner) handleInput(callCtx engine.Context, monitor Monitor, env []string, input string) (string, error) { - inputToolRefs, err := callCtx.Tool.GetInputFilterTools(*callCtx.Program) + inputToolRefs, err := callCtx.Tool.GetToolsByType(callCtx.Program, types.ToolTypeInput) if err != nil { return "", err } diff --git a/pkg/runner/output.go b/pkg/runner/output.go index d4cb4b9b..e5fe849d 100644 --- a/pkg/runner/output.go +++ b/pkg/runner/output.go @@ -6,10 +6,11 @@ import ( "fmt" "github.com/gptscript-ai/gptscript/pkg/engine" + "github.com/gptscript-ai/gptscript/pkg/types" ) func (r *Runner) handleOutput(callCtx engine.Context, monitor Monitor, env []string, state *State, retErr error) (*State, error) { - outputToolRefs, err := callCtx.Tool.GetOutputFilterTools(*callCtx.Program) + outputToolRefs, err := callCtx.Tool.GetToolsByType(callCtx.Program, types.ToolTypeOutput) if err != nil { return nil, err } diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 93e40670..c843b6b5 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -172,11 +172,7 @@ func (r *Runner) Chat(ctx context.Context, prevState ChatState, prg types.Progra return resp, err } - if state == nil || state.StartContinuation { - if state != nil { - state = state.WithResumeInput(&input) - input = state.InputContextContinuationInput - } + if state == nil { state, err = r.start(callCtx, state, monitor, env, input) if err != nil { return resp, err @@ -186,11 +182,9 @@ func (r *Runner) Chat(ctx context.Context, prevState ChatState, prg types.Progra state.ResumeInput = &input } - if !state.StartContinuation { - state, err = r.resume(callCtx, monitor, env, state) - if err != nil { - return resp, err - } + state, err = r.resume(callCtx, monitor, env, state) + if err != nil { + return resp, err } if state.Result != nil { @@ -260,6 +254,10 @@ func getToolRefInput(prg *types.Program, ref types.ToolReference, input string) targetArgs := prg.ToolSet[ref.ToolID].Arguments targetKeys := map[string]string{} + if ref.Arg == "*" { + return input, nil + } + if targetArgs == nil { return "", nil } @@ -331,24 +329,10 @@ func getToolRefInput(prg *types.Program, ref types.ToolReference, input string) return string(output), err } -func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (result []engine.InputContext, _ *State, _ error) { - toolRefs, err := callCtx.Tool.GetContextTools(*callCtx.Program) +func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monitor, env []string, input string) (result []engine.InputContext, _ error) { + toolRefs, err := callCtx.Tool.GetToolsByType(callCtx.Program, types.ToolTypeContext) if err != nil { - return nil, nil, err - } - - var newState *State - if state != nil { - cp := *state - newState = &cp - if newState.InputContextContinuation != nil { - newState.InputContexts = nil - newState.InputContextContinuation = nil - newState.InputContextContinuationInput = "" - newState.ResumeInput = state.InputContextContinuationResumeInput - - input = state.InputContextContinuationInput - } + return nil, err } for i, toolRef := range toolRefs { @@ -359,29 +343,16 @@ func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monito contextInput, err := getToolRefInput(callCtx.Program, toolRef, input) if err != nil { - return nil, nil, err + return nil, err } var content *State - if state != nil && state.InputContextContinuation != nil { - content, err = r.subCallResume(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, "", state.InputContextContinuation.WithResumeInput(state.ResumeInput), engine.ContextToolCategory) - } else { - content, err = r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) - } + content, err = r.subCall(callCtx.Ctx, callCtx, monitor, env, toolRef.ToolID, contextInput, "", engine.ContextToolCategory) if err != nil { - return nil, nil, err + return nil, err } if content.Continuation != nil { - if newState == nil { - newState = &State{} - } - newState.InputContexts = result - newState.InputContextContinuation = content - newState.InputContextContinuationInput = input - if state != nil { - newState.InputContextContinuationResumeInput = state.ResumeInput - } - return nil, newState, nil + return nil, fmt.Errorf("invalid state: context tool [%s] can not result in a continuation", toolRef.ToolID) } result = append(result, engine.InputContext{ ToolID: toolRef.ToolID, @@ -389,7 +360,7 @@ func (r *Runner) getContext(callCtx engine.Context, state *State, monitor Monito }) } - return result, newState, nil + return result, nil } func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, input string) (*State, error) { @@ -397,9 +368,6 @@ func (r *Runner) call(callCtx engine.Context, monitor Monitor, env []string, inp if err != nil { return nil, err } - if result.StartContinuation { - return result, nil - } return r.resume(callCtx, monitor, env, result) } @@ -419,7 +387,7 @@ func (r *Runner) start(callCtx engine.Context, state *State, monitor Monitor, en return nil, err } - credTools, err := callCtx.Tool.GetCredentialTools(*callCtx.Program, callCtx.AgentGroup) + credTools, err := callCtx.Tool.GetToolsByType(callCtx.Program, types.ToolTypeCredential) if err != nil { return nil, err } @@ -431,15 +399,10 @@ func (r *Runner) start(callCtx engine.Context, state *State, monitor Monitor, en } } - var newState *State - callCtx.InputContext, newState, err = r.getContext(callCtx, state, monitor, env, input) + callCtx.InputContext, err = r.getContext(callCtx, state, monitor, env, input) if err != nil { return nil, err } - if newState != nil && newState.InputContextContinuation != nil { - newState.StartContinuation = true - return newState, nil - } e := engine.Engine{ Model: r.c, @@ -489,11 +452,7 @@ type State struct { SubCalls []SubCallResult `json:"subCalls,omitempty"` SubCallID string `json:"subCallID,omitempty"` - InputContexts []engine.InputContext `json:"inputContexts,omitempty"` - InputContextContinuation *State `json:"inputContextContinuation,omitempty"` - InputContextContinuationInput string `json:"inputContextContinuationInput,omitempty"` - InputContextContinuationResumeInput *string `json:"inputContextContinuationResumeInput,omitempty"` - StartContinuation bool `json:"startContinuation,omitempty"` + InputContexts []engine.InputContext `json:"inputContexts,omitempty"` } func (s State) WithResumeInput(input *string) *State { @@ -506,10 +465,6 @@ func (s State) ContinuationContentToolID() (string, error) { return s.ContinuationToolID, nil } - if s.InputContextContinuation != nil { - return s.InputContextContinuation.ContinuationContentToolID() - } - for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContentToolID() @@ -523,10 +478,6 @@ func (s State) ContinuationContent() (string, error) { return *s.Continuation.Result, nil } - if s.InputContextContinuation != nil { - return s.InputContextContinuation.ContinuationContent() - } - for _, subCall := range s.SubCalls { if s.SubCallID == subCall.CallID { return subCall.State.ContinuationContent() @@ -545,10 +496,6 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s retState, retErr = r.handleOutput(callCtx, monitor, env, retState, retErr) }() - if state.StartContinuation { - return nil, fmt.Errorf("invalid state, resume should not have StartContinuation set to true") - } - if state.Continuation == nil { return nil, errors.New("invalid state, resume should have Continuation data") } @@ -556,7 +503,7 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s progress, progressClose := streamProgress(&callCtx, monitor) defer progressClose() - credTools, err := callCtx.Tool.GetCredentialTools(*callCtx.Program, callCtx.AgentGroup) + credTools, err := callCtx.Tool.GetToolsByType(callCtx.Program, types.ToolTypeCredential) if err != nil { return nil, err } @@ -653,8 +600,12 @@ func (r *Runner) resume(callCtx engine.Context, monitor Monitor, env []string, s contentInput = state.Continuation.State.Input } - callCtx.InputContext, state, err = r.getContext(callCtx, state, monitor, env, contentInput) - if err != nil || state.InputContextContinuation != nil { + if state.ResumeInput != nil { + contentInput = *state.ResumeInput + } + + callCtx.InputContext, err = r.getContext(callCtx, state, monitor, env, contentInput) + if err != nil { return state, err } @@ -764,10 +715,6 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, callCtx.LastReturn = state.Continuation } - if state.InputContextContinuation != nil { - return state, nil, nil - } - if state.SubCallID != "" { if state.ResumeInput == nil { return nil, nil, fmt.Errorf("invalid state, input must be set for sub call continuation on callID [%s]", state.SubCallID) diff --git a/pkg/sdkserver/server.go b/pkg/sdkserver/server.go index 4ef28267..f72e7ae9 100644 --- a/pkg/sdkserver/server.go +++ b/pkg/sdkserver/server.go @@ -32,6 +32,8 @@ type Options struct { // Run will start the server and block until the server is shut down. func Run(ctx context.Context, opts Options) error { + opts = complete(opts) + listener, err := newListener(opts) if err != nil { return err diff --git a/pkg/tests/runner2_test.go b/pkg/tests/runner2_test.go new file mode 100644 index 00000000..27d4c226 --- /dev/null +++ b/pkg/tests/runner2_test.go @@ -0,0 +1,57 @@ +package tests + +import ( + "context" + "testing" + + "github.com/gptscript-ai/gptscript/pkg/loader" + "github.com/gptscript-ai/gptscript/pkg/tests/tester" + "github.com/stretchr/testify/require" +) + +func TestContextWithAsterick(t *testing.T) { + r := tester.NewRunner(t) + prg, err := loader.ProgramFromSource(context.Background(), ` +chat: true +context: foo with * + +Say hi + +--- +name: foo + +#!/bin/bash + +echo This is the input: ${GPTSCRIPT_INPUT} +`, "") + require.NoError(t, err) + + resp, err := r.Chat(context.Background(), nil, prg, nil, "input 1") + r.AssertStep(t, resp, err) + + resp, err = r.Chat(context.Background(), resp.State, prg, nil, "input 2") + r.AssertStep(t, resp, err) +} + +func TestContextShareBug(t *testing.T) { + r := tester.NewRunner(t) + prg, err := loader.ProgramFromSource(context.Background(), ` +chat: true +tools: sharecontext + +Say hi + +--- +name: sharecontext +share context: realcontext +--- +name: realcontext + +#!sys.echo + +Yo dawg`, "") + require.NoError(t, err) + + resp, err := r.Chat(context.Background(), nil, prg, nil, "input 1") + r.AssertStep(t, resp, err) +} diff --git a/pkg/tests/runner_test.go b/pkg/tests/runner_test.go index 141e6aff..18871ed6 100644 --- a/pkg/tests/runner_test.go +++ b/pkg/tests/runner_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/gptscript-ai/gptscript/pkg/engine" + "github.com/gptscript-ai/gptscript/pkg/loader" "github.com/gptscript-ai/gptscript/pkg/tests/tester" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/hexops/autogold/v2" @@ -212,82 +213,8 @@ func TestContextSubChat(t *testing.T) { prg, err := r.Load("") require.NoError(t, err) - resp, err := r.Chat(context.Background(), nil, prg, os.Environ(), "User 1") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 1 - from chatbot1").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step1")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_2", - Function: types.CompletionFunctionCall{ - Name: types.ToolNormalizer("sys.chat.finish"), - Arguments: "Response from context chatbot", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 2 - from context tool", - }, tester.Result{ - Text: "Assistant Response 3 - from main chat tool", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 2") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step2")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_3", - Function: types.CompletionFunctionCall{ - Name: "chatbot", - Arguments: "Input to chatbot1 on resume", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 4 - from chatbot1", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 3") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 3 - from main chat tool").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step3")) - - r.RespondWith(tester.Result{ - Content: []types.ContentPart{ - { - ToolCall: &types.CompletionToolCall{ - ID: "call_4", - Function: types.CompletionFunctionCall{ - Name: types.ToolNormalizer("sys.chat.finish"), - Arguments: "Response from context chatbot after resume", - }, - }, - }, - }, - }, tester.Result{ - Text: "Assistant Response 5 - from context tool resume", - }, tester.Result{ - Text: "Assistant Response 6 - from main chat tool resume", - }) - resp, err = r.Chat(context.Background(), resp.State, prg, os.Environ(), "User 4") - require.NoError(t, err) - r.AssertResponded(t) - assert.False(t, resp.Done) - autogold.Expect("Assistant Response 6 - from main chat tool resume").Equal(t, resp.Content) - autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step4")) + _, err = r.Chat(context.Background(), nil, prg, os.Environ(), "User 1") + autogold.Expect("invalid state: context tool [testdata/TestContextSubChat/test.gpt:subtool] can not result in a continuation").Equal(t, err.Error()) } func TestSubChat(t *testing.T) { @@ -1041,3 +968,33 @@ func TestRuntimesLocalDev(t *testing.T) { _ = os.RemoveAll("testdata/TestRuntimesLocalDev/node_modules") _ = os.RemoveAll("testdata/TestRuntimesLocalDev/package-lock.json") } + +func TestToolsChange(t *testing.T) { + r := tester.NewRunner(t) + prg, err := loader.ProgramFromSource(context.Background(), ` +chat: true +tools: sys.ls, sys.read, sys.write +`, "") + require.NoError(t, err) + + resp, err := r.Chat(context.Background(), nil, prg, nil, "input 1") + require.NoError(t, err) + r.AssertResponded(t) + assert.False(t, resp.Done) + autogold.Expect("TEST RESULT CALL: 1").Equal(t, resp.Content) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step1")) + + prg, err = loader.ProgramFromSource(context.Background(), ` +chat: true +temperature: 0.6 +tools: sys.ls, sys.write +`, "") + require.NoError(t, err) + + resp, err = r.Chat(context.Background(), resp.State, prg, nil, "input 2") + require.NoError(t, err) + r.AssertResponded(t) + assert.False(t, resp.Done) + autogold.Expect("TEST RESULT CALL: 2").Equal(t, resp.Content) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+"/step2")) +} diff --git a/pkg/tests/testdata/TestAgentOnly/call2.golden b/pkg/tests/testdata/TestAgentOnly/call2.golden index 82f95523..7f6b155b 100644 --- a/pkg/tests/testdata/TestAgentOnly/call2.golden +++ b/pkg/tests/testdata/TestAgentOnly/call2.golden @@ -4,8 +4,8 @@ "tools": [ { "function": { - "toolID": "testdata/TestAgentOnly/test.gpt:agent1", - "name": "agent1", + "toolID": "testdata/TestAgentOnly/test.gpt:agent3", + "name": "agent3", "parameters": { "properties": { "defaultPromptParameter": { @@ -19,8 +19,8 @@ }, { "function": { - "toolID": "testdata/TestAgentOnly/test.gpt:agent3", - "name": "agent3", + "toolID": "testdata/TestAgentOnly/test.gpt:agent1", + "name": "agent1", "parameters": { "properties": { "defaultPromptParameter": { diff --git a/pkg/tests/testdata/TestAgentOnly/step1.golden b/pkg/tests/testdata/TestAgentOnly/step1.golden index 662dbf04..2cda2025 100644 --- a/pkg/tests/testdata/TestAgentOnly/step1.golden +++ b/pkg/tests/testdata/TestAgentOnly/step1.golden @@ -96,8 +96,8 @@ "tools": [ { "function": { - "toolID": "testdata/TestAgentOnly/test.gpt:agent1", - "name": "agent1", + "toolID": "testdata/TestAgentOnly/test.gpt:agent3", + "name": "agent3", "parameters": { "properties": { "defaultPromptParameter": { @@ -111,8 +111,8 @@ }, { "function": { - "toolID": "testdata/TestAgentOnly/test.gpt:agent3", - "name": "agent3", + "toolID": "testdata/TestAgentOnly/test.gpt:agent1", + "name": "agent1", "parameters": { "properties": { "defaultPromptParameter": { diff --git a/pkg/tests/testdata/TestAgents/call3-resp.golden b/pkg/tests/testdata/TestAgents/call3-resp.golden index e2a65c99..7568fc69 100644 --- a/pkg/tests/testdata/TestAgents/call3-resp.golden +++ b/pkg/tests/testdata/TestAgents/call3-resp.golden @@ -3,7 +3,7 @@ "content": [ { "toolCall": { - "index": 1, + "index": 0, "id": "call_3", "function": { "name": "agent3" diff --git a/pkg/tests/testdata/TestAgents/call3.golden b/pkg/tests/testdata/TestAgents/call3.golden index f9b45a1b..5b1638e0 100644 --- a/pkg/tests/testdata/TestAgents/call3.golden +++ b/pkg/tests/testdata/TestAgents/call3.golden @@ -4,8 +4,8 @@ "tools": [ { "function": { - "toolID": "testdata/TestAgents/test.gpt:agent1", - "name": "agent1", + "toolID": "testdata/TestAgents/test.gpt:agent3", + "name": "agent3", "parameters": { "properties": { "defaultPromptParameter": { @@ -19,8 +19,8 @@ }, { "function": { - "toolID": "testdata/TestAgents/test.gpt:agent3", - "name": "agent3", + "toolID": "testdata/TestAgents/test.gpt:agent1", + "name": "agent1", "parameters": { "properties": { "defaultPromptParameter": { diff --git a/pkg/tests/testdata/TestAgents/step1.golden b/pkg/tests/testdata/TestAgents/step1.golden index 3047e695..72e01114 100644 --- a/pkg/tests/testdata/TestAgents/step1.golden +++ b/pkg/tests/testdata/TestAgents/step1.golden @@ -178,8 +178,8 @@ "tools": [ { "function": { - "toolID": "testdata/TestAgents/test.gpt:agent1", - "name": "agent1", + "toolID": "testdata/TestAgents/test.gpt:agent3", + "name": "agent3", "parameters": { "properties": { "defaultPromptParameter": { @@ -193,8 +193,8 @@ }, { "function": { - "toolID": "testdata/TestAgents/test.gpt:agent3", - "name": "agent3", + "toolID": "testdata/TestAgents/test.gpt:agent1", + "name": "agent1", "parameters": { "properties": { "defaultPromptParameter": { @@ -222,7 +222,7 @@ "content": [ { "toolCall": { - "index": 1, + "index": 0, "id": "call_3", "function": { "name": "agent3" @@ -237,7 +237,7 @@ }, "pending": { "call_3": { - "index": 1, + "index": 0, "id": "call_3", "function": { "name": "agent3" diff --git a/pkg/tests/testdata/TestContextSubChat/call7-resp.golden b/pkg/tests/testdata/TestContextShareBug/call1-resp.golden similarity index 58% rename from pkg/tests/testdata/TestContextSubChat/call7-resp.golden rename to pkg/tests/testdata/TestContextShareBug/call1-resp.golden index 3e0c5f3c..2861a036 100644 --- a/pkg/tests/testdata/TestContextSubChat/call7-resp.golden +++ b/pkg/tests/testdata/TestContextShareBug/call1-resp.golden @@ -2,7 +2,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 4 - from chatbot1" + "text": "TEST RESULT CALL: 1" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/call5.golden b/pkg/tests/testdata/TestContextShareBug/call1.golden similarity index 75% rename from pkg/tests/testdata/TestContextSubChat/call5.golden rename to pkg/tests/testdata/TestContextShareBug/call1.golden index 2b8cf41e..0a46f0ca 100644 --- a/pkg/tests/testdata/TestContextSubChat/call5.golden +++ b/pkg/tests/testdata/TestContextShareBug/call1.golden @@ -6,7 +6,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 2 - from context tool\nHello" + "text": "\nYo dawg\nSay hi" } ], "usage": {} @@ -15,7 +15,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/step2.golden b/pkg/tests/testdata/TestContextShareBug/step1.golden similarity index 62% rename from pkg/tests/testdata/TestContextSubChat/step2.golden rename to pkg/tests/testdata/TestContextShareBug/step1.golden index dfcb2b96..cb17be6d 100644 --- a/pkg/tests/testdata/TestContextSubChat/step2.golden +++ b/pkg/tests/testdata/TestContextShareBug/step1.golden @@ -1,11 +1,11 @@ `{ "done": false, - "content": "Assistant Response 3 - from main chat tool", - "toolID": "testdata/TestContextSubChat/test.gpt:", + "content": "TEST RESULT CALL: 1", + "toolID": "inline:", "state": { "continuation": { "state": { - "input": "User 1", + "input": "input 1", "completion": { "model": "gpt-4o", "internalSystemPrompt": false, @@ -14,7 +14,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 2 - from context tool\nHello" + "text": "\nYo dawg\nSay hi" } ], "usage": {} @@ -23,7 +23,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} @@ -32,7 +32,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 3 - from main chat tool" + "text": "TEST RESULT CALL: 1" } ], "usage": {} @@ -41,8 +41,8 @@ "chat": true } }, - "result": "Assistant Response 3 - from main chat tool" + "result": "TEST RESULT CALL: 1" }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:" + "continuationToolID": "inline:" } }` diff --git a/pkg/tests/testdata/TestContextSubChat/call10-resp.golden b/pkg/tests/testdata/TestContextSubChat/call10-resp.golden deleted file mode 100644 index 144ca8d9..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call10-resp.golden +++ /dev/null @@ -1,9 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "text": "Assistant Response 6 - from main chat tool resume" - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call3-resp.golden b/pkg/tests/testdata/TestContextSubChat/call3-resp.golden deleted file mode 100644 index b116d066..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call3-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_2", - "function": { - "name": "chatFinish", - "arguments": "Response from context chatbot" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call3.golden b/pkg/tests/testdata/TestContextSubChat/call3.golden deleted file mode 100644 index 55ad402f..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call3.golden +++ /dev/null @@ -1,61 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 1 - from chatbot1" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call4.golden b/pkg/tests/testdata/TestContextSubChat/call4.golden deleted file mode 100644 index e1fb91ea..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call4.golden +++ /dev/null @@ -1,64 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - ], - "usage": {} - }, - { - "role": "tool", - "content": [ - { - "text": "Response from context chatbot" - } - ], - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - }, - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call6-resp.golden b/pkg/tests/testdata/TestContextSubChat/call6-resp.golden deleted file mode 100644 index 6807fce9..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call6-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call6.golden b/pkg/tests/testdata/TestContextSubChat/call6.golden deleted file mode 100644 index 225401db..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call6.golden +++ /dev/null @@ -1,31 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call7.golden b/pkg/tests/testdata/TestContextSubChat/call7.golden deleted file mode 100644 index b0ef4e39..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call7.golden +++ /dev/null @@ -1,43 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call8-resp.golden b/pkg/tests/testdata/TestContextSubChat/call8-resp.golden deleted file mode 100644 index 2e608b31..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call8-resp.golden +++ /dev/null @@ -1,16 +0,0 @@ -`{ - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_4", - "function": { - "name": "chatFinish", - "arguments": "Response from context chatbot after resume" - } - } - } - ], - "usage": {} -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call8.golden b/pkg/tests/testdata/TestContextSubChat/call8.golden deleted file mode 100644 index 3d0db61b..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call8.golden +++ /dev/null @@ -1,61 +0,0 @@ -`{ - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 4 - from chatbot1" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 4" - } - ], - "usage": {} - } - ], - "chat": true -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call9.golden b/pkg/tests/testdata/TestContextSubChat/call9.golden deleted file mode 100644 index 33768f26..00000000 --- a/pkg/tests/testdata/TestContextSubChat/call9.golden +++ /dev/null @@ -1,64 +0,0 @@ -`{ - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} - }, - { - "role": "tool", - "content": [ - { - "text": "Response from context chatbot after resume" - } - ], - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - }, - "usage": {} - } - ] -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step1.golden b/pkg/tests/testdata/TestContextSubChat/step1.golden deleted file mode 100644 index 2ffb138e..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step1.golden +++ /dev/null @@ -1,146 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 1 - from chatbot1", - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "state": { - "inputContextContinuation": { - "continuation": { - "state": { - "completion": { - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - ], - "usage": {} - } - ] - }, - "pending": { - "call_1": { - "index": 0, - "id": "call_1", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1" - } - } - } - }, - "calls": { - "call_1": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "input": "Input to chatbot1" - } - } - }, - "subCalls": [ - { - "toolId": "testdata/TestContextSubChat/test.gpt:chatbot", - "callId": "call_1", - "state": { - "continuation": { - "state": { - "input": "Input to chatbot1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 1 - from chatbot1" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 1 - from chatbot1" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:chatbot" - } - } - ], - "subCallID": "call_1" - }, - "inputContextContinuationInput": "User 1", - "startContinuation": true - } -}` diff --git a/pkg/tests/testdata/TestContextSubChat/step3.golden b/pkg/tests/testdata/TestContextSubChat/step3.golden deleted file mode 100644 index 0ccb188b..00000000 --- a/pkg/tests/testdata/TestContextSubChat/step3.golden +++ /dev/null @@ -1,188 +0,0 @@ -`{ - "done": false, - "content": "Assistant Response 3 - from main chat tool", - "toolID": "testdata/TestContextSubChat/test.gpt:", - "state": { - "continuation": { - "state": { - "input": "User 1", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Assistant Response 2 - from context tool\nHello" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "User 1" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 3 - from main chat tool" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 3 - from main chat tool" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:", - "resumeInput": "User 3", - "inputContextContinuation": { - "continuation": { - "state": { - "completion": { - "model": "gpt-4o", - "tools": [ - { - "function": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "name": "chatbot", - "parameters": { - "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the assistant. This may be an instruction or question.", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "Call chatbot" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "toolCall": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - ], - "usage": {} - } - ] - }, - "pending": { - "call_3": { - "index": 0, - "id": "call_3", - "function": { - "name": "chatbot", - "arguments": "Input to chatbot1 on resume" - } - } - } - }, - "calls": { - "call_3": { - "toolID": "testdata/TestContextSubChat/test.gpt:chatbot", - "input": "Input to chatbot1 on resume" - } - } - }, - "subCalls": [ - { - "toolId": "testdata/TestContextSubChat/test.gpt:chatbot", - "callId": "call_3", - "state": { - "continuation": { - "state": { - "input": "Input to chatbot1 on resume", - "completion": { - "model": "gpt-4o", - "internalSystemPrompt": false, - "tools": [ - { - "function": { - "toolID": "sys.chat.finish", - "name": "chatFinish", - "description": "Concludes the conversation. This can not be used to ask a question.", - "parameters": { - "properties": { - "return": { - "description": "The instructed value to return or a summary of the dialog if no value is instructed", - "type": "string" - } - }, - "type": "object" - } - } - } - ], - "messages": [ - { - "role": "system", - "content": [ - { - "text": "This is a chatbot" - } - ], - "usage": {} - }, - { - "role": "user", - "content": [ - { - "text": "Input to chatbot1 on resume" - } - ], - "usage": {} - }, - { - "role": "assistant", - "content": [ - { - "text": "Assistant Response 4 - from chatbot1" - } - ], - "usage": {} - } - ], - "chat": true - } - }, - "result": "Assistant Response 4 - from chatbot1" - }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:chatbot" - } - } - ], - "subCallID": "call_3" - }, - "inputContextContinuationInput": "User 1", - "inputContextContinuationResumeInput": "User 3" - } -}` diff --git a/pkg/tests/testdata/TestContextSubChat/call4-resp.golden b/pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden similarity index 56% rename from pkg/tests/testdata/TestContextSubChat/call4-resp.golden rename to pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden index a86ae187..2861a036 100644 --- a/pkg/tests/testdata/TestContextSubChat/call4-resp.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/call1-resp.golden @@ -2,7 +2,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 2 - from context tool" + "text": "TEST RESULT CALL: 1" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextWithAsterick/call1.golden b/pkg/tests/testdata/TestContextWithAsterick/call1.golden new file mode 100644 index 00000000..6d9538ce --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/call1.golden @@ -0,0 +1,25 @@ +`{ + "model": "gpt-4o", + "internalSystemPrompt": false, + "messages": [ + { + "role": "system", + "content": [ + { + "text": "This is the input: input 1\n\nSay hi" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + } + ], + "chat": true +}` diff --git a/pkg/tests/testdata/TestContextSubChat/call5-resp.golden b/pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden similarity index 55% rename from pkg/tests/testdata/TestContextSubChat/call5-resp.golden rename to pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden index e49a8481..997ca1b9 100644 --- a/pkg/tests/testdata/TestContextSubChat/call5-resp.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/call2-resp.golden @@ -2,7 +2,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 3 - from main chat tool" + "text": "TEST RESULT CALL: 2" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/call10.golden b/pkg/tests/testdata/TestContextWithAsterick/call2.golden similarity index 72% rename from pkg/tests/testdata/TestContextSubChat/call10.golden rename to pkg/tests/testdata/TestContextWithAsterick/call2.golden index c8c98651..f159014c 100644 --- a/pkg/tests/testdata/TestContextSubChat/call10.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/call2.golden @@ -6,7 +6,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 5 - from context tool resume\nHello" + "text": "This is the input: input 2\n\nSay hi" } ], "usage": {} @@ -15,7 +15,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} @@ -24,7 +24,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 3 - from main chat tool" + "text": "TEST RESULT CALL: 1" } ], "usage": {} @@ -33,7 +33,7 @@ "role": "user", "content": [ { - "text": "User 3" + "text": "input 2" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextWithAsterick/step1.golden b/pkg/tests/testdata/TestContextWithAsterick/step1.golden new file mode 100644 index 00000000..cc42e6a6 --- /dev/null +++ b/pkg/tests/testdata/TestContextWithAsterick/step1.golden @@ -0,0 +1,48 @@ +`{ + "done": false, + "content": "TEST RESULT CALL: 1", + "toolID": "inline:", + "state": { + "continuation": { + "state": { + "input": "input 1", + "completion": { + "model": "gpt-4o", + "internalSystemPrompt": false, + "messages": [ + { + "role": "system", + "content": [ + { + "text": "This is the input: input 1\n\nSay hi" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} + } + ], + "chat": true + } + }, + "result": "TEST RESULT CALL: 1" + }, + "continuationToolID": "inline:" + } +}` diff --git a/pkg/tests/testdata/TestContextSubChat/step4.golden b/pkg/tests/testdata/TestContextWithAsterick/step2.golden similarity index 65% rename from pkg/tests/testdata/TestContextSubChat/step4.golden rename to pkg/tests/testdata/TestContextWithAsterick/step2.golden index 5e95d626..02bc92fe 100644 --- a/pkg/tests/testdata/TestContextSubChat/step4.golden +++ b/pkg/tests/testdata/TestContextWithAsterick/step2.golden @@ -1,11 +1,11 @@ `{ "done": false, - "content": "Assistant Response 6 - from main chat tool resume", - "toolID": "testdata/TestContextSubChat/test.gpt:", + "content": "TEST RESULT CALL: 2", + "toolID": "inline:", "state": { "continuation": { "state": { - "input": "User 1", + "input": "input 1", "completion": { "model": "gpt-4o", "internalSystemPrompt": false, @@ -14,7 +14,7 @@ "role": "system", "content": [ { - "text": "Assistant Response 5 - from context tool resume\nHello" + "text": "This is the input: input 2\n\nSay hi" } ], "usage": {} @@ -23,7 +23,7 @@ "role": "user", "content": [ { - "text": "User 1" + "text": "input 1" } ], "usage": {} @@ -32,7 +32,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 3 - from main chat tool" + "text": "TEST RESULT CALL: 1" } ], "usage": {} @@ -41,7 +41,7 @@ "role": "user", "content": [ { - "text": "User 3" + "text": "input 2" } ], "usage": {} @@ -50,7 +50,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 6 - from main chat tool resume" + "text": "TEST RESULT CALL: 2" } ], "usage": {} @@ -59,8 +59,8 @@ "chat": true } }, - "result": "Assistant Response 6 - from main chat tool resume" + "result": "TEST RESULT CALL: 2" }, - "continuationToolID": "testdata/TestContextSubChat/test.gpt:" + "continuationToolID": "inline:" } }` diff --git a/pkg/tests/testdata/TestExport/call1-resp.golden b/pkg/tests/testdata/TestExport/call1-resp.golden index 8462d188..7fe59586 100644 --- a/pkg/tests/testdata/TestExport/call1-resp.golden +++ b/pkg/tests/testdata/TestExport/call1-resp.golden @@ -3,7 +3,7 @@ "content": [ { "toolCall": { - "index": 2, + "index": 1, "id": "call_1", "function": { "name": "transient" diff --git a/pkg/tests/testdata/TestExport/call1.golden b/pkg/tests/testdata/TestExport/call1.golden index 9f8b650d..b700ee55 100644 --- a/pkg/tests/testdata/TestExport/call1.golden +++ b/pkg/tests/testdata/TestExport/call1.golden @@ -18,8 +18,8 @@ }, { "function": { - "toolID": "testdata/TestExport/parent.gpt:parent-local", - "name": "parentLocal", + "toolID": "testdata/TestExport/sub/child.gpt:transient", + "name": "transient", "parameters": { "properties": { "defaultPromptParameter": { @@ -33,8 +33,8 @@ }, { "function": { - "toolID": "testdata/TestExport/sub/child.gpt:transient", - "name": "transient", + "toolID": "testdata/TestExport/parent.gpt:parent-local", + "name": "parentLocal", "parameters": { "properties": { "defaultPromptParameter": { diff --git a/pkg/tests/testdata/TestExport/call3.golden b/pkg/tests/testdata/TestExport/call3.golden index ccf7e980..d2abca0c 100644 --- a/pkg/tests/testdata/TestExport/call3.golden +++ b/pkg/tests/testdata/TestExport/call3.golden @@ -18,8 +18,8 @@ }, { "function": { - "toolID": "testdata/TestExport/parent.gpt:parent-local", - "name": "parentLocal", + "toolID": "testdata/TestExport/sub/child.gpt:transient", + "name": "transient", "parameters": { "properties": { "defaultPromptParameter": { @@ -33,8 +33,8 @@ }, { "function": { - "toolID": "testdata/TestExport/sub/child.gpt:transient", - "name": "transient", + "toolID": "testdata/TestExport/parent.gpt:parent-local", + "name": "parentLocal", "parameters": { "properties": { "defaultPromptParameter": { @@ -62,7 +62,7 @@ "content": [ { "toolCall": { - "index": 2, + "index": 1, "id": "call_1", "function": { "name": "transient" @@ -80,7 +80,7 @@ } ], "toolCall": { - "index": 2, + "index": 1, "id": "call_1", "function": { "name": "transient" diff --git a/pkg/tests/testdata/TestExportContext/call1.golden b/pkg/tests/testdata/TestExportContext/call1.golden index bec15478..0ee8f9fe 100644 --- a/pkg/tests/testdata/TestExportContext/call1.golden +++ b/pkg/tests/testdata/TestExportContext/call1.golden @@ -38,7 +38,7 @@ "role": "system", "content": [ { - "text": "this is from external context\nthis is from context\nThis is from tool" + "text": "this is from context\nthis is from external context\nThis is from tool" } ], "usage": {} diff --git a/pkg/tests/testdata/TestToolRefAll/call1.golden b/pkg/tests/testdata/TestToolRefAll/call1.golden index ef36e3fb..9289affa 100644 --- a/pkg/tests/testdata/TestToolRefAll/call1.golden +++ b/pkg/tests/testdata/TestToolRefAll/call1.golden @@ -18,12 +18,12 @@ }, { "function": { - "toolID": "testdata/TestToolRefAll/test.gpt:none", - "name": "none", + "toolID": "testdata/TestToolRefAll/test.gpt:agentAssistant", + "name": "agentAssistant", "parameters": { "properties": { - "noneArg": { - "description": "stuff", + "defaultPromptParameter": { + "description": "Prompt to send to the tool. This may be an instruction or question.", "type": "string" } }, @@ -33,12 +33,12 @@ }, { "function": { - "toolID": "testdata/TestToolRefAll/test.gpt:agentAssistant", - "name": "agent", + "toolID": "testdata/TestToolRefAll/test.gpt:none", + "name": "none", "parameters": { "properties": { - "defaultPromptParameter": { - "description": "Prompt to send to the tool. This may be an instruction or question.", + "noneArg": { + "description": "stuff", "type": "string" } }, @@ -52,7 +52,7 @@ "role": "system", "content": [ { - "text": "\nShared context\n\nContext Body\nMain tool" + "text": "\nContext Body\n\nShared context\nMain tool" } ], "usage": {} diff --git a/pkg/tests/testdata/TestContextSubChat/call9-resp.golden b/pkg/tests/testdata/TestToolsChange/call1-resp.golden similarity index 53% rename from pkg/tests/testdata/TestContextSubChat/call9-resp.golden rename to pkg/tests/testdata/TestToolsChange/call1-resp.golden index 4424246d..2861a036 100644 --- a/pkg/tests/testdata/TestContextSubChat/call9-resp.golden +++ b/pkg/tests/testdata/TestToolsChange/call1-resp.golden @@ -2,7 +2,7 @@ "role": "assistant", "content": [ { - "text": "Assistant Response 5 - from context tool resume" + "text": "TEST RESULT CALL: 1" } ], "usage": {} diff --git a/pkg/tests/testdata/TestToolsChange/call1.golden b/pkg/tests/testdata/TestToolsChange/call1.golden new file mode 100644 index 00000000..6c7c2d55 --- /dev/null +++ b/pkg/tests/testdata/TestToolsChange/call1.golden @@ -0,0 +1,70 @@ +`{ + "model": "gpt-4o", + "internalSystemPrompt": false, + "tools": [ + { + "function": { + "toolID": "sys.ls", + "name": "ls", + "description": "Lists the contents of a directory", + "parameters": { + "properties": { + "dir": { + "description": "The directory to list", + "type": "string" + } + }, + "type": "object" + } + } + }, + { + "function": { + "toolID": "sys.read", + "name": "read", + "description": "Reads the contents of a file", + "parameters": { + "properties": { + "filename": { + "description": "The name of the file to read", + "type": "string" + } + }, + "type": "object" + } + } + }, + { + "function": { + "toolID": "sys.write", + "name": "write", + "description": "Write the contents to a file", + "parameters": { + "properties": { + "content": { + "description": "The content to write", + "type": "string" + }, + "filename": { + "description": "The name of the file to write to", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "messages": [ + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + } + ], + "chat": true +}` diff --git a/pkg/tests/testdata/TestToolsChange/call2-resp.golden b/pkg/tests/testdata/TestToolsChange/call2-resp.golden new file mode 100644 index 00000000..997ca1b9 --- /dev/null +++ b/pkg/tests/testdata/TestToolsChange/call2-resp.golden @@ -0,0 +1,9 @@ +`{ + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 2" + } + ], + "usage": {} +}` diff --git a/pkg/tests/testdata/TestToolsChange/call2.golden b/pkg/tests/testdata/TestToolsChange/call2.golden new file mode 100644 index 00000000..ad86b7ce --- /dev/null +++ b/pkg/tests/testdata/TestToolsChange/call2.golden @@ -0,0 +1,73 @@ +`{ + "model": "gpt-4o", + "internalSystemPrompt": false, + "tools": [ + { + "function": { + "toolID": "sys.ls", + "name": "ls", + "description": "Lists the contents of a directory", + "parameters": { + "properties": { + "dir": { + "description": "The directory to list", + "type": "string" + } + }, + "type": "object" + } + } + }, + { + "function": { + "toolID": "sys.write", + "name": "write", + "description": "Write the contents to a file", + "parameters": { + "properties": { + "content": { + "description": "The content to write", + "type": "string" + }, + "filename": { + "description": "The name of the file to write to", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "messages": [ + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 2" + } + ], + "usage": {} + } + ], + "chat": true, + "temperature": 0.6 +}` diff --git a/pkg/tests/testdata/TestToolsChange/step1.golden b/pkg/tests/testdata/TestToolsChange/step1.golden new file mode 100644 index 00000000..1aae05d1 --- /dev/null +++ b/pkg/tests/testdata/TestToolsChange/step1.golden @@ -0,0 +1,93 @@ +`{ + "done": false, + "content": "TEST RESULT CALL: 1", + "toolID": "inline:", + "state": { + "continuation": { + "state": { + "input": "input 1", + "completion": { + "model": "gpt-4o", + "internalSystemPrompt": false, + "tools": [ + { + "function": { + "toolID": "sys.ls", + "name": "ls", + "description": "Lists the contents of a directory", + "parameters": { + "properties": { + "dir": { + "description": "The directory to list", + "type": "string" + } + }, + "type": "object" + } + } + }, + { + "function": { + "toolID": "sys.read", + "name": "read", + "description": "Reads the contents of a file", + "parameters": { + "properties": { + "filename": { + "description": "The name of the file to read", + "type": "string" + } + }, + "type": "object" + } + } + }, + { + "function": { + "toolID": "sys.write", + "name": "write", + "description": "Write the contents to a file", + "parameters": { + "properties": { + "content": { + "description": "The content to write", + "type": "string" + }, + "filename": { + "description": "The name of the file to write to", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "messages": [ + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} + } + ], + "chat": true + } + }, + "result": "TEST RESULT CALL: 1" + }, + "continuationToolID": "inline:" + } +}` diff --git a/pkg/tests/testdata/TestToolsChange/step2.golden b/pkg/tests/testdata/TestToolsChange/step2.golden new file mode 100644 index 00000000..9c9dbad7 --- /dev/null +++ b/pkg/tests/testdata/TestToolsChange/step2.golden @@ -0,0 +1,96 @@ +`{ + "done": false, + "content": "TEST RESULT CALL: 2", + "toolID": "inline:", + "state": { + "continuation": { + "state": { + "input": "input 1", + "completion": { + "model": "gpt-4o", + "internalSystemPrompt": false, + "tools": [ + { + "function": { + "toolID": "sys.ls", + "name": "ls", + "description": "Lists the contents of a directory", + "parameters": { + "properties": { + "dir": { + "description": "The directory to list", + "type": "string" + } + }, + "type": "object" + } + } + }, + { + "function": { + "toolID": "sys.write", + "name": "write", + "description": "Write the contents to a file", + "parameters": { + "properties": { + "content": { + "description": "The content to write", + "type": "string" + }, + "filename": { + "description": "The name of the file to write to", + "type": "string" + } + }, + "type": "object" + } + } + } + ], + "messages": [ + { + "role": "user", + "content": [ + { + "text": "input 1" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 1" + } + ], + "usage": {} + }, + { + "role": "user", + "content": [ + { + "text": "input 2" + } + ], + "usage": {} + }, + { + "role": "assistant", + "content": [ + { + "text": "TEST RESULT CALL: 2" + } + ], + "usage": {} + } + ], + "chat": true, + "temperature": 0.6 + } + }, + "result": "TEST RESULT CALL: 2" + }, + "continuationToolID": "inline:" + } +}` diff --git a/pkg/tests/tester/runner.go b/pkg/tests/tester/runner.go index 66337ff5..b460ce18 100644 --- a/pkg/tests/tester/runner.go +++ b/pkg/tests/tester/runner.go @@ -135,7 +135,8 @@ func (c *Client) Call(_ context.Context, messageRequest types.CompletionRequest, type Runner struct { *runner.Runner - Client *Client + Client *Client + StepAsserted int } func (r *Runner) RunDefault() string { @@ -166,6 +167,21 @@ func (r *Runner) AssertResponded(t *testing.T) { require.Len(t, r.Client.result, 0) } +func toJSONString(t *testing.T, v interface{}) string { + t.Helper() + x, err := json.MarshalIndent(v, "", " ") + require.NoError(t, err) + return string(x) +} + +func (r *Runner) AssertStep(t *testing.T, resp runner.ChatResponse, err error) { + t.Helper() + r.StepAsserted++ + require.NoError(t, err) + r.AssertResponded(t) + autogold.ExpectFile(t, toJSONString(t, resp), autogold.Name(t.Name()+fmt.Sprintf("/step%d", r.StepAsserted))) +} + func (r *Runner) RespondWith(result ...Result) { r.Client.result = append(r.Client.result, result...) } diff --git a/pkg/types/completion.go b/pkg/types/completion.go index dd70ad50..5b3899c3 100644 --- a/pkg/types/completion.go +++ b/pkg/types/completion.go @@ -9,15 +9,15 @@ import ( ) type CompletionRequest struct { - Model string `json:"model,omitempty"` - InternalSystemPrompt *bool `json:"internalSystemPrompt,omitempty"` - Tools []CompletionTool `json:"tools,omitempty"` - Messages []CompletionMessage `json:"messages,omitempty"` - MaxTokens int `json:"maxTokens,omitempty"` - Chat bool `json:"chat,omitempty"` - Temperature *float32 `json:"temperature,omitempty"` - JSONResponse bool `json:"jsonResponse,omitempty"` - Cache *bool `json:"cache,omitempty"` + Model string `json:"model,omitempty"` + InternalSystemPrompt *bool `json:"internalSystemPrompt,omitempty"` + Tools []ChatCompletionTool `json:"tools,omitempty"` + Messages []CompletionMessage `json:"messages,omitempty"` + MaxTokens int `json:"maxTokens,omitempty"` + Chat bool `json:"chat,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + JSONResponse bool `json:"jsonResponse,omitempty"` + Cache *bool `json:"cache,omitempty"` } func (r *CompletionRequest) GetCache() bool { @@ -27,7 +27,7 @@ func (r *CompletionRequest) GetCache() bool { return *r.Cache } -type CompletionTool struct { +type ChatCompletionTool struct { Function CompletionFunctionDefinition `json:"function,omitempty"` } diff --git a/pkg/types/set.go b/pkg/types/set.go index 230e112b..65b73d22 100644 --- a/pkg/types/set.go +++ b/pkg/types/set.go @@ -19,6 +19,17 @@ func (t *toolRefSet) List() (result []ToolReference, err error) { return result, t.err } +func (t *toolRefSet) Contains(value ToolReference) bool { + key := toolRefKey{ + name: value.Named, + toolID: value.ToolID, + arg: value.Arg, + } + + _, ok := t.set[key] + return ok +} + func (t *toolRefSet) HasTool(toolID string) bool { for _, ref := range t.set { if ref.ToolID == toolID { diff --git a/pkg/types/tool.go b/pkg/types/tool.go index 57ce3fbf..d9d59837 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -33,11 +33,15 @@ const ( ToolTypeAgent = ToolType("agent") ToolTypeOutput = ToolType("output") ToolTypeInput = ToolType("input") - ToolTypeAssistant = ToolType("assistant") ToolTypeTool = ToolType("tool") ToolTypeCredential = ToolType("credential") - ToolTypeProvider = ToolType("provider") ToolTypeDefault = ToolType("") + + // The following types logically exist but have no real code reference. These are kept + // here just so that we have a comprehensive list + + ToolTypeAssistant = ToolType("assistant") + ToolTypeProvider = ToolType("provider") ) type ErrToolNotFound struct { @@ -140,6 +144,28 @@ type Parameters struct { Type ToolType `json:"type,omitempty"` } +func (p Parameters) allExports() []string { + return slices.Concat( + p.ExportContext, + p.Export, + p.ExportCredentials, + p.ExportInputFilters, + p.ExportOutputFilters, + ) +} + +func (p Parameters) allReferences() []string { + return slices.Concat( + p.GlobalTools, + p.Tools, + p.Context, + p.Agents, + p.Credentials, + p.InputFilters, + p.OutputFilters, + ) +} + func (p Parameters) ToolRefNames() []string { return slices.Concat( p.Tools, @@ -335,39 +361,6 @@ func ParseCredentialArgs(toolName string, input string) (string, string, map[str return originalName, alias, args, nil } -func (t Tool) GetAgents(prg Program) (result []ToolReference, _ error) { - toolRefs, err := t.GetToolRefsFromNames(t.Agents) - if err != nil { - return nil, err - } - - genericToolRefs, err := t.getCompletionToolRefs(prg, nil, ToolTypeAgent) - if err != nil { - return nil, err - } - - toolRefs = append(toolRefs, genericToolRefs...) - - // Agent Tool refs must be named - for i, toolRef := range toolRefs { - if toolRef.Named != "" { - continue - } - tool := prg.ToolSet[toolRef.ToolID] - name := tool.Name - if name == "" { - name = toolRef.Reference - } - normed := ToolNormalizer(name) - if trimmed := strings.TrimSuffix(strings.TrimSuffix(normed, "Agent"), "Assistant"); trimmed != "" { - normed = trimmed - } - toolRefs[i].Named = normed - } - - return toolRefs, nil -} - func (t Tool) GetToolRefsFromNames(names []string) (result []ToolReference, _ error) { for _, toolName := range names { toolRefs, ok := t.ToolMapping[toolName] @@ -507,284 +500,185 @@ func (t ToolDef) String() string { return buf.String() } -func (t Tool) getExportedContext(prg Program) ([]ToolReference, error) { - result := &toolRefSet{} - - exportRefs, err := t.GetToolRefsFromNames(t.ExportContext) - if err != nil { - return nil, err - } - - for _, exportRef := range exportRefs { - result.Add(exportRef) - - tool := prg.ToolSet[exportRef.ToolID] - result.AddAll(tool.getExportedContext(prg)) - } - - return result.List() -} - -func (t Tool) getExportedTools(prg Program) ([]ToolReference, error) { - result := &toolRefSet{} - - exportRefs, err := t.GetToolRefsFromNames(t.Export) - if err != nil { - return nil, err - } - - for _, exportRef := range exportRefs { - result.Add(exportRef) - result.AddAll(prg.ToolSet[exportRef.ToolID].getExportedTools(prg)) - } - - return result.List() -} - -// GetContextTools returns all tools that are in the context of the tool including all the -// contexts that are exported by the context tools. This will recurse all exports. -func (t Tool) GetContextTools(prg Program) ([]ToolReference, error) { - result := &toolRefSet{} - result.AddAll(t.getDirectContextToolRefs(prg)) - - contextRefs, err := t.getCompletionToolRefs(prg, nil, ToolTypeContext) - if err != nil { - return nil, err - } +func (t Tool) GetNextAgentGroup(prg *Program, agentGroup []ToolReference, toolID string) (result []ToolReference, _ error) { + newAgentGroup := toolRefSet{} + newAgentGroup.AddAll(t.GetToolsByType(prg, ToolTypeAgent)) - for _, contextRef := range contextRefs { - result.AddAll(prg.ToolSet[contextRef.ToolID].getExportedContext(prg)) - result.Add(contextRef) + if newAgentGroup.HasTool(toolID) { + // Join new agent group + return newAgentGroup.List() } - return result.List() + return agentGroup, nil } -// GetContextTools returns all tools that are in the context of the tool including all the -// contexts that are exported by the context tools. This will recurse all exports. -func (t Tool) getDirectContextToolRefs(prg Program) ([]ToolReference, error) { - result := &toolRefSet{} - - contextRefs, err := t.GetToolRefsFromNames(t.Context) +func (t Tool) getAgents(prg *Program) (result []ToolReference, _ error) { + toolRefs, err := t.GetToolRefsFromNames(t.Agents) if err != nil { return nil, err } - for _, contextRef := range contextRefs { - result.AddAll(prg.ToolSet[contextRef.ToolID].getExportedContext(prg)) - result.Add(contextRef) + // Agent Tool refs must be named + for i, toolRef := range toolRefs { + if toolRef.Named != "" { + continue + } + tool := prg.ToolSet[toolRef.ToolID] + name := tool.Name + if name == "" { + name = toolRef.Reference + } + normed := ToolNormalizer(name) + if trimmed := strings.TrimSuffix(strings.TrimSuffix(normed, "Agent"), "Assistant"); trimmed != "" { + normed = trimmed + } + toolRefs[i].Named = normed } - return result.List() + return toolRefs, nil } -func (t Tool) GetOutputFilterTools(program Program) ([]ToolReference, error) { - result := &toolRefSet{} - - outputFilterRefs, err := t.GetToolRefsFromNames(t.OutputFilters) - if err != nil { - return nil, err - } - - for _, outputFilterRef := range outputFilterRefs { - result.Add(outputFilterRef) - } - - result.AddAll(t.getCompletionToolRefs(program, nil, ToolTypeOutput)) - - contextRefs, err := t.getDirectContextToolRefs(program) - if err != nil { - return nil, err - } - - for _, contextRef := range contextRefs { - contextTool := program.ToolSet[contextRef.ToolID] - result.AddAll(contextTool.GetToolRefsFromNames(contextTool.ExportOutputFilters)) +func (t Tool) GetToolsByType(prg *Program, toolType ToolType) ([]ToolReference, error) { + if toolType == ToolTypeAgent { + // Agents are special, they can only be sourced from direct references and not the generic 'tool:' or shared by references + return t.getAgents(prg) } - return result.List() -} - -func (t Tool) GetInputFilterTools(program Program) ([]ToolReference, error) { - result := &toolRefSet{} + toolSet := &toolRefSet{} - inputFilterRefs, err := t.GetToolRefsFromNames(t.InputFilters) - if err != nil { - return nil, err - } + var ( + directRefs []string + toolsListFilterType = []ToolType{toolType} + ) - for _, inputFilterRef := range inputFilterRefs { - result.Add(inputFilterRef) + switch toolType { + case ToolTypeContext: + directRefs = t.Context + case ToolTypeOutput: + directRefs = t.OutputFilters + case ToolTypeInput: + directRefs = t.InputFilters + case ToolTypeTool: + toolsListFilterType = append(toolsListFilterType, ToolTypeDefault, ToolTypeAgent) + case ToolTypeCredential: + directRefs = t.Credentials + default: + return nil, fmt.Errorf("unknown tool type %v", toolType) } - result.AddAll(t.getCompletionToolRefs(program, nil, ToolTypeInput)) + toolSet.AddAll(t.GetToolRefsFromNames(directRefs)) - contextRefs, err := t.getDirectContextToolRefs(program) + toolRefs, err := t.GetToolRefsFromNames(t.Tools) if err != nil { return nil, err } - for _, contextRef := range contextRefs { - contextTool := program.ToolSet[contextRef.ToolID] - result.AddAll(contextTool.GetToolRefsFromNames(contextTool.ExportInputFilters)) - } - - return result.List() -} - -func (t Tool) GetNextAgentGroup(prg Program, agentGroup []ToolReference, toolID string) (result []ToolReference, _ error) { - newAgentGroup := toolRefSet{} - if err := t.addAgents(prg, &newAgentGroup); err != nil { - return nil, err - } - - if newAgentGroup.HasTool(toolID) { - // Join new agent group - return newAgentGroup.List() - } - - return agentGroup, nil -} - -func filterRefs(prg Program, refs []ToolReference, types ...ToolType) (result []ToolReference) { - for _, ref := range refs { - if slices.Contains(types, prg.ToolSet[ref.ToolID].Type) { - result = append(result, ref) + for _, toolRef := range toolRefs { + tool, ok := prg.ToolSet[toolRef.ToolID] + if !ok { + continue + } + if slices.Contains(toolsListFilterType, tool.Type) { + toolSet.Add(toolRef) } - } - return -} - -func (t Tool) GetCompletionTools(prg Program, agentGroup ...ToolReference) (result []CompletionTool, err error) { - toolSet := &toolRefSet{} - toolSet.AddAll(t.getCompletionToolRefs(prg, agentGroup, ToolTypeDefault, ToolTypeTool)) - - if err := t.addAgents(prg, toolSet); err != nil { - return nil, err } - refs, err := toolSet.List() + exportSources, err := t.getExportSources(prg) if err != nil { return nil, err } - return toolRefsToCompletionTools(refs, prg), nil -} - -func (t Tool) addAgents(prg Program, result *toolRefSet) error { - subToolRefs, err := t.GetAgents(prg) - if err != nil { - return err - } - - for _, subToolRef := range subToolRefs { - // don't add yourself - if subToolRef.ToolID != t.ID { - // Add the tool itself and no exports - result.Add(subToolRef) + for _, exportSource := range exportSources { + var ( + tool = prg.ToolSet[exportSource.ToolID] + exportRefs []string + ) + + switch toolType { + case ToolTypeContext: + exportRefs = tool.ExportContext + case ToolTypeOutput: + exportRefs = tool.ExportOutputFilters + case ToolTypeInput: + exportRefs = tool.ExportInputFilters + case ToolTypeTool: + exportRefs = tool.Export + case ToolTypeCredential: + exportRefs = tool.ExportCredentials + default: + return nil, fmt.Errorf("unknown tool type %v", toolType) } + toolSet.AddAll(tool.GetToolRefsFromNames(exportRefs)) } - return nil + return toolSet.List() } -func (t Tool) addReferencedTools(prg Program, result *toolRefSet) error { - subToolRefs, err := t.GetToolRefsFromNames(t.Parameters.Tools) +func (t Tool) addExportsRecursively(prg *Program, toolSet *toolRefSet) error { + toolRefs, err := t.GetToolRefsFromNames(t.allExports()) if err != nil { return err } - for _, subToolRef := range subToolRefs { - // Add the tool - result.Add(subToolRef) + for _, toolRef := range toolRefs { + if toolSet.Contains(toolRef) { + continue + } - // Get all tools exports - result.AddAll(prg.ToolSet[subToolRef.ToolID].getExportedTools(prg)) + toolSet.Add(toolRef) + if err := prg.ToolSet[toolRef.ToolID].addExportsRecursively(prg, toolSet); err != nil { + return err + } } return nil } -func (t Tool) addContextExportedTools(prg Program, result *toolRefSet) error { - contextTools, err := t.getDirectContextToolRefs(prg) +func (t Tool) getExportSources(prg *Program) ([]ToolReference, error) { + // We start first with all references from this tool. This gives us the + // initial set of export sources. + // Then all tools in the export sources in the set we look for exports of those tools recursively. + // So a share of a share of a share should be added. + + toolSet := toolRefSet{} + toolRefs, err := t.GetToolRefsFromNames(t.allReferences()) if err != nil { - return err + return nil, err } - for _, contextTool := range contextTools { - result.AddAll(prg.ToolSet[contextTool.ToolID].getExportedTools(prg)) + for _, toolRef := range toolRefs { + if err := prg.ToolSet[toolRef.ToolID].addExportsRecursively(prg, &toolSet); err != nil { + return nil, err + } + toolSet.Add(toolRef) } - return nil + return toolSet.List() } -func (t Tool) getCompletionToolRefs(prg Program, agentGroup []ToolReference, types ...ToolType) ([]ToolReference, error) { - if len(types) == 0 { - types = []ToolType{ToolTypeDefault, ToolTypeTool} - } - - result := toolRefSet{} +func (t Tool) GetChatCompletionTools(prg Program, agentGroup ...ToolReference) (result []ChatCompletionTool, err error) { + toolSet := &toolRefSet{} + toolSet.AddAll(t.GetToolsByType(&prg, ToolTypeTool)) + toolSet.AddAll(t.GetToolsByType(&prg, ToolTypeAgent)) if t.Chat { for _, agent := range agentGroup { // don't add yourself if agent.ToolID != t.ID { - result.Add(agent) + toolSet.Add(agent) } } } - if err := t.addReferencedTools(prg, &result); err != nil { - return nil, err - } - - if err := t.addContextExportedTools(prg, &result); err != nil { - return nil, err - } - - refs, err := result.List() - return filterRefs(prg, refs, types...), err -} - -func (t Tool) GetCredentialTools(prg Program, agentGroup []ToolReference) ([]ToolReference, error) { - result := toolRefSet{} - - result.AddAll(t.GetToolRefsFromNames(t.Credentials)) - - result.AddAll(t.getCompletionToolRefs(prg, nil, ToolTypeCredential)) - - toolRefs, err := result.List() - if err != nil { - return nil, err - } - for _, toolRef := range toolRefs { - referencedTool := prg.ToolSet[toolRef.ToolID] - result.AddAll(referencedTool.GetToolRefsFromNames(referencedTool.ExportCredentials)) - } - - toolRefs, err = t.getCompletionToolRefs(prg, agentGroup) - if err != nil { - return nil, err - } - for _, toolRef := range toolRefs { - referencedTool := prg.ToolSet[toolRef.ToolID] - result.AddAll(referencedTool.GetToolRefsFromNames(referencedTool.ExportCredentials)) - } - - contextToolRefs, err := t.GetContextTools(prg) + refs, err := toolSet.List() if err != nil { return nil, err } - for _, contextToolRef := range contextToolRefs { - contextTool := prg.ToolSet[contextToolRef.ToolID] - result.AddAll(contextTool.GetToolRefsFromNames(contextTool.ExportCredentials)) - } - - return result.List() + return toolRefsToCompletionTools(refs, prg), nil } -func toolRefsToCompletionTools(completionTools []ToolReference, prg Program) (result []CompletionTool) { +func toolRefsToCompletionTools(completionTools []ToolReference, prg Program) (result []ChatCompletionTool) { toolNames := map[string]struct{}{} for _, subToolRef := range completionTools { @@ -805,7 +699,7 @@ func toolRefsToCompletionTools(completionTools []ToolReference, prg Program) (re if subTool.Instructions == "" { log.Debugf("Skipping zero instruction tool %s (%s)", subToolName, subTool.ID) } else { - result = append(result, CompletionTool{ + result = append(result, ChatCompletionTool{ Function: CompletionFunctionDefinition{ ToolID: subTool.ID, Name: PickToolName(subToolName, toolNames),