From c31a4c95e721747e2427bb86a5a1456fa9689faa Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Tue, 23 Jul 2024 00:00:42 +0100 Subject: [PATCH 1/3] Release/2.0.0 (#2844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🥳 flowise@2.0.0 release --- package.json | 2 +- packages/components/package.json | 2 +- packages/server/package.json | 2 +- packages/ui/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 381c079d610..22384a4e67c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flowise", - "version": "1.8.4", + "version": "2.0.0", "private": true, "homepage": "https://flowiseai.com", "workspaces": [ diff --git a/packages/components/package.json b/packages/components/package.json index 36b4c5d3f2e..11ab150072b 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -1,6 +1,6 @@ { "name": "flowise-components", - "version": "1.8.6", + "version": "2.0.0", "description": "Flowiseai Components", "main": "dist/src/index", "types": "dist/src/index.d.ts", diff --git a/packages/server/package.json b/packages/server/package.json index 7b21afa8844..f0cb2c2ed63 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "flowise", - "version": "1.8.4", + "version": "2.0.0", "description": "Flowiseai Server", "main": "dist/index", "types": "dist/index.d.ts", diff --git a/packages/ui/package.json b/packages/ui/package.json index 39b8ccd4e76..96e9aa43a88 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "flowise-ui", - "version": "1.8.4", + "version": "2.0.0", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://flowiseai.com", "author": { From 368c69cbc53132e6cb8ac13b55441f944d25724d Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Tue, 23 Jul 2024 15:15:41 +0100 Subject: [PATCH 2/3] Bugfix/Move summarization as optional for multi agents (#2858) add summarization as optional for multi agents --- .../multiagents/Supervisor/Supervisor.ts | 271 +++++++++++++++++- .../nodes/multiagents/Worker/Worker.ts | 2 +- packages/components/src/Interface.ts | 2 +- packages/server/src/utils/buildAgentGraph.ts | 28 +- 4 files changed, 290 insertions(+), 13 deletions(-) diff --git a/packages/components/nodes/multiagents/Supervisor/Supervisor.ts b/packages/components/nodes/multiagents/Supervisor/Supervisor.ts index fd1cd35c25e..cccd23486ef 100644 --- a/packages/components/nodes/multiagents/Supervisor/Supervisor.ts +++ b/packages/components/nodes/multiagents/Supervisor/Supervisor.ts @@ -49,7 +49,7 @@ class Supervisor_MultiAgents implements INode { constructor() { this.label = 'Supervisor' this.name = 'supervisor' - this.version = 2.0 + this.version = 3.0 this.type = 'Supervisor' this.icon = 'supervisor.svg' this.category = 'Multi Agents' @@ -84,6 +84,14 @@ class Supervisor_MultiAgents implements INode { description: 'Save the state of the agent', optional: true }, + { + label: 'Summarization', + name: 'summarization', + type: 'boolean', + description: 'Return final output as a summarization of the conversation', + optional: true, + additionalParams: true + }, { label: 'Recursion Limit', name: 'recursionLimit', @@ -110,6 +118,7 @@ class Supervisor_MultiAgents implements INode { const _recursionLimit = nodeData.inputs?.recursionLimit as string const recursionLimit = _recursionLimit ? parseFloat(_recursionLimit) : 100 const moderations = (nodeData.inputs?.inputModeration as Moderation[]) ?? [] + const summarization = nodeData.inputs?.summarization as string const abortControllerSignal = options.signal as AbortController @@ -128,6 +137,257 @@ class Supervisor_MultiAgents implements INode { systemPrompt = systemPrompt.replaceAll('{team_members}', members.join(', ')) + let userPrompt = `Given the conversation above, who should act next? Or should we FINISH? Select one of: ${memberOptions.join( + ', ' + )}` + + const tool = new RouteTool({ + schema: z.object({ + reasoning: z.string(), + next: z.enum(['FINISH', ...members]), + instructions: z.string().describe('The specific instructions of the sub-task the next role should accomplish.') + }) + }) + + let supervisor + + if (llm instanceof ChatMistralAI) { + let prompt = ChatPromptTemplate.fromMessages([ + ['system', systemPrompt], + new MessagesPlaceholder('messages'), + ['human', userPrompt] + ]) + + const messages = await processImageMessage(1, llm, prompt, nodeData, options) + prompt = messages.prompt + multiModalMessageContent = messages.multiModalMessageContent + + // Force Mistral to use tool + // @ts-ignore + const modelWithTool = llm.bind({ + tools: [tool], + tool_choice: 'any', + signal: abortControllerSignal ? abortControllerSignal.signal : undefined + }) + + const outputParser = new JsonOutputToolsParser() + + supervisor = prompt + .pipe(modelWithTool) + .pipe(outputParser) + .pipe((x) => { + if (Array.isArray(x) && x.length) { + const toolAgentAction = x[0] + return { + next: Object.keys(toolAgentAction.args).length ? toolAgentAction.args.next : 'FINISH', + instructions: Object.keys(toolAgentAction.args).length + ? toolAgentAction.args.instructions + : 'Conversation finished', + team_members: members.join(', ') + } + } else { + return { + next: 'FINISH', + instructions: 'Conversation finished', + team_members: members.join(', ') + } + } + }) + } else if (llm instanceof ChatAnthropic) { + // Force Anthropic to use tool : https://docs.anthropic.com/claude/docs/tool-use#forcing-tool-use + userPrompt = `Given the conversation above, who should act next? Or should we FINISH? Select one of: ${memberOptions.join( + ', ' + )}. Use the ${routerToolName} tool in your response.` + + let prompt = ChatPromptTemplate.fromMessages([ + ['system', systemPrompt], + new MessagesPlaceholder('messages'), + ['human', userPrompt] + ]) + + const messages = await processImageMessage(1, llm, prompt, nodeData, options) + prompt = messages.prompt + multiModalMessageContent = messages.multiModalMessageContent + + if (llm.bindTools === undefined) { + throw new Error(`This agent only compatible with function calling models.`) + } + + const modelWithTool = llm.bindTools([tool]) + + const outputParser = new ToolCallingAgentOutputParser() + + supervisor = prompt + .pipe(modelWithTool) + .pipe(outputParser) + .pipe((x) => { + if (Array.isArray(x) && x.length) { + const toolAgentAction = x[0] as any + return { + next: toolAgentAction.toolInput.next, + instructions: toolAgentAction.toolInput.instructions, + team_members: members.join(', ') + } + } else if (typeof x === 'object' && 'returnValues' in x) { + return { + next: 'FINISH', + instructions: x.returnValues?.output, + team_members: members.join(', ') + } + } else { + return { + next: 'FINISH', + instructions: 'Conversation finished', + team_members: members.join(', ') + } + } + }) + } else if (llm instanceof ChatOpenAI) { + let prompt = ChatPromptTemplate.fromMessages([ + ['system', systemPrompt], + new MessagesPlaceholder('messages'), + ['human', userPrompt] + ]) + + // @ts-ignore + const messages = await processImageMessage(1, llm, prompt, nodeData, options) + prompt = messages.prompt + multiModalMessageContent = messages.multiModalMessageContent + + // Force OpenAI to use tool + const modelWithTool = llm.bind({ + tools: [tool], + tool_choice: { type: 'function', function: { name: routerToolName } }, + signal: abortControllerSignal ? abortControllerSignal.signal : undefined + }) + + const outputParser = new ToolCallingAgentOutputParser() + + supervisor = prompt + .pipe(modelWithTool) + .pipe(outputParser) + .pipe((x) => { + if (Array.isArray(x) && x.length) { + const toolAgentAction = x[0] as any + return { + next: toolAgentAction.toolInput.next, + instructions: toolAgentAction.toolInput.instructions, + team_members: members.join(', ') + } + } else if (typeof x === 'object' && 'returnValues' in x) { + return { + next: 'FINISH', + instructions: x.returnValues?.output, + team_members: members.join(', ') + } + } else { + return { + next: 'FINISH', + instructions: 'Conversation finished', + team_members: members.join(', ') + } + } + }) + } else if (llm instanceof ChatGoogleGenerativeAI) { + /* + * Gemini doesn't have system message and messages have to be alternate between model and user + * So we have to place the system + human prompt at last + */ + let prompt = ChatPromptTemplate.fromMessages([ + ['system', systemPrompt], + new MessagesPlaceholder('messages'), + ['human', userPrompt] + ]) + + const messages = await processImageMessage(2, llm, prompt, nodeData, options) + prompt = messages.prompt + multiModalMessageContent = messages.multiModalMessageContent + + if (llm.bindTools === undefined) { + throw new Error(`This agent only compatible with function calling models.`) + } + const modelWithTool = llm.bindTools([tool]) + + const outputParser = new ToolCallingAgentOutputParser() + + supervisor = prompt + .pipe(modelWithTool) + .pipe(outputParser) + .pipe((x) => { + if (Array.isArray(x) && x.length) { + const toolAgentAction = x[0] as any + return { + next: toolAgentAction.toolInput.next, + instructions: toolAgentAction.toolInput.instructions, + team_members: members.join(', ') + } + } else if (typeof x === 'object' && 'returnValues' in x) { + return { + next: 'FINISH', + instructions: x.returnValues?.output, + team_members: members.join(', ') + } + } else { + return { + next: 'FINISH', + instructions: 'Conversation finished', + team_members: members.join(', ') + } + } + }) + } else { + let prompt = ChatPromptTemplate.fromMessages([ + ['system', systemPrompt], + new MessagesPlaceholder('messages'), + ['human', userPrompt] + ]) + + const messages = await processImageMessage(1, llm, prompt, nodeData, options) + prompt = messages.prompt + multiModalMessageContent = messages.multiModalMessageContent + + if (llm.bindTools === undefined) { + throw new Error(`This agent only compatible with function calling models.`) + } + const modelWithTool = llm.bindTools([tool]) + + const outputParser = new ToolCallingAgentOutputParser() + + supervisor = prompt + .pipe(modelWithTool) + .pipe(outputParser) + .pipe((x) => { + if (Array.isArray(x) && x.length) { + const toolAgentAction = x[0] as any + return { + next: toolAgentAction.toolInput.next, + instructions: toolAgentAction.toolInput.instructions, + team_members: members.join(', ') + } + } else if (typeof x === 'object' && 'returnValues' in x) { + return { + next: 'FINISH', + instructions: x.returnValues?.output, + team_members: members.join(', ') + } + } else { + return { + next: 'FINISH', + instructions: 'Conversation finished', + team_members: members.join(', ') + } + } + }) + } + + return supervisor + } + + async function createTeamSupervisorWithSummarize(llm: BaseChatModel, systemPrompt: string, members: string[]): Promise { + const memberOptions = ['FINISH', ...members] + + systemPrompt = systemPrompt.replaceAll('{team_members}', members.join(', ')) + let userPrompt = `Given the conversation above, who should act next? Or should we FINISH? Select one of: ${memberOptions.join( ', ' )} @@ -247,7 +507,8 @@ class Supervisor_MultiAgents implements INode { ['human', userPrompt] ]) - const messages = await processImageMessage(1, llm as any, prompt, nodeData, options) + // @ts-ignore + const messages = await processImageMessage(1, llm, prompt, nodeData, options) prompt = messages.prompt multiModalMessageContent = messages.multiModalMessageContent @@ -389,7 +650,9 @@ class Supervisor_MultiAgents implements INode { return supervisor } - const supervisorAgent = await createTeamSupervisor(llm, supervisorPrompt ? supervisorPrompt : sysPrompt, workersNodeNames) + const supervisorAgent = summarization + ? await createTeamSupervisorWithSummarize(llm, supervisorPrompt ? supervisorPrompt : sysPrompt, workersNodeNames) + : await createTeamSupervisor(llm, supervisorPrompt ? supervisorPrompt : sysPrompt, workersNodeNames) const supervisorNode = async (state: ITeamState, config: RunnableConfig) => await agentNode( @@ -433,7 +696,7 @@ async function agentNode( throw new Error('Aborted!') } const result = await agent.invoke({ ...state, signal: abortControllerSignal.signal }, config) - const additional_kwargs: ICommonObject = { nodeId } + const additional_kwargs: ICommonObject = { nodeId, type: 'supervisor' } result.additional_kwargs = { ...result.additional_kwargs, ...additional_kwargs } return result } catch (error) { diff --git a/packages/components/nodes/multiagents/Worker/Worker.ts b/packages/components/nodes/multiagents/Worker/Worker.ts index 4b7b904c563..06a5d059cc2 100644 --- a/packages/components/nodes/multiagents/Worker/Worker.ts +++ b/packages/components/nodes/multiagents/Worker/Worker.ts @@ -283,7 +283,7 @@ async function agentNode( } const result = await agent.invoke({ ...state, signal: abortControllerSignal.signal }, config) - const additional_kwargs: ICommonObject = { nodeId } + const additional_kwargs: ICommonObject = { nodeId, type: 'worker' } if (result.usedTools) { additional_kwargs.usedTools = result.usedTools } diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 0fc86e36a11..5b4c4d23d9a 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -206,7 +206,7 @@ export interface ITeamState { team_members: string[] next: string instructions: string - summarization: string + summarization?: string } export interface ISeqAgentsState { diff --git a/packages/server/src/utils/buildAgentGraph.ts b/packages/server/src/utils/buildAgentGraph.ts index 220adf060b4..e6a61a4cbf6 100644 --- a/packages/server/src/utils/buildAgentGraph.ts +++ b/packages/server/src/utils/buildAgentGraph.ts @@ -147,6 +147,7 @@ export const buildAgentGraph = async ( let streamResults let finalResult = '' let finalSummarization = '' + let lastWorkerResult = '' let agentReasoning: IAgentReasoning[] = [] let isSequential = false let lastMessageRaw = {} as AIMessageChunk @@ -182,7 +183,8 @@ export const buildAgentGraph = async ( incomingInput.question, chatHistory, incomingInput?.overrideConfig, - sessionId || chatId + sessionId || chatId, + seqAgentNodes.some((node) => node.data.inputs?.summarization) ) } else { isSequential = true @@ -277,6 +279,12 @@ export const buildAgentGraph = async ( finalSummarization = output[agentName]?.summarization ?? '' + lastWorkerResult = + output[agentName]?.messages?.length && + output[agentName].messages[output[agentName].messages.length - 1]?.additional_kwargs?.type === 'worker' + ? output[agentName].messages[output[agentName].messages.length - 1].content + : lastWorkerResult + if (socketIO && incomingInput.socketIOClientId) { if (!isStreamingStarted) { isStreamingStarted = true @@ -305,10 +313,13 @@ export const buildAgentGraph = async ( /* * For multi agents mode, sometimes finalResult is empty - * Provide summary as final result + * 1.) Provide lastWorkerResult as final result if available + * 2.) Provide summary as final result if available */ - if (!isSequential && !finalResult && finalSummarization) { - finalResult = finalSummarization + if (!isSequential && !finalResult) { + if (lastWorkerResult) finalResult = lastWorkerResult + else if (finalSummarization) finalResult = finalSummarization + if (socketIO && incomingInput.socketIOClientId) { socketIO.to(incomingInput.socketIOClientId).emit('token', finalResult) } @@ -425,6 +436,7 @@ export const buildAgentGraph = async ( * @param {string} question * @param {ICommonObject} overrideConfig * @param {string} threadId + * @param {boolean} summarization */ const compileMultiAgentsGraph = async ( chatflow: IChatFlow, @@ -437,7 +449,8 @@ const compileMultiAgentsGraph = async ( question: string, chatHistory: IMessage[] = [], overrideConfig?: ICommonObject, - threadId?: string + threadId?: string, + summarization?: boolean ) => { const appServer = getRunningExpressApp() const channels: ITeamState = { @@ -447,10 +460,11 @@ const compileMultiAgentsGraph = async ( }, next: 'initialState', instructions: "Solve the user's request.", - team_members: [], - summarization: 'summarize' + team_members: [] } + if (summarization) channels.summarization = 'summarize' + const workflowGraph = new StateGraph({ //@ts-ignore channels From f606f86a7915c4d4e380928c2675f9a4b6f4e77c Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 23 Jul 2024 15:17:14 +0100 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=A5=B3=20flowise@2.0.1=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- packages/components/package.json | 2 +- packages/server/package.json | 2 +- packages/ui/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 22384a4e67c..cc7a959a2e2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flowise", - "version": "2.0.0", + "version": "2.0.1", "private": true, "homepage": "https://flowiseai.com", "workspaces": [ diff --git a/packages/components/package.json b/packages/components/package.json index 11ab150072b..63c278e81a6 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -1,6 +1,6 @@ { "name": "flowise-components", - "version": "2.0.0", + "version": "2.0.1", "description": "Flowiseai Components", "main": "dist/src/index", "types": "dist/src/index.d.ts", diff --git a/packages/server/package.json b/packages/server/package.json index f0cb2c2ed63..01341096ecd 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "flowise", - "version": "2.0.0", + "version": "2.0.1", "description": "Flowiseai Server", "main": "dist/index", "types": "dist/index.d.ts", diff --git a/packages/ui/package.json b/packages/ui/package.json index 96e9aa43a88..b7a461d0c81 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "flowise-ui", - "version": "2.0.0", + "version": "2.0.1", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://flowiseai.com", "author": {