From 472e70feeeafb9a0c44c00cb52851601704e89a5 Mon Sep 17 00:00:00 2001
From: Alex Yang <himself65@outlook.com>
Date: Fri, 19 Apr 2024 17:52:36 -0500
Subject: [PATCH] refactor: full typed & iterator of agent worker/runner (part
3) (#728)
Fixes: https://github.com/run-llama/LlamaIndexTS/issues/692, https://github.com/run-llama/LlamaIndexTS/issues/557
Refs: https://github.com/run-llama/llama_index/blob/5a6ffe32faa75db0b4737d1e7a85e6fe4afe94af/docs/module_guides/deploying/agents/agent_runner.md
---
.npmrc | 3 +-
apps/docs/docs/examples/agent.mdx | 80 +-
apps/docs/docs/modules/agent/index.md | 8 +-
.../modules/agent/multi_document_agent.mdx | 307 ---
apps/docs/docs/modules/agent/openai.mdx | 185 --
.../docs/modules/agent/query_engine_tool.mdx | 130 -
apps/docs/docs/modules/agent/react_agent.mdx | 201 --
examples/agent/multi_document_agent.ts | 2 +-
examples/agent/openai.ts | 92 +-
examples/agent/react_agent.ts | 8 +-
examples/agent/step_wise_openai.ts | 19 +-
examples/agent/step_wise_query_tool.ts | 30 +-
examples/agent/step_wise_react.ts | 25 +-
examples/package.json | 5 +-
examples/readers/package.json | 2 +-
examples/recipes/cost-analysis.ts | 6 +-
examples/tsconfig.json | 3 +-
packages/core/.gitignore | 2 +-
packages/core/e2e/fixtures/llm/open_ai.ts | 15 +-
packages/core/e2e/node/claude.e2e.ts | 16 +-
packages/core/e2e/node/fixtures/data/Alex.txt | 2 +
packages/core/e2e/node/fixtures/tools.ts | 21 +
packages/core/e2e/node/openai.e2e.ts | 132 +-
packages/core/e2e/node/react.e2e.ts | 29 +
.../core/e2e/node/snapshot/agent_stream.snap | 2164 +----------------
.../snapshot/agent_with_object_retriever.snap | 103 +
.../snapshot/openai_agent_system_prompt.snap | 83 +
.../core/e2e/node/snapshot/react-agent.snap | 63 +
packages/core/e2e/node/utils.ts | 19 +-
packages/core/src/agent/README.md | 73 +
packages/core/src/agent/anthropic.ts | 231 +-
packages/core/src/agent/base.ts | 411 ++++
packages/core/src/agent/index.ts | 25 +-
packages/core/src/agent/openai.ts | 194 ++
packages/core/src/agent/openai/base.ts | 80 -
packages/core/src/agent/openai/types/chat.ts | 13 -
packages/core/src/agent/openai/worker.ts | 397 ---
packages/core/src/agent/react.ts | 397 +++
packages/core/src/agent/react/base.ts | 48 -
packages/core/src/agent/react/formatter.ts | 84 -
packages/core/src/agent/react/outputParser.ts | 105 -
packages/core/src/agent/react/types.ts | 91 -
packages/core/src/agent/react/worker.ts | 325 ---
packages/core/src/agent/runner/base.ts | 384 ---
packages/core/src/agent/runner/types.ts | 106 -
packages/core/src/agent/type.ts | 4 +
packages/core/src/agent/types.ts | 202 --
packages/core/src/agent/utils.ts | 131 +-
.../core/src/callbacks/CallbackManager.ts | 10 +-
packages/core/src/engines/chat/types.ts | 47 -
.../core/src/internal/context/EventCaller.ts | 12 +-
.../prompts.ts => internal/prompt/react.ts} | 29 +-
packages/core/src/internal/type.ts | 5 +
packages/core/src/internal/utils.ts | 6 +-
packages/core/src/llm/types.ts | 30 +-
packages/core/src/objects/base.ts | 9 +-
packages/core/src/tools/index.ts | 2 -
packages/core/src/tools/types.ts | 22 -
packages/core/src/tools/utils.ts | 33 -
packages/core/src/types.ts | 7 +
.../tests/agent/runner/AgentRunner.test.ts | 85 -
pnpm-lock.yaml | 96 +-
tsconfig.json | 2 +-
63 files changed, 2038 insertions(+), 5413 deletions(-)
delete mode 100644 apps/docs/docs/modules/agent/multi_document_agent.mdx
delete mode 100644 apps/docs/docs/modules/agent/openai.mdx
delete mode 100644 apps/docs/docs/modules/agent/query_engine_tool.mdx
delete mode 100644 apps/docs/docs/modules/agent/react_agent.mdx
create mode 100644 packages/core/e2e/node/fixtures/data/Alex.txt
create mode 100644 packages/core/e2e/node/react.e2e.ts
create mode 100644 packages/core/e2e/node/snapshot/agent_with_object_retriever.snap
create mode 100644 packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap
create mode 100644 packages/core/e2e/node/snapshot/react-agent.snap
create mode 100644 packages/core/src/agent/README.md
create mode 100644 packages/core/src/agent/base.ts
create mode 100644 packages/core/src/agent/openai.ts
delete mode 100644 packages/core/src/agent/openai/base.ts
delete mode 100644 packages/core/src/agent/openai/types/chat.ts
delete mode 100644 packages/core/src/agent/openai/worker.ts
create mode 100644 packages/core/src/agent/react.ts
delete mode 100644 packages/core/src/agent/react/base.ts
delete mode 100644 packages/core/src/agent/react/formatter.ts
delete mode 100644 packages/core/src/agent/react/outputParser.ts
delete mode 100644 packages/core/src/agent/react/types.ts
delete mode 100644 packages/core/src/agent/react/worker.ts
delete mode 100644 packages/core/src/agent/runner/base.ts
delete mode 100644 packages/core/src/agent/runner/types.ts
create mode 100644 packages/core/src/agent/type.ts
delete mode 100644 packages/core/src/agent/types.ts
rename packages/core/src/{agent/react/prompts.ts => internal/prompt/react.ts} (68%)
create mode 100644 packages/core/src/internal/type.ts
delete mode 100644 packages/core/src/tools/types.ts
delete mode 100644 packages/core/src/tools/utils.ts
delete mode 100644 packages/core/tests/agent/runner/AgentRunner.test.ts
diff --git a/.npmrc b/.npmrc
index b151845bf..5da1bc1c6 100644
--- a/.npmrc
+++ b/.npmrc
@@ -1,3 +1,4 @@
auto-install-peers = true
enable-pre-post-scripts = true
-prefer-workspace-packages: true
+prefer-workspace-packages = true
+save-workspace-protocol = true
diff --git a/apps/docs/docs/examples/agent.mdx b/apps/docs/docs/examples/agent.mdx
index 2b37e9cce..08ce623b5 100644
--- a/apps/docs/docs/examples/agent.mdx
+++ b/apps/docs/docs/examples/agent.mdx
@@ -4,81 +4,7 @@ A built-in agent that can take decisions and reasoning based on the tools provid
## OpenAI Agent
-```ts
-import { FunctionTool, OpenAIAgent } from "llamaindex";
+import CodeBlock from "@theme/CodeBlock";
+import CodeSource from "!raw-loader!../../../../examples/agent/openai";
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-// Define the parameters of the divide function as a JSON schema
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend to divide",
- },
- b: {
- type: "number",
- description: "The divisor to divide by",
- },
- },
- required: ["a", "b"],
-};
-
-async function main() {
- // Create a function tool from the sum function
- const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers"
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
+<CodeBlock language="ts">{CodeSource}</CodeBlock>
diff --git a/apps/docs/docs/modules/agent/index.md b/apps/docs/docs/modules/agent/index.md
index 99e2d1f4a..65a3f8cd4 100644
--- a/apps/docs/docs/modules/agent/index.md
+++ b/apps/docs/docs/modules/agent/index.md
@@ -11,4 +11,10 @@ An “agent†is an automated reasoning and decision engine. It takes in a user
LlamaIndex.TS comes with a few built-in agents, but you can also create your own. The built-in agents include:
-- [OpenAI Agent](./openai.mdx)
+- OpenAI Agent
+- Anthropic Agent
+- ReACT Agent
+
+## Examples
+
+- [OpenAI Agent](../../examples/agent.mdx)
diff --git a/apps/docs/docs/modules/agent/multi_document_agent.mdx b/apps/docs/docs/modules/agent/multi_document_agent.mdx
deleted file mode 100644
index f51f459ff..000000000
--- a/apps/docs/docs/modules/agent/multi_document_agent.mdx
+++ /dev/null
@@ -1,307 +0,0 @@
-# Multi-Document Agent
-
-In this guide, you learn towards setting up an agent that can effectively answer different types of questions over a larger set of documents.
-
-These questions include the following
-
-- QA over a specific doc
-- QA comparing different docs
-- Summaries over a specific doc
-- Comparing summaries between different docs
-
-We do this with the following architecture:
-
-- setup a “document agent†over each Document: each doc agent can do QA/summarization within its doc
-- setup a top-level agent over this set of document agents. Do tool retrieval and then do CoT over the set of tools to answer a question.
-
-## Setup and Download Data
-
-We first start by installing the necessary libraries and downloading the data.
-
-```bash
-pnpm i llamaindex
-```
-
-```ts
-import {
- Document,
- ObjectIndex,
- OpenAI,
- OpenAIAgent,
- QueryEngineTool,
- SimpleNodeParser,
- SimpleToolNodeMapping,
- SummaryIndex,
- VectorStoreIndex,
- Settings,
- storageContextFromDefaults,
-} from "llamaindex";
-```
-
-And then for the data we will run through a list of countries and download the wikipedia page for each country.
-
-```ts
-import fs from "fs";
-import path from "path";
-
-const dataPath = path.join(__dirname, "tmp_data");
-
-const extractWikipediaTitle = async (title: string) => {
- const fileExists = fs.existsSync(path.join(dataPath, `${title}.txt`));
-
- if (fileExists) {
- console.log(`File already exists for the title: ${title}`);
- return;
- }
-
- const queryParams = new URLSearchParams({
- action: "query",
- format: "json",
- titles: title,
- prop: "extracts",
- explaintext: "true",
- });
-
- const url = `https://en.wikipedia.org/w/api.php?${queryParams}`;
-
- const response = await fetch(url);
- const data: any = await response.json();
-
- const pages = data.query.pages;
- const page = pages[Object.keys(pages)[0]];
- const wikiText = page.extract;
-
- await new Promise((resolve) => {
- fs.writeFile(path.join(dataPath, `${title}.txt`), wikiText, (err: any) => {
- if (err) {
- console.error(err);
- resolve(title);
- return;
- }
- console.log(`${title} stored in file!`);
-
- resolve(title);
- });
- });
-};
-```
-
-```ts
-export const extractWikipedia = async (titles: string[]) => {
- if (!fs.existsSync(dataPath)) {
- fs.mkdirSync(dataPath);
- }
-
- for await (const title of titles) {
- await extractWikipediaTitle(title);
- }
-
- console.log("Extration finished!");
-```
-
-These files will be saved in the `tmp_data` folder.
-
-Now we can call the function to download the data for each country.
-
-```ts
-await extractWikipedia([
- "Brazil",
- "United States",
- "Canada",
- "Mexico",
- "Argentina",
- "Chile",
- "Colombia",
- "Peru",
- "Venezuela",
- "Ecuador",
- "Bolivia",
- "Paraguay",
- "Uruguay",
- "Guyana",
- "Suriname",
- "French Guiana",
- "Falkland Islands",
-]);
-```
-
-## Load the data
-
-Now that we have the data, we can load it into the LlamaIndex and store as a document.
-
-```ts
-import { Document } from "llamaindex";
-
-const countryDocs: Record<string, Document> = {};
-
-for (const title of wikiTitles) {
- const path = `./agent/helpers/tmp_data/${title}.txt`;
- const text = await fs.readFile(path, "utf-8");
- const document = new Document({ text: text, id_: path });
- countryDocs[title] = document;
-}
-```
-
-## Setup LLM and StorageContext
-
-We will be using gpt-4 for this example and we will use the `StorageContext` to store the documents in-memory.
-
-```ts
-Settings.llm = new OpenAI({
- model: "gpt-4",
-});
-
-const storageContext = await storageContextFromDefaults({
- persistDir: "./storage",
-});
-```
-
-## Building Multi-Document Agents
-
-In this section we show you how to construct the multi-document agent. We first build a document agent for each document, and then define the top-level parent agent with an object index.
-
-```ts
-const documentAgents: Record<string, any> = {};
-const queryEngines: Record<string, any> = {};
-```
-
-Now we iterate over each country and create a document agent for each one.
-
-### Build Agent for each Document
-
-In this section we define “document agents†for each document.
-
-We define both a vector index (for semantic search) and summary index (for summarization) for each document. The two query engines are then converted into tools that are passed to an OpenAI function calling agent.
-
-This document agent can dynamically choose to perform semantic search or summarization within a given document.
-
-We create a separate document agent for each coutnry.
-
-```ts
-for (const title of wikiTitles) {
- // parse the document into nodes
- const nodes = new SimpleNodeParser({
- chunkSize: 200,
- chunkOverlap: 20,
- }).getNodesFromDocuments([countryDocs[title]]);
-
- // create the vector index for specific search
- const vectorIndex = await VectorStoreIndex.init({
- storageContext: storageContext,
- nodes,
- });
-
- // create the summary index for broader search
- const summaryIndex = await SummaryIndex.init({
- nodes,
- });
-
- const vectorQueryEngine = summaryIndex.asQueryEngine();
- const summaryQueryEngine = summaryIndex.asQueryEngine();
-
- // create the query engines for each task
- const queryEngineTools = [
- new QueryEngineTool({
- queryEngine: vectorQueryEngine,
- metadata: {
- name: "vector_tool",
- description: `Useful for questions related to specific aspects of ${title} (e.g. the history, arts and culture, sports, demographics, or more).`,
- },
- }),
- new QueryEngineTool({
- queryEngine: summaryQueryEngine,
- metadata: {
- name: "summary_tool",
- description: `Useful for any requests that require a holistic summary of EVERYTHING about ${title}. For questions about more specific sections, please use the vector_tool.`,
- },
- }),
- ];
-
- // create the document agent
- const agent = new OpenAIAgent({
- tools: queryEngineTools,
- llm,
- });
-
- documentAgents[title] = agent;
- queryEngines[title] = vectorIndex.asQueryEngine();
-}
-```
-
-## Build Top-Level Agent
-
-Now we define the top-level agent that can answer questions over the set of document agents.
-
-This agent takes in all document agents as tools. This specific agent RetrieverOpenAIAgent performs tool retrieval before tool use (unlike a default agent that tries to put all tools in the prompt).
-
-Here we use a top-k retriever, but we encourage you to customize the tool retriever method!
-
-Firstly, we create a tool for each document agent
-
-```ts
-const allTools: QueryEngineTool[] = [];
-```
-
-```ts
-for (const title of wikiTitles) {
- const wikiSummary = `
- This content contains Wikipedia articles about ${title}.
- Use this tool if you want to answer any questions about ${title}
- `;
-
- const docTool = new QueryEngineTool({
- queryEngine: documentAgents[title],
- metadata: {
- name: `tool_${title}`,
- description: wikiSummary,
- },
- });
-
- allTools.push(docTool);
-}
-```
-
-Our top level agent will use this document agents as tools and use toolRetriever to retrieve the best tool to answer a question.
-
-```ts
-// map the tools to nodes
-const toolMapping = SimpleToolNodeMapping.fromObjects(allTools);
-
-// create the object index
-const objectIndex = await ObjectIndex.fromObjects(
- allTools,
- toolMapping,
- VectorStoreIndex,
- {
- storageContext,
- },
-);
-
-// create the top agent
-const topAgent = new OpenAIAgent({
- toolRetriever: await objectIndex.asRetriever({}),
- llm,
- prefixMessages: [
- {
- content:
- "You are an agent designed to answer queries about a set of given countries. Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
- role: "system",
- },
- ],
-});
-```
-
-## Use the Agent
-
-Now we can use the agent to answer questions.
-
-```ts
-const response = await topAgent.chat({
- message: "Tell me the differences between Brazil and Canada economics?",
-});
-
-// print output
-console.log(response);
-```
-
-You can find the full code for this example [here](https://github.com/run-llama/LlamaIndexTS/tree/main/examples/agent/multi-document-agent.ts)
diff --git a/apps/docs/docs/modules/agent/openai.mdx b/apps/docs/docs/modules/agent/openai.mdx
deleted file mode 100644
index 7b47dd78e..000000000
--- a/apps/docs/docs/modules/agent/openai.mdx
+++ /dev/null
@@ -1,185 +0,0 @@
----
-sidebar_position: 0
----
-
-# OpenAI Agent
-
-OpenAI API that supports function calling, it’s never been easier to build your own agent!
-
-In this notebook tutorial, we showcase how to write your own OpenAI agent
-
-## Setup
-
-First, you need to install the `llamaindex` package. You can do this by running the following command in your terminal:
-
-```bash
-pnpm i llamaindex
-```
-
-Then we can define a function to sum two numbers and another function to divide two numbers.
-
-```ts
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-```
-
-## Create a function tool
-
-Now we can create a function tool from the sum function and another function tool from the divide function.
-
-For the parameters of the sum function, we can define a JSON schema.
-
-### JSON Schema
-
-```ts
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend a to divide",
- },
- b: {
- type: "number",
- description: "The divisor b to divide by",
- },
- },
- required: ["a", "b"],
-};
-
-const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
-});
-
-const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
-});
-```
-
-## Create an OpenAIAgent
-
-Now we can create an OpenAIAgent with the function tools.
-
-```ts
-const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
-});
-```
-
-## Chat with the agent
-
-Now we can chat with the agent.
-
-```ts
-const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
-});
-
-console.log(String(response));
-```
-
-## Full code
-
-```ts
-import { FunctionTool, OpenAIAgent } from "llamaindex";
-
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-// Define the parameters of the divide function as a JSON schema
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The argument a to divide",
- },
- b: {
- type: "number",
- description: "The argument b to divide",
- },
- },
- required: ["a", "b"],
-};
-
-async function main() {
- // Create a function tool from the sum function
- const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
diff --git a/apps/docs/docs/modules/agent/query_engine_tool.mdx b/apps/docs/docs/modules/agent/query_engine_tool.mdx
deleted file mode 100644
index 4b9a3c399..000000000
--- a/apps/docs/docs/modules/agent/query_engine_tool.mdx
+++ /dev/null
@@ -1,130 +0,0 @@
----
-sidebar_position: 1
----
-
-# OpenAI Agent + QueryEngineTool
-
-QueryEngineTool is a tool that allows you to query a vector index. In this example, we will create a vector index from a set of documents and then create a QueryEngineTool from the vector index. We will then create an OpenAIAgent with the QueryEngineTool and chat with the agent.
-
-## Setup
-
-First, you need to install the `llamaindex` package. You can do this by running the following command in your terminal:
-
-```bash
-pnpm i llamaindex
-```
-
-Then you can import the necessary classes and functions.
-
-```ts
-import {
- OpenAIAgent,
- SimpleDirectoryReader,
- VectorStoreIndex,
- QueryEngineTool,
-} from "llamaindex";
-```
-
-## Create a vector index
-
-Now we can create a vector index from a set of documents.
-
-```ts
-// Load the documents
-const documents = await new SimpleDirectoryReader().loadData({
- directoryPath: "node_modules/llamaindex/examples/",
-});
-
-// Create a vector index from the documents
-const vectorIndex = await VectorStoreIndex.fromDocuments(documents);
-```
-
-## Create a QueryEngineTool
-
-Now we can create a QueryEngineTool from the vector index.
-
-```ts
-// Create a query engine from the vector index
-const abramovQueryEngine = vectorIndex.asQueryEngine();
-
-// Create a QueryEngineTool with the query engine
-const queryEngineTool = new QueryEngineTool({
- queryEngine: abramovQueryEngine,
- metadata: {
- name: "abramov_query_engine",
- description: "A query engine for the Abramov documents",
- },
-});
-```
-
-## Create an OpenAIAgent
-
-```ts
-// Create an OpenAIAgent with the query engine tool tools
-
-const agent = new OpenAIAgent({
- tools: [queryEngineTool],
-});
-```
-
-## Chat with the agent
-
-Now we can chat with the agent.
-
-```ts
-const response = await agent.chat({
- message: "What was his salary?",
-});
-
-console.log(String(response));
-```
-
-## Full code
-
-```ts
-import {
- OpenAIAgent,
- SimpleDirectoryReader,
- VectorStoreIndex,
- QueryEngineTool,
-} from "llamaindex";
-
-async function main() {
- // Load the documents
- const documents = await new SimpleDirectoryReader().loadData({
- directoryPath: "node_modules/llamaindex/examples/",
- });
-
- // Create a vector index from the documents
- const vectorIndex = await VectorStoreIndex.fromDocuments(documents);
-
- // Create a query engine from the vector index
- const abramovQueryEngine = vectorIndex.asQueryEngine();
-
- // Create a QueryEngineTool with the query engine
- const queryEngineTool = new QueryEngineTool({
- queryEngine: abramovQueryEngine,
- metadata: {
- name: "abramov_query_engine",
- description: "A query engine for the Abramov documents",
- },
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [queryEngineTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "What was his salary?",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
diff --git a/apps/docs/docs/modules/agent/react_agent.mdx b/apps/docs/docs/modules/agent/react_agent.mdx
deleted file mode 100644
index ba31674f0..000000000
--- a/apps/docs/docs/modules/agent/react_agent.mdx
+++ /dev/null
@@ -1,201 +0,0 @@
-# ReAct Agent
-
-The ReAct agent is an AI agent that can reason over the next action, construct an action command, execute the action, and repeat these steps in an iterative loop until the task is complete.
-
-In this notebook tutorial, we showcase how to write your ReAct agent using the `llamaindex` package.
-
-## Setup
-
-First, you need to install the `llamaindex` package. You can do this by running the following command in your terminal:
-
-```bash
-pnpm i llamaindex
-```
-
-And then you can import the `OpenAIAgent` and `FunctionTool` from the `llamaindex` package.
-
-```ts
-import { FunctionTool, OpenAIAgent } from "llamaindex";
-```
-
-Then we can define a function to sum two numbers and another function to divide two numbers.
-
-```ts
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-```
-
-## Create a function tool
-
-Now we can create a function tool from the sum function and another function tool from the divide function.
-
-For the parameters of the sum function, we can define a JSON schema.
-
-### JSON Schema
-
-```ts
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend a to divide",
- },
- b: {
- type: "number",
- description: "The divisor b to divide by",
- },
- },
- required: ["a", "b"],
-};
-
-const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
-});
-
-const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
-});
-```
-
-## Create an ReAct
-
-Now we can create an OpenAIAgent with the function tools.
-
-```ts
-const agent = new ReActAgent({
- tools: [sumFunctionTool, divideFunctionTool],
-});
-```
-
-## Chat with the agent
-
-Now we can chat with the agent.
-
-```ts
-const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
-});
-
-console.log(String(response));
-```
-
-The output will be:
-
-```bash
-Thought: I need to use a tool to help me answer the question.
-Action: sumNumbers
-Action Input: {"a":5,"b":5}
-
-Observation: 10
-Thought: I can answer without using any more tools.
-Answer: The sum of 5 and 5 is 10, and when divided by 2, the result is 5.
-
-The sum of 5 and 5 is 10, and when divided by 2, the result is 5.
-```
-
-## Full code
-
-```ts
-import { FunctionTool, ReActAgent } from "llamaindex";
-
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-// Define the parameters of the divide function as a JSON schema
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The argument a to divide",
- },
- b: {
- type: "number",
- description: "The argument b to divide",
- },
- },
- required: ["a", "b"],
-};
-
-async function main() {
- // Create a function tool from the sum function
- const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "I want to sum 5 and 5 and then divide by 2",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
diff --git a/examples/agent/multi_document_agent.ts b/examples/agent/multi_document_agent.ts
index d4d0407ad..8a7861a15 100644
--- a/examples/agent/multi_document_agent.ts
+++ b/examples/agent/multi_document_agent.ts
@@ -125,7 +125,7 @@ async function main() {
const topAgent = new OpenAIAgent({
toolRetriever: await objectIndex.asRetriever({}),
llm: new OpenAI({ model: "gpt-4" }),
- prefixMessages: [
+ chatHistory: [
{
content:
"You are an agent designed to answer queries about a set of given countries. Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
diff --git a/examples/agent/openai.ts b/examples/agent/openai.ts
index 3bc65c2b3..857008054 100644
--- a/examples/agent/openai.ts
+++ b/examples/agent/openai.ts
@@ -1,72 +1,58 @@
import { FunctionTool, OpenAIAgent } from "llamaindex";
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }) {
- return `${a + b}`;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }) {
- return `${a / b}`;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
+const sumNumbers = FunctionTool.from(
+ ({ a, b }: { a: number; b: number }) => `${a + b}`,
+ {
+ name: "sumNumbers",
+ description: "Use this function to sum two numbers",
+ parameters: {
+ type: "object",
+ properties: {
+ a: {
+ type: "number",
+ description: "The first number",
+ },
+ b: {
+ type: "number",
+ description: "The second number",
+ },
+ },
+ required: ["a", "b"],
},
},
- required: ["a", "b"],
-} as const;
+);
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend a to divide",
- },
- b: {
- type: "number",
- description: "The divisor b to divide by",
+const divideNumbers = FunctionTool.from(
+ ({ a, b }: { a: number; b: number }) => `${a / b}`,
+ {
+ name: "divideNumbers",
+ description: "Use this function to divide two numbers",
+ parameters: {
+ type: "object",
+ properties: {
+ a: {
+ type: "number",
+ description: "The dividend a to divide",
+ },
+ b: {
+ type: "number",
+ description: "The divisor b to divide by",
+ },
+ },
+ required: ["a", "b"],
},
},
- required: ["a", "b"],
-} as const;
+);
async function main() {
- // Create a function tool from the sum function
- const functionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const functionTool2 = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
const agent = new OpenAIAgent({
- tools: [functionTool, functionTool2],
+ tools: [sumNumbers, divideNumbers],
});
- // Chat with the agent
const response = await agent.chat({
message: "How much is 5 + 5? then divide by 2",
});
- // Print the response
console.log(String(response));
}
diff --git a/examples/agent/react_agent.ts b/examples/agent/react_agent.ts
index d86fe4707..6171c35f8 100644
--- a/examples/agent/react_agent.ts
+++ b/examples/agent/react_agent.ts
@@ -1,4 +1,4 @@
-import { Anthropic, FunctionTool, ReActAgent } from "llamaindex";
+import { Anthropic, FunctionTool, ReACTAgent } from "llamaindex";
// Define a function to sum two numbers
function sumNumbers({ a, b }: { a: number; b: number }) {
@@ -62,18 +62,18 @@ async function main() {
});
// Create an ReActAgent with the function tools
- const agent = new ReActAgent({
+ const agent = new ReACTAgent({
llm: anthropic,
tools: [functionTool, functionTool2],
});
// Chat with the agent
- const response = await agent.chat({
+ const { response } = await agent.chat({
message: "Divide 16 by 2 then add 20",
});
// Print the response
- console.log(String(response));
+ console.log(response.message);
}
void main().then(() => {
diff --git a/examples/agent/step_wise_openai.ts b/examples/agent/step_wise_openai.ts
index 08b20509b..d487f74a3 100644
--- a/examples/agent/step_wise_openai.ts
+++ b/examples/agent/step_wise_openai.ts
@@ -62,29 +62,18 @@ async function main() {
});
// Create a task to sum and divide numbers
- const task = agent.createTask("How much is 5 + 5? then divide by 2");
+ const task = await agent.createTask("How much is 5 + 5? then divide by 2");
let count = 0;
- while (true) {
- const stepOutput = await agent.runStep(task.taskId);
-
+ for await (const stepOutput of task) {
console.log(`Runnning step ${count++}`);
console.log(`======== OUTPUT ==========`);
- if (stepOutput.output.response) {
- console.log(stepOutput.output.response);
- } else {
- console.log(stepOutput.output.sources);
- }
+ console.log(stepOutput.output.message.content);
console.log(`==========================`);
if (stepOutput.isLast) {
- const finalResponse = await agent.finalizeResponse(
- task.taskId,
- stepOutput,
- );
- console.log({ finalResponse });
- break;
+ console.log(stepOutput.output.message.content);
}
}
}
diff --git a/examples/agent/step_wise_query_tool.ts b/examples/agent/step_wise_query_tool.ts
index b098f1a4e..92a5ebe0f 100644
--- a/examples/agent/step_wise_query_tool.ts
+++ b/examples/agent/step_wise_query_tool.ts
@@ -31,31 +31,11 @@ async function main() {
tools: [queryEngineTool],
});
- const task = agent.createTask("What was his salary?");
-
- let count = 0;
-
- while (true) {
- const stepOutput = await agent.runStep(task.taskId);
-
- console.log(`Runnning step ${count++}`);
- console.log(`======== OUTPUT ==========`);
- if (stepOutput.output.response) {
- console.log(stepOutput.output.response);
- } else {
- console.log(stepOutput.output.sources);
- }
- console.log(`==========================`);
-
- if (stepOutput.isLast) {
- const finalResponse = await agent.finalizeResponse(
- task.taskId,
- stepOutput,
- );
- console.log({ finalResponse });
- break;
- }
- }
+ const { response } = await agent.chat({
+ message: "What was his salary?",
+ });
+
+ console.log(response.message.content);
}
void main().then(() => {
diff --git a/examples/agent/step_wise_react.ts b/examples/agent/step_wise_react.ts
index 185e52821..97bd88e83 100644
--- a/examples/agent/step_wise_react.ts
+++ b/examples/agent/step_wise_react.ts
@@ -1,4 +1,4 @@
-import { FunctionTool, ReActAgent } from "llamaindex";
+import { Anthropic, FunctionTool, ReACTAgent } from "llamaindex";
// Define a function to sum two numbers
function sumNumbers({ a, b }: { a: number; b: number }) {
@@ -7,6 +7,7 @@ function sumNumbers({ a, b }: { a: number; b: number }) {
// Define a function to divide two numbers
function divideNumbers({ a, b }: { a: number; b: number }) {
+ console.log("get input", a, b);
return `${a / b}`;
}
@@ -57,30 +58,22 @@ async function main() {
});
// Create an OpenAIAgent with the function tools
- const agent = new ReActAgent({
+ const agent = new ReACTAgent({
+ llm: new Anthropic({
+ model: "claude-3-opus",
+ }),
tools: [functionTool, functionTool2],
});
- const task = agent.createTask("Divide 16 by 2 then add 20");
+ const task = await agent.createTask("Divide 16 by 2 then add 20");
let count = 0;
- while (true) {
- const stepOutput = await agent.runStep(task.taskId);
-
+ for await (const stepOutput of task) {
console.log(`Runnning step ${count++}`);
console.log(`======== OUTPUT ==========`);
- console.log(stepOutput.output);
+ console.log(stepOutput);
console.log(`==========================`);
-
- if (stepOutput.isLast) {
- const finalResponse = await agent.finalizeResponse(
- task.taskId,
- stepOutput,
- );
- console.log({ finalResponse });
- break;
- }
}
}
diff --git a/examples/package.json b/examples/package.json
index eb4dd3a1a..3403f7b13 100644
--- a/examples/package.json
+++ b/examples/package.json
@@ -12,7 +12,7 @@
"commander": "^11.1.0",
"dotenv": "^16.4.5",
"js-tiktoken": "^1.0.11",
- "llamaindex": "latest",
+ "llamaindex": "*",
"mongodb": "^6.5.0",
"pathe": "^1.1.2"
},
@@ -24,5 +24,8 @@
},
"scripts": {
"lint": "eslint ."
+ },
+ "stackblitz": {
+ "startCommand": "npm start"
}
}
diff --git a/examples/readers/package.json b/examples/readers/package.json
index fd5e7aee7..8202eb59c 100644
--- a/examples/readers/package.json
+++ b/examples/readers/package.json
@@ -12,7 +12,7 @@
"start:llamaparse": "node --loader ts-node/esm ./src/llamaparse.ts"
},
"dependencies": {
- "llamaindex": "latest"
+ "llamaindex": "*"
},
"devDependencies": {
"@types/node": "^20.12.7",
diff --git a/examples/recipes/cost-analysis.ts b/examples/recipes/cost-analysis.ts
index 79b162c3b..37b4aef36 100644
--- a/examples/recipes/cost-analysis.ts
+++ b/examples/recipes/cost-analysis.ts
@@ -1,5 +1,5 @@
import { encodingForModel } from "js-tiktoken";
-import { OpenAI } from "llamaindex";
+import { ChatMessage, OpenAI, type LLMStartEvent } from "llamaindex";
import { Settings } from "llamaindex/Settings";
import { extractText } from "llamaindex/llm/utils";
@@ -12,9 +12,9 @@ const llm = new OpenAI({
let tokenCount = 0;
-Settings.callbackManager.on("llm-start", (event) => {
+Settings.callbackManager.on("llm-start", (event: LLMStartEvent) => {
const { messages } = event.detail.payload;
- tokenCount += messages.reduce((count, message) => {
+ messages.reduce((count: number, message: ChatMessage) => {
return count + encoding.encode(extractText(message.content)).length;
}, 0);
console.log("Token count:", tokenCount);
diff --git a/examples/tsconfig.json b/examples/tsconfig.json
index 722089b2e..b87cae708 100644
--- a/examples/tsconfig.json
+++ b/examples/tsconfig.json
@@ -1,12 +1,13 @@
{
"compilerOptions": {
- "target": "es2017",
+ "target": "ES2022",
"module": "esnext",
"moduleResolution": "bundler",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": true,
"skipLibCheck": true,
+ "lib": ["ES2022"],
"outDir": "./lib",
"tsBuildInfoFile": "./lib/.tsbuildinfo",
"incremental": true,
diff --git a/packages/core/.gitignore b/packages/core/.gitignore
index 1e2995d5d..485ee85c2 100644
--- a/packages/core/.gitignore
+++ b/packages/core/.gitignore
@@ -1,3 +1,3 @@
.turbo
-README.md
+/README.md
LICENSE
\ No newline at end of file
diff --git a/packages/core/e2e/fixtures/llm/open_ai.ts b/packages/core/e2e/fixtures/llm/open_ai.ts
index 2c1741b8c..76d314807 100644
--- a/packages/core/e2e/fixtures/llm/open_ai.ts
+++ b/packages/core/e2e/fixtures/llm/open_ai.ts
@@ -10,6 +10,7 @@ import type {
} from "llamaindex/llm/types";
import { extractText } from "llamaindex/llm/utils";
import { deepStrictEqual, strictEqual } from "node:assert";
+import { inspect } from "node:util";
import { llmCompleteMockStorage } from "../../node/utils.js";
export function getOpenAISession() {
@@ -46,10 +47,12 @@ export class OpenAI implements LLM {
if (llmCompleteMockStorage.llmEventStart.length > 0) {
const chatMessage =
llmCompleteMockStorage.llmEventStart.shift()!["messages"];
- strictEqual(chatMessage.length, params.messages.length);
+ console.log(inspect(params.messages, { depth: 1 }));
+ console.log(inspect(chatMessage, { depth: 1 }));
+ strictEqual(params.messages.length, chatMessage.length);
for (let i = 0; i < chatMessage.length; i++) {
- strictEqual(chatMessage[i].role, params.messages[i].role);
- deepStrictEqual(chatMessage[i].content, params.messages[i].content);
+ strictEqual(params.messages[i].role, chatMessage[i].role);
+ deepStrictEqual(params.messages[i].content, chatMessage[i].content);
}
if (llmCompleteMockStorage.llmEventEnd.length > 0) {
@@ -89,9 +92,9 @@ export class OpenAI implements LLM {
if (llmCompleteMockStorage.llmEventStart.length > 0) {
const chatMessage =
llmCompleteMockStorage.llmEventStart.shift()!["messages"];
- strictEqual(chatMessage.length, 1);
- strictEqual(chatMessage[0].role, "user");
- strictEqual(chatMessage[0].content, params.prompt);
+ strictEqual(1, chatMessage.length);
+ strictEqual("user", chatMessage[0].role);
+ strictEqual(params.prompt, chatMessage[0].content);
}
if (llmCompleteMockStorage.llmEventEnd.length > 0) {
const response = llmCompleteMockStorage.llmEventEnd.shift()!["response"];
diff --git a/packages/core/e2e/node/claude.e2e.ts b/packages/core/e2e/node/claude.e2e.ts
index c4e82a954..1afda5e5b 100644
--- a/packages/core/e2e/node/claude.e2e.ts
+++ b/packages/core/e2e/node/claude.e2e.ts
@@ -1,6 +1,7 @@
import { consola } from "consola";
import { Anthropic, FunctionTool, Settings, type LLM } from "llamaindex";
import { AnthropicAgent } from "llamaindex/agent/anthropic";
+import { extractText } from "llamaindex/llm/utils";
import { ok } from "node:assert";
import { beforeEach, test } from "node:test";
import { sumNumbersTool } from "./fixtures/tools.js";
@@ -70,12 +71,11 @@ await test("anthropic agent", async (t) => {
},
],
});
- const result = await agent.chat({
+ const { response } = await agent.chat({
message: "What is the weather in San Francisco?",
});
- consola.debug("response:", result.response);
- ok(typeof result.response === "string");
- ok(result.response.includes("35"));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("35"));
});
await t.test("async function", async () => {
@@ -111,8 +111,8 @@ await test("anthropic agent", async (t) => {
const { response } = await agent.chat({
message: "My name is Alex Yang. What is my unique id?",
});
- consola.debug("response:", response);
- ok(response.includes(uniqueId));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes(uniqueId));
});
await t.test("sum numbers", async () => {
@@ -120,10 +120,10 @@ await test("anthropic agent", async (t) => {
tools: [sumNumbersTool],
});
- const response = await openaiAgent.chat({
+ const { response } = await openaiAgent.chat({
message: "how much is 1 + 1?",
});
- ok(response.response.includes("2"));
+ ok(extractText(response.message.content).includes("2"));
});
});
diff --git a/packages/core/e2e/node/fixtures/data/Alex.txt b/packages/core/e2e/node/fixtures/data/Alex.txt
new file mode 100644
index 000000000..9aeae43f9
--- /dev/null
+++ b/packages/core/e2e/node/fixtures/data/Alex.txt
@@ -0,0 +1,2 @@
+Alex is a male.
+What's very important, Alex is not in the Brazil.
diff --git a/packages/core/e2e/node/fixtures/tools.ts b/packages/core/e2e/node/fixtures/tools.ts
index 6d523991a..0060415c9 100644
--- a/packages/core/e2e/node/fixtures/tools.ts
+++ b/packages/core/e2e/node/fixtures/tools.ts
@@ -45,3 +45,24 @@ export const divideNumbersTool = FunctionTool.from(divideNumbers, {
required: ["a", "b"],
},
});
+
+// should always return the 72 degrees
+export const getWeatherTool = FunctionTool.from(
+ async ({ city }: { city: string }) => {
+ return `The weather in ${city} is 72 degrees`;
+ },
+ {
+ name: "getWeather",
+ description: "Get the weather for a city",
+ parameters: {
+ type: "object",
+ properties: {
+ city: {
+ type: "string",
+ description: "The city to get the weather for",
+ },
+ },
+ required: ["city"],
+ },
+ },
+);
diff --git a/packages/core/e2e/node/openai.e2e.ts b/packages/core/e2e/node/openai.e2e.ts
index 06c78a309..4f853e538 100644
--- a/packages/core/e2e/node/openai.e2e.ts
+++ b/packages/core/e2e/node/openai.e2e.ts
@@ -2,18 +2,29 @@ import { consola } from "consola";
import {
Document,
FunctionTool,
+ ObjectIndex,
OpenAI,
OpenAIAgent,
QueryEngineTool,
Settings,
+ SimpleNodeParser,
+ SimpleToolNodeMapping,
SubQuestionQueryEngine,
+ SummaryIndex,
VectorStoreIndex,
type LLM,
} from "llamaindex";
+import { extractText } from "llamaindex/llm/utils";
import { ok, strictEqual } from "node:assert";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
import { beforeEach, test } from "node:test";
-import { divideNumbersTool, sumNumbersTool } from "./fixtures/tools.js";
-import { mockLLMEvent } from "./utils.js";
+import {
+ divideNumbersTool,
+ getWeatherTool,
+ sumNumbersTool,
+} from "./fixtures/tools.js";
+import { mockLLMEvent, testRootDir } from "./utils.js";
let llm: LLM;
beforeEach(async () => {
@@ -84,12 +95,107 @@ await test("gpt-4-turbo", async (t) => {
const { response } = await agent.chat({
message: "What is the weather in San Jose?",
});
- consola.debug("response:", response);
- ok(typeof response === "string");
- ok(response.includes("45"));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("45"));
+ });
+});
+
+await test("agent system prompt", async (t) => {
+ await mockLLMEvent(t, "openai_agent_system_prompt");
+ await t.test("chat", async (t) => {
+ const agent = new OpenAIAgent({
+ tools: [getWeatherTool],
+ systemPrompt:
+ "You are a pirate. You MUST speak every words staring with a 'Arhgs'",
+ });
+ const { response } = await agent.chat({
+ message: "What is the weather in San Francisco?",
+ });
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("72"));
+ ok(extractText(response.message.content).includes("Arhg"));
});
});
+await test("agent with object retriever", async (t) => {
+ await mockLLMEvent(t, "agent_with_object_retriever");
+
+ const alexInfoPath = join(testRootDir, "./fixtures/data/Alex.txt");
+ const alexInfoText = await readFile(alexInfoPath, "utf-8");
+ const alexDocument = new Document({ text: alexInfoText, id_: alexInfoPath });
+
+ const nodes = new SimpleNodeParser({
+ chunkSize: 200,
+ chunkOverlap: 20,
+ }).getNodesFromDocuments([alexDocument]);
+
+ const summaryIndex = await SummaryIndex.init({
+ nodes,
+ });
+
+ const summaryQueryEngine = summaryIndex.asQueryEngine();
+
+ const queryEngineTools = [
+ FunctionTool.from(
+ ({ query }: { query?: string }) => {
+ throw new Error("This tool should not be called");
+ },
+ {
+ name: "vector_tool",
+ description:
+ "This tool should not be called, never use this tool in any cases.",
+ parameters: {
+ type: "object",
+ properties: {
+ query: { type: "string", nullable: true },
+ },
+ },
+ },
+ ),
+ new QueryEngineTool({
+ queryEngine: summaryQueryEngine,
+ metadata: {
+ name: "summary_tool",
+ description: `Useful for any requests that short information about Alex.
+For questions about Alex, please use this tool.
+For questions about more specific sections, please use the vector_tool.`,
+ },
+ }),
+ ];
+
+ const originalCall = queryEngineTools[1].call.bind(queryEngineTools[1]);
+ const mockCall = t.mock.fn(({ query }: { query: string }) => {
+ return originalCall({ query });
+ });
+ queryEngineTools[1].call = mockCall;
+
+ const toolMapping = SimpleToolNodeMapping.fromObjects(queryEngineTools);
+
+ const objectIndex = await ObjectIndex.fromObjects(
+ queryEngineTools,
+ toolMapping,
+ VectorStoreIndex,
+ );
+
+ const toolRetriever = await objectIndex.asRetriever({});
+
+ const agent = new OpenAIAgent({
+ toolRetriever,
+ systemPrompt:
+ "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
+ });
+
+ strictEqual(mockCall.mock.callCount(), 0);
+ const { response } = await agent.chat({
+ message:
+ "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no.",
+ });
+ strictEqual(mockCall.mock.callCount(), 1);
+
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).toLowerCase().includes("no"));
+});
+
await test("agent", async (t) => {
await mockLLMEvent(t, "agent");
await t.test("chat", async () => {
@@ -113,12 +219,11 @@ await test("agent", async (t) => {
},
],
});
- const result = await agent.chat({
+ const { response } = await agent.chat({
message: "What is the weather in San Francisco?",
});
- consola.debug("response:", result.response);
- ok(typeof result.response === "string");
- ok(result.response.includes("35"));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("35"));
});
await t.test("async function", async () => {
@@ -154,8 +259,7 @@ await test("agent", async (t) => {
const { response } = await agent.chat({
message: "My name is Alex Yang. What is my unique id?",
});
- consola.debug("response:", response);
- ok(response.includes(uniqueId));
+ ok(extractText(response.message.content).includes(uniqueId));
});
await t.test("sum numbers", async () => {
@@ -163,11 +267,11 @@ await test("agent", async (t) => {
tools: [sumNumbersTool],
});
- const response = await openaiAgent.chat({
+ const { response } = await openaiAgent.chat({
message: "how much is 1 + 1?",
});
- ok(response.response.includes("2"));
+ ok(extractText(response.message.content).includes("2"));
});
});
@@ -189,7 +293,7 @@ await test("agent stream", async (t) => {
let message = "";
for await (const chunk of response) {
- message += chunk.response;
+ message += chunk.delta;
}
strictEqual(fn.mock.callCount(), 2);
diff --git a/packages/core/e2e/node/react.e2e.ts b/packages/core/e2e/node/react.e2e.ts
new file mode 100644
index 000000000..ecffb72a0
--- /dev/null
+++ b/packages/core/e2e/node/react.e2e.ts
@@ -0,0 +1,29 @@
+import { OpenAI, ReACTAgent, Settings, type LLM } from "llamaindex";
+import { extractText } from "llamaindex/llm/utils";
+import { ok } from "node:assert";
+import { beforeEach, test } from "node:test";
+import { getWeatherTool } from "./fixtures/tools.js";
+import { mockLLMEvent } from "./utils.js";
+
+let llm: LLM;
+beforeEach(async () => {
+ Settings.llm = new OpenAI({
+ model: "gpt-3.5-turbo",
+ });
+ llm = Settings.llm;
+});
+
+await test("react agent", async (t) => {
+ await mockLLMEvent(t, "react-agent");
+ await t.test("get weather", async () => {
+ const agent = new ReACTAgent({
+ tools: [getWeatherTool],
+ });
+ const { response } = await agent.chat({
+ stream: false,
+ message: "What is the weather like in San Francisco?",
+ });
+
+ ok(extractText(response.message.content).includes("72"));
+ });
+});
diff --git a/packages/core/e2e/node/snapshot/agent_stream.snap b/packages/core/e2e/node/snapshot/agent_stream.snap
index e9e9aa161..be29a6826 100644
--- a/packages/core/e2e/node/snapshot/agent_stream.snap
+++ b/packages/core/e2e/node/snapshot/agent_stream.snap
@@ -4,8 +4,8 @@
"id": "PRESERVE_0",
"messages": [
{
- "content": "Divide 16 by 2 then add 20",
- "role": "user"
+ "role": "user",
+ "content": "Divide 16 by 2 then add 20"
}
]
},
@@ -13,78 +13,50 @@
"id": "PRESERVE_1",
"messages": [
{
- "content": "Divide 16 by 2 then add 20",
- "role": "user"
- },
- {
- "content": "",
- "role": "assistant",
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- }
- },
- {
- "content": "8",
"role": "user",
- "options": {
- "toolResult": {
- "id": "HIDDEN",
- "isError": false
- }
- }
- }
- ]
- },
- {
- "id": "PRESERVE_2",
- "messages": [
- {
- "content": "Divide 16 by 2 then add 20",
- "role": "user"
+ "content": "Divide 16 by 2 then add 20"
},
{
- "content": "",
"role": "assistant",
+ "content": "",
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
}
},
{
- "content": "8",
"role": "user",
+ "content": "8",
"options": {
"toolResult": {
- "id": "HIDDEN",
- "isError": false
+ "result": "8",
+ "isError": false,
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn"
}
}
},
{
- "content": "",
"role": "assistant",
+ "content": "",
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
+ "input": "{\"a\": 8, \"b\": 20}"
}
}
},
{
- "content": "28",
"role": "user",
+ "content": "28",
"options": {
"toolResult": {
- "id": "HIDDEN",
- "isError": false
+ "result": "28",
+ "isError": false,
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2"
}
}
}
@@ -95,361 +67,14 @@
{
"id": "PRESERVE_0",
"response": {
- "raw": [
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "divideNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ": 16,"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": " \"b\": "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "2}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": ": 8, "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "\"b\": 2"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "0}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- }
- ],
+ "raw": null,
"message": {
"content": "",
"role": "assistant",
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
}
@@ -459,749 +84,7 @@
{
"id": "PRESERVE_1",
"response": {
- "raw": [
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "role": "assistant",
- "content": null,
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "a"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "8"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ",\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "b"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "20"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- ],
- "message": {
- "content": "",
- "role": "assistant",
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- }
- }
- }
- },
- {
- "id": "PRESERVE_2",
- "response": {
- "raw": [
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "The"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "The"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " result"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " result"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " of"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " of"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " dividing"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " dividing"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "16"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "16"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " by"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " by"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "2"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "2"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " and"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " and"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " then"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " then"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " adding"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " adding"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "20"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "20"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " is"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " is"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "28"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "28"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "."
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "."
- }
- ],
+ "raw": null,
"message": {
"content": "The result of dividing 16 by 2 and then adding 20 is 28.",
"role": "assistant",
@@ -1214,37 +97,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "divideNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1254,34 +111,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1291,34 +125,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ": 16,"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1328,34 +139,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": " \"b\": "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1365,34 +153,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "2}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1402,37 +167,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1442,34 +181,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1479,34 +195,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": ": 8, "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1516,34 +209,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "\"b\": 2"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1553,34 +223,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "0}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1590,806 +237,143 @@
{
"id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "role": "assistant",
- "content": null,
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "a"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "8"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ",\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "b"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "20"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_2",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "The"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "The"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " result"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " result"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " of"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " of"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " dividing"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " dividing"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "16"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "16"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " by"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " by"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "2"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "2"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " and"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " and"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " then"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " then"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " adding"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " adding"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "20"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "20"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " is"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " is"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "28"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "28"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "."
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "."
}
diff --git a/packages/core/e2e/node/snapshot/agent_with_object_retriever.snap b/packages/core/e2e/node/snapshot/agent_with_object_retriever.snap
new file mode 100644
index 000000000..627dac39d
--- /dev/null
+++ b/packages/core/e2e/node/snapshot/agent_with_object_retriever.snap
@@ -0,0 +1,103 @@
+{
+ "llmEventStart": [
+ {
+ "id": "PRESERVE_0",
+ "messages": [
+ {
+ "content": "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no."
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_1",
+ "messages": [
+ {
+ "content": "Context information is below.\n---------------------\nAlex is a male. What's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Alex\nAnswer:",
+ "role": "user"
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_2",
+ "messages": [
+ {
+ "content": "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no."
+ },
+ {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_EVThrsiOylO0p6ZmGdsA31x9",
+ "name": "summary_tool",
+ "input": "{\"query\": \"Alex\"}"
+ }
+ }
+ },
+ {
+ "content": "Alex is not in Brazil.",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "result": "Alex is not in Brazil.",
+ "isError": false,
+ "id": "call_EVThrsiOylO0p6ZmGdsA31x9"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "llmEventEnd": [
+ {
+ "id": "PRESERVE_0",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_EVThrsiOylO0p6ZmGdsA31x9",
+ "name": "summary_tool",
+ "input": "{\"query\": \"Alex\"}"
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_1",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Alex is not in Brazil.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_2",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "No, Alex does not live in Brazil based on the brief information available.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ }
+ ],
+ "llmEventStream": []
+}
\ No newline at end of file
diff --git a/packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap b/packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap
new file mode 100644
index 000000000..eaa9dbecc
--- /dev/null
+++ b/packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap
@@ -0,0 +1,83 @@
+{
+ "llmEventStart": [
+ {
+ "id": "PRESERVE_0",
+ "messages": [
+ {
+ "content": "You are a pirate. You MUST speak every words staring with a 'Arhgs'",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What is the weather in San Francisco?"
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_1",
+ "messages": [
+ {
+ "content": "You are a pirate. You MUST speak every words staring with a 'Arhgs'",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What is the weather in San Francisco?"
+ },
+ {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_h4gSNrz7MhhkOod7W4WKZ1iZ",
+ "name": "getWeather",
+ "input": "{\"city\":\"San Francisco\"}"
+ }
+ }
+ },
+ {
+ "content": "The weather in San Francisco is 72 degrees",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "result": "The weather in San Francisco is 72 degrees",
+ "isError": false,
+ "id": "call_h4gSNrz7MhhkOod7W4WKZ1iZ"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "llmEventEnd": [
+ {
+ "id": "PRESERVE_0",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_h4gSNrz7MhhkOod7W4WKZ1iZ",
+ "name": "getWeather",
+ "input": "{\"city\":\"San Francisco\"}"
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_1",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Arhgs the weather in San Francisco be 72 degrees.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ }
+ ],
+ "llmEventStream": []
+}
\ No newline at end of file
diff --git a/packages/core/e2e/node/snapshot/react-agent.snap b/packages/core/e2e/node/snapshot/react-agent.snap
new file mode 100644
index 000000000..48884cc45
--- /dev/null
+++ b/packages/core/e2e/node/snapshot/react-agent.snap
@@ -0,0 +1,63 @@
+{
+ "llmEventStart": [
+ {
+ "id": "PRESERVE_0",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.\n\n## Tools\nYou have access to a wide variety of tools. You are responsible for using\nthe tools in any sequence you deem appropriate to complete the task at hand.\nThis may require breaking the task into subtasks and using different tools\nto complete each subtask.\n\nYou have access to the following tools:\n- getWeather: Get the weather for a city with schema: {\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"description\":\"The city to get the weather for\"}},\"required\":[\"city\"]}\n\n## Output Format\nTo answer the question, please use the following format.\n\n\"\"\"\nThought: I need to use a tool to help me answer the question.\nAction: tool name (one of getWeather) if using a tool.\nAction Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{\"input\": \"hello world\", \"num_beams\": 5}})\n\"\"\"\n\nPlease ALWAYS start with a Thought.\n\nPlease use a valid JSON format for the Action Input. Do NOT do this {{'input': 'hello world', 'num_beams': 5}}.\n\nIf this format is used, the user will respond in the following format:\n\n\"\"\"\"\nObservation: tool response\n\"\"\"\"\n\nYou should keep repeating the above format until you have enough information\nto answer the question without using any more tools. At that point, you MUST respond\nin the one of the following two formats:\n\n\"\"\"\"\nThought: I can answer without using any more tools.\nAnswer: [your answer here]\n\"\"\"\"\n\n\"\"\"\"\nThought: I cannot answer the question with the provided tools.\nAnswer: Sorry, I cannot answer your query.\n\"\"\"\"\n\n## Current Conversation\nBelow is the current conversation consisting of interleaving human and assistant messages."
+ },
+ {
+ "role": "user",
+ "content": "What is the weather like in San Francisco?"
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_1",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.\n\n## Tools\nYou have access to a wide variety of tools. You are responsible for using\nthe tools in any sequence you deem appropriate to complete the task at hand.\nThis may require breaking the task into subtasks and using different tools\nto complete each subtask.\n\nYou have access to the following tools:\n- getWeather: Get the weather for a city with schema: {\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"description\":\"The city to get the weather for\"}},\"required\":[\"city\"]}\n\n## Output Format\nTo answer the question, please use the following format.\n\n\"\"\"\nThought: I need to use a tool to help me answer the question.\nAction: tool name (one of getWeather) if using a tool.\nAction Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{\"input\": \"hello world\", \"num_beams\": 5}})\n\"\"\"\n\nPlease ALWAYS start with a Thought.\n\nPlease use a valid JSON format for the Action Input. Do NOT do this {{'input': 'hello world', 'num_beams': 5}}.\n\nIf this format is used, the user will respond in the following format:\n\n\"\"\"\"\nObservation: tool response\n\"\"\"\"\n\nYou should keep repeating the above format until you have enough information\nto answer the question without using any more tools. At that point, you MUST respond\nin the one of the following two formats:\n\n\"\"\"\"\nThought: I can answer without using any more tools.\nAnswer: [your answer here]\n\"\"\"\"\n\n\"\"\"\"\nThought: I cannot answer the question with the provided tools.\nAnswer: Sorry, I cannot answer your query.\n\"\"\"\"\n\n## Current Conversation\nBelow is the current conversation consisting of interleaving human and assistant messages."
+ },
+ {
+ "role": "user",
+ "content": "What is the weather like in San Francisco?"
+ },
+ {
+ "role": "assistant",
+ "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nInput: {\"city\":\"San Francisco\"}"
+ },
+ {
+ "role": "user",
+ "content": "Observation: The weather in San Francisco is 72 degrees"
+ }
+ ]
+ }
+ ],
+ "llmEventEnd": [
+ {
+ "id": "PRESERVE_0",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_1",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Thought: I can answer without using any more tools.\nAnswer: The weather in San Francisco is 72 degrees.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ }
+ ],
+ "llmEventStream": []
+}
\ No newline at end of file
diff --git a/packages/core/e2e/node/utils.ts b/packages/core/e2e/node/utils.ts
index 7b7c6916b..92d58b84a 100644
--- a/packages/core/e2e/node/utils.ts
+++ b/packages/core/e2e/node/utils.ts
@@ -50,6 +50,11 @@ export async function mockLLMEvent(
...event.detail.payload,
// @ts-expect-error id is not UUID, but it is fine for testing
id: idMap.get(event.detail.payload.id)!,
+ response: {
+ ...event.detail.payload.response,
+ // hide raw object since it might too big
+ raw: null,
+ },
});
}
@@ -58,6 +63,11 @@ export async function mockLLMEvent(
...event.detail.payload,
// @ts-expect-error id is not UUID, but it is fine for testing
id: idMap.get(event.detail.payload.id)!,
+ chunk: {
+ ...event.detail.payload.chunk,
+ // hide raw object since it might too big
+ raw: null,
+ },
});
}
@@ -92,14 +102,7 @@ export async function mockLLMEvent(
Settings.callbackManager.off("llm-start", captureLLMStart);
// eslint-disable-next-line turbo/no-undeclared-env-vars
if (process.env.UPDATE_SNAPSHOT === "1") {
- const data = JSON.stringify(newLLMCompleteMockStorage, null, 2)
- .replace(/"id": "(?!PRESERVE_).*"/g, '"id": "HIDDEN"')
- .replace(/"created": \d+/g, `"created": 114514`)
- .replace(
- /"system_fingerprint": ".*"/g,
- '"system_fingerprint": "HIDDEN"',
- )
- .replace(/"tool_call_id": ".*"/g, '"tool_call_id": "HIDDEN"');
+ const data = JSON.stringify(newLLMCompleteMockStorage, null, 2);
await writeFile(
join(testRootDir, "snapshot", `${snapshotName}.snap`),
data,
diff --git a/packages/core/src/agent/README.md b/packages/core/src/agent/README.md
new file mode 100644
index 000000000..8d21725f1
--- /dev/null
+++ b/packages/core/src/agent/README.md
@@ -0,0 +1,73 @@
+# Agent
+
+> This is an internal code design document for the agent API.
+>
+> APIs are not exactly the same as the final version, but it is a good reference for what we are going to do.
+
+Most of the agent logic is same with Python, but we have some changes to make it more suitable for JavaScript.
+
+> https://github.com/run-llama/llama_index/tree/6b97753dec4a9c33b16c63a8333ddba3f49ec40f/docs/docs/module_guides/deploying/agents
+
+## API Changes
+
+### Classes changes
+
+- Task: we don't have `Task` class in JS, we use `ReadableStream` instead.
+
+- TaskStep: this is the step for each task run, includes like the input, the context, etc. This class will be used in taskHandler.
+
+- TaskOutput: this is the output for each task run, includes like is last step, the output, etc.
+
+### taskHandler
+
+taskHandler is a function that takes a TaskStep and returns a TaskOutput.
+
+```typescript
+type TaskHandler = (step: TaskStep) => Promise<TaskOutput>;
+```
+
+### `createTask` to be AsyncGenerator
+
+We use async generator to create task, since it's more suitable for JavaScript.
+
+```typescript
+const agent = new MyAgent();
+const task = agent.createTask();
+for await (const taskOutput of task) {
+ // do something
+}
+```
+
+### `from_*` -> `from`
+
+In python, there is `from_tools`, `from_defaults`... etc.
+But in JS/TS, we normalize them to `from`, since we can do this way
+using [function overloads](https://www.typescriptlang.org/docs/handbook/2/functions.html#function-overloads) in
+TypeScript.
+
+```typescript
+class Agent {
+ from(tools: BaseTool[]): Agent;
+ from(defaults: Defaults): Agent;
+ from(toolsOrDefaults: BaseTool[] | Defaults): Agent {
+ // runtime check
+ }
+}
+```
+
+### No sync method for chat/query method
+
+Force all methods to be async, since the all LLMs returns Promise.
+
+### Cancelable
+
+Use `AbortController` to cancel the task.
+
+```typescript
+const controller = new AbortController();
+const task = agent.createTask({ signal: controller.signal });
+process.on("SIGINT", () => controller.abort());
+for await (const taskOutput of task) {
+ // do something
+}
+```
diff --git a/packages/core/src/agent/anthropic.ts b/packages/core/src/agent/anthropic.ts
index 393483492..2d05759e4 100644
--- a/packages/core/src/agent/anthropic.ts
+++ b/packages/core/src/agent/anthropic.ts
@@ -1,27 +1,22 @@
import { Settings } from "../Settings.js";
import {
- AgentChatResponse,
type ChatEngineParamsNonStreaming,
+ type ChatEngineParamsStreaming,
} from "../engines/chat/index.js";
-import { wrapEventCaller } from "../internal/context/EventCaller.js";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import { prettifyError } from "../internal/utils.js";
import { Anthropic } from "../llm/anthropic.js";
-import type {
- ChatMessage,
- ChatResponse,
- ToolCallLLMMessageOptions,
-} from "../llm/index.js";
-import { extractText } from "../llm/utils.js";
+import type { ToolCallLLMMessageOptions } from "../llm/index.js";
import { ObjectRetriever } from "../objects/index.js";
import type { BaseToolWithCall } from "../types.js";
+import {
+ AgentRunner,
+ AgentWorker,
+ type AgentChatResponse,
+ type AgentParamsBase,
+ type TaskHandler,
+} from "./base.js";
+import { callTool } from "./utils.js";
-const MAX_TOOL_CALLS = 10;
-
-type AnthropicParamsBase = {
- llm?: Anthropic;
- chatHistory?: ChatMessage<ToolCallLLMMessageOptions>[];
-};
+type AnthropicParamsBase = AgentParamsBase<Anthropic>;
type AnthropicParamsWithTools = AnthropicParamsBase & {
tools: BaseToolWithCall[];
@@ -35,136 +30,94 @@ export type AnthropicAgentParams =
| AnthropicParamsWithTools
| AnthropicParamsWithToolRetriever;
-type AgentContext = {
- toolCalls: number;
- llm: Anthropic;
- tools: BaseToolWithCall[];
- messages: ChatMessage<ToolCallLLMMessageOptions>[];
- shouldContinue: (context: AgentContext) => boolean;
-};
-
-type TaskResult = {
- response: ChatResponse<ToolCallLLMMessageOptions>;
- chatHistory: ChatMessage<ToolCallLLMMessageOptions>[];
-};
-
-async function task(
- context: AgentContext,
- input: ChatMessage<ToolCallLLMMessageOptions>,
-): Promise<TaskResult> {
- const { llm, tools, messages } = context;
- const nextMessages: ChatMessage<ToolCallLLMMessageOptions>[] = [
- ...messages,
- input,
- ];
- const response = await llm.chat({
- stream: false,
- tools,
- messages: nextMessages,
- });
- const options = response.message.options ?? {};
- if ("toolCall" in options) {
- const { toolCall } = options;
- const { input, name, id } = toolCall;
- const targetTool = tools.find((tool) => tool.metadata.name === name);
- let output: string;
- let isError = true;
- if (!context.shouldContinue(context)) {
- output = "Error: Tool call limit reached";
- } else if (!targetTool) {
- output = `Error: Tool ${name} not found`;
- } else {
- try {
- getCallbackManager().dispatchEvent("llm-tool-call", {
- payload: {
- toolCall: {
- name,
- input,
- },
- },
- });
- output = await targetTool.call(input);
- isError = false;
- } catch (error: unknown) {
- output = prettifyError(error);
- }
- }
- return task(
- {
- ...context,
- toolCalls: context.toolCalls + 1,
- messages: [...nextMessages, response.message],
- },
- {
- content: output,
- role: "user",
- options: {
- toolResult: {
- isError,
- id,
- },
- },
- },
- );
- } else {
- return { response, chatHistory: [...nextMessages, response.message] };
- }
+export class AnthropicAgentWorker extends AgentWorker<Anthropic> {
+ taskHandler = AnthropicAgent.taskHandler;
}
-export class AnthropicAgent {
- readonly #llm: Anthropic;
- readonly #tools:
- | BaseToolWithCall[]
- | ((query: string) => Promise<BaseToolWithCall[]>) = [];
- #chatHistory: ChatMessage<ToolCallLLMMessageOptions>[] = [];
-
+export class AnthropicAgent extends AgentRunner<Anthropic> {
constructor(params: AnthropicAgentParams) {
- this.#llm =
- params.llm ?? Settings.llm instanceof Anthropic
- ? (Settings.llm as Anthropic)
- : new Anthropic();
- if ("tools" in params) {
- this.#tools = params.tools;
- } else if ("toolRetriever" in params) {
- this.#tools = params.toolRetriever.retrieve.bind(params.toolRetriever);
- }
- if (Array.isArray(params.chatHistory)) {
- this.#chatHistory = params.chatHistory;
- }
- }
-
- static shouldContinue(context: AgentContext): boolean {
- return context.toolCalls < MAX_TOOL_CALLS;
+ super({
+ llm:
+ params.llm ?? Settings.llm instanceof Anthropic
+ ? (Settings.llm as Anthropic)
+ : new Anthropic(),
+ chatHistory: params.chatHistory ?? [],
+ systemPrompt: params.systemPrompt ?? null,
+ runner: new AnthropicAgentWorker(),
+ tools:
+ "tools" in params
+ ? params.tools
+ : params.toolRetriever.retrieve.bind(params.toolRetriever),
+ });
}
- public reset(): void {
- this.#chatHistory = [];
- }
-
- getTools(query: string): Promise<BaseToolWithCall[]> | BaseToolWithCall[] {
- return typeof this.#tools === "function" ? this.#tools(query) : this.#tools;
- }
+ createStore = AgentRunner.defaultCreateStore;
- @wrapEventCaller
async chat(
params: ChatEngineParamsNonStreaming,
- ): Promise<Promise<AgentChatResponse>> {
- const { chatHistory, response } = await task(
- {
- llm: this.#llm,
- tools: await this.getTools(extractText(params.message)),
- toolCalls: 0,
- messages: [...this.#chatHistory],
- // do we need this?
- shouldContinue: AnthropicAgent.shouldContinue,
- },
- {
- role: "user",
- content: params.message,
- options: {},
- },
- );
- this.#chatHistory = [...chatHistory];
- return new AgentChatResponse(extractText(response.message.content));
+ ): Promise<AgentChatResponse<ToolCallLLMMessageOptions>>;
+ async chat(params: ChatEngineParamsStreaming): Promise<never>;
+ override async chat(
+ params: ChatEngineParamsNonStreaming | ChatEngineParamsStreaming,
+ ) {
+ if (params.stream) {
+ throw new Error("Anthropic does not support streaming");
+ }
+ return super.chat(params);
}
+
+ static taskHandler: TaskHandler<Anthropic> = async (step) => {
+ const { input } = step;
+ const { llm, getTools, stream } = step.context;
+ if (input) {
+ step.context.store.messages = [...step.context.store.messages, input];
+ }
+ const lastMessage = step.context.store.messages.at(-1)!.content;
+ const tools = await getTools(lastMessage);
+ if (stream === true) {
+ throw new Error("Anthropic does not support streaming");
+ }
+ const response = await llm.chat({
+ stream,
+ tools,
+ messages: step.context.store.messages,
+ });
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ response.message,
+ ];
+ const options = response.message.options ?? {};
+ if ("toolCall" in options) {
+ const { toolCall } = options;
+ const targetTool = tools.find(
+ (tool) => tool.metadata.name === toolCall.name,
+ );
+ const toolOutput = await callTool(targetTool, toolCall);
+ step.context.store.toolOutputs.push(toolOutput);
+ return {
+ taskStep: step,
+ output: {
+ raw: response.raw,
+ message: {
+ content: toolOutput.output,
+ role: "user",
+ options: {
+ toolResult: {
+ result: toolOutput.output,
+ isError: toolOutput.isError,
+ id: toolCall.id,
+ },
+ },
+ },
+ },
+ isLast: false,
+ };
+ } else {
+ return {
+ taskStep: step,
+ output: response,
+ isLast: true,
+ };
+ }
+ };
}
diff --git a/packages/core/src/agent/base.ts b/packages/core/src/agent/base.ts
new file mode 100644
index 000000000..9c1b804c5
--- /dev/null
+++ b/packages/core/src/agent/base.ts
@@ -0,0 +1,411 @@
+import { pipeline, randomUUID } from "@llamaindex/env";
+import {
+ type ChatEngineParamsNonStreaming,
+ type ChatEngineParamsStreaming,
+} from "../engines/chat/index.js";
+import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { isAsyncIterable } from "../internal/utils.js";
+import type {
+ ChatMessage,
+ ChatResponse,
+ ChatResponseChunk,
+ LLM,
+ MessageContent,
+} from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
+import type { BaseToolWithCall, ToolOutput, UUID } from "../types.js";
+import { consumeAsyncIterable } from "./utils.js";
+
+export const MAX_TOOL_CALLS = 10;
+
+export type AgentTaskContext<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ readonly stream: boolean;
+ readonly toolCallCount: number;
+ readonly llm: Model;
+ readonly getTools: (
+ input: MessageContent,
+ ) => BaseToolWithCall[] | Promise<BaseToolWithCall[]>;
+ shouldContinue: (
+ taskStep: Readonly<TaskStep<Model, Store, AdditionalMessageOptions>>,
+ ) => boolean;
+ store: {
+ toolOutputs: ToolOutput[];
+ messages: ChatMessage<AdditionalMessageOptions>[];
+ } & Store;
+};
+
+export type TaskStep<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ id: UUID;
+ input: ChatMessage<AdditionalMessageOptions> | null;
+ context: AgentTaskContext<Model, Store, AdditionalMessageOptions>;
+
+ // linked list
+ prevStep: TaskStep<Model, Store, AdditionalMessageOptions> | null;
+ nextSteps: Set<TaskStep<Model, Store, AdditionalMessageOptions>>;
+};
+
+export type TaskStepOutput<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> =
+ | {
+ taskStep: TaskStep<Model, Store, AdditionalMessageOptions>;
+ output:
+ | null
+ | ChatResponse<AdditionalMessageOptions>
+ | ReadableStream<ChatResponseChunk<AdditionalMessageOptions>>;
+ isLast: false;
+ }
+ | {
+ taskStep: TaskStep<Model, Store, AdditionalMessageOptions>;
+ output:
+ | ChatResponse<AdditionalMessageOptions>
+ | ReadableStream<ChatResponseChunk<AdditionalMessageOptions>>;
+ isLast: true;
+ };
+
+export type TaskHandler<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = (
+ step: TaskStep<Model, Store, AdditionalMessageOptions>,
+) => Promise<TaskStepOutput<Model, Store, AdditionalMessageOptions>>;
+
+/**
+ * @internal
+ */
+export async function* createTaskImpl<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+>(
+ handler: TaskHandler<Model, Store, AdditionalMessageOptions>,
+ context: AgentTaskContext<Model, Store, AdditionalMessageOptions>,
+ _input: ChatMessage<AdditionalMessageOptions>,
+): AsyncGenerator<TaskStepOutput<Model, Store, AdditionalMessageOptions>> {
+ let isDone = false;
+ let input: ChatMessage<AdditionalMessageOptions> | null = _input;
+ let prevStep: TaskStep<Model, Store, AdditionalMessageOptions> | null = null;
+ while (!isDone) {
+ const step: TaskStep<Model, Store, AdditionalMessageOptions> = {
+ id: randomUUID(),
+ input,
+ context,
+ prevStep,
+ nextSteps: new Set(),
+ };
+ if (prevStep) {
+ prevStep.nextSteps.add(step);
+ }
+ const prevToolCallCount = step.context.toolCallCount;
+ if (!step.context.shouldContinue(step)) {
+ throw new Error("Tool call count exceeded limit");
+ }
+ getCallbackManager().dispatchEvent("agent-start", {
+ payload: {},
+ });
+ const taskOutput = await handler(step);
+ const { isLast, output, taskStep } = taskOutput;
+ // do not consume last output
+ if (!isLast) {
+ if (output) {
+ input = isAsyncIterable(output)
+ ? await consumeAsyncIterable(output)
+ : output.message;
+ } else {
+ input = null;
+ }
+ }
+ context = {
+ ...taskStep.context,
+ store: {
+ ...taskStep.context.store,
+ },
+ toolCallCount: prevToolCallCount + 1,
+ };
+ if (isLast) {
+ isDone = true;
+ getCallbackManager().dispatchEvent("agent-end", {
+ payload: {},
+ });
+ }
+ prevStep = taskStep;
+ yield taskOutput;
+ }
+}
+
+export type AgentStreamChatResponse<Options extends object> = {
+ response: ReadableStream<ChatResponseChunk<Options>>;
+ sources: ToolOutput[];
+};
+
+export type AgentChatResponse<Options extends object> = {
+ response: ChatResponse<Options>;
+ sources: ToolOutput[];
+};
+
+export type AgentRunnerParams<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ llm: AI;
+ chatHistory: ChatMessage<AdditionalMessageOptions>[];
+ systemPrompt: MessageContent | null;
+ runner: AgentWorker<AI, Store, AdditionalMessageOptions>;
+ tools:
+ | BaseToolWithCall[]
+ | ((query: MessageContent) => Promise<BaseToolWithCall[]>);
+};
+
+export type AgentParamsBase<
+ AI extends LLM,
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ llm?: AI;
+ chatHistory?: ChatMessage<AdditionalMessageOptions>[];
+ systemPrompt?: MessageContent;
+};
+
+/**
+ * Worker will schedule tasks and handle the task execution
+ */
+export abstract class AgentWorker<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> {
+ #taskSet = new Set<TaskStep<AI, Store, AdditionalMessageOptions>>();
+ abstract taskHandler: TaskHandler<AI, Store, AdditionalMessageOptions>;
+
+ public createTask(
+ query: string,
+ context: AgentTaskContext<AI, Store, AdditionalMessageOptions>,
+ ): ReadableStream<TaskStepOutput<AI, Store, AdditionalMessageOptions>> {
+ const taskGenerator = createTaskImpl(this.taskHandler, context, {
+ role: "user",
+ content: query,
+ });
+ return new ReadableStream<
+ TaskStepOutput<AI, Store, AdditionalMessageOptions>
+ >({
+ start: async (controller) => {
+ for await (const stepOutput of taskGenerator) {
+ this.#taskSet.add(stepOutput.taskStep);
+ controller.enqueue(stepOutput);
+ if (stepOutput.isLast) {
+ let currentStep: TaskStep<
+ AI,
+ Store,
+ AdditionalMessageOptions
+ > | null = stepOutput.taskStep;
+ while (currentStep) {
+ this.#taskSet.delete(currentStep);
+ currentStep = currentStep.prevStep;
+ }
+ controller.close();
+ }
+ }
+ },
+ });
+ }
+
+ [Symbol.toStringTag] = "AgentWorker";
+}
+
+/**
+ * Runner will manage the task execution and provide a high-level API for the user
+ */
+export abstract class AgentRunner<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> {
+ readonly #llm: AI;
+ readonly #tools:
+ | BaseToolWithCall[]
+ | ((query: MessageContent) => Promise<BaseToolWithCall[]>);
+ readonly #systemPrompt: MessageContent | null = null;
+ #chatHistory: ChatMessage<AdditionalMessageOptions>[];
+ readonly #runner: AgentWorker<AI, Store, AdditionalMessageOptions>;
+
+ // create extra store
+ abstract createStore(): Store;
+
+ static defaultCreateStore(): object {
+ return Object.create(null);
+ }
+
+ protected constructor(
+ params: AgentRunnerParams<AI, Store, AdditionalMessageOptions>,
+ ) {
+ const { llm, chatHistory, runner, tools } = params;
+ this.#llm = llm;
+ this.#chatHistory = chatHistory;
+ this.#runner = runner;
+ if (params.systemPrompt) {
+ this.#systemPrompt = params.systemPrompt;
+ }
+ this.#tools = tools;
+ }
+
+ get llm() {
+ return this.#llm;
+ }
+
+ get chatHistory(): ChatMessage<AdditionalMessageOptions>[] {
+ return this.#chatHistory;
+ }
+
+ public reset(): void {
+ this.#chatHistory = [];
+ }
+
+ public getTools(
+ query: MessageContent,
+ ): Promise<BaseToolWithCall[]> | BaseToolWithCall[] {
+ return typeof this.#tools === "function" ? this.#tools(query) : this.#tools;
+ }
+
+ static shouldContinue<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+ >(task: Readonly<TaskStep<AI, Store, AdditionalMessageOptions>>): boolean {
+ return task.context.toolCallCount < MAX_TOOL_CALLS;
+ }
+
+ // fixme: this shouldn't be async
+ async createTask(message: MessageContent, stream: boolean = false) {
+ const initialMessages = [...this.#chatHistory];
+ if (this.#systemPrompt !== null) {
+ const systemPrompt = this.#systemPrompt;
+ const alreadyHasSystemPrompt = initialMessages
+ .filter((msg) => msg.role === "system")
+ .some((msg) => Object.is(msg.content, systemPrompt));
+ if (!alreadyHasSystemPrompt) {
+ initialMessages.push({
+ content: systemPrompt,
+ role: "system",
+ });
+ }
+ }
+ return this.#runner.createTask(extractText(message), {
+ stream,
+ toolCallCount: 0,
+ llm: this.#llm,
+ getTools: (message) => this.getTools(message),
+ store: {
+ ...this.createStore(),
+ messages: initialMessages,
+ toolOutputs: [] as ToolOutput[],
+ },
+ shouldContinue: AgentRunner.shouldContinue,
+ });
+ }
+
+ async chat(
+ params: ChatEngineParamsNonStreaming,
+ ): Promise<AgentChatResponse<AdditionalMessageOptions>>;
+ async chat(
+ params: ChatEngineParamsStreaming,
+ ): Promise<AgentStreamChatResponse<AdditionalMessageOptions>>;
+ @wrapEventCaller
+ async chat(
+ params: ChatEngineParamsNonStreaming | ChatEngineParamsStreaming,
+ ): Promise<
+ | AgentChatResponse<AdditionalMessageOptions>
+ | AgentStreamChatResponse<AdditionalMessageOptions>
+ > {
+ const task = await this.createTask(params.message, !!params.stream);
+ const stepOutput = await pipeline(
+ task,
+ async (
+ iter: AsyncIterable<
+ TaskStepOutput<AI, Store, AdditionalMessageOptions>
+ >,
+ ) => {
+ for await (const stepOutput of iter) {
+ if (stepOutput.isLast) {
+ return stepOutput;
+ }
+ }
+ throw new Error("Task did not complete");
+ },
+ );
+ const { output, taskStep } = stepOutput;
+ this.#chatHistory = [...taskStep.context.store.messages];
+ if (isAsyncIterable(output)) {
+ return {
+ response: output,
+ sources: [...taskStep.context.store.toolOutputs],
+ } satisfies AgentStreamChatResponse<AdditionalMessageOptions>;
+ } else {
+ return {
+ response: output,
+ sources: [...taskStep.context.store.toolOutputs],
+ } satisfies AgentChatResponse<AdditionalMessageOptions>;
+ }
+ }
+}
diff --git a/packages/core/src/agent/index.ts b/packages/core/src/agent/index.ts
index 652403e03..2fc557f4b 100644
--- a/packages/core/src/agent/index.ts
+++ b/packages/core/src/agent/index.ts
@@ -1,7 +1,18 @@
-// Not exporting the AnthropicAgent because it is not ready to ship yet.
-// export { AnthropicAgent, type AnthropicAgentParams } from "./anthropic.js";
-export * from "./openai/base.js";
-export * from "./openai/worker.js";
-export * from "./react/base.js";
-export * from "./react/worker.js";
-export * from "./types.js";
+export {
+ AnthropicAgent,
+ AnthropicAgentWorker,
+ type AnthropicAgentParams,
+} from "./anthropic.js";
+export {
+ OpenAIAgent,
+ OpenAIAgentWorker,
+ type OpenAIAgentParams,
+} from "./openai.js";
+export {
+ ReACTAgent,
+ ReACTAgentWorker,
+ type ReACTAgentParams,
+} from "./react.js";
+// todo: ParallelAgent
+// todo: CustomAgent
+// todo: ReactMultiModal
diff --git a/packages/core/src/agent/openai.ts b/packages/core/src/agent/openai.ts
new file mode 100644
index 000000000..c0d59b076
--- /dev/null
+++ b/packages/core/src/agent/openai.ts
@@ -0,0 +1,194 @@
+import { pipeline } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
+import type {
+ ChatResponseChunk,
+ ToolCall,
+ ToolCallLLMMessageOptions,
+} from "../llm/index.js";
+import { OpenAI } from "../llm/open_ai.js";
+import { ObjectRetriever } from "../objects/index.js";
+import type { BaseToolWithCall } from "../types.js";
+import {
+ AgentRunner,
+ AgentWorker,
+ type AgentParamsBase,
+ type TaskHandler,
+} from "./base.js";
+import { callTool } from "./utils.js";
+
+type OpenAIParamsBase = AgentParamsBase<OpenAI>;
+
+type OpenAIParamsWithTools = OpenAIParamsBase & {
+ tools: BaseToolWithCall[];
+};
+
+type OpenAIParamsWithToolRetriever = OpenAIParamsBase & {
+ toolRetriever: ObjectRetriever<BaseToolWithCall>;
+};
+
+export type OpenAIAgentParams =
+ | OpenAIParamsWithTools
+ | OpenAIParamsWithToolRetriever;
+
+export class OpenAIAgentWorker extends AgentWorker<OpenAI> {
+ taskHandler = OpenAIAgent.taskHandler;
+}
+
+export class OpenAIAgent extends AgentRunner<OpenAI> {
+ constructor(params: OpenAIAgentParams) {
+ super({
+ llm:
+ params.llm ?? Settings.llm instanceof OpenAI
+ ? (Settings.llm as OpenAI)
+ : new OpenAI(),
+ chatHistory: params.chatHistory ?? [],
+ runner: new OpenAIAgentWorker(),
+ systemPrompt: params.systemPrompt ?? null,
+ tools:
+ "tools" in params
+ ? params.tools
+ : params.toolRetriever.retrieve.bind(params.toolRetriever),
+ });
+ }
+
+ createStore = AgentRunner.defaultCreateStore;
+
+ static taskHandler: TaskHandler<OpenAI> = async (step) => {
+ const { input } = step;
+ const { llm, stream, getTools } = step.context;
+ if (input) {
+ step.context.store.messages = [...step.context.store.messages, input];
+ }
+ const lastMessage = step.context.store.messages.at(-1)!.content;
+ const tools = await getTools(lastMessage);
+ const response = await llm.chat({
+ // @ts-expect-error
+ stream,
+ tools,
+ messages: [...step.context.store.messages],
+ });
+ if (!stream) {
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ response.message,
+ ];
+ const options = response.message.options ?? {};
+ if ("toolCall" in options) {
+ const { toolCall } = options;
+ const targetTool = tools.find(
+ (tool) => tool.metadata.name === toolCall.name,
+ );
+ const toolOutput = await callTool(targetTool, toolCall);
+ step.context.store.toolOutputs.push(toolOutput);
+ return {
+ taskStep: step,
+ output: {
+ raw: response.raw,
+ message: {
+ content: toolOutput.output,
+ role: "user",
+ options: {
+ toolResult: {
+ result: toolOutput.output,
+ isError: toolOutput.isError,
+ id: toolCall.id,
+ },
+ },
+ },
+ },
+ isLast: false,
+ };
+ } else {
+ return {
+ taskStep: step,
+ output: response,
+ isLast: true,
+ };
+ }
+ } else {
+ const responseChunkStream = new ReadableStream<
+ ChatResponseChunk<ToolCallLLMMessageOptions>
+ >({
+ async start(controller) {
+ for await (const chunk of response) {
+ controller.enqueue(chunk);
+ }
+ controller.close();
+ },
+ });
+ const [pipStream, finalStream] = responseChunkStream.tee();
+ const reader = pipStream.getReader();
+ const { value } = await reader.read();
+ reader.releaseLock();
+ if (value === undefined) {
+ throw new Error(
+ "first chunk value is undefined, this should not happen",
+ );
+ }
+ // check if first chunk has tool calls, if so, this is a function call
+ // otherwise, it's a regular message
+ const hasToolCall = !!(value.options && "toolCall" in value.options);
+
+ if (hasToolCall) {
+ // you need to consume the response to get the full toolCalls
+ const toolCalls = await pipeline(
+ pipStream,
+ async (
+ iter: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>,
+ ) => {
+ const toolCalls = new Map<string, ToolCall>();
+ for await (const chunk of iter) {
+ if (chunk.options && "toolCall" in chunk.options) {
+ const toolCall = chunk.options.toolCall;
+ toolCalls.set(toolCall.id, toolCall);
+ }
+ }
+ return [...toolCalls.values()];
+ },
+ );
+ for (const toolCall of toolCalls) {
+ const targetTool = tools.find(
+ (tool) => tool.metadata.name === toolCall.name,
+ );
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ {
+ role: "assistant" as const,
+ content: "",
+ options: {
+ toolCall,
+ },
+ },
+ ];
+ const toolOutput = await callTool(targetTool, toolCall);
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ {
+ role: "user" as const,
+ content: toolOutput.output,
+ options: {
+ toolResult: {
+ result: toolOutput.output,
+ isError: toolOutput.isError,
+ id: toolCall.id,
+ },
+ },
+ },
+ ];
+ step.context.store.toolOutputs.push(toolOutput);
+ }
+ return {
+ taskStep: step,
+ output: null,
+ isLast: false,
+ };
+ } else {
+ return {
+ taskStep: step,
+ output: finalStream,
+ isLast: true,
+ };
+ }
+ }
+ };
+}
diff --git a/packages/core/src/agent/openai/base.ts b/packages/core/src/agent/openai/base.ts
deleted file mode 100644
index 6a9af20c7..000000000
--- a/packages/core/src/agent/openai/base.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-import { Settings } from "../../Settings.js";
-import type { ChatMessage } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
-import type { BaseMemory } from "../../memory/types.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import type { BaseTool } from "../../types.js";
-import { AgentRunner } from "../runner/base.js";
-import { OpenAIAgentWorker } from "./worker.js";
-
-type OpenAIAgentParams = {
- tools?: BaseTool[];
- llm?: OpenAI;
- memory?: BaseMemory;
- prefixMessages?: ChatMessage[];
- maxFunctionCalls?: number;
- defaultToolChoice?: string;
- toolRetriever?: ObjectRetriever<BaseTool>;
- systemPrompt?: string;
-};
-
-/**
- * An agent that uses OpenAI's API to generate text.
- *
- * @category OpenAI
- */
-export class OpenAIAgent extends AgentRunner {
- constructor({
- tools,
- llm,
- memory,
- prefixMessages,
- maxFunctionCalls = 5,
- defaultToolChoice = "auto",
- toolRetriever,
- systemPrompt,
- }: OpenAIAgentParams) {
- if (!llm) {
- if (Settings.llm instanceof OpenAI) {
- llm = Settings.llm;
- } else {
- console.warn("No OpenAI model provided, creating a new one");
- llm = new OpenAI({ model: "gpt-3.5-turbo-0613" });
- }
- }
-
- if (systemPrompt) {
- if (prefixMessages) {
- throw new Error("Cannot provide both systemPrompt and prefixMessages");
- }
-
- prefixMessages = [
- {
- content: systemPrompt,
- role: "system",
- },
- ];
- }
-
- if (!llm?.supportToolCall) {
- throw new Error("LLM model must be a function-calling model");
- }
-
- const stepEngine = new OpenAIAgentWorker({
- tools,
- llm,
- prefixMessages,
- maxFunctionCalls,
- toolRetriever,
- });
-
- super({
- agentWorker: stepEngine,
- llm,
- memory,
- defaultToolChoice,
- // @ts-expect-error 2322
- chatHistory: prefixMessages,
- });
- }
-}
diff --git a/packages/core/src/agent/openai/types/chat.ts b/packages/core/src/agent/openai/types/chat.ts
deleted file mode 100644
index afe922198..000000000
--- a/packages/core/src/agent/openai/types/chat.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-export type OpenAIToolCall = ChatCompletionMessageToolCall;
-
-export interface Function {
- arguments: string;
- name: string;
- type: "function";
-}
-
-export interface ChatCompletionMessageToolCall {
- id: string;
- function: Function;
- type: "function";
-}
diff --git a/packages/core/src/agent/openai/worker.ts b/packages/core/src/agent/openai/worker.ts
deleted file mode 100644
index 38e4d72ed..000000000
--- a/packages/core/src/agent/openai/worker.ts
+++ /dev/null
@@ -1,397 +0,0 @@
-import { pipeline, randomUUID } from "@llamaindex/env";
-import type { ChatCompletionToolChoiceOption } from "openai/resources/chat/completions";
-import { Response } from "../../Response.js";
-import { Settings } from "../../Settings.js";
-import {
- AgentChatResponse,
- ChatResponseMode,
- StreamingAgentChatResponse,
-} from "../../engines/chat/types.js";
-import {
- OpenAI,
- isFunctionCallingModel,
- type ChatMessage,
- type ChatResponseChunk,
- type LLMChatParamsBase,
- type OpenAIAdditionalChatOptions,
- type ToolCallLLMMessageOptions,
- type ToolCallOptions,
-} from "../../llm/index.js";
-import { extractText } from "../../llm/utils.js";
-import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import type { ToolOutput } from "../../tools/types.js";
-import { callToolWithErrorHandling } from "../../tools/utils.js";
-import type { BaseTool } from "../../types.js";
-import type { AgentWorker, Task } from "../types.js";
-import { TaskStep, TaskStepOutput } from "../types.js";
-import { addUserStepToMemory, getFunctionByName } from "../utils.js";
-
-async function callFunction(
- tools: BaseTool[],
- toolCall: ToolCallOptions["toolCall"],
-): Promise<[ChatMessage<ToolCallLLMMessageOptions>, ToolOutput]> {
- const id = toolCall.id;
- const name = toolCall.name;
- const input = toolCall.input;
-
- if (Settings.debug) {
- console.log("=== Calling Function ===");
- console.log(`Calling function: ${name} with args: ${input}`);
- }
-
- const tool = getFunctionByName(tools, name);
-
- // Call tool
- // Use default error message
- const output = await callToolWithErrorHandling(tool, input);
-
- if (Settings.debug) {
- console.log(`Got output ${output}`);
- console.log("==========================");
- }
-
- return [
- {
- content: `${output}`,
- role: "user",
- options: {
- toolResult: {
- id,
- isError: false,
- },
- },
- },
- output,
- ];
-}
-
-type OpenAIAgentWorkerParams = {
- tools?: BaseTool[];
- llm?: OpenAI;
- prefixMessages?: ChatMessage[];
- maxFunctionCalls?: number;
- toolRetriever?: ObjectRetriever<BaseTool>;
-};
-
-type CallFunctionOutput = {
- message: ChatMessage;
- toolOutput: ToolOutput;
-};
-
-export class OpenAIAgentWorker
- implements AgentWorker<LLMChatParamsBase<OpenAIAdditionalChatOptions>>
-{
- private llm: OpenAI;
- private maxFunctionCalls: number = 5;
-
- public prefixMessages: ChatMessage[];
-
- private _getTools: (input: string) => Promise<BaseTool[]>;
-
- constructor({
- tools = [],
- llm,
- prefixMessages,
- maxFunctionCalls,
- toolRetriever,
- }: OpenAIAgentWorkerParams) {
- this.llm =
- llm ?? isFunctionCallingModel(Settings.llm)
- ? (Settings.llm as OpenAI)
- : new OpenAI({
- model: "gpt-3.5-turbo-0613",
- });
- if (maxFunctionCalls) {
- this.maxFunctionCalls = maxFunctionCalls;
- }
- this.prefixMessages = prefixMessages || [];
-
- if (tools.length > 0 && toolRetriever) {
- throw new Error("Cannot specify both tools and tool_retriever");
- } else if (tools.length > 0) {
- this._getTools = async () => tools;
- } else if (toolRetriever) {
- // fixme: this won't work, type mismatch
- this._getTools = async (message: string) =>
- toolRetriever.retrieve(message);
- } else {
- this._getTools = async () => [];
- }
- }
-
- public getAllMessages(task: Task): ChatMessage<ToolCallLLMMessageOptions>[] {
- return [
- ...this.prefixMessages,
- ...task.memory.get(),
- ...task.extraState.newMemory.get(),
- ];
- }
-
- public getLatestToolCall(task: Task): ToolCallOptions["toolCall"] | null {
- const chatHistory: ChatMessage[] = task.extraState.newMemory.getAll();
-
- if (chatHistory.length === 0) {
- return null;
- }
-
- // @ts-expect-error fixme
- return chatHistory[chatHistory.length - 1].options?.toolCall;
- }
-
- private _getLlmChatParams(
- task: Task,
- tools: BaseTool[],
- toolChoice: ChatCompletionToolChoiceOption = "auto",
- ): LLMChatParamsBase<OpenAIAdditionalChatOptions, ToolCallLLMMessageOptions> {
- const llmChatParams = {
- messages: this.getAllMessages(task),
- tools: undefined as BaseTool[] | undefined,
- additionalChatOptions: {} as OpenAIAdditionalChatOptions,
- } satisfies LLMChatParamsBase<
- OpenAIAdditionalChatOptions,
- ToolCallLLMMessageOptions
- >;
-
- if (tools.length > 0) {
- llmChatParams.tools = tools;
- llmChatParams.additionalChatOptions.tool_choice = toolChoice;
- }
-
- return llmChatParams;
- }
-
- private _processMessage(
- task: Task,
- aiMessage: ChatMessage,
- ): AgentChatResponse {
- task.extraState.newMemory.put(aiMessage);
-
- return new AgentChatResponse(
- extractText(aiMessage.content),
- task.extraState.sources,
- );
- }
-
- private async _getStreamAiResponse(
- task: Task,
- llmChatParams: LLMChatParamsBase<
- OpenAIAdditionalChatOptions,
- ToolCallLLMMessageOptions
- >,
- ): Promise<StreamingAgentChatResponse | AgentChatResponse> {
- const stream = await this.llm.chat({
- stream: true,
- ...llmChatParams,
- });
-
- const responseChunkStream = new ReadableStream<
- ChatResponseChunk<ToolCallLLMMessageOptions>
- >({
- async start(controller) {
- for await (const chunk of stream) {
- controller.enqueue(chunk);
- }
- controller.close();
- },
- });
- const [pipStream, finalStream] = responseChunkStream.tee();
- const reader = pipStream.getReader();
- const { value } = await reader.read();
- reader.releaseLock();
- if (value === undefined) {
- throw new Error("first chunk value is undefined, this should not happen");
- }
- // check if first chunk has tool calls, if so, this is a function call
- // otherwise, it's a regular message
- const hasToolCall: boolean = !!(
- value.options && "toolCall" in value.options
- );
-
- if (hasToolCall) {
- return this._processMessage(task, {
- content: await pipeline(finalStream, async (iterator) => {
- let content = "";
- for await (const value of iterator) {
- content += value.delta;
- }
- return content;
- }),
- role: "assistant",
- options: value.options,
- });
- } else {
- const [responseStream, chunkStream] = finalStream.tee();
- let content = "";
- return new StreamingAgentChatResponse(
- responseStream.pipeThrough<Response>({
- readable: new ReadableStream({
- async start(controller) {
- for await (const chunk of chunkStream) {
- controller.enqueue(new Response(chunk.delta));
- }
- controller.close();
- },
- }),
- writable: new WritableStream({
- write(chunk) {
- content += chunk.delta;
- },
- close() {
- task.extraState.newMemory.put({
- content,
- role: "assistant",
- });
- },
- }),
- }),
- task.extraState.sources,
- );
- }
- }
-
- private async _getAgentResponse(
- task: Task,
- mode: ChatResponseMode,
- llmChatParams: LLMChatParamsBase<
- OpenAIAdditionalChatOptions,
- ToolCallLLMMessageOptions
- >,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
- if (mode === ChatResponseMode.WAIT) {
- const chatResponse = await this.llm.chat({
- stream: false,
- ...llmChatParams,
- });
-
- return this._processMessage(
- task,
- chatResponse.message,
- ) as AgentChatResponse;
- } else if (mode === ChatResponseMode.STREAM) {
- return this._getStreamAiResponse(task, llmChatParams);
- }
-
- throw new Error("Invalid mode");
- }
-
- async callFunction(
- tools: BaseTool[],
- toolCall: ToolCallOptions["toolCall"],
- ): Promise<CallFunctionOutput> {
- const functionMessage = await callFunction(tools, toolCall);
-
- const message = functionMessage[0];
- const toolOutput = functionMessage[1];
-
- return {
- message,
- toolOutput,
- };
- }
-
- initializeStep(task: Task): TaskStep {
- const sources: ToolOutput[] = [];
-
- const newMemory = new ChatMemoryBuffer({
- tokenLimit: task.memory.tokenLimit,
- });
-
- const taskState = {
- sources,
- nFunctionCalls: 0,
- newMemory,
- };
-
- task.extraState = {
- ...task.extraState,
- ...taskState,
- };
-
- return new TaskStep(task.taskId, randomUUID(), task.input);
- }
-
- private _shouldContinue(
- toolCall: ToolCallOptions["toolCall"] | null,
- nFunctionCalls: number,
- ): toolCall is ToolCallOptions["toolCall"] {
- if (nFunctionCalls > this.maxFunctionCalls) {
- return false;
- }
-
- return !!toolCall;
- }
-
- async getTools(input: string): Promise<BaseTool[]> {
- return this._getTools(input);
- }
-
- private async _runStep(
- step: TaskStep,
- task: Task,
- mode: ChatResponseMode = ChatResponseMode.WAIT,
- toolChoice: ChatCompletionToolChoiceOption = "auto",
- ): Promise<TaskStepOutput> {
- const tools = await this.getTools(task.input);
-
- if (step.input) {
- addUserStepToMemory(step, task.extraState.newMemory);
- }
-
- const llmChatParams = this._getLlmChatParams(task, tools, toolChoice);
-
- const agentChatResponse = await this._getAgentResponse(
- task,
- mode,
- llmChatParams,
- );
-
- const latestToolCall = this.getLatestToolCall(task) ?? null;
-
- let isDone: boolean;
- let newSteps: TaskStep[];
-
- if (!this._shouldContinue(latestToolCall, task.extraState.nFunctionCalls)) {
- isDone = true;
- newSteps = [];
- } else {
- isDone = false;
- const { message, toolOutput } = await this.callFunction(
- tools,
- latestToolCall,
- );
-
- task.extraState.sources.push(toolOutput);
- task.extraState.newMemory.put(message);
-
- task.extraState.nFunctionCalls += 1;
-
- newSteps = [step.getNextStep(randomUUID(), undefined)];
- }
-
- return new TaskStepOutput(agentChatResponse, step, newSteps, isDone);
- }
-
- async runStep(
- step: TaskStep,
- task: Task,
- chatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
- ): Promise<TaskStepOutput> {
- const toolChoice = chatParams?.additionalChatOptions?.tool_choice ?? "auto";
- return this._runStep(step, task, ChatResponseMode.WAIT, toolChoice);
- }
-
- async streamStep(
- step: TaskStep,
- task: Task,
- chatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
- ): Promise<TaskStepOutput> {
- const toolChoice = chatParams?.additionalChatOptions?.tool_choice ?? "auto";
- return this._runStep(step, task, ChatResponseMode.STREAM, toolChoice);
- }
-
- finalizeTask(task: Task): void {
- task.memory.set(task.memory.get().concat(task.extraState.newMemory.get()));
- task.extraState.newMemory.reset();
- }
-}
diff --git a/packages/core/src/agent/react.ts b/packages/core/src/agent/react.ts
new file mode 100644
index 000000000..1201c93e2
--- /dev/null
+++ b/packages/core/src/agent/react.ts
@@ -0,0 +1,397 @@
+import { pipeline, randomUUID } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
+import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
+import { isAsyncIterable } from "../internal/utils.js";
+import {
+ type ChatMessage,
+ type ChatResponse,
+ type ChatResponseChunk,
+ type LLM,
+} from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
+import { ObjectRetriever } from "../objects/index.js";
+import type { BaseTool, BaseToolWithCall } from "../types.js";
+import {
+ AgentRunner,
+ AgentWorker,
+ type AgentParamsBase,
+ type TaskHandler,
+} from "./base.js";
+import {
+ callTool,
+ consumeAsyncIterable,
+ createReadableStream,
+} from "./utils.js";
+
+type ReACTAgentParamsBase = AgentParamsBase<LLM>;
+
+type ReACTAgentParamsWithTools = ReACTAgentParamsBase & {
+ tools: BaseToolWithCall[];
+};
+
+type ReACTAgentParamsWithToolRetriever = ReACTAgentParamsBase & {
+ toolRetriever: ObjectRetriever<BaseToolWithCall>;
+};
+
+export type ReACTAgentParams =
+ | ReACTAgentParamsWithTools
+ | ReACTAgentParamsWithToolRetriever;
+
+type BaseReason = {
+ type: unknown;
+};
+
+type ObservationReason = BaseReason & {
+ type: "observation";
+ observation: string;
+};
+
+type ActionReason = BaseReason & {
+ type: "action";
+ thought: string;
+ action: string;
+ input: Record<string, unknown>;
+};
+
+type ResponseReason = BaseReason & {
+ type: "response";
+ thought: string;
+ response: ChatResponse | AsyncIterable<ChatResponseChunk>;
+};
+
+type Reason = ObservationReason | ActionReason | ResponseReason;
+
+function reasonFormatter(reason: Reason): string | Promise<string> {
+ switch (reason.type) {
+ case "observation":
+ return `Observation: ${reason.observation}`;
+ case "action":
+ return `Thought: ${reason.thought}\nAction: ${reason.action}\nInput: ${JSON.stringify(
+ reason.input,
+ )}`;
+ case "response": {
+ if (isAsyncIterable(reason.response)) {
+ return consumeAsyncIterable(reason.response).then(
+ (message) =>
+ `Thought: ${reason.thought}\nAnswer: ${extractText(message.content)}`,
+ );
+ } else {
+ return `Thought: ${reason.thought}\nAnswer: ${extractText(
+ reason.response.message.content,
+ )}`;
+ }
+ }
+ }
+}
+
+function extractJsonStr(text: string): string {
+ const pattern = /\{.*}/s;
+ const match = text.match(pattern);
+
+ if (!match) {
+ throw new SyntaxError(`Could not extract json string from output: ${text}`);
+ }
+
+ return match[0];
+}
+
+function extractFinalResponse(
+ inputText: string,
+): [thought: string, answer: string] {
+ const pattern = /\s*Thought:(.*?)Answer:(.*?)$/s;
+
+ const match = inputText.match(pattern);
+
+ if (!match) {
+ throw new Error(
+ `Could not extract final answer from input text: ${inputText}`,
+ );
+ }
+
+ const thought = match[1].trim();
+ const answer = match[2].trim();
+ return [thought, answer];
+}
+
+function extractToolUse(
+ inputText: string,
+): [thought: string, action: string, input: string] {
+ const pattern =
+ /\s*Thought: (.*?)\nAction: ([a-zA-Z0-9_]+).*?\.*Input: .*?(\{.*?})/s;
+
+ const match = inputText.match(pattern);
+
+ if (!match) {
+ throw new Error(
+ `Could not extract tool use from input text: "${inputText}"`,
+ );
+ }
+
+ const thought = match[1].trim();
+ const action = match[2].trim();
+ const actionInput = match[3].trim();
+ return [thought, action, actionInput];
+}
+
+function actionInputParser(jsonStr: string): Record<string, unknown> {
+ const processedString = jsonStr.replace(/(?<!\w)'|'(?!\w)/g, '"');
+ const pattern = /"(\w+)":\s*"([^"]*)"/g;
+ const matches = [...processedString.matchAll(pattern)];
+ return Object.fromEntries(matches);
+}
+
+type ReACTOutputParser = <Options extends object>(
+ output: ChatResponse<Options> | AsyncIterable<ChatResponseChunk<Options>>,
+) => Promise<Reason>;
+
+const reACTOutputParser: ReACTOutputParser = async (
+ output,
+): Promise<Reason> => {
+ let reason: Reason | null = null;
+
+ if (isAsyncIterable(output)) {
+ const [peakStream, finalStream] = createReadableStream(output).tee();
+ const type = await pipeline(peakStream, async (iter) => {
+ let content = "";
+ for await (const chunk of iter) {
+ content += chunk.delta;
+ if (content.includes("Action:")) {
+ return "action";
+ } else if (content.includes("Answer:")) {
+ return "answer";
+ } else if (content.includes("Thought:")) {
+ return "thought";
+ }
+ }
+ });
+ // step 2: do the parsing from content
+ switch (type) {
+ case "action": {
+ // have to consume the stream to get the full content
+ const response = await consumeAsyncIterable(finalStream);
+ const { content } = response;
+ const [thought, action, input] = extractToolUse(content);
+ const jsonStr = extractJsonStr(input);
+ let json: Record<string, unknown>;
+ try {
+ json = JSON.parse(jsonStr);
+ } catch (e) {
+ json = actionInputParser(jsonStr);
+ }
+ reason = {
+ type: "action",
+ thought,
+ action,
+ input: json,
+ };
+ break;
+ }
+ case "thought": {
+ const thought = "(Implicit) I can answer without any more tools!";
+ reason = {
+ type: "response",
+ thought,
+ // bypass the response, because here we don't need to do anything with it
+ response: finalStream,
+ };
+ break;
+ }
+ case "answer": {
+ const response = await consumeAsyncIterable(finalStream);
+ const { content } = response;
+ const [thought, answer] = extractFinalResponse(content);
+ reason = {
+ type: "response",
+ thought,
+ response: {
+ raw: response,
+ message: {
+ role: "assistant",
+ content: answer,
+ },
+ },
+ };
+ break;
+ }
+ default: {
+ throw new Error(`Invalid type: ${type}`);
+ }
+ }
+ } else {
+ const content = extractText(output.message.content);
+ const type = content.includes("Answer:")
+ ? "answer"
+ : content.includes("Action:")
+ ? "action"
+ : "thought";
+
+ // step 2: do the parsing from content
+ switch (type) {
+ case "action": {
+ const [thought, action, input] = extractToolUse(content);
+ const jsonStr = extractJsonStr(input);
+ let json: Record<string, unknown>;
+ try {
+ json = JSON.parse(jsonStr);
+ } catch (e) {
+ json = actionInputParser(jsonStr);
+ }
+ reason = {
+ type: "action",
+ thought,
+ action,
+ input: json,
+ };
+ break;
+ }
+ case "thought": {
+ const thought = "(Implicit) I can answer without any more tools!";
+ reason = {
+ type: "response",
+ thought,
+ response: {
+ raw: output,
+ message: {
+ role: "assistant",
+ content: extractText(output.message.content),
+ },
+ },
+ };
+ break;
+ }
+ case "answer": {
+ const [thought, answer] = extractFinalResponse(content);
+ reason = {
+ type: "response",
+ thought,
+ response: {
+ raw: output,
+ message: {
+ role: "assistant",
+ content: answer,
+ },
+ },
+ };
+ break;
+ }
+ default: {
+ throw new Error(`Invalid type: ${type}`);
+ }
+ }
+ }
+ if (reason === null) {
+ throw new TypeError("Reason is null");
+ }
+ return reason;
+};
+
+type ReACTAgentStore = {
+ reasons: Reason[];
+};
+
+type ChatFormatter = <Options extends object>(
+ tools: BaseTool[],
+ messages: ChatMessage<Options>[],
+ currentReasons: Reason[],
+) => Promise<ChatMessage<Options>[]>;
+
+const chatFormatter: ChatFormatter = async <Options extends object>(
+ tools: BaseTool[],
+ messages: ChatMessage<Options>[],
+ currentReasons: Reason[],
+): Promise<ChatMessage<Options>[]> => {
+ const header = getReACTAgentSystemHeader(tools);
+ const reasonMessages: ChatMessage<Options>[] = [];
+ for (const reason of currentReasons) {
+ const response = await reasonFormatter(reason);
+ reasonMessages.push({
+ role: reason.type === "observation" ? "user" : "assistant",
+ content: response,
+ });
+ }
+ return [
+ {
+ role: "system",
+ content: header,
+ },
+ ...messages,
+ ...reasonMessages,
+ ];
+};
+
+export class ReACTAgentWorker extends AgentWorker<LLM, ReACTAgentStore> {
+ taskHandler = ReACTAgent.taskHandler;
+}
+
+export class ReACTAgent extends AgentRunner<LLM, ReACTAgentStore> {
+ constructor(
+ params: ReACTAgentParamsWithTools | ReACTAgentParamsWithToolRetriever,
+ ) {
+ super({
+ llm: params.llm ?? Settings.llm,
+ chatHistory: params.chatHistory ?? [],
+ runner: new ReACTAgentWorker(),
+ systemPrompt: params.systemPrompt ?? null,
+ tools:
+ "tools" in params
+ ? params.tools
+ : params.toolRetriever.retrieve.bind(params.toolRetriever),
+ });
+ }
+
+ createStore() {
+ return {
+ reasons: [],
+ };
+ }
+
+ static taskHandler: TaskHandler<LLM, ReACTAgentStore> = async (step) => {
+ const { llm, stream, getTools } = step.context;
+ const input = step.input;
+ if (input) {
+ step.context.store.messages.push(input);
+ }
+ const lastMessage = step.context.store.messages.at(-1)!.content;
+ const tools = await getTools(lastMessage);
+ const messages = await chatFormatter(
+ tools,
+ step.context.store.messages,
+ step.context.store.reasons,
+ );
+ const response = await llm.chat({
+ // @ts-expect-error
+ stream,
+ messages,
+ });
+ const reason = await reACTOutputParser(response);
+ step.context.store.reasons = [...step.context.store.reasons, reason];
+ if (reason.type === "response") {
+ return {
+ isLast: true,
+ output: response,
+ taskStep: step,
+ };
+ } else {
+ if (reason.type === "action") {
+ const tool = tools.find((tool) => tool.metadata.name === reason.action);
+ const toolOutput = await callTool(tool, {
+ id: randomUUID(),
+ input: reason.input,
+ name: reason.action,
+ });
+ step.context.store.reasons = [
+ ...step.context.store.reasons,
+ {
+ type: "observation",
+ observation: toolOutput.output,
+ },
+ ];
+ }
+ return {
+ isLast: false,
+ output: null,
+ taskStep: step,
+ };
+ }
+ };
+}
diff --git a/packages/core/src/agent/react/base.ts b/packages/core/src/agent/react/base.ts
deleted file mode 100644
index 763e6f487..000000000
--- a/packages/core/src/agent/react/base.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-import type { ChatMessage, LLM } from "../../llm/index.js";
-import type { BaseMemory } from "../../memory/types.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import type { BaseTool } from "../../types.js";
-import { AgentRunner } from "../runner/base.js";
-import { ReActAgentWorker } from "./worker.js";
-
-type ReActAgentParams = {
- tools: BaseTool[];
- llm?: LLM;
- memory?: BaseMemory;
- prefixMessages?: ChatMessage[];
- maxInteractions?: number;
- defaultToolChoice?: string;
- toolRetriever?: ObjectRetriever<BaseTool>;
-};
-
-/**
- * An agent that uses OpenAI's API to generate text.
- *
- * @category OpenAI
- */
-export class ReActAgent extends AgentRunner {
- constructor({
- tools,
- llm,
- memory,
- prefixMessages,
- maxInteractions = 10,
- defaultToolChoice = "auto",
- toolRetriever,
- }: Partial<ReActAgentParams>) {
- const stepEngine = new ReActAgentWorker({
- tools: tools ?? [],
- llm,
- maxInteractions,
- toolRetriever,
- });
-
- super({
- agentWorker: stepEngine,
- memory,
- defaultToolChoice,
- // @ts-expect-error 2322
- chatHistory: prefixMessages,
- });
- }
-}
diff --git a/packages/core/src/agent/react/formatter.ts b/packages/core/src/agent/react/formatter.ts
deleted file mode 100644
index aff8200ec..000000000
--- a/packages/core/src/agent/react/formatter.ts
+++ /dev/null
@@ -1,84 +0,0 @@
-import type { ChatMessage } from "../../llm/index.js";
-import type { BaseTool } from "../../types.js";
-import { getReactChatSystemHeader } from "./prompts.js";
-import type { BaseReasoningStep } from "./types.js";
-import { ObservationReasoningStep } from "./types.js";
-
-function getReactToolDescriptions(tools: BaseTool[]): string[] {
- const toolDescs: string[] = [];
- for (const tool of tools) {
- // @ts-ignore
- const toolDesc = `> Tool Name: ${tool.metadata.name}\nTool Description: ${tool.metadata.description}\nTool Args: ${JSON.stringify(tool?.metadata?.parameters?.properties)}\n`;
- toolDescs.push(toolDesc);
- }
- return toolDescs;
-}
-
-export interface BaseAgentChatFormatter {
- format(
- tools: BaseTool[],
- chatHistory: ChatMessage[],
- currentReasoning?: BaseReasoningStep[],
- ): ChatMessage[];
-}
-
-export class ReActChatFormatter implements BaseAgentChatFormatter {
- systemHeader: string = "";
- context: string = "'";
-
- constructor(init?: Partial<ReActChatFormatter>) {
- Object.assign(this, init);
- }
-
- format(
- tools: BaseTool[],
- chatHistory: ChatMessage[],
- currentReasoning?: BaseReasoningStep[],
- ): ChatMessage[] {
- currentReasoning = currentReasoning ?? [];
-
- const formatArgs = {
- toolDesc: getReactToolDescriptions(tools).join("\n"),
- toolNames: tools.map((tool) => tool.metadata.name).join(", "),
- context: "",
- };
-
- if (this.context) {
- formatArgs["context"] = this.context;
- }
-
- const reasoningHistory = [];
-
- for (const reasoningStep of currentReasoning) {
- let message: ChatMessage | undefined;
-
- if (reasoningStep instanceof ObservationReasoningStep) {
- message = {
- content: reasoningStep.getContent(),
- role: "user",
- };
- } else {
- message = {
- content: reasoningStep.getContent(),
- role: "assistant",
- };
- }
-
- reasoningHistory.push(message);
- }
-
- const systemContent = getReactChatSystemHeader({
- toolDesc: formatArgs.toolDesc,
- toolNames: formatArgs.toolNames,
- });
-
- return [
- {
- content: systemContent,
- role: "system",
- },
- ...chatHistory,
- ...reasoningHistory,
- ];
- }
-}
diff --git a/packages/core/src/agent/react/outputParser.ts b/packages/core/src/agent/react/outputParser.ts
deleted file mode 100644
index 593cdc187..000000000
--- a/packages/core/src/agent/react/outputParser.ts
+++ /dev/null
@@ -1,105 +0,0 @@
-import type { BaseReasoningStep } from "./types.js";
-import {
- ActionReasoningStep,
- BaseOutputParser,
- ResponseReasoningStep,
-} from "./types.js";
-
-function extractJsonStr(text: string): string {
- const pattern = /\{.*\}/s;
- const match = text.match(pattern);
-
- if (!match) {
- throw new Error(`Could not extract json string from output: ${text}`);
- }
-
- return match[0];
-}
-
-function extractToolUse(inputText: string): [string, string, string] {
- const pattern =
- /\s*Thought: (.*?)\nAction: ([a-zA-Z0-9_]+).*?\nAction Input: .*?(\{.*?\})/s;
-
- const match = inputText.match(pattern);
-
- if (!match) {
- throw new Error(`Could not extract tool use from input text: ${inputText}`);
- }
-
- const thought = match[1].trim();
- const action = match[2].trim();
- const actionInput = match[3].trim();
- return [thought, action, actionInput];
-}
-
-function actionInputParser(jsonStr: string): object {
- const processedString = jsonStr.replace(/(?<!\w)\'|\'(?!\w)/g, '"');
- const pattern = /"(\w+)":\s*"([^"]*)"/g;
- const matches = [...processedString.matchAll(pattern)];
- return Object.fromEntries(matches);
-}
-
-function extractFinalResponse(inputText: string): [string, string] {
- const pattern = /\s*Thought:(.*?)Answer:(.*?)(?:$)/s;
-
- const match = inputText.match(pattern);
-
- if (!match) {
- throw new Error(
- `Could not extract final answer from input text: ${inputText}`,
- );
- }
-
- const thought = match[1].trim();
- const answer = match[2].trim();
- return [thought, answer];
-}
-
-export class ReActOutputParser extends BaseOutputParser {
- parse(output: string, isStreaming: boolean = false): BaseReasoningStep {
- if (!output.includes("Thought:")) {
- // NOTE: handle the case where the agent directly outputs the answer
- // instead of following the thought-answer format
- return new ResponseReasoningStep({
- thought: "(Implicit) I can answer without any more tools!",
- response: output,
- isStreaming,
- });
- }
-
- if (output.includes("Answer:")) {
- const [thought, answer] = extractFinalResponse(output);
- return new ResponseReasoningStep({
- thought: thought,
- response: answer,
- isStreaming,
- });
- }
-
- if (output.includes("Action:")) {
- const [thought, action, action_input] = extractToolUse(output);
- const json_str = extractJsonStr(action_input);
-
- // First we try json, if this fails we use ast
- let actionInputDict;
-
- try {
- actionInputDict = JSON.parse(json_str);
- } catch (e) {
- actionInputDict = actionInputParser(json_str);
- }
-
- return new ActionReasoningStep({
- thought: thought,
- action: action,
- actionInput: actionInputDict,
- });
- }
-
- throw new Error(`Could not parse output: ${output}`);
- }
-
- format(output: string): string {
- throw new Error("Not implemented");
- }
-}
diff --git a/packages/core/src/agent/react/types.ts b/packages/core/src/agent/react/types.ts
deleted file mode 100644
index e8849275f..000000000
--- a/packages/core/src/agent/react/types.ts
+++ /dev/null
@@ -1,91 +0,0 @@
-import type { ChatMessage } from "../../llm/index.js";
-import { extractText } from "../../llm/utils.js";
-
-export interface BaseReasoningStep {
- getContent(): string;
- isDone(): boolean;
-}
-
-export class ObservationReasoningStep implements BaseReasoningStep {
- observation: string;
-
- constructor(init?: Partial<ObservationReasoningStep>) {
- this.observation = init?.observation ?? "";
- }
-
- getContent(): string {
- return `Observation: ${this.observation}`;
- }
-
- isDone(): boolean {
- return false;
- }
-}
-
-export class ActionReasoningStep implements BaseReasoningStep {
- thought: string;
- action: string;
- actionInput: Record<string, any>;
-
- constructor(init?: Partial<ActionReasoningStep>) {
- this.thought = init?.thought ?? "";
- this.action = init?.action ?? "";
- this.actionInput = init?.actionInput ?? {};
- }
-
- getContent(): string {
- return `Thought: ${this.thought}\nAction: ${this.action}\nAction Input: ${JSON.stringify(this.actionInput)}`;
- }
-
- isDone(): boolean {
- return false;
- }
-}
-
-export abstract class BaseOutputParser {
- abstract parse(output: string, isStreaming?: boolean): BaseReasoningStep;
-
- format(output: string) {
- return output;
- }
-
- formatMessages(messages: ChatMessage[]): ChatMessage[] {
- if (messages) {
- if (messages[0].role === "system") {
- messages[0].content = this.format(
- extractText(messages[0].content) || "",
- );
- } else {
- messages[messages.length - 1].content = this.format(
- extractText(messages[messages.length - 1].content) || "",
- );
- }
- }
-
- return messages;
- }
-}
-
-export class ResponseReasoningStep implements BaseReasoningStep {
- thought: string;
- response: string;
- isStreaming: boolean = false;
-
- constructor(init?: Partial<ResponseReasoningStep>) {
- this.thought = init?.thought ?? "";
- this.response = init?.response ?? "";
- this.isStreaming = init?.isStreaming ?? false;
- }
-
- getContent(): string {
- if (this.isStreaming) {
- return `Thought: ${this.thought}\nAnswer (Starts With): ${this.response} ...`;
- } else {
- return `Thought: ${this.thought}\nAnswer: ${this.response}`;
- }
- }
-
- isDone(): boolean {
- return true;
- }
-}
diff --git a/packages/core/src/agent/react/worker.ts b/packages/core/src/agent/react/worker.ts
deleted file mode 100644
index 3fba7b04c..000000000
--- a/packages/core/src/agent/react/worker.ts
+++ /dev/null
@@ -1,325 +0,0 @@
-import { randomUUID } from "@llamaindex/env";
-import type { ChatMessage } from "cohere-ai/api";
-import { Settings } from "../../Settings.js";
-import { AgentChatResponse } from "../../engines/chat/index.js";
-import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
-import { type ChatResponse, type LLM } from "../../llm/index.js";
-import { extractText } from "../../llm/utils.js";
-import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import { ToolOutput } from "../../tools/index.js";
-import type { BaseTool } from "../../types.js";
-import type { AgentWorker, Task } from "../types.js";
-import { TaskStep, TaskStepOutput } from "../types.js";
-import { ReActChatFormatter } from "./formatter.js";
-import { ReActOutputParser } from "./outputParser.js";
-import type { BaseReasoningStep } from "./types.js";
-import {
- ActionReasoningStep,
- ObservationReasoningStep,
- ResponseReasoningStep,
-} from "./types.js";
-
-type ReActAgentWorkerParams = {
- tools: BaseTool[];
- llm?: LLM;
- maxInteractions?: number;
- reactChatFormatter?: ReActChatFormatter | undefined;
- outputParser?: ReActOutputParser | undefined;
- toolRetriever?: ObjectRetriever<BaseTool> | undefined;
-};
-
-function addUserStepToReasoning(
- step: TaskStep,
- memory: ChatMemoryBuffer,
- currentReasoning: BaseReasoningStep[],
-): void {
- if (step.stepState.isFirst) {
- memory.put({
- content: step.input ?? "",
- role: "user",
- });
- step.stepState.isFirst = false;
- } else {
- const reasoningStep = new ObservationReasoningStep({
- observation: step.input ?? undefined,
- });
- currentReasoning.push(reasoningStep);
- if (Settings.debug) {
- console.log(`Added user message to memory: ${step.input}`);
- }
- }
-}
-
-type ChatParams = {
- messages: ChatMessage[];
- tools?: BaseTool[];
-};
-
-/**
- * ReAct agent worker.
- */
-export class ReActAgentWorker implements AgentWorker<ChatParams> {
- llm: LLM;
-
- maxInteractions: number = 10;
- reactChatFormatter: ReActChatFormatter;
- outputParser: ReActOutputParser;
-
- _getTools: (message: string) => Promise<BaseTool[]>;
-
- constructor({
- tools,
- llm,
- maxInteractions,
- reactChatFormatter,
- outputParser,
- toolRetriever,
- }: ReActAgentWorkerParams) {
- this.llm = llm ?? Settings.llm;
-
- this.maxInteractions = maxInteractions ?? 10;
- this.reactChatFormatter = reactChatFormatter ?? new ReActChatFormatter();
- this.outputParser = outputParser ?? new ReActOutputParser();
-
- if (tools.length > 0 && toolRetriever) {
- throw new Error("Cannot specify both tools and tool_retriever");
- } else if (tools.length > 0) {
- this._getTools = async () => tools;
- } else if (toolRetriever) {
- this._getTools = async (message: string) =>
- toolRetriever.retrieve(message);
- } else {
- this._getTools = async () => [];
- }
- }
-
- initializeStep(task: Task): TaskStep {
- const sources: ToolOutput[] = [];
- const currentReasoning: BaseReasoningStep[] = [];
- const newMemory = new ChatMemoryBuffer({
- tokenLimit: task.memory.tokenLimit,
- });
-
- const taskState = {
- sources,
- currentReasoning,
- newMemory,
- };
-
- task.extraState = {
- ...task.extraState,
- ...taskState,
- };
-
- return new TaskStep(task.taskId, randomUUID(), task.input, {
- isFirst: true,
- });
- }
-
- extractReasoningStep(
- output: ChatResponse,
- isStreaming: boolean,
- ): [string, BaseReasoningStep[], boolean] {
- if (!output.message.content) {
- throw new Error("Got empty message.");
- }
-
- const messageContent = output.message.content;
- const currentReasoning: BaseReasoningStep[] = [];
-
- let reasoningStep;
-
- try {
- reasoningStep = this.outputParser.parse(
- extractText(messageContent),
- isStreaming,
- ) as ActionReasoningStep;
- } catch (e) {
- throw new Error(`Could not parse output: ${e}`);
- }
-
- if (Settings.debug) {
- console.log(`${reasoningStep.getContent()}\n`);
- }
-
- currentReasoning.push(reasoningStep);
-
- if (reasoningStep.isDone()) {
- return [extractText(messageContent), currentReasoning, true];
- }
-
- const actionReasoningStep = new ActionReasoningStep({
- thought: reasoningStep.getContent(),
- action: reasoningStep.action,
- actionInput: reasoningStep.actionInput,
- });
-
- if (!(actionReasoningStep instanceof ActionReasoningStep)) {
- throw new Error(`Expected ActionReasoningStep, got ${reasoningStep}`);
- }
-
- return [extractText(messageContent), currentReasoning, false];
- }
-
- async _processActions(
- task: Task,
- tools: BaseTool[],
- output: ChatResponse,
- isStreaming: boolean = false,
- ): Promise<[BaseReasoningStep[], boolean]> {
- const toolsDict: Record<string, BaseTool> = {};
-
- for (const tool of tools) {
- toolsDict[tool.metadata.name] = tool;
- }
-
- const [_, currentReasoning, isDone] = this.extractReasoningStep(
- output,
- isStreaming,
- );
-
- if (isDone) {
- return [currentReasoning, true];
- }
-
- const reasoningStep = currentReasoning[
- currentReasoning.length - 1
- ] as ActionReasoningStep;
-
- const actionReasoningStep = new ActionReasoningStep({
- thought: reasoningStep.getContent(),
- action: reasoningStep.action,
- actionInput: reasoningStep.actionInput,
- });
-
- const tool = toolsDict[actionReasoningStep.action];
-
- getCallbackManager().dispatchEvent("llm-tool-call", {
- payload: {
- toolCall: {
- name: tool.metadata.name,
- input: JSON.stringify(actionReasoningStep.actionInput),
- },
- },
- });
- const toolOutput = await tool.call!(actionReasoningStep.actionInput);
-
- task.extraState.sources.push(
- new ToolOutput(
- toolOutput,
- tool.metadata.name,
- actionReasoningStep.actionInput,
- toolOutput,
- ),
- );
-
- const observationStep = new ObservationReasoningStep({
- observation: toolOutput,
- });
-
- currentReasoning.push(observationStep);
-
- if (Settings.debug) {
- console.log(`${observationStep.getContent()}`);
- }
-
- return [currentReasoning, false];
- }
-
- _getResponse(
- currentReasoning: BaseReasoningStep[],
- sources: ToolOutput[],
- ): AgentChatResponse {
- if (currentReasoning.length === 0) {
- throw new Error("No reasoning steps were taken.");
- } else if (currentReasoning.length === this.maxInteractions) {
- throw new Error("Reached max iterations.");
- }
-
- const responseStep = currentReasoning[currentReasoning.length - 1];
-
- let responseStr: string;
-
- if (responseStep instanceof ResponseReasoningStep) {
- responseStr = responseStep.response;
- } else {
- responseStr = responseStep.getContent();
- }
-
- return new AgentChatResponse(responseStr, sources);
- }
-
- _getTaskStepResponse(
- agentResponse: AgentChatResponse,
- step: TaskStep,
- isDone: boolean,
- ): TaskStepOutput {
- let newSteps: TaskStep[] = [];
-
- if (isDone) {
- newSteps = [];
- } else {
- newSteps = [step.getNextStep(randomUUID(), undefined)];
- }
-
- return new TaskStepOutput(agentResponse, step, newSteps, isDone);
- }
-
- async _runStep(step: TaskStep, task: Task): Promise<TaskStepOutput> {
- if (step.input) {
- addUserStepToReasoning(
- step,
- task.extraState.newMemory,
- task.extraState.currentReasoning,
- );
- }
-
- const tools = await this._getTools(task.input);
-
- const inputChat = this.reactChatFormatter.format(
- tools,
- [...task.memory.getAll(), ...task.extraState.newMemory.getAll()],
- task.extraState.currentReasoning,
- );
-
- const chatResponse = await this.llm.chat({
- messages: inputChat,
- });
-
- const [reasoningSteps, isDone] = await this._processActions(
- task,
- tools,
- chatResponse,
- );
-
- task.extraState.currentReasoning.push(...reasoningSteps);
-
- const agentResponse = this._getResponse(
- task.extraState.currentReasoning,
- task.extraState.sources,
- );
-
- if (isDone) {
- task.extraState.newMemory.put({
- content: agentResponse.response,
- role: "assistant",
- });
- }
-
- return this._getTaskStepResponse(agentResponse, step, isDone);
- }
-
- async runStep(step: TaskStep, task: Task): Promise<TaskStepOutput> {
- return await this._runStep(step, task);
- }
-
- streamStep(): Promise<TaskStepOutput> {
- throw new Error("Method not implemented.");
- }
-
- finalizeTask(task: Task): void {
- task.memory.set(task.memory.get() + task.extraState.newMemory.get());
- task.extraState.newMemory.reset();
- }
-}
diff --git a/packages/core/src/agent/runner/base.ts b/packages/core/src/agent/runner/base.ts
deleted file mode 100644
index 915f996b3..000000000
--- a/packages/core/src/agent/runner/base.ts
+++ /dev/null
@@ -1,384 +0,0 @@
-import { randomUUID } from "@llamaindex/env";
-import type { ChatHistory } from "../../ChatHistory.js";
-import type { ChatEngineAgentParams } from "../../engines/chat/index.js";
-import {
- AgentChatResponse,
- ChatResponseMode,
- StreamingAgentChatResponse,
-} from "../../engines/chat/index.js";
-import type { LLM } from "../../llm/index.js";
-import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
-import type { BaseMemory } from "../../memory/types.js";
-import type { AgentWorker, TaskStepOutput } from "../types.js";
-import { Task, TaskStep } from "../types.js";
-import { AgentState, BaseAgentRunner, TaskState } from "./types.js";
-const validateStepFromArgs = (
- taskId: string,
- input?: string | null,
- step?: any,
- kwargs?: any,
-): TaskStep | undefined => {
- if (step) {
- if (input) {
- throw new Error("Cannot specify both `step` and `input`");
- }
- return step;
- } else {
- if (!input) return;
- return new TaskStep(taskId, step, input, kwargs);
- }
-};
-
-type AgentRunnerParams = {
- agentWorker: AgentWorker;
- chatHistory?: ChatHistory;
- state?: AgentState;
- memory?: BaseMemory;
- llm?: LLM;
- initTaskStateKwargs?: Record<string, any>;
- deleteTaskOnFinish?: boolean;
- defaultToolChoice?: string;
-};
-
-export class AgentRunner extends BaseAgentRunner {
- agentWorker: AgentWorker;
- state: AgentState;
- memory: BaseMemory;
- initTaskStateKwargs: Record<string, any>;
- deleteTaskOnFinish: boolean;
- defaultToolChoice: string;
-
- /**
- * Creates an AgentRunner.
- */
- constructor(params: AgentRunnerParams) {
- super();
-
- this.agentWorker = params.agentWorker;
- this.state = params.state ?? new AgentState();
- this.memory =
- params.memory ??
- new ChatMemoryBuffer({
- llm: params.llm,
- chatHistory: params.chatHistory,
- });
- this.initTaskStateKwargs = params.initTaskStateKwargs ?? {};
- this.deleteTaskOnFinish = params.deleteTaskOnFinish ?? false;
- this.defaultToolChoice = params.defaultToolChoice ?? "auto";
- }
-
- /**
- * Creates a task.
- * @param input
- * @param kwargs
- */
- createTask(input: string, kwargs?: any): Task {
- let extraState;
-
- if (!this.initTaskStateKwargs) {
- if (kwargs && "extraState" in kwargs) {
- if (extraState) {
- delete extraState["extraState"];
- }
- }
- } else {
- if (kwargs && "extraState" in kwargs) {
- throw new Error(
- "Cannot specify both `extraState` and `initTaskStateKwargs`",
- );
- } else {
- extraState = this.initTaskStateKwargs;
- }
- }
-
- const task = new Task({
- taskId: randomUUID(),
- input,
- memory: this.memory,
- extraState,
- ...kwargs,
- });
-
- const initialStep = this.agentWorker.initializeStep(task);
-
- const taskState = new TaskState({
- task,
- stepQueue: [initialStep],
- });
-
- this.state.taskDict[task.taskId] = taskState;
-
- return task;
- }
-
- /**
- * Deletes the task.
- * @param taskId
- */
- deleteTask(taskId: string): void {
- delete this.state.taskDict[taskId];
- }
-
- /**
- * Returns the list of tasks.
- */
- listTasks(): Task[] {
- return Object.values(this.state.taskDict).map(
- (taskState) => taskState.task,
- );
- }
-
- /**
- * Returns the task.
- */
- getTask(taskId: string): Task {
- return this.state.taskDict[taskId].task;
- }
-
- /**
- * Returns the completed steps in the task.
- * @param taskId
- * @param kwargs
- */
- getCompletedSteps(taskId: string): TaskStepOutput[] {
- return this.state.taskDict[taskId].completedSteps;
- }
-
- /**
- * Returns the next steps in the task.
- * @param taskId
- * @param kwargs
- */
- getUpcomingSteps(taskId: string, kwargs: any): TaskStep[] {
- return this.state.taskDict[taskId].stepQueue;
- }
-
- private async _runStep(
- taskId: string,
- step?: TaskStep,
- mode: ChatResponseMode = ChatResponseMode.WAIT,
- kwargs?: any,
- ): Promise<TaskStepOutput> {
- const task = this.state.getTask(taskId);
- const curStep = step || this.state.getStepQueue(taskId).shift();
-
- let curStepOutput: TaskStepOutput;
-
- if (!curStep) {
- throw new Error(`No step found for task ${taskId}`);
- }
-
- if (mode === ChatResponseMode.WAIT) {
- curStepOutput = await this.agentWorker.runStep(curStep, task, kwargs);
- } else if (mode === ChatResponseMode.STREAM) {
- curStepOutput = await this.agentWorker.streamStep(curStep, task, kwargs);
- } else {
- throw new Error(`Invalid mode: ${mode}`);
- }
-
- const nextSteps = curStepOutput.nextSteps;
-
- this.state.addSteps(taskId, nextSteps);
- this.state.addCompletedStep(taskId, [curStepOutput]);
-
- return curStepOutput;
- }
-
- /**
- * Runs the next step in the task.
- * @param taskId
- * @param kwargs
- * @param step
- * @returns
- */
- async runStep(
- taskId: string,
- input?: string | null,
- step?: TaskStep,
- kwargs: any = {},
- ): Promise<TaskStepOutput> {
- const curStep = validateStepFromArgs(taskId, input, step, kwargs);
- return this._runStep(taskId, curStep, ChatResponseMode.WAIT, kwargs);
- }
-
- /**
- * Runs the step and returns the response.
- * @param taskId
- * @param input
- * @param step
- * @param kwargs
- */
- async streamStep(
- taskId: string,
- input: string,
- step?: TaskStep,
- kwargs?: any,
- ): Promise<TaskStepOutput> {
- const curStep = validateStepFromArgs(taskId, input, step, kwargs);
- return this._runStep(taskId, curStep, ChatResponseMode.STREAM, kwargs);
- }
-
- /**
- * Finalizes the response and returns it.
- * @param taskId
- * @param kwargs
- * @param stepOutput
- * @returns
- */
- async finalizeResponse(
- taskId: string,
- stepOutput: TaskStepOutput,
- kwargs?: any,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
- if (!stepOutput) {
- stepOutput =
- this.getCompletedSteps(taskId)[
- this.getCompletedSteps(taskId).length - 1
- ];
- }
-
- if (!stepOutput.isLast) {
- throw new Error(
- "finalizeResponse can only be called on the last step output",
- );
- }
-
- if (!(stepOutput.output instanceof StreamingAgentChatResponse)) {
- if (!(stepOutput.output instanceof AgentChatResponse)) {
- throw new Error(
- `When \`isLast\` is True, cur_step_output.output must be AGENT_CHAT_RESPONSE_TYPE: ${stepOutput.output}`,
- );
- }
- }
-
- this.agentWorker.finalizeTask(this.getTask(taskId), kwargs);
-
- if (this.deleteTaskOnFinish) {
- this.deleteTask(taskId);
- }
-
- return stepOutput.output;
- }
-
- protected async _chat({
- message,
- toolChoice,
- stream,
- }: ChatEngineAgentParams): Promise<AgentChatResponse>;
- protected async _chat({
- message,
- toolChoice,
- stream,
- }: ChatEngineAgentParams & {
- stream: true;
- }): Promise<StreamingAgentChatResponse>;
- protected async _chat({
- message,
- toolChoice,
- stream,
- }: ChatEngineAgentParams): Promise<
- AgentChatResponse | StreamingAgentChatResponse
- > {
- const task = this.createTask(message as string);
-
- let resultOutput;
-
- const mode = stream ? ChatResponseMode.STREAM : ChatResponseMode.WAIT;
-
- while (true) {
- const curStepOutput = await this._runStep(task.taskId, undefined, mode, {
- toolChoice,
- });
-
- if (curStepOutput.isLast) {
- resultOutput = curStepOutput;
- break;
- }
-
- toolChoice = "auto";
- }
-
- return this.finalizeResponse(task.taskId, resultOutput);
- }
-
- /**
- * Sends a message to the LLM and returns the response.
- * @param message
- * @param chatHistory
- * @param toolChoice
- * @returns
- */
- public async chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- }: ChatEngineAgentParams & {
- stream?: false;
- }): Promise<AgentChatResponse>;
- public async chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- }: ChatEngineAgentParams & {
- stream: true;
- }): Promise<StreamingAgentChatResponse>;
- public async chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- }: ChatEngineAgentParams): Promise<
- AgentChatResponse | StreamingAgentChatResponse
- > {
- if (!toolChoice) {
- toolChoice = this.defaultToolChoice;
- }
-
- const chatResponse = await this._chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- });
-
- return chatResponse;
- }
-
- protected _getPromptModules(): string[] {
- return [];
- }
-
- protected _getPrompts(): string[] {
- return [];
- }
-
- /**
- * Resets the agent.
- */
- reset(): void {
- this.state = new AgentState();
- }
-
- getCompletedStep(
- taskId: string,
- stepId: string,
- kwargs: any,
- ): TaskStepOutput {
- const completedSteps = this.getCompletedSteps(taskId);
- for (const stepOutput of completedSteps) {
- if (stepOutput.taskStep.stepId === stepId) {
- return stepOutput;
- }
- }
-
- throw new Error(`Step ${stepId} not found in task ${taskId}`);
- }
-
- /**
- * Undoes the step.
- * @param taskId
- */
- undoStep(taskId: string): void {}
-}
diff --git a/packages/core/src/agent/runner/types.ts b/packages/core/src/agent/runner/types.ts
deleted file mode 100644
index c6fb3b8b6..000000000
--- a/packages/core/src/agent/runner/types.ts
+++ /dev/null
@@ -1,106 +0,0 @@
-import type {
- AgentChatResponse,
- StreamingAgentChatResponse,
-} from "../../engines/chat/index.js";
-import type { Task, TaskStep, TaskStepOutput } from "../types.js";
-import { BaseAgent } from "../types.js";
-
-export class TaskState {
- task!: Task;
- stepQueue!: TaskStep[];
- completedSteps!: TaskStepOutput[];
-
- constructor(init?: Partial<TaskState>) {
- Object.assign(this, init);
- }
-}
-
-export abstract class BaseAgentRunner extends BaseAgent {
- constructor(init?: Partial<BaseAgentRunner>) {
- super();
- }
-
- abstract createTask(input: string, kwargs: any): Task;
- abstract deleteTask(taskId: string): void;
- abstract getTask(taskId: string, kwargs: any): Task;
- abstract listTasks(kwargs: any): Task[];
- abstract getUpcomingSteps(taskId: string, kwargs: any): TaskStep[];
- abstract getCompletedSteps(taskId: string, kwargs: any): TaskStepOutput[];
-
- getCompletedStep(
- taskId: string,
- stepId: string,
- kwargs: any,
- ): TaskStepOutput {
- const completedSteps = this.getCompletedSteps(taskId, kwargs);
- for (const stepOutput of completedSteps) {
- if (stepOutput.taskStep.stepId === stepId) {
- return stepOutput;
- }
- }
-
- throw new Error(`Step ${stepId} not found in task ${taskId}`);
- }
-
- abstract runStep(
- taskId: string,
- input: string,
- step: TaskStep,
- kwargs: any,
- ): Promise<TaskStepOutput>;
-
- abstract streamStep(
- taskId: string,
- input: string,
- step: TaskStep,
- kwargs?: any,
- ): Promise<TaskStepOutput>;
-
- abstract finalizeResponse(
- taskId: string,
- stepOutput: TaskStepOutput,
- kwargs?: any,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-
- abstract undoStep(taskId: string): void;
-}
-
-export class AgentState {
- taskDict!: Record<string, TaskState>;
-
- constructor(init?: Partial<AgentState>) {
- Object.assign(this, init);
-
- if (!this.taskDict) {
- this.taskDict = {};
- }
- }
-
- getTask(taskId: string): Task {
- return this.taskDict[taskId].task;
- }
-
- getCompletedSteps(taskId: string): TaskStepOutput[] {
- return this.taskDict[taskId].completedSteps || [];
- }
-
- getStepQueue(taskId: string): TaskStep[] {
- return this.taskDict[taskId].stepQueue || [];
- }
-
- addSteps(taskId: string, steps: TaskStep[]): void {
- if (!this.taskDict[taskId].stepQueue) {
- this.taskDict[taskId].stepQueue = [];
- }
-
- this.taskDict[taskId].stepQueue.push(...steps);
- }
-
- addCompletedStep(taskId: string, stepOutputs: TaskStepOutput[]): void {
- if (!this.taskDict[taskId].completedSteps) {
- this.taskDict[taskId].completedSteps = [];
- }
-
- this.taskDict[taskId].completedSteps.push(...stepOutputs);
- }
-}
diff --git a/packages/core/src/agent/type.ts b/packages/core/src/agent/type.ts
new file mode 100644
index 000000000..38d974cf0
--- /dev/null
+++ b/packages/core/src/agent/type.ts
@@ -0,0 +1,4 @@
+import type { BaseEvent } from "../internal/type.js";
+
+export type AgentStartEvent = BaseEvent<{}>;
+export type AgentEndEvent = BaseEvent<{}>;
diff --git a/packages/core/src/agent/types.ts b/packages/core/src/agent/types.ts
deleted file mode 100644
index 5864c3202..000000000
--- a/packages/core/src/agent/types.ts
+++ /dev/null
@@ -1,202 +0,0 @@
-import type {
- AgentChatResponse,
- ChatEngineAgentParams,
- StreamingAgentChatResponse,
-} from "../engines/chat/index.js";
-
-import type { BaseMemory } from "../memory/types.js";
-import type { QueryEngineParamsNonStreaming } from "../types.js";
-
-export interface AgentWorker<ExtraParams extends object = object> {
- initializeStep(task: Task, params?: ExtraParams): TaskStep;
- runStep(
- step: TaskStep,
- task: Task,
- params?: ExtraParams,
- ): Promise<TaskStepOutput>;
- streamStep(
- step: TaskStep,
- task: Task,
- params?: ExtraParams,
- ): Promise<TaskStepOutput>;
- finalizeTask(task: Task, params?: ExtraParams): void;
-}
-
-interface BaseChatEngine {
- chat(
- params: ChatEngineAgentParams,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-}
-
-interface BaseQueryEngine {
- query(
- params: QueryEngineParamsNonStreaming,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-}
-
-/**
- * BaseAgent is the base class for all agents.
- */
-export abstract class BaseAgent implements BaseChatEngine, BaseQueryEngine {
- protected _getPrompts(): string[] {
- return [];
- }
-
- protected _getPromptModules(): string[] {
- return [];
- }
-
- abstract chat(
- params: ChatEngineAgentParams,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-
- abstract reset(): void;
-
- /**
- * query is the main entrypoint for the agent. It takes a query and returns a response.
- * @param params
- * @returns
- */
- async query(
- params: QueryEngineParamsNonStreaming,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
- // Handle non-streaming query
- const agentResponse = await this.chat({
- message: params.query,
- chatHistory: [],
- });
-
- return agentResponse;
- }
-}
-
-type TaskParams = {
- taskId: string;
- input: string;
- memory: BaseMemory;
- extraState: Record<string, any>;
-};
-
-/**
- * Task is a unit of work for the agent.
- * @param taskId: taskId
- */
-export class Task {
- taskId: string;
- input: string;
-
- memory: BaseMemory;
- extraState: Record<string, any>;
-
- constructor({ taskId, input, memory, extraState }: TaskParams) {
- this.taskId = taskId;
- this.input = input;
- this.memory = memory;
- this.extraState = extraState ?? {};
- }
-}
-
-interface ITaskStep {
- taskId: string;
- stepId: string;
- input?: string | null;
- stepState: Record<string, any>;
- nextSteps: Record<string, TaskStep>;
- prevSteps: Record<string, TaskStep>;
- isReady: boolean;
- getNextStep(
- stepId: string,
- input?: string,
- stepState?: Record<string, any>,
- ): TaskStep;
- linkStep(nextStep: TaskStep): void;
-}
-
-/**
- * TaskStep is a unit of work for the agent.
- * @param taskId: taskId
- * @param stepId: stepId
- * @param input: input
- * @param stepState: stepState
- */
-export class TaskStep implements ITaskStep {
- taskId: string;
- stepId: string;
- input?: string | null;
- stepState: Record<string, any> = {};
- nextSteps: Record<string, TaskStep> = {};
- prevSteps: Record<string, TaskStep> = {};
- isReady: boolean = true;
-
- constructor(
- taskId: string,
- stepId: string,
- input?: string | null,
- stepState?: Record<string, any> | null,
- ) {
- this.taskId = taskId;
- this.stepId = stepId;
- this.input = input;
- this.stepState = stepState ?? this.stepState;
- }
-
- /*
- * getNextStep is a function that returns the next step.
- * @param stepId: stepId
- * @param input: input
- * @param stepState: stepState
- * @returns: TaskStep
- */
- getNextStep(
- stepId: string,
- input?: string,
- stepState?: Record<string, unknown>,
- ): TaskStep {
- return new TaskStep(
- this.taskId,
- stepId,
- input,
- stepState ?? this.stepState,
- );
- }
-
- /*
- * linkStep is a function that links the next step.
- * @param nextStep: nextStep
- * @returns: void
- */
- linkStep(nextStep: TaskStep): void {
- this.nextSteps[nextStep.stepId] = nextStep;
- nextStep.prevSteps[this.stepId] = this;
- }
-}
-
-/**
- * TaskStepOutput is a unit of work for the agent.
- * @param output: output
- * @param taskStep: taskStep
- * @param nextSteps: nextSteps
- * @param isLast: isLast
- */
-export class TaskStepOutput {
- output: AgentChatResponse | StreamingAgentChatResponse;
- taskStep: TaskStep;
- nextSteps: TaskStep[];
- isLast: boolean;
-
- constructor(
- output: AgentChatResponse | StreamingAgentChatResponse,
- taskStep: TaskStep,
- nextSteps: TaskStep[],
- isLast: boolean = false,
- ) {
- this.output = output;
- this.taskStep = taskStep;
- this.nextSteps = nextSteps;
- this.isLast = isLast;
- }
-
- toString(): string {
- return String(this.output);
- }
-}
diff --git a/packages/core/src/agent/utils.ts b/packages/core/src/agent/utils.ts
index a22c882ec..08a601edf 100644
--- a/packages/core/src/agent/utils.ts
+++ b/packages/core/src/agent/utils.ts
@@ -1,35 +1,112 @@
-import { Settings } from "../Settings.js";
-import type { ChatMessage } from "../llm/index.js";
-import type { ChatMemoryBuffer } from "../memory/ChatMemoryBuffer.js";
-import type { BaseTool } from "../types.js";
-import type { TaskStep } from "./types.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { isAsyncIterable, prettifyError } from "../internal/utils.js";
+import type {
+ ChatMessage,
+ ChatResponseChunk,
+ TextChatMessage,
+ ToolCall,
+} from "../llm/index.js";
+import type { BaseTool, ToolOutput } from "../types.js";
-export function addUserStepToMemory(
- step: TaskStep,
- memory: ChatMemoryBuffer,
-): void {
- if (!step.input) {
- return;
+export async function callTool(
+ tool: BaseTool | undefined,
+ toolCall: ToolCall,
+): Promise<ToolOutput> {
+ if (!tool) {
+ const output = `Tool ${toolCall.name} does not exist.`;
+ return {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: true,
+ };
}
-
- const userMessage: ChatMessage = {
- content: step.input,
- role: "user",
- };
-
- memory.put(userMessage);
-
- if (Settings.debug) {
- console.log(`Added user message to memory!: ${userMessage.content}`);
+ const call = tool.call;
+ let output: string;
+ if (!call) {
+ output = `Tool ${tool.metadata.name} (remote:${toolCall.name}) does not have a implementation.`;
+ return {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: true,
+ };
}
+ try {
+ let input = toolCall.input;
+ if (typeof input === "string") {
+ input = JSON.parse(input);
+ }
+ getCallbackManager().dispatchEvent("llm-tool-call", {
+ payload: {
+ toolCall: { ...toolCall },
+ },
+ });
+ output = await call.call(tool, input);
+ const toolOutput: ToolOutput = {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: false,
+ };
+ getCallbackManager().dispatchEvent("llm-tool-result", {
+ payload: {
+ toolCall: { ...toolCall },
+ toolResult: { ...toolOutput },
+ },
+ });
+ return toolOutput;
+ } catch (e) {
+ output = prettifyError(e);
+ }
+ return {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: true,
+ };
}
-export function getFunctionByName(tools: BaseTool[], name: string): BaseTool {
- const exist = tools.find((tool) => tool.metadata.name === name);
-
- if (!exist) {
- throw new Error(`Tool with name ${name} not found`);
+export async function consumeAsyncIterable<Options extends object>(
+ input: ChatMessage<Options>,
+): Promise<ChatMessage<Options>>;
+export async function consumeAsyncIterable<Options extends object>(
+ input: AsyncIterable<ChatResponseChunk<Options>>,
+): Promise<TextChatMessage<Options>>;
+export async function consumeAsyncIterable<Options extends object>(
+ input: ChatMessage<Options> | AsyncIterable<ChatResponseChunk<Options>>,
+): Promise<ChatMessage<Options>> {
+ if (isAsyncIterable(input)) {
+ const result: ChatMessage<Options> = {
+ content: "",
+ // only assistant will give streaming response
+ role: "assistant",
+ options: {} as Options,
+ };
+ for await (const chunk of input) {
+ result.content += chunk.delta;
+ if (chunk.options) {
+ result.options = {
+ ...result.options,
+ ...chunk.options,
+ };
+ }
+ }
+ return result;
+ } else {
+ return input;
}
+}
- return exist;
+export function createReadableStream<T>(
+ asyncIterable: AsyncIterable<T>,
+): ReadableStream<T> {
+ return new ReadableStream<T>({
+ async start(controller) {
+ for await (const chunk of asyncIterable) {
+ controller.enqueue(chunk);
+ }
+ controller.close();
+ },
+ });
}
diff --git a/packages/core/src/callbacks/CallbackManager.ts b/packages/core/src/callbacks/CallbackManager.ts
index 22c534c18..81bdc10b9 100644
--- a/packages/core/src/callbacks/CallbackManager.ts
+++ b/packages/core/src/callbacks/CallbackManager.ts
@@ -1,6 +1,7 @@
import type { Anthropic } from "@anthropic-ai/sdk";
import { CustomEvent } from "@llamaindex/env";
import type { NodeWithScore } from "../Node.js";
+import type { AgentEndEvent, AgentStartEvent } from "../agent/type.js";
import {
EventCaller,
getEventCaller,
@@ -10,6 +11,7 @@ import type {
LLMStartEvent,
LLMStreamEvent,
LLMToolCallEvent,
+ LLMToolResultEvent,
} from "../llm/types.js";
export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
@@ -47,10 +49,15 @@ export interface LlamaIndexEventMaps {
* @deprecated
*/
stream: CustomEvent<StreamCallbackResponse>;
+ // llm events
"llm-start": LLMStartEvent;
"llm-end": LLMEndEvent;
"llm-tool-call": LLMToolCallEvent;
+ "llm-tool-result": LLMToolResultEvent;
"llm-stream": LLMStreamEvent;
+ // agent events
+ "agent-start": AgentStartEvent;
+ "agent-end": AgentEndEvent;
}
//#region @deprecated remove in the next major version
@@ -205,9 +212,10 @@ export class CallbackManager implements CallbackManagerMethods {
if (!handlers) {
return;
}
+ const clone = structuredClone(detail);
queueMicrotask(() => {
handlers.forEach((handler) =>
- handler(LlamaIndexCustomEvent.fromEvent(event, detail)),
+ handler(LlamaIndexCustomEvent.fromEvent(event, clone)),
);
});
}
diff --git a/packages/core/src/engines/chat/types.ts b/packages/core/src/engines/chat/types.ts
index 768119afe..a3d554156 100644
--- a/packages/core/src/engines/chat/types.ts
+++ b/packages/core/src/engines/chat/types.ts
@@ -3,7 +3,6 @@ import type { NodeWithScore } from "../../Node.js";
import type { Response } from "../../Response.js";
import type { ChatMessage } from "../../llm/index.js";
import type { MessageContent } from "../../llm/types.js";
-import type { ToolOutput } from "../../tools/types.js";
/**
* Represents the base parameters for ChatEngine.
@@ -57,49 +56,3 @@ export interface Context {
export interface ContextGenerator {
generate(message: string): Promise<Context>;
}
-
-export enum ChatResponseMode {
- WAIT = "wait",
- STREAM = "stream",
-}
-
-export class AgentChatResponse {
- response: string;
- sources: ToolOutput[];
- sourceNodes?: NodeWithScore[];
-
- constructor(
- response: string,
- sources?: ToolOutput[],
- sourceNodes?: NodeWithScore[],
- ) {
- this.response = response;
- this.sources = sources || [];
- this.sourceNodes = sourceNodes || [];
- }
-
- protected _getFormattedSources() {
- throw new Error("Not implemented yet");
- }
-
- toString() {
- return this.response ?? "";
- }
-}
-
-export class StreamingAgentChatResponse {
- response: AsyncIterable<Response>;
-
- sources: ToolOutput[];
- sourceNodes?: NodeWithScore[];
-
- constructor(
- response: AsyncIterable<Response>,
- sources?: ToolOutput[],
- sourceNodes?: NodeWithScore[],
- ) {
- this.response = response;
- this.sources = sources ?? [];
- this.sourceNodes = sourceNodes ?? [];
- }
-}
diff --git a/packages/core/src/internal/context/EventCaller.ts b/packages/core/src/internal/context/EventCaller.ts
index 26c6a631f..ab7a2d9a1 100644
--- a/packages/core/src/internal/context/EventCaller.ts
+++ b/packages/core/src/internal/context/EventCaller.ts
@@ -1,5 +1,5 @@
import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
-import { isAsyncGenerator, isGenerator } from "../utils.js";
+import { isAsyncIterable, isIterable } from "../utils.js";
const eventReasonAsyncLocalStorage = new AsyncLocalStorage<EventCaller>();
@@ -71,22 +71,24 @@ export function wrapEventCaller<This, Result, Args extends unknown[]>(
return function (this: This, ...args: Args): Result {
const result = originalMethod.call(this, ...args);
// patch for iterators because AsyncLocalStorage doesn't work with them
- if (isAsyncGenerator(result)) {
+ if (isAsyncIterable(result)) {
+ const iter = result[Symbol.asyncIterator]();
const snapshot = AsyncLocalStorage.snapshot();
return (async function* asyncGeneratorWrapper() {
while (true) {
- const { value, done } = await snapshot(() => result.next());
+ const { value, done } = await snapshot(() => iter.next());
if (done) {
break;
}
yield value;
}
})() as Result;
- } else if (isGenerator(result)) {
+ } else if (isIterable(result)) {
+ const iter = result[Symbol.iterator]();
const snapshot = AsyncLocalStorage.snapshot();
return (function* generatorWrapper() {
while (true) {
- const { value, done } = snapshot(() => result.next());
+ const { value, done } = snapshot(() => iter.next());
if (done) {
break;
}
diff --git a/packages/core/src/agent/react/prompts.ts b/packages/core/src/internal/prompt/react.ts
similarity index 68%
rename from packages/core/src/agent/react/prompts.ts
rename to packages/core/src/internal/prompt/react.ts
index 75e98a468..3f4450b91 100644
--- a/packages/core/src/agent/react/prompts.ts
+++ b/packages/core/src/internal/prompt/react.ts
@@ -1,13 +1,14 @@
-type ReactChatSystemHeaderParams = {
- toolDesc: string;
- toolNames: string;
-};
-
-export const getReactChatSystemHeader = ({
- toolDesc,
- toolNames,
-}: ReactChatSystemHeaderParams) =>
- `You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.
+import type { BaseTool } from "../../types.js";
+
+export const getReACTAgentSystemHeader = (tools: BaseTool[]) => {
+ const description = tools
+ .map(
+ (tool) =>
+ `- ${tool.metadata.name}: ${tool.metadata.description} with schema: ${JSON.stringify(tool.metadata.parameters)}`,
+ )
+ .join("\n");
+ const names = tools.map((tool) => tool.metadata.name).join(", ");
+ return `You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.
## Tools
You have access to a wide variety of tools. You are responsible for using
@@ -16,14 +17,14 @@ This may require breaking the task into subtasks and using different tools
to complete each subtask.
You have access to the following tools:
-${toolDesc}
+${description}
## Output Format
To answer the question, please use the following format.
"""
Thought: I need to use a tool to help me answer the question.
-Action: tool name (one of ${toolNames}) if using a tool.
+Action: tool name (one of ${names}) if using a tool.
Action Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{"input": "hello world", "num_beams": 5}})
"""
@@ -52,5 +53,5 @@ Answer: Sorry, I cannot answer your query.
""""
## Current Conversation
-Below is the current conversation consisting of interleaving human and assistant messages.
-`;
+Below is the current conversation consisting of interleaving human and assistant messages.`;
+};
diff --git a/packages/core/src/internal/type.ts b/packages/core/src/internal/type.ts
new file mode 100644
index 000000000..b93af22a0
--- /dev/null
+++ b/packages/core/src/internal/type.ts
@@ -0,0 +1,5 @@
+import { CustomEvent } from "@llamaindex/env";
+
+export type BaseEvent<Payload extends Record<string, unknown>> = CustomEvent<{
+ payload: Payload;
+}>;
diff --git a/packages/core/src/internal/utils.ts b/packages/core/src/internal/utils.ts
index 63133c61a..d142bf1db 100644
--- a/packages/core/src/internal/utils.ts
+++ b/packages/core/src/internal/utils.ts
@@ -1,8 +1,10 @@
-export const isAsyncGenerator = (obj: unknown): obj is AsyncGenerator => {
+export const isAsyncIterable = (
+ obj: unknown,
+): obj is AsyncIterable<unknown> => {
return obj != null && typeof obj === "object" && Symbol.asyncIterator in obj;
};
-export const isGenerator = (obj: unknown): obj is Generator => {
+export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
return obj != null && typeof obj === "object" && Symbol.iterator in obj;
};
diff --git a/packages/core/src/llm/types.ts b/packages/core/src/llm/types.ts
index 26da7b4ee..211269536 100644
--- a/packages/core/src/llm/types.ts
+++ b/packages/core/src/llm/types.ts
@@ -1,24 +1,23 @@
import type { Tokenizers } from "../GlobalsHelper.js";
-import type { BaseTool, UUID } from "../types.js";
+import type { BaseEvent } from "../internal/type.js";
+import type { BaseTool, ToolOutput, UUID } from "../types.js";
-type LLMBaseEvent<Payload extends Record<string, unknown>> = CustomEvent<{
- payload: Payload;
-}>;
-
-export type LLMStartEvent = LLMBaseEvent<{
+export type LLMStartEvent = BaseEvent<{
id: UUID;
messages: ChatMessage[];
}>;
-export type LLMToolCallEvent = LLMBaseEvent<{
- // fixme: id is missing in the context
- // id: UUID;
- toolCall: Omit<ToolCallOptions["toolCall"], "id">;
+export type LLMToolCallEvent = BaseEvent<{
+ toolCall: ToolCall;
+}>;
+export type LLMToolResultEvent = BaseEvent<{
+ toolCall: ToolCall;
+ toolResult: ToolOutput;
}>;
-export type LLMEndEvent = LLMBaseEvent<{
+export type LLMEndEvent = BaseEvent<{
id: UUID;
response: ChatResponse;
}>;
-export type LLMStreamEvent = LLMBaseEvent<{
+export type LLMStreamEvent = BaseEvent<{
id: UUID;
chunk: ChatResponseChunk;
}>;
@@ -77,6 +76,13 @@ export interface LLM<
export type MessageType = "user" | "assistant" | "system" | "memory";
+export type TextChatMessage<AdditionalMessageOptions extends object = object> =
+ {
+ content: string;
+ role: MessageType;
+ options?: undefined | AdditionalMessageOptions;
+ };
+
export type ChatMessage<AdditionalMessageOptions extends object = object> = {
content: MessageContent;
role: MessageType;
diff --git a/packages/core/src/objects/base.ts b/packages/core/src/objects/base.ts
index 3ef6832d4..3eade2994 100644
--- a/packages/core/src/objects/base.ts
+++ b/packages/core/src/objects/base.ts
@@ -2,6 +2,8 @@ import type { BaseNode, Metadata } from "../Node.js";
import { TextNode } from "../Node.js";
import type { BaseRetriever } from "../Retriever.js";
import type { VectorStoreIndex } from "../indices/index.js";
+import type { MessageContent } from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
import type { BaseTool } from "../types.js";
// Assuming that necessary interfaces and classes (like OT, TextNode, BaseNode, etc.) are defined elsewhere
@@ -48,7 +50,8 @@ export abstract class BaseObjectNodeMapping {
// You will need to implement specific subclasses of BaseObjectNodeMapping as per your project requirements.
-type QueryType = string;
+// todo: multimodal support
+type QueryType = MessageContent;
export class ObjectRetriever<T = unknown> {
_retriever: BaseRetriever;
@@ -69,7 +72,9 @@ export class ObjectRetriever<T = unknown> {
// Translating the retrieve method
async retrieve(strOrQueryBundle: QueryType): Promise<T[]> {
- const nodes = await this.retriever.retrieve({ query: strOrQueryBundle });
+ const nodes = await this.retriever.retrieve({
+ query: extractText(strOrQueryBundle),
+ });
const objs = nodes.map((n) => this._objectNodeMapping.fromNode(n.node));
return objs;
}
diff --git a/packages/core/src/tools/index.ts b/packages/core/src/tools/index.ts
index 2b52c32ed..d8629c0e3 100644
--- a/packages/core/src/tools/index.ts
+++ b/packages/core/src/tools/index.ts
@@ -1,5 +1,3 @@
export * from "./QueryEngineTool.js";
export * from "./WikipediaTool.js";
export * from "./functionTool.js";
-export * from "./types.js";
-export * from "./utils.js";
diff --git a/packages/core/src/tools/types.ts b/packages/core/src/tools/types.ts
deleted file mode 100644
index f98f14f72..000000000
--- a/packages/core/src/tools/types.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-export class ToolOutput {
- content: string;
- toolName: string;
- rawInput: any;
- rawOutput: any;
-
- constructor(
- content: string,
- toolName: string,
- rawInput: any,
- rawOutput: any,
- ) {
- this.content = content;
- this.toolName = toolName;
- this.rawInput = rawInput;
- this.rawOutput = rawOutput;
- }
-
- toString(): string {
- return this.content;
- }
-}
diff --git a/packages/core/src/tools/utils.ts b/packages/core/src/tools/utils.ts
deleted file mode 100644
index afc5280aa..000000000
--- a/packages/core/src/tools/utils.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import type { BaseTool } from "../types.js";
-import { ToolOutput } from "./types.js";
-
-export async function callToolWithErrorHandling(
- tool: BaseTool,
- input: unknown,
-): Promise<ToolOutput> {
- if (!tool.call) {
- return new ToolOutput(
- "Error: Tool does not have a call function.",
- tool.metadata.name,
- input,
- null,
- );
- }
- try {
- getCallbackManager().dispatchEvent("llm-tool-call", {
- payload: {
- toolCall: {
- name: tool.metadata.name,
- input,
- },
- },
- });
- const value = await tool.call(
- typeof input === "string" ? JSON.parse(input) : input,
- );
- return new ToolOutput(value, tool.metadata.name, input, value);
- } catch (e) {
- return new ToolOutput(`Error: ${e}`, tool.metadata.name, input, e);
- }
-}
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index a51b1182e..17433e08a 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -103,3 +103,10 @@ export class QueryBundle {
}
export type UUID = `${string}-${string}-${string}-${string}-${string}`;
+
+export type ToolOutput = {
+ tool: BaseTool | undefined;
+ input: unknown;
+ output: string;
+ isError: boolean;
+};
diff --git a/packages/core/tests/agent/runner/AgentRunner.test.ts b/packages/core/tests/agent/runner/AgentRunner.test.ts
deleted file mode 100644
index 4b8b4af76..000000000
--- a/packages/core/tests/agent/runner/AgentRunner.test.ts
+++ /dev/null
@@ -1,85 +0,0 @@
-import { OpenAIAgentWorker } from "llamaindex/agent/index";
-import { AgentRunner } from "llamaindex/agent/runner/base";
-import { OpenAI } from "llamaindex/llm/open_ai";
-import { beforeEach, describe, expect, it } from "vitest";
-
-import { Settings } from "llamaindex";
-import {
- DEFAULT_LLM_TEXT_OUTPUT,
- mockLlmGeneration,
-} from "../../utility/mockOpenAI.js";
-
-describe("Agent Runner", () => {
- let agentRunner: AgentRunner;
-
- beforeEach(() => {
- Settings.llm = new OpenAI({
- model: "gpt-3.5-turbo",
- });
-
- mockLlmGeneration();
-
- agentRunner = new AgentRunner({
- agentWorker: new OpenAIAgentWorker({
- tools: [],
- }),
- });
- });
-
- it("should be able to initialize a task", () => {
- const task = agentRunner.createTask("hello world");
-
- expect(task.input).toEqual("hello world");
- expect(task.taskId in agentRunner.state.taskDict).toEqual(true);
-
- expect(agentRunner.listTasks().length).toEqual(1);
- });
-
- it("should be able to run a step", async () => {
- const task = agentRunner.createTask("hello world");
-
- expect(agentRunner.getCompletedSteps(task.taskId)).toBeUndefined();
-
- const stepOutput = await agentRunner.runStep(task.taskId, task.input);
-
- const completedSteps = agentRunner.getCompletedSteps(task.taskId);
-
- expect(completedSteps.length).toEqual(1);
-
- expect(stepOutput.isLast).toEqual(true);
- });
-
- it("should be able to finalize a task", async () => {
- const task = agentRunner.createTask("hello world");
-
- expect(agentRunner.getCompletedSteps(task.taskId)).toBeUndefined();
-
- const stepOutput1 = await agentRunner.runStep(task.taskId, task.input);
-
- expect(stepOutput1.isLast).toEqual(true);
- });
-
- it("should be able to delete a task", () => {
- const task = agentRunner.createTask("hello world");
-
- expect(agentRunner.listTasks().length).toEqual(1);
-
- agentRunner.deleteTask(task.taskId);
-
- expect(agentRunner.listTasks().length).toEqual(0);
- });
-
- it("should be able to run a chat", async () => {
- const response = await agentRunner.chat({
- message: "hello world",
- });
-
- expect(agentRunner.listTasks().length).toEqual(1);
-
- expect(response).toEqual({
- response: DEFAULT_LLM_TEXT_OUTPUT,
- sourceNodes: [],
- sources: [],
- });
- });
-});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index cb4615cce..df4ddf454 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -147,8 +147,8 @@ importers:
specifier: ^1.0.11
version: 1.0.11
llamaindex:
- specifier: latest
- version: link:../packages/core
+ specifier: '*'
+ version: 0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.5)
mongodb:
specifier: ^6.5.0
version: 6.5.0
@@ -172,8 +172,8 @@ importers:
examples/readers:
dependencies:
llamaindex:
- specifier: latest
- version: 0.2.9(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3)
+ specifier: '*'
+ version: 0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3)
devDependencies:
'@types/node':
specifier: ^20.12.7
@@ -1410,11 +1410,6 @@ packages:
'@dabh/diagnostics@2.0.3':
resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==}
- '@datastax/astra-db-ts@0.1.4':
- resolution: {integrity: sha512-EG/7UUuEdxpeyGV1fkGIUX5jjUcESToCtohoti0rNMEm01T1E4NXOPHXMnkyXo71zqrlUoTlGn5du+acnlbslQ==}
- engines: {node: '>=14.0.0'}
- hasBin: true
-
'@datastax/astra-db-ts@1.0.1':
resolution: {integrity: sha512-TSfP/Lv0ghPc01lVqiXGdunM+2CsBt8tG/9cqTs8K3LaM5vRJJj760Q2+udif834rrbXM4xN8LZQeWPfVPy6BA==}
engines: {node: '>=14.0.0'}
@@ -5586,8 +5581,8 @@ packages:
resolution: {integrity: sha512-ovJXBXkKGfq+CwmKTjluEqFi3p4h8xvkxGQQAQan22YCgef4KZ1mKGjzfGh6PL6AW5Csw0QiQPNuQyH+6Xk3hA==}
engines: {node: '>=18.0.0'}
- llamaindex@0.2.9:
- resolution: {integrity: sha512-Ifmk4Bu0CEKI3cwfER3WyZSP7+gOBHgtWobsEJ1c+1pYcIMipwxpqBUba/YXQceuEmh7xsCd0+ROkoNumUX3CQ==}
+ llamaindex@0.2.10:
+ resolution: {integrity: sha512-GXO/H4k6iF0dQStg1kOTYYm0pnMbD1gM8LwRKEPOeC/mY+Q2pyIyDB22cPc8nOTf+ah3rbLiXOxTORTUmC1xKA==}
engines: {node: '>=18.0.0'}
load-yaml-file@0.2.0:
@@ -7326,10 +7321,6 @@ packages:
renderkid@3.0.0:
resolution: {integrity: sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==}
- replicate@0.25.2:
- resolution: {integrity: sha512-c5otBJ5E66XLS0X196pBCsyy85b03ZBLeV/lbKfU8cqfkt3Qd6NGEiPwTtxtsQ4AznggMJNn2Qq68t/bV85M2w==}
- engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'}
-
require-directory@2.1.1:
resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
engines: {node: '>=0.10.0'}
@@ -9897,14 +9888,6 @@ snapshots:
enabled: 2.0.0
kuler: 2.0.0
- '@datastax/astra-db-ts@0.1.4':
- dependencies:
- axios: 1.6.8
- bson: 6.6.0
- winston: 3.13.0
- transitivePeerDependencies:
- - debug
-
'@datastax/astra-db-ts@1.0.1':
dependencies:
bson-objectid: 2.0.4
@@ -15114,11 +15097,11 @@ snapshots:
rfdc: 1.3.1
wrap-ansi: 9.0.0
- llamaindex@0.2.9(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3):
+ llamaindex@0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3):
dependencies:
'@anthropic-ai/sdk': 0.20.6(encoding@0.1.13)
'@aws-crypto/sha256-js': 5.2.0
- '@datastax/astra-db-ts': 0.1.4
+ '@datastax/astra-db-ts': 1.0.1
'@grpc/grpc-js': 1.10.6
'@llamaindex/cloud': 0.0.5(node-fetch@2.7.0(encoding@0.1.13))
'@llamaindex/env': 0.0.7(@aws-crypto/sha256-js@5.2.0)(pathe@1.1.2)(readable-stream@4.5.2)
@@ -15151,7 +15134,64 @@ snapshots:
pgvector: 0.1.8
portkey-ai: 0.1.16
rake-modified: 1.0.8
- replicate: 0.25.2
+ string-strip-html: 13.4.8
+ wikipedia: 2.1.2
+ wink-nlp: 1.14.3
+ transitivePeerDependencies:
+ - '@aws-sdk/credential-providers'
+ - '@google/generative-ai'
+ - '@mongodb-js/zstd'
+ - bufferutil
+ - debug
+ - encoding
+ - gcp-metadata
+ - kerberos
+ - mongodb-client-encryption
+ - node-fetch
+ - pg-native
+ - readable-stream
+ - snappy
+ - socks
+ - typescript
+ - utf-8-validate
+
+ llamaindex@0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.5):
+ dependencies:
+ '@anthropic-ai/sdk': 0.20.6(encoding@0.1.13)
+ '@aws-crypto/sha256-js': 5.2.0
+ '@datastax/astra-db-ts': 1.0.1
+ '@grpc/grpc-js': 1.10.6
+ '@llamaindex/cloud': 0.0.5(node-fetch@2.7.0(encoding@0.1.13))
+ '@llamaindex/env': 0.0.7(@aws-crypto/sha256-js@5.2.0)(pathe@1.1.2)(readable-stream@4.5.2)
+ '@mistralai/mistralai': 0.1.3(encoding@0.1.13)
+ '@notionhq/client': 2.2.15(encoding@0.1.13)
+ '@pinecone-database/pinecone': 2.2.0
+ '@qdrant/js-client-rest': 1.8.2(typescript@5.4.5)
+ '@types/lodash': 4.17.0
+ '@types/node': 20.12.7
+ '@types/papaparse': 5.3.14
+ '@types/pg': 8.11.5
+ '@xenova/transformers': 2.17.1
+ '@zilliz/milvus2-sdk-node': 2.4.1
+ ajv: 8.12.0
+ assemblyai: 4.4.1
+ chromadb: 1.7.3(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.38.0(encoding@0.1.13))
+ cohere-ai: 7.9.5(encoding@0.1.13)
+ js-tiktoken: 1.0.11
+ lodash: 4.17.21
+ magic-bytes.js: 1.10.0
+ mammoth: 1.7.1
+ md-utils-ts: 2.0.0
+ mongodb: 6.5.0
+ notion-md-crawler: 0.0.2(encoding@0.1.13)
+ openai: 4.38.0(encoding@0.1.13)
+ papaparse: 5.4.1
+ pathe: 1.1.2
+ pdf2json: 3.0.5
+ pg: 8.11.5
+ pgvector: 0.1.8
+ portkey-ai: 0.1.16
+ rake-modified: 1.0.8
string-strip-html: 13.4.8
wikipedia: 2.1.2
wink-nlp: 1.14.3
@@ -17329,10 +17369,6 @@ snapshots:
lodash: 4.17.21
strip-ansi: 6.0.1
- replicate@0.25.2:
- optionalDependencies:
- readable-stream: 4.5.2
-
require-directory@2.1.1: {}
require-from-string@2.0.2: {}
diff --git a/tsconfig.json b/tsconfig.json
index 43f6f276e..f2cac2115 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,6 +1,6 @@
{
"compilerOptions": {
- "target": "es2016",
+ "target": "es2022",
"module": "esnext",
"moduleResolution": "bundler",
"verbatimModuleSyntax": true,
--
GitLab