diff --git a/.npmrc b/.npmrc
index b151845bf3bba5af548b9196680a3fa4039ccb19..5da1bc1c63972cde6364e06aed0f0b8fecba8f31 100644
--- a/.npmrc
+++ b/.npmrc
@@ -1,3 +1,4 @@
auto-install-peers = true
enable-pre-post-scripts = true
-prefer-workspace-packages: true
+prefer-workspace-packages = true
+save-workspace-protocol = true
diff --git a/apps/docs/docs/examples/agent.mdx b/apps/docs/docs/examples/agent.mdx
index 2b37e9ccebfa4fee72e493e5f3d33268f52c116f..08ce623b5cde01e194f684d05a0f144117db3297 100644
--- a/apps/docs/docs/examples/agent.mdx
+++ b/apps/docs/docs/examples/agent.mdx
@@ -4,81 +4,7 @@ A built-in agent that can take decisions and reasoning based on the tools provid
## OpenAI Agent
-```ts
-import { FunctionTool, OpenAIAgent } from "llamaindex";
+import CodeBlock from "@theme/CodeBlock";
+import CodeSource from "!raw-loader!../../../../examples/agent/openai";
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-// Define the parameters of the divide function as a JSON schema
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend to divide",
- },
- b: {
- type: "number",
- description: "The divisor to divide by",
- },
- },
- required: ["a", "b"],
-};
-
-async function main() {
- // Create a function tool from the sum function
- const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers"
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
+<CodeBlock language="ts">{CodeSource}</CodeBlock>
diff --git a/apps/docs/docs/modules/agent/index.md b/apps/docs/docs/modules/agent/index.md
index 99e2d1f4ab5fa04e9898b4906c4d746a2207de2c..65a3f8cd4502d0320f10055e15a27d9a0ce111cb 100644
--- a/apps/docs/docs/modules/agent/index.md
+++ b/apps/docs/docs/modules/agent/index.md
@@ -11,4 +11,10 @@ An “agent†is an automated reasoning and decision engine. It takes in a user
LlamaIndex.TS comes with a few built-in agents, but you can also create your own. The built-in agents include:
-- [OpenAI Agent](./openai.mdx)
+- OpenAI Agent
+- Anthropic Agent
+- ReACT Agent
+
+## Examples
+
+- [OpenAI Agent](../../examples/agent.mdx)
diff --git a/apps/docs/docs/modules/agent/multi_document_agent.mdx b/apps/docs/docs/modules/agent/multi_document_agent.mdx
deleted file mode 100644
index f51f459ff9902c5b25a34be3350f3acaaf88baf7..0000000000000000000000000000000000000000
--- a/apps/docs/docs/modules/agent/multi_document_agent.mdx
+++ /dev/null
@@ -1,307 +0,0 @@
-# Multi-Document Agent
-
-In this guide, you learn towards setting up an agent that can effectively answer different types of questions over a larger set of documents.
-
-These questions include the following
-
-- QA over a specific doc
-- QA comparing different docs
-- Summaries over a specific doc
-- Comparing summaries between different docs
-
-We do this with the following architecture:
-
-- setup a “document agent†over each Document: each doc agent can do QA/summarization within its doc
-- setup a top-level agent over this set of document agents. Do tool retrieval and then do CoT over the set of tools to answer a question.
-
-## Setup and Download Data
-
-We first start by installing the necessary libraries and downloading the data.
-
-```bash
-pnpm i llamaindex
-```
-
-```ts
-import {
- Document,
- ObjectIndex,
- OpenAI,
- OpenAIAgent,
- QueryEngineTool,
- SimpleNodeParser,
- SimpleToolNodeMapping,
- SummaryIndex,
- VectorStoreIndex,
- Settings,
- storageContextFromDefaults,
-} from "llamaindex";
-```
-
-And then for the data we will run through a list of countries and download the wikipedia page for each country.
-
-```ts
-import fs from "fs";
-import path from "path";
-
-const dataPath = path.join(__dirname, "tmp_data");
-
-const extractWikipediaTitle = async (title: string) => {
- const fileExists = fs.existsSync(path.join(dataPath, `${title}.txt`));
-
- if (fileExists) {
- console.log(`File already exists for the title: ${title}`);
- return;
- }
-
- const queryParams = new URLSearchParams({
- action: "query",
- format: "json",
- titles: title,
- prop: "extracts",
- explaintext: "true",
- });
-
- const url = `https://en.wikipedia.org/w/api.php?${queryParams}`;
-
- const response = await fetch(url);
- const data: any = await response.json();
-
- const pages = data.query.pages;
- const page = pages[Object.keys(pages)[0]];
- const wikiText = page.extract;
-
- await new Promise((resolve) => {
- fs.writeFile(path.join(dataPath, `${title}.txt`), wikiText, (err: any) => {
- if (err) {
- console.error(err);
- resolve(title);
- return;
- }
- console.log(`${title} stored in file!`);
-
- resolve(title);
- });
- });
-};
-```
-
-```ts
-export const extractWikipedia = async (titles: string[]) => {
- if (!fs.existsSync(dataPath)) {
- fs.mkdirSync(dataPath);
- }
-
- for await (const title of titles) {
- await extractWikipediaTitle(title);
- }
-
- console.log("Extration finished!");
-```
-
-These files will be saved in the `tmp_data` folder.
-
-Now we can call the function to download the data for each country.
-
-```ts
-await extractWikipedia([
- "Brazil",
- "United States",
- "Canada",
- "Mexico",
- "Argentina",
- "Chile",
- "Colombia",
- "Peru",
- "Venezuela",
- "Ecuador",
- "Bolivia",
- "Paraguay",
- "Uruguay",
- "Guyana",
- "Suriname",
- "French Guiana",
- "Falkland Islands",
-]);
-```
-
-## Load the data
-
-Now that we have the data, we can load it into the LlamaIndex and store as a document.
-
-```ts
-import { Document } from "llamaindex";
-
-const countryDocs: Record<string, Document> = {};
-
-for (const title of wikiTitles) {
- const path = `./agent/helpers/tmp_data/${title}.txt`;
- const text = await fs.readFile(path, "utf-8");
- const document = new Document({ text: text, id_: path });
- countryDocs[title] = document;
-}
-```
-
-## Setup LLM and StorageContext
-
-We will be using gpt-4 for this example and we will use the `StorageContext` to store the documents in-memory.
-
-```ts
-Settings.llm = new OpenAI({
- model: "gpt-4",
-});
-
-const storageContext = await storageContextFromDefaults({
- persistDir: "./storage",
-});
-```
-
-## Building Multi-Document Agents
-
-In this section we show you how to construct the multi-document agent. We first build a document agent for each document, and then define the top-level parent agent with an object index.
-
-```ts
-const documentAgents: Record<string, any> = {};
-const queryEngines: Record<string, any> = {};
-```
-
-Now we iterate over each country and create a document agent for each one.
-
-### Build Agent for each Document
-
-In this section we define “document agents†for each document.
-
-We define both a vector index (for semantic search) and summary index (for summarization) for each document. The two query engines are then converted into tools that are passed to an OpenAI function calling agent.
-
-This document agent can dynamically choose to perform semantic search or summarization within a given document.
-
-We create a separate document agent for each coutnry.
-
-```ts
-for (const title of wikiTitles) {
- // parse the document into nodes
- const nodes = new SimpleNodeParser({
- chunkSize: 200,
- chunkOverlap: 20,
- }).getNodesFromDocuments([countryDocs[title]]);
-
- // create the vector index for specific search
- const vectorIndex = await VectorStoreIndex.init({
- storageContext: storageContext,
- nodes,
- });
-
- // create the summary index for broader search
- const summaryIndex = await SummaryIndex.init({
- nodes,
- });
-
- const vectorQueryEngine = summaryIndex.asQueryEngine();
- const summaryQueryEngine = summaryIndex.asQueryEngine();
-
- // create the query engines for each task
- const queryEngineTools = [
- new QueryEngineTool({
- queryEngine: vectorQueryEngine,
- metadata: {
- name: "vector_tool",
- description: `Useful for questions related to specific aspects of ${title} (e.g. the history, arts and culture, sports, demographics, or more).`,
- },
- }),
- new QueryEngineTool({
- queryEngine: summaryQueryEngine,
- metadata: {
- name: "summary_tool",
- description: `Useful for any requests that require a holistic summary of EVERYTHING about ${title}. For questions about more specific sections, please use the vector_tool.`,
- },
- }),
- ];
-
- // create the document agent
- const agent = new OpenAIAgent({
- tools: queryEngineTools,
- llm,
- });
-
- documentAgents[title] = agent;
- queryEngines[title] = vectorIndex.asQueryEngine();
-}
-```
-
-## Build Top-Level Agent
-
-Now we define the top-level agent that can answer questions over the set of document agents.
-
-This agent takes in all document agents as tools. This specific agent RetrieverOpenAIAgent performs tool retrieval before tool use (unlike a default agent that tries to put all tools in the prompt).
-
-Here we use a top-k retriever, but we encourage you to customize the tool retriever method!
-
-Firstly, we create a tool for each document agent
-
-```ts
-const allTools: QueryEngineTool[] = [];
-```
-
-```ts
-for (const title of wikiTitles) {
- const wikiSummary = `
- This content contains Wikipedia articles about ${title}.
- Use this tool if you want to answer any questions about ${title}
- `;
-
- const docTool = new QueryEngineTool({
- queryEngine: documentAgents[title],
- metadata: {
- name: `tool_${title}`,
- description: wikiSummary,
- },
- });
-
- allTools.push(docTool);
-}
-```
-
-Our top level agent will use this document agents as tools and use toolRetriever to retrieve the best tool to answer a question.
-
-```ts
-// map the tools to nodes
-const toolMapping = SimpleToolNodeMapping.fromObjects(allTools);
-
-// create the object index
-const objectIndex = await ObjectIndex.fromObjects(
- allTools,
- toolMapping,
- VectorStoreIndex,
- {
- storageContext,
- },
-);
-
-// create the top agent
-const topAgent = new OpenAIAgent({
- toolRetriever: await objectIndex.asRetriever({}),
- llm,
- prefixMessages: [
- {
- content:
- "You are an agent designed to answer queries about a set of given countries. Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
- role: "system",
- },
- ],
-});
-```
-
-## Use the Agent
-
-Now we can use the agent to answer questions.
-
-```ts
-const response = await topAgent.chat({
- message: "Tell me the differences between Brazil and Canada economics?",
-});
-
-// print output
-console.log(response);
-```
-
-You can find the full code for this example [here](https://github.com/run-llama/LlamaIndexTS/tree/main/examples/agent/multi-document-agent.ts)
diff --git a/apps/docs/docs/modules/agent/openai.mdx b/apps/docs/docs/modules/agent/openai.mdx
deleted file mode 100644
index 7b47dd78e2f9f296dbcab496311c0d99d852aabd..0000000000000000000000000000000000000000
--- a/apps/docs/docs/modules/agent/openai.mdx
+++ /dev/null
@@ -1,185 +0,0 @@
----
-sidebar_position: 0
----
-
-# OpenAI Agent
-
-OpenAI API that supports function calling, it’s never been easier to build your own agent!
-
-In this notebook tutorial, we showcase how to write your own OpenAI agent
-
-## Setup
-
-First, you need to install the `llamaindex` package. You can do this by running the following command in your terminal:
-
-```bash
-pnpm i llamaindex
-```
-
-Then we can define a function to sum two numbers and another function to divide two numbers.
-
-```ts
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-```
-
-## Create a function tool
-
-Now we can create a function tool from the sum function and another function tool from the divide function.
-
-For the parameters of the sum function, we can define a JSON schema.
-
-### JSON Schema
-
-```ts
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend a to divide",
- },
- b: {
- type: "number",
- description: "The divisor b to divide by",
- },
- },
- required: ["a", "b"],
-};
-
-const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
-});
-
-const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
-});
-```
-
-## Create an OpenAIAgent
-
-Now we can create an OpenAIAgent with the function tools.
-
-```ts
-const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
-});
-```
-
-## Chat with the agent
-
-Now we can chat with the agent.
-
-```ts
-const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
-});
-
-console.log(String(response));
-```
-
-## Full code
-
-```ts
-import { FunctionTool, OpenAIAgent } from "llamaindex";
-
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-// Define the parameters of the divide function as a JSON schema
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The argument a to divide",
- },
- b: {
- type: "number",
- description: "The argument b to divide",
- },
- },
- required: ["a", "b"],
-};
-
-async function main() {
- // Create a function tool from the sum function
- const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
diff --git a/apps/docs/docs/modules/agent/query_engine_tool.mdx b/apps/docs/docs/modules/agent/query_engine_tool.mdx
deleted file mode 100644
index 4b9a3c399552f00e86577bc64aa15add427cd4da..0000000000000000000000000000000000000000
--- a/apps/docs/docs/modules/agent/query_engine_tool.mdx
+++ /dev/null
@@ -1,130 +0,0 @@
----
-sidebar_position: 1
----
-
-# OpenAI Agent + QueryEngineTool
-
-QueryEngineTool is a tool that allows you to query a vector index. In this example, we will create a vector index from a set of documents and then create a QueryEngineTool from the vector index. We will then create an OpenAIAgent with the QueryEngineTool and chat with the agent.
-
-## Setup
-
-First, you need to install the `llamaindex` package. You can do this by running the following command in your terminal:
-
-```bash
-pnpm i llamaindex
-```
-
-Then you can import the necessary classes and functions.
-
-```ts
-import {
- OpenAIAgent,
- SimpleDirectoryReader,
- VectorStoreIndex,
- QueryEngineTool,
-} from "llamaindex";
-```
-
-## Create a vector index
-
-Now we can create a vector index from a set of documents.
-
-```ts
-// Load the documents
-const documents = await new SimpleDirectoryReader().loadData({
- directoryPath: "node_modules/llamaindex/examples/",
-});
-
-// Create a vector index from the documents
-const vectorIndex = await VectorStoreIndex.fromDocuments(documents);
-```
-
-## Create a QueryEngineTool
-
-Now we can create a QueryEngineTool from the vector index.
-
-```ts
-// Create a query engine from the vector index
-const abramovQueryEngine = vectorIndex.asQueryEngine();
-
-// Create a QueryEngineTool with the query engine
-const queryEngineTool = new QueryEngineTool({
- queryEngine: abramovQueryEngine,
- metadata: {
- name: "abramov_query_engine",
- description: "A query engine for the Abramov documents",
- },
-});
-```
-
-## Create an OpenAIAgent
-
-```ts
-// Create an OpenAIAgent with the query engine tool tools
-
-const agent = new OpenAIAgent({
- tools: [queryEngineTool],
-});
-```
-
-## Chat with the agent
-
-Now we can chat with the agent.
-
-```ts
-const response = await agent.chat({
- message: "What was his salary?",
-});
-
-console.log(String(response));
-```
-
-## Full code
-
-```ts
-import {
- OpenAIAgent,
- SimpleDirectoryReader,
- VectorStoreIndex,
- QueryEngineTool,
-} from "llamaindex";
-
-async function main() {
- // Load the documents
- const documents = await new SimpleDirectoryReader().loadData({
- directoryPath: "node_modules/llamaindex/examples/",
- });
-
- // Create a vector index from the documents
- const vectorIndex = await VectorStoreIndex.fromDocuments(documents);
-
- // Create a query engine from the vector index
- const abramovQueryEngine = vectorIndex.asQueryEngine();
-
- // Create a QueryEngineTool with the query engine
- const queryEngineTool = new QueryEngineTool({
- queryEngine: abramovQueryEngine,
- metadata: {
- name: "abramov_query_engine",
- description: "A query engine for the Abramov documents",
- },
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [queryEngineTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "What was his salary?",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
diff --git a/apps/docs/docs/modules/agent/react_agent.mdx b/apps/docs/docs/modules/agent/react_agent.mdx
deleted file mode 100644
index ba31674f004eab4bbf8d36bfb8fe8614cecfa741..0000000000000000000000000000000000000000
--- a/apps/docs/docs/modules/agent/react_agent.mdx
+++ /dev/null
@@ -1,201 +0,0 @@
-# ReAct Agent
-
-The ReAct agent is an AI agent that can reason over the next action, construct an action command, execute the action, and repeat these steps in an iterative loop until the task is complete.
-
-In this notebook tutorial, we showcase how to write your ReAct agent using the `llamaindex` package.
-
-## Setup
-
-First, you need to install the `llamaindex` package. You can do this by running the following command in your terminal:
-
-```bash
-pnpm i llamaindex
-```
-
-And then you can import the `OpenAIAgent` and `FunctionTool` from the `llamaindex` package.
-
-```ts
-import { FunctionTool, OpenAIAgent } from "llamaindex";
-```
-
-Then we can define a function to sum two numbers and another function to divide two numbers.
-
-```ts
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-```
-
-## Create a function tool
-
-Now we can create a function tool from the sum function and another function tool from the divide function.
-
-For the parameters of the sum function, we can define a JSON schema.
-
-### JSON Schema
-
-```ts
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend a to divide",
- },
- b: {
- type: "number",
- description: "The divisor b to divide by",
- },
- },
- required: ["a", "b"],
-};
-
-const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
-});
-
-const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
-});
-```
-
-## Create an ReAct
-
-Now we can create an OpenAIAgent with the function tools.
-
-```ts
-const agent = new ReActAgent({
- tools: [sumFunctionTool, divideFunctionTool],
-});
-```
-
-## Chat with the agent
-
-Now we can chat with the agent.
-
-```ts
-const response = await agent.chat({
- message: "How much is 5 + 5? then divide by 2",
-});
-
-console.log(String(response));
-```
-
-The output will be:
-
-```bash
-Thought: I need to use a tool to help me answer the question.
-Action: sumNumbers
-Action Input: {"a":5,"b":5}
-
-Observation: 10
-Thought: I can answer without using any more tools.
-Answer: The sum of 5 and 5 is 10, and when divided by 2, the result is 5.
-
-The sum of 5 and 5 is 10, and when divided by 2, the result is 5.
-```
-
-## Full code
-
-```ts
-import { FunctionTool, ReActAgent } from "llamaindex";
-
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
- return a + b;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
- return a / b;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
-};
-
-// Define the parameters of the divide function as a JSON schema
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The argument a to divide",
- },
- b: {
- type: "number",
- description: "The argument b to divide",
- },
- },
- required: ["a", "b"],
-};
-
-async function main() {
- // Create a function tool from the sum function
- const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const divideFunctionTool = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
- const agent = new OpenAIAgent({
- tools: [sumFunctionTool, divideFunctionTool],
- });
-
- // Chat with the agent
- const response = await agent.chat({
- message: "I want to sum 5 and 5 and then divide by 2",
- });
-
- // Print the response
- console.log(String(response));
-}
-
-main().then(() => {
- console.log("Done");
-});
-```
diff --git a/examples/agent/multi_document_agent.ts b/examples/agent/multi_document_agent.ts
index d4d0407add2b7fd7958d6a8739e1dc0815f65f4d..8a7861a15efdda0d2eab87a459cf62717cfa3c37 100644
--- a/examples/agent/multi_document_agent.ts
+++ b/examples/agent/multi_document_agent.ts
@@ -125,7 +125,7 @@ async function main() {
const topAgent = new OpenAIAgent({
toolRetriever: await objectIndex.asRetriever({}),
llm: new OpenAI({ model: "gpt-4" }),
- prefixMessages: [
+ chatHistory: [
{
content:
"You are an agent designed to answer queries about a set of given countries. Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
diff --git a/examples/agent/openai.ts b/examples/agent/openai.ts
index 3bc65c2b37c1b5d472f1f6ea4ba010bb4e81c077..857008054ebb31b60fa0426d6e5abd7c00a5c33d 100644
--- a/examples/agent/openai.ts
+++ b/examples/agent/openai.ts
@@ -1,72 +1,58 @@
import { FunctionTool, OpenAIAgent } from "llamaindex";
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }) {
- return `${a + b}`;
-}
-
-// Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }) {
- return `${a / b}`;
-}
-
-// Define the parameters of the sum function as a JSON schema
-const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
+const sumNumbers = FunctionTool.from(
+ ({ a, b }: { a: number; b: number }) => `${a + b}`,
+ {
+ name: "sumNumbers",
+ description: "Use this function to sum two numbers",
+ parameters: {
+ type: "object",
+ properties: {
+ a: {
+ type: "number",
+ description: "The first number",
+ },
+ b: {
+ type: "number",
+ description: "The second number",
+ },
+ },
+ required: ["a", "b"],
},
},
- required: ["a", "b"],
-} as const;
+);
-const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend a to divide",
- },
- b: {
- type: "number",
- description: "The divisor b to divide by",
+const divideNumbers = FunctionTool.from(
+ ({ a, b }: { a: number; b: number }) => `${a / b}`,
+ {
+ name: "divideNumbers",
+ description: "Use this function to divide two numbers",
+ parameters: {
+ type: "object",
+ properties: {
+ a: {
+ type: "number",
+ description: "The dividend a to divide",
+ },
+ b: {
+ type: "number",
+ description: "The divisor b to divide by",
+ },
+ },
+ required: ["a", "b"],
},
},
- required: ["a", "b"],
-} as const;
+);
async function main() {
- // Create a function tool from the sum function
- const functionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- // Create a function tool from the divide function
- const functionTool2 = new FunctionTool(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
- });
-
- // Create an OpenAIAgent with the function tools
const agent = new OpenAIAgent({
- tools: [functionTool, functionTool2],
+ tools: [sumNumbers, divideNumbers],
});
- // Chat with the agent
const response = await agent.chat({
message: "How much is 5 + 5? then divide by 2",
});
- // Print the response
console.log(String(response));
}
diff --git a/examples/agent/react_agent.ts b/examples/agent/react_agent.ts
index d86fe47073dfb9c78679343b06d1a10fce2d3393..6171c35f8b435c334f110f70ef52aeebea8e045e 100644
--- a/examples/agent/react_agent.ts
+++ b/examples/agent/react_agent.ts
@@ -1,4 +1,4 @@
-import { Anthropic, FunctionTool, ReActAgent } from "llamaindex";
+import { Anthropic, FunctionTool, ReACTAgent } from "llamaindex";
// Define a function to sum two numbers
function sumNumbers({ a, b }: { a: number; b: number }) {
@@ -62,18 +62,18 @@ async function main() {
});
// Create an ReActAgent with the function tools
- const agent = new ReActAgent({
+ const agent = new ReACTAgent({
llm: anthropic,
tools: [functionTool, functionTool2],
});
// Chat with the agent
- const response = await agent.chat({
+ const { response } = await agent.chat({
message: "Divide 16 by 2 then add 20",
});
// Print the response
- console.log(String(response));
+ console.log(response.message);
}
void main().then(() => {
diff --git a/examples/agent/step_wise_openai.ts b/examples/agent/step_wise_openai.ts
index 08b20509bac64e1ba45a443942458b0c6561df85..d487f74a3d830efeb15123c3fea6e28517c096d2 100644
--- a/examples/agent/step_wise_openai.ts
+++ b/examples/agent/step_wise_openai.ts
@@ -62,29 +62,18 @@ async function main() {
});
// Create a task to sum and divide numbers
- const task = agent.createTask("How much is 5 + 5? then divide by 2");
+ const task = await agent.createTask("How much is 5 + 5? then divide by 2");
let count = 0;
- while (true) {
- const stepOutput = await agent.runStep(task.taskId);
-
+ for await (const stepOutput of task) {
console.log(`Runnning step ${count++}`);
console.log(`======== OUTPUT ==========`);
- if (stepOutput.output.response) {
- console.log(stepOutput.output.response);
- } else {
- console.log(stepOutput.output.sources);
- }
+ console.log(stepOutput.output.message.content);
console.log(`==========================`);
if (stepOutput.isLast) {
- const finalResponse = await agent.finalizeResponse(
- task.taskId,
- stepOutput,
- );
- console.log({ finalResponse });
- break;
+ console.log(stepOutput.output.message.content);
}
}
}
diff --git a/examples/agent/step_wise_query_tool.ts b/examples/agent/step_wise_query_tool.ts
index b098f1a4e2689571c43850d9186b316509e00c3d..92a5ebe0f2740a623f0d63df62b8773b7eaf914e 100644
--- a/examples/agent/step_wise_query_tool.ts
+++ b/examples/agent/step_wise_query_tool.ts
@@ -31,31 +31,11 @@ async function main() {
tools: [queryEngineTool],
});
- const task = agent.createTask("What was his salary?");
-
- let count = 0;
-
- while (true) {
- const stepOutput = await agent.runStep(task.taskId);
-
- console.log(`Runnning step ${count++}`);
- console.log(`======== OUTPUT ==========`);
- if (stepOutput.output.response) {
- console.log(stepOutput.output.response);
- } else {
- console.log(stepOutput.output.sources);
- }
- console.log(`==========================`);
-
- if (stepOutput.isLast) {
- const finalResponse = await agent.finalizeResponse(
- task.taskId,
- stepOutput,
- );
- console.log({ finalResponse });
- break;
- }
- }
+ const { response } = await agent.chat({
+ message: "What was his salary?",
+ });
+
+ console.log(response.message.content);
}
void main().then(() => {
diff --git a/examples/agent/step_wise_react.ts b/examples/agent/step_wise_react.ts
index 185e528212419c8981458fc0007ff1465a9a69b1..97bd88e8302a2bc143bb3298c343229132fc82f3 100644
--- a/examples/agent/step_wise_react.ts
+++ b/examples/agent/step_wise_react.ts
@@ -1,4 +1,4 @@
-import { FunctionTool, ReActAgent } from "llamaindex";
+import { Anthropic, FunctionTool, ReACTAgent } from "llamaindex";
// Define a function to sum two numbers
function sumNumbers({ a, b }: { a: number; b: number }) {
@@ -7,6 +7,7 @@ function sumNumbers({ a, b }: { a: number; b: number }) {
// Define a function to divide two numbers
function divideNumbers({ a, b }: { a: number; b: number }) {
+ console.log("get input", a, b);
return `${a / b}`;
}
@@ -57,30 +58,22 @@ async function main() {
});
// Create an OpenAIAgent with the function tools
- const agent = new ReActAgent({
+ const agent = new ReACTAgent({
+ llm: new Anthropic({
+ model: "claude-3-opus",
+ }),
tools: [functionTool, functionTool2],
});
- const task = agent.createTask("Divide 16 by 2 then add 20");
+ const task = await agent.createTask("Divide 16 by 2 then add 20");
let count = 0;
- while (true) {
- const stepOutput = await agent.runStep(task.taskId);
-
+ for await (const stepOutput of task) {
console.log(`Runnning step ${count++}`);
console.log(`======== OUTPUT ==========`);
- console.log(stepOutput.output);
+ console.log(stepOutput);
console.log(`==========================`);
-
- if (stepOutput.isLast) {
- const finalResponse = await agent.finalizeResponse(
- task.taskId,
- stepOutput,
- );
- console.log({ finalResponse });
- break;
- }
}
}
diff --git a/examples/package.json b/examples/package.json
index eb4dd3a1a4edb4a301d91136b9d49a13297b262e..3403f7b133d68c9e638cd45fc21e492bc9d8c9f9 100644
--- a/examples/package.json
+++ b/examples/package.json
@@ -12,7 +12,7 @@
"commander": "^11.1.0",
"dotenv": "^16.4.5",
"js-tiktoken": "^1.0.11",
- "llamaindex": "latest",
+ "llamaindex": "*",
"mongodb": "^6.5.0",
"pathe": "^1.1.2"
},
@@ -24,5 +24,8 @@
},
"scripts": {
"lint": "eslint ."
+ },
+ "stackblitz": {
+ "startCommand": "npm start"
}
}
diff --git a/examples/readers/package.json b/examples/readers/package.json
index fd5e7aee79f583cfee5bf6ed06c7366a9c796f7a..8202eb59ca66152c44589c3d9b8e384769f729a7 100644
--- a/examples/readers/package.json
+++ b/examples/readers/package.json
@@ -12,7 +12,7 @@
"start:llamaparse": "node --loader ts-node/esm ./src/llamaparse.ts"
},
"dependencies": {
- "llamaindex": "latest"
+ "llamaindex": "*"
},
"devDependencies": {
"@types/node": "^20.12.7",
diff --git a/examples/recipes/cost-analysis.ts b/examples/recipes/cost-analysis.ts
index 79b162c3bdee6cf9c5c521c45cf890c38a4a9c3c..37b4aef3638a6a7055933404df0492648464005f 100644
--- a/examples/recipes/cost-analysis.ts
+++ b/examples/recipes/cost-analysis.ts
@@ -1,5 +1,5 @@
import { encodingForModel } from "js-tiktoken";
-import { OpenAI } from "llamaindex";
+import { ChatMessage, OpenAI, type LLMStartEvent } from "llamaindex";
import { Settings } from "llamaindex/Settings";
import { extractText } from "llamaindex/llm/utils";
@@ -12,9 +12,9 @@ const llm = new OpenAI({
let tokenCount = 0;
-Settings.callbackManager.on("llm-start", (event) => {
+Settings.callbackManager.on("llm-start", (event: LLMStartEvent) => {
const { messages } = event.detail.payload;
- tokenCount += messages.reduce((count, message) => {
+ messages.reduce((count: number, message: ChatMessage) => {
return count + encoding.encode(extractText(message.content)).length;
}, 0);
console.log("Token count:", tokenCount);
diff --git a/examples/tsconfig.json b/examples/tsconfig.json
index 722089b2e539c89c5ae4aef1b248d0aa0ab63345..b87cae7085f089b67958a7125f89063ceccab52d 100644
--- a/examples/tsconfig.json
+++ b/examples/tsconfig.json
@@ -1,12 +1,13 @@
{
"compilerOptions": {
- "target": "es2017",
+ "target": "ES2022",
"module": "esnext",
"moduleResolution": "bundler",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": true,
"skipLibCheck": true,
+ "lib": ["ES2022"],
"outDir": "./lib",
"tsBuildInfoFile": "./lib/.tsbuildinfo",
"incremental": true,
diff --git a/packages/core/.gitignore b/packages/core/.gitignore
index 1e2995d5dae5bd97e601ce96de933cf12e5c3c1f..485ee85c2944728f046f93e2e3c5100a4242ea01 100644
--- a/packages/core/.gitignore
+++ b/packages/core/.gitignore
@@ -1,3 +1,3 @@
.turbo
-README.md
+/README.md
LICENSE
\ No newline at end of file
diff --git a/packages/core/e2e/fixtures/llm/open_ai.ts b/packages/core/e2e/fixtures/llm/open_ai.ts
index 2c1741b8ca0c3b32613a5f3d677210179972eda0..76d31480737daaf3211b6ff75d51889e79d6081b 100644
--- a/packages/core/e2e/fixtures/llm/open_ai.ts
+++ b/packages/core/e2e/fixtures/llm/open_ai.ts
@@ -10,6 +10,7 @@ import type {
} from "llamaindex/llm/types";
import { extractText } from "llamaindex/llm/utils";
import { deepStrictEqual, strictEqual } from "node:assert";
+import { inspect } from "node:util";
import { llmCompleteMockStorage } from "../../node/utils.js";
export function getOpenAISession() {
@@ -46,10 +47,12 @@ export class OpenAI implements LLM {
if (llmCompleteMockStorage.llmEventStart.length > 0) {
const chatMessage =
llmCompleteMockStorage.llmEventStart.shift()!["messages"];
- strictEqual(chatMessage.length, params.messages.length);
+ console.log(inspect(params.messages, { depth: 1 }));
+ console.log(inspect(chatMessage, { depth: 1 }));
+ strictEqual(params.messages.length, chatMessage.length);
for (let i = 0; i < chatMessage.length; i++) {
- strictEqual(chatMessage[i].role, params.messages[i].role);
- deepStrictEqual(chatMessage[i].content, params.messages[i].content);
+ strictEqual(params.messages[i].role, chatMessage[i].role);
+ deepStrictEqual(params.messages[i].content, chatMessage[i].content);
}
if (llmCompleteMockStorage.llmEventEnd.length > 0) {
@@ -89,9 +92,9 @@ export class OpenAI implements LLM {
if (llmCompleteMockStorage.llmEventStart.length > 0) {
const chatMessage =
llmCompleteMockStorage.llmEventStart.shift()!["messages"];
- strictEqual(chatMessage.length, 1);
- strictEqual(chatMessage[0].role, "user");
- strictEqual(chatMessage[0].content, params.prompt);
+ strictEqual(1, chatMessage.length);
+ strictEqual("user", chatMessage[0].role);
+ strictEqual(params.prompt, chatMessage[0].content);
}
if (llmCompleteMockStorage.llmEventEnd.length > 0) {
const response = llmCompleteMockStorage.llmEventEnd.shift()!["response"];
diff --git a/packages/core/e2e/node/claude.e2e.ts b/packages/core/e2e/node/claude.e2e.ts
index c4e82a954a5c9399a539b744dabdace29eee7631..1afda5e5b0757484d0caa1845f7b3087cba5437f 100644
--- a/packages/core/e2e/node/claude.e2e.ts
+++ b/packages/core/e2e/node/claude.e2e.ts
@@ -1,6 +1,7 @@
import { consola } from "consola";
import { Anthropic, FunctionTool, Settings, type LLM } from "llamaindex";
import { AnthropicAgent } from "llamaindex/agent/anthropic";
+import { extractText } from "llamaindex/llm/utils";
import { ok } from "node:assert";
import { beforeEach, test } from "node:test";
import { sumNumbersTool } from "./fixtures/tools.js";
@@ -70,12 +71,11 @@ await test("anthropic agent", async (t) => {
},
],
});
- const result = await agent.chat({
+ const { response } = await agent.chat({
message: "What is the weather in San Francisco?",
});
- consola.debug("response:", result.response);
- ok(typeof result.response === "string");
- ok(result.response.includes("35"));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("35"));
});
await t.test("async function", async () => {
@@ -111,8 +111,8 @@ await test("anthropic agent", async (t) => {
const { response } = await agent.chat({
message: "My name is Alex Yang. What is my unique id?",
});
- consola.debug("response:", response);
- ok(response.includes(uniqueId));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes(uniqueId));
});
await t.test("sum numbers", async () => {
@@ -120,10 +120,10 @@ await test("anthropic agent", async (t) => {
tools: [sumNumbersTool],
});
- const response = await openaiAgent.chat({
+ const { response } = await openaiAgent.chat({
message: "how much is 1 + 1?",
});
- ok(response.response.includes("2"));
+ ok(extractText(response.message.content).includes("2"));
});
});
diff --git a/packages/core/e2e/node/fixtures/data/Alex.txt b/packages/core/e2e/node/fixtures/data/Alex.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9aeae43f90e207ecd1c665dac787ad7a4c157a1a
--- /dev/null
+++ b/packages/core/e2e/node/fixtures/data/Alex.txt
@@ -0,0 +1,2 @@
+Alex is a male.
+What's very important, Alex is not in the Brazil.
diff --git a/packages/core/e2e/node/fixtures/tools.ts b/packages/core/e2e/node/fixtures/tools.ts
index 6d523991a9354e7884aabef2de97062fbf957b6a..0060415c909a14c1fc0a2212cd479946a7a0e557 100644
--- a/packages/core/e2e/node/fixtures/tools.ts
+++ b/packages/core/e2e/node/fixtures/tools.ts
@@ -45,3 +45,24 @@ export const divideNumbersTool = FunctionTool.from(divideNumbers, {
required: ["a", "b"],
},
});
+
+// should always return the 72 degrees
+export const getWeatherTool = FunctionTool.from(
+ async ({ city }: { city: string }) => {
+ return `The weather in ${city} is 72 degrees`;
+ },
+ {
+ name: "getWeather",
+ description: "Get the weather for a city",
+ parameters: {
+ type: "object",
+ properties: {
+ city: {
+ type: "string",
+ description: "The city to get the weather for",
+ },
+ },
+ required: ["city"],
+ },
+ },
+);
diff --git a/packages/core/e2e/node/openai.e2e.ts b/packages/core/e2e/node/openai.e2e.ts
index 06c78a3093d54ec2b3ca61fb6e03f7140a763d22..4f853e5383bd07de39540d3d7f926f9dc97c73dd 100644
--- a/packages/core/e2e/node/openai.e2e.ts
+++ b/packages/core/e2e/node/openai.e2e.ts
@@ -2,18 +2,29 @@ import { consola } from "consola";
import {
Document,
FunctionTool,
+ ObjectIndex,
OpenAI,
OpenAIAgent,
QueryEngineTool,
Settings,
+ SimpleNodeParser,
+ SimpleToolNodeMapping,
SubQuestionQueryEngine,
+ SummaryIndex,
VectorStoreIndex,
type LLM,
} from "llamaindex";
+import { extractText } from "llamaindex/llm/utils";
import { ok, strictEqual } from "node:assert";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
import { beforeEach, test } from "node:test";
-import { divideNumbersTool, sumNumbersTool } from "./fixtures/tools.js";
-import { mockLLMEvent } from "./utils.js";
+import {
+ divideNumbersTool,
+ getWeatherTool,
+ sumNumbersTool,
+} from "./fixtures/tools.js";
+import { mockLLMEvent, testRootDir } from "./utils.js";
let llm: LLM;
beforeEach(async () => {
@@ -84,12 +95,107 @@ await test("gpt-4-turbo", async (t) => {
const { response } = await agent.chat({
message: "What is the weather in San Jose?",
});
- consola.debug("response:", response);
- ok(typeof response === "string");
- ok(response.includes("45"));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("45"));
+ });
+});
+
+await test("agent system prompt", async (t) => {
+ await mockLLMEvent(t, "openai_agent_system_prompt");
+ await t.test("chat", async (t) => {
+ const agent = new OpenAIAgent({
+ tools: [getWeatherTool],
+ systemPrompt:
+ "You are a pirate. You MUST speak every words staring with a 'Arhgs'",
+ });
+ const { response } = await agent.chat({
+ message: "What is the weather in San Francisco?",
+ });
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("72"));
+ ok(extractText(response.message.content).includes("Arhg"));
});
});
+await test("agent with object retriever", async (t) => {
+ await mockLLMEvent(t, "agent_with_object_retriever");
+
+ const alexInfoPath = join(testRootDir, "./fixtures/data/Alex.txt");
+ const alexInfoText = await readFile(alexInfoPath, "utf-8");
+ const alexDocument = new Document({ text: alexInfoText, id_: alexInfoPath });
+
+ const nodes = new SimpleNodeParser({
+ chunkSize: 200,
+ chunkOverlap: 20,
+ }).getNodesFromDocuments([alexDocument]);
+
+ const summaryIndex = await SummaryIndex.init({
+ nodes,
+ });
+
+ const summaryQueryEngine = summaryIndex.asQueryEngine();
+
+ const queryEngineTools = [
+ FunctionTool.from(
+ ({ query }: { query?: string }) => {
+ throw new Error("This tool should not be called");
+ },
+ {
+ name: "vector_tool",
+ description:
+ "This tool should not be called, never use this tool in any cases.",
+ parameters: {
+ type: "object",
+ properties: {
+ query: { type: "string", nullable: true },
+ },
+ },
+ },
+ ),
+ new QueryEngineTool({
+ queryEngine: summaryQueryEngine,
+ metadata: {
+ name: "summary_tool",
+ description: `Useful for any requests that short information about Alex.
+For questions about Alex, please use this tool.
+For questions about more specific sections, please use the vector_tool.`,
+ },
+ }),
+ ];
+
+ const originalCall = queryEngineTools[1].call.bind(queryEngineTools[1]);
+ const mockCall = t.mock.fn(({ query }: { query: string }) => {
+ return originalCall({ query });
+ });
+ queryEngineTools[1].call = mockCall;
+
+ const toolMapping = SimpleToolNodeMapping.fromObjects(queryEngineTools);
+
+ const objectIndex = await ObjectIndex.fromObjects(
+ queryEngineTools,
+ toolMapping,
+ VectorStoreIndex,
+ );
+
+ const toolRetriever = await objectIndex.asRetriever({});
+
+ const agent = new OpenAIAgent({
+ toolRetriever,
+ systemPrompt:
+ "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
+ });
+
+ strictEqual(mockCall.mock.callCount(), 0);
+ const { response } = await agent.chat({
+ message:
+ "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no.",
+ });
+ strictEqual(mockCall.mock.callCount(), 1);
+
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).toLowerCase().includes("no"));
+});
+
await test("agent", async (t) => {
await mockLLMEvent(t, "agent");
await t.test("chat", async () => {
@@ -113,12 +219,11 @@ await test("agent", async (t) => {
},
],
});
- const result = await agent.chat({
+ const { response } = await agent.chat({
message: "What is the weather in San Francisco?",
});
- consola.debug("response:", result.response);
- ok(typeof result.response === "string");
- ok(result.response.includes("35"));
+ consola.debug("response:", response.message.content);
+ ok(extractText(response.message.content).includes("35"));
});
await t.test("async function", async () => {
@@ -154,8 +259,7 @@ await test("agent", async (t) => {
const { response } = await agent.chat({
message: "My name is Alex Yang. What is my unique id?",
});
- consola.debug("response:", response);
- ok(response.includes(uniqueId));
+ ok(extractText(response.message.content).includes(uniqueId));
});
await t.test("sum numbers", async () => {
@@ -163,11 +267,11 @@ await test("agent", async (t) => {
tools: [sumNumbersTool],
});
- const response = await openaiAgent.chat({
+ const { response } = await openaiAgent.chat({
message: "how much is 1 + 1?",
});
- ok(response.response.includes("2"));
+ ok(extractText(response.message.content).includes("2"));
});
});
@@ -189,7 +293,7 @@ await test("agent stream", async (t) => {
let message = "";
for await (const chunk of response) {
- message += chunk.response;
+ message += chunk.delta;
}
strictEqual(fn.mock.callCount(), 2);
diff --git a/packages/core/e2e/node/react.e2e.ts b/packages/core/e2e/node/react.e2e.ts
new file mode 100644
index 0000000000000000000000000000000000000000..ecffb72a0f690d0e52918c5f00e8456a9f86a743
--- /dev/null
+++ b/packages/core/e2e/node/react.e2e.ts
@@ -0,0 +1,29 @@
+import { OpenAI, ReACTAgent, Settings, type LLM } from "llamaindex";
+import { extractText } from "llamaindex/llm/utils";
+import { ok } from "node:assert";
+import { beforeEach, test } from "node:test";
+import { getWeatherTool } from "./fixtures/tools.js";
+import { mockLLMEvent } from "./utils.js";
+
+let llm: LLM;
+beforeEach(async () => {
+ Settings.llm = new OpenAI({
+ model: "gpt-3.5-turbo",
+ });
+ llm = Settings.llm;
+});
+
+await test("react agent", async (t) => {
+ await mockLLMEvent(t, "react-agent");
+ await t.test("get weather", async () => {
+ const agent = new ReACTAgent({
+ tools: [getWeatherTool],
+ });
+ const { response } = await agent.chat({
+ stream: false,
+ message: "What is the weather like in San Francisco?",
+ });
+
+ ok(extractText(response.message.content).includes("72"));
+ });
+});
diff --git a/packages/core/e2e/node/snapshot/agent_stream.snap b/packages/core/e2e/node/snapshot/agent_stream.snap
index e9e9aa161d5834e0b097ada9ff8b86271e77f893..be29a6826dd866bfbc4669b119a6bc7f536799e2 100644
--- a/packages/core/e2e/node/snapshot/agent_stream.snap
+++ b/packages/core/e2e/node/snapshot/agent_stream.snap
@@ -4,8 +4,8 @@
"id": "PRESERVE_0",
"messages": [
{
- "content": "Divide 16 by 2 then add 20",
- "role": "user"
+ "role": "user",
+ "content": "Divide 16 by 2 then add 20"
}
]
},
@@ -13,78 +13,50 @@
"id": "PRESERVE_1",
"messages": [
{
- "content": "Divide 16 by 2 then add 20",
- "role": "user"
- },
- {
- "content": "",
- "role": "assistant",
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- }
- },
- {
- "content": "8",
"role": "user",
- "options": {
- "toolResult": {
- "id": "HIDDEN",
- "isError": false
- }
- }
- }
- ]
- },
- {
- "id": "PRESERVE_2",
- "messages": [
- {
- "content": "Divide 16 by 2 then add 20",
- "role": "user"
+ "content": "Divide 16 by 2 then add 20"
},
{
- "content": "",
"role": "assistant",
+ "content": "",
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
}
},
{
- "content": "8",
"role": "user",
+ "content": "8",
"options": {
"toolResult": {
- "id": "HIDDEN",
- "isError": false
+ "result": "8",
+ "isError": false,
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn"
}
}
},
{
- "content": "",
"role": "assistant",
+ "content": "",
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
+ "input": "{\"a\": 8, \"b\": 20}"
}
}
},
{
- "content": "28",
"role": "user",
+ "content": "28",
"options": {
"toolResult": {
- "id": "HIDDEN",
- "isError": false
+ "result": "28",
+ "isError": false,
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2"
}
}
}
@@ -95,361 +67,14 @@
{
"id": "PRESERVE_0",
"response": {
- "raw": [
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "divideNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ": 16,"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": " \"b\": "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "2}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "divideNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 16, \"b\": 2}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": ": 8, "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "\"b\": 2"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "0}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\": 8, \"b\": 20}"
- }
- },
- "delta": ""
- }
- ],
+ "raw": null,
"message": {
"content": "",
"role": "assistant",
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
}
@@ -459,749 +84,7 @@
{
"id": "PRESERVE_1",
"response": {
- "raw": [
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "role": "assistant",
- "content": null,
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "a"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "8"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ",\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "b"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "20"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- ],
- "message": {
- "content": "",
- "role": "assistant",
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- }
- }
- }
- },
- {
- "id": "PRESERVE_2",
- "response": {
- "raw": [
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "The"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "The"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " result"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " result"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " of"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " of"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " dividing"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " dividing"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "16"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "16"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " by"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " by"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "2"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "2"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " and"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " and"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " then"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " then"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " adding"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " adding"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "20"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "20"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " is"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " is"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": " "
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "28"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "28"
- },
- {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "."
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {},
- "delta": "."
- }
- ],
+ "raw": null,
"message": {
"content": "The result of dividing 16 by 2 and then adding 20 is 28.",
"role": "assistant",
@@ -1214,37 +97,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "divideNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1254,34 +111,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1291,34 +125,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ": 16,"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1328,34 +139,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": " \"b\": "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1365,34 +153,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "2}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "divideNumbers",
- "id": "HIDDEN",
+ "id": "call_t0vy4M815ncAQnfRqoflW5hn",
"input": "{\"a\": 16, \"b\": 2}"
}
},
@@ -1402,37 +167,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1442,34 +181,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "{\"a\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1479,34 +195,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": ": 8, "
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1516,34 +209,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "\"b\": 2"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1553,34 +223,11 @@
{
"id": "PRESERVE_0",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 1,
- "function": {
- "arguments": "0}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {
"toolCall": {
"name": "sumNumbers",
- "id": "HIDDEN",
+ "id": "call_08QNOtWYlDoqPPXHMtbvr7A2",
"input": "{\"a\": 8, \"b\": 20}"
}
},
@@ -1590,806 +237,143 @@
{
"id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "role": "assistant",
- "content": null,
- "tool_calls": [
- {
- "index": 0,
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": ""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "{\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "a"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "8"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": ",\""
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "b"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "\":"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "20"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_1",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "tool_calls": [
- {
- "index": 0,
- "function": {
- "arguments": "}"
- }
- }
- ]
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
- "options": {
- "toolCall": {
- "name": "sumNumbers",
- "id": "HIDDEN",
- "input": "{\"a\":8,\"b\":20}"
- }
- },
- "delta": ""
- }
- },
- {
- "id": "PRESERVE_2",
- "chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "The"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "The"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " result"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " result"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " of"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " of"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " dividing"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " dividing"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "16"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "16"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " by"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " by"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "2"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "2"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " and"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " and"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " then"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " then"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " adding"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " adding"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "20"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "20"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " is"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " is"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": " "
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": " "
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "28"
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "28"
}
},
{
- "id": "PRESERVE_2",
+ "id": "PRESERVE_1",
"chunk": {
- "raw": {
- "id": "HIDDEN",
- "object": "chat.completion.chunk",
- "created": 114514,
- "model": "gpt-3.5-turbo-0125",
- "system_fingerprint": "HIDDEN",
- "choices": [
- {
- "index": 0,
- "delta": {
- "content": "."
- },
- "logprobs": null,
- "finish_reason": null
- }
- ]
- },
+ "raw": null,
"options": {},
"delta": "."
}
diff --git a/packages/core/e2e/node/snapshot/agent_with_object_retriever.snap b/packages/core/e2e/node/snapshot/agent_with_object_retriever.snap
new file mode 100644
index 0000000000000000000000000000000000000000..627dac39daf811cb973d082ad77a851c25396da8
--- /dev/null
+++ b/packages/core/e2e/node/snapshot/agent_with_object_retriever.snap
@@ -0,0 +1,103 @@
+{
+ "llmEventStart": [
+ {
+ "id": "PRESERVE_0",
+ "messages": [
+ {
+ "content": "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no."
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_1",
+ "messages": [
+ {
+ "content": "Context information is below.\n---------------------\nAlex is a male. What's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Alex\nAnswer:",
+ "role": "user"
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_2",
+ "messages": [
+ {
+ "content": "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no."
+ },
+ {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_EVThrsiOylO0p6ZmGdsA31x9",
+ "name": "summary_tool",
+ "input": "{\"query\": \"Alex\"}"
+ }
+ }
+ },
+ {
+ "content": "Alex is not in Brazil.",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "result": "Alex is not in Brazil.",
+ "isError": false,
+ "id": "call_EVThrsiOylO0p6ZmGdsA31x9"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "llmEventEnd": [
+ {
+ "id": "PRESERVE_0",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_EVThrsiOylO0p6ZmGdsA31x9",
+ "name": "summary_tool",
+ "input": "{\"query\": \"Alex\"}"
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_1",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Alex is not in Brazil.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_2",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "No, Alex does not live in Brazil based on the brief information available.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ }
+ ],
+ "llmEventStream": []
+}
\ No newline at end of file
diff --git a/packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap b/packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap
new file mode 100644
index 0000000000000000000000000000000000000000..eaa9dbecc1df7df88d85b6e9ac2254cdcc5cb071
--- /dev/null
+++ b/packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap
@@ -0,0 +1,83 @@
+{
+ "llmEventStart": [
+ {
+ "id": "PRESERVE_0",
+ "messages": [
+ {
+ "content": "You are a pirate. You MUST speak every words staring with a 'Arhgs'",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What is the weather in San Francisco?"
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_1",
+ "messages": [
+ {
+ "content": "You are a pirate. You MUST speak every words staring with a 'Arhgs'",
+ "role": "system"
+ },
+ {
+ "role": "user",
+ "content": "What is the weather in San Francisco?"
+ },
+ {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_h4gSNrz7MhhkOod7W4WKZ1iZ",
+ "name": "getWeather",
+ "input": "{\"city\":\"San Francisco\"}"
+ }
+ }
+ },
+ {
+ "content": "The weather in San Francisco is 72 degrees",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "result": "The weather in San Francisco is 72 degrees",
+ "isError": false,
+ "id": "call_h4gSNrz7MhhkOod7W4WKZ1iZ"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "llmEventEnd": [
+ {
+ "id": "PRESERVE_0",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "call_h4gSNrz7MhhkOod7W4WKZ1iZ",
+ "name": "getWeather",
+ "input": "{\"city\":\"San Francisco\"}"
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_1",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Arhgs the weather in San Francisco be 72 degrees.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ }
+ ],
+ "llmEventStream": []
+}
\ No newline at end of file
diff --git a/packages/core/e2e/node/snapshot/react-agent.snap b/packages/core/e2e/node/snapshot/react-agent.snap
new file mode 100644
index 0000000000000000000000000000000000000000..48884cc45adf87768e8c373699e4ba9056c606f9
--- /dev/null
+++ b/packages/core/e2e/node/snapshot/react-agent.snap
@@ -0,0 +1,63 @@
+{
+ "llmEventStart": [
+ {
+ "id": "PRESERVE_0",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.\n\n## Tools\nYou have access to a wide variety of tools. You are responsible for using\nthe tools in any sequence you deem appropriate to complete the task at hand.\nThis may require breaking the task into subtasks and using different tools\nto complete each subtask.\n\nYou have access to the following tools:\n- getWeather: Get the weather for a city with schema: {\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"description\":\"The city to get the weather for\"}},\"required\":[\"city\"]}\n\n## Output Format\nTo answer the question, please use the following format.\n\n\"\"\"\nThought: I need to use a tool to help me answer the question.\nAction: tool name (one of getWeather) if using a tool.\nAction Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{\"input\": \"hello world\", \"num_beams\": 5}})\n\"\"\"\n\nPlease ALWAYS start with a Thought.\n\nPlease use a valid JSON format for the Action Input. Do NOT do this {{'input': 'hello world', 'num_beams': 5}}.\n\nIf this format is used, the user will respond in the following format:\n\n\"\"\"\"\nObservation: tool response\n\"\"\"\"\n\nYou should keep repeating the above format until you have enough information\nto answer the question without using any more tools. At that point, you MUST respond\nin the one of the following two formats:\n\n\"\"\"\"\nThought: I can answer without using any more tools.\nAnswer: [your answer here]\n\"\"\"\"\n\n\"\"\"\"\nThought: I cannot answer the question with the provided tools.\nAnswer: Sorry, I cannot answer your query.\n\"\"\"\"\n\n## Current Conversation\nBelow is the current conversation consisting of interleaving human and assistant messages."
+ },
+ {
+ "role": "user",
+ "content": "What is the weather like in San Francisco?"
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_1",
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.\n\n## Tools\nYou have access to a wide variety of tools. You are responsible for using\nthe tools in any sequence you deem appropriate to complete the task at hand.\nThis may require breaking the task into subtasks and using different tools\nto complete each subtask.\n\nYou have access to the following tools:\n- getWeather: Get the weather for a city with schema: {\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\",\"description\":\"The city to get the weather for\"}},\"required\":[\"city\"]}\n\n## Output Format\nTo answer the question, please use the following format.\n\n\"\"\"\nThought: I need to use a tool to help me answer the question.\nAction: tool name (one of getWeather) if using a tool.\nAction Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{\"input\": \"hello world\", \"num_beams\": 5}})\n\"\"\"\n\nPlease ALWAYS start with a Thought.\n\nPlease use a valid JSON format for the Action Input. Do NOT do this {{'input': 'hello world', 'num_beams': 5}}.\n\nIf this format is used, the user will respond in the following format:\n\n\"\"\"\"\nObservation: tool response\n\"\"\"\"\n\nYou should keep repeating the above format until you have enough information\nto answer the question without using any more tools. At that point, you MUST respond\nin the one of the following two formats:\n\n\"\"\"\"\nThought: I can answer without using any more tools.\nAnswer: [your answer here]\n\"\"\"\"\n\n\"\"\"\"\nThought: I cannot answer the question with the provided tools.\nAnswer: Sorry, I cannot answer your query.\n\"\"\"\"\n\n## Current Conversation\nBelow is the current conversation consisting of interleaving human and assistant messages."
+ },
+ {
+ "role": "user",
+ "content": "What is the weather like in San Francisco?"
+ },
+ {
+ "role": "assistant",
+ "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nInput: {\"city\":\"San Francisco\"}"
+ },
+ {
+ "role": "user",
+ "content": "Observation: The weather in San Francisco is 72 degrees"
+ }
+ ]
+ }
+ ],
+ "llmEventEnd": [
+ {
+ "id": "PRESERVE_0",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_1",
+ "response": {
+ "raw": null,
+ "message": {
+ "content": "Thought: I can answer without using any more tools.\nAnswer: The weather in San Francisco is 72 degrees.",
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ }
+ ],
+ "llmEventStream": []
+}
\ No newline at end of file
diff --git a/packages/core/e2e/node/utils.ts b/packages/core/e2e/node/utils.ts
index 7b7c6916bbe19d13b1ca229faf26fa1dd7456aef..92d58b84ab77d0c67b6009ba840ddd5c09d9b67a 100644
--- a/packages/core/e2e/node/utils.ts
+++ b/packages/core/e2e/node/utils.ts
@@ -50,6 +50,11 @@ export async function mockLLMEvent(
...event.detail.payload,
// @ts-expect-error id is not UUID, but it is fine for testing
id: idMap.get(event.detail.payload.id)!,
+ response: {
+ ...event.detail.payload.response,
+ // hide raw object since it might too big
+ raw: null,
+ },
});
}
@@ -58,6 +63,11 @@ export async function mockLLMEvent(
...event.detail.payload,
// @ts-expect-error id is not UUID, but it is fine for testing
id: idMap.get(event.detail.payload.id)!,
+ chunk: {
+ ...event.detail.payload.chunk,
+ // hide raw object since it might too big
+ raw: null,
+ },
});
}
@@ -92,14 +102,7 @@ export async function mockLLMEvent(
Settings.callbackManager.off("llm-start", captureLLMStart);
// eslint-disable-next-line turbo/no-undeclared-env-vars
if (process.env.UPDATE_SNAPSHOT === "1") {
- const data = JSON.stringify(newLLMCompleteMockStorage, null, 2)
- .replace(/"id": "(?!PRESERVE_).*"/g, '"id": "HIDDEN"')
- .replace(/"created": \d+/g, `"created": 114514`)
- .replace(
- /"system_fingerprint": ".*"/g,
- '"system_fingerprint": "HIDDEN"',
- )
- .replace(/"tool_call_id": ".*"/g, '"tool_call_id": "HIDDEN"');
+ const data = JSON.stringify(newLLMCompleteMockStorage, null, 2);
await writeFile(
join(testRootDir, "snapshot", `${snapshotName}.snap`),
data,
diff --git a/packages/core/src/agent/README.md b/packages/core/src/agent/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..8d21725f1c0f2ef680cc8dd8755a2b37ff8e9a7d
--- /dev/null
+++ b/packages/core/src/agent/README.md
@@ -0,0 +1,73 @@
+# Agent
+
+> This is an internal code design document for the agent API.
+>
+> APIs are not exactly the same as the final version, but it is a good reference for what we are going to do.
+
+Most of the agent logic is same with Python, but we have some changes to make it more suitable for JavaScript.
+
+> https://github.com/run-llama/llama_index/tree/6b97753dec4a9c33b16c63a8333ddba3f49ec40f/docs/docs/module_guides/deploying/agents
+
+## API Changes
+
+### Classes changes
+
+- Task: we don't have `Task` class in JS, we use `ReadableStream` instead.
+
+- TaskStep: this is the step for each task run, includes like the input, the context, etc. This class will be used in taskHandler.
+
+- TaskOutput: this is the output for each task run, includes like is last step, the output, etc.
+
+### taskHandler
+
+taskHandler is a function that takes a TaskStep and returns a TaskOutput.
+
+```typescript
+type TaskHandler = (step: TaskStep) => Promise<TaskOutput>;
+```
+
+### `createTask` to be AsyncGenerator
+
+We use async generator to create task, since it's more suitable for JavaScript.
+
+```typescript
+const agent = new MyAgent();
+const task = agent.createTask();
+for await (const taskOutput of task) {
+ // do something
+}
+```
+
+### `from_*` -> `from`
+
+In python, there is `from_tools`, `from_defaults`... etc.
+But in JS/TS, we normalize them to `from`, since we can do this way
+using [function overloads](https://www.typescriptlang.org/docs/handbook/2/functions.html#function-overloads) in
+TypeScript.
+
+```typescript
+class Agent {
+ from(tools: BaseTool[]): Agent;
+ from(defaults: Defaults): Agent;
+ from(toolsOrDefaults: BaseTool[] | Defaults): Agent {
+ // runtime check
+ }
+}
+```
+
+### No sync method for chat/query method
+
+Force all methods to be async, since the all LLMs returns Promise.
+
+### Cancelable
+
+Use `AbortController` to cancel the task.
+
+```typescript
+const controller = new AbortController();
+const task = agent.createTask({ signal: controller.signal });
+process.on("SIGINT", () => controller.abort());
+for await (const taskOutput of task) {
+ // do something
+}
+```
diff --git a/packages/core/src/agent/anthropic.ts b/packages/core/src/agent/anthropic.ts
index 3934834923ad4b37ccdc0826fff9487739936490..2d05759e4efd03debc90ee35c0334eaba1a414a4 100644
--- a/packages/core/src/agent/anthropic.ts
+++ b/packages/core/src/agent/anthropic.ts
@@ -1,27 +1,22 @@
import { Settings } from "../Settings.js";
import {
- AgentChatResponse,
type ChatEngineParamsNonStreaming,
+ type ChatEngineParamsStreaming,
} from "../engines/chat/index.js";
-import { wrapEventCaller } from "../internal/context/EventCaller.js";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import { prettifyError } from "../internal/utils.js";
import { Anthropic } from "../llm/anthropic.js";
-import type {
- ChatMessage,
- ChatResponse,
- ToolCallLLMMessageOptions,
-} from "../llm/index.js";
-import { extractText } from "../llm/utils.js";
+import type { ToolCallLLMMessageOptions } from "../llm/index.js";
import { ObjectRetriever } from "../objects/index.js";
import type { BaseToolWithCall } from "../types.js";
+import {
+ AgentRunner,
+ AgentWorker,
+ type AgentChatResponse,
+ type AgentParamsBase,
+ type TaskHandler,
+} from "./base.js";
+import { callTool } from "./utils.js";
-const MAX_TOOL_CALLS = 10;
-
-type AnthropicParamsBase = {
- llm?: Anthropic;
- chatHistory?: ChatMessage<ToolCallLLMMessageOptions>[];
-};
+type AnthropicParamsBase = AgentParamsBase<Anthropic>;
type AnthropicParamsWithTools = AnthropicParamsBase & {
tools: BaseToolWithCall[];
@@ -35,136 +30,94 @@ export type AnthropicAgentParams =
| AnthropicParamsWithTools
| AnthropicParamsWithToolRetriever;
-type AgentContext = {
- toolCalls: number;
- llm: Anthropic;
- tools: BaseToolWithCall[];
- messages: ChatMessage<ToolCallLLMMessageOptions>[];
- shouldContinue: (context: AgentContext) => boolean;
-};
-
-type TaskResult = {
- response: ChatResponse<ToolCallLLMMessageOptions>;
- chatHistory: ChatMessage<ToolCallLLMMessageOptions>[];
-};
-
-async function task(
- context: AgentContext,
- input: ChatMessage<ToolCallLLMMessageOptions>,
-): Promise<TaskResult> {
- const { llm, tools, messages } = context;
- const nextMessages: ChatMessage<ToolCallLLMMessageOptions>[] = [
- ...messages,
- input,
- ];
- const response = await llm.chat({
- stream: false,
- tools,
- messages: nextMessages,
- });
- const options = response.message.options ?? {};
- if ("toolCall" in options) {
- const { toolCall } = options;
- const { input, name, id } = toolCall;
- const targetTool = tools.find((tool) => tool.metadata.name === name);
- let output: string;
- let isError = true;
- if (!context.shouldContinue(context)) {
- output = "Error: Tool call limit reached";
- } else if (!targetTool) {
- output = `Error: Tool ${name} not found`;
- } else {
- try {
- getCallbackManager().dispatchEvent("llm-tool-call", {
- payload: {
- toolCall: {
- name,
- input,
- },
- },
- });
- output = await targetTool.call(input);
- isError = false;
- } catch (error: unknown) {
- output = prettifyError(error);
- }
- }
- return task(
- {
- ...context,
- toolCalls: context.toolCalls + 1,
- messages: [...nextMessages, response.message],
- },
- {
- content: output,
- role: "user",
- options: {
- toolResult: {
- isError,
- id,
- },
- },
- },
- );
- } else {
- return { response, chatHistory: [...nextMessages, response.message] };
- }
+export class AnthropicAgentWorker extends AgentWorker<Anthropic> {
+ taskHandler = AnthropicAgent.taskHandler;
}
-export class AnthropicAgent {
- readonly #llm: Anthropic;
- readonly #tools:
- | BaseToolWithCall[]
- | ((query: string) => Promise<BaseToolWithCall[]>) = [];
- #chatHistory: ChatMessage<ToolCallLLMMessageOptions>[] = [];
-
+export class AnthropicAgent extends AgentRunner<Anthropic> {
constructor(params: AnthropicAgentParams) {
- this.#llm =
- params.llm ?? Settings.llm instanceof Anthropic
- ? (Settings.llm as Anthropic)
- : new Anthropic();
- if ("tools" in params) {
- this.#tools = params.tools;
- } else if ("toolRetriever" in params) {
- this.#tools = params.toolRetriever.retrieve.bind(params.toolRetriever);
- }
- if (Array.isArray(params.chatHistory)) {
- this.#chatHistory = params.chatHistory;
- }
- }
-
- static shouldContinue(context: AgentContext): boolean {
- return context.toolCalls < MAX_TOOL_CALLS;
+ super({
+ llm:
+ params.llm ?? Settings.llm instanceof Anthropic
+ ? (Settings.llm as Anthropic)
+ : new Anthropic(),
+ chatHistory: params.chatHistory ?? [],
+ systemPrompt: params.systemPrompt ?? null,
+ runner: new AnthropicAgentWorker(),
+ tools:
+ "tools" in params
+ ? params.tools
+ : params.toolRetriever.retrieve.bind(params.toolRetriever),
+ });
}
- public reset(): void {
- this.#chatHistory = [];
- }
-
- getTools(query: string): Promise<BaseToolWithCall[]> | BaseToolWithCall[] {
- return typeof this.#tools === "function" ? this.#tools(query) : this.#tools;
- }
+ createStore = AgentRunner.defaultCreateStore;
- @wrapEventCaller
async chat(
params: ChatEngineParamsNonStreaming,
- ): Promise<Promise<AgentChatResponse>> {
- const { chatHistory, response } = await task(
- {
- llm: this.#llm,
- tools: await this.getTools(extractText(params.message)),
- toolCalls: 0,
- messages: [...this.#chatHistory],
- // do we need this?
- shouldContinue: AnthropicAgent.shouldContinue,
- },
- {
- role: "user",
- content: params.message,
- options: {},
- },
- );
- this.#chatHistory = [...chatHistory];
- return new AgentChatResponse(extractText(response.message.content));
+ ): Promise<AgentChatResponse<ToolCallLLMMessageOptions>>;
+ async chat(params: ChatEngineParamsStreaming): Promise<never>;
+ override async chat(
+ params: ChatEngineParamsNonStreaming | ChatEngineParamsStreaming,
+ ) {
+ if (params.stream) {
+ throw new Error("Anthropic does not support streaming");
+ }
+ return super.chat(params);
}
+
+ static taskHandler: TaskHandler<Anthropic> = async (step) => {
+ const { input } = step;
+ const { llm, getTools, stream } = step.context;
+ if (input) {
+ step.context.store.messages = [...step.context.store.messages, input];
+ }
+ const lastMessage = step.context.store.messages.at(-1)!.content;
+ const tools = await getTools(lastMessage);
+ if (stream === true) {
+ throw new Error("Anthropic does not support streaming");
+ }
+ const response = await llm.chat({
+ stream,
+ tools,
+ messages: step.context.store.messages,
+ });
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ response.message,
+ ];
+ const options = response.message.options ?? {};
+ if ("toolCall" in options) {
+ const { toolCall } = options;
+ const targetTool = tools.find(
+ (tool) => tool.metadata.name === toolCall.name,
+ );
+ const toolOutput = await callTool(targetTool, toolCall);
+ step.context.store.toolOutputs.push(toolOutput);
+ return {
+ taskStep: step,
+ output: {
+ raw: response.raw,
+ message: {
+ content: toolOutput.output,
+ role: "user",
+ options: {
+ toolResult: {
+ result: toolOutput.output,
+ isError: toolOutput.isError,
+ id: toolCall.id,
+ },
+ },
+ },
+ },
+ isLast: false,
+ };
+ } else {
+ return {
+ taskStep: step,
+ output: response,
+ isLast: true,
+ };
+ }
+ };
}
diff --git a/packages/core/src/agent/base.ts b/packages/core/src/agent/base.ts
new file mode 100644
index 0000000000000000000000000000000000000000..9c1b804c5639a4865c662d1dc03fd301cc3ef682
--- /dev/null
+++ b/packages/core/src/agent/base.ts
@@ -0,0 +1,411 @@
+import { pipeline, randomUUID } from "@llamaindex/env";
+import {
+ type ChatEngineParamsNonStreaming,
+ type ChatEngineParamsStreaming,
+} from "../engines/chat/index.js";
+import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { isAsyncIterable } from "../internal/utils.js";
+import type {
+ ChatMessage,
+ ChatResponse,
+ ChatResponseChunk,
+ LLM,
+ MessageContent,
+} from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
+import type { BaseToolWithCall, ToolOutput, UUID } from "../types.js";
+import { consumeAsyncIterable } from "./utils.js";
+
+export const MAX_TOOL_CALLS = 10;
+
+export type AgentTaskContext<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ readonly stream: boolean;
+ readonly toolCallCount: number;
+ readonly llm: Model;
+ readonly getTools: (
+ input: MessageContent,
+ ) => BaseToolWithCall[] | Promise<BaseToolWithCall[]>;
+ shouldContinue: (
+ taskStep: Readonly<TaskStep<Model, Store, AdditionalMessageOptions>>,
+ ) => boolean;
+ store: {
+ toolOutputs: ToolOutput[];
+ messages: ChatMessage<AdditionalMessageOptions>[];
+ } & Store;
+};
+
+export type TaskStep<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ id: UUID;
+ input: ChatMessage<AdditionalMessageOptions> | null;
+ context: AgentTaskContext<Model, Store, AdditionalMessageOptions>;
+
+ // linked list
+ prevStep: TaskStep<Model, Store, AdditionalMessageOptions> | null;
+ nextSteps: Set<TaskStep<Model, Store, AdditionalMessageOptions>>;
+};
+
+export type TaskStepOutput<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> =
+ | {
+ taskStep: TaskStep<Model, Store, AdditionalMessageOptions>;
+ output:
+ | null
+ | ChatResponse<AdditionalMessageOptions>
+ | ReadableStream<ChatResponseChunk<AdditionalMessageOptions>>;
+ isLast: false;
+ }
+ | {
+ taskStep: TaskStep<Model, Store, AdditionalMessageOptions>;
+ output:
+ | ChatResponse<AdditionalMessageOptions>
+ | ReadableStream<ChatResponseChunk<AdditionalMessageOptions>>;
+ isLast: true;
+ };
+
+export type TaskHandler<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = (
+ step: TaskStep<Model, Store, AdditionalMessageOptions>,
+) => Promise<TaskStepOutput<Model, Store, AdditionalMessageOptions>>;
+
+/**
+ * @internal
+ */
+export async function* createTaskImpl<
+ Model extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = Model extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+>(
+ handler: TaskHandler<Model, Store, AdditionalMessageOptions>,
+ context: AgentTaskContext<Model, Store, AdditionalMessageOptions>,
+ _input: ChatMessage<AdditionalMessageOptions>,
+): AsyncGenerator<TaskStepOutput<Model, Store, AdditionalMessageOptions>> {
+ let isDone = false;
+ let input: ChatMessage<AdditionalMessageOptions> | null = _input;
+ let prevStep: TaskStep<Model, Store, AdditionalMessageOptions> | null = null;
+ while (!isDone) {
+ const step: TaskStep<Model, Store, AdditionalMessageOptions> = {
+ id: randomUUID(),
+ input,
+ context,
+ prevStep,
+ nextSteps: new Set(),
+ };
+ if (prevStep) {
+ prevStep.nextSteps.add(step);
+ }
+ const prevToolCallCount = step.context.toolCallCount;
+ if (!step.context.shouldContinue(step)) {
+ throw new Error("Tool call count exceeded limit");
+ }
+ getCallbackManager().dispatchEvent("agent-start", {
+ payload: {},
+ });
+ const taskOutput = await handler(step);
+ const { isLast, output, taskStep } = taskOutput;
+ // do not consume last output
+ if (!isLast) {
+ if (output) {
+ input = isAsyncIterable(output)
+ ? await consumeAsyncIterable(output)
+ : output.message;
+ } else {
+ input = null;
+ }
+ }
+ context = {
+ ...taskStep.context,
+ store: {
+ ...taskStep.context.store,
+ },
+ toolCallCount: prevToolCallCount + 1,
+ };
+ if (isLast) {
+ isDone = true;
+ getCallbackManager().dispatchEvent("agent-end", {
+ payload: {},
+ });
+ }
+ prevStep = taskStep;
+ yield taskOutput;
+ }
+}
+
+export type AgentStreamChatResponse<Options extends object> = {
+ response: ReadableStream<ChatResponseChunk<Options>>;
+ sources: ToolOutput[];
+};
+
+export type AgentChatResponse<Options extends object> = {
+ response: ChatResponse<Options>;
+ sources: ToolOutput[];
+};
+
+export type AgentRunnerParams<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ llm: AI;
+ chatHistory: ChatMessage<AdditionalMessageOptions>[];
+ systemPrompt: MessageContent | null;
+ runner: AgentWorker<AI, Store, AdditionalMessageOptions>;
+ tools:
+ | BaseToolWithCall[]
+ | ((query: MessageContent) => Promise<BaseToolWithCall[]>);
+};
+
+export type AgentParamsBase<
+ AI extends LLM,
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> = {
+ llm?: AI;
+ chatHistory?: ChatMessage<AdditionalMessageOptions>[];
+ systemPrompt?: MessageContent;
+};
+
+/**
+ * Worker will schedule tasks and handle the task execution
+ */
+export abstract class AgentWorker<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> {
+ #taskSet = new Set<TaskStep<AI, Store, AdditionalMessageOptions>>();
+ abstract taskHandler: TaskHandler<AI, Store, AdditionalMessageOptions>;
+
+ public createTask(
+ query: string,
+ context: AgentTaskContext<AI, Store, AdditionalMessageOptions>,
+ ): ReadableStream<TaskStepOutput<AI, Store, AdditionalMessageOptions>> {
+ const taskGenerator = createTaskImpl(this.taskHandler, context, {
+ role: "user",
+ content: query,
+ });
+ return new ReadableStream<
+ TaskStepOutput<AI, Store, AdditionalMessageOptions>
+ >({
+ start: async (controller) => {
+ for await (const stepOutput of taskGenerator) {
+ this.#taskSet.add(stepOutput.taskStep);
+ controller.enqueue(stepOutput);
+ if (stepOutput.isLast) {
+ let currentStep: TaskStep<
+ AI,
+ Store,
+ AdditionalMessageOptions
+ > | null = stepOutput.taskStep;
+ while (currentStep) {
+ this.#taskSet.delete(currentStep);
+ currentStep = currentStep.prevStep;
+ }
+ controller.close();
+ }
+ }
+ },
+ });
+ }
+
+ [Symbol.toStringTag] = "AgentWorker";
+}
+
+/**
+ * Runner will manage the task execution and provide a high-level API for the user
+ */
+export abstract class AgentRunner<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+> {
+ readonly #llm: AI;
+ readonly #tools:
+ | BaseToolWithCall[]
+ | ((query: MessageContent) => Promise<BaseToolWithCall[]>);
+ readonly #systemPrompt: MessageContent | null = null;
+ #chatHistory: ChatMessage<AdditionalMessageOptions>[];
+ readonly #runner: AgentWorker<AI, Store, AdditionalMessageOptions>;
+
+ // create extra store
+ abstract createStore(): Store;
+
+ static defaultCreateStore(): object {
+ return Object.create(null);
+ }
+
+ protected constructor(
+ params: AgentRunnerParams<AI, Store, AdditionalMessageOptions>,
+ ) {
+ const { llm, chatHistory, runner, tools } = params;
+ this.#llm = llm;
+ this.#chatHistory = chatHistory;
+ this.#runner = runner;
+ if (params.systemPrompt) {
+ this.#systemPrompt = params.systemPrompt;
+ }
+ this.#tools = tools;
+ }
+
+ get llm() {
+ return this.#llm;
+ }
+
+ get chatHistory(): ChatMessage<AdditionalMessageOptions>[] {
+ return this.#chatHistory;
+ }
+
+ public reset(): void {
+ this.#chatHistory = [];
+ }
+
+ public getTools(
+ query: MessageContent,
+ ): Promise<BaseToolWithCall[]> | BaseToolWithCall[] {
+ return typeof this.#tools === "function" ? this.#tools(query) : this.#tools;
+ }
+
+ static shouldContinue<
+ AI extends LLM,
+ Store extends object = {},
+ AdditionalMessageOptions extends object = AI extends LLM<
+ object,
+ infer AdditionalMessageOptions
+ >
+ ? AdditionalMessageOptions
+ : never,
+ >(task: Readonly<TaskStep<AI, Store, AdditionalMessageOptions>>): boolean {
+ return task.context.toolCallCount < MAX_TOOL_CALLS;
+ }
+
+ // fixme: this shouldn't be async
+ async createTask(message: MessageContent, stream: boolean = false) {
+ const initialMessages = [...this.#chatHistory];
+ if (this.#systemPrompt !== null) {
+ const systemPrompt = this.#systemPrompt;
+ const alreadyHasSystemPrompt = initialMessages
+ .filter((msg) => msg.role === "system")
+ .some((msg) => Object.is(msg.content, systemPrompt));
+ if (!alreadyHasSystemPrompt) {
+ initialMessages.push({
+ content: systemPrompt,
+ role: "system",
+ });
+ }
+ }
+ return this.#runner.createTask(extractText(message), {
+ stream,
+ toolCallCount: 0,
+ llm: this.#llm,
+ getTools: (message) => this.getTools(message),
+ store: {
+ ...this.createStore(),
+ messages: initialMessages,
+ toolOutputs: [] as ToolOutput[],
+ },
+ shouldContinue: AgentRunner.shouldContinue,
+ });
+ }
+
+ async chat(
+ params: ChatEngineParamsNonStreaming,
+ ): Promise<AgentChatResponse<AdditionalMessageOptions>>;
+ async chat(
+ params: ChatEngineParamsStreaming,
+ ): Promise<AgentStreamChatResponse<AdditionalMessageOptions>>;
+ @wrapEventCaller
+ async chat(
+ params: ChatEngineParamsNonStreaming | ChatEngineParamsStreaming,
+ ): Promise<
+ | AgentChatResponse<AdditionalMessageOptions>
+ | AgentStreamChatResponse<AdditionalMessageOptions>
+ > {
+ const task = await this.createTask(params.message, !!params.stream);
+ const stepOutput = await pipeline(
+ task,
+ async (
+ iter: AsyncIterable<
+ TaskStepOutput<AI, Store, AdditionalMessageOptions>
+ >,
+ ) => {
+ for await (const stepOutput of iter) {
+ if (stepOutput.isLast) {
+ return stepOutput;
+ }
+ }
+ throw new Error("Task did not complete");
+ },
+ );
+ const { output, taskStep } = stepOutput;
+ this.#chatHistory = [...taskStep.context.store.messages];
+ if (isAsyncIterable(output)) {
+ return {
+ response: output,
+ sources: [...taskStep.context.store.toolOutputs],
+ } satisfies AgentStreamChatResponse<AdditionalMessageOptions>;
+ } else {
+ return {
+ response: output,
+ sources: [...taskStep.context.store.toolOutputs],
+ } satisfies AgentChatResponse<AdditionalMessageOptions>;
+ }
+ }
+}
diff --git a/packages/core/src/agent/index.ts b/packages/core/src/agent/index.ts
index 652403e03a9f1af0a2d79d758167bfd8af7430f0..2fc557f4b0202374e62704d99405da1736f08031 100644
--- a/packages/core/src/agent/index.ts
+++ b/packages/core/src/agent/index.ts
@@ -1,7 +1,18 @@
-// Not exporting the AnthropicAgent because it is not ready to ship yet.
-// export { AnthropicAgent, type AnthropicAgentParams } from "./anthropic.js";
-export * from "./openai/base.js";
-export * from "./openai/worker.js";
-export * from "./react/base.js";
-export * from "./react/worker.js";
-export * from "./types.js";
+export {
+ AnthropicAgent,
+ AnthropicAgentWorker,
+ type AnthropicAgentParams,
+} from "./anthropic.js";
+export {
+ OpenAIAgent,
+ OpenAIAgentWorker,
+ type OpenAIAgentParams,
+} from "./openai.js";
+export {
+ ReACTAgent,
+ ReACTAgentWorker,
+ type ReACTAgentParams,
+} from "./react.js";
+// todo: ParallelAgent
+// todo: CustomAgent
+// todo: ReactMultiModal
diff --git a/packages/core/src/agent/openai.ts b/packages/core/src/agent/openai.ts
new file mode 100644
index 0000000000000000000000000000000000000000..c0d59b076f56072ee83f9bf17dd072b5ec8741ad
--- /dev/null
+++ b/packages/core/src/agent/openai.ts
@@ -0,0 +1,194 @@
+import { pipeline } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
+import type {
+ ChatResponseChunk,
+ ToolCall,
+ ToolCallLLMMessageOptions,
+} from "../llm/index.js";
+import { OpenAI } from "../llm/open_ai.js";
+import { ObjectRetriever } from "../objects/index.js";
+import type { BaseToolWithCall } from "../types.js";
+import {
+ AgentRunner,
+ AgentWorker,
+ type AgentParamsBase,
+ type TaskHandler,
+} from "./base.js";
+import { callTool } from "./utils.js";
+
+type OpenAIParamsBase = AgentParamsBase<OpenAI>;
+
+type OpenAIParamsWithTools = OpenAIParamsBase & {
+ tools: BaseToolWithCall[];
+};
+
+type OpenAIParamsWithToolRetriever = OpenAIParamsBase & {
+ toolRetriever: ObjectRetriever<BaseToolWithCall>;
+};
+
+export type OpenAIAgentParams =
+ | OpenAIParamsWithTools
+ | OpenAIParamsWithToolRetriever;
+
+export class OpenAIAgentWorker extends AgentWorker<OpenAI> {
+ taskHandler = OpenAIAgent.taskHandler;
+}
+
+export class OpenAIAgent extends AgentRunner<OpenAI> {
+ constructor(params: OpenAIAgentParams) {
+ super({
+ llm:
+ params.llm ?? Settings.llm instanceof OpenAI
+ ? (Settings.llm as OpenAI)
+ : new OpenAI(),
+ chatHistory: params.chatHistory ?? [],
+ runner: new OpenAIAgentWorker(),
+ systemPrompt: params.systemPrompt ?? null,
+ tools:
+ "tools" in params
+ ? params.tools
+ : params.toolRetriever.retrieve.bind(params.toolRetriever),
+ });
+ }
+
+ createStore = AgentRunner.defaultCreateStore;
+
+ static taskHandler: TaskHandler<OpenAI> = async (step) => {
+ const { input } = step;
+ const { llm, stream, getTools } = step.context;
+ if (input) {
+ step.context.store.messages = [...step.context.store.messages, input];
+ }
+ const lastMessage = step.context.store.messages.at(-1)!.content;
+ const tools = await getTools(lastMessage);
+ const response = await llm.chat({
+ // @ts-expect-error
+ stream,
+ tools,
+ messages: [...step.context.store.messages],
+ });
+ if (!stream) {
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ response.message,
+ ];
+ const options = response.message.options ?? {};
+ if ("toolCall" in options) {
+ const { toolCall } = options;
+ const targetTool = tools.find(
+ (tool) => tool.metadata.name === toolCall.name,
+ );
+ const toolOutput = await callTool(targetTool, toolCall);
+ step.context.store.toolOutputs.push(toolOutput);
+ return {
+ taskStep: step,
+ output: {
+ raw: response.raw,
+ message: {
+ content: toolOutput.output,
+ role: "user",
+ options: {
+ toolResult: {
+ result: toolOutput.output,
+ isError: toolOutput.isError,
+ id: toolCall.id,
+ },
+ },
+ },
+ },
+ isLast: false,
+ };
+ } else {
+ return {
+ taskStep: step,
+ output: response,
+ isLast: true,
+ };
+ }
+ } else {
+ const responseChunkStream = new ReadableStream<
+ ChatResponseChunk<ToolCallLLMMessageOptions>
+ >({
+ async start(controller) {
+ for await (const chunk of response) {
+ controller.enqueue(chunk);
+ }
+ controller.close();
+ },
+ });
+ const [pipStream, finalStream] = responseChunkStream.tee();
+ const reader = pipStream.getReader();
+ const { value } = await reader.read();
+ reader.releaseLock();
+ if (value === undefined) {
+ throw new Error(
+ "first chunk value is undefined, this should not happen",
+ );
+ }
+ // check if first chunk has tool calls, if so, this is a function call
+ // otherwise, it's a regular message
+ const hasToolCall = !!(value.options && "toolCall" in value.options);
+
+ if (hasToolCall) {
+ // you need to consume the response to get the full toolCalls
+ const toolCalls = await pipeline(
+ pipStream,
+ async (
+ iter: AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>,
+ ) => {
+ const toolCalls = new Map<string, ToolCall>();
+ for await (const chunk of iter) {
+ if (chunk.options && "toolCall" in chunk.options) {
+ const toolCall = chunk.options.toolCall;
+ toolCalls.set(toolCall.id, toolCall);
+ }
+ }
+ return [...toolCalls.values()];
+ },
+ );
+ for (const toolCall of toolCalls) {
+ const targetTool = tools.find(
+ (tool) => tool.metadata.name === toolCall.name,
+ );
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ {
+ role: "assistant" as const,
+ content: "",
+ options: {
+ toolCall,
+ },
+ },
+ ];
+ const toolOutput = await callTool(targetTool, toolCall);
+ step.context.store.messages = [
+ ...step.context.store.messages,
+ {
+ role: "user" as const,
+ content: toolOutput.output,
+ options: {
+ toolResult: {
+ result: toolOutput.output,
+ isError: toolOutput.isError,
+ id: toolCall.id,
+ },
+ },
+ },
+ ];
+ step.context.store.toolOutputs.push(toolOutput);
+ }
+ return {
+ taskStep: step,
+ output: null,
+ isLast: false,
+ };
+ } else {
+ return {
+ taskStep: step,
+ output: finalStream,
+ isLast: true,
+ };
+ }
+ }
+ };
+}
diff --git a/packages/core/src/agent/openai/base.ts b/packages/core/src/agent/openai/base.ts
deleted file mode 100644
index 6a9af20c7b56fba332af52044ed8d58fa9552259..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/openai/base.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-import { Settings } from "../../Settings.js";
-import type { ChatMessage } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
-import type { BaseMemory } from "../../memory/types.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import type { BaseTool } from "../../types.js";
-import { AgentRunner } from "../runner/base.js";
-import { OpenAIAgentWorker } from "./worker.js";
-
-type OpenAIAgentParams = {
- tools?: BaseTool[];
- llm?: OpenAI;
- memory?: BaseMemory;
- prefixMessages?: ChatMessage[];
- maxFunctionCalls?: number;
- defaultToolChoice?: string;
- toolRetriever?: ObjectRetriever<BaseTool>;
- systemPrompt?: string;
-};
-
-/**
- * An agent that uses OpenAI's API to generate text.
- *
- * @category OpenAI
- */
-export class OpenAIAgent extends AgentRunner {
- constructor({
- tools,
- llm,
- memory,
- prefixMessages,
- maxFunctionCalls = 5,
- defaultToolChoice = "auto",
- toolRetriever,
- systemPrompt,
- }: OpenAIAgentParams) {
- if (!llm) {
- if (Settings.llm instanceof OpenAI) {
- llm = Settings.llm;
- } else {
- console.warn("No OpenAI model provided, creating a new one");
- llm = new OpenAI({ model: "gpt-3.5-turbo-0613" });
- }
- }
-
- if (systemPrompt) {
- if (prefixMessages) {
- throw new Error("Cannot provide both systemPrompt and prefixMessages");
- }
-
- prefixMessages = [
- {
- content: systemPrompt,
- role: "system",
- },
- ];
- }
-
- if (!llm?.supportToolCall) {
- throw new Error("LLM model must be a function-calling model");
- }
-
- const stepEngine = new OpenAIAgentWorker({
- tools,
- llm,
- prefixMessages,
- maxFunctionCalls,
- toolRetriever,
- });
-
- super({
- agentWorker: stepEngine,
- llm,
- memory,
- defaultToolChoice,
- // @ts-expect-error 2322
- chatHistory: prefixMessages,
- });
- }
-}
diff --git a/packages/core/src/agent/openai/types/chat.ts b/packages/core/src/agent/openai/types/chat.ts
deleted file mode 100644
index afe92219808e0503886000257197cacf0833e7df..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/openai/types/chat.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-export type OpenAIToolCall = ChatCompletionMessageToolCall;
-
-export interface Function {
- arguments: string;
- name: string;
- type: "function";
-}
-
-export interface ChatCompletionMessageToolCall {
- id: string;
- function: Function;
- type: "function";
-}
diff --git a/packages/core/src/agent/openai/worker.ts b/packages/core/src/agent/openai/worker.ts
deleted file mode 100644
index 38e4d72ed5f011b28cea9a21fe49fa521c277f5f..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/openai/worker.ts
+++ /dev/null
@@ -1,397 +0,0 @@
-import { pipeline, randomUUID } from "@llamaindex/env";
-import type { ChatCompletionToolChoiceOption } from "openai/resources/chat/completions";
-import { Response } from "../../Response.js";
-import { Settings } from "../../Settings.js";
-import {
- AgentChatResponse,
- ChatResponseMode,
- StreamingAgentChatResponse,
-} from "../../engines/chat/types.js";
-import {
- OpenAI,
- isFunctionCallingModel,
- type ChatMessage,
- type ChatResponseChunk,
- type LLMChatParamsBase,
- type OpenAIAdditionalChatOptions,
- type ToolCallLLMMessageOptions,
- type ToolCallOptions,
-} from "../../llm/index.js";
-import { extractText } from "../../llm/utils.js";
-import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import type { ToolOutput } from "../../tools/types.js";
-import { callToolWithErrorHandling } from "../../tools/utils.js";
-import type { BaseTool } from "../../types.js";
-import type { AgentWorker, Task } from "../types.js";
-import { TaskStep, TaskStepOutput } from "../types.js";
-import { addUserStepToMemory, getFunctionByName } from "../utils.js";
-
-async function callFunction(
- tools: BaseTool[],
- toolCall: ToolCallOptions["toolCall"],
-): Promise<[ChatMessage<ToolCallLLMMessageOptions>, ToolOutput]> {
- const id = toolCall.id;
- const name = toolCall.name;
- const input = toolCall.input;
-
- if (Settings.debug) {
- console.log("=== Calling Function ===");
- console.log(`Calling function: ${name} with args: ${input}`);
- }
-
- const tool = getFunctionByName(tools, name);
-
- // Call tool
- // Use default error message
- const output = await callToolWithErrorHandling(tool, input);
-
- if (Settings.debug) {
- console.log(`Got output ${output}`);
- console.log("==========================");
- }
-
- return [
- {
- content: `${output}`,
- role: "user",
- options: {
- toolResult: {
- id,
- isError: false,
- },
- },
- },
- output,
- ];
-}
-
-type OpenAIAgentWorkerParams = {
- tools?: BaseTool[];
- llm?: OpenAI;
- prefixMessages?: ChatMessage[];
- maxFunctionCalls?: number;
- toolRetriever?: ObjectRetriever<BaseTool>;
-};
-
-type CallFunctionOutput = {
- message: ChatMessage;
- toolOutput: ToolOutput;
-};
-
-export class OpenAIAgentWorker
- implements AgentWorker<LLMChatParamsBase<OpenAIAdditionalChatOptions>>
-{
- private llm: OpenAI;
- private maxFunctionCalls: number = 5;
-
- public prefixMessages: ChatMessage[];
-
- private _getTools: (input: string) => Promise<BaseTool[]>;
-
- constructor({
- tools = [],
- llm,
- prefixMessages,
- maxFunctionCalls,
- toolRetriever,
- }: OpenAIAgentWorkerParams) {
- this.llm =
- llm ?? isFunctionCallingModel(Settings.llm)
- ? (Settings.llm as OpenAI)
- : new OpenAI({
- model: "gpt-3.5-turbo-0613",
- });
- if (maxFunctionCalls) {
- this.maxFunctionCalls = maxFunctionCalls;
- }
- this.prefixMessages = prefixMessages || [];
-
- if (tools.length > 0 && toolRetriever) {
- throw new Error("Cannot specify both tools and tool_retriever");
- } else if (tools.length > 0) {
- this._getTools = async () => tools;
- } else if (toolRetriever) {
- // fixme: this won't work, type mismatch
- this._getTools = async (message: string) =>
- toolRetriever.retrieve(message);
- } else {
- this._getTools = async () => [];
- }
- }
-
- public getAllMessages(task: Task): ChatMessage<ToolCallLLMMessageOptions>[] {
- return [
- ...this.prefixMessages,
- ...task.memory.get(),
- ...task.extraState.newMemory.get(),
- ];
- }
-
- public getLatestToolCall(task: Task): ToolCallOptions["toolCall"] | null {
- const chatHistory: ChatMessage[] = task.extraState.newMemory.getAll();
-
- if (chatHistory.length === 0) {
- return null;
- }
-
- // @ts-expect-error fixme
- return chatHistory[chatHistory.length - 1].options?.toolCall;
- }
-
- private _getLlmChatParams(
- task: Task,
- tools: BaseTool[],
- toolChoice: ChatCompletionToolChoiceOption = "auto",
- ): LLMChatParamsBase<OpenAIAdditionalChatOptions, ToolCallLLMMessageOptions> {
- const llmChatParams = {
- messages: this.getAllMessages(task),
- tools: undefined as BaseTool[] | undefined,
- additionalChatOptions: {} as OpenAIAdditionalChatOptions,
- } satisfies LLMChatParamsBase<
- OpenAIAdditionalChatOptions,
- ToolCallLLMMessageOptions
- >;
-
- if (tools.length > 0) {
- llmChatParams.tools = tools;
- llmChatParams.additionalChatOptions.tool_choice = toolChoice;
- }
-
- return llmChatParams;
- }
-
- private _processMessage(
- task: Task,
- aiMessage: ChatMessage,
- ): AgentChatResponse {
- task.extraState.newMemory.put(aiMessage);
-
- return new AgentChatResponse(
- extractText(aiMessage.content),
- task.extraState.sources,
- );
- }
-
- private async _getStreamAiResponse(
- task: Task,
- llmChatParams: LLMChatParamsBase<
- OpenAIAdditionalChatOptions,
- ToolCallLLMMessageOptions
- >,
- ): Promise<StreamingAgentChatResponse | AgentChatResponse> {
- const stream = await this.llm.chat({
- stream: true,
- ...llmChatParams,
- });
-
- const responseChunkStream = new ReadableStream<
- ChatResponseChunk<ToolCallLLMMessageOptions>
- >({
- async start(controller) {
- for await (const chunk of stream) {
- controller.enqueue(chunk);
- }
- controller.close();
- },
- });
- const [pipStream, finalStream] = responseChunkStream.tee();
- const reader = pipStream.getReader();
- const { value } = await reader.read();
- reader.releaseLock();
- if (value === undefined) {
- throw new Error("first chunk value is undefined, this should not happen");
- }
- // check if first chunk has tool calls, if so, this is a function call
- // otherwise, it's a regular message
- const hasToolCall: boolean = !!(
- value.options && "toolCall" in value.options
- );
-
- if (hasToolCall) {
- return this._processMessage(task, {
- content: await pipeline(finalStream, async (iterator) => {
- let content = "";
- for await (const value of iterator) {
- content += value.delta;
- }
- return content;
- }),
- role: "assistant",
- options: value.options,
- });
- } else {
- const [responseStream, chunkStream] = finalStream.tee();
- let content = "";
- return new StreamingAgentChatResponse(
- responseStream.pipeThrough<Response>({
- readable: new ReadableStream({
- async start(controller) {
- for await (const chunk of chunkStream) {
- controller.enqueue(new Response(chunk.delta));
- }
- controller.close();
- },
- }),
- writable: new WritableStream({
- write(chunk) {
- content += chunk.delta;
- },
- close() {
- task.extraState.newMemory.put({
- content,
- role: "assistant",
- });
- },
- }),
- }),
- task.extraState.sources,
- );
- }
- }
-
- private async _getAgentResponse(
- task: Task,
- mode: ChatResponseMode,
- llmChatParams: LLMChatParamsBase<
- OpenAIAdditionalChatOptions,
- ToolCallLLMMessageOptions
- >,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
- if (mode === ChatResponseMode.WAIT) {
- const chatResponse = await this.llm.chat({
- stream: false,
- ...llmChatParams,
- });
-
- return this._processMessage(
- task,
- chatResponse.message,
- ) as AgentChatResponse;
- } else if (mode === ChatResponseMode.STREAM) {
- return this._getStreamAiResponse(task, llmChatParams);
- }
-
- throw new Error("Invalid mode");
- }
-
- async callFunction(
- tools: BaseTool[],
- toolCall: ToolCallOptions["toolCall"],
- ): Promise<CallFunctionOutput> {
- const functionMessage = await callFunction(tools, toolCall);
-
- const message = functionMessage[0];
- const toolOutput = functionMessage[1];
-
- return {
- message,
- toolOutput,
- };
- }
-
- initializeStep(task: Task): TaskStep {
- const sources: ToolOutput[] = [];
-
- const newMemory = new ChatMemoryBuffer({
- tokenLimit: task.memory.tokenLimit,
- });
-
- const taskState = {
- sources,
- nFunctionCalls: 0,
- newMemory,
- };
-
- task.extraState = {
- ...task.extraState,
- ...taskState,
- };
-
- return new TaskStep(task.taskId, randomUUID(), task.input);
- }
-
- private _shouldContinue(
- toolCall: ToolCallOptions["toolCall"] | null,
- nFunctionCalls: number,
- ): toolCall is ToolCallOptions["toolCall"] {
- if (nFunctionCalls > this.maxFunctionCalls) {
- return false;
- }
-
- return !!toolCall;
- }
-
- async getTools(input: string): Promise<BaseTool[]> {
- return this._getTools(input);
- }
-
- private async _runStep(
- step: TaskStep,
- task: Task,
- mode: ChatResponseMode = ChatResponseMode.WAIT,
- toolChoice: ChatCompletionToolChoiceOption = "auto",
- ): Promise<TaskStepOutput> {
- const tools = await this.getTools(task.input);
-
- if (step.input) {
- addUserStepToMemory(step, task.extraState.newMemory);
- }
-
- const llmChatParams = this._getLlmChatParams(task, tools, toolChoice);
-
- const agentChatResponse = await this._getAgentResponse(
- task,
- mode,
- llmChatParams,
- );
-
- const latestToolCall = this.getLatestToolCall(task) ?? null;
-
- let isDone: boolean;
- let newSteps: TaskStep[];
-
- if (!this._shouldContinue(latestToolCall, task.extraState.nFunctionCalls)) {
- isDone = true;
- newSteps = [];
- } else {
- isDone = false;
- const { message, toolOutput } = await this.callFunction(
- tools,
- latestToolCall,
- );
-
- task.extraState.sources.push(toolOutput);
- task.extraState.newMemory.put(message);
-
- task.extraState.nFunctionCalls += 1;
-
- newSteps = [step.getNextStep(randomUUID(), undefined)];
- }
-
- return new TaskStepOutput(agentChatResponse, step, newSteps, isDone);
- }
-
- async runStep(
- step: TaskStep,
- task: Task,
- chatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
- ): Promise<TaskStepOutput> {
- const toolChoice = chatParams?.additionalChatOptions?.tool_choice ?? "auto";
- return this._runStep(step, task, ChatResponseMode.WAIT, toolChoice);
- }
-
- async streamStep(
- step: TaskStep,
- task: Task,
- chatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
- ): Promise<TaskStepOutput> {
- const toolChoice = chatParams?.additionalChatOptions?.tool_choice ?? "auto";
- return this._runStep(step, task, ChatResponseMode.STREAM, toolChoice);
- }
-
- finalizeTask(task: Task): void {
- task.memory.set(task.memory.get().concat(task.extraState.newMemory.get()));
- task.extraState.newMemory.reset();
- }
-}
diff --git a/packages/core/src/agent/react.ts b/packages/core/src/agent/react.ts
new file mode 100644
index 0000000000000000000000000000000000000000..1201c93e2321fc844e1e62ed2056d728c26e5470
--- /dev/null
+++ b/packages/core/src/agent/react.ts
@@ -0,0 +1,397 @@
+import { pipeline, randomUUID } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
+import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
+import { isAsyncIterable } from "../internal/utils.js";
+import {
+ type ChatMessage,
+ type ChatResponse,
+ type ChatResponseChunk,
+ type LLM,
+} from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
+import { ObjectRetriever } from "../objects/index.js";
+import type { BaseTool, BaseToolWithCall } from "../types.js";
+import {
+ AgentRunner,
+ AgentWorker,
+ type AgentParamsBase,
+ type TaskHandler,
+} from "./base.js";
+import {
+ callTool,
+ consumeAsyncIterable,
+ createReadableStream,
+} from "./utils.js";
+
+type ReACTAgentParamsBase = AgentParamsBase<LLM>;
+
+type ReACTAgentParamsWithTools = ReACTAgentParamsBase & {
+ tools: BaseToolWithCall[];
+};
+
+type ReACTAgentParamsWithToolRetriever = ReACTAgentParamsBase & {
+ toolRetriever: ObjectRetriever<BaseToolWithCall>;
+};
+
+export type ReACTAgentParams =
+ | ReACTAgentParamsWithTools
+ | ReACTAgentParamsWithToolRetriever;
+
+type BaseReason = {
+ type: unknown;
+};
+
+type ObservationReason = BaseReason & {
+ type: "observation";
+ observation: string;
+};
+
+type ActionReason = BaseReason & {
+ type: "action";
+ thought: string;
+ action: string;
+ input: Record<string, unknown>;
+};
+
+type ResponseReason = BaseReason & {
+ type: "response";
+ thought: string;
+ response: ChatResponse | AsyncIterable<ChatResponseChunk>;
+};
+
+type Reason = ObservationReason | ActionReason | ResponseReason;
+
+function reasonFormatter(reason: Reason): string | Promise<string> {
+ switch (reason.type) {
+ case "observation":
+ return `Observation: ${reason.observation}`;
+ case "action":
+ return `Thought: ${reason.thought}\nAction: ${reason.action}\nInput: ${JSON.stringify(
+ reason.input,
+ )}`;
+ case "response": {
+ if (isAsyncIterable(reason.response)) {
+ return consumeAsyncIterable(reason.response).then(
+ (message) =>
+ `Thought: ${reason.thought}\nAnswer: ${extractText(message.content)}`,
+ );
+ } else {
+ return `Thought: ${reason.thought}\nAnswer: ${extractText(
+ reason.response.message.content,
+ )}`;
+ }
+ }
+ }
+}
+
+function extractJsonStr(text: string): string {
+ const pattern = /\{.*}/s;
+ const match = text.match(pattern);
+
+ if (!match) {
+ throw new SyntaxError(`Could not extract json string from output: ${text}`);
+ }
+
+ return match[0];
+}
+
+function extractFinalResponse(
+ inputText: string,
+): [thought: string, answer: string] {
+ const pattern = /\s*Thought:(.*?)Answer:(.*?)$/s;
+
+ const match = inputText.match(pattern);
+
+ if (!match) {
+ throw new Error(
+ `Could not extract final answer from input text: ${inputText}`,
+ );
+ }
+
+ const thought = match[1].trim();
+ const answer = match[2].trim();
+ return [thought, answer];
+}
+
+function extractToolUse(
+ inputText: string,
+): [thought: string, action: string, input: string] {
+ const pattern =
+ /\s*Thought: (.*?)\nAction: ([a-zA-Z0-9_]+).*?\.*Input: .*?(\{.*?})/s;
+
+ const match = inputText.match(pattern);
+
+ if (!match) {
+ throw new Error(
+ `Could not extract tool use from input text: "${inputText}"`,
+ );
+ }
+
+ const thought = match[1].trim();
+ const action = match[2].trim();
+ const actionInput = match[3].trim();
+ return [thought, action, actionInput];
+}
+
+function actionInputParser(jsonStr: string): Record<string, unknown> {
+ const processedString = jsonStr.replace(/(?<!\w)'|'(?!\w)/g, '"');
+ const pattern = /"(\w+)":\s*"([^"]*)"/g;
+ const matches = [...processedString.matchAll(pattern)];
+ return Object.fromEntries(matches);
+}
+
+type ReACTOutputParser = <Options extends object>(
+ output: ChatResponse<Options> | AsyncIterable<ChatResponseChunk<Options>>,
+) => Promise<Reason>;
+
+const reACTOutputParser: ReACTOutputParser = async (
+ output,
+): Promise<Reason> => {
+ let reason: Reason | null = null;
+
+ if (isAsyncIterable(output)) {
+ const [peakStream, finalStream] = createReadableStream(output).tee();
+ const type = await pipeline(peakStream, async (iter) => {
+ let content = "";
+ for await (const chunk of iter) {
+ content += chunk.delta;
+ if (content.includes("Action:")) {
+ return "action";
+ } else if (content.includes("Answer:")) {
+ return "answer";
+ } else if (content.includes("Thought:")) {
+ return "thought";
+ }
+ }
+ });
+ // step 2: do the parsing from content
+ switch (type) {
+ case "action": {
+ // have to consume the stream to get the full content
+ const response = await consumeAsyncIterable(finalStream);
+ const { content } = response;
+ const [thought, action, input] = extractToolUse(content);
+ const jsonStr = extractJsonStr(input);
+ let json: Record<string, unknown>;
+ try {
+ json = JSON.parse(jsonStr);
+ } catch (e) {
+ json = actionInputParser(jsonStr);
+ }
+ reason = {
+ type: "action",
+ thought,
+ action,
+ input: json,
+ };
+ break;
+ }
+ case "thought": {
+ const thought = "(Implicit) I can answer without any more tools!";
+ reason = {
+ type: "response",
+ thought,
+ // bypass the response, because here we don't need to do anything with it
+ response: finalStream,
+ };
+ break;
+ }
+ case "answer": {
+ const response = await consumeAsyncIterable(finalStream);
+ const { content } = response;
+ const [thought, answer] = extractFinalResponse(content);
+ reason = {
+ type: "response",
+ thought,
+ response: {
+ raw: response,
+ message: {
+ role: "assistant",
+ content: answer,
+ },
+ },
+ };
+ break;
+ }
+ default: {
+ throw new Error(`Invalid type: ${type}`);
+ }
+ }
+ } else {
+ const content = extractText(output.message.content);
+ const type = content.includes("Answer:")
+ ? "answer"
+ : content.includes("Action:")
+ ? "action"
+ : "thought";
+
+ // step 2: do the parsing from content
+ switch (type) {
+ case "action": {
+ const [thought, action, input] = extractToolUse(content);
+ const jsonStr = extractJsonStr(input);
+ let json: Record<string, unknown>;
+ try {
+ json = JSON.parse(jsonStr);
+ } catch (e) {
+ json = actionInputParser(jsonStr);
+ }
+ reason = {
+ type: "action",
+ thought,
+ action,
+ input: json,
+ };
+ break;
+ }
+ case "thought": {
+ const thought = "(Implicit) I can answer without any more tools!";
+ reason = {
+ type: "response",
+ thought,
+ response: {
+ raw: output,
+ message: {
+ role: "assistant",
+ content: extractText(output.message.content),
+ },
+ },
+ };
+ break;
+ }
+ case "answer": {
+ const [thought, answer] = extractFinalResponse(content);
+ reason = {
+ type: "response",
+ thought,
+ response: {
+ raw: output,
+ message: {
+ role: "assistant",
+ content: answer,
+ },
+ },
+ };
+ break;
+ }
+ default: {
+ throw new Error(`Invalid type: ${type}`);
+ }
+ }
+ }
+ if (reason === null) {
+ throw new TypeError("Reason is null");
+ }
+ return reason;
+};
+
+type ReACTAgentStore = {
+ reasons: Reason[];
+};
+
+type ChatFormatter = <Options extends object>(
+ tools: BaseTool[],
+ messages: ChatMessage<Options>[],
+ currentReasons: Reason[],
+) => Promise<ChatMessage<Options>[]>;
+
+const chatFormatter: ChatFormatter = async <Options extends object>(
+ tools: BaseTool[],
+ messages: ChatMessage<Options>[],
+ currentReasons: Reason[],
+): Promise<ChatMessage<Options>[]> => {
+ const header = getReACTAgentSystemHeader(tools);
+ const reasonMessages: ChatMessage<Options>[] = [];
+ for (const reason of currentReasons) {
+ const response = await reasonFormatter(reason);
+ reasonMessages.push({
+ role: reason.type === "observation" ? "user" : "assistant",
+ content: response,
+ });
+ }
+ return [
+ {
+ role: "system",
+ content: header,
+ },
+ ...messages,
+ ...reasonMessages,
+ ];
+};
+
+export class ReACTAgentWorker extends AgentWorker<LLM, ReACTAgentStore> {
+ taskHandler = ReACTAgent.taskHandler;
+}
+
+export class ReACTAgent extends AgentRunner<LLM, ReACTAgentStore> {
+ constructor(
+ params: ReACTAgentParamsWithTools | ReACTAgentParamsWithToolRetriever,
+ ) {
+ super({
+ llm: params.llm ?? Settings.llm,
+ chatHistory: params.chatHistory ?? [],
+ runner: new ReACTAgentWorker(),
+ systemPrompt: params.systemPrompt ?? null,
+ tools:
+ "tools" in params
+ ? params.tools
+ : params.toolRetriever.retrieve.bind(params.toolRetriever),
+ });
+ }
+
+ createStore() {
+ return {
+ reasons: [],
+ };
+ }
+
+ static taskHandler: TaskHandler<LLM, ReACTAgentStore> = async (step) => {
+ const { llm, stream, getTools } = step.context;
+ const input = step.input;
+ if (input) {
+ step.context.store.messages.push(input);
+ }
+ const lastMessage = step.context.store.messages.at(-1)!.content;
+ const tools = await getTools(lastMessage);
+ const messages = await chatFormatter(
+ tools,
+ step.context.store.messages,
+ step.context.store.reasons,
+ );
+ const response = await llm.chat({
+ // @ts-expect-error
+ stream,
+ messages,
+ });
+ const reason = await reACTOutputParser(response);
+ step.context.store.reasons = [...step.context.store.reasons, reason];
+ if (reason.type === "response") {
+ return {
+ isLast: true,
+ output: response,
+ taskStep: step,
+ };
+ } else {
+ if (reason.type === "action") {
+ const tool = tools.find((tool) => tool.metadata.name === reason.action);
+ const toolOutput = await callTool(tool, {
+ id: randomUUID(),
+ input: reason.input,
+ name: reason.action,
+ });
+ step.context.store.reasons = [
+ ...step.context.store.reasons,
+ {
+ type: "observation",
+ observation: toolOutput.output,
+ },
+ ];
+ }
+ return {
+ isLast: false,
+ output: null,
+ taskStep: step,
+ };
+ }
+ };
+}
diff --git a/packages/core/src/agent/react/base.ts b/packages/core/src/agent/react/base.ts
deleted file mode 100644
index 763e6f4874684dc75f9116f7265456dc96da8e0e..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/react/base.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-import type { ChatMessage, LLM } from "../../llm/index.js";
-import type { BaseMemory } from "../../memory/types.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import type { BaseTool } from "../../types.js";
-import { AgentRunner } from "../runner/base.js";
-import { ReActAgentWorker } from "./worker.js";
-
-type ReActAgentParams = {
- tools: BaseTool[];
- llm?: LLM;
- memory?: BaseMemory;
- prefixMessages?: ChatMessage[];
- maxInteractions?: number;
- defaultToolChoice?: string;
- toolRetriever?: ObjectRetriever<BaseTool>;
-};
-
-/**
- * An agent that uses OpenAI's API to generate text.
- *
- * @category OpenAI
- */
-export class ReActAgent extends AgentRunner {
- constructor({
- tools,
- llm,
- memory,
- prefixMessages,
- maxInteractions = 10,
- defaultToolChoice = "auto",
- toolRetriever,
- }: Partial<ReActAgentParams>) {
- const stepEngine = new ReActAgentWorker({
- tools: tools ?? [],
- llm,
- maxInteractions,
- toolRetriever,
- });
-
- super({
- agentWorker: stepEngine,
- memory,
- defaultToolChoice,
- // @ts-expect-error 2322
- chatHistory: prefixMessages,
- });
- }
-}
diff --git a/packages/core/src/agent/react/formatter.ts b/packages/core/src/agent/react/formatter.ts
deleted file mode 100644
index aff8200ecb770754e15a571296fdfef07631a8a8..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/react/formatter.ts
+++ /dev/null
@@ -1,84 +0,0 @@
-import type { ChatMessage } from "../../llm/index.js";
-import type { BaseTool } from "../../types.js";
-import { getReactChatSystemHeader } from "./prompts.js";
-import type { BaseReasoningStep } from "./types.js";
-import { ObservationReasoningStep } from "./types.js";
-
-function getReactToolDescriptions(tools: BaseTool[]): string[] {
- const toolDescs: string[] = [];
- for (const tool of tools) {
- // @ts-ignore
- const toolDesc = `> Tool Name: ${tool.metadata.name}\nTool Description: ${tool.metadata.description}\nTool Args: ${JSON.stringify(tool?.metadata?.parameters?.properties)}\n`;
- toolDescs.push(toolDesc);
- }
- return toolDescs;
-}
-
-export interface BaseAgentChatFormatter {
- format(
- tools: BaseTool[],
- chatHistory: ChatMessage[],
- currentReasoning?: BaseReasoningStep[],
- ): ChatMessage[];
-}
-
-export class ReActChatFormatter implements BaseAgentChatFormatter {
- systemHeader: string = "";
- context: string = "'";
-
- constructor(init?: Partial<ReActChatFormatter>) {
- Object.assign(this, init);
- }
-
- format(
- tools: BaseTool[],
- chatHistory: ChatMessage[],
- currentReasoning?: BaseReasoningStep[],
- ): ChatMessage[] {
- currentReasoning = currentReasoning ?? [];
-
- const formatArgs = {
- toolDesc: getReactToolDescriptions(tools).join("\n"),
- toolNames: tools.map((tool) => tool.metadata.name).join(", "),
- context: "",
- };
-
- if (this.context) {
- formatArgs["context"] = this.context;
- }
-
- const reasoningHistory = [];
-
- for (const reasoningStep of currentReasoning) {
- let message: ChatMessage | undefined;
-
- if (reasoningStep instanceof ObservationReasoningStep) {
- message = {
- content: reasoningStep.getContent(),
- role: "user",
- };
- } else {
- message = {
- content: reasoningStep.getContent(),
- role: "assistant",
- };
- }
-
- reasoningHistory.push(message);
- }
-
- const systemContent = getReactChatSystemHeader({
- toolDesc: formatArgs.toolDesc,
- toolNames: formatArgs.toolNames,
- });
-
- return [
- {
- content: systemContent,
- role: "system",
- },
- ...chatHistory,
- ...reasoningHistory,
- ];
- }
-}
diff --git a/packages/core/src/agent/react/outputParser.ts b/packages/core/src/agent/react/outputParser.ts
deleted file mode 100644
index 593cdc18784c0d7f0e559ee20db2994a80cee227..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/react/outputParser.ts
+++ /dev/null
@@ -1,105 +0,0 @@
-import type { BaseReasoningStep } from "./types.js";
-import {
- ActionReasoningStep,
- BaseOutputParser,
- ResponseReasoningStep,
-} from "./types.js";
-
-function extractJsonStr(text: string): string {
- const pattern = /\{.*\}/s;
- const match = text.match(pattern);
-
- if (!match) {
- throw new Error(`Could not extract json string from output: ${text}`);
- }
-
- return match[0];
-}
-
-function extractToolUse(inputText: string): [string, string, string] {
- const pattern =
- /\s*Thought: (.*?)\nAction: ([a-zA-Z0-9_]+).*?\nAction Input: .*?(\{.*?\})/s;
-
- const match = inputText.match(pattern);
-
- if (!match) {
- throw new Error(`Could not extract tool use from input text: ${inputText}`);
- }
-
- const thought = match[1].trim();
- const action = match[2].trim();
- const actionInput = match[3].trim();
- return [thought, action, actionInput];
-}
-
-function actionInputParser(jsonStr: string): object {
- const processedString = jsonStr.replace(/(?<!\w)\'|\'(?!\w)/g, '"');
- const pattern = /"(\w+)":\s*"([^"]*)"/g;
- const matches = [...processedString.matchAll(pattern)];
- return Object.fromEntries(matches);
-}
-
-function extractFinalResponse(inputText: string): [string, string] {
- const pattern = /\s*Thought:(.*?)Answer:(.*?)(?:$)/s;
-
- const match = inputText.match(pattern);
-
- if (!match) {
- throw new Error(
- `Could not extract final answer from input text: ${inputText}`,
- );
- }
-
- const thought = match[1].trim();
- const answer = match[2].trim();
- return [thought, answer];
-}
-
-export class ReActOutputParser extends BaseOutputParser {
- parse(output: string, isStreaming: boolean = false): BaseReasoningStep {
- if (!output.includes("Thought:")) {
- // NOTE: handle the case where the agent directly outputs the answer
- // instead of following the thought-answer format
- return new ResponseReasoningStep({
- thought: "(Implicit) I can answer without any more tools!",
- response: output,
- isStreaming,
- });
- }
-
- if (output.includes("Answer:")) {
- const [thought, answer] = extractFinalResponse(output);
- return new ResponseReasoningStep({
- thought: thought,
- response: answer,
- isStreaming,
- });
- }
-
- if (output.includes("Action:")) {
- const [thought, action, action_input] = extractToolUse(output);
- const json_str = extractJsonStr(action_input);
-
- // First we try json, if this fails we use ast
- let actionInputDict;
-
- try {
- actionInputDict = JSON.parse(json_str);
- } catch (e) {
- actionInputDict = actionInputParser(json_str);
- }
-
- return new ActionReasoningStep({
- thought: thought,
- action: action,
- actionInput: actionInputDict,
- });
- }
-
- throw new Error(`Could not parse output: ${output}`);
- }
-
- format(output: string): string {
- throw new Error("Not implemented");
- }
-}
diff --git a/packages/core/src/agent/react/types.ts b/packages/core/src/agent/react/types.ts
deleted file mode 100644
index e8849275f3c5e3e9e889c98fe8668e0e232dc69c..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/react/types.ts
+++ /dev/null
@@ -1,91 +0,0 @@
-import type { ChatMessage } from "../../llm/index.js";
-import { extractText } from "../../llm/utils.js";
-
-export interface BaseReasoningStep {
- getContent(): string;
- isDone(): boolean;
-}
-
-export class ObservationReasoningStep implements BaseReasoningStep {
- observation: string;
-
- constructor(init?: Partial<ObservationReasoningStep>) {
- this.observation = init?.observation ?? "";
- }
-
- getContent(): string {
- return `Observation: ${this.observation}`;
- }
-
- isDone(): boolean {
- return false;
- }
-}
-
-export class ActionReasoningStep implements BaseReasoningStep {
- thought: string;
- action: string;
- actionInput: Record<string, any>;
-
- constructor(init?: Partial<ActionReasoningStep>) {
- this.thought = init?.thought ?? "";
- this.action = init?.action ?? "";
- this.actionInput = init?.actionInput ?? {};
- }
-
- getContent(): string {
- return `Thought: ${this.thought}\nAction: ${this.action}\nAction Input: ${JSON.stringify(this.actionInput)}`;
- }
-
- isDone(): boolean {
- return false;
- }
-}
-
-export abstract class BaseOutputParser {
- abstract parse(output: string, isStreaming?: boolean): BaseReasoningStep;
-
- format(output: string) {
- return output;
- }
-
- formatMessages(messages: ChatMessage[]): ChatMessage[] {
- if (messages) {
- if (messages[0].role === "system") {
- messages[0].content = this.format(
- extractText(messages[0].content) || "",
- );
- } else {
- messages[messages.length - 1].content = this.format(
- extractText(messages[messages.length - 1].content) || "",
- );
- }
- }
-
- return messages;
- }
-}
-
-export class ResponseReasoningStep implements BaseReasoningStep {
- thought: string;
- response: string;
- isStreaming: boolean = false;
-
- constructor(init?: Partial<ResponseReasoningStep>) {
- this.thought = init?.thought ?? "";
- this.response = init?.response ?? "";
- this.isStreaming = init?.isStreaming ?? false;
- }
-
- getContent(): string {
- if (this.isStreaming) {
- return `Thought: ${this.thought}\nAnswer (Starts With): ${this.response} ...`;
- } else {
- return `Thought: ${this.thought}\nAnswer: ${this.response}`;
- }
- }
-
- isDone(): boolean {
- return true;
- }
-}
diff --git a/packages/core/src/agent/react/worker.ts b/packages/core/src/agent/react/worker.ts
deleted file mode 100644
index 3fba7b04cc0eb945f6bd6d8d8de7303f0755f129..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/react/worker.ts
+++ /dev/null
@@ -1,325 +0,0 @@
-import { randomUUID } from "@llamaindex/env";
-import type { ChatMessage } from "cohere-ai/api";
-import { Settings } from "../../Settings.js";
-import { AgentChatResponse } from "../../engines/chat/index.js";
-import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
-import { type ChatResponse, type LLM } from "../../llm/index.js";
-import { extractText } from "../../llm/utils.js";
-import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
-import type { ObjectRetriever } from "../../objects/base.js";
-import { ToolOutput } from "../../tools/index.js";
-import type { BaseTool } from "../../types.js";
-import type { AgentWorker, Task } from "../types.js";
-import { TaskStep, TaskStepOutput } from "../types.js";
-import { ReActChatFormatter } from "./formatter.js";
-import { ReActOutputParser } from "./outputParser.js";
-import type { BaseReasoningStep } from "./types.js";
-import {
- ActionReasoningStep,
- ObservationReasoningStep,
- ResponseReasoningStep,
-} from "./types.js";
-
-type ReActAgentWorkerParams = {
- tools: BaseTool[];
- llm?: LLM;
- maxInteractions?: number;
- reactChatFormatter?: ReActChatFormatter | undefined;
- outputParser?: ReActOutputParser | undefined;
- toolRetriever?: ObjectRetriever<BaseTool> | undefined;
-};
-
-function addUserStepToReasoning(
- step: TaskStep,
- memory: ChatMemoryBuffer,
- currentReasoning: BaseReasoningStep[],
-): void {
- if (step.stepState.isFirst) {
- memory.put({
- content: step.input ?? "",
- role: "user",
- });
- step.stepState.isFirst = false;
- } else {
- const reasoningStep = new ObservationReasoningStep({
- observation: step.input ?? undefined,
- });
- currentReasoning.push(reasoningStep);
- if (Settings.debug) {
- console.log(`Added user message to memory: ${step.input}`);
- }
- }
-}
-
-type ChatParams = {
- messages: ChatMessage[];
- tools?: BaseTool[];
-};
-
-/**
- * ReAct agent worker.
- */
-export class ReActAgentWorker implements AgentWorker<ChatParams> {
- llm: LLM;
-
- maxInteractions: number = 10;
- reactChatFormatter: ReActChatFormatter;
- outputParser: ReActOutputParser;
-
- _getTools: (message: string) => Promise<BaseTool[]>;
-
- constructor({
- tools,
- llm,
- maxInteractions,
- reactChatFormatter,
- outputParser,
- toolRetriever,
- }: ReActAgentWorkerParams) {
- this.llm = llm ?? Settings.llm;
-
- this.maxInteractions = maxInteractions ?? 10;
- this.reactChatFormatter = reactChatFormatter ?? new ReActChatFormatter();
- this.outputParser = outputParser ?? new ReActOutputParser();
-
- if (tools.length > 0 && toolRetriever) {
- throw new Error("Cannot specify both tools and tool_retriever");
- } else if (tools.length > 0) {
- this._getTools = async () => tools;
- } else if (toolRetriever) {
- this._getTools = async (message: string) =>
- toolRetriever.retrieve(message);
- } else {
- this._getTools = async () => [];
- }
- }
-
- initializeStep(task: Task): TaskStep {
- const sources: ToolOutput[] = [];
- const currentReasoning: BaseReasoningStep[] = [];
- const newMemory = new ChatMemoryBuffer({
- tokenLimit: task.memory.tokenLimit,
- });
-
- const taskState = {
- sources,
- currentReasoning,
- newMemory,
- };
-
- task.extraState = {
- ...task.extraState,
- ...taskState,
- };
-
- return new TaskStep(task.taskId, randomUUID(), task.input, {
- isFirst: true,
- });
- }
-
- extractReasoningStep(
- output: ChatResponse,
- isStreaming: boolean,
- ): [string, BaseReasoningStep[], boolean] {
- if (!output.message.content) {
- throw new Error("Got empty message.");
- }
-
- const messageContent = output.message.content;
- const currentReasoning: BaseReasoningStep[] = [];
-
- let reasoningStep;
-
- try {
- reasoningStep = this.outputParser.parse(
- extractText(messageContent),
- isStreaming,
- ) as ActionReasoningStep;
- } catch (e) {
- throw new Error(`Could not parse output: ${e}`);
- }
-
- if (Settings.debug) {
- console.log(`${reasoningStep.getContent()}\n`);
- }
-
- currentReasoning.push(reasoningStep);
-
- if (reasoningStep.isDone()) {
- return [extractText(messageContent), currentReasoning, true];
- }
-
- const actionReasoningStep = new ActionReasoningStep({
- thought: reasoningStep.getContent(),
- action: reasoningStep.action,
- actionInput: reasoningStep.actionInput,
- });
-
- if (!(actionReasoningStep instanceof ActionReasoningStep)) {
- throw new Error(`Expected ActionReasoningStep, got ${reasoningStep}`);
- }
-
- return [extractText(messageContent), currentReasoning, false];
- }
-
- async _processActions(
- task: Task,
- tools: BaseTool[],
- output: ChatResponse,
- isStreaming: boolean = false,
- ): Promise<[BaseReasoningStep[], boolean]> {
- const toolsDict: Record<string, BaseTool> = {};
-
- for (const tool of tools) {
- toolsDict[tool.metadata.name] = tool;
- }
-
- const [_, currentReasoning, isDone] = this.extractReasoningStep(
- output,
- isStreaming,
- );
-
- if (isDone) {
- return [currentReasoning, true];
- }
-
- const reasoningStep = currentReasoning[
- currentReasoning.length - 1
- ] as ActionReasoningStep;
-
- const actionReasoningStep = new ActionReasoningStep({
- thought: reasoningStep.getContent(),
- action: reasoningStep.action,
- actionInput: reasoningStep.actionInput,
- });
-
- const tool = toolsDict[actionReasoningStep.action];
-
- getCallbackManager().dispatchEvent("llm-tool-call", {
- payload: {
- toolCall: {
- name: tool.metadata.name,
- input: JSON.stringify(actionReasoningStep.actionInput),
- },
- },
- });
- const toolOutput = await tool.call!(actionReasoningStep.actionInput);
-
- task.extraState.sources.push(
- new ToolOutput(
- toolOutput,
- tool.metadata.name,
- actionReasoningStep.actionInput,
- toolOutput,
- ),
- );
-
- const observationStep = new ObservationReasoningStep({
- observation: toolOutput,
- });
-
- currentReasoning.push(observationStep);
-
- if (Settings.debug) {
- console.log(`${observationStep.getContent()}`);
- }
-
- return [currentReasoning, false];
- }
-
- _getResponse(
- currentReasoning: BaseReasoningStep[],
- sources: ToolOutput[],
- ): AgentChatResponse {
- if (currentReasoning.length === 0) {
- throw new Error("No reasoning steps were taken.");
- } else if (currentReasoning.length === this.maxInteractions) {
- throw new Error("Reached max iterations.");
- }
-
- const responseStep = currentReasoning[currentReasoning.length - 1];
-
- let responseStr: string;
-
- if (responseStep instanceof ResponseReasoningStep) {
- responseStr = responseStep.response;
- } else {
- responseStr = responseStep.getContent();
- }
-
- return new AgentChatResponse(responseStr, sources);
- }
-
- _getTaskStepResponse(
- agentResponse: AgentChatResponse,
- step: TaskStep,
- isDone: boolean,
- ): TaskStepOutput {
- let newSteps: TaskStep[] = [];
-
- if (isDone) {
- newSteps = [];
- } else {
- newSteps = [step.getNextStep(randomUUID(), undefined)];
- }
-
- return new TaskStepOutput(agentResponse, step, newSteps, isDone);
- }
-
- async _runStep(step: TaskStep, task: Task): Promise<TaskStepOutput> {
- if (step.input) {
- addUserStepToReasoning(
- step,
- task.extraState.newMemory,
- task.extraState.currentReasoning,
- );
- }
-
- const tools = await this._getTools(task.input);
-
- const inputChat = this.reactChatFormatter.format(
- tools,
- [...task.memory.getAll(), ...task.extraState.newMemory.getAll()],
- task.extraState.currentReasoning,
- );
-
- const chatResponse = await this.llm.chat({
- messages: inputChat,
- });
-
- const [reasoningSteps, isDone] = await this._processActions(
- task,
- tools,
- chatResponse,
- );
-
- task.extraState.currentReasoning.push(...reasoningSteps);
-
- const agentResponse = this._getResponse(
- task.extraState.currentReasoning,
- task.extraState.sources,
- );
-
- if (isDone) {
- task.extraState.newMemory.put({
- content: agentResponse.response,
- role: "assistant",
- });
- }
-
- return this._getTaskStepResponse(agentResponse, step, isDone);
- }
-
- async runStep(step: TaskStep, task: Task): Promise<TaskStepOutput> {
- return await this._runStep(step, task);
- }
-
- streamStep(): Promise<TaskStepOutput> {
- throw new Error("Method not implemented.");
- }
-
- finalizeTask(task: Task): void {
- task.memory.set(task.memory.get() + task.extraState.newMemory.get());
- task.extraState.newMemory.reset();
- }
-}
diff --git a/packages/core/src/agent/runner/base.ts b/packages/core/src/agent/runner/base.ts
deleted file mode 100644
index 915f996b3a0c831e861f32f135cd41aa41485408..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/runner/base.ts
+++ /dev/null
@@ -1,384 +0,0 @@
-import { randomUUID } from "@llamaindex/env";
-import type { ChatHistory } from "../../ChatHistory.js";
-import type { ChatEngineAgentParams } from "../../engines/chat/index.js";
-import {
- AgentChatResponse,
- ChatResponseMode,
- StreamingAgentChatResponse,
-} from "../../engines/chat/index.js";
-import type { LLM } from "../../llm/index.js";
-import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
-import type { BaseMemory } from "../../memory/types.js";
-import type { AgentWorker, TaskStepOutput } from "../types.js";
-import { Task, TaskStep } from "../types.js";
-import { AgentState, BaseAgentRunner, TaskState } from "./types.js";
-const validateStepFromArgs = (
- taskId: string,
- input?: string | null,
- step?: any,
- kwargs?: any,
-): TaskStep | undefined => {
- if (step) {
- if (input) {
- throw new Error("Cannot specify both `step` and `input`");
- }
- return step;
- } else {
- if (!input) return;
- return new TaskStep(taskId, step, input, kwargs);
- }
-};
-
-type AgentRunnerParams = {
- agentWorker: AgentWorker;
- chatHistory?: ChatHistory;
- state?: AgentState;
- memory?: BaseMemory;
- llm?: LLM;
- initTaskStateKwargs?: Record<string, any>;
- deleteTaskOnFinish?: boolean;
- defaultToolChoice?: string;
-};
-
-export class AgentRunner extends BaseAgentRunner {
- agentWorker: AgentWorker;
- state: AgentState;
- memory: BaseMemory;
- initTaskStateKwargs: Record<string, any>;
- deleteTaskOnFinish: boolean;
- defaultToolChoice: string;
-
- /**
- * Creates an AgentRunner.
- */
- constructor(params: AgentRunnerParams) {
- super();
-
- this.agentWorker = params.agentWorker;
- this.state = params.state ?? new AgentState();
- this.memory =
- params.memory ??
- new ChatMemoryBuffer({
- llm: params.llm,
- chatHistory: params.chatHistory,
- });
- this.initTaskStateKwargs = params.initTaskStateKwargs ?? {};
- this.deleteTaskOnFinish = params.deleteTaskOnFinish ?? false;
- this.defaultToolChoice = params.defaultToolChoice ?? "auto";
- }
-
- /**
- * Creates a task.
- * @param input
- * @param kwargs
- */
- createTask(input: string, kwargs?: any): Task {
- let extraState;
-
- if (!this.initTaskStateKwargs) {
- if (kwargs && "extraState" in kwargs) {
- if (extraState) {
- delete extraState["extraState"];
- }
- }
- } else {
- if (kwargs && "extraState" in kwargs) {
- throw new Error(
- "Cannot specify both `extraState` and `initTaskStateKwargs`",
- );
- } else {
- extraState = this.initTaskStateKwargs;
- }
- }
-
- const task = new Task({
- taskId: randomUUID(),
- input,
- memory: this.memory,
- extraState,
- ...kwargs,
- });
-
- const initialStep = this.agentWorker.initializeStep(task);
-
- const taskState = new TaskState({
- task,
- stepQueue: [initialStep],
- });
-
- this.state.taskDict[task.taskId] = taskState;
-
- return task;
- }
-
- /**
- * Deletes the task.
- * @param taskId
- */
- deleteTask(taskId: string): void {
- delete this.state.taskDict[taskId];
- }
-
- /**
- * Returns the list of tasks.
- */
- listTasks(): Task[] {
- return Object.values(this.state.taskDict).map(
- (taskState) => taskState.task,
- );
- }
-
- /**
- * Returns the task.
- */
- getTask(taskId: string): Task {
- return this.state.taskDict[taskId].task;
- }
-
- /**
- * Returns the completed steps in the task.
- * @param taskId
- * @param kwargs
- */
- getCompletedSteps(taskId: string): TaskStepOutput[] {
- return this.state.taskDict[taskId].completedSteps;
- }
-
- /**
- * Returns the next steps in the task.
- * @param taskId
- * @param kwargs
- */
- getUpcomingSteps(taskId: string, kwargs: any): TaskStep[] {
- return this.state.taskDict[taskId].stepQueue;
- }
-
- private async _runStep(
- taskId: string,
- step?: TaskStep,
- mode: ChatResponseMode = ChatResponseMode.WAIT,
- kwargs?: any,
- ): Promise<TaskStepOutput> {
- const task = this.state.getTask(taskId);
- const curStep = step || this.state.getStepQueue(taskId).shift();
-
- let curStepOutput: TaskStepOutput;
-
- if (!curStep) {
- throw new Error(`No step found for task ${taskId}`);
- }
-
- if (mode === ChatResponseMode.WAIT) {
- curStepOutput = await this.agentWorker.runStep(curStep, task, kwargs);
- } else if (mode === ChatResponseMode.STREAM) {
- curStepOutput = await this.agentWorker.streamStep(curStep, task, kwargs);
- } else {
- throw new Error(`Invalid mode: ${mode}`);
- }
-
- const nextSteps = curStepOutput.nextSteps;
-
- this.state.addSteps(taskId, nextSteps);
- this.state.addCompletedStep(taskId, [curStepOutput]);
-
- return curStepOutput;
- }
-
- /**
- * Runs the next step in the task.
- * @param taskId
- * @param kwargs
- * @param step
- * @returns
- */
- async runStep(
- taskId: string,
- input?: string | null,
- step?: TaskStep,
- kwargs: any = {},
- ): Promise<TaskStepOutput> {
- const curStep = validateStepFromArgs(taskId, input, step, kwargs);
- return this._runStep(taskId, curStep, ChatResponseMode.WAIT, kwargs);
- }
-
- /**
- * Runs the step and returns the response.
- * @param taskId
- * @param input
- * @param step
- * @param kwargs
- */
- async streamStep(
- taskId: string,
- input: string,
- step?: TaskStep,
- kwargs?: any,
- ): Promise<TaskStepOutput> {
- const curStep = validateStepFromArgs(taskId, input, step, kwargs);
- return this._runStep(taskId, curStep, ChatResponseMode.STREAM, kwargs);
- }
-
- /**
- * Finalizes the response and returns it.
- * @param taskId
- * @param kwargs
- * @param stepOutput
- * @returns
- */
- async finalizeResponse(
- taskId: string,
- stepOutput: TaskStepOutput,
- kwargs?: any,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
- if (!stepOutput) {
- stepOutput =
- this.getCompletedSteps(taskId)[
- this.getCompletedSteps(taskId).length - 1
- ];
- }
-
- if (!stepOutput.isLast) {
- throw new Error(
- "finalizeResponse can only be called on the last step output",
- );
- }
-
- if (!(stepOutput.output instanceof StreamingAgentChatResponse)) {
- if (!(stepOutput.output instanceof AgentChatResponse)) {
- throw new Error(
- `When \`isLast\` is True, cur_step_output.output must be AGENT_CHAT_RESPONSE_TYPE: ${stepOutput.output}`,
- );
- }
- }
-
- this.agentWorker.finalizeTask(this.getTask(taskId), kwargs);
-
- if (this.deleteTaskOnFinish) {
- this.deleteTask(taskId);
- }
-
- return stepOutput.output;
- }
-
- protected async _chat({
- message,
- toolChoice,
- stream,
- }: ChatEngineAgentParams): Promise<AgentChatResponse>;
- protected async _chat({
- message,
- toolChoice,
- stream,
- }: ChatEngineAgentParams & {
- stream: true;
- }): Promise<StreamingAgentChatResponse>;
- protected async _chat({
- message,
- toolChoice,
- stream,
- }: ChatEngineAgentParams): Promise<
- AgentChatResponse | StreamingAgentChatResponse
- > {
- const task = this.createTask(message as string);
-
- let resultOutput;
-
- const mode = stream ? ChatResponseMode.STREAM : ChatResponseMode.WAIT;
-
- while (true) {
- const curStepOutput = await this._runStep(task.taskId, undefined, mode, {
- toolChoice,
- });
-
- if (curStepOutput.isLast) {
- resultOutput = curStepOutput;
- break;
- }
-
- toolChoice = "auto";
- }
-
- return this.finalizeResponse(task.taskId, resultOutput);
- }
-
- /**
- * Sends a message to the LLM and returns the response.
- * @param message
- * @param chatHistory
- * @param toolChoice
- * @returns
- */
- public async chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- }: ChatEngineAgentParams & {
- stream?: false;
- }): Promise<AgentChatResponse>;
- public async chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- }: ChatEngineAgentParams & {
- stream: true;
- }): Promise<StreamingAgentChatResponse>;
- public async chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- }: ChatEngineAgentParams): Promise<
- AgentChatResponse | StreamingAgentChatResponse
- > {
- if (!toolChoice) {
- toolChoice = this.defaultToolChoice;
- }
-
- const chatResponse = await this._chat({
- message,
- chatHistory,
- toolChoice,
- stream,
- });
-
- return chatResponse;
- }
-
- protected _getPromptModules(): string[] {
- return [];
- }
-
- protected _getPrompts(): string[] {
- return [];
- }
-
- /**
- * Resets the agent.
- */
- reset(): void {
- this.state = new AgentState();
- }
-
- getCompletedStep(
- taskId: string,
- stepId: string,
- kwargs: any,
- ): TaskStepOutput {
- const completedSteps = this.getCompletedSteps(taskId);
- for (const stepOutput of completedSteps) {
- if (stepOutput.taskStep.stepId === stepId) {
- return stepOutput;
- }
- }
-
- throw new Error(`Step ${stepId} not found in task ${taskId}`);
- }
-
- /**
- * Undoes the step.
- * @param taskId
- */
- undoStep(taskId: string): void {}
-}
diff --git a/packages/core/src/agent/runner/types.ts b/packages/core/src/agent/runner/types.ts
deleted file mode 100644
index c6fb3b8b6231aa9d2f33f700425c9d1c15a7ffb4..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/runner/types.ts
+++ /dev/null
@@ -1,106 +0,0 @@
-import type {
- AgentChatResponse,
- StreamingAgentChatResponse,
-} from "../../engines/chat/index.js";
-import type { Task, TaskStep, TaskStepOutput } from "../types.js";
-import { BaseAgent } from "../types.js";
-
-export class TaskState {
- task!: Task;
- stepQueue!: TaskStep[];
- completedSteps!: TaskStepOutput[];
-
- constructor(init?: Partial<TaskState>) {
- Object.assign(this, init);
- }
-}
-
-export abstract class BaseAgentRunner extends BaseAgent {
- constructor(init?: Partial<BaseAgentRunner>) {
- super();
- }
-
- abstract createTask(input: string, kwargs: any): Task;
- abstract deleteTask(taskId: string): void;
- abstract getTask(taskId: string, kwargs: any): Task;
- abstract listTasks(kwargs: any): Task[];
- abstract getUpcomingSteps(taskId: string, kwargs: any): TaskStep[];
- abstract getCompletedSteps(taskId: string, kwargs: any): TaskStepOutput[];
-
- getCompletedStep(
- taskId: string,
- stepId: string,
- kwargs: any,
- ): TaskStepOutput {
- const completedSteps = this.getCompletedSteps(taskId, kwargs);
- for (const stepOutput of completedSteps) {
- if (stepOutput.taskStep.stepId === stepId) {
- return stepOutput;
- }
- }
-
- throw new Error(`Step ${stepId} not found in task ${taskId}`);
- }
-
- abstract runStep(
- taskId: string,
- input: string,
- step: TaskStep,
- kwargs: any,
- ): Promise<TaskStepOutput>;
-
- abstract streamStep(
- taskId: string,
- input: string,
- step: TaskStep,
- kwargs?: any,
- ): Promise<TaskStepOutput>;
-
- abstract finalizeResponse(
- taskId: string,
- stepOutput: TaskStepOutput,
- kwargs?: any,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-
- abstract undoStep(taskId: string): void;
-}
-
-export class AgentState {
- taskDict!: Record<string, TaskState>;
-
- constructor(init?: Partial<AgentState>) {
- Object.assign(this, init);
-
- if (!this.taskDict) {
- this.taskDict = {};
- }
- }
-
- getTask(taskId: string): Task {
- return this.taskDict[taskId].task;
- }
-
- getCompletedSteps(taskId: string): TaskStepOutput[] {
- return this.taskDict[taskId].completedSteps || [];
- }
-
- getStepQueue(taskId: string): TaskStep[] {
- return this.taskDict[taskId].stepQueue || [];
- }
-
- addSteps(taskId: string, steps: TaskStep[]): void {
- if (!this.taskDict[taskId].stepQueue) {
- this.taskDict[taskId].stepQueue = [];
- }
-
- this.taskDict[taskId].stepQueue.push(...steps);
- }
-
- addCompletedStep(taskId: string, stepOutputs: TaskStepOutput[]): void {
- if (!this.taskDict[taskId].completedSteps) {
- this.taskDict[taskId].completedSteps = [];
- }
-
- this.taskDict[taskId].completedSteps.push(...stepOutputs);
- }
-}
diff --git a/packages/core/src/agent/type.ts b/packages/core/src/agent/type.ts
new file mode 100644
index 0000000000000000000000000000000000000000..38d974cf0bf060010f9ff82571de7adafdfb47de
--- /dev/null
+++ b/packages/core/src/agent/type.ts
@@ -0,0 +1,4 @@
+import type { BaseEvent } from "../internal/type.js";
+
+export type AgentStartEvent = BaseEvent<{}>;
+export type AgentEndEvent = BaseEvent<{}>;
diff --git a/packages/core/src/agent/types.ts b/packages/core/src/agent/types.ts
deleted file mode 100644
index 5864c3202f01b53f8ef404417f3105791ea5bb1b..0000000000000000000000000000000000000000
--- a/packages/core/src/agent/types.ts
+++ /dev/null
@@ -1,202 +0,0 @@
-import type {
- AgentChatResponse,
- ChatEngineAgentParams,
- StreamingAgentChatResponse,
-} from "../engines/chat/index.js";
-
-import type { BaseMemory } from "../memory/types.js";
-import type { QueryEngineParamsNonStreaming } from "../types.js";
-
-export interface AgentWorker<ExtraParams extends object = object> {
- initializeStep(task: Task, params?: ExtraParams): TaskStep;
- runStep(
- step: TaskStep,
- task: Task,
- params?: ExtraParams,
- ): Promise<TaskStepOutput>;
- streamStep(
- step: TaskStep,
- task: Task,
- params?: ExtraParams,
- ): Promise<TaskStepOutput>;
- finalizeTask(task: Task, params?: ExtraParams): void;
-}
-
-interface BaseChatEngine {
- chat(
- params: ChatEngineAgentParams,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-}
-
-interface BaseQueryEngine {
- query(
- params: QueryEngineParamsNonStreaming,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-}
-
-/**
- * BaseAgent is the base class for all agents.
- */
-export abstract class BaseAgent implements BaseChatEngine, BaseQueryEngine {
- protected _getPrompts(): string[] {
- return [];
- }
-
- protected _getPromptModules(): string[] {
- return [];
- }
-
- abstract chat(
- params: ChatEngineAgentParams,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse>;
-
- abstract reset(): void;
-
- /**
- * query is the main entrypoint for the agent. It takes a query and returns a response.
- * @param params
- * @returns
- */
- async query(
- params: QueryEngineParamsNonStreaming,
- ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
- // Handle non-streaming query
- const agentResponse = await this.chat({
- message: params.query,
- chatHistory: [],
- });
-
- return agentResponse;
- }
-}
-
-type TaskParams = {
- taskId: string;
- input: string;
- memory: BaseMemory;
- extraState: Record<string, any>;
-};
-
-/**
- * Task is a unit of work for the agent.
- * @param taskId: taskId
- */
-export class Task {
- taskId: string;
- input: string;
-
- memory: BaseMemory;
- extraState: Record<string, any>;
-
- constructor({ taskId, input, memory, extraState }: TaskParams) {
- this.taskId = taskId;
- this.input = input;
- this.memory = memory;
- this.extraState = extraState ?? {};
- }
-}
-
-interface ITaskStep {
- taskId: string;
- stepId: string;
- input?: string | null;
- stepState: Record<string, any>;
- nextSteps: Record<string, TaskStep>;
- prevSteps: Record<string, TaskStep>;
- isReady: boolean;
- getNextStep(
- stepId: string,
- input?: string,
- stepState?: Record<string, any>,
- ): TaskStep;
- linkStep(nextStep: TaskStep): void;
-}
-
-/**
- * TaskStep is a unit of work for the agent.
- * @param taskId: taskId
- * @param stepId: stepId
- * @param input: input
- * @param stepState: stepState
- */
-export class TaskStep implements ITaskStep {
- taskId: string;
- stepId: string;
- input?: string | null;
- stepState: Record<string, any> = {};
- nextSteps: Record<string, TaskStep> = {};
- prevSteps: Record<string, TaskStep> = {};
- isReady: boolean = true;
-
- constructor(
- taskId: string,
- stepId: string,
- input?: string | null,
- stepState?: Record<string, any> | null,
- ) {
- this.taskId = taskId;
- this.stepId = stepId;
- this.input = input;
- this.stepState = stepState ?? this.stepState;
- }
-
- /*
- * getNextStep is a function that returns the next step.
- * @param stepId: stepId
- * @param input: input
- * @param stepState: stepState
- * @returns: TaskStep
- */
- getNextStep(
- stepId: string,
- input?: string,
- stepState?: Record<string, unknown>,
- ): TaskStep {
- return new TaskStep(
- this.taskId,
- stepId,
- input,
- stepState ?? this.stepState,
- );
- }
-
- /*
- * linkStep is a function that links the next step.
- * @param nextStep: nextStep
- * @returns: void
- */
- linkStep(nextStep: TaskStep): void {
- this.nextSteps[nextStep.stepId] = nextStep;
- nextStep.prevSteps[this.stepId] = this;
- }
-}
-
-/**
- * TaskStepOutput is a unit of work for the agent.
- * @param output: output
- * @param taskStep: taskStep
- * @param nextSteps: nextSteps
- * @param isLast: isLast
- */
-export class TaskStepOutput {
- output: AgentChatResponse | StreamingAgentChatResponse;
- taskStep: TaskStep;
- nextSteps: TaskStep[];
- isLast: boolean;
-
- constructor(
- output: AgentChatResponse | StreamingAgentChatResponse,
- taskStep: TaskStep,
- nextSteps: TaskStep[],
- isLast: boolean = false,
- ) {
- this.output = output;
- this.taskStep = taskStep;
- this.nextSteps = nextSteps;
- this.isLast = isLast;
- }
-
- toString(): string {
- return String(this.output);
- }
-}
diff --git a/packages/core/src/agent/utils.ts b/packages/core/src/agent/utils.ts
index a22c882ec0286b4f281f0b7ebd858711d05e48c0..08a601edfdba9b2a31be406587c71ec27443148f 100644
--- a/packages/core/src/agent/utils.ts
+++ b/packages/core/src/agent/utils.ts
@@ -1,35 +1,112 @@
-import { Settings } from "../Settings.js";
-import type { ChatMessage } from "../llm/index.js";
-import type { ChatMemoryBuffer } from "../memory/ChatMemoryBuffer.js";
-import type { BaseTool } from "../types.js";
-import type { TaskStep } from "./types.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { isAsyncIterable, prettifyError } from "../internal/utils.js";
+import type {
+ ChatMessage,
+ ChatResponseChunk,
+ TextChatMessage,
+ ToolCall,
+} from "../llm/index.js";
+import type { BaseTool, ToolOutput } from "../types.js";
-export function addUserStepToMemory(
- step: TaskStep,
- memory: ChatMemoryBuffer,
-): void {
- if (!step.input) {
- return;
+export async function callTool(
+ tool: BaseTool | undefined,
+ toolCall: ToolCall,
+): Promise<ToolOutput> {
+ if (!tool) {
+ const output = `Tool ${toolCall.name} does not exist.`;
+ return {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: true,
+ };
}
-
- const userMessage: ChatMessage = {
- content: step.input,
- role: "user",
- };
-
- memory.put(userMessage);
-
- if (Settings.debug) {
- console.log(`Added user message to memory!: ${userMessage.content}`);
+ const call = tool.call;
+ let output: string;
+ if (!call) {
+ output = `Tool ${tool.metadata.name} (remote:${toolCall.name}) does not have a implementation.`;
+ return {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: true,
+ };
}
+ try {
+ let input = toolCall.input;
+ if (typeof input === "string") {
+ input = JSON.parse(input);
+ }
+ getCallbackManager().dispatchEvent("llm-tool-call", {
+ payload: {
+ toolCall: { ...toolCall },
+ },
+ });
+ output = await call.call(tool, input);
+ const toolOutput: ToolOutput = {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: false,
+ };
+ getCallbackManager().dispatchEvent("llm-tool-result", {
+ payload: {
+ toolCall: { ...toolCall },
+ toolResult: { ...toolOutput },
+ },
+ });
+ return toolOutput;
+ } catch (e) {
+ output = prettifyError(e);
+ }
+ return {
+ tool,
+ input: toolCall.input,
+ output,
+ isError: true,
+ };
}
-export function getFunctionByName(tools: BaseTool[], name: string): BaseTool {
- const exist = tools.find((tool) => tool.metadata.name === name);
-
- if (!exist) {
- throw new Error(`Tool with name ${name} not found`);
+export async function consumeAsyncIterable<Options extends object>(
+ input: ChatMessage<Options>,
+): Promise<ChatMessage<Options>>;
+export async function consumeAsyncIterable<Options extends object>(
+ input: AsyncIterable<ChatResponseChunk<Options>>,
+): Promise<TextChatMessage<Options>>;
+export async function consumeAsyncIterable<Options extends object>(
+ input: ChatMessage<Options> | AsyncIterable<ChatResponseChunk<Options>>,
+): Promise<ChatMessage<Options>> {
+ if (isAsyncIterable(input)) {
+ const result: ChatMessage<Options> = {
+ content: "",
+ // only assistant will give streaming response
+ role: "assistant",
+ options: {} as Options,
+ };
+ for await (const chunk of input) {
+ result.content += chunk.delta;
+ if (chunk.options) {
+ result.options = {
+ ...result.options,
+ ...chunk.options,
+ };
+ }
+ }
+ return result;
+ } else {
+ return input;
}
+}
- return exist;
+export function createReadableStream<T>(
+ asyncIterable: AsyncIterable<T>,
+): ReadableStream<T> {
+ return new ReadableStream<T>({
+ async start(controller) {
+ for await (const chunk of asyncIterable) {
+ controller.enqueue(chunk);
+ }
+ controller.close();
+ },
+ });
}
diff --git a/packages/core/src/callbacks/CallbackManager.ts b/packages/core/src/callbacks/CallbackManager.ts
index 22c534c18ba1a0654d7d4c21c14e106b9a4dfbec..81bdc10b944df711a22673cec55105853a0320fc 100644
--- a/packages/core/src/callbacks/CallbackManager.ts
+++ b/packages/core/src/callbacks/CallbackManager.ts
@@ -1,6 +1,7 @@
import type { Anthropic } from "@anthropic-ai/sdk";
import { CustomEvent } from "@llamaindex/env";
import type { NodeWithScore } from "../Node.js";
+import type { AgentEndEvent, AgentStartEvent } from "../agent/type.js";
import {
EventCaller,
getEventCaller,
@@ -10,6 +11,7 @@ import type {
LLMStartEvent,
LLMStreamEvent,
LLMToolCallEvent,
+ LLMToolResultEvent,
} from "../llm/types.js";
export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
@@ -47,10 +49,15 @@ export interface LlamaIndexEventMaps {
* @deprecated
*/
stream: CustomEvent<StreamCallbackResponse>;
+ // llm events
"llm-start": LLMStartEvent;
"llm-end": LLMEndEvent;
"llm-tool-call": LLMToolCallEvent;
+ "llm-tool-result": LLMToolResultEvent;
"llm-stream": LLMStreamEvent;
+ // agent events
+ "agent-start": AgentStartEvent;
+ "agent-end": AgentEndEvent;
}
//#region @deprecated remove in the next major version
@@ -205,9 +212,10 @@ export class CallbackManager implements CallbackManagerMethods {
if (!handlers) {
return;
}
+ const clone = structuredClone(detail);
queueMicrotask(() => {
handlers.forEach((handler) =>
- handler(LlamaIndexCustomEvent.fromEvent(event, detail)),
+ handler(LlamaIndexCustomEvent.fromEvent(event, clone)),
);
});
}
diff --git a/packages/core/src/engines/chat/types.ts b/packages/core/src/engines/chat/types.ts
index 768119afe51330624227ead17ea7408effa0aecc..a3d5541569907ac911743eb5c7efe98fcfdcc63a 100644
--- a/packages/core/src/engines/chat/types.ts
+++ b/packages/core/src/engines/chat/types.ts
@@ -3,7 +3,6 @@ import type { NodeWithScore } from "../../Node.js";
import type { Response } from "../../Response.js";
import type { ChatMessage } from "../../llm/index.js";
import type { MessageContent } from "../../llm/types.js";
-import type { ToolOutput } from "../../tools/types.js";
/**
* Represents the base parameters for ChatEngine.
@@ -57,49 +56,3 @@ export interface Context {
export interface ContextGenerator {
generate(message: string): Promise<Context>;
}
-
-export enum ChatResponseMode {
- WAIT = "wait",
- STREAM = "stream",
-}
-
-export class AgentChatResponse {
- response: string;
- sources: ToolOutput[];
- sourceNodes?: NodeWithScore[];
-
- constructor(
- response: string,
- sources?: ToolOutput[],
- sourceNodes?: NodeWithScore[],
- ) {
- this.response = response;
- this.sources = sources || [];
- this.sourceNodes = sourceNodes || [];
- }
-
- protected _getFormattedSources() {
- throw new Error("Not implemented yet");
- }
-
- toString() {
- return this.response ?? "";
- }
-}
-
-export class StreamingAgentChatResponse {
- response: AsyncIterable<Response>;
-
- sources: ToolOutput[];
- sourceNodes?: NodeWithScore[];
-
- constructor(
- response: AsyncIterable<Response>,
- sources?: ToolOutput[],
- sourceNodes?: NodeWithScore[],
- ) {
- this.response = response;
- this.sources = sources ?? [];
- this.sourceNodes = sourceNodes ?? [];
- }
-}
diff --git a/packages/core/src/internal/context/EventCaller.ts b/packages/core/src/internal/context/EventCaller.ts
index 26c6a631f1d138bf15563df5cb0133364b961054..ab7a2d9a1712bfd6d1904c9b0112b9d6b2704643 100644
--- a/packages/core/src/internal/context/EventCaller.ts
+++ b/packages/core/src/internal/context/EventCaller.ts
@@ -1,5 +1,5 @@
import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
-import { isAsyncGenerator, isGenerator } from "../utils.js";
+import { isAsyncIterable, isIterable } from "../utils.js";
const eventReasonAsyncLocalStorage = new AsyncLocalStorage<EventCaller>();
@@ -71,22 +71,24 @@ export function wrapEventCaller<This, Result, Args extends unknown[]>(
return function (this: This, ...args: Args): Result {
const result = originalMethod.call(this, ...args);
// patch for iterators because AsyncLocalStorage doesn't work with them
- if (isAsyncGenerator(result)) {
+ if (isAsyncIterable(result)) {
+ const iter = result[Symbol.asyncIterator]();
const snapshot = AsyncLocalStorage.snapshot();
return (async function* asyncGeneratorWrapper() {
while (true) {
- const { value, done } = await snapshot(() => result.next());
+ const { value, done } = await snapshot(() => iter.next());
if (done) {
break;
}
yield value;
}
})() as Result;
- } else if (isGenerator(result)) {
+ } else if (isIterable(result)) {
+ const iter = result[Symbol.iterator]();
const snapshot = AsyncLocalStorage.snapshot();
return (function* generatorWrapper() {
while (true) {
- const { value, done } = snapshot(() => result.next());
+ const { value, done } = snapshot(() => iter.next());
if (done) {
break;
}
diff --git a/packages/core/src/agent/react/prompts.ts b/packages/core/src/internal/prompt/react.ts
similarity index 68%
rename from packages/core/src/agent/react/prompts.ts
rename to packages/core/src/internal/prompt/react.ts
index 75e98a468abd69254bf308639ffec99d791b7629..3f4450b91b3aa781a12c598f0618a3c7e3d303b4 100644
--- a/packages/core/src/agent/react/prompts.ts
+++ b/packages/core/src/internal/prompt/react.ts
@@ -1,13 +1,14 @@
-type ReactChatSystemHeaderParams = {
- toolDesc: string;
- toolNames: string;
-};
-
-export const getReactChatSystemHeader = ({
- toolDesc,
- toolNames,
-}: ReactChatSystemHeaderParams) =>
- `You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.
+import type { BaseTool } from "../../types.js";
+
+export const getReACTAgentSystemHeader = (tools: BaseTool[]) => {
+ const description = tools
+ .map(
+ (tool) =>
+ `- ${tool.metadata.name}: ${tool.metadata.description} with schema: ${JSON.stringify(tool.metadata.parameters)}`,
+ )
+ .join("\n");
+ const names = tools.map((tool) => tool.metadata.name).join(", ");
+ return `You are designed to help with a variety of tasks, from answering questions to providing summaries to other types of analyses.
## Tools
You have access to a wide variety of tools. You are responsible for using
@@ -16,14 +17,14 @@ This may require breaking the task into subtasks and using different tools
to complete each subtask.
You have access to the following tools:
-${toolDesc}
+${description}
## Output Format
To answer the question, please use the following format.
"""
Thought: I need to use a tool to help me answer the question.
-Action: tool name (one of ${toolNames}) if using a tool.
+Action: tool name (one of ${names}) if using a tool.
Action Input: the input to the tool, in a JSON format representing the kwargs (e.g. {{"input": "hello world", "num_beams": 5}})
"""
@@ -52,5 +53,5 @@ Answer: Sorry, I cannot answer your query.
""""
## Current Conversation
-Below is the current conversation consisting of interleaving human and assistant messages.
-`;
+Below is the current conversation consisting of interleaving human and assistant messages.`;
+};
diff --git a/packages/core/src/internal/type.ts b/packages/core/src/internal/type.ts
new file mode 100644
index 0000000000000000000000000000000000000000..b93af22a0132c2fa848231f851c41a27e348d709
--- /dev/null
+++ b/packages/core/src/internal/type.ts
@@ -0,0 +1,5 @@
+import { CustomEvent } from "@llamaindex/env";
+
+export type BaseEvent<Payload extends Record<string, unknown>> = CustomEvent<{
+ payload: Payload;
+}>;
diff --git a/packages/core/src/internal/utils.ts b/packages/core/src/internal/utils.ts
index 63133c61a8c54eb49604b06f65c64cfed6e3c0ad..d142bf1db887c6745c73490fa25550e3ea392f6f 100644
--- a/packages/core/src/internal/utils.ts
+++ b/packages/core/src/internal/utils.ts
@@ -1,8 +1,10 @@
-export const isAsyncGenerator = (obj: unknown): obj is AsyncGenerator => {
+export const isAsyncIterable = (
+ obj: unknown,
+): obj is AsyncIterable<unknown> => {
return obj != null && typeof obj === "object" && Symbol.asyncIterator in obj;
};
-export const isGenerator = (obj: unknown): obj is Generator => {
+export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
return obj != null && typeof obj === "object" && Symbol.iterator in obj;
};
diff --git a/packages/core/src/llm/types.ts b/packages/core/src/llm/types.ts
index 26da7b4ee6e4f37694d7ef98c1da8e3fa37ca774..21126953635d96fed9fe75cf06fbee4aac5f3703 100644
--- a/packages/core/src/llm/types.ts
+++ b/packages/core/src/llm/types.ts
@@ -1,24 +1,23 @@
import type { Tokenizers } from "../GlobalsHelper.js";
-import type { BaseTool, UUID } from "../types.js";
+import type { BaseEvent } from "../internal/type.js";
+import type { BaseTool, ToolOutput, UUID } from "../types.js";
-type LLMBaseEvent<Payload extends Record<string, unknown>> = CustomEvent<{
- payload: Payload;
-}>;
-
-export type LLMStartEvent = LLMBaseEvent<{
+export type LLMStartEvent = BaseEvent<{
id: UUID;
messages: ChatMessage[];
}>;
-export type LLMToolCallEvent = LLMBaseEvent<{
- // fixme: id is missing in the context
- // id: UUID;
- toolCall: Omit<ToolCallOptions["toolCall"], "id">;
+export type LLMToolCallEvent = BaseEvent<{
+ toolCall: ToolCall;
+}>;
+export type LLMToolResultEvent = BaseEvent<{
+ toolCall: ToolCall;
+ toolResult: ToolOutput;
}>;
-export type LLMEndEvent = LLMBaseEvent<{
+export type LLMEndEvent = BaseEvent<{
id: UUID;
response: ChatResponse;
}>;
-export type LLMStreamEvent = LLMBaseEvent<{
+export type LLMStreamEvent = BaseEvent<{
id: UUID;
chunk: ChatResponseChunk;
}>;
@@ -77,6 +76,13 @@ export interface LLM<
export type MessageType = "user" | "assistant" | "system" | "memory";
+export type TextChatMessage<AdditionalMessageOptions extends object = object> =
+ {
+ content: string;
+ role: MessageType;
+ options?: undefined | AdditionalMessageOptions;
+ };
+
export type ChatMessage<AdditionalMessageOptions extends object = object> = {
content: MessageContent;
role: MessageType;
diff --git a/packages/core/src/objects/base.ts b/packages/core/src/objects/base.ts
index 3ef6832d4824afad698efa7981053c692c554de1..3eade2994c735a1188172a2b5c31e542deae2aaa 100644
--- a/packages/core/src/objects/base.ts
+++ b/packages/core/src/objects/base.ts
@@ -2,6 +2,8 @@ import type { BaseNode, Metadata } from "../Node.js";
import { TextNode } from "../Node.js";
import type { BaseRetriever } from "../Retriever.js";
import type { VectorStoreIndex } from "../indices/index.js";
+import type { MessageContent } from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
import type { BaseTool } from "../types.js";
// Assuming that necessary interfaces and classes (like OT, TextNode, BaseNode, etc.) are defined elsewhere
@@ -48,7 +50,8 @@ export abstract class BaseObjectNodeMapping {
// You will need to implement specific subclasses of BaseObjectNodeMapping as per your project requirements.
-type QueryType = string;
+// todo: multimodal support
+type QueryType = MessageContent;
export class ObjectRetriever<T = unknown> {
_retriever: BaseRetriever;
@@ -69,7 +72,9 @@ export class ObjectRetriever<T = unknown> {
// Translating the retrieve method
async retrieve(strOrQueryBundle: QueryType): Promise<T[]> {
- const nodes = await this.retriever.retrieve({ query: strOrQueryBundle });
+ const nodes = await this.retriever.retrieve({
+ query: extractText(strOrQueryBundle),
+ });
const objs = nodes.map((n) => this._objectNodeMapping.fromNode(n.node));
return objs;
}
diff --git a/packages/core/src/tools/index.ts b/packages/core/src/tools/index.ts
index 2b52c32edf8fbc07dd31882a66ace92095400d61..d8629c0e3538680dcadc5c6b49ca954153acec7e 100644
--- a/packages/core/src/tools/index.ts
+++ b/packages/core/src/tools/index.ts
@@ -1,5 +1,3 @@
export * from "./QueryEngineTool.js";
export * from "./WikipediaTool.js";
export * from "./functionTool.js";
-export * from "./types.js";
-export * from "./utils.js";
diff --git a/packages/core/src/tools/types.ts b/packages/core/src/tools/types.ts
deleted file mode 100644
index f98f14f7275c3b2f69ef0aafc438aa3eb0167e6d..0000000000000000000000000000000000000000
--- a/packages/core/src/tools/types.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-export class ToolOutput {
- content: string;
- toolName: string;
- rawInput: any;
- rawOutput: any;
-
- constructor(
- content: string,
- toolName: string,
- rawInput: any,
- rawOutput: any,
- ) {
- this.content = content;
- this.toolName = toolName;
- this.rawInput = rawInput;
- this.rawOutput = rawOutput;
- }
-
- toString(): string {
- return this.content;
- }
-}
diff --git a/packages/core/src/tools/utils.ts b/packages/core/src/tools/utils.ts
deleted file mode 100644
index afc5280aa7224e634a43b6cb724d477fa9699d8a..0000000000000000000000000000000000000000
--- a/packages/core/src/tools/utils.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import type { BaseTool } from "../types.js";
-import { ToolOutput } from "./types.js";
-
-export async function callToolWithErrorHandling(
- tool: BaseTool,
- input: unknown,
-): Promise<ToolOutput> {
- if (!tool.call) {
- return new ToolOutput(
- "Error: Tool does not have a call function.",
- tool.metadata.name,
- input,
- null,
- );
- }
- try {
- getCallbackManager().dispatchEvent("llm-tool-call", {
- payload: {
- toolCall: {
- name: tool.metadata.name,
- input,
- },
- },
- });
- const value = await tool.call(
- typeof input === "string" ? JSON.parse(input) : input,
- );
- return new ToolOutput(value, tool.metadata.name, input, value);
- } catch (e) {
- return new ToolOutput(`Error: ${e}`, tool.metadata.name, input, e);
- }
-}
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index a51b1182ec543d25c90eeb416a9b6634aa6c942f..17433e08a2ea02359817045d58ee59e4b4672d37 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -103,3 +103,10 @@ export class QueryBundle {
}
export type UUID = `${string}-${string}-${string}-${string}-${string}`;
+
+export type ToolOutput = {
+ tool: BaseTool | undefined;
+ input: unknown;
+ output: string;
+ isError: boolean;
+};
diff --git a/packages/core/tests/agent/runner/AgentRunner.test.ts b/packages/core/tests/agent/runner/AgentRunner.test.ts
deleted file mode 100644
index 4b8b4af76785aec589a5c68e9dcc2fedc9fe6b4e..0000000000000000000000000000000000000000
--- a/packages/core/tests/agent/runner/AgentRunner.test.ts
+++ /dev/null
@@ -1,85 +0,0 @@
-import { OpenAIAgentWorker } from "llamaindex/agent/index";
-import { AgentRunner } from "llamaindex/agent/runner/base";
-import { OpenAI } from "llamaindex/llm/open_ai";
-import { beforeEach, describe, expect, it } from "vitest";
-
-import { Settings } from "llamaindex";
-import {
- DEFAULT_LLM_TEXT_OUTPUT,
- mockLlmGeneration,
-} from "../../utility/mockOpenAI.js";
-
-describe("Agent Runner", () => {
- let agentRunner: AgentRunner;
-
- beforeEach(() => {
- Settings.llm = new OpenAI({
- model: "gpt-3.5-turbo",
- });
-
- mockLlmGeneration();
-
- agentRunner = new AgentRunner({
- agentWorker: new OpenAIAgentWorker({
- tools: [],
- }),
- });
- });
-
- it("should be able to initialize a task", () => {
- const task = agentRunner.createTask("hello world");
-
- expect(task.input).toEqual("hello world");
- expect(task.taskId in agentRunner.state.taskDict).toEqual(true);
-
- expect(agentRunner.listTasks().length).toEqual(1);
- });
-
- it("should be able to run a step", async () => {
- const task = agentRunner.createTask("hello world");
-
- expect(agentRunner.getCompletedSteps(task.taskId)).toBeUndefined();
-
- const stepOutput = await agentRunner.runStep(task.taskId, task.input);
-
- const completedSteps = agentRunner.getCompletedSteps(task.taskId);
-
- expect(completedSteps.length).toEqual(1);
-
- expect(stepOutput.isLast).toEqual(true);
- });
-
- it("should be able to finalize a task", async () => {
- const task = agentRunner.createTask("hello world");
-
- expect(agentRunner.getCompletedSteps(task.taskId)).toBeUndefined();
-
- const stepOutput1 = await agentRunner.runStep(task.taskId, task.input);
-
- expect(stepOutput1.isLast).toEqual(true);
- });
-
- it("should be able to delete a task", () => {
- const task = agentRunner.createTask("hello world");
-
- expect(agentRunner.listTasks().length).toEqual(1);
-
- agentRunner.deleteTask(task.taskId);
-
- expect(agentRunner.listTasks().length).toEqual(0);
- });
-
- it("should be able to run a chat", async () => {
- const response = await agentRunner.chat({
- message: "hello world",
- });
-
- expect(agentRunner.listTasks().length).toEqual(1);
-
- expect(response).toEqual({
- response: DEFAULT_LLM_TEXT_OUTPUT,
- sourceNodes: [],
- sources: [],
- });
- });
-});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index cb4615cce9f7fe87ef028542522e917d2b389b4f..df4ddf45402aaee833ed0bd12dea6ef0045fdea6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -147,8 +147,8 @@ importers:
specifier: ^1.0.11
version: 1.0.11
llamaindex:
- specifier: latest
- version: link:../packages/core
+ specifier: '*'
+ version: 0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.5)
mongodb:
specifier: ^6.5.0
version: 6.5.0
@@ -172,8 +172,8 @@ importers:
examples/readers:
dependencies:
llamaindex:
- specifier: latest
- version: 0.2.9(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3)
+ specifier: '*'
+ version: 0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3)
devDependencies:
'@types/node':
specifier: ^20.12.7
@@ -1410,11 +1410,6 @@ packages:
'@dabh/diagnostics@2.0.3':
resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==}
- '@datastax/astra-db-ts@0.1.4':
- resolution: {integrity: sha512-EG/7UUuEdxpeyGV1fkGIUX5jjUcESToCtohoti0rNMEm01T1E4NXOPHXMnkyXo71zqrlUoTlGn5du+acnlbslQ==}
- engines: {node: '>=14.0.0'}
- hasBin: true
-
'@datastax/astra-db-ts@1.0.1':
resolution: {integrity: sha512-TSfP/Lv0ghPc01lVqiXGdunM+2CsBt8tG/9cqTs8K3LaM5vRJJj760Q2+udif834rrbXM4xN8LZQeWPfVPy6BA==}
engines: {node: '>=14.0.0'}
@@ -5586,8 +5581,8 @@ packages:
resolution: {integrity: sha512-ovJXBXkKGfq+CwmKTjluEqFi3p4h8xvkxGQQAQan22YCgef4KZ1mKGjzfGh6PL6AW5Csw0QiQPNuQyH+6Xk3hA==}
engines: {node: '>=18.0.0'}
- llamaindex@0.2.9:
- resolution: {integrity: sha512-Ifmk4Bu0CEKI3cwfER3WyZSP7+gOBHgtWobsEJ1c+1pYcIMipwxpqBUba/YXQceuEmh7xsCd0+ROkoNumUX3CQ==}
+ llamaindex@0.2.10:
+ resolution: {integrity: sha512-GXO/H4k6iF0dQStg1kOTYYm0pnMbD1gM8LwRKEPOeC/mY+Q2pyIyDB22cPc8nOTf+ah3rbLiXOxTORTUmC1xKA==}
engines: {node: '>=18.0.0'}
load-yaml-file@0.2.0:
@@ -7326,10 +7321,6 @@ packages:
renderkid@3.0.0:
resolution: {integrity: sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==}
- replicate@0.25.2:
- resolution: {integrity: sha512-c5otBJ5E66XLS0X196pBCsyy85b03ZBLeV/lbKfU8cqfkt3Qd6NGEiPwTtxtsQ4AznggMJNn2Qq68t/bV85M2w==}
- engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'}
-
require-directory@2.1.1:
resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
engines: {node: '>=0.10.0'}
@@ -9897,14 +9888,6 @@ snapshots:
enabled: 2.0.0
kuler: 2.0.0
- '@datastax/astra-db-ts@0.1.4':
- dependencies:
- axios: 1.6.8
- bson: 6.6.0
- winston: 3.13.0
- transitivePeerDependencies:
- - debug
-
'@datastax/astra-db-ts@1.0.1':
dependencies:
bson-objectid: 2.0.4
@@ -15114,11 +15097,11 @@ snapshots:
rfdc: 1.3.1
wrap-ansi: 9.0.0
- llamaindex@0.2.9(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3):
+ llamaindex@0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.3):
dependencies:
'@anthropic-ai/sdk': 0.20.6(encoding@0.1.13)
'@aws-crypto/sha256-js': 5.2.0
- '@datastax/astra-db-ts': 0.1.4
+ '@datastax/astra-db-ts': 1.0.1
'@grpc/grpc-js': 1.10.6
'@llamaindex/cloud': 0.0.5(node-fetch@2.7.0(encoding@0.1.13))
'@llamaindex/env': 0.0.7(@aws-crypto/sha256-js@5.2.0)(pathe@1.1.2)(readable-stream@4.5.2)
@@ -15151,7 +15134,64 @@ snapshots:
pgvector: 0.1.8
portkey-ai: 0.1.16
rake-modified: 1.0.8
- replicate: 0.25.2
+ string-strip-html: 13.4.8
+ wikipedia: 2.1.2
+ wink-nlp: 1.14.3
+ transitivePeerDependencies:
+ - '@aws-sdk/credential-providers'
+ - '@google/generative-ai'
+ - '@mongodb-js/zstd'
+ - bufferutil
+ - debug
+ - encoding
+ - gcp-metadata
+ - kerberos
+ - mongodb-client-encryption
+ - node-fetch
+ - pg-native
+ - readable-stream
+ - snappy
+ - socks
+ - typescript
+ - utf-8-validate
+
+ llamaindex@0.2.10(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(readable-stream@4.5.2)(typescript@5.4.5):
+ dependencies:
+ '@anthropic-ai/sdk': 0.20.6(encoding@0.1.13)
+ '@aws-crypto/sha256-js': 5.2.0
+ '@datastax/astra-db-ts': 1.0.1
+ '@grpc/grpc-js': 1.10.6
+ '@llamaindex/cloud': 0.0.5(node-fetch@2.7.0(encoding@0.1.13))
+ '@llamaindex/env': 0.0.7(@aws-crypto/sha256-js@5.2.0)(pathe@1.1.2)(readable-stream@4.5.2)
+ '@mistralai/mistralai': 0.1.3(encoding@0.1.13)
+ '@notionhq/client': 2.2.15(encoding@0.1.13)
+ '@pinecone-database/pinecone': 2.2.0
+ '@qdrant/js-client-rest': 1.8.2(typescript@5.4.5)
+ '@types/lodash': 4.17.0
+ '@types/node': 20.12.7
+ '@types/papaparse': 5.3.14
+ '@types/pg': 8.11.5
+ '@xenova/transformers': 2.17.1
+ '@zilliz/milvus2-sdk-node': 2.4.1
+ ajv: 8.12.0
+ assemblyai: 4.4.1
+ chromadb: 1.7.3(cohere-ai@7.9.5(encoding@0.1.13))(encoding@0.1.13)(openai@4.38.0(encoding@0.1.13))
+ cohere-ai: 7.9.5(encoding@0.1.13)
+ js-tiktoken: 1.0.11
+ lodash: 4.17.21
+ magic-bytes.js: 1.10.0
+ mammoth: 1.7.1
+ md-utils-ts: 2.0.0
+ mongodb: 6.5.0
+ notion-md-crawler: 0.0.2(encoding@0.1.13)
+ openai: 4.38.0(encoding@0.1.13)
+ papaparse: 5.4.1
+ pathe: 1.1.2
+ pdf2json: 3.0.5
+ pg: 8.11.5
+ pgvector: 0.1.8
+ portkey-ai: 0.1.16
+ rake-modified: 1.0.8
string-strip-html: 13.4.8
wikipedia: 2.1.2
wink-nlp: 1.14.3
@@ -17329,10 +17369,6 @@ snapshots:
lodash: 4.17.21
strip-ansi: 6.0.1
- replicate@0.25.2:
- optionalDependencies:
- readable-stream: 4.5.2
-
require-directory@2.1.1: {}
require-from-string@2.0.2: {}
diff --git a/tsconfig.json b/tsconfig.json
index 43f6f276e1019b05271a91b08413e0ee225bd02e..f2cac2115f7017df00819509497158c7e31dd35a 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,6 +1,6 @@
{
"compilerOptions": {
- "target": "es2016",
+ "target": "es2022",
"module": "esnext",
"moduleResolution": "bundler",
"verbatimModuleSyntax": true,