From 208282d62fdcdb4e91c0b0800727f00a8456bba7 Mon Sep 17 00:00:00 2001
From: Alex Yang <himself65@outlook.com>
Date: Mon, 15 Apr 2024 16:22:47 -0500
Subject: [PATCH] feat: init anthropic agent (part 2) (#719)
---
.changeset/perfect-sloths-vanish.md | 10 +
examples/anthropic/agent.ts | 43 +
examples/toolsStream.ts | 5 +-
packages/core/e2e/fixtures/llm/open_ai.ts | 5 +-
packages/core/e2e/node/claude.e2e.ts | 86 +-
packages/core/e2e/node/fixtures/tools.ts | 47 +
packages/core/e2e/node/openai.e2e.ts | 85 +-
packages/core/e2e/node/snapshot/agent.snap | 122 +-
.../core/e2e/node/snapshot/agent_stream.snap | 1123 +++++++++--------
.../e2e/node/snapshot/anthropic-agent.snap | 403 ++++++
.../core/e2e/node/snapshot/gpt-4-turbo.snap | 38 +-
.../core/e2e/node/snapshot/llm-anthropic.snap | 60 +-
packages/core/src/ChatHistory.ts | 19 +-
packages/core/src/agent/anthropic.ts | 170 +++
packages/core/src/agent/index.ts | 2 +
packages/core/src/agent/openai/base.ts | 5 +-
packages/core/src/agent/openai/worker.ts | 113 +-
packages/core/src/agent/react/base.ts | 3 +-
packages/core/src/agent/react/worker.ts | 11 +-
packages/core/src/agent/runner/base.ts | 5 +-
.../core/src/callbacks/CallbackManager.ts | 10 +-
packages/core/src/internal/utils.ts | 11 +
packages/core/src/llm/anthropic.ts | 131 +-
packages/core/src/llm/base.ts | 27 +-
packages/core/src/llm/open_ai.ts | 176 ++-
packages/core/src/llm/types.ts | 203 ++-
packages/core/src/llm/utils.ts | 20 +-
packages/core/src/memory/ChatMemoryBuffer.ts | 33 +-
packages/core/src/memory/types.ts | 7 +-
packages/core/src/objects/base.ts | 6 +-
.../src/storage/chatStore/SimpleChatStore.ts | 5 +-
packages/core/src/storage/chatStore/types.ts | 5 +-
packages/core/src/tools/utils.ts | 26 +-
packages/core/src/types.ts | 2 +-
packages/core/tests/type.test.ts | 56 +
35 files changed, 1956 insertions(+), 1117 deletions(-)
create mode 100644 .changeset/perfect-sloths-vanish.md
create mode 100644 examples/anthropic/agent.ts
create mode 100644 packages/core/e2e/node/fixtures/tools.ts
create mode 100644 packages/core/e2e/node/snapshot/anthropic-agent.snap
create mode 100644 packages/core/src/agent/anthropic.ts
create mode 100644 packages/core/tests/type.test.ts
diff --git a/.changeset/perfect-sloths-vanish.md b/.changeset/perfect-sloths-vanish.md
new file mode 100644
index 000000000..77b6d3510
--- /dev/null
+++ b/.changeset/perfect-sloths-vanish.md
@@ -0,0 +1,10 @@
+---
+"llamaindex": patch
+"@llamaindex/edge": patch
+---
+
+feat: init anthropic agent
+
+remove the `tool` | `function` type in `MessageType`. Replace with `assistant` instead.
+This is because these two types are only available for `OpenAI`.
+Since `OpenAI` deprecates the function type, we support the Claude 3 tool call.
diff --git a/examples/anthropic/agent.ts b/examples/anthropic/agent.ts
new file mode 100644
index 000000000..94c2d23f8
--- /dev/null
+++ b/examples/anthropic/agent.ts
@@ -0,0 +1,43 @@
+import { FunctionTool, Settings, WikipediaTool } from "llamaindex";
+import { AnthropicAgent } from "llamaindex/agent/anthropic";
+
+Settings.callbackManager.on("llm-tool-call", (event) => {
+ console.log("llm-tool-call", event.detail.payload.toolCall);
+});
+
+const agent = new AnthropicAgent({
+ tools: [
+ FunctionTool.from<{ location: string }>(
+ (query) => {
+ return `The weather in ${query.location} is sunny`;
+ },
+ {
+ name: "weather",
+ description: "Get the weather",
+ parameters: {
+ type: "object",
+ properties: {
+ location: {
+ type: "string",
+ description: "The location to get the weather for",
+ },
+ },
+ required: ["location"],
+ },
+ },
+ ),
+ new WikipediaTool(),
+ ],
+});
+
+async function main() {
+ // https://docs.anthropic.com/claude/docs/tool-use#tool-use-best-practices-and-limitations
+ const { response } = await agent.chat({
+ message:
+ "What is the weather in New York? What's the history of New York from Wikipedia in 3 sentences?",
+ });
+
+ console.log(response);
+}
+
+void main();
diff --git a/examples/toolsStream.ts b/examples/toolsStream.ts
index 3b1e86619..7108ddd06 100644
--- a/examples/toolsStream.ts
+++ b/examples/toolsStream.ts
@@ -35,7 +35,10 @@ async function main() {
const stream = await llm.chat({ ...args, stream: true });
for await (const chunk of stream) {
process.stdout.write(chunk.delta);
- console.log(chunk.options?.toolCalls?.[0]);
+ if (chunk.options && "toolCall" in chunk.options) {
+ console.log("Tool call:");
+ console.log(chunk.options.toolCall);
+ }
}
}
diff --git a/packages/core/e2e/fixtures/llm/open_ai.ts b/packages/core/e2e/fixtures/llm/open_ai.ts
index bef6a4b40..2c1741b8c 100644
--- a/packages/core/e2e/fixtures/llm/open_ai.ts
+++ b/packages/core/e2e/fixtures/llm/open_ai.ts
@@ -9,7 +9,7 @@ import type {
LLMCompletionParamsStreaming,
} from "llamaindex/llm/types";
import { extractText } from "llamaindex/llm/utils";
-import { strictEqual } from "node:assert";
+import { deepStrictEqual, strictEqual } from "node:assert";
import { llmCompleteMockStorage } from "../../node/utils.js";
export function getOpenAISession() {
@@ -21,6 +21,7 @@ export function isFunctionCallingModel() {
}
export class OpenAI implements LLM {
+ supportToolCall = true;
get metadata() {
return {
model: "mock-model",
@@ -48,7 +49,7 @@ export class OpenAI implements LLM {
strictEqual(chatMessage.length, params.messages.length);
for (let i = 0; i < chatMessage.length; i++) {
strictEqual(chatMessage[i].role, params.messages[i].role);
- strictEqual(chatMessage[i].content, params.messages[i].content);
+ deepStrictEqual(chatMessage[i].content, params.messages[i].content);
}
if (llmCompleteMockStorage.llmEventEnd.length > 0) {
diff --git a/packages/core/e2e/node/claude.e2e.ts b/packages/core/e2e/node/claude.e2e.ts
index 23325866b..c4e82a954 100644
--- a/packages/core/e2e/node/claude.e2e.ts
+++ b/packages/core/e2e/node/claude.e2e.ts
@@ -1,7 +1,9 @@
import { consola } from "consola";
-import { Anthropic, Settings, type LLM } from "llamaindex";
+import { Anthropic, FunctionTool, Settings, type LLM } from "llamaindex";
+import { AnthropicAgent } from "llamaindex/agent/anthropic";
import { ok } from "node:assert";
import { beforeEach, test } from "node:test";
+import { sumNumbersTool } from "./fixtures/tools.js";
import { mockLLMEvent } from "./utils.js";
let llm: LLM;
@@ -20,6 +22,7 @@ await test("anthropic llm", async (t) => {
{
content: "Hello",
role: "user",
+ options: {},
},
],
});
@@ -43,3 +46,84 @@ await test("anthropic llm", async (t) => {
}
});
});
+
+await test("anthropic agent", async (t) => {
+ await mockLLMEvent(t, "anthropic-agent");
+ await t.test("chat", async () => {
+ const agent = new AnthropicAgent({
+ tools: [
+ {
+ call: async () => {
+ return "35 degrees and sunny in San Francisco";
+ },
+ metadata: {
+ name: "Weather",
+ description: "Get the weather",
+ parameters: {
+ type: "object",
+ properties: {
+ location: { type: "string" },
+ },
+ required: ["location"],
+ },
+ },
+ },
+ ],
+ });
+ const result = await agent.chat({
+ message: "What is the weather in San Francisco?",
+ });
+ consola.debug("response:", result.response);
+ ok(typeof result.response === "string");
+ ok(result.response.includes("35"));
+ });
+
+ await t.test("async function", async () => {
+ const uniqueId = "123456789";
+ const showUniqueId = FunctionTool.from<{
+ firstName: string;
+ lastName: string;
+ }>(
+ async ({ firstName, lastName }) => {
+ ok(typeof firstName === "string");
+ ok(typeof lastName === "string");
+ const fullName = firstName + lastName;
+ ok(fullName.toLowerCase().includes("alex"));
+ ok(fullName.toLowerCase().includes("yang"));
+ return uniqueId;
+ },
+ {
+ name: "unique_id",
+ description: "show user unique id",
+ parameters: {
+ type: "object",
+ properties: {
+ firstName: { type: "string" },
+ lastName: { type: "string" },
+ },
+ required: ["firstName", "lastName"],
+ },
+ },
+ );
+ const agent = new AnthropicAgent({
+ tools: [showUniqueId],
+ });
+ const { response } = await agent.chat({
+ message: "My name is Alex Yang. What is my unique id?",
+ });
+ consola.debug("response:", response);
+ ok(response.includes(uniqueId));
+ });
+
+ await t.test("sum numbers", async () => {
+ const openaiAgent = new AnthropicAgent({
+ tools: [sumNumbersTool],
+ });
+
+ const response = await openaiAgent.chat({
+ message: "how much is 1 + 1?",
+ });
+
+ ok(response.response.includes("2"));
+ });
+});
diff --git a/packages/core/e2e/node/fixtures/tools.ts b/packages/core/e2e/node/fixtures/tools.ts
new file mode 100644
index 000000000..6d523991a
--- /dev/null
+++ b/packages/core/e2e/node/fixtures/tools.ts
@@ -0,0 +1,47 @@
+import { FunctionTool } from "llamaindex";
+
+function sumNumbers({ a, b }: { a: number; b: number }) {
+ return `${a + b}`;
+}
+
+function divideNumbers({ a, b }: { a: number; b: number }) {
+ return `${a / b}`;
+}
+
+export const sumNumbersTool = FunctionTool.from(sumNumbers, {
+ name: "sumNumbers",
+ description: "Use this function to sum two numbers",
+ parameters: {
+ type: "object",
+ properties: {
+ a: {
+ type: "number",
+ description: "The first number",
+ },
+ b: {
+ type: "number",
+ description: "The second number",
+ },
+ },
+ required: ["a", "b"],
+ },
+});
+
+export const divideNumbersTool = FunctionTool.from(divideNumbers, {
+ name: "divideNumbers",
+ description: "Use this function to divide two numbers",
+ parameters: {
+ type: "object",
+ properties: {
+ a: {
+ type: "number",
+ description: "The first number",
+ },
+ b: {
+ type: "number",
+ description: "The second number",
+ },
+ },
+ required: ["a", "b"],
+ },
+});
diff --git a/packages/core/e2e/node/openai.e2e.ts b/packages/core/e2e/node/openai.e2e.ts
index 710ea5bc9..06c78a309 100644
--- a/packages/core/e2e/node/openai.e2e.ts
+++ b/packages/core/e2e/node/openai.e2e.ts
@@ -10,8 +10,9 @@ import {
VectorStoreIndex,
type LLM,
} from "llamaindex";
-import { ok } from "node:assert";
+import { ok, strictEqual } from "node:assert";
import { beforeEach, test } from "node:test";
+import { divideNumbersTool, sumNumbersTool } from "./fixtures/tools.js";
import { mockLLMEvent } from "./utils.js";
let llm: LLM;
@@ -22,14 +23,6 @@ beforeEach(async () => {
llm = Settings.llm;
});
-function sumNumbers({ a, b }: { a: number; b: number }) {
- return `${a + b}`;
-}
-
-function divideNumbers({ a, b }: { a: number; b: number }) {
- return `${a / b}`;
-}
-
await test("openai llm", async (t) => {
await mockLLMEvent(t, "llm");
await t.test("llm.chat", async () => {
@@ -166,27 +159,8 @@ await test("agent", async (t) => {
});
await t.test("sum numbers", async () => {
- const sumFunctionTool = new FunctionTool(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
- },
- });
-
const openaiAgent = new OpenAIAgent({
- tools: [sumFunctionTool],
+ tools: [sumNumbersTool],
});
const response = await openaiAgent.chat({
@@ -199,51 +173,12 @@ await test("agent", async (t) => {
await test("agent stream", async (t) => {
await mockLLMEvent(t, "agent_stream");
- await t.test("sum numbers stream", async () => {
- const sumJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The first number",
- },
- b: {
- type: "number",
- description: "The second number",
- },
- },
- required: ["a", "b"],
- } as const;
-
- const divideJSON = {
- type: "object",
- properties: {
- a: {
- type: "number",
- description: "The dividend",
- },
- b: {
- type: "number",
- description: "The divisor",
- },
- },
- required: ["a", "b"],
- } as const;
-
- const functionTool = FunctionTool.from(sumNumbers, {
- name: "sumNumbers",
- description: "Use this function to sum two numbers",
- parameters: sumJSON,
- });
-
- const functionTool2 = FunctionTool.from(divideNumbers, {
- name: "divideNumbers",
- description: "Use this function to divide two numbers",
- parameters: divideJSON,
- });
+ await t.test("sum numbers stream", async (t) => {
+ const fn = t.mock.fn(() => {});
+ Settings.callbackManager.on("llm-tool-call", fn);
const agent = new OpenAIAgent({
- tools: [functionTool, functionTool2],
+ tools: [sumNumbersTool, divideNumbersTool],
});
const { response } = await agent.chat({
@@ -257,13 +192,17 @@ await test("agent stream", async (t) => {
message += chunk.response;
}
+ strictEqual(fn.mock.callCount(), 2);
ok(message.includes("28"));
+ Settings.callbackManager.off("llm-tool-call", fn);
});
});
await test("queryEngine", async (t) => {
await mockLLMEvent(t, "queryEngine_subquestion");
await t.test("subquestion", async () => {
+ const fn = t.mock.fn(() => {});
+ Settings.callbackManager.on("llm-tool-call", fn);
const document = new Document({
text: "Bill Gates stole from Apple.\n Steve Jobs stole from Xerox.",
});
@@ -288,5 +227,7 @@ await test("queryEngine", async (t) => {
});
ok(response.includes("Apple"));
+ strictEqual(fn.mock.callCount(), 0);
+ Settings.callbackManager.off("llm-tool-call", fn);
});
});
diff --git a/packages/core/e2e/node/snapshot/agent.snap b/packages/core/e2e/node/snapshot/agent.snap
index c283affff..11cb01b92 100644
--- a/packages/core/e2e/node/snapshot/agent.snap
+++ b/packages/core/e2e/node/snapshot/agent.snap
@@ -20,24 +20,21 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "Weather",
- "arguments": "{\"location\":\"San Francisco\"}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "Weather",
+ "input": "{\"location\":\"San Francisco\"}"
+ }
}
},
{
"content": "35 degrees and sunny in San Francisco",
- "role": "tool",
+ "role": "user",
"options": {
- "name": "Weather",
- "tool_call_id": "HIDDEN"
+ "toolResult": {
+ "id": "HIDDEN",
+ "isError": false
+ }
}
}
]
@@ -62,24 +59,21 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "unique_id",
- "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "unique_id",
+ "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+ }
}
},
{
"content": "123456789",
- "role": "tool",
+ "role": "user",
"options": {
- "name": "unique_id",
- "tool_call_id": "HIDDEN"
+ "toolResult": {
+ "id": "HIDDEN",
+ "isError": false
+ }
}
}
]
@@ -104,24 +98,21 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\":1,\"b\":1}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "sumNumbers",
+ "input": "{\"a\":1,\"b\":1}"
+ }
}
},
{
"content": "2",
- "role": "tool",
+ "role": "user",
"options": {
- "name": "sumNumbers",
- "tool_call_id": "HIDDEN"
+ "toolResult": {
+ "id": "HIDDEN",
+ "isError": false
+ }
}
}
]
@@ -168,16 +159,11 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "Weather",
- "arguments": "{\"location\":\"San Francisco\"}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "Weather",
+ "input": "{\"location\":\"San Francisco\"}"
+ }
}
}
}
@@ -255,16 +241,11 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "unique_id",
- "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "unique_id",
+ "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+ }
}
}
}
@@ -342,16 +323,11 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\":1,\"b\":1}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "sumNumbers",
+ "input": "{\"a\":1,\"b\":1}"
+ }
}
}
}
@@ -369,7 +345,7 @@
"index": 0,
"message": {
"role": "assistant",
- "content": "The sum of 1 + 1 is 2."
+ "content": "1 + 1 is equal to 2."
},
"logprobs": null,
"finish_reason": "stop"
@@ -377,13 +353,13 @@
],
"usage": {
"prompt_tokens": 97,
- "completion_tokens": 13,
- "total_tokens": 110
+ "completion_tokens": 11,
+ "total_tokens": 108
},
"system_fingerprint": "HIDDEN"
},
"message": {
- "content": "The sum of 1 + 1 is 2.",
+ "content": "1 + 1 is equal to 2.",
"role": "assistant",
"options": {}
}
diff --git a/packages/core/e2e/node/snapshot/agent_stream.snap b/packages/core/e2e/node/snapshot/agent_stream.snap
index ecd85f162..e9e9aa161 100644
--- a/packages/core/e2e/node/snapshot/agent_stream.snap
+++ b/packages/core/e2e/node/snapshot/agent_stream.snap
@@ -20,40 +20,72 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
}
},
{
"content": "8",
- "role": "tool",
+ "role": "user",
"options": {
- "name": "divideNumbers",
- "tool_call_id": "HIDDEN"
+ "toolResult": {
+ "id": "HIDDEN",
+ "isError": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_2",
+ "messages": [
+ {
+ "content": "Divide 16 by 2 then add 20",
+ "role": "user"
+ },
+ {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
+ }
+ },
+ {
+ "content": "8",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "id": "HIDDEN",
+ "isError": false
+ }
+ }
+ },
+ {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
}
},
{
"content": "28",
- "role": "tool",
+ "role": "user",
"options": {
- "name": "sumNumbers",
- "tool_call_id": "HIDDEN"
+ "toolResult": {
+ "id": "HIDDEN",
+ "isError": false
+ }
}
}
]
@@ -93,24 +125,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
},
@@ -140,24 +159,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
},
@@ -187,24 +193,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
},
@@ -234,24 +227,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
},
@@ -281,24 +261,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
},
@@ -331,24 +298,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
},
@@ -378,24 +332,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
},
@@ -425,24 +366,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
},
@@ -472,24 +400,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
},
@@ -519,24 +434,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
}
@@ -545,24 +447,11 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
}
}
}
@@ -582,15 +471,33 @@
{
"index": 0,
"delta": {
- "content": "The"
+ "role": "assistant",
+ "content": null,
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "HIDDEN",
+ "type": "function",
+ "function": {
+ "name": "sumNumbers",
+ "arguments": ""
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": "The"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -603,15 +510,28 @@
{
"index": 0,
"delta": {
- "content": " result"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "{\""
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " result"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -624,15 +544,28 @@
{
"index": 0,
"delta": {
- "content": " of"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "a"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " of"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -645,15 +578,28 @@
{
"index": 0,
"delta": {
- "content": " dividing"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "\":"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " dividing"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -666,15 +612,28 @@
{
"index": 0,
"delta": {
- "content": " "
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "8"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " "
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -687,15 +646,28 @@
{
"index": 0,
"delta": {
- "content": "16"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": ",\""
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": "16"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -708,15 +680,28 @@
{
"index": 0,
"delta": {
- "content": " by"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "b"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " by"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -729,15 +714,28 @@
{
"index": 0,
"delta": {
- "content": " "
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "\":"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " "
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -750,15 +748,28 @@
{
"index": 0,
"delta": {
- "content": "2"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "20"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": "2"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
},
{
"raw": {
@@ -771,16 +782,47 @@
{
"index": 0,
"delta": {
- "content": " is"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "}"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " is"
- },
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
+ }
+ ],
+ "message": {
+ "content": "",
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_2",
+ "response": {
+ "raw": [
{
"raw": {
"id": "HIDDEN",
@@ -792,7 +834,7 @@
{
"index": 0,
"delta": {
- "content": " "
+ "content": "The"
},
"logprobs": null,
"finish_reason": null
@@ -800,7 +842,7 @@
]
},
"options": {},
- "delta": " "
+ "delta": "The"
},
{
"raw": {
@@ -813,7 +855,7 @@
{
"index": 0,
"delta": {
- "content": "8"
+ "content": " result"
},
"logprobs": null,
"finish_reason": null
@@ -821,7 +863,7 @@
]
},
"options": {},
- "delta": "8"
+ "delta": " result"
},
{
"raw": {
@@ -834,7 +876,7 @@
{
"index": 0,
"delta": {
- "content": ","
+ "content": " of"
},
"logprobs": null,
"finish_reason": null
@@ -842,7 +884,7 @@
]
},
"options": {},
- "delta": ","
+ "delta": " of"
},
{
"raw": {
@@ -855,7 +897,7 @@
{
"index": 0,
"delta": {
- "content": " and"
+ "content": " dividing"
},
"logprobs": null,
"finish_reason": null
@@ -863,7 +905,7 @@
]
},
"options": {},
- "delta": " and"
+ "delta": " dividing"
},
{
"raw": {
@@ -876,7 +918,7 @@
{
"index": 0,
"delta": {
- "content": " when"
+ "content": " "
},
"logprobs": null,
"finish_reason": null
@@ -884,7 +926,7 @@
]
},
"options": {},
- "delta": " when"
+ "delta": " "
},
{
"raw": {
@@ -897,7 +939,7 @@
{
"index": 0,
"delta": {
- "content": " you"
+ "content": "16"
},
"logprobs": null,
"finish_reason": null
@@ -905,7 +947,7 @@
]
},
"options": {},
- "delta": " you"
+ "delta": "16"
},
{
"raw": {
@@ -918,7 +960,7 @@
{
"index": 0,
"delta": {
- "content": " add"
+ "content": " by"
},
"logprobs": null,
"finish_reason": null
@@ -926,7 +968,7 @@
]
},
"options": {},
- "delta": " add"
+ "delta": " by"
},
{
"raw": {
@@ -960,7 +1002,7 @@
{
"index": 0,
"delta": {
- "content": "20"
+ "content": "2"
},
"logprobs": null,
"finish_reason": null
@@ -968,7 +1010,7 @@
]
},
"options": {},
- "delta": "20"
+ "delta": "2"
},
{
"raw": {
@@ -981,7 +1023,7 @@
{
"index": 0,
"delta": {
- "content": " to"
+ "content": " and"
},
"logprobs": null,
"finish_reason": null
@@ -989,7 +1031,7 @@
]
},
"options": {},
- "delta": " to"
+ "delta": " and"
},
{
"raw": {
@@ -1002,7 +1044,7 @@
{
"index": 0,
"delta": {
- "content": " "
+ "content": " then"
},
"logprobs": null,
"finish_reason": null
@@ -1010,7 +1052,7 @@
]
},
"options": {},
- "delta": " "
+ "delta": " then"
},
{
"raw": {
@@ -1023,7 +1065,7 @@
{
"index": 0,
"delta": {
- "content": "8"
+ "content": " adding"
},
"logprobs": null,
"finish_reason": null
@@ -1031,7 +1073,7 @@
]
},
"options": {},
- "delta": "8"
+ "delta": " adding"
},
{
"raw": {
@@ -1044,7 +1086,7 @@
{
"index": 0,
"delta": {
- "content": ","
+ "content": " "
},
"logprobs": null,
"finish_reason": null
@@ -1052,7 +1094,7 @@
]
},
"options": {},
- "delta": ","
+ "delta": " "
},
{
"raw": {
@@ -1065,7 +1107,7 @@
{
"index": 0,
"delta": {
- "content": " you"
+ "content": "20"
},
"logprobs": null,
"finish_reason": null
@@ -1073,7 +1115,7 @@
]
},
"options": {},
- "delta": " you"
+ "delta": "20"
},
{
"raw": {
@@ -1086,7 +1128,7 @@
{
"index": 0,
"delta": {
- "content": " get"
+ "content": " is"
},
"logprobs": null,
"finish_reason": null
@@ -1094,7 +1136,7 @@
]
},
"options": {},
- "delta": " get"
+ "delta": " is"
},
{
"raw": {
@@ -1161,7 +1203,7 @@
}
],
"message": {
- "content": "The result of dividing 16 by 2 is 8, and when you add 20 to 8, you get 28.",
+ "content": "The result of dividing 16 by 2 and then adding 20 is 28.",
"role": "assistant",
"options": {}
}
@@ -1200,24 +1242,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
}
@@ -1250,24 +1279,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
}
@@ -1300,24 +1316,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
}
@@ -1350,24 +1353,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
}
@@ -1400,24 +1390,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "divideNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 16, \"b\": 2}"
+ }
},
"delta": ""
}
@@ -1453,24 +1430,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
}
@@ -1503,24 +1467,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
}
@@ -1553,24 +1504,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
}
@@ -1603,24 +1541,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
}
@@ -1653,24 +1578,11 @@
]
},
"options": {
- "toolCalls": [
- {
- "function": {
- "name": "divideNumbers",
- "arguments": "{\"a\": 16, \"b\": 2}"
- },
- "id": "HIDDEN",
- "type": "function"
- },
- {
- "function": {
- "name": "sumNumbers",
- "arguments": "{\"a\": 8, \"b\": 20}"
- },
- "id": "HIDDEN",
- "type": "function"
- }
- ]
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\": 8, \"b\": 20}"
+ }
},
"delta": ""
}
@@ -1688,15 +1600,33 @@
{
"index": 0,
"delta": {
- "content": "The"
+ "role": "assistant",
+ "content": null,
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "HIDDEN",
+ "type": "function",
+ "function": {
+ "name": "sumNumbers",
+ "arguments": ""
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": "The"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1712,15 +1642,28 @@
{
"index": 0,
"delta": {
- "content": " result"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "{\""
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " result"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1736,15 +1679,28 @@
{
"index": 0,
"delta": {
- "content": " of"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "a"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " of"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1760,15 +1716,28 @@
{
"index": 0,
"delta": {
- "content": " dividing"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "\":"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " dividing"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1784,15 +1753,28 @@
{
"index": 0,
"delta": {
- "content": " "
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "8"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " "
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1808,15 +1790,28 @@
{
"index": 0,
"delta": {
- "content": "16"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": ",\""
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": "16"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1832,15 +1827,28 @@
{
"index": 0,
"delta": {
- "content": " by"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "b"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " by"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1856,15 +1864,28 @@
{
"index": 0,
"delta": {
- "content": " "
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "\":"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " "
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1880,15 +1901,28 @@
{
"index": 0,
"delta": {
- "content": "2"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "20"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": "2"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
@@ -1904,19 +1938,32 @@
{
"index": 0,
"delta": {
- "content": " is"
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "arguments": "}"
+ }
+ }
+ ]
},
"logprobs": null,
"finish_reason": null
}
]
},
- "options": {},
- "delta": " is"
+ "options": {
+ "toolCall": {
+ "name": "sumNumbers",
+ "id": "HIDDEN",
+ "input": "{\"a\":8,\"b\":20}"
+ }
+ },
+ "delta": ""
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -1928,7 +1975,7 @@
{
"index": 0,
"delta": {
- "content": " "
+ "content": "The"
},
"logprobs": null,
"finish_reason": null
@@ -1936,11 +1983,11 @@
]
},
"options": {},
- "delta": " "
+ "delta": "The"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -1952,7 +1999,7 @@
{
"index": 0,
"delta": {
- "content": "8"
+ "content": " result"
},
"logprobs": null,
"finish_reason": null
@@ -1960,11 +2007,11 @@
]
},
"options": {},
- "delta": "8"
+ "delta": " result"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -1976,7 +2023,7 @@
{
"index": 0,
"delta": {
- "content": ","
+ "content": " of"
},
"logprobs": null,
"finish_reason": null
@@ -1984,11 +2031,11 @@
]
},
"options": {},
- "delta": ","
+ "delta": " of"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2000,7 +2047,7 @@
{
"index": 0,
"delta": {
- "content": " and"
+ "content": " dividing"
},
"logprobs": null,
"finish_reason": null
@@ -2008,11 +2055,11 @@
]
},
"options": {},
- "delta": " and"
+ "delta": " dividing"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2024,7 +2071,7 @@
{
"index": 0,
"delta": {
- "content": " when"
+ "content": " "
},
"logprobs": null,
"finish_reason": null
@@ -2032,11 +2079,11 @@
]
},
"options": {},
- "delta": " when"
+ "delta": " "
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2048,7 +2095,7 @@
{
"index": 0,
"delta": {
- "content": " you"
+ "content": "16"
},
"logprobs": null,
"finish_reason": null
@@ -2056,11 +2103,11 @@
]
},
"options": {},
- "delta": " you"
+ "delta": "16"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2072,7 +2119,7 @@
{
"index": 0,
"delta": {
- "content": " add"
+ "content": " by"
},
"logprobs": null,
"finish_reason": null
@@ -2080,11 +2127,11 @@
]
},
"options": {},
- "delta": " add"
+ "delta": " by"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2108,7 +2155,7 @@
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2120,7 +2167,7 @@
{
"index": 0,
"delta": {
- "content": "20"
+ "content": "2"
},
"logprobs": null,
"finish_reason": null
@@ -2128,11 +2175,11 @@
]
},
"options": {},
- "delta": "20"
+ "delta": "2"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2144,7 +2191,7 @@
{
"index": 0,
"delta": {
- "content": " to"
+ "content": " and"
},
"logprobs": null,
"finish_reason": null
@@ -2152,11 +2199,11 @@
]
},
"options": {},
- "delta": " to"
+ "delta": " and"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2168,7 +2215,7 @@
{
"index": 0,
"delta": {
- "content": " "
+ "content": " then"
},
"logprobs": null,
"finish_reason": null
@@ -2176,11 +2223,11 @@
]
},
"options": {},
- "delta": " "
+ "delta": " then"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2192,7 +2239,7 @@
{
"index": 0,
"delta": {
- "content": "8"
+ "content": " adding"
},
"logprobs": null,
"finish_reason": null
@@ -2200,11 +2247,11 @@
]
},
"options": {},
- "delta": "8"
+ "delta": " adding"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2216,7 +2263,7 @@
{
"index": 0,
"delta": {
- "content": ","
+ "content": " "
},
"logprobs": null,
"finish_reason": null
@@ -2224,11 +2271,11 @@
]
},
"options": {},
- "delta": ","
+ "delta": " "
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2240,7 +2287,7 @@
{
"index": 0,
"delta": {
- "content": " you"
+ "content": "20"
},
"logprobs": null,
"finish_reason": null
@@ -2248,11 +2295,11 @@
]
},
"options": {},
- "delta": " you"
+ "delta": "20"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2264,7 +2311,7 @@
{
"index": 0,
"delta": {
- "content": " get"
+ "content": " is"
},
"logprobs": null,
"finish_reason": null
@@ -2272,11 +2319,11 @@
]
},
"options": {},
- "delta": " get"
+ "delta": " is"
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2300,7 +2347,7 @@
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
@@ -2324,7 +2371,7 @@
}
},
{
- "id": "PRESERVE_1",
+ "id": "PRESERVE_2",
"chunk": {
"raw": {
"id": "HIDDEN",
diff --git a/packages/core/e2e/node/snapshot/anthropic-agent.snap b/packages/core/e2e/node/snapshot/anthropic-agent.snap
new file mode 100644
index 000000000..8d57bad7e
--- /dev/null
+++ b/packages/core/e2e/node/snapshot/anthropic-agent.snap
@@ -0,0 +1,403 @@
+{
+ "llmEventStart": [
+ {
+ "id": "PRESERVE_0",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the weather in San Francisco?",
+ "options": {}
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the weather in San Francisco?",
+ "options": {}
+ },
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+ }
+ ],
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "Weather",
+ "input": {
+ "location": "San Francisco"
+ }
+ }
+ }
+ },
+ {
+ "content": "35 degrees and sunny in San Francisco",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "isError": false,
+ "id": "HIDDEN"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_2",
+ "messages": [
+ {
+ "role": "user",
+ "content": "My name is Alex Yang. What is my unique id?",
+ "options": {}
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_3",
+ "messages": [
+ {
+ "role": "user",
+ "content": "My name is Alex Yang. What is my unique id?",
+ "options": {}
+ },
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+ }
+ ],
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "unique_id",
+ "input": {
+ "firstName": "Alex",
+ "lastName": "Yang"
+ }
+ }
+ }
+ },
+ {
+ "content": "123456789",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "isError": false,
+ "id": "HIDDEN"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_4",
+ "messages": [
+ {
+ "role": "user",
+ "content": "how much is 1 + 1?",
+ "options": {}
+ }
+ ]
+ },
+ {
+ "id": "PRESERVE_5",
+ "messages": [
+ {
+ "role": "user",
+ "content": "how much is 1 + 1?",
+ "options": {}
+ },
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+ }
+ ],
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "sumNumbers",
+ "input": {
+ "a": 1,
+ "b": 1
+ }
+ }
+ }
+ },
+ {
+ "content": "2",
+ "role": "user",
+ "options": {
+ "toolResult": {
+ "isError": false,
+ "id": "HIDDEN"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "llmEventEnd": [
+ {
+ "id": "PRESERVE_0",
+ "response": {
+ "raw": {
+ "id": "HIDDEN",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-opus-20240229",
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 462,
+ "output_tokens": 147
+ },
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+ },
+ {
+ "type": "tool_use",
+ "id": "HIDDEN",
+ "name": "Weather",
+ "input": {
+ "location": "San Francisco"
+ }
+ }
+ ],
+ "stop_reason": "tool_use"
+ },
+ "message": {
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+ }
+ ],
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "Weather",
+ "input": {
+ "location": "San Francisco"
+ }
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_1",
+ "response": {
+ "raw": {
+ "id": "HIDDEN",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-opus-20240229",
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 628,
+ "output_tokens": 18
+ },
+ "content": [
+ {
+ "type": "text",
+ "text": "The current weather in San Francisco is 35 degrees and sunny."
+ }
+ ],
+ "stop_reason": "end_turn"
+ },
+ "message": {
+ "content": [
+ {
+ "type": "text",
+ "text": "The current weather in San Francisco is 35 degrees and sunny."
+ }
+ ],
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_2",
+ "response": {
+ "raw": {
+ "id": "HIDDEN",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-opus-20240229",
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 482,
+ "output_tokens": 152
+ },
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+ },
+ {
+ "type": "tool_use",
+ "id": "HIDDEN",
+ "name": "unique_id",
+ "input": {
+ "firstName": "Alex",
+ "lastName": "Yang"
+ }
+ }
+ ],
+ "stop_reason": "tool_use"
+ },
+ "message": {
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+ }
+ ],
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "unique_id",
+ "input": {
+ "firstName": "Alex",
+ "lastName": "Yang"
+ }
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_3",
+ "response": {
+ "raw": {
+ "id": "HIDDEN",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-opus-20240229",
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 648,
+ "output_tokens": 13
+ },
+ "content": [
+ {
+ "type": "text",
+ "text": "Your unique ID is 123456789."
+ }
+ ],
+ "stop_reason": "end_turn"
+ },
+ "message": {
+ "content": [
+ {
+ "type": "text",
+ "text": "Your unique ID is 123456789."
+ }
+ ],
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_4",
+ "response": {
+ "raw": {
+ "id": "HIDDEN",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-opus-20240229",
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 498,
+ "output_tokens": 151
+ },
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+ },
+ {
+ "type": "tool_use",
+ "id": "HIDDEN",
+ "name": "sumNumbers",
+ "input": {
+ "a": 1,
+ "b": 1
+ }
+ }
+ ],
+ "stop_reason": "tool_use"
+ },
+ "message": {
+ "content": [
+ {
+ "type": "text",
+ "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+ }
+ ],
+ "role": "assistant",
+ "options": {
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "sumNumbers",
+ "input": {
+ "a": 1,
+ "b": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ {
+ "id": "PRESERVE_5",
+ "response": {
+ "raw": {
+ "id": "HIDDEN",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-3-opus-20240229",
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 661,
+ "output_tokens": 16
+ },
+ "content": [
+ {
+ "type": "text",
+ "text": "So 1 + 1 = 2."
+ }
+ ],
+ "stop_reason": "end_turn"
+ },
+ "message": {
+ "content": [
+ {
+ "type": "text",
+ "text": "So 1 + 1 = 2."
+ }
+ ],
+ "role": "assistant",
+ "options": {}
+ }
+ }
+ }
+ ],
+ "llmEventStream": []
+}
\ No newline at end of file
diff --git a/packages/core/e2e/node/snapshot/gpt-4-turbo.snap b/packages/core/e2e/node/snapshot/gpt-4-turbo.snap
index de400bac6..2e5658921 100644
--- a/packages/core/e2e/node/snapshot/gpt-4-turbo.snap
+++ b/packages/core/e2e/node/snapshot/gpt-4-turbo.snap
@@ -20,24 +20,21 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "Weather",
- "arguments": "{\"location\":\"San Jose\"}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "Weather",
+ "input": "{\"location\":\"San Jose\"}"
+ }
}
},
{
"content": "45 degrees and sunny in San Jose",
- "role": "tool",
+ "role": "user",
"options": {
- "name": "Weather",
- "tool_call_id": "HIDDEN"
+ "toolResult": {
+ "id": "HIDDEN",
+ "isError": false
+ }
}
}
]
@@ -84,16 +81,11 @@
"content": "",
"role": "assistant",
"options": {
- "toolCalls": [
- {
- "id": "HIDDEN",
- "type": "function",
- "function": {
- "name": "Weather",
- "arguments": "{\"location\":\"San Jose\"}"
- }
- }
- ]
+ "toolCall": {
+ "id": "HIDDEN",
+ "name": "Weather",
+ "input": "{\"location\":\"San Jose\"}"
+ }
}
}
}
diff --git a/packages/core/e2e/node/snapshot/llm-anthropic.snap b/packages/core/e2e/node/snapshot/llm-anthropic.snap
index 6b18dccbd..9c4892063 100644
--- a/packages/core/e2e/node/snapshot/llm-anthropic.snap
+++ b/packages/core/e2e/node/snapshot/llm-anthropic.snap
@@ -5,7 +5,8 @@
"messages": [
{
"content": "Hello",
- "role": "user"
+ "role": "user",
+ "options": {}
}
]
},
@@ -43,7 +44,8 @@
},
"message": {
"content": "Hello! How can I assist you today?",
- "role": "assistant"
+ "role": "assistant",
+ "options": {}
}
}
},
@@ -60,7 +62,8 @@
"text": "Hello"
}
},
- "delta": "Hello"
+ "delta": "Hello",
+ "options": {}
},
{
"raw": {
@@ -71,7 +74,8 @@
"text": "!"
}
},
- "delta": "!"
+ "delta": "!",
+ "options": {}
},
{
"raw": {
@@ -82,7 +86,8 @@
"text": " How"
}
},
- "delta": " How"
+ "delta": " How",
+ "options": {}
},
{
"raw": {
@@ -93,7 +98,8 @@
"text": " can"
}
},
- "delta": " can"
+ "delta": " can",
+ "options": {}
},
{
"raw": {
@@ -104,7 +110,8 @@
"text": " I"
}
},
- "delta": " I"
+ "delta": " I",
+ "options": {}
},
{
"raw": {
@@ -115,7 +122,8 @@
"text": " assist"
}
},
- "delta": " assist"
+ "delta": " assist",
+ "options": {}
},
{
"raw": {
@@ -126,7 +134,8 @@
"text": " you"
}
},
- "delta": " you"
+ "delta": " you",
+ "options": {}
},
{
"raw": {
@@ -137,7 +146,8 @@
"text": " today"
}
},
- "delta": " today"
+ "delta": " today",
+ "options": {}
},
{
"raw": {
@@ -148,7 +158,8 @@
"text": "?"
}
},
- "delta": "?"
+ "delta": "?",
+ "options": {}
}
],
"message": {
@@ -171,7 +182,8 @@
"text": "Hello"
}
},
- "delta": "Hello"
+ "delta": "Hello",
+ "options": {}
}
},
{
@@ -185,7 +197,8 @@
"text": "!"
}
},
- "delta": "!"
+ "delta": "!",
+ "options": {}
}
},
{
@@ -199,7 +212,8 @@
"text": " How"
}
},
- "delta": " How"
+ "delta": " How",
+ "options": {}
}
},
{
@@ -213,7 +227,8 @@
"text": " can"
}
},
- "delta": " can"
+ "delta": " can",
+ "options": {}
}
},
{
@@ -227,7 +242,8 @@
"text": " I"
}
},
- "delta": " I"
+ "delta": " I",
+ "options": {}
}
},
{
@@ -241,7 +257,8 @@
"text": " assist"
}
},
- "delta": " assist"
+ "delta": " assist",
+ "options": {}
}
},
{
@@ -255,7 +272,8 @@
"text": " you"
}
},
- "delta": " you"
+ "delta": " you",
+ "options": {}
}
},
{
@@ -269,7 +287,8 @@
"text": " today"
}
},
- "delta": " today"
+ "delta": " today",
+ "options": {}
}
},
{
@@ -283,7 +302,8 @@
"text": "?"
}
},
- "delta": "?"
+ "delta": "?",
+ "options": {}
}
}
]
diff --git a/packages/core/src/ChatHistory.ts b/packages/core/src/ChatHistory.ts
index 1b2a04a95..470ad6507 100644
--- a/packages/core/src/ChatHistory.ts
+++ b/packages/core/src/ChatHistory.ts
@@ -8,20 +8,22 @@ import { extractText } from "./llm/utils.js";
/**
* A ChatHistory is used to keep the state of back and forth chat messages
*/
-export abstract class ChatHistory {
- abstract get messages(): ChatMessage[];
+export abstract class ChatHistory<
+ AdditionalMessageOptions extends object = object,
+> {
+ abstract get messages(): ChatMessage<AdditionalMessageOptions>[];
/**
* Adds a message to the chat history.
* @param message
*/
- abstract addMessage(message: ChatMessage): void;
+ abstract addMessage(message: ChatMessage<AdditionalMessageOptions>): void;
/**
* Returns the messages that should be used as input to the LLM.
*/
abstract requestMessages(
- transientMessages?: ChatMessage[],
- ): Promise<ChatMessage[]>;
+ transientMessages?: ChatMessage<AdditionalMessageOptions>[],
+ ): Promise<ChatMessage<AdditionalMessageOptions>[]>;
/**
* Resets the chat history so that it's empty.
@@ -31,7 +33,7 @@ export abstract class ChatHistory {
/**
* Returns the new messages since the last call to this function (or since calling the constructor)
*/
- abstract newMessages(): ChatMessage[];
+ abstract newMessages(): ChatMessage<AdditionalMessageOptions>[];
}
export class SimpleChatHistory extends ChatHistory {
@@ -108,6 +110,7 @@ export class SummaryChatHistory extends ChatHistory {
context: messagesToHistoryStr(messagesToSummarize),
}),
role: "user" as MessageType,
+ options: {},
},
];
// remove oldest message until the chat history is short enough for the context window
@@ -116,7 +119,9 @@ export class SummaryChatHistory extends ChatHistory {
this.tokenizer(promptMessages[0].content).length > this.tokensToSummarize
);
- const response = await this.llm.chat({ messages: promptMessages });
+ const response = await this.llm.chat({
+ messages: promptMessages,
+ });
return { content: response.message.content, role: "memory" };
}
diff --git a/packages/core/src/agent/anthropic.ts b/packages/core/src/agent/anthropic.ts
new file mode 100644
index 000000000..393483492
--- /dev/null
+++ b/packages/core/src/agent/anthropic.ts
@@ -0,0 +1,170 @@
+import { Settings } from "../Settings.js";
+import {
+ AgentChatResponse,
+ type ChatEngineParamsNonStreaming,
+} from "../engines/chat/index.js";
+import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { prettifyError } from "../internal/utils.js";
+import { Anthropic } from "../llm/anthropic.js";
+import type {
+ ChatMessage,
+ ChatResponse,
+ ToolCallLLMMessageOptions,
+} from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
+import { ObjectRetriever } from "../objects/index.js";
+import type { BaseToolWithCall } from "../types.js";
+
+const MAX_TOOL_CALLS = 10;
+
+type AnthropicParamsBase = {
+ llm?: Anthropic;
+ chatHistory?: ChatMessage<ToolCallLLMMessageOptions>[];
+};
+
+type AnthropicParamsWithTools = AnthropicParamsBase & {
+ tools: BaseToolWithCall[];
+};
+
+type AnthropicParamsWithToolRetriever = AnthropicParamsBase & {
+ toolRetriever: ObjectRetriever<BaseToolWithCall>;
+};
+
+export type AnthropicAgentParams =
+ | AnthropicParamsWithTools
+ | AnthropicParamsWithToolRetriever;
+
+type AgentContext = {
+ toolCalls: number;
+ llm: Anthropic;
+ tools: BaseToolWithCall[];
+ messages: ChatMessage<ToolCallLLMMessageOptions>[];
+ shouldContinue: (context: AgentContext) => boolean;
+};
+
+type TaskResult = {
+ response: ChatResponse<ToolCallLLMMessageOptions>;
+ chatHistory: ChatMessage<ToolCallLLMMessageOptions>[];
+};
+
+async function task(
+ context: AgentContext,
+ input: ChatMessage<ToolCallLLMMessageOptions>,
+): Promise<TaskResult> {
+ const { llm, tools, messages } = context;
+ const nextMessages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+ ...messages,
+ input,
+ ];
+ const response = await llm.chat({
+ stream: false,
+ tools,
+ messages: nextMessages,
+ });
+ const options = response.message.options ?? {};
+ if ("toolCall" in options) {
+ const { toolCall } = options;
+ const { input, name, id } = toolCall;
+ const targetTool = tools.find((tool) => tool.metadata.name === name);
+ let output: string;
+ let isError = true;
+ if (!context.shouldContinue(context)) {
+ output = "Error: Tool call limit reached";
+ } else if (!targetTool) {
+ output = `Error: Tool ${name} not found`;
+ } else {
+ try {
+ getCallbackManager().dispatchEvent("llm-tool-call", {
+ payload: {
+ toolCall: {
+ name,
+ input,
+ },
+ },
+ });
+ output = await targetTool.call(input);
+ isError = false;
+ } catch (error: unknown) {
+ output = prettifyError(error);
+ }
+ }
+ return task(
+ {
+ ...context,
+ toolCalls: context.toolCalls + 1,
+ messages: [...nextMessages, response.message],
+ },
+ {
+ content: output,
+ role: "user",
+ options: {
+ toolResult: {
+ isError,
+ id,
+ },
+ },
+ },
+ );
+ } else {
+ return { response, chatHistory: [...nextMessages, response.message] };
+ }
+}
+
+export class AnthropicAgent {
+ readonly #llm: Anthropic;
+ readonly #tools:
+ | BaseToolWithCall[]
+ | ((query: string) => Promise<BaseToolWithCall[]>) = [];
+ #chatHistory: ChatMessage<ToolCallLLMMessageOptions>[] = [];
+
+ constructor(params: AnthropicAgentParams) {
+ this.#llm =
+ params.llm ?? Settings.llm instanceof Anthropic
+ ? (Settings.llm as Anthropic)
+ : new Anthropic();
+ if ("tools" in params) {
+ this.#tools = params.tools;
+ } else if ("toolRetriever" in params) {
+ this.#tools = params.toolRetriever.retrieve.bind(params.toolRetriever);
+ }
+ if (Array.isArray(params.chatHistory)) {
+ this.#chatHistory = params.chatHistory;
+ }
+ }
+
+ static shouldContinue(context: AgentContext): boolean {
+ return context.toolCalls < MAX_TOOL_CALLS;
+ }
+
+ public reset(): void {
+ this.#chatHistory = [];
+ }
+
+ getTools(query: string): Promise<BaseToolWithCall[]> | BaseToolWithCall[] {
+ return typeof this.#tools === "function" ? this.#tools(query) : this.#tools;
+ }
+
+ @wrapEventCaller
+ async chat(
+ params: ChatEngineParamsNonStreaming,
+ ): Promise<Promise<AgentChatResponse>> {
+ const { chatHistory, response } = await task(
+ {
+ llm: this.#llm,
+ tools: await this.getTools(extractText(params.message)),
+ toolCalls: 0,
+ messages: [...this.#chatHistory],
+ // do we need this?
+ shouldContinue: AnthropicAgent.shouldContinue,
+ },
+ {
+ role: "user",
+ content: params.message,
+ options: {},
+ },
+ );
+ this.#chatHistory = [...chatHistory];
+ return new AgentChatResponse(extractText(response.message.content));
+ }
+}
diff --git a/packages/core/src/agent/index.ts b/packages/core/src/agent/index.ts
index bbf0a4481..652403e03 100644
--- a/packages/core/src/agent/index.ts
+++ b/packages/core/src/agent/index.ts
@@ -1,3 +1,5 @@
+// Not exporting the AnthropicAgent because it is not ready to ship yet.
+// export { AnthropicAgent, type AnthropicAgentParams } from "./anthropic.js";
export * from "./openai/base.js";
export * from "./openai/worker.js";
export * from "./react/base.js";
diff --git a/packages/core/src/agent/openai/base.ts b/packages/core/src/agent/openai/base.ts
index eb89d2317..6a9af20c7 100644
--- a/packages/core/src/agent/openai/base.ts
+++ b/packages/core/src/agent/openai/base.ts
@@ -14,7 +14,7 @@ type OpenAIAgentParams = {
prefixMessages?: ChatMessage[];
maxFunctionCalls?: number;
defaultToolChoice?: string;
- toolRetriever?: ObjectRetriever;
+ toolRetriever?: ObjectRetriever<BaseTool>;
systemPrompt?: string;
};
@@ -56,7 +56,7 @@ export class OpenAIAgent extends AgentRunner {
];
}
- if (!llm?.metadata.isFunctionCallingModel) {
+ if (!llm?.supportToolCall) {
throw new Error("LLM model must be a function-calling model");
}
@@ -73,6 +73,7 @@ export class OpenAIAgent extends AgentRunner {
llm,
memory,
defaultToolChoice,
+ // @ts-expect-error 2322
chatHistory: prefixMessages,
});
}
diff --git a/packages/core/src/agent/openai/worker.ts b/packages/core/src/agent/openai/worker.ts
index d7939cd13..38e4d72ed 100644
--- a/packages/core/src/agent/openai/worker.ts
+++ b/packages/core/src/agent/openai/worker.ts
@@ -14,7 +14,8 @@ import {
type ChatResponseChunk,
type LLMChatParamsBase,
type OpenAIAdditionalChatOptions,
- type OpenAIAdditionalMessageOptions,
+ type ToolCallLLMMessageOptions,
+ type ToolCallOptions,
} from "../../llm/index.js";
import { extractText } from "../../llm/utils.js";
import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
@@ -25,28 +26,25 @@ import type { BaseTool } from "../../types.js";
import type { AgentWorker, Task } from "../types.js";
import { TaskStep, TaskStepOutput } from "../types.js";
import { addUserStepToMemory, getFunctionByName } from "../utils.js";
-import type { OpenAIToolCall } from "./types/chat.js";
async function callFunction(
tools: BaseTool[],
- toolCall: OpenAIToolCall,
-): Promise<[ChatMessage, ToolOutput]> {
- const id_ = toolCall.id;
- const functionCall = toolCall.function;
- const name = toolCall.function.name;
- const argumentsStr = toolCall.function.arguments;
+ toolCall: ToolCallOptions["toolCall"],
+): Promise<[ChatMessage<ToolCallLLMMessageOptions>, ToolOutput]> {
+ const id = toolCall.id;
+ const name = toolCall.name;
+ const input = toolCall.input;
if (Settings.debug) {
console.log("=== Calling Function ===");
- console.log(`Calling function: ${name} with args: ${argumentsStr}`);
+ console.log(`Calling function: ${name} with args: ${input}`);
}
const tool = getFunctionByName(tools, name);
- const argumentDict = JSON.parse(argumentsStr);
// Call tool
// Use default error message
- const output = await callToolWithErrorHandling(tool, argumentDict);
+ const output = await callToolWithErrorHandling(tool, input);
if (Settings.debug) {
console.log(`Got output ${output}`);
@@ -56,10 +54,12 @@ async function callFunction(
return [
{
content: `${output}`,
- role: "tool",
+ role: "user",
options: {
- name,
- tool_call_id: id_,
+ toolResult: {
+ id,
+ isError: false,
+ },
},
},
output,
@@ -71,7 +71,7 @@ type OpenAIAgentWorkerParams = {
llm?: OpenAI;
prefixMessages?: ChatMessage[];
maxFunctionCalls?: number;
- toolRetriever?: ObjectRetriever;
+ toolRetriever?: ObjectRetriever<BaseTool>;
};
type CallFunctionOutput = {
@@ -120,7 +120,7 @@ export class OpenAIAgentWorker
}
}
- public getAllMessages(task: Task): ChatMessage[] {
+ public getAllMessages(task: Task): ChatMessage<ToolCallLLMMessageOptions>[] {
return [
...this.prefixMessages,
...task.memory.get(),
@@ -128,30 +128,33 @@ export class OpenAIAgentWorker
];
}
- public getLatestToolCalls(task: Task): OpenAIToolCall[] | null {
+ public getLatestToolCall(task: Task): ToolCallOptions["toolCall"] | null {
const chatHistory: ChatMessage[] = task.extraState.newMemory.getAll();
if (chatHistory.length === 0) {
return null;
}
- // fixme
- return chatHistory[chatHistory.length - 1].options?.toolCalls as any;
+ // @ts-expect-error fixme
+ return chatHistory[chatHistory.length - 1].options?.toolCall;
}
private _getLlmChatParams(
task: Task,
- openaiTools: BaseTool[],
+ tools: BaseTool[],
toolChoice: ChatCompletionToolChoiceOption = "auto",
- ): LLMChatParamsBase<OpenAIAdditionalChatOptions> {
+ ): LLMChatParamsBase<OpenAIAdditionalChatOptions, ToolCallLLMMessageOptions> {
const llmChatParams = {
messages: this.getAllMessages(task),
tools: undefined as BaseTool[] | undefined,
additionalChatOptions: {} as OpenAIAdditionalChatOptions,
- } satisfies LLMChatParamsBase<OpenAIAdditionalChatOptions>;
+ } satisfies LLMChatParamsBase<
+ OpenAIAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >;
- if (openaiTools.length > 0) {
- llmChatParams.tools = openaiTools;
+ if (tools.length > 0) {
+ llmChatParams.tools = tools;
llmChatParams.additionalChatOptions.tool_choice = toolChoice;
}
@@ -172,7 +175,10 @@ export class OpenAIAgentWorker
private async _getStreamAiResponse(
task: Task,
- llmChatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
+ llmChatParams: LLMChatParamsBase<
+ OpenAIAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
): Promise<StreamingAgentChatResponse | AgentChatResponse> {
const stream = await this.llm.chat({
stream: true,
@@ -180,7 +186,7 @@ export class OpenAIAgentWorker
});
const responseChunkStream = new ReadableStream<
- ChatResponseChunk<OpenAIAdditionalMessageOptions>
+ ChatResponseChunk<ToolCallLLMMessageOptions>
>({
async start(controller) {
for await (const chunk of stream) {
@@ -198,11 +204,11 @@ export class OpenAIAgentWorker
}
// check if first chunk has tool calls, if so, this is a function call
// otherwise, it's a regular message
- const hasToolCalls: boolean =
- !!value.options?.toolCalls?.length &&
- value.options?.toolCalls?.length > 0;
+ const hasToolCall: boolean = !!(
+ value.options && "toolCall" in value.options
+ );
- if (hasToolCalls) {
+ if (hasToolCall) {
return this._processMessage(task, {
content: await pipeline(finalStream, async (iterator) => {
let content = "";
@@ -247,7 +253,10 @@ export class OpenAIAgentWorker
private async _getAgentResponse(
task: Task,
mode: ChatResponseMode,
- llmChatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
+ llmChatParams: LLMChatParamsBase<
+ OpenAIAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
): Promise<AgentChatResponse | StreamingAgentChatResponse> {
if (mode === ChatResponseMode.WAIT) {
const chatResponse = await this.llm.chat({
@@ -268,14 +277,8 @@ export class OpenAIAgentWorker
async callFunction(
tools: BaseTool[],
- toolCall: OpenAIToolCall,
+ toolCall: ToolCallOptions["toolCall"],
): Promise<CallFunctionOutput> {
- const functionCall = toolCall.function;
-
- if (!functionCall) {
- throw new Error("Invalid tool_call object");
- }
-
const functionMessage = await callFunction(tools, toolCall);
const message = functionMessage[0];
@@ -309,18 +312,14 @@ export class OpenAIAgentWorker
}
private _shouldContinue(
- toolCalls: OpenAIToolCall[] | null,
+ toolCall: ToolCallOptions["toolCall"] | null,
nFunctionCalls: number,
- ): boolean {
+ ): toolCall is ToolCallOptions["toolCall"] {
if (nFunctionCalls > this.maxFunctionCalls) {
return false;
}
- if (toolCalls?.length === 0) {
- return false;
- }
-
- return true;
+ return !!toolCall;
}
async getTools(input: string): Promise<BaseTool[]> {
@@ -347,29 +346,25 @@ export class OpenAIAgentWorker
llmChatParams,
);
- const latestToolCalls = this.getLatestToolCalls(task) || [];
+ const latestToolCall = this.getLatestToolCall(task) ?? null;
let isDone: boolean;
- let newSteps: TaskStep[] = [];
+ let newSteps: TaskStep[];
- if (
- !this._shouldContinue(latestToolCalls, task.extraState.nFunctionCalls)
- ) {
+ if (!this._shouldContinue(latestToolCall, task.extraState.nFunctionCalls)) {
isDone = true;
newSteps = [];
} else {
isDone = false;
- for (const toolCall of latestToolCalls) {
- const { message, toolOutput } = await this.callFunction(
- tools,
- toolCall,
- );
+ const { message, toolOutput } = await this.callFunction(
+ tools,
+ latestToolCall,
+ );
- task.extraState.sources.push(toolOutput);
- task.extraState.newMemory.put(message);
+ task.extraState.sources.push(toolOutput);
+ task.extraState.newMemory.put(message);
- task.extraState.nFunctionCalls += 1;
- }
+ task.extraState.nFunctionCalls += 1;
newSteps = [step.getNextStep(randomUUID(), undefined)];
}
diff --git a/packages/core/src/agent/react/base.ts b/packages/core/src/agent/react/base.ts
index 5f7f1b06a..763e6f487 100644
--- a/packages/core/src/agent/react/base.ts
+++ b/packages/core/src/agent/react/base.ts
@@ -12,7 +12,7 @@ type ReActAgentParams = {
prefixMessages?: ChatMessage[];
maxInteractions?: number;
defaultToolChoice?: string;
- toolRetriever?: ObjectRetriever;
+ toolRetriever?: ObjectRetriever<BaseTool>;
};
/**
@@ -41,6 +41,7 @@ export class ReActAgent extends AgentRunner {
agentWorker: stepEngine,
memory,
defaultToolChoice,
+ // @ts-expect-error 2322
chatHistory: prefixMessages,
});
}
diff --git a/packages/core/src/agent/react/worker.ts b/packages/core/src/agent/react/worker.ts
index 7987d33c0..3fba7b04c 100644
--- a/packages/core/src/agent/react/worker.ts
+++ b/packages/core/src/agent/react/worker.ts
@@ -2,6 +2,7 @@ import { randomUUID } from "@llamaindex/env";
import type { ChatMessage } from "cohere-ai/api";
import { Settings } from "../../Settings.js";
import { AgentChatResponse } from "../../engines/chat/index.js";
+import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
import { type ChatResponse, type LLM } from "../../llm/index.js";
import { extractText } from "../../llm/utils.js";
import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
@@ -25,7 +26,7 @@ type ReActAgentWorkerParams = {
maxInteractions?: number;
reactChatFormatter?: ReActChatFormatter | undefined;
outputParser?: ReActOutputParser | undefined;
- toolRetriever?: ObjectRetriever | undefined;
+ toolRetriever?: ObjectRetriever<BaseTool> | undefined;
};
function addUserStepToReasoning(
@@ -194,6 +195,14 @@ export class ReActAgentWorker implements AgentWorker<ChatParams> {
const tool = toolsDict[actionReasoningStep.action];
+ getCallbackManager().dispatchEvent("llm-tool-call", {
+ payload: {
+ toolCall: {
+ name: tool.metadata.name,
+ input: JSON.stringify(actionReasoningStep.actionInput),
+ },
+ },
+ });
const toolOutput = await tool.call!(actionReasoningStep.actionInput);
task.extraState.sources.push(
diff --git a/packages/core/src/agent/runner/base.ts b/packages/core/src/agent/runner/base.ts
index 361fe9aa5..915f996b3 100644
--- a/packages/core/src/agent/runner/base.ts
+++ b/packages/core/src/agent/runner/base.ts
@@ -1,11 +1,12 @@
import { randomUUID } from "@llamaindex/env";
+import type { ChatHistory } from "../../ChatHistory.js";
import type { ChatEngineAgentParams } from "../../engines/chat/index.js";
import {
AgentChatResponse,
ChatResponseMode,
StreamingAgentChatResponse,
} from "../../engines/chat/index.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
+import type { LLM } from "../../llm/index.js";
import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
import type { BaseMemory } from "../../memory/types.js";
import type { AgentWorker, TaskStepOutput } from "../types.js";
@@ -30,7 +31,7 @@ const validateStepFromArgs = (
type AgentRunnerParams = {
agentWorker: AgentWorker;
- chatHistory?: ChatMessage[];
+ chatHistory?: ChatHistory;
state?: AgentState;
memory?: BaseMemory;
llm?: LLM;
diff --git a/packages/core/src/callbacks/CallbackManager.ts b/packages/core/src/callbacks/CallbackManager.ts
index d596615ac..22c534c18 100644
--- a/packages/core/src/callbacks/CallbackManager.ts
+++ b/packages/core/src/callbacks/CallbackManager.ts
@@ -9,6 +9,7 @@ import type {
LLMEndEvent,
LLMStartEvent,
LLMStreamEvent,
+ LLMToolCallEvent,
} from "../llm/types.js";
export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
@@ -48,6 +49,7 @@ export interface LlamaIndexEventMaps {
stream: CustomEvent<StreamCallbackResponse>;
"llm-start": LLMStartEvent;
"llm-end": LLMEndEvent;
+ "llm-tool-call": LLMToolCallEvent;
"llm-stream": LLMStreamEvent;
}
@@ -203,8 +205,10 @@ export class CallbackManager implements CallbackManagerMethods {
if (!handlers) {
return;
}
- handlers.forEach((handler) =>
- handler(LlamaIndexCustomEvent.fromEvent(event, detail)),
- );
+ queueMicrotask(() => {
+ handlers.forEach((handler) =>
+ handler(LlamaIndexCustomEvent.fromEvent(event, detail)),
+ );
+ });
}
}
diff --git a/packages/core/src/internal/utils.ts b/packages/core/src/internal/utils.ts
index c04db0b39..63133c61a 100644
--- a/packages/core/src/internal/utils.ts
+++ b/packages/core/src/internal/utils.ts
@@ -5,3 +5,14 @@ export const isAsyncGenerator = (obj: unknown): obj is AsyncGenerator => {
export const isGenerator = (obj: unknown): obj is Generator => {
return obj != null && typeof obj === "object" && Symbol.iterator in obj;
};
+
+/**
+ * Prettify an error for AI to read
+ */
+export function prettifyError(error: unknown): string {
+ if (error instanceof Error) {
+ return `Error(${error.name}): ${error.message}`;
+ } else {
+ return `${error}`;
+ }
+}
diff --git a/packages/core/src/llm/anthropic.ts b/packages/core/src/llm/anthropic.ts
index c6dfd5473..a708d9b0a 100644
--- a/packages/core/src/llm/anthropic.ts
+++ b/packages/core/src/llm/anthropic.ts
@@ -2,19 +2,28 @@ import type { ClientOptions } from "@anthropic-ai/sdk";
import { Anthropic as SDKAnthropic } from "@anthropic-ai/sdk";
import type {
Tool,
+ ToolResultBlockParam,
ToolUseBlock,
+ ToolUseBlockParam,
+ ToolsBetaContentBlock,
+ ToolsBetaMessageParam,
} from "@anthropic-ai/sdk/resources/beta/tools/messages";
-import type { TextBlock } from "@anthropic-ai/sdk/resources/index";
+import type {
+ TextBlock,
+ TextBlockParam,
+} from "@anthropic-ai/sdk/resources/index";
+import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
import { getEnv } from "@llamaindex/env";
import _ from "lodash";
import type { BaseTool } from "../types.js";
-import { BaseLLM } from "./base.js";
+import { ToolCallLLM } from "./base.js";
import type {
ChatMessage,
ChatResponse,
ChatResponseChunk,
LLMChatParamsNonStreaming,
LLMChatParamsStreaming,
+ ToolCallLLMMessageOptions,
} from "./types.js";
import { extractText, wrapLLMEvent } from "./utils.js";
@@ -89,19 +98,7 @@ const AVAILABLE_ANTHROPIC_MODELS_WITHOUT_DATE: { [key: string]: string } = {
export type AnthropicAdditionalChatOptions = {};
-export type AnthropicAdditionalMessageOptions =
- | {
- toolCall: string;
- }
- | {
- toolUse: ToolUseBlock;
- }
- | {};
-
-export class Anthropic extends BaseLLM<
- AnthropicAdditionalChatOptions,
- AnthropicAdditionalMessageOptions
-> {
+export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
// Per completion Anthropic params
model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
temperature: number;
@@ -133,6 +130,10 @@ export class Anthropic extends BaseLLM<
});
}
+ get supportToolCall() {
+ return this.model.startsWith("claude-3");
+ }
+
get metadata() {
return {
model: this.model,
@@ -151,27 +152,90 @@ export class Anthropic extends BaseLLM<
return model;
};
- formatMessages(messages: ChatMessage[]) {
- return messages.map((message) => {
+ formatMessages<Beta = false>(
+ messages: ChatMessage<ToolCallLLMMessageOptions>[],
+ ): Beta extends true ? ToolsBetaMessageParam[] : MessageParam[] {
+ return messages.map<any>((message) => {
if (message.role !== "user" && message.role !== "assistant") {
throw new Error("Unsupported Anthropic role");
}
+ const options = message.options ?? {};
+ if ("toolResult" in options) {
+ const { id, isError } = options.toolResult;
+ return {
+ role: "user",
+ content: [
+ {
+ type: "tool_result",
+ is_error: isError,
+ content: [
+ {
+ type: "text",
+ text: extractText(message.content),
+ },
+ ],
+ tool_use_id: id,
+ },
+ ] satisfies ToolResultBlockParam[],
+ } satisfies ToolsBetaMessageParam;
+ } else if ("toolCall" in options) {
+ const aiThinkingText = extractText(message.content);
+ return {
+ role: "assistant",
+ content: [
+ // this could be empty when you call two tools in one query
+ ...(aiThinkingText.trim()
+ ? [
+ {
+ type: "text",
+ text: aiThinkingText,
+ } satisfies TextBlockParam,
+ ]
+ : []),
+ {
+ type: "tool_use",
+ id: options.toolCall.id,
+ name: options.toolCall.name,
+ input: options.toolCall.input,
+ } satisfies ToolUseBlockParam,
+ ] satisfies ToolsBetaContentBlock[],
+ } satisfies ToolsBetaMessageParam;
+ }
return {
content: extractText(message.content),
role: message.role,
- };
+ } satisfies MessageParam;
});
}
chat(
- params: LLMChatParamsStreaming,
- ): Promise<AsyncIterable<ChatResponseChunk>>;
- chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+ params: LLMChatParamsStreaming<
+ AnthropicAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
+ ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
+ chat(
+ params: LLMChatParamsNonStreaming<
+ AnthropicAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
+ ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
@wrapLLMEvent
async chat(
- params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
- ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+ params:
+ | LLMChatParamsNonStreaming<
+ AnthropicAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >
+ | LLMChatParamsStreaming<
+ AnthropicAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
+ ): Promise<
+ | ChatResponse<ToolCallLLMMessageOptions>
+ | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
+ > {
let { messages } = params;
const { stream, tools } = params;
@@ -201,7 +265,7 @@ export class Anthropic extends BaseLLM<
if (tools) {
const response = await anthropic.beta.tools.messages.create({
- messages: this.formatMessages(messages),
+ messages: this.formatMessages<true>(messages),
tools: tools.map(Anthropic.toTool),
model: this.getModelName(this.model),
temperature: this.temperature,
@@ -226,7 +290,11 @@ export class Anthropic extends BaseLLM<
role: "assistant",
options: toolUseBlock
? {
- toolUse: toolUseBlock,
+ toolCall: {
+ id: toolUseBlock.id,
+ name: toolUseBlock.name,
+ input: toolUseBlock.input,
+ },
}
: {},
},
@@ -243,18 +311,22 @@ export class Anthropic extends BaseLLM<
return {
raw: response,
- message: { content: response.content[0].text, role: "assistant" },
+ message: {
+ content: response.content[0].text,
+ role: "assistant",
+ options: {},
+ },
};
}
}
protected async *streamChat(
- messages: ChatMessage[],
+ messages: ChatMessage<ToolCallLLMMessageOptions>[],
systemPrompt?: string | null,
- ): AsyncIterable<ChatResponseChunk> {
+ ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
const stream = await this.session.anthropic.messages.create({
model: this.getModelName(this.model),
- messages: this.formatMessages(messages),
+ messages: this.formatMessages<false>(messages),
max_tokens: this.maxTokens ?? 4096,
temperature: this.temperature,
top_p: this.topP,
@@ -273,6 +345,7 @@ export class Anthropic extends BaseLLM<
yield {
raw: part,
delta: content,
+ options: {},
};
}
return;
diff --git a/packages/core/src/llm/base.ts b/packages/core/src/llm/base.ts
index d0f434909..8db28bc9b 100644
--- a/packages/core/src/llm/base.ts
+++ b/packages/core/src/llm/base.ts
@@ -8,18 +8,13 @@ import type {
LLMCompletionParamsNonStreaming,
LLMCompletionParamsStreaming,
LLMMetadata,
+ ToolCallLLMMessageOptions,
} from "./types.js";
import { extractText, streamConverter } from "./utils.js";
export abstract class BaseLLM<
- AdditionalChatOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
+ AdditionalChatOptions extends object = object,
+ AdditionalMessageOptions extends object = object,
> implements LLM<AdditionalChatOptions>
{
abstract metadata: LLMMetadata;
@@ -56,9 +51,21 @@ export abstract class BaseLLM<
}
abstract chat(
- params: LLMChatParamsStreaming<AdditionalChatOptions>,
+ params: LLMChatParamsStreaming<
+ AdditionalChatOptions,
+ AdditionalMessageOptions
+ >,
): Promise<AsyncIterable<ChatResponseChunk>>;
abstract chat(
- params: LLMChatParamsNonStreaming<AdditionalChatOptions>,
+ params: LLMChatParamsNonStreaming<
+ AdditionalChatOptions,
+ AdditionalMessageOptions
+ >,
): Promise<ChatResponse<AdditionalMessageOptions>>;
}
+
+export abstract class ToolCallLLM<
+ AdditionalChatOptions extends object = object,
+> extends BaseLLM<AdditionalChatOptions, ToolCallLLMMessageOptions> {
+ abstract supportToolCall: boolean;
+}
diff --git a/packages/core/src/llm/open_ai.ts b/packages/core/src/llm/open_ai.ts
index 6279f924e..78590ec89 100644
--- a/packages/core/src/llm/open_ai.ts
+++ b/packages/core/src/llm/open_ai.ts
@@ -9,11 +9,11 @@ import { OpenAI as OrigOpenAI } from "openai";
import type {
ChatCompletionAssistantMessageParam,
- ChatCompletionFunctionMessageParam,
ChatCompletionMessageToolCall,
ChatCompletionRole,
ChatCompletionSystemMessageParam,
ChatCompletionTool,
+ ChatCompletionToolMessageParam,
ChatCompletionUserMessageParam,
} from "openai/resources/chat/completions";
import type { ChatCompletionMessageParam } from "openai/resources/index.js";
@@ -28,7 +28,7 @@ import {
getAzureModel,
shouldUseAzure,
} from "./azure.js";
-import { BaseLLM } from "./base.js";
+import { ToolCallLLM } from "./base.js";
import type {
ChatMessage,
ChatResponse,
@@ -37,8 +37,9 @@ import type {
LLMChatParamsNonStreaming,
LLMChatParamsStreaming,
LLMMetadata,
- MessageToolCall,
MessageType,
+ ToolCallLLMMessageOptions,
+ ToolCallOptions,
} from "./types.js";
import { extractText, wrapLLMEvent } from "./utils.js";
@@ -143,14 +144,7 @@ export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
return isChatModel && !isOld;
}
-export type OpenAIAdditionalMetadata = {
- isFunctionCallingModel: boolean;
-};
-
-export type OpenAIAdditionalMessageOptions = {
- functionName?: string;
- toolCalls?: ChatCompletionMessageToolCall[];
-};
+export type OpenAIAdditionalMetadata = {};
export type OpenAIAdditionalChatOptions = Omit<
Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
@@ -164,10 +158,7 @@ export type OpenAIAdditionalChatOptions = Omit<
| "toolChoice"
>;
-export class OpenAI extends BaseLLM<
- OpenAIAdditionalChatOptions,
- OpenAIAdditionalMessageOptions
-> {
+export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
// Per completion OpenAI params
model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
temperature: number;
@@ -238,6 +229,10 @@ export class OpenAI extends BaseLLM<
}
}
+ get supportToolCall() {
+ return isFunctionCallingModel(this);
+ }
+
get metadata(): LLMMetadata & OpenAIAdditionalMetadata {
const contextWindow =
ALL_AVAILABLE_OPENAI_MODELS[
@@ -250,7 +245,6 @@ export class OpenAI extends BaseLLM<
maxTokens: this.maxTokens,
contextWindow,
tokenizer: Tokenizers.CL100K_BASE,
- isFunctionCallingModel: isFunctionCallingModel(this),
};
}
@@ -262,46 +256,45 @@ export class OpenAI extends BaseLLM<
return "assistant";
case "system":
return "system";
- case "function":
- return "function";
- case "tool":
- return "tool";
default:
return "user";
}
}
static toOpenAIMessage(
- messages: ChatMessage<OpenAIAdditionalMessageOptions>[],
+ messages: ChatMessage<ToolCallLLMMessageOptions>[],
): ChatCompletionMessageParam[] {
return messages.map((message) => {
- const options: OpenAIAdditionalMessageOptions = message.options ?? {};
- if (message.role === "user") {
- return {
- role: "user",
- content: message.content,
- } satisfies ChatCompletionUserMessageParam;
- }
- if (typeof message.content !== "string") {
- console.warn("Message content is not a string");
- }
- if (message.role === "function") {
- if (!options.functionName) {
- console.warn("Function message does not have a name");
- }
+ const options = message.options ?? {};
+ if ("toolResult" in options) {
return {
- role: "function",
- name: options.functionName ?? "UNKNOWN",
+ tool_call_id: options.toolResult.id,
+ role: "tool",
content: extractText(message.content),
- // todo: remove this since this is deprecated in the OpenAI API
- } satisfies ChatCompletionFunctionMessageParam;
- }
- if (message.role === "assistant") {
+ } satisfies ChatCompletionToolMessageParam;
+ } else if ("toolCall" in options) {
return {
role: "assistant",
content: extractText(message.content),
- tool_calls: options.toolCalls,
+ tool_calls: [
+ {
+ id: options.toolCall.id,
+ type: "function",
+ function: {
+ name: options.toolCall.name,
+ arguments:
+ typeof options.toolCall.input === "string"
+ ? options.toolCall.input
+ : JSON.stringify(options.toolCall.input),
+ },
+ },
+ ],
} satisfies ChatCompletionAssistantMessageParam;
+ } else if (message.role === "user") {
+ return {
+ role: "user",
+ content: message.content,
+ } satisfies ChatCompletionUserMessageParam;
}
const response:
@@ -312,27 +305,38 @@ export class OpenAI extends BaseLLM<
role: OpenAI.toOpenAIRole(message.role) as never,
// fixme: should not extract text, but assert content is string
content: extractText(message.content),
- ...options,
};
return response;
});
}
chat(
- params: LLMChatParamsStreaming<OpenAIAdditionalChatOptions>,
- ): Promise<AsyncIterable<ChatResponseChunk<OpenAIAdditionalMessageOptions>>>;
+ params: LLMChatParamsStreaming<
+ OpenAIAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
+ ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
chat(
- params: LLMChatParamsNonStreaming<OpenAIAdditionalChatOptions>,
- ): Promise<ChatResponse<OpenAIAdditionalMessageOptions>>;
+ params: LLMChatParamsNonStreaming<
+ OpenAIAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
+ ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
@wrapEventCaller
@wrapLLMEvent
async chat(
params:
- | LLMChatParamsNonStreaming<OpenAIAdditionalChatOptions>
- | LLMChatParamsStreaming<OpenAIAdditionalChatOptions>,
+ | LLMChatParamsNonStreaming<
+ OpenAIAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >
+ | LLMChatParamsStreaming<
+ OpenAIAdditionalChatOptions,
+ ToolCallLLMMessageOptions
+ >,
): Promise<
- | ChatResponse<OpenAIAdditionalMessageOptions>
- | AsyncIterable<ChatResponseChunk<OpenAIAdditionalMessageOptions>>
+ | ChatResponse<ToolCallLLMMessageOptions>
+ | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
> {
const { messages, stream, tools, additionalChatOptions } = params;
const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
@@ -358,18 +362,21 @@ export class OpenAI extends BaseLLM<
const content = response.choices[0].message?.content ?? "";
- const options: OpenAIAdditionalMessageOptions = {};
-
- if (response.choices[0].message?.tool_calls) {
- options.toolCalls = response.choices[0].message.tool_calls;
- }
-
return {
raw: response,
message: {
content,
role: response.choices[0].message.role,
- options,
+ options: response.choices[0].message?.tool_calls
+ ? {
+ toolCall: {
+ id: response.choices[0].message.tool_calls[0].id,
+ name: response.choices[0].message.tool_calls[0].function.name,
+ input:
+ response.choices[0].message.tool_calls[0].function.arguments,
+ },
+ }
+ : {},
},
};
}
@@ -377,7 +384,7 @@ export class OpenAI extends BaseLLM<
@wrapEventCaller
protected async *streamChat(
baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams,
- ): AsyncIterable<ChatResponseChunk<OpenAIAdditionalMessageOptions>> {
+ ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
const stream: AsyncIterable<OpenAILLM.Chat.ChatCompletionChunk> =
await this.session.openai.chat.completions.create({
...baseRequestParams,
@@ -387,13 +394,26 @@ export class OpenAI extends BaseLLM<
// TODO: add callback to streamConverter and use streamConverter here
//Indices
let idxCounter: number = 0;
- const toolCalls: MessageToolCall[] = [];
+ let toolCallOptions: ToolCallOptions | null = null;
for await (const part of stream) {
if (!part.choices.length) continue;
const choice = part.choices[0];
// skip parts that don't have any content
if (!(choice.delta.content || choice.delta.tool_calls)) continue;
- updateToolCalls(toolCalls, choice.delta.tool_calls);
+ if (choice.delta.tool_calls?.[0].id) {
+ toolCallOptions = {
+ toolCall: {
+ name: choice.delta.tool_calls[0].function!.name!,
+ id: choice.delta.tool_calls[0].id,
+ input: choice.delta.tool_calls[0].function!.arguments!,
+ },
+ };
+ } else {
+ if (choice.delta.tool_calls?.[0].function?.arguments) {
+ toolCallOptions!.toolCall.input +=
+ choice.delta.tool_calls[0].function.arguments;
+ }
+ }
const isDone: boolean = choice.finish_reason !== null;
@@ -405,8 +425,7 @@ export class OpenAI extends BaseLLM<
yield {
raw: part,
- // add tool calls to final chunk
- options: toolCalls.length > 0 ? { toolCalls: toolCalls } : {},
+ options: toolCallOptions ? toolCallOptions : {},
delta: choice.delta.content ?? "",
};
}
@@ -424,34 +443,3 @@ export class OpenAI extends BaseLLM<
};
}
}
-
-function updateToolCalls(
- toolCalls: MessageToolCall[],
- toolCallDeltas?: OpenAILLM.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall[],
-) {
- function augmentToolCall(
- toolCall?: MessageToolCall,
- toolCallDelta?: OpenAILLM.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall,
- ) {
- toolCall =
- toolCall ??
- ({ function: { name: "", arguments: "" } } as MessageToolCall);
- toolCall.id = toolCall.id ?? toolCallDelta?.id;
- toolCall.type = toolCall.type ?? toolCallDelta?.type;
- if (toolCallDelta?.function?.arguments) {
- toolCall.function.arguments += toolCallDelta.function.arguments;
- }
- if (toolCallDelta?.function?.name) {
- toolCall.function.name += toolCallDelta.function.name;
- }
- return toolCall;
- }
- if (toolCallDeltas) {
- toolCallDeltas?.forEach((toolCall) => {
- toolCalls[toolCall.index] = augmentToolCall(
- toolCalls[toolCall.index],
- toolCall,
- );
- });
- }
-}
diff --git a/packages/core/src/llm/types.ts b/packages/core/src/llm/types.ts
index bbe8aa8e0..26da7b4ee 100644
--- a/packages/core/src/llm/types.ts
+++ b/packages/core/src/llm/types.ts
@@ -1,54 +1,42 @@
import type { Tokenizers } from "../GlobalsHelper.js";
import type { BaseTool, UUID } from "../types.js";
-type LLMBaseEvent<
- Type extends string,
- Payload extends Record<string, unknown>,
-> = CustomEvent<{
+type LLMBaseEvent<Payload extends Record<string, unknown>> = CustomEvent<{
payload: Payload;
}>;
-export type LLMStartEvent = LLMBaseEvent<
- "llm-start",
- {
- id: UUID;
- messages: ChatMessage[];
- }
->;
-export type LLMEndEvent = LLMBaseEvent<
- "llm-end",
- {
- id: UUID;
- response: ChatResponse;
- }
->;
-export type LLMStreamEvent = LLMBaseEvent<
- "llm-stream",
- {
- id: UUID;
- chunk: ChatResponseChunk;
- }
->;
+export type LLMStartEvent = LLMBaseEvent<{
+ id: UUID;
+ messages: ChatMessage[];
+}>;
+export type LLMToolCallEvent = LLMBaseEvent<{
+ // fixme: id is missing in the context
+ // id: UUID;
+ toolCall: Omit<ToolCallOptions["toolCall"], "id">;
+}>;
+export type LLMEndEvent = LLMBaseEvent<{
+ id: UUID;
+ response: ChatResponse;
+}>;
+export type LLMStreamEvent = LLMBaseEvent<{
+ id: UUID;
+ chunk: ChatResponseChunk;
+}>;
/**
* @internal
*/
export interface LLMChat<
- AdditionalChatOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
+ AdditionalChatOptions extends object = object,
+ AdditionalMessageOptions extends object = object,
> {
chat(
params:
| LLMChatParamsStreaming<AdditionalChatOptions>
| LLMChatParamsNonStreaming<AdditionalChatOptions>,
): Promise<
- ChatResponse<AdditionalMessageOptions> | AsyncIterable<ChatResponseChunk>
+ | ChatResponse<AdditionalMessageOptions>
+ | AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>
>;
}
@@ -56,24 +44,24 @@ export interface LLMChat<
* Unified language model interface
*/
export interface LLM<
- AdditionalChatOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
+ AdditionalChatOptions extends object = object,
+ AdditionalMessageOptions extends object = object,
> extends LLMChat<AdditionalChatOptions> {
metadata: LLMMetadata;
/**
* Get a chat response from the LLM
*/
chat(
- params: LLMChatParamsStreaming<AdditionalChatOptions>,
+ params: LLMChatParamsStreaming<
+ AdditionalChatOptions,
+ AdditionalMessageOptions
+ >,
): Promise<AsyncIterable<ChatResponseChunk>>;
chat(
- params: LLMChatParamsNonStreaming<AdditionalChatOptions>,
+ params: LLMChatParamsNonStreaming<
+ AdditionalChatOptions,
+ AdditionalMessageOptions
+ >,
): Promise<ChatResponse<AdditionalMessageOptions>>;
/**
@@ -87,48 +75,16 @@ export interface LLM<
): Promise<CompletionResponse>;
}
-// todo: remove "generic", "function", "memory";
-export type MessageType =
- | "user"
- | "assistant"
- | "system"
- /**
- * @deprecated
- */
- | "generic"
- /**
- * @deprecated
- */
- | "function"
- /**
- * @deprecated
- */
- | "memory"
- | "tool";
-
-export type ChatMessage<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
-> =
- AdditionalMessageOptions extends Record<string, unknown>
- ? {
- content: MessageContent;
- role: MessageType;
- options?: AdditionalMessageOptions;
- }
- : {
- content: MessageContent;
- role: MessageType;
- options: AdditionalMessageOptions;
- };
+export type MessageType = "user" | "assistant" | "system" | "memory";
+
+export type ChatMessage<AdditionalMessageOptions extends object = object> = {
+ content: MessageContent;
+ role: MessageType;
+ options?: undefined | AdditionalMessageOptions;
+};
export interface ChatResponse<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
+ AdditionalMessageOptions extends object = object,
> {
message: ChatMessage<AdditionalMessageOptions>;
/**
@@ -140,22 +96,12 @@ export interface ChatResponse<
}
export type ChatResponseChunk<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
-> =
- AdditionalMessageOptions extends Record<string, unknown>
- ? {
- raw: object | null;
- delta: string;
- options?: AdditionalMessageOptions;
- }
- : {
- raw: object | null;
- delta: string;
- options: AdditionalMessageOptions;
- };
+ AdditionalMessageOptions extends object = object,
+> = {
+ raw: object | null;
+ delta: string;
+ options?: undefined | AdditionalMessageOptions;
+};
export interface CompletionResponse {
text: string;
@@ -177,36 +123,25 @@ export type LLMMetadata = {
};
export interface LLMChatParamsBase<
- AdditionalChatOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
+ AdditionalChatOptions extends object = object,
+ AdditionalMessageOptions extends object = object,
> {
messages: ChatMessage<AdditionalMessageOptions>[];
additionalChatOptions?: AdditionalChatOptions;
tools?: BaseTool[];
- additionalKwargs?: Record<string, unknown>;
}
export interface LLMChatParamsStreaming<
- AdditionalChatOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
-> extends LLMChatParamsBase<AdditionalChatOptions> {
+ AdditionalChatOptions extends object = object,
+ AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
stream: true;
}
export interface LLMChatParamsNonStreaming<
- AdditionalChatOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
-> extends LLMChatParamsBase<AdditionalChatOptions> {
+ AdditionalChatOptions extends object = object,
+ AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
stream?: false;
}
@@ -242,13 +177,29 @@ export type MessageContentDetail =
*/
export type MessageContent = string | MessageContentDetail[];
-interface Function {
- arguments: string;
+export type ToolCall = {
name: string;
-}
+ // for now, claude-3-opus will give object, gpt-3/4 will give string
+ // todo: unify this to always be an object
+ input: unknown;
+ id: string;
+};
-export interface MessageToolCall {
+export type ToolResult = {
id: string;
- function: Function;
- type: "function";
-}
+ result: string;
+ isError: boolean;
+};
+
+export type ToolCallOptions = {
+ toolCall: ToolCall;
+};
+
+export type ToolResultOptions = {
+ toolResult: ToolResult;
+};
+
+export type ToolCallLLMMessageOptions =
+ | ToolResultOptions
+ | ToolCallOptions
+ | {};
diff --git a/packages/core/src/llm/utils.ts b/packages/core/src/llm/utils.ts
index 44f256cbc..4d90a66b0 100644
--- a/packages/core/src/llm/utils.ts
+++ b/packages/core/src/llm/utils.ts
@@ -61,14 +61,24 @@ export function extractText(message: MessageContent): string {
/**
* @internal
*/
-export function wrapLLMEvent(
- originalMethod: LLMChat["chat"],
+export function wrapLLMEvent<
+ AdditionalChatOptions extends object = object,
+ AdditionalMessageOptions extends object = object,
+>(
+ originalMethod: LLMChat<
+ AdditionalChatOptions,
+ AdditionalMessageOptions
+ >["chat"],
_context: ClassMethodDecoratorContext,
) {
return async function withLLMEvent(
- this: LLM,
- ...params: Parameters<LLMChat["chat"]>
- ): ReturnType<LLMChat["chat"]> {
+ this: LLM<AdditionalChatOptions, AdditionalMessageOptions>,
+ ...params: Parameters<
+ LLMChat<AdditionalChatOptions, AdditionalMessageOptions>["chat"]
+ >
+ ): ReturnType<
+ LLMChat<AdditionalChatOptions, AdditionalMessageOptions>["chat"]
+ > {
const id = randomUUID();
getCallbackManager().dispatchEvent("llm-start", {
payload: {
diff --git a/packages/core/src/memory/ChatMemoryBuffer.ts b/packages/core/src/memory/ChatMemoryBuffer.ts
index 54367a84a..eda6bbb31 100644
--- a/packages/core/src/memory/ChatMemoryBuffer.ts
+++ b/packages/core/src/memory/ChatMemoryBuffer.ts
@@ -1,3 +1,4 @@
+import type { ChatHistory } from "../ChatHistory.js";
import type { ChatMessage, LLM } from "../llm/index.js";
import { SimpleChatStore } from "../storage/chatStore/SimpleChatStore.js";
import type { BaseChatStore } from "../storage/chatStore/types.js";
@@ -6,25 +7,17 @@ import type { BaseMemory } from "./types.js";
const DEFAULT_TOKEN_LIMIT_RATIO = 0.75;
const DEFAULT_TOKEN_LIMIT = 3000;
-type ChatMemoryBufferParams<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
-> = {
- tokenLimit?: number;
- chatStore?: BaseChatStore<AdditionalMessageOptions>;
- chatStoreKey?: string;
- chatHistory?: ChatMessage<AdditionalMessageOptions>[];
- llm?: LLM<Record<string, unknown>, AdditionalMessageOptions>;
-};
-
-export class ChatMemoryBuffer<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
-> implements BaseMemory<AdditionalMessageOptions>
+type ChatMemoryBufferParams<AdditionalMessageOptions extends object = object> =
+ {
+ tokenLimit?: number;
+ chatStore?: BaseChatStore<AdditionalMessageOptions>;
+ chatStoreKey?: string;
+ chatHistory?: ChatHistory<AdditionalMessageOptions>;
+ llm?: LLM<object, AdditionalMessageOptions>;
+ };
+
+export class ChatMemoryBuffer<AdditionalMessageOptions extends object = object>
+ implements BaseMemory<AdditionalMessageOptions>
{
tokenLimit: number;
@@ -47,7 +40,7 @@ export class ChatMemoryBuffer<
}
if (init?.chatHistory) {
- this.chatStore.setMessages(this.chatStoreKey, init.chatHistory);
+ this.chatStore.setMessages(this.chatStoreKey, init.chatHistory.messages);
}
}
diff --git a/packages/core/src/memory/types.ts b/packages/core/src/memory/types.ts
index 5a19c431b..8af16e5a1 100644
--- a/packages/core/src/memory/types.ts
+++ b/packages/core/src/memory/types.ts
@@ -1,11 +1,6 @@
import type { ChatMessage } from "../llm/index.js";
-export interface BaseMemory<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
-> {
+export interface BaseMemory<AdditionalMessageOptions extends object = object> {
tokenLimit: number;
get(...args: unknown[]): ChatMessage<AdditionalMessageOptions>[];
getAll(): ChatMessage<AdditionalMessageOptions>[];
diff --git a/packages/core/src/objects/base.ts b/packages/core/src/objects/base.ts
index b7bd96399..3ef6832d4 100644
--- a/packages/core/src/objects/base.ts
+++ b/packages/core/src/objects/base.ts
@@ -50,7 +50,7 @@ export abstract class BaseObjectNodeMapping {
type QueryType = string;
-export class ObjectRetriever {
+export class ObjectRetriever<T = unknown> {
_retriever: BaseRetriever;
_objectNodeMapping: BaseObjectNodeMapping;
@@ -68,7 +68,7 @@ export class ObjectRetriever {
}
// Translating the retrieve method
- async retrieve(strOrQueryBundle: QueryType): Promise<any> {
+ async retrieve(strOrQueryBundle: QueryType): Promise<T[]> {
const nodes = await this.retriever.retrieve({ query: strOrQueryBundle });
const objs = nodes.map((n) => this._objectNodeMapping.fromNode(n.node));
return objs;
@@ -180,7 +180,7 @@ export class ObjectIndex {
return this._objectNodeMapping.objNodeMapping();
}
- async asRetriever(kwargs: any): Promise<ObjectRetriever> {
+ async asRetriever(kwargs: any): Promise<ObjectRetriever<any>> {
return new ObjectRetriever(
this._index.asRetriever(kwargs),
this._objectNodeMapping,
diff --git a/packages/core/src/storage/chatStore/SimpleChatStore.ts b/packages/core/src/storage/chatStore/SimpleChatStore.ts
index 4c09f2550..4f6df4718 100644
--- a/packages/core/src/storage/chatStore/SimpleChatStore.ts
+++ b/packages/core/src/storage/chatStore/SimpleChatStore.ts
@@ -6,10 +6,7 @@ import type { BaseChatStore } from "./types.js";
* This could lead to memory leaks if the messages are not properly cleaned up.
*/
export class SimpleChatStore<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
+ AdditionalMessageOptions extends object = Record<string, unknown>,
> implements BaseChatStore<AdditionalMessageOptions>
{
store: { [key: string]: ChatMessage<AdditionalMessageOptions>[] } = {};
diff --git a/packages/core/src/storage/chatStore/types.ts b/packages/core/src/storage/chatStore/types.ts
index 7b3a2acda..c809f91be 100644
--- a/packages/core/src/storage/chatStore/types.ts
+++ b/packages/core/src/storage/chatStore/types.ts
@@ -1,10 +1,7 @@
import type { ChatMessage } from "../../llm/index.js";
export interface BaseChatStore<
- AdditionalMessageOptions extends Record<string, unknown> = Record<
- string,
- unknown
- >,
+ AdditionalMessageOptions extends object = object,
> {
setMessages(
key: string,
diff --git a/packages/core/src/tools/utils.ts b/packages/core/src/tools/utils.ts
index 3c19c20a4..afc5280aa 100644
--- a/packages/core/src/tools/utils.ts
+++ b/packages/core/src/tools/utils.ts
@@ -1,27 +1,33 @@
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
import type { BaseTool } from "../types.js";
import { ToolOutput } from "./types.js";
export async function callToolWithErrorHandling(
tool: BaseTool,
- inputDict: { [key: string]: any },
+ input: unknown,
): Promise<ToolOutput> {
if (!tool.call) {
return new ToolOutput(
"Error: Tool does not have a call function.",
tool.metadata.name,
- { kwargs: inputDict },
+ input,
null,
);
}
try {
- const value = await tool.call(inputDict);
- return new ToolOutput(value, tool.metadata.name, inputDict, value);
- } catch (e) {
- return new ToolOutput(
- `Error: ${e}`,
- tool.metadata.name,
- { kwargs: inputDict },
- e,
+ getCallbackManager().dispatchEvent("llm-tool-call", {
+ payload: {
+ toolCall: {
+ name: tool.metadata.name,
+ input,
+ },
+ },
+ });
+ const value = await tool.call(
+ typeof input === "string" ? JSON.parse(input) : input,
);
+ return new ToolOutput(value, tool.metadata.name, input, value);
+ } catch (e) {
+ return new ToolOutput(`Error: ${e}`, tool.metadata.name, input, e);
}
}
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index f1b0073e3..a51b1182e 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -67,7 +67,7 @@ export interface BaseTool<Input = any> {
Input extends Known ? ToolMetadata<JSONSchemaType<Input>> : ToolMetadata;
}
-export type ToolWithCall<Input = unknown> = Omit<BaseTool<Input>, "call"> & {
+export type BaseToolWithCall<Input = any> = Omit<BaseTool<Input>, "call"> & {
call: NonNullable<Pick<BaseTool<Input>, "call">["call"]>;
};
diff --git a/packages/core/tests/type.test.ts b/packages/core/tests/type.test.ts
new file mode 100644
index 000000000..3612f0c00
--- /dev/null
+++ b/packages/core/tests/type.test.ts
@@ -0,0 +1,56 @@
+import type { ChatMessage, MessageContent, MessageType } from "llamaindex";
+import { expectTypeOf, test } from "vitest";
+import type { ChatResponse } from "../src/index.js";
+
+test("chat message type", () => {
+ // if generic is not provided, `options` is not required
+ expectTypeOf<ChatMessage>().toMatchTypeOf<{
+ content: MessageContent;
+ role: MessageType;
+ }>();
+ expectTypeOf<ChatMessage>().toMatchTypeOf<{
+ content: MessageContent;
+ role: MessageType;
+ options?: object;
+ }>();
+ expectTypeOf<ChatMessage>().not.toMatchTypeOf<{
+ content: MessageContent;
+ role: MessageType;
+ options: Record<string, unknown>;
+ }>();
+ type Options = {
+ a: string;
+ b: number;
+ };
+ expectTypeOf<ChatMessage<Options>>().toMatchTypeOf<{
+ content: MessageContent;
+ role: MessageType;
+ options?: Options;
+ }>();
+});
+
+test("chat response type", () => {
+ // if generic is not provided, `options` is not required
+ expectTypeOf<ChatResponse>().toMatchTypeOf<{
+ message: ChatMessage;
+ raw: object | null;
+ }>();
+ expectTypeOf<ChatResponse>().toMatchTypeOf<{
+ message: ChatMessage;
+ raw: object | null;
+ options?: Record<string, unknown>;
+ }>();
+ expectTypeOf<ChatResponse>().not.toMatchTypeOf<{
+ message: ChatMessage;
+ raw: object | null;
+ options: Record<string, unknown>;
+ }>();
+ type Options = {
+ a: string;
+ b: number;
+ };
+ expectTypeOf<ChatResponse<Options>>().toMatchTypeOf<{
+ message: ChatMessage<Options>;
+ raw: object | null;
+ }>();
+});
--
GitLab