diff --git a/.changeset/cuddly-radios-fetch.md b/.changeset/cuddly-radios-fetch.md new file mode 100644 index 0000000000000000000000000000000000000000..12af763f63ae7612436b16e1d0bdb6002208de87 --- /dev/null +++ b/.changeset/cuddly-radios-fetch.md @@ -0,0 +1,5 @@ +--- +"llamaindex": patch +--- + +fix: anthroipic image input diff --git a/packages/llamaindex/src/llm/anthropic.ts b/packages/llamaindex/src/llm/anthropic.ts index f1662f244ad597917e1c90f9c41164732139f223..7ee5362000f0021ecb2bcf2c4be16407847932f2 100644 --- a/packages/llamaindex/src/llm/anthropic.ts +++ b/packages/llamaindex/src/llm/anthropic.ts @@ -13,7 +13,10 @@ import type { TextBlock, TextBlockParam, } from "@anthropic-ai/sdk/resources/index"; -import type { MessageParam } from "@anthropic-ai/sdk/resources/messages"; +import type { + ImageBlockParam, + MessageParam, +} from "@anthropic-ai/sdk/resources/messages"; import { getEnv } from "@llamaindex/env"; import _ from "lodash"; import type { BaseTool } from "../types.js"; @@ -214,7 +217,32 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> { } return { - content: extractText(message.content), + content: + typeof message.content === "string" + ? message.content + : message.content.map( + (content): TextBlockParam | ImageBlockParam => + content.type === "text" + ? { + type: "text", + text: content.text, + } + : { + type: "image", + source: { + data: content.image_url.url.substring( + content.image_url.url.indexOf(",") + 1, + ), + media_type: + `image/${content.image_url.url.substring("data:image/".length, content.image_url.url.indexOf(";base64"))}` as + | "image/jpeg" + | "image/png" + | "image/gif" + | "image/webp", + type: "base64", + }, + }, + ), role: message.role as "user" | "assistant", } satisfies MessageParam; }); diff --git a/packages/llamaindex/tests/llm/index.test.ts b/packages/llamaindex/tests/llm/index.test.ts index fbde9f13caef427d947284dc8e2c262f2e09d94e..4a88cf9c5ed01ec74cfcd6ffd64c1a0ef12f6331 100644 --- a/packages/llamaindex/tests/llm/index.test.ts +++ b/packages/llamaindex/tests/llm/index.test.ts @@ -62,5 +62,51 @@ describe("Anthropic llm", () => { role: "user", }, ]); + + expect( + anthropic.formatMessages([ + { + content: "You are a helpful assistant.", + role: "assistant", + }, + { + content: [ + { + text: "What do you see in the image?", + type: "text", + }, + { + type: "image_url", + image_url: { + url: `data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAQDAwQDAwQEAwQFBAQFBgoHBgYGBg0JCggKDw0QEA8NDw4RExgUERIXEg4PFRwVFxkZGxsbEBQdHx0aHxgaGxr/2wBDAQQFBQYFBgwHBwwaEQ8RGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhr/wAARCAAgACADASIAAhEBAxEB/8QAGAABAAMBAAAAAAAAAAAAAAAACAQHCQb/xAAvEAABAgUCBAUDBQEAAAAAAAACAQMEBQYHERIhAAgTYSIxMkJxI2KCFBVBUVKh/8QAGAEAAwEBAAAAAAAAAAAAAAAAAwQFAQL/xAAnEQABBAECAwkAAAAAAAAAAAACAQMEEQAFMiExYRITFCJBcXKBof/aAAwDAQACEQMRAD8Aufmb5mnbWREFRdvIMZ3cWcaBh2NHUGEFwtIKQp63CX0h+S7YQgRzGSq6kgqGAS8NQRc6fmkIMWwSxJEyP+m0bwggQr5iIom6KnnxXty61jK+uJUVUxzxm/M5g5EASr6G9WGwTsIIIp2FOHJfi0kyvzS9Cv0zGwEF+2whOAUY4a6mnm2lREURLPoTggNG5tS6xpmOT4GQptwNUZc6sbexzcZRVSTKTOgudMPEL0j7E2uQNOxIqcaYcqXNaxe2HKnauBiAraDZ6n0k0tTBpPNwE9pptqDP3DtlBC1Q8qNw5K4AwLEunYkWMwcYg6fnqoH/ADPHA2/qeZWquhJJ3pODmEhmg/qGl2XAloebL5HWK/K8dOMOM7xVPfJrMhmQiq0SFXOlyPc+jIq3lwakpeYNq27K491kfvbzls07ECiSdlThhWKvj1LLx0VVLWGqSBuFJ1jc3WBEUb8K4TUieHz3xni7ea3lSZvZDhUVImxAVtBso39VdLUe0nk2a+0030n+K7YUc95/J66tRIp3SVXUpGyUI7wvPxDBoJ/UaLIuIqtuInRwiiqp4z3XbBYr3cGp9P30zJXiSjk1HLsqdIvxvzV1q8ZtB3ppa5bkwZkDz7LsF09Qxgi0Roa6UUU1LnxYH5JP74D1LUjNrkXigabc6kZM5vPFZi3NPi3dVXnFT+EQUM17IvEi1tL1xUkcEHb+lo6duvRUO644wwSDpaPWgG7sAApIKqqqm4jvxo1yvcrjdoTiqtrQ2I+u5nr19ItbUA2a5IAX3GvuP8U2ypMS5pSwFC5peTtM0lnSkMWVVUJb48a+8//Z`, + }, + }, + ], + role: "user", + }, + ]), + ).toEqual([ + { + content: "You are a helpful assistant.", + role: "assistant", + }, + { + content: [ + { + text: "What do you see in the image?", + type: "text", + }, + { + source: { + data: "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAQDAwQDAwQEAwQFBAQFBgoHBgYGBg0JCggKDw0QEA8NDw4RExgUERIXEg4PFRwVFxkZGxsbEBQdHx0aHxgaGxr/2wBDAQQFBQYFBgwHBwwaEQ8RGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhr/wAARCAAgACADASIAAhEBAxEB/8QAGAABAAMBAAAAAAAAAAAAAAAACAQHCQb/xAAvEAABAgUCBAUDBQEAAAAAAAACAQMEBQYHERIhAAgTYSIxMkJxI2KCFBVBUVKh/8QAGAEAAwEBAAAAAAAAAAAAAAAAAwQFAQL/xAAnEQABBAECAwkAAAAAAAAAAAACAQMEEQAFMiExYRITFCJBcXKBof/aAAwDAQACEQMRAD8Aufmb5mnbWREFRdvIMZ3cWcaBh2NHUGEFwtIKQp63CX0h+S7YQgRzGSq6kgqGAS8NQRc6fmkIMWwSxJEyP+m0bwggQr5iIom6KnnxXty61jK+uJUVUxzxm/M5g5EASr6G9WGwTsIIIp2FOHJfi0kyvzS9Cv0zGwEF+2whOAUY4a6mnm2lREURLPoTggNG5tS6xpmOT4GQptwNUZc6sbexzcZRVSTKTOgudMPEL0j7E2uQNOxIqcaYcqXNaxe2HKnauBiAraDZ6n0k0tTBpPNwE9pptqDP3DtlBC1Q8qNw5K4AwLEunYkWMwcYg6fnqoH/ADPHA2/qeZWquhJJ3pODmEhmg/qGl2XAloebL5HWK/K8dOMOM7xVPfJrMhmQiq0SFXOlyPc+jIq3lwakpeYNq27K491kfvbzls07ECiSdlThhWKvj1LLx0VVLWGqSBuFJ1jc3WBEUb8K4TUieHz3xni7ea3lSZvZDhUVImxAVtBso39VdLUe0nk2a+0030n+K7YUc95/J66tRIp3SVXUpGyUI7wvPxDBoJ/UaLIuIqtuInRwiiqp4z3XbBYr3cGp9P30zJXiSjk1HLsqdIvxvzV1q8ZtB3ppa5bkwZkDz7LsF09Qxgi0Roa6UUU1LnxYH5JP74D1LUjNrkXigabc6kZM5vPFZi3NPi3dVXnFT+EQUM17IvEi1tL1xUkcEHb+lo6duvRUO644wwSDpaPWgG7sAApIKqqqm4jvxo1yvcrjdoTiqtrQ2I+u5nr19ItbUA2a5IAX3GvuP8U2ypMS5pSwFC5peTtM0lnSkMWVVUJb48a+8//Z", + media_type: "image/jpeg", + type: "base64", + }, + type: "image", + }, + ], + role: "user", + }, + ]); }); });