fix: Export imageToDataUrl for using images in chat (#1146)

.changeset/tasty-hornets-repeat.md

+---
+"llamaindex": patch
+---
+
+Export imageToDataUrl for using images in chat

examples/multimodal/chat.ts

+// call pnpm tsx multimodal/load.ts first to init the storage
+import { OpenAI, Settings, SimpleChatEngine, imageToDataUrl } from "llamaindex";
+import fs from "node:fs/promises";
+
+import path from "path";
+// Update llm
+Settings.llm = new OpenAI({ model: "gpt-4o-mini", maxTokens: 512 });
+
+async function main() {
+  const chatEngine = new SimpleChatEngine();
+
+  // Load the image and convert it to a data URL
+  const imagePath = path.join(__dirname, ".", "data", "60.jpg");
+
+  // 1. you can read the buffer from the file
+  const imageBuffer = await fs.readFile(imagePath);
+  const dataUrl = await imageToDataUrl(imageBuffer);
+  // or 2. you can just pass the file path to the imageToDataUrl function
+  // const dataUrl = await imageToDataUrl(imagePath);
+
+  // Update the image_url in the chat message
+  const response = await chatEngine.chat({
+    message: [
+      {
+        type: "text",
+        text: "What is in this image?",
+      },
+      {
+        type: "image_url",
+        image_url: {
+          url: dataUrl,
+        },
+      },
+    ],
+  });
+
+  console.log(response.message.content);
+}
+
+main().catch(console.error);

packages/llamaindex/src/index.edge.ts

@@ -37,6 +37,7 @@ export * from "./evaluation/index.js";
 export * from "./extractors/index.js";
 export * from "./indices/index.js";
 export * from "./ingestion/index.js";
+export { imageToDataUrl } from "./internal/utils.js";
 export * from "./llm/index.js";
 export * from "./nodeParsers/index.js";
 export * from "./objects/index.js";

packages/llamaindex/src/internal/utils.ts

@@ -182,7 +182,9 @@ export function stringToImage(input: string): ImageType {
   }
 }
 
-export async function imageToDataUrl(input: ImageType): Promise<string> {
+export async function imageToDataUrl(
+  input: ImageType | Uint8Array,
+): Promise<string> {
   // first ensure, that the input is a Blob
   if (
     (input instanceof URL && input.protocol === "file:") ||
@@ -196,6 +198,8 @@ export async function imageToDataUrl(input: ImageType): Promise<string> {
   } else if (!(input instanceof Blob)) {
     if (input instanceof URL) {
       throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
+    } else if (input instanceof Uint8Array) {
+      input = new Blob([input]); // convert Uint8Array to Blob
     } else {
       throw new Error(`Unsupported input type: ${typeof input}`);
     }