diff --git a/.changeset/spicy-colts-dream.md b/.changeset/spicy-colts-dream.md new file mode 100644 index 0000000000000000000000000000000000000000..392dca0f4080ff720d367d8d53adfc0fe3cebc8a --- /dev/null +++ b/.changeset/spicy-colts-dream.md @@ -0,0 +1,5 @@ +--- +"create-llama": patch +--- + +feat: support showing image on chat message diff --git a/packages/create-llama/templates/types/streaming/express/src/controllers/chat.controller.ts b/packages/create-llama/templates/types/streaming/express/src/controllers/chat.controller.ts index e82658016cbfb036f31485a79d8e83a2c8fe376f..9d1eb0c69b50ae4f258e5321eec1ffcc5d0bda1e 100644 --- a/packages/create-llama/templates/types/streaming/express/src/controllers/chat.controller.ts +++ b/packages/create-llama/templates/types/streaming/express/src/controllers/chat.controller.ts @@ -35,7 +35,7 @@ export const chat = async (req: Request, res: Response) => { } const llm = new OpenAI({ - model: process.env.MODEL || "gpt-3.5-turbo", + model: (process.env.MODEL as any) || "gpt-3.5-turbo", }); const chatEngine = await createChatEngine(llm); @@ -54,9 +54,24 @@ export const chat = async (req: Request, res: Response) => { }); // Return a stream, which can be consumed by the Vercel/AI client - const stream = LlamaIndexStream(response); + const { stream, data: streamData } = LlamaIndexStream(response, { + parserOptions: { + image_url: data?.imageUrl, + }, + }); - streamToResponse(stream, res); + // Pipe LlamaIndexStream to response + const processedStream = stream.pipeThrough(streamData.stream); + return streamToResponse(processedStream, res, { + headers: { + // response MUST have the `X-Experimental-Stream-Data: 'true'` header + // so that the client uses the correct parsing logic, see + // https://sdk.vercel.ai/docs/api-reference/stream-data#on-the-server + "X-Experimental-Stream-Data": "true", + "Content-Type": "text/plain; charset=utf-8", + "Access-Control-Expose-Headers": "X-Experimental-Stream-Data", + }, + }); } catch (error) { console.error("[LlamaIndex]", error); return res.status(500).json({ diff --git a/packages/create-llama/templates/types/streaming/express/src/controllers/llamaindex-stream.ts b/packages/create-llama/templates/types/streaming/express/src/controllers/llamaindex-stream.ts index e86c7626f09e2beca9ce15d922a37679c9906e43..6ddd8eae68bc199188d07a0af8f27a12b2a6abb3 100644 --- a/packages/create-llama/templates/types/streaming/express/src/controllers/llamaindex-stream.ts +++ b/packages/create-llama/templates/types/streaming/express/src/controllers/llamaindex-stream.ts @@ -1,19 +1,45 @@ import { + JSONValue, createCallbacksTransformer, createStreamDataTransformer, + experimental_StreamData, trimStartOfStreamHelper, type AIStreamCallbacksAndOptions, } from "ai"; import { Response } from "llamaindex"; -function createParser(res: AsyncIterable<Response>) { +type ParserOptions = { + image_url?: string; +}; + +function createParser( + res: AsyncIterable<Response>, + data: experimental_StreamData, + opts?: ParserOptions, +) { const it = res[Symbol.asyncIterator](); const trimStartOfStream = trimStartOfStreamHelper(); return new ReadableStream<string>({ + start() { + // if image_url is provided, send it via the data stream + if (opts?.image_url) { + const message: JSONValue = { + type: "image_url", + image_url: { + url: opts.image_url, + }, + }; + data.append(message); + } else { + data.append({}); // send an empty image response for the user's message + } + }, async pull(controller): Promise<void> { const { value, done } = await it.next(); if (done) { controller.close(); + data.append({}); // send an empty image response for the assistant's message + data.close(); return; } @@ -27,11 +53,16 @@ function createParser(res: AsyncIterable<Response>) { export function LlamaIndexStream( res: AsyncIterable<Response>, - callbacks?: AIStreamCallbacksAndOptions, -): ReadableStream { - return createParser(res) - .pipeThrough(createCallbacksTransformer(callbacks)) - .pipeThrough( - createStreamDataTransformer(callbacks?.experimental_streamData), - ); + opts?: { + callbacks?: AIStreamCallbacksAndOptions; + parserOptions?: ParserOptions; + }, +): { stream: ReadableStream; data: experimental_StreamData } { + const data = new experimental_StreamData(); + return { + stream: createParser(res, data, opts?.parserOptions) + .pipeThrough(createCallbacksTransformer(opts?.callbacks)) + .pipeThrough(createStreamDataTransformer(true)), + data, + }; }