From 2091fea2b41d1cf802053c0df1bfa563024541ab Mon Sep 17 00:00:00 2001 From: Thuc Pham <51660321+thucpn@users.noreply.github.com> Date: Thu, 6 Jun 2024 14:24:31 +0700 Subject: [PATCH] feat: display attachments in user messages (#114) * use same csv card for message and upload box * do not send csv and image data back to client * fix: use LLM_MAX_TOKENS --------- Co-authored-by: leehuwuj <leehuwuj@gmail.com> Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de> --- .../engines/python/agent/tools/interpreter.py | 2 +- .../components/ui/html/chat/chat-input.tsx | 1 + .../src/controllers/engine/settings.ts | 4 +- .../src/controllers/llamaindex-stream.ts | 11 +-- .../express/src/controllers/stream-helper.ts | 20 ----- .../streaming/fastapi/app/api/routers/chat.py | 5 -- .../fastapi/app/api/routers/models.py | 19 ---- .../nextjs/app/api/chat/engine/settings.ts | 4 +- .../nextjs/app/api/chat/llamaindex-stream.ts | 11 +-- .../nextjs/app/api/chat/stream-helper.ts | 20 ----- .../nextjs/app/components/chat-section.tsx | 3 + .../app/components/ui/chat/chat-input.tsx | 71 ++++++++++++--- .../app/components/ui/chat/chat-message.tsx | 6 +- .../app/components/ui/chat/chat.interface.ts | 4 + .../app/components/ui/chat/csv-content.tsx | 13 ++- .../components/ui/chat/widgets/CsvDialog.tsx | 62 ------------- .../app/components/ui/upload-csv-preview.tsx | 90 ++++++++++++++----- 17 files changed, 150 insertions(+), 196 deletions(-) delete mode 100644 templates/types/streaming/nextjs/app/components/ui/chat/widgets/CsvDialog.tsx diff --git a/templates/components/engines/python/agent/tools/interpreter.py b/templates/components/engines/python/agent/tools/interpreter.py index bc913935..e2a5300a 100644 --- a/templates/components/engines/python/agent/tools/interpreter.py +++ b/templates/components/engines/python/agent/tools/interpreter.py @@ -96,7 +96,7 @@ class E2BCodeInterpreter: exec = interpreter.notebook.exec_cell(code) if exec.error: - output = E2BToolOutput(is_error=True, logs=[exec.error]) + output = E2BToolOutput(is_error=True, logs=exec.logs, results=[]) else: if len(exec.results) == 0: output = E2BToolOutput(is_error=False, logs=exec.logs, results=[]) diff --git a/templates/components/ui/html/chat/chat-input.tsx b/templates/components/ui/html/chat/chat-input.tsx index e9e11c86..562d08a0 100644 --- a/templates/components/ui/html/chat/chat-input.tsx +++ b/templates/components/ui/html/chat/chat-input.tsx @@ -15,6 +15,7 @@ export interface ChatInputProps { handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void; isLoading: boolean; messages: Message[]; + setInput?: (input: string) => void; } export default function ChatInput(props: ChatInputProps) { diff --git a/templates/types/streaming/express/src/controllers/engine/settings.ts b/templates/types/streaming/express/src/controllers/engine/settings.ts index a46feb7e..d2ccf190 100644 --- a/templates/types/streaming/express/src/controllers/engine/settings.ts +++ b/templates/types/streaming/express/src/controllers/engine/settings.ts @@ -45,7 +45,9 @@ export const initSettings = async () => { function initOpenAI() { Settings.llm = new OpenAI({ model: process.env.MODEL ?? "gpt-3.5-turbo", - maxTokens: 512, + maxTokens: process.env.LLM_MAX_TOKENS + ? Number(process.env.LLM_MAX_TOKENS) + : undefined, }); Settings.embedModel = new OpenAIEmbedding({ model: process.env.EMBEDDING_MODEL, diff --git a/templates/types/streaming/express/src/controllers/llamaindex-stream.ts b/templates/types/streaming/express/src/controllers/llamaindex-stream.ts index 779b6a65..a8c055a4 100644 --- a/templates/types/streaming/express/src/controllers/llamaindex-stream.ts +++ b/templates/types/streaming/express/src/controllers/llamaindex-stream.ts @@ -14,12 +14,7 @@ import { } from "llamaindex"; import { AgentStreamChatResponse } from "llamaindex/agent/base"; -import { - CsvFile, - appendCsvData, - appendImageData, - appendSourceData, -} from "./stream-helper"; +import { CsvFile, appendSourceData } from "./stream-helper"; type LlamaIndexResponse = | AgentStreamChatResponse<ToolCallLLMMessageOptions> @@ -75,10 +70,6 @@ function createParser( let sourceNodes: NodeWithScore<Metadata>[] | undefined; return new ReadableStream<string>({ - start() { - appendImageData(data, opts?.imageUrl); - appendCsvData(data, opts?.csvFiles); - }, async pull(controller): Promise<void> { const { value, done } = await it.next(); if (done) { diff --git a/templates/types/streaming/express/src/controllers/stream-helper.ts b/templates/types/streaming/express/src/controllers/stream-helper.ts index 05988a2f..15527ba8 100644 --- a/templates/types/streaming/express/src/controllers/stream-helper.ts +++ b/templates/types/streaming/express/src/controllers/stream-helper.ts @@ -7,16 +7,6 @@ import { ToolOutput, } from "llamaindex"; -export function appendImageData(data: StreamData, imageUrl?: string) { - if (!imageUrl) return; - data.appendMessageAnnotation({ - type: "image", - data: { - url: imageUrl, - }, - }); -} - function getNodeUrl(metadata: Metadata) { const url = metadata["URL"]; if (url) return url; @@ -128,13 +118,3 @@ export type CsvFile = { filesize: number; id: string; }; - -export function appendCsvData(data: StreamData, csvFiles?: CsvFile[]) { - if (!csvFiles) return; - data.appendMessageAnnotation({ - type: "csv", - data: { - csvFiles, - }, - }); -} diff --git a/templates/types/streaming/fastapi/app/api/routers/chat.py b/templates/types/streaming/fastapi/app/api/routers/chat.py index 5ff2ce05..446f9cc0 100644 --- a/templates/types/streaming/fastapi/app/api/routers/chat.py +++ b/templates/types/streaming/fastapi/app/api/routers/chat.py @@ -35,11 +35,6 @@ async def chat( chat_engine.callback_manager.handlers.append(event_handler) # type: ignore async def content_generator(): - # Yield the additional data - if data.data is not None: - for data_response in data.get_additional_data_response(): - yield VercelStreamResponse.convert_data(data_response) - # Yield the text response async def _chat_response_generator(): response = await chat_engine.astream_chat(last_message_content, messages) diff --git a/templates/types/streaming/fastapi/app/api/routers/models.py b/templates/types/streaming/fastapi/app/api/routers/models.py index b64e86f0..5b1ebca5 100644 --- a/templates/types/streaming/fastapi/app/api/routers/models.py +++ b/templates/types/streaming/fastapi/app/api/routers/models.py @@ -51,19 +51,6 @@ class DataParserOptions(BaseModel): [f"```csv\n{csv_file.content}\n```" for csv_file in self.csv_files] ) - def to_response_data(self) -> list[dict] | None: - output = [] - if self.csv_files is not None and len(self.csv_files) > 0: - output.append( - { - "type": "csv", - "data": { - "csvFiles": [csv_file.dict() for csv_file in self.csv_files] - }, - } - ) - return output if len(output) > 0 else None - class ChatData(BaseModel): data: DataParserOptions | None = Field( @@ -107,12 +94,6 @@ class ChatData(BaseModel): for message in self.messages[:-1] ] - def get_additional_data_response(self) -> list[dict] | None: - """ - Get the additional data - """ - return self.data.to_response_data() - def is_last_message_from_user(self) -> bool: return self.messages[-1].role == MessageRole.USER diff --git a/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts b/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts index 9b7bb00e..f8bfd7be 100644 --- a/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts +++ b/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts @@ -45,7 +45,9 @@ export const initSettings = async () => { function initOpenAI() { Settings.llm = new OpenAI({ model: process.env.MODEL ?? "gpt-3.5-turbo", - maxTokens: 512, + maxTokens: process.env.LLM_MAX_TOKENS + ? Number(process.env.LLM_MAX_TOKENS) + : undefined, }); Settings.embedModel = new OpenAIEmbedding({ model: process.env.EMBEDDING_MODEL, diff --git a/templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts b/templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts index 779b6a65..a8c055a4 100644 --- a/templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts +++ b/templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts @@ -14,12 +14,7 @@ import { } from "llamaindex"; import { AgentStreamChatResponse } from "llamaindex/agent/base"; -import { - CsvFile, - appendCsvData, - appendImageData, - appendSourceData, -} from "./stream-helper"; +import { CsvFile, appendSourceData } from "./stream-helper"; type LlamaIndexResponse = | AgentStreamChatResponse<ToolCallLLMMessageOptions> @@ -75,10 +70,6 @@ function createParser( let sourceNodes: NodeWithScore<Metadata>[] | undefined; return new ReadableStream<string>({ - start() { - appendImageData(data, opts?.imageUrl); - appendCsvData(data, opts?.csvFiles); - }, async pull(controller): Promise<void> { const { value, done } = await it.next(); if (done) { diff --git a/templates/types/streaming/nextjs/app/api/chat/stream-helper.ts b/templates/types/streaming/nextjs/app/api/chat/stream-helper.ts index 05988a2f..15527ba8 100644 --- a/templates/types/streaming/nextjs/app/api/chat/stream-helper.ts +++ b/templates/types/streaming/nextjs/app/api/chat/stream-helper.ts @@ -7,16 +7,6 @@ import { ToolOutput, } from "llamaindex"; -export function appendImageData(data: StreamData, imageUrl?: string) { - if (!imageUrl) return; - data.appendMessageAnnotation({ - type: "image", - data: { - url: imageUrl, - }, - }); -} - function getNodeUrl(metadata: Metadata) { const url = metadata["URL"]; if (url) return url; @@ -128,13 +118,3 @@ export type CsvFile = { filesize: number; id: string; }; - -export function appendCsvData(data: StreamData, csvFiles?: CsvFile[]) { - if (!csvFiles) return; - data.appendMessageAnnotation({ - type: "csv", - data: { - csvFiles, - }, - }); -} diff --git a/templates/types/streaming/nextjs/app/components/chat-section.tsx b/templates/types/streaming/nextjs/app/components/chat-section.tsx index 1f9ecf93..9a4ea6b9 100644 --- a/templates/types/streaming/nextjs/app/components/chat-section.tsx +++ b/templates/types/streaming/nextjs/app/components/chat-section.tsx @@ -15,6 +15,7 @@ export default function ChatSection() { reload, stop, append, + setInput, } = useChat({ api: chatAPI, headers: { @@ -42,6 +43,8 @@ export default function ChatSection() { handleInputChange={handleInputChange} isLoading={isLoading} messages={messages} + append={append} + setInput={setInput} /> </div> ); diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx index 128e4aef..531361be 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx @@ -1,6 +1,8 @@ +import { JSONValue } from "ai"; import { Loader2 } from "lucide-react"; import { useState } from "react"; import { v4 as uuidv4 } from "uuid"; +import { MessageAnnotation, MessageAnnotationType } from "."; import { Button } from "../button"; import FileUploader from "../file-uploader"; import { Input } from "../input"; @@ -19,6 +21,8 @@ export default function ChatInput( | "handleSubmit" | "handleInputChange" | "messages" + | "setInput" + | "append" >, ) { const [imageUrl, setImageUrl] = useState<string | null>(null); @@ -26,23 +30,61 @@ export default function ChatInput( props.messages, ); - const onSubmit = (e: React.FormEvent<HTMLFormElement>) => { + const getAttachments = () => { + if (!imageUrl && files.length === 0) return undefined; + const annotations: MessageAnnotation[] = []; if (imageUrl) { - props.handleSubmit(e, { - data: { imageUrl: imageUrl }, + annotations.push({ + type: MessageAnnotationType.IMAGE, + data: { url: imageUrl }, }); - setImageUrl(null); - return; } - if (files.length > 0) { - props.handleSubmit(e, { - data: { csvFiles: files }, + annotations.push({ + type: MessageAnnotationType.CSV, + data: { + csvFiles: files.map((file) => ({ + id: file.id, + content: file.content, + filename: file.filename, + filesize: file.filesize, + type: "available", + })), + }, }); - resetUploadedFiles(); - return; } + return annotations as JSONValue[]; + }; + // default submit function does not handle including annotations in the message + // so we need to use append function to submit new message with annotations + const submitWithAttachment = ( + e: React.FormEvent<HTMLFormElement>, + attachments: JSONValue[] | undefined, + ) => { + e.preventDefault(); + props.append!( + { + content: props.input, + role: "user", + createdAt: new Date(), + annotations: attachments, + }, + { + data: { imageUrl, csvFiles: files }, + }, + ); + setImageUrl(null); + resetUploadedFiles(); + props.setInput!(""); + }; + + const onSubmit = (e: React.FormEvent<HTMLFormElement>) => { + const attachments = getAttachments(); + if (attachments) { + submitWithAttachment(e, attachments); + return; + } props.handleSubmit(e); }; @@ -82,6 +124,10 @@ export default function ChatInput( return await handleUploadImageFile(file); } if (file.type === "text/csv") { + if (files.length > 0) { + alert("You can only upload one csv file at a time."); + return; + } return await handleUploadCsvFile(file); } props.onFileUpload?.(file); @@ -111,8 +157,7 @@ export default function ChatInput( return ( <UploadCsvPreview key={csv.id} - filename={csv.filename} - filesize={csv.filesize} + csv={csv} onRemove={() => removeFile(csv)} isNew={csv.type === "new_upload"} /> @@ -135,7 +180,7 @@ export default function ChatInput( onFileUpload={handleUploadFile} onFileError={props.onFileError} /> - <Button type="submit" disabled={props.isLoading}> + <Button type="submit" disabled={props.isLoading || !props.input.trim()}> Send message </Button> </div> diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message.tsx index fcd56306..7d346a7e 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat-message.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat-message.tsx @@ -60,7 +60,7 @@ function ChatMessageContent({ const contents: ContentDisplayConfig[] = [ { - order: -4, + order: 1, component: imageData[0] ? <ChatImage data={imageData[0]} /> : null, }, { @@ -71,7 +71,7 @@ function ChatMessageContent({ ) : null, }, { - order: -2, + order: 2, component: csvData[0] ? <CsvContent data={csvData[0]} /> : null, }, { @@ -83,7 +83,7 @@ function ChatMessageContent({ component: <Markdown content={message.content} />, }, { - order: 1, + order: 3, component: sourceData[0] ? <ChatSources data={sourceData[0]} /> : null, }, ]; diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/chat.interface.ts b/templates/types/streaming/nextjs/app/components/ui/chat/chat.interface.ts index 5fcbbdae..6b74d4fa 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/chat.interface.ts +++ b/templates/types/streaming/nextjs/app/components/ui/chat/chat.interface.ts @@ -15,7 +15,11 @@ export interface ChatHandler { stop?: () => void; onFileUpload?: (file: File) => Promise<void>; onFileError?: (errMsg: string) => void; + setInput?: (input: string) => void; append?: ( message: Message | Omit<Message, "id">, + ops?: { + data: any; + }, ) => Promise<string | null | undefined>; } diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/csv-content.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/csv-content.tsx index e4005cf2..1a84d978 100644 --- a/templates/types/streaming/nextjs/app/components/ui/chat/csv-content.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/chat/csv-content.tsx @@ -1,16 +1,13 @@ import { CsvData } from "."; -import CsvDialog from "./widgets/CsvDialog"; +import UploadCsvPreview from "../upload-csv-preview"; export default function CsvContent({ data }: { data: CsvData }) { if (!data.csvFiles.length) return null; return ( - <div> - <p className="font-semibold mb-2">Using data from following CSV files:</p> - <div className="flex gap-2 items-center"> - {data.csvFiles.map((csv, index) => ( - <CsvDialog key={index} csv={csv} /> - ))} - </div> + <div className="flex gap-2 items-center"> + {data.csvFiles.map((csv, index) => ( + <UploadCsvPreview key={index} csv={csv} /> + ))} </div> ); } diff --git a/templates/types/streaming/nextjs/app/components/ui/chat/widgets/CsvDialog.tsx b/templates/types/streaming/nextjs/app/components/ui/chat/widgets/CsvDialog.tsx deleted file mode 100644 index adcb53db..00000000 --- a/templates/types/streaming/nextjs/app/components/ui/chat/widgets/CsvDialog.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import Image from "next/image"; -import { CsvFile } from ".."; -import SheetIcon from "../../../ui/icons/sheet.svg"; -import { Button } from "../../button"; -import { - Drawer, - DrawerClose, - DrawerContent, - DrawerDescription, - DrawerHeader, - DrawerTitle, - DrawerTrigger, -} from "../../drawer"; - -export interface CsvDialogProps { - csv: CsvFile; -} - -export default function CsvDialog(props: CsvDialogProps) { - const { filename, filesize, content } = props.csv; - const fileSizeInKB = Math.round((filesize / 1024) * 10) / 10; - return ( - <Drawer direction="left"> - <DrawerTrigger asChild> - <div - className="border-2 border-green-700 py-1.5 px-3 rounded-lg flex gap-2 items-center cursor-pointer text-sm hover:bg-green-700 hover:text-white transition-colors duration-200 ease-in-out" - key={filename} - > - <div className="h-4 w-4 shrink-0 rounded-md"> - <Image - className="h-full w-auto" - priority - src={SheetIcon} - alt="SheetIcon" - /> - </div> - <span> - {filename} - {fileSizeInKB} KB - </span> - </div> - </DrawerTrigger> - <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] "> - <DrawerHeader className="flex justify-between"> - <div className="space-y-2"> - <DrawerTitle>Csv Raw Content</DrawerTitle> - <DrawerDescription> - {filename} ({fileSizeInKB} KB) - </DrawerDescription> - </div> - <DrawerClose asChild> - <Button variant="outline">Close</Button> - </DrawerClose> - </DrawerHeader> - <div className="m-4 max-h-[80%] overflow-auto"> - <pre className="bg-secondary rounded-md p-4 block text-sm"> - {content} - </pre> - </div> - </DrawerContent> - </Drawer> - ); -} diff --git a/templates/types/streaming/nextjs/app/components/ui/upload-csv-preview.tsx b/templates/types/streaming/nextjs/app/components/ui/upload-csv-preview.tsx index ca397a79..767ed63c 100644 --- a/templates/types/streaming/nextjs/app/components/ui/upload-csv-preview.tsx +++ b/templates/types/streaming/nextjs/app/components/ui/upload-csv-preview.tsx @@ -1,22 +1,60 @@ import { XCircleIcon } from "lucide-react"; import Image from "next/image"; import SheetIcon from "../ui/icons/sheet.svg"; +import { Button } from "./button"; +import { CsvFile } from "./chat"; +import { + Drawer, + DrawerClose, + DrawerContent, + DrawerDescription, + DrawerHeader, + DrawerTitle, + DrawerTrigger, +} from "./drawer"; import { cn } from "./lib/utils"; -export default function UploadCsvPreview({ - filename, - filesize, - onRemove, - isNew, -}: { - filename: string; - filesize: number; - onRemove: () => void; +export interface UploadCsvPreviewProps { + csv: CsvFile; + onRemove?: () => void; isNew?: boolean; -}) { - const fileSizeInKB = Math.round((filesize / 1024) * 10) / 10; +} + +export default function UploadCsvPreview(props: UploadCsvPreviewProps) { + const { filename, filesize, content } = props.csv; + return ( + <Drawer direction="left"> + <DrawerTrigger asChild> + <div> + <CSVSummaryCard {...props} /> + </div> + </DrawerTrigger> + <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] "> + <DrawerHeader className="flex justify-between"> + <div className="space-y-2"> + <DrawerTitle>Csv Raw Content</DrawerTitle> + <DrawerDescription> + {filename} ({inKB(filesize)} KB) + </DrawerDescription> + </div> + <DrawerClose asChild> + <Button variant="outline">Close</Button> + </DrawerClose> + </DrawerHeader> + <div className="m-4 max-h-[80%] overflow-auto"> + <pre className="bg-secondary rounded-md p-4 block text-sm"> + {content} + </pre> + </div> + </DrawerContent> + </Drawer> + ); +} + +function CSVSummaryCard(props: UploadCsvPreviewProps) { + const { onRemove, isNew, csv } = props; return ( - <div className="p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative"> + <div className="p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative cursor-pointer"> <div className="flex flex-row items-center gap-2"> <div className="relative h-10 w-10 shrink-0 overflow-hidden rounded-md"> <Image @@ -28,7 +66,7 @@ export default function UploadCsvPreview({ </div> <div className="overflow-hidden"> <div className="truncate font-semibold"> - {filename} ({fileSizeInKB} KB) + {csv.filename} ({inKB(csv.filesize)} KB) </div> <div className="truncate text-token-text-tertiary flex items-center gap-2"> <span>Spreadsheet</span> @@ -40,16 +78,22 @@ export default function UploadCsvPreview({ </div> </div> </div> - <div - className={cn( - "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full", - )} - > - <XCircleIcon - className="w-6 h-6 bg-gray-500 text-white rounded-full" - onClick={onRemove} - /> - </div> + {onRemove && ( + <div + className={cn( + "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full", + )} + > + <XCircleIcon + className="w-6 h-6 bg-gray-500 text-white rounded-full" + onClick={onRemove} + /> + </div> + )} </div> ); } + +function inKB(size: number) { + return Math.round((size / 1024) * 10) / 10; +} -- GitLab