From 8ed98bcb0713d7e6c2a0efe498d469e4a4a9511c Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Wed, 1 Nov 2023 12:33:15 +0700
Subject: [PATCH] feat: add streaming express example and align with
non-streaming one
---
packages/create-llama/templates/index.ts | 2 +-
.../simple/express/README-template.md | 11 +++---
.../templates/simple/express/index.ts | 4 +--
.../templates/simple/express/package.json | 12 ++++---
.../{llm.controller.ts => chat.controller.ts} | 16 ++++-----
.../routes/{llm.route.ts => chat.route.ts} | 2 +-
.../templates/simple/express/tsconfig.json | 2 --
.../streaming/express/README-template.md | 34 ++++++++++++++++++
.../templates/streaming/express/eslintrc.json | 3 ++
.../templates/streaming/express/index.ts | 17 +++++++++
.../templates/streaming/express/package.json | 25 +++++++++++++
.../src/controllers/chat.controller.ts | 36 +++++++++++++++++++
.../src/controllers/llamaindex-stream.ts | 35 ++++++++++++++++++
.../express/src/routes/chat.route.ts | 8 +++++
.../templates/streaming/express/tsconfig.json | 9 +++++
15 files changed, 188 insertions(+), 28 deletions(-)
rename packages/create-llama/templates/simple/express/src/controllers/{llm.controller.ts => chat.controller.ts} (67%)
rename packages/create-llama/templates/simple/express/src/routes/{llm.route.ts => chat.route.ts} (70%)
create mode 100644 packages/create-llama/templates/streaming/express/README-template.md
create mode 100644 packages/create-llama/templates/streaming/express/eslintrc.json
create mode 100644 packages/create-llama/templates/streaming/express/index.ts
create mode 100644 packages/create-llama/templates/streaming/express/package.json
create mode 100644 packages/create-llama/templates/streaming/express/src/controllers/chat.controller.ts
create mode 100644 packages/create-llama/templates/streaming/express/src/controllers/llamaindex-stream.ts
create mode 100644 packages/create-llama/templates/streaming/express/src/routes/chat.route.ts
create mode 100644 packages/create-llama/templates/streaming/express/tsconfig.json
diff --git a/packages/create-llama/templates/index.ts b/packages/create-llama/templates/index.ts
index 5bf8a7d3b..6f338b635 100644
--- a/packages/create-llama/templates/index.ts
+++ b/packages/create-llama/templates/index.ts
@@ -72,7 +72,7 @@ export const installTemplate = async ({
const routeFile = path.join(
root,
relativeEngineDestPath,
- framework === "nextjs" ? "route.ts" : "llm.controller.ts",
+ framework === "nextjs" ? "route.ts" : "chat.controller.ts",
);
const routeFileContent = await fs.readFile(routeFile, "utf8");
const newContent = routeFileContent.replace(
diff --git a/packages/create-llama/templates/simple/express/README-template.md b/packages/create-llama/templates/simple/express/README-template.md
index 2b4d11531..2da2865d5 100644
--- a/packages/create-llama/templates/simple/express/README-template.md
+++ b/packages/create-llama/templates/simple/express/README-template.md
@@ -14,18 +14,15 @@ Second, run the development server:
npm run dev
```
-Then call the express API endpoint `/api/llm` to see the result:
+Then call the express API endpoint `/api/chat` to see the result:
```
-curl --location 'localhost:3000/api/llm' \
+curl --location 'localhost:3000/api/chat' \
--header 'Content-Type: application/json' \
---data '{
- "message": "Hello",
- "chatHistory": []
-}'
+--data '{ "messages": [{ "role": "user", "content": "Hello" }] }'
```
-You can start editing the API by modifying `src/controllers/llm.controller.ts`. The endpoint auto-updates as you save the file.
+You can start editing the API by modifying `src/controllers/chat.controller.ts`. The endpoint auto-updates as you save the file.
## Learn More
diff --git a/packages/create-llama/templates/simple/express/index.ts b/packages/create-llama/templates/simple/express/index.ts
index 157555572..b73784295 100644
--- a/packages/create-llama/templates/simple/express/index.ts
+++ b/packages/create-llama/templates/simple/express/index.ts
@@ -1,5 +1,5 @@
import express, { Express, Request, Response } from "express";
-import llmRouter from "./src/routes/llm.route";
+import chatRouter from "./src/routes/chat.route";
const app: Express = express();
const port = 3000;
@@ -10,7 +10,7 @@ app.get("/", (req: Request, res: Response) => {
res.send("LlamaIndex Express Server");
});
-app.use("/api/llm", llmRouter);
+app.use("/api/chat", chatRouter);
app.listen(port, () => {
console.log(`⚡️[server]: Server is running at http://localhost:${port}`);
diff --git a/packages/create-llama/templates/simple/express/package.json b/packages/create-llama/templates/simple/express/package.json
index 3b57d610e..1e20573d8 100644
--- a/packages/create-llama/templates/simple/express/package.json
+++ b/packages/create-llama/templates/simple/express/package.json
@@ -1,11 +1,12 @@
{
"name": "llama-index-express",
"version": "1.0.0",
- "main": "index.js",
+ "main": "dist/index.js",
+ "type": "module",
"scripts": {
- "build": "tsc",
+ "build": "tsup index.ts --format esm --dts",
"start": "node dist/index.js",
- "dev": "concurrently \"tsc --watch\" \"nodemon -q dist/index.js\""
+ "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon -q dist/index.js\""
},
"dependencies": {
"express": "^4",
@@ -15,8 +16,9 @@
"@types/express": "^4",
"@types/node": "^20",
"concurrently": "^8",
+ "eslint": "^8",
"nodemon": "^3",
- "typescript": "^5",
- "eslint": "^8"
+ "tsup": "^7",
+ "typescript": "^5"
}
}
\ No newline at end of file
diff --git a/packages/create-llama/templates/simple/express/src/controllers/llm.controller.ts b/packages/create-llama/templates/simple/express/src/controllers/chat.controller.ts
similarity index 67%
rename from packages/create-llama/templates/simple/express/src/controllers/llm.controller.ts
rename to packages/create-llama/templates/simple/express/src/controllers/chat.controller.ts
index 7647b649b..b11e10502 100644
--- a/packages/create-llama/templates/simple/express/src/controllers/llm.controller.ts
+++ b/packages/create-llama/templates/simple/express/src/controllers/chat.controller.ts
@@ -4,16 +4,12 @@ import { createChatEngine } from "../../../../engines/context";
export const chat = async (req: Request, res: Response, next: NextFunction) => {
try {
- const {
- message,
- chatHistory,
- }: {
- message: string;
- chatHistory: ChatMessage[];
- } = req.body;
- if (!message || !chatHistory) {
+ const { messages }: { messages: ChatMessage[] } = req.body;
+ const lastMessage = messages.pop();
+ if (!messages || !lastMessage || lastMessage.role !== "user") {
return res.status(400).json({
- error: "message, chatHistory are required in the request body",
+ error:
+ "messages are required in the request body and the last message must be from the user",
});
}
@@ -23,7 +19,7 @@ export const chat = async (req: Request, res: Response, next: NextFunction) => {
const chatEngine = await createChatEngine(llm);
- const response = await chatEngine.chat(message, chatHistory);
+ const response = await chatEngine.chat(lastMessage.content, messages);
const result: ChatMessage = {
role: "assistant",
content: response.response,
diff --git a/packages/create-llama/templates/simple/express/src/routes/llm.route.ts b/packages/create-llama/templates/simple/express/src/routes/chat.route.ts
similarity index 70%
rename from packages/create-llama/templates/simple/express/src/routes/llm.route.ts
rename to packages/create-llama/templates/simple/express/src/routes/chat.route.ts
index 3711c71b9..bdfeb0853 100644
--- a/packages/create-llama/templates/simple/express/src/routes/llm.route.ts
+++ b/packages/create-llama/templates/simple/express/src/routes/chat.route.ts
@@ -1,5 +1,5 @@
import express from "express";
-import { chat } from "../controllers/llm.controller";
+import { chat } from "../controllers/chat.controller";
const llmRouter = express.Router();
diff --git a/packages/create-llama/templates/simple/express/tsconfig.json b/packages/create-llama/templates/simple/express/tsconfig.json
index fd70902d6..57c4a6281 100644
--- a/packages/create-llama/templates/simple/express/tsconfig.json
+++ b/packages/create-llama/templates/simple/express/tsconfig.json
@@ -1,8 +1,6 @@
{
"compilerOptions": {
"target": "es2016",
- "module": "commonjs",
- "outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": true,
diff --git a/packages/create-llama/templates/streaming/express/README-template.md b/packages/create-llama/templates/streaming/express/README-template.md
new file mode 100644
index 000000000..2da2865d5
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/README-template.md
@@ -0,0 +1,34 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Express](https://expressjs.com/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Second, run the development server:
+
+```
+npm run dev
+```
+
+Then call the express API endpoint `/api/chat` to see the result:
+
+```
+curl --location 'localhost:3000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Hello" }] }'
+```
+
+You can start editing the API by modifying `src/controllers/chat.controller.ts`. The endpoint auto-updates as you save the file.
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!
diff --git a/packages/create-llama/templates/streaming/express/eslintrc.json b/packages/create-llama/templates/streaming/express/eslintrc.json
new file mode 100644
index 000000000..c19581799
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/eslintrc.json
@@ -0,0 +1,3 @@
+{
+ "extends": "eslint:recommended"
+}
\ No newline at end of file
diff --git a/packages/create-llama/templates/streaming/express/index.ts b/packages/create-llama/templates/streaming/express/index.ts
new file mode 100644
index 000000000..b73784295
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/index.ts
@@ -0,0 +1,17 @@
+import express, { Express, Request, Response } from "express";
+import chatRouter from "./src/routes/chat.route";
+
+const app: Express = express();
+const port = 3000;
+
+app.use(express.json());
+
+app.get("/", (req: Request, res: Response) => {
+ res.send("LlamaIndex Express Server");
+});
+
+app.use("/api/chat", chatRouter);
+
+app.listen(port, () => {
+ console.log(`⚡️[server]: Server is running at http://localhost:${port}`);
+});
diff --git a/packages/create-llama/templates/streaming/express/package.json b/packages/create-llama/templates/streaming/express/package.json
new file mode 100644
index 000000000..ac90ee8a6
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/package.json
@@ -0,0 +1,25 @@
+{
+ "name": "llama-index-express-streaming",
+ "version": "1.0.0",
+ "main": "dist/index.js",
+ "type": "module",
+ "scripts": {
+ "build": "tsup index.ts --format esm --dts",
+ "start": "node dist/index.js",
+ "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon -q dist/index.js\""
+ },
+ "dependencies": {
+ "ai": "^2",
+ "express": "^4",
+ "llamaindex": "0.0.31"
+ },
+ "devDependencies": {
+ "@types/express": "^4",
+ "@types/node": "^20",
+ "concurrently": "^8",
+ "eslint": "^8",
+ "nodemon": "^3",
+ "tsup": "^7",
+ "typescript": "^5"
+ }
+}
\ No newline at end of file
diff --git a/packages/create-llama/templates/streaming/express/src/controllers/chat.controller.ts b/packages/create-llama/templates/streaming/express/src/controllers/chat.controller.ts
new file mode 100644
index 000000000..58f96b035
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/src/controllers/chat.controller.ts
@@ -0,0 +1,36 @@
+import { streamToResponse } from "ai";
+import { NextFunction, Request, Response } from "express";
+import { ChatMessage, OpenAI } from "llamaindex";
+import { createChatEngine } from "../../../../engines/context";
+import { LlamaIndexStream } from "./llamaindex-stream";
+
+export const chat = async (req: Request, res: Response, next: NextFunction) => {
+ try {
+ const { messages }: { messages: ChatMessage[] } = req.body;
+ const lastMessage = messages.pop();
+ if (!messages || !lastMessage || lastMessage.role !== "user") {
+ return res.status(400).json({
+ error:
+ "messages are required in the request body and the last message must be from the user",
+ });
+ }
+
+ const llm = new OpenAI({
+ model: "gpt-3.5-turbo",
+ });
+
+ const chatEngine = await createChatEngine(llm);
+
+ const response = await chatEngine.chat(lastMessage.content, messages, true);
+
+ // Transform the response into a readable stream
+ const stream = LlamaIndexStream(response);
+
+ streamToResponse(stream, res);
+ } catch (error) {
+ console.error("[LlamaIndex]", error);
+ return res.status(500).json({
+ error: (error as Error).message,
+ });
+ }
+};
diff --git a/packages/create-llama/templates/streaming/express/src/controllers/llamaindex-stream.ts b/packages/create-llama/templates/streaming/express/src/controllers/llamaindex-stream.ts
new file mode 100644
index 000000000..12328de87
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/src/controllers/llamaindex-stream.ts
@@ -0,0 +1,35 @@
+import {
+ createCallbacksTransformer,
+ createStreamDataTransformer,
+ trimStartOfStreamHelper,
+ type AIStreamCallbacksAndOptions,
+} from "ai";
+
+function createParser(res: AsyncGenerator<any>) {
+ const trimStartOfStream = trimStartOfStreamHelper();
+ return new ReadableStream<string>({
+ async pull(controller): Promise<void> {
+ const { value, done } = await res.next();
+ if (done) {
+ controller.close();
+ return;
+ }
+
+ const text = trimStartOfStream(value ?? "");
+ if (text) {
+ controller.enqueue(text);
+ }
+ },
+ });
+}
+
+export function LlamaIndexStream(
+ res: AsyncGenerator<any>,
+ callbacks?: AIStreamCallbacksAndOptions,
+): ReadableStream {
+ return createParser(res)
+ .pipeThrough(createCallbacksTransformer(callbacks))
+ .pipeThrough(
+ createStreamDataTransformer(callbacks?.experimental_streamData),
+ );
+}
diff --git a/packages/create-llama/templates/streaming/express/src/routes/chat.route.ts b/packages/create-llama/templates/streaming/express/src/routes/chat.route.ts
new file mode 100644
index 000000000..bdfeb0853
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/src/routes/chat.route.ts
@@ -0,0 +1,8 @@
+import express from "express";
+import { chat } from "../controllers/chat.controller";
+
+const llmRouter = express.Router();
+
+llmRouter.route("/").post(chat);
+
+export default llmRouter;
diff --git a/packages/create-llama/templates/streaming/express/tsconfig.json b/packages/create-llama/templates/streaming/express/tsconfig.json
new file mode 100644
index 000000000..bf2e3ec6e
--- /dev/null
+++ b/packages/create-llama/templates/streaming/express/tsconfig.json
@@ -0,0 +1,9 @@
+{
+ "compilerOptions": {
+ "target": "es2016",
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "strict": true,
+ "skipLibCheck": true
+ }
+}
\ No newline at end of file
--
GitLab