diff --git a/templates/components/engines/typescript/agent/chat.ts b/templates/components/engines/typescript/agent/chat.ts
index 51a61ba0613830f1a6744ef5a96ac244833b328c..9c82dbabc74526a6a7232a58cf56ebcd2feee902 100644
--- a/templates/components/engines/typescript/agent/chat.ts
+++ b/templates/components/engines/typescript/agent/chat.ts
@@ -1,8 +1,8 @@
import {
BaseTool,
- OpenAI,
OpenAIAgent,
QueryEngineTool,
+ Settings,
ToolFactory,
} from "llamaindex";
import fs from "node:fs/promises";
@@ -10,12 +10,12 @@ import path from "node:path";
import { getDataSource } from "./index";
import { STORAGE_CACHE_DIR } from "./shared";
-export async function createChatEngine(llm: OpenAI) {
+export async function createChatEngine() {
let tools: BaseTool[] = [];
// Add a query engine tool if we have a data source
// Delete this code if you don't have a data source
- const index = await getDataSource(llm);
+ const index = await getDataSource();
if (index) {
tools.push(
new QueryEngineTool({
@@ -38,7 +38,7 @@ export async function createChatEngine(llm: OpenAI) {
return new OpenAIAgent({
tools,
- llm,
+ llm: Settings.llm,
verbose: true,
});
}
diff --git a/templates/components/engines/typescript/chat/chat.ts b/templates/components/engines/typescript/chat/chat.ts
index 2feea01be85f29a5a670f2392431d0997fede43f..62cc77df080546cab14c4bbc7ec48ab892b3a89f 100644
--- a/templates/components/engines/typescript/chat/chat.ts
+++ b/templates/components/engines/typescript/chat/chat.ts
@@ -1,8 +1,8 @@
-import { ContextChatEngine, LLM } from "llamaindex";
+import { ContextChatEngine, Settings } from "llamaindex";
import { getDataSource } from "./index";
-export async function createChatEngine(llm: LLM) {
- const index = await getDataSource(llm);
+export async function createChatEngine() {
+ const index = await getDataSource();
if (!index) {
throw new Error(
`StorageContext is empty - call 'npm run generate' to generate the storage first`,
@@ -12,7 +12,7 @@ export async function createChatEngine(llm: LLM) {
retriever.similarityTopK = 3;
return new ContextChatEngine({
- chatModel: llm,
+ chatModel: Settings.llm,
retriever,
});
}
diff --git a/templates/components/vectordbs/typescript/astra/generate.ts b/templates/components/vectordbs/typescript/astra/generate.ts
index ca13f51e7bd8ef33da34025e499be3b21f95f680..c29e390ca83ea06fcec18682ed402358f98724e0 100644
--- a/templates/components/vectordbs/typescript/astra/generate.ts
+++ b/templates/components/vectordbs/typescript/astra/generate.ts
@@ -6,6 +6,7 @@ import {
storageContextFromDefaults,
} from "llamaindex";
import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
dotenv.config();
@@ -33,6 +34,7 @@ async function loadAndIndex() {
(async () => {
checkRequiredEnvVars();
+ initSettings();
await loadAndIndex();
console.log("Finished generating storage.");
})();
diff --git a/templates/components/vectordbs/typescript/astra/index.ts b/templates/components/vectordbs/typescript/astra/index.ts
index d1ac6c11d69baefcab54087eee45b67006474bdc..c659bc92598b0d2bc2bd6a752ded20c035ceeeeb 100644
--- a/templates/components/vectordbs/typescript/astra/index.ts
+++ b/templates/components/vectordbs/typescript/astra/index.ts
@@ -1,20 +1,10 @@
/* eslint-disable turbo/no-undeclared-env-vars */
-import {
- AstraDBVectorStore,
- LLM,
- VectorStoreIndex,
- serviceContextFromDefaults,
-} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared";
+import { AstraDBVectorStore, VectorStoreIndex } from "llamaindex";
+import { checkRequiredEnvVars } from "./shared";
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
checkRequiredEnvVars();
- const serviceContext = serviceContextFromDefaults({
- llm,
- chunkSize: CHUNK_SIZE,
- chunkOverlap: CHUNK_OVERLAP,
- });
const store = new AstraDBVectorStore();
await store.connect(process.env.ASTRA_DB_COLLECTION!);
- return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+ return await VectorStoreIndex.fromVectorStore(store);
}
diff --git a/templates/components/vectordbs/typescript/astra/shared.ts b/templates/components/vectordbs/typescript/astra/shared.ts
index fb240187a3a0b58c89e79b0e3eb15f474093cf30..d923d1b2befdd5842333111e9898eb365f552457 100644
--- a/templates/components/vectordbs/typescript/astra/shared.ts
+++ b/templates/components/vectordbs/typescript/astra/shared.ts
@@ -1,6 +1,3 @@
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
const REQUIRED_ENV_VARS = [
"ASTRA_DB_APPLICATION_TOKEN",
"ASTRA_DB_ENDPOINT",
diff --git a/templates/components/vectordbs/typescript/milvus/generate.ts b/templates/components/vectordbs/typescript/milvus/generate.ts
index bb2a33cfcccc2d6ad758b84603937fdfb9adefb5..709d35467305a064ce5bd9cca099b218f07394c5 100644
--- a/templates/components/vectordbs/typescript/milvus/generate.ts
+++ b/templates/components/vectordbs/typescript/milvus/generate.ts
@@ -6,6 +6,7 @@ import {
storageContextFromDefaults,
} from "llamaindex";
import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
import { checkRequiredEnvVars, getMilvusClient } from "./shared";
dotenv.config();
@@ -32,6 +33,7 @@ async function loadAndIndex() {
(async () => {
checkRequiredEnvVars();
+ initSettings();
await loadAndIndex();
console.log("Finished generating storage.");
})();
diff --git a/templates/components/vectordbs/typescript/milvus/index.ts b/templates/components/vectordbs/typescript/milvus/index.ts
index 05652b3065877771c5b08e42a49dbbda203ba238..c10bad6d1707aad6ed63df7a254848ddc7b42200 100644
--- a/templates/components/vectordbs/typescript/milvus/index.ts
+++ b/templates/components/vectordbs/typescript/milvus/index.ts
@@ -1,25 +1,10 @@
-import {
- LLM,
- MilvusVectorStore,
- serviceContextFromDefaults,
- VectorStoreIndex,
-} from "llamaindex";
-import {
- checkRequiredEnvVars,
- CHUNK_OVERLAP,
- CHUNK_SIZE,
- getMilvusClient,
-} from "./shared";
+import { MilvusVectorStore, VectorStoreIndex } from "llamaindex";
+import { checkRequiredEnvVars, getMilvusClient } from "./shared";
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
checkRequiredEnvVars();
- const serviceContext = serviceContextFromDefaults({
- llm,
- chunkSize: CHUNK_SIZE,
- chunkOverlap: CHUNK_OVERLAP,
- });
const milvusClient = getMilvusClient();
const store = new MilvusVectorStore({ milvusClient });
- return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+ return await VectorStoreIndex.fromVectorStore(store);
}
diff --git a/templates/components/vectordbs/typescript/milvus/shared.ts b/templates/components/vectordbs/typescript/milvus/shared.ts
index 5cdb356b157e4c019594c497559f29e8cae8ebf3..5f4139aa32e95066df8e9a60bab1c1999e83aff5 100644
--- a/templates/components/vectordbs/typescript/milvus/shared.ts
+++ b/templates/components/vectordbs/typescript/milvus/shared.ts
@@ -1,8 +1,5 @@
import { MilvusClient } from "@zilliz/milvus2-sdk-node";
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
const REQUIRED_ENV_VARS = [
"MILVUS_ADDRESS",
"MILVUS_USERNAME",
diff --git a/templates/components/vectordbs/typescript/mongo/generate.ts b/templates/components/vectordbs/typescript/mongo/generate.ts
index abfd0a6e90c7553997def49054de698e2e60b34f..da7aed57cdd9844bdbcdf41b63682effbfa5802d 100644
--- a/templates/components/vectordbs/typescript/mongo/generate.ts
+++ b/templates/components/vectordbs/typescript/mongo/generate.ts
@@ -7,6 +7,7 @@ import {
} from "llamaindex";
import { MongoClient } from "mongodb";
import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
dotenv.config();
@@ -42,6 +43,7 @@ async function loadAndIndex() {
(async () => {
checkRequiredEnvVars();
+ initSettings();
await loadAndIndex();
console.log("Finished generating storage.");
})();
diff --git a/templates/components/vectordbs/typescript/mongo/index.ts b/templates/components/vectordbs/typescript/mongo/index.ts
index ea2a3962879321a2e8fa8820f374d100825d748a..77dc7b4ddc22765a6d6f976dc37b92669a21b917 100644
--- a/templates/components/vectordbs/typescript/mongo/index.ts
+++ b/templates/components/vectordbs/typescript/mongo/index.ts
@@ -1,21 +1,11 @@
/* eslint-disable turbo/no-undeclared-env-vars */
-import {
- LLM,
- MongoDBAtlasVectorSearch,
- serviceContextFromDefaults,
- VectorStoreIndex,
-} from "llamaindex";
+import { MongoDBAtlasVectorSearch, VectorStoreIndex } from "llamaindex";
import { MongoClient } from "mongodb";
-import { checkRequiredEnvVars, CHUNK_OVERLAP, CHUNK_SIZE } from "./shared";
+import { checkRequiredEnvVars } from "./shared";
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
checkRequiredEnvVars();
const client = new MongoClient(process.env.MONGO_URI!);
- const serviceContext = serviceContextFromDefaults({
- llm,
- chunkSize: CHUNK_SIZE,
- chunkOverlap: CHUNK_OVERLAP,
- });
const store = new MongoDBAtlasVectorSearch({
mongodbClient: client,
dbName: process.env.MONGODB_DATABASE!,
@@ -23,5 +13,5 @@ export async function getDataSource(llm: LLM) {
indexName: process.env.MONGODB_VECTOR_INDEX,
});
- return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+ return await VectorStoreIndex.fromVectorStore(store);
}
diff --git a/templates/components/vectordbs/typescript/mongo/shared.ts b/templates/components/vectordbs/typescript/mongo/shared.ts
index ab46718271fc8e73963740d4b79f58deafda22cb..d6532a566ccc7618f545878084329ef5d8f29518 100644
--- a/templates/components/vectordbs/typescript/mongo/shared.ts
+++ b/templates/components/vectordbs/typescript/mongo/shared.ts
@@ -1,6 +1,3 @@
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
const REQUIRED_ENV_VARS = [
"MONGO_URI",
"MONGODB_DATABASE",
diff --git a/templates/components/vectordbs/typescript/none/generate.ts b/templates/components/vectordbs/typescript/none/generate.ts
index 2773fbeff639b4bc8b50c1c6a5333b34862efb53..732ba21129c8aab1dc7de78df2e03df54092a791 100644
--- a/templates/components/vectordbs/typescript/none/generate.ts
+++ b/templates/components/vectordbs/typescript/none/generate.ts
@@ -1,14 +1,10 @@
-import {
- ServiceContext,
- serviceContextFromDefaults,
- storageContextFromDefaults,
- VectorStoreIndex,
-} from "llamaindex";
+import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
import * as dotenv from "dotenv";
import { getDocuments } from "./loader";
-import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./shared";
+import { initSettings } from "./settings";
+import { STORAGE_CACHE_DIR } from "./shared";
// Load environment variables from local .env file
dotenv.config();
@@ -20,7 +16,7 @@ async function getRuntime(func: any) {
return end - start;
}
-async function generateDatasource(serviceContext: ServiceContext) {
+async function generateDatasource() {
console.log(`Generating storage context...`);
// Split documents, create embeddings and store them in the storage context
const ms = await getRuntime(async () => {
@@ -30,18 +26,13 @@ async function generateDatasource(serviceContext: ServiceContext) {
const documents = await getDocuments();
await VectorStoreIndex.fromDocuments(documents, {
storageContext,
- serviceContext,
});
});
console.log(`Storage context successfully generated in ${ms / 1000}s.`);
}
(async () => {
- const serviceContext = serviceContextFromDefaults({
- chunkSize: CHUNK_SIZE,
- chunkOverlap: CHUNK_OVERLAP,
- });
-
- await generateDatasource(serviceContext);
+ initSettings();
+ await generateDatasource();
console.log("Finished generating storage.");
})();
diff --git a/templates/components/vectordbs/typescript/none/index.ts b/templates/components/vectordbs/typescript/none/index.ts
index 474d7e95a3476d2deffd864d52b50b71b62c9bad..919ba3a615b454dbcc9230f00dcff57e69fd33ec 100644
--- a/templates/components/vectordbs/typescript/none/index.ts
+++ b/templates/components/vectordbs/typescript/none/index.ts
@@ -1,18 +1,11 @@
import {
- LLM,
- serviceContextFromDefaults,
SimpleDocumentStore,
storageContextFromDefaults,
VectorStoreIndex,
} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./shared";
+import { STORAGE_CACHE_DIR } from "./shared";
-export async function getDataSource(llm: LLM) {
- const serviceContext = serviceContextFromDefaults({
- llm,
- chunkSize: CHUNK_SIZE,
- chunkOverlap: CHUNK_OVERLAP,
- });
+export async function getDataSource() {
const storageContext = await storageContextFromDefaults({
persistDir: `${STORAGE_CACHE_DIR}`,
});
@@ -25,6 +18,5 @@ export async function getDataSource(llm: LLM) {
}
return await VectorStoreIndex.init({
storageContext,
- serviceContext,
});
}
diff --git a/templates/components/vectordbs/typescript/none/shared.ts b/templates/components/vectordbs/typescript/none/shared.ts
index 42a8664afed8aa72f0fa1e159f856eacdc193bf5..e7736e5b3f830112d7619a5fba199ac9479ea92d 100644
--- a/templates/components/vectordbs/typescript/none/shared.ts
+++ b/templates/components/vectordbs/typescript/none/shared.ts
@@ -1,3 +1 @@
export const STORAGE_CACHE_DIR = "./cache";
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
diff --git a/templates/components/vectordbs/typescript/pg/generate.ts b/templates/components/vectordbs/typescript/pg/generate.ts
index b41e4c20fe9b4f71c6ed17b3aed9ae29a0f28d55..f5664b6f070d69658d4391ddbe0f3422c903c71b 100644
--- a/templates/components/vectordbs/typescript/pg/generate.ts
+++ b/templates/components/vectordbs/typescript/pg/generate.ts
@@ -6,6 +6,7 @@ import {
storageContextFromDefaults,
} from "llamaindex";
import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
import {
PGVECTOR_COLLECTION,
PGVECTOR_SCHEMA,
@@ -37,6 +38,7 @@ async function loadAndIndex() {
(async () => {
checkRequiredEnvVars();
+ initSettings();
await loadAndIndex();
console.log("Finished generating storage.");
process.exit(0);
diff --git a/templates/components/vectordbs/typescript/pg/index.ts b/templates/components/vectordbs/typescript/pg/index.ts
index 8c504e71ced6dd68de6f53a7e8ec54ca566123ec..0b572d74dd98ce7c180cdc78e5334efd767e2a5a 100644
--- a/templates/components/vectordbs/typescript/pg/index.ts
+++ b/templates/components/vectordbs/typescript/pg/index.ts
@@ -1,29 +1,17 @@
/* eslint-disable turbo/no-undeclared-env-vars */
+import { PGVectorStore, VectorStoreIndex } from "llamaindex";
import {
- LLM,
- PGVectorStore,
- VectorStoreIndex,
- serviceContextFromDefaults,
-} from "llamaindex";
-import {
- CHUNK_OVERLAP,
- CHUNK_SIZE,
PGVECTOR_SCHEMA,
PGVECTOR_TABLE,
checkRequiredEnvVars,
} from "./shared";
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
checkRequiredEnvVars();
const pgvs = new PGVectorStore({
connectionString: process.env.PG_CONNECTION_STRING,
schemaName: PGVECTOR_SCHEMA,
tableName: PGVECTOR_TABLE,
});
- const serviceContext = serviceContextFromDefaults({
- llm,
- chunkSize: CHUNK_SIZE,
- chunkOverlap: CHUNK_OVERLAP,
- });
- return await VectorStoreIndex.fromVectorStore(pgvs, serviceContext);
+ return await VectorStoreIndex.fromVectorStore(pgvs);
}
diff --git a/templates/components/vectordbs/typescript/pg/shared.ts b/templates/components/vectordbs/typescript/pg/shared.ts
index ba7479340b796bcedcb4b661cb6eb97159767450..88774df1226c715fc539ca93110c9009d7dfda44 100644
--- a/templates/components/vectordbs/typescript/pg/shared.ts
+++ b/templates/components/vectordbs/typescript/pg/shared.ts
@@ -1,6 +1,4 @@
export const PGVECTOR_COLLECTION = "data";
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
export const PGVECTOR_SCHEMA = "public";
export const PGVECTOR_TABLE = "llamaindex_embedding";
diff --git a/templates/components/vectordbs/typescript/pinecone/generate.ts b/templates/components/vectordbs/typescript/pinecone/generate.ts
index d879c6b10073320b57266cab735138b166431cbe..c275bc5856b54d0d57a4a9e78e1dd43bc05e0fc6 100644
--- a/templates/components/vectordbs/typescript/pinecone/generate.ts
+++ b/templates/components/vectordbs/typescript/pinecone/generate.ts
@@ -6,6 +6,7 @@ import {
storageContextFromDefaults,
} from "llamaindex";
import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
dotenv.config();
@@ -28,6 +29,7 @@ async function loadAndIndex() {
(async () => {
checkRequiredEnvVars();
+ initSettings();
await loadAndIndex();
console.log("Finished generating storage.");
})();
diff --git a/templates/components/vectordbs/typescript/pinecone/index.ts b/templates/components/vectordbs/typescript/pinecone/index.ts
index 8fa949e7c79336beb057e6290183c0a404242bf4..226713d99330c22127158356c11cb36b3dac0d82 100644
--- a/templates/components/vectordbs/typescript/pinecone/index.ts
+++ b/templates/components/vectordbs/typescript/pinecone/index.ts
@@ -1,19 +1,9 @@
/* eslint-disable turbo/no-undeclared-env-vars */
-import {
- LLM,
- PineconeVectorStore,
- VectorStoreIndex,
- serviceContextFromDefaults,
-} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared";
+import { PineconeVectorStore, VectorStoreIndex } from "llamaindex";
+import { checkRequiredEnvVars } from "./shared";
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
checkRequiredEnvVars();
- const serviceContext = serviceContextFromDefaults({
- llm,
- chunkSize: CHUNK_SIZE,
- chunkOverlap: CHUNK_OVERLAP,
- });
const store = new PineconeVectorStore();
- return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+ return await VectorStoreIndex.fromVectorStore(store);
}
diff --git a/templates/components/vectordbs/typescript/pinecone/shared.ts b/templates/components/vectordbs/typescript/pinecone/shared.ts
index ae2fd6b1ffafb6df3adf6d25f53c65fc0a790464..c4e9911d74baf43d17861e4295f836c3b04be21f 100644
--- a/templates/components/vectordbs/typescript/pinecone/shared.ts
+++ b/templates/components/vectordbs/typescript/pinecone/shared.ts
@@ -1,6 +1,3 @@
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
const REQUIRED_ENV_VARS = ["PINECONE_ENVIRONMENT", "PINECONE_API_KEY"];
export function checkRequiredEnvVars() {
diff --git a/templates/types/streaming/express/src/controllers/chat-request.controller.ts b/templates/types/streaming/express/src/controllers/chat-request.controller.ts
index b81b5999744dd70361ddb606de35126f56841d29..056edc5d615e452301980be9e08f5b08a57c8d08 100644
--- a/templates/types/streaming/express/src/controllers/chat-request.controller.ts
+++ b/templates/types/streaming/express/src/controllers/chat-request.controller.ts
@@ -1,5 +1,5 @@
import { Request, Response } from "express";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
+import { ChatMessage, MessageContent } from "llamaindex";
import { createChatEngine } from "./engine/chat";
const convertMessageContent = (
@@ -32,10 +32,6 @@ export const chatRequest = async (req: Request, res: Response) => {
});
}
- const llm = new OpenAI({
- model: process.env.MODEL || "gpt-3.5-turbo",
- });
-
// Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
// Note: The non-streaming template does not need the Vercel/AI format, we're still using it for consistency with the streaming template
const userMessageContent = convertMessageContent(
@@ -43,7 +39,7 @@ export const chatRequest = async (req: Request, res: Response) => {
data?.imageUrl,
);
- const chatEngine = await createChatEngine(llm);
+ const chatEngine = await createChatEngine();
// Calling LlamaIndex's ChatEngine to get a response
const response = await chatEngine.chat({
diff --git a/templates/types/streaming/express/src/controllers/chat.controller.ts b/templates/types/streaming/express/src/controllers/chat.controller.ts
index 1d9cd56a79b3b9ce4ea89f82b6030aeaf8a84625..7bdf98da61a69b1550d05fa363f174df22050e5e 100644
--- a/templates/types/streaming/express/src/controllers/chat.controller.ts
+++ b/templates/types/streaming/express/src/controllers/chat.controller.ts
@@ -1,6 +1,6 @@
import { streamToResponse } from "ai";
import { Request, Response } from "express";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
+import { ChatMessage, MessageContent } from "llamaindex";
import { createChatEngine } from "./engine/chat";
import { LlamaIndexStream } from "./llamaindex-stream";
@@ -34,11 +34,7 @@ export const chat = async (req: Request, res: Response) => {
});
}
- const llm = new OpenAI({
- model: (process.env.MODEL as any) || "gpt-3.5-turbo",
- });
-
- const chatEngine = await createChatEngine(llm);
+ const chatEngine = await createChatEngine();
// Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
const userMessageContent = convertMessageContent(
diff --git a/templates/types/streaming/express/src/controllers/engine/chat.ts b/templates/types/streaming/express/src/controllers/engine/chat.ts
index abb02e90cd2ce91096791bf10c4665afcbe11d38..a42ac41cc86814fc712724caaec2c3152ac04fd4 100644
--- a/templates/types/streaming/express/src/controllers/engine/chat.ts
+++ b/templates/types/streaming/express/src/controllers/engine/chat.ts
@@ -1,7 +1,7 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
+import { Settings, SimpleChatEngine } from "llamaindex";
-export async function createChatEngine(llm: LLM) {
+export async function createChatEngine() {
return new SimpleChatEngine({
- llm,
+ llm: Settings.llm,
});
}
diff --git a/templates/types/streaming/express/src/controllers/engine/settings.ts b/templates/types/streaming/express/src/controllers/engine/settings.ts
new file mode 100644
index 0000000000000000000000000000000000000000..25c077a5d81b4e427a70579339df3cb805dbe217
--- /dev/null
+++ b/templates/types/streaming/express/src/controllers/engine/settings.ts
@@ -0,0 +1,13 @@
+import { OpenAI, Settings } from "llamaindex";
+
+const CHUNK_SIZE = 512;
+const CHUNK_OVERLAP = 20;
+
+export const initSettings = async () => {
+ Settings.llm = new OpenAI({
+ model: process.env.MODEL ?? "gpt-3.5-turbo",
+ maxTokens: 512,
+ });
+ Settings.chunkSize = CHUNK_SIZE;
+ Settings.chunkOverlap = CHUNK_OVERLAP;
+};
diff --git a/templates/types/streaming/express/src/routes/chat.route.ts b/templates/types/streaming/express/src/routes/chat.route.ts
index 0fc0d4c958ea83876c9c53226e79be42c4a4e01e..34a3e005ff34a3fc9afa87c49d2a40fee8db3db1 100644
--- a/templates/types/streaming/express/src/routes/chat.route.ts
+++ b/templates/types/streaming/express/src/routes/chat.route.ts
@@ -1,9 +1,11 @@
import express, { Router } from "express";
import { chatRequest } from "../controllers/chat-request.controller";
import { chat } from "../controllers/chat.controller";
+import { initSettings } from "../controllers/engine/settings";
const llmRouter: Router = express.Router();
+initSettings();
llmRouter.route("/").post(chat);
llmRouter.route("/request").post(chatRequest);
diff --git a/templates/types/streaming/nextjs/app/api/chat/engine/chat.ts b/templates/types/streaming/nextjs/app/api/chat/engine/chat.ts
index abb02e90cd2ce91096791bf10c4665afcbe11d38..a42ac41cc86814fc712724caaec2c3152ac04fd4 100644
--- a/templates/types/streaming/nextjs/app/api/chat/engine/chat.ts
+++ b/templates/types/streaming/nextjs/app/api/chat/engine/chat.ts
@@ -1,7 +1,7 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
+import { Settings, SimpleChatEngine } from "llamaindex";
-export async function createChatEngine(llm: LLM) {
+export async function createChatEngine() {
return new SimpleChatEngine({
- llm,
+ llm: Settings.llm,
});
}
diff --git a/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts b/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts
new file mode 100644
index 0000000000000000000000000000000000000000..25c077a5d81b4e427a70579339df3cb805dbe217
--- /dev/null
+++ b/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts
@@ -0,0 +1,13 @@
+import { OpenAI, Settings } from "llamaindex";
+
+const CHUNK_SIZE = 512;
+const CHUNK_OVERLAP = 20;
+
+export const initSettings = async () => {
+ Settings.llm = new OpenAI({
+ model: process.env.MODEL ?? "gpt-3.5-turbo",
+ maxTokens: 512,
+ });
+ Settings.chunkSize = CHUNK_SIZE;
+ Settings.chunkOverlap = CHUNK_OVERLAP;
+};
diff --git a/templates/types/streaming/nextjs/app/api/chat/route.ts b/templates/types/streaming/nextjs/app/api/chat/route.ts
index 484262f2c3f97cad8754a90ee02f2a071e5cf0c4..92e2f3b4947cd729ed44c7dadd2687cc844c9d33 100644
--- a/templates/types/streaming/nextjs/app/api/chat/route.ts
+++ b/templates/types/streaming/nextjs/app/api/chat/route.ts
@@ -1,11 +1,13 @@
import { initObservability } from "@/app/observability";
import { StreamingTextResponse } from "ai";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
+import { ChatMessage, MessageContent } from "llamaindex";
import { NextRequest, NextResponse } from "next/server";
import { createChatEngine } from "./engine/chat";
+import { initSettings } from "./engine/settings";
import { LlamaIndexStream } from "./llamaindex-stream";
initObservability();
+initSettings();
export const runtime = "nodejs";
export const dynamic = "force-dynamic";
@@ -44,12 +46,7 @@ export async function POST(request: NextRequest) {
);
}
- const llm = new OpenAI({
- model: (process.env.MODEL as any) ?? "gpt-3.5-turbo",
- maxTokens: 512,
- });
-
- const chatEngine = await createChatEngine(llm);
+ const chatEngine = await createChatEngine();
// Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
const userMessageContent = convertMessageContent(