From 8201e2f5c27f490f5d2259c65149a2b44fbdf46e Mon Sep 17 00:00:00 2001
From: Thuc Pham <51660321+thucpn@users.noreply.github.com>
Date: Fri, 5 Apr 2024 12:37:17 +0700
Subject: [PATCH] refactor: replace mjs vectordb files by ts files (#36)
---
helpers/typescript.ts | 4 ++--
.../engines/typescript/agent/chat.ts | 2 +-
.../typescript/file/{loader.mjs => loader.ts} | 0
.../llama_parse/{loader.mjs => loader.ts} | 0
.../astra/{generate.mjs => generate.ts} | 6 +++---
.../vectordbs/typescript/astra/index.ts | 2 +-
.../typescript/astra/{shared.mjs => shared.ts} | 0
.../milvus/{generate.mjs => generate.ts} | 4 ++--
.../vectordbs/typescript/milvus/index.ts | 14 ++------------
.../milvus/{shared.mjs => shared.ts} | 2 +-
.../mongo/{generate.mjs => generate.ts} | 10 +++++-----
.../vectordbs/typescript/mongo/index.ts | 18 ++++--------------
.../typescript/mongo/{shared.mjs => shared.ts} | 0
.../none/{generate.mjs => generate.ts} | 9 +++++----
.../vectordbs/typescript/none/index.ts | 2 +-
.../none/{constants.mjs => shared.ts} | 0
.../pg/{generate.mjs => generate.ts} | 4 ++--
.../vectordbs/typescript/pg/index.ts | 14 ++------------
.../typescript/pg/{shared.mjs => shared.ts} | 0
.../pinecone/{generate.mjs => generate.ts} | 4 ++--
.../vectordbs/typescript/pinecone/index.ts | 14 ++------------
.../pinecone/{shared.mjs => shared.ts} | 0
templates/types/streaming/express/package.json | 3 ++-
.../types/streaming/express/tsconfig.json | 5 +++++
templates/types/streaming/nextjs/package.json | 3 ++-
templates/types/streaming/nextjs/tsconfig.json | 7 ++++++-
26 files changed, 50 insertions(+), 77 deletions(-)
rename templates/components/loaders/typescript/file/{loader.mjs => loader.ts} (100%)
rename templates/components/loaders/typescript/llama_parse/{loader.mjs => loader.ts} (100%)
rename templates/components/vectordbs/typescript/astra/{generate.mjs => generate.ts} (87%)
rename templates/components/vectordbs/typescript/astra/{shared.mjs => shared.ts} (100%)
rename templates/components/vectordbs/typescript/milvus/{generate.mjs => generate.ts} (89%)
rename templates/components/vectordbs/typescript/milvus/{shared.mjs => shared.ts} (95%)
rename templates/components/vectordbs/typescript/mongo/{generate.mjs => generate.ts} (84%)
rename templates/components/vectordbs/typescript/mongo/{shared.mjs => shared.ts} (100%)
rename templates/components/vectordbs/typescript/none/{generate.mjs => generate.ts} (86%)
rename templates/components/vectordbs/typescript/none/{constants.mjs => shared.ts} (100%)
rename templates/components/vectordbs/typescript/pg/{generate.mjs => generate.ts} (94%)
rename templates/components/vectordbs/typescript/pg/{shared.mjs => shared.ts} (100%)
rename templates/components/vectordbs/typescript/pinecone/{generate.mjs => generate.ts} (90%)
rename templates/components/vectordbs/typescript/pinecone/{shared.mjs => shared.ts} (100%)
diff --git a/helpers/typescript.ts b/helpers/typescript.ts
index 3ffa2131..db159a3f 100644
--- a/helpers/typescript.ts
+++ b/helpers/typescript.ts
@@ -205,10 +205,10 @@ async function updatePackageJson({
// add generate script if using context engine
packageJson.scripts = {
...packageJson.scripts,
- generate: `node ${path.join(
+ generate: `ts-node ${path.join(
relativeEngineDestPath,
"engine",
- "generate.mjs",
+ "generate.ts",
)}`,
};
}
diff --git a/templates/components/engines/typescript/agent/chat.ts b/templates/components/engines/typescript/agent/chat.ts
index 3de82345..51a61ba0 100644
--- a/templates/components/engines/typescript/agent/chat.ts
+++ b/templates/components/engines/typescript/agent/chat.ts
@@ -7,8 +7,8 @@ import {
} from "llamaindex";
import fs from "node:fs/promises";
import path from "node:path";
-import { STORAGE_CACHE_DIR } from "./constants.mjs";
import { getDataSource } from "./index";
+import { STORAGE_CACHE_DIR } from "./shared";
export async function createChatEngine(llm: OpenAI) {
let tools: BaseTool[] = [];
diff --git a/templates/components/loaders/typescript/file/loader.mjs b/templates/components/loaders/typescript/file/loader.ts
similarity index 100%
rename from templates/components/loaders/typescript/file/loader.mjs
rename to templates/components/loaders/typescript/file/loader.ts
diff --git a/templates/components/loaders/typescript/llama_parse/loader.mjs b/templates/components/loaders/typescript/llama_parse/loader.ts
similarity index 100%
rename from templates/components/loaders/typescript/llama_parse/loader.mjs
rename to templates/components/loaders/typescript/llama_parse/loader.ts
diff --git a/templates/components/vectordbs/typescript/astra/generate.mjs b/templates/components/vectordbs/typescript/astra/generate.ts
similarity index 87%
rename from templates/components/vectordbs/typescript/astra/generate.mjs
rename to templates/components/vectordbs/typescript/astra/generate.ts
index 904ec009..ca13f51e 100644
--- a/templates/components/vectordbs/typescript/astra/generate.mjs
+++ b/templates/components/vectordbs/typescript/astra/generate.ts
@@ -5,8 +5,8 @@ import {
VectorStoreIndex,
storageContextFromDefaults,
} from "llamaindex";
-import { getDocuments } from "./loader.mjs";
-import { checkRequiredEnvVars } from "./shared.mjs";
+import { getDocuments } from "./loader";
+import { checkRequiredEnvVars } from "./shared";
dotenv.config();
@@ -15,7 +15,7 @@ async function loadAndIndex() {
const documents = await getDocuments();
// create vector store and a collection
- const collectionName = process.env.ASTRA_DB_COLLECTION;
+ const collectionName = process.env.ASTRA_DB_COLLECTION!;
const vectorStore = new AstraDBVectorStore();
await vectorStore.create(collectionName, {
vector: { dimension: 1536, metric: "cosine" },
diff --git a/templates/components/vectordbs/typescript/astra/index.ts b/templates/components/vectordbs/typescript/astra/index.ts
index 4a44a11a..d1ac6c11 100644
--- a/templates/components/vectordbs/typescript/astra/index.ts
+++ b/templates/components/vectordbs/typescript/astra/index.ts
@@ -5,7 +5,7 @@ import {
VectorStoreIndex,
serviceContextFromDefaults,
} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared.mjs";
+import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared";
export async function getDataSource(llm: LLM) {
checkRequiredEnvVars();
diff --git a/templates/components/vectordbs/typescript/astra/shared.mjs b/templates/components/vectordbs/typescript/astra/shared.ts
similarity index 100%
rename from templates/components/vectordbs/typescript/astra/shared.mjs
rename to templates/components/vectordbs/typescript/astra/shared.ts
diff --git a/templates/components/vectordbs/typescript/milvus/generate.mjs b/templates/components/vectordbs/typescript/milvus/generate.ts
similarity index 89%
rename from templates/components/vectordbs/typescript/milvus/generate.mjs
rename to templates/components/vectordbs/typescript/milvus/generate.ts
index 11c3184d..bb2a33cf 100644
--- a/templates/components/vectordbs/typescript/milvus/generate.mjs
+++ b/templates/components/vectordbs/typescript/milvus/generate.ts
@@ -5,8 +5,8 @@ import {
VectorStoreIndex,
storageContextFromDefaults,
} from "llamaindex";
-import { getDocuments } from "./loader.mjs";
-import { checkRequiredEnvVars, getMilvusClient } from "./shared.mjs";
+import { getDocuments } from "./loader";
+import { checkRequiredEnvVars, getMilvusClient } from "./shared";
dotenv.config();
diff --git a/templates/components/vectordbs/typescript/milvus/index.ts b/templates/components/vectordbs/typescript/milvus/index.ts
index cf487ad3..05652b30 100644
--- a/templates/components/vectordbs/typescript/milvus/index.ts
+++ b/templates/components/vectordbs/typescript/milvus/index.ts
@@ -1,5 +1,4 @@
import {
- ContextChatEngine,
LLM,
MilvusVectorStore,
serviceContextFromDefaults,
@@ -10,9 +9,9 @@ import {
CHUNK_OVERLAP,
CHUNK_SIZE,
getMilvusClient,
-} from "./shared.mjs";
+} from "./shared";
-async function getDataSource(llm: LLM) {
+export async function getDataSource(llm: LLM) {
checkRequiredEnvVars();
const serviceContext = serviceContextFromDefaults({
llm,
@@ -24,12 +23,3 @@ async function getDataSource(llm: LLM) {
return await VectorStoreIndex.fromVectorStore(store, serviceContext);
}
-
-export async function createChatEngine(llm: LLM) {
- const index = await getDataSource(llm);
- const retriever = index.asRetriever({ similarityTopK: 3 });
- return new ContextChatEngine({
- chatModel: llm,
- retriever,
- });
-}
diff --git a/templates/components/vectordbs/typescript/milvus/shared.mjs b/templates/components/vectordbs/typescript/milvus/shared.ts
similarity index 95%
rename from templates/components/vectordbs/typescript/milvus/shared.mjs
rename to templates/components/vectordbs/typescript/milvus/shared.ts
index a02ea57c..5cdb356b 100644
--- a/templates/components/vectordbs/typescript/milvus/shared.mjs
+++ b/templates/components/vectordbs/typescript/milvus/shared.ts
@@ -16,7 +16,7 @@ export function getMilvusClient() {
throw new Error("MILVUS_ADDRESS environment variable is required");
}
return new MilvusClient({
- address: process.env.MILVUS_ADDRESS,
+ address: process.env.MILVUS_ADDRESS!,
username: process.env.MILVUS_USERNAME,
password: process.env.MILVUS_PASSWORD,
});
diff --git a/templates/components/vectordbs/typescript/mongo/generate.mjs b/templates/components/vectordbs/typescript/mongo/generate.ts
similarity index 84%
rename from templates/components/vectordbs/typescript/mongo/generate.mjs
rename to templates/components/vectordbs/typescript/mongo/generate.ts
index 618859eb..abfd0a6e 100644
--- a/templates/components/vectordbs/typescript/mongo/generate.mjs
+++ b/templates/components/vectordbs/typescript/mongo/generate.ts
@@ -6,14 +6,14 @@ import {
storageContextFromDefaults,
} from "llamaindex";
import { MongoClient } from "mongodb";
-import { getDocuments } from "./loader.mjs";
-import { checkRequiredEnvVars } from "./shared.mjs";
+import { getDocuments } from "./loader";
+import { checkRequiredEnvVars } from "./shared";
dotenv.config();
-const mongoUri = process.env.MONGO_URI;
-const databaseName = process.env.MONGODB_DATABASE;
-const vectorCollectionName = process.env.MONGODB_VECTORS;
+const mongoUri = process.env.MONGO_URI!;
+const databaseName = process.env.MONGODB_DATABASE!;
+const vectorCollectionName = process.env.MONGODB_VECTORS!;
const indexName = process.env.MONGODB_VECTOR_INDEX;
async function loadAndIndex() {
diff --git a/templates/components/vectordbs/typescript/mongo/index.ts b/templates/components/vectordbs/typescript/mongo/index.ts
index 844789c6..ea2a3962 100644
--- a/templates/components/vectordbs/typescript/mongo/index.ts
+++ b/templates/components/vectordbs/typescript/mongo/index.ts
@@ -1,15 +1,14 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import {
- ContextChatEngine,
LLM,
MongoDBAtlasVectorSearch,
serviceContextFromDefaults,
VectorStoreIndex,
} from "llamaindex";
import { MongoClient } from "mongodb";
-import { checkRequiredEnvVars, CHUNK_OVERLAP, CHUNK_SIZE } from "./shared.mjs";
+import { checkRequiredEnvVars, CHUNK_OVERLAP, CHUNK_SIZE } from "./shared";
-async function getDataSource(llm: LLM) {
+export async function getDataSource(llm: LLM) {
checkRequiredEnvVars();
const client = new MongoClient(process.env.MONGO_URI!);
const serviceContext = serviceContextFromDefaults({
@@ -19,19 +18,10 @@ async function getDataSource(llm: LLM) {
});
const store = new MongoDBAtlasVectorSearch({
mongodbClient: client,
- dbName: process.env.MONGODB_DATABASE,
- collectionName: process.env.MONGODB_VECTORS,
+ dbName: process.env.MONGODB_DATABASE!,
+ collectionName: process.env.MONGODB_VECTORS!,
indexName: process.env.MONGODB_VECTOR_INDEX,
});
return await VectorStoreIndex.fromVectorStore(store, serviceContext);
}
-
-export async function createChatEngine(llm: LLM) {
- const index = await getDataSource(llm);
- const retriever = index.asRetriever({ similarityTopK: 3 });
- return new ContextChatEngine({
- chatModel: llm,
- retriever,
- });
-}
diff --git a/templates/components/vectordbs/typescript/mongo/shared.mjs b/templates/components/vectordbs/typescript/mongo/shared.ts
similarity index 100%
rename from templates/components/vectordbs/typescript/mongo/shared.mjs
rename to templates/components/vectordbs/typescript/mongo/shared.ts
diff --git a/templates/components/vectordbs/typescript/none/generate.mjs b/templates/components/vectordbs/typescript/none/generate.ts
similarity index 86%
rename from templates/components/vectordbs/typescript/none/generate.mjs
rename to templates/components/vectordbs/typescript/none/generate.ts
index 5b3987c1..2773fbef 100644
--- a/templates/components/vectordbs/typescript/none/generate.mjs
+++ b/templates/components/vectordbs/typescript/none/generate.ts
@@ -1,4 +1,5 @@
import {
+ ServiceContext,
serviceContextFromDefaults,
storageContextFromDefaults,
VectorStoreIndex,
@@ -6,20 +7,20 @@ import {
import * as dotenv from "dotenv";
-import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./constants.mjs";
-import { getDocuments } from "./loader.mjs";
+import { getDocuments } from "./loader";
+import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./shared";
// Load environment variables from local .env file
dotenv.config();
-async function getRuntime(func) {
+async function getRuntime(func: any) {
const start = Date.now();
await func();
const end = Date.now();
return end - start;
}
-async function generateDatasource(serviceContext) {
+async function generateDatasource(serviceContext: ServiceContext) {
console.log(`Generating storage context...`);
// Split documents, create embeddings and store them in the storage context
const ms = await getRuntime(async () => {
diff --git a/templates/components/vectordbs/typescript/none/index.ts b/templates/components/vectordbs/typescript/none/index.ts
index f3819b51..474d7e95 100644
--- a/templates/components/vectordbs/typescript/none/index.ts
+++ b/templates/components/vectordbs/typescript/none/index.ts
@@ -5,7 +5,7 @@ import {
storageContextFromDefaults,
VectorStoreIndex,
} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./constants.mjs";
+import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./shared";
export async function getDataSource(llm: LLM) {
const serviceContext = serviceContextFromDefaults({
diff --git a/templates/components/vectordbs/typescript/none/constants.mjs b/templates/components/vectordbs/typescript/none/shared.ts
similarity index 100%
rename from templates/components/vectordbs/typescript/none/constants.mjs
rename to templates/components/vectordbs/typescript/none/shared.ts
diff --git a/templates/components/vectordbs/typescript/pg/generate.mjs b/templates/components/vectordbs/typescript/pg/generate.ts
similarity index 94%
rename from templates/components/vectordbs/typescript/pg/generate.mjs
rename to templates/components/vectordbs/typescript/pg/generate.ts
index ca8410bf..b41e4c20 100644
--- a/templates/components/vectordbs/typescript/pg/generate.mjs
+++ b/templates/components/vectordbs/typescript/pg/generate.ts
@@ -5,13 +5,13 @@ import {
VectorStoreIndex,
storageContextFromDefaults,
} from "llamaindex";
-import { getDocuments } from "./loader.mjs";
+import { getDocuments } from "./loader";
import {
PGVECTOR_COLLECTION,
PGVECTOR_SCHEMA,
PGVECTOR_TABLE,
checkRequiredEnvVars,
-} from "./shared.mjs";
+} from "./shared";
dotenv.config();
diff --git a/templates/components/vectordbs/typescript/pg/index.ts b/templates/components/vectordbs/typescript/pg/index.ts
index 7de66a2e..8c504e71 100644
--- a/templates/components/vectordbs/typescript/pg/index.ts
+++ b/templates/components/vectordbs/typescript/pg/index.ts
@@ -1,6 +1,5 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import {
- ContextChatEngine,
LLM,
PGVectorStore,
VectorStoreIndex,
@@ -12,9 +11,9 @@ import {
PGVECTOR_SCHEMA,
PGVECTOR_TABLE,
checkRequiredEnvVars,
-} from "./shared.mjs";
+} from "./shared";
-async function getDataSource(llm: LLM) {
+export async function getDataSource(llm: LLM) {
checkRequiredEnvVars();
const pgvs = new PGVectorStore({
connectionString: process.env.PG_CONNECTION_STRING,
@@ -28,12 +27,3 @@ async function getDataSource(llm: LLM) {
});
return await VectorStoreIndex.fromVectorStore(pgvs, serviceContext);
}
-
-export async function createChatEngine(llm: LLM) {
- const index = await getDataSource(llm);
- const retriever = index.asRetriever({ similarityTopK: 3 });
- return new ContextChatEngine({
- chatModel: llm,
- retriever,
- });
-}
diff --git a/templates/components/vectordbs/typescript/pg/shared.mjs b/templates/components/vectordbs/typescript/pg/shared.ts
similarity index 100%
rename from templates/components/vectordbs/typescript/pg/shared.mjs
rename to templates/components/vectordbs/typescript/pg/shared.ts
diff --git a/templates/components/vectordbs/typescript/pinecone/generate.mjs b/templates/components/vectordbs/typescript/pinecone/generate.ts
similarity index 90%
rename from templates/components/vectordbs/typescript/pinecone/generate.mjs
rename to templates/components/vectordbs/typescript/pinecone/generate.ts
index 3e1fcaa0..d879c6b1 100644
--- a/templates/components/vectordbs/typescript/pinecone/generate.mjs
+++ b/templates/components/vectordbs/typescript/pinecone/generate.ts
@@ -5,8 +5,8 @@ import {
VectorStoreIndex,
storageContextFromDefaults,
} from "llamaindex";
-import { getDocuments } from "./loader.mjs";
-import { checkRequiredEnvVars } from "./shared.mjs";
+import { getDocuments } from "./loader";
+import { checkRequiredEnvVars } from "./shared";
dotenv.config();
diff --git a/templates/components/vectordbs/typescript/pinecone/index.ts b/templates/components/vectordbs/typescript/pinecone/index.ts
index be18486c..8fa949e7 100644
--- a/templates/components/vectordbs/typescript/pinecone/index.ts
+++ b/templates/components/vectordbs/typescript/pinecone/index.ts
@@ -1,14 +1,13 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import {
- ContextChatEngine,
LLM,
PineconeVectorStore,
VectorStoreIndex,
serviceContextFromDefaults,
} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared.mjs";
+import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared";
-async function getDataSource(llm: LLM) {
+export async function getDataSource(llm: LLM) {
checkRequiredEnvVars();
const serviceContext = serviceContextFromDefaults({
llm,
@@ -18,12 +17,3 @@ async function getDataSource(llm: LLM) {
const store = new PineconeVectorStore();
return await VectorStoreIndex.fromVectorStore(store, serviceContext);
}
-
-export async function createChatEngine(llm: LLM) {
- const index = await getDataSource(llm);
- const retriever = index.asRetriever({ similarityTopK: 5 });
- return new ContextChatEngine({
- chatModel: llm,
- retriever,
- });
-}
diff --git a/templates/components/vectordbs/typescript/pinecone/shared.mjs b/templates/components/vectordbs/typescript/pinecone/shared.ts
similarity index 100%
rename from templates/components/vectordbs/typescript/pinecone/shared.mjs
rename to templates/components/vectordbs/typescript/pinecone/shared.ts
diff --git a/templates/types/streaming/express/package.json b/templates/types/streaming/express/package.json
index 12530b24..5ac4b099 100644
--- a/templates/types/streaming/express/package.json
+++ b/templates/types/streaming/express/package.json
@@ -27,6 +27,7 @@
"typescript": "^5.3.2",
"prettier": "^3.2.5",
"prettier-plugin-organize-imports": "^3.2.4",
- "eslint-config-prettier": "^8.10.0"
+ "eslint-config-prettier": "^8.10.0",
+ "ts-node": "^10.9.2"
}
}
diff --git a/templates/types/streaming/express/tsconfig.json b/templates/types/streaming/express/tsconfig.json
index c242d5fa..b103d12f 100644
--- a/templates/types/streaming/express/tsconfig.json
+++ b/templates/types/streaming/express/tsconfig.json
@@ -9,5 +9,10 @@
"paths": {
"@/*": ["./*"]
}
+ },
+ "ts-node": {
+ "compilerOptions": {
+ "module": "commonjs"
+ }
}
}
diff --git a/templates/types/streaming/nextjs/package.json b/templates/types/streaming/nextjs/package.json
index 0f97c3e8..2471f7bf 100644
--- a/templates/types/streaming/nextjs/package.json
+++ b/templates/types/streaming/nextjs/package.json
@@ -43,6 +43,7 @@
"cross-env": "^7.0.3",
"prettier": "^3.2.5",
"prettier-plugin-organize-imports": "^3.2.4",
- "eslint-config-prettier": "^8.10.0"
+ "eslint-config-prettier": "^8.10.0",
+ "ts-node": "^10.9.2"
}
}
diff --git a/templates/types/streaming/nextjs/tsconfig.json b/templates/types/streaming/nextjs/tsconfig.json
index 40c136b8..65e2ba60 100644
--- a/templates/types/streaming/nextjs/tsconfig.json
+++ b/templates/types/streaming/nextjs/tsconfig.json
@@ -24,5 +24,10 @@
"forceConsistentCasingInFileNames": true
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
- "exclude": ["node_modules"]
+ "exclude": ["node_modules"],
+ "ts-node": {
+ "compilerOptions": {
+ "module": "commonjs"
+ }
+ }
}
--
GitLab