From 0213fe07dd4f8ce441f144cd21987b1fac1b9df6 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Tue, 24 Sep 2024 16:11:43 +0700
Subject: [PATCH] fix: add dependencies for pg vector store (#312)
---
.changeset/strong-wasps-nail.md | 5 +
.coderabbit.yaml | 6 ++
e2e/resolve_ts_dependencies.spec.ts | 97 +++++++++++++++++++
e2e/utils.ts | 6 +-
helpers/typescript.ts | 37 ++++++-
.../engines/typescript/agent/chat.ts | 13 ++-
.../vectordbs/typescript/astra/generate.ts | 5 +-
.../vectordbs/typescript/astra/index.ts | 2 +-
.../vectordbs/typescript/chroma/generate.ts | 4 +-
.../vectordbs/typescript/chroma/index.ts | 4 +-
.../typescript/llamacloud/queryFilter.ts | 1 -
.../vectordbs/typescript/milvus/generate.ts | 2 +-
.../vectordbs/typescript/milvus/index.ts | 2 +-
.../vectordbs/typescript/mongo/generate.ts | 2 +-
.../vectordbs/typescript/mongo/index.ts | 2 +-
.../vectordbs/typescript/pg/generate.ts | 7 +-
.../vectordbs/typescript/pg/index.ts | 3 +-
.../vectordbs/typescript/pinecone/generate.ts | 2 +-
.../vectordbs/typescript/pinecone/index.ts | 2 +-
.../vectordbs/typescript/qdrant/generate.ts | 2 +-
.../vectordbs/typescript/qdrant/index.ts | 2 +-
.../vectordbs/typescript/weaviate/index.ts | 2 +-
.../controllers/chat-request.controller.ts | 2 +-
23 files changed, 181 insertions(+), 29 deletions(-)
create mode 100644 .changeset/strong-wasps-nail.md
create mode 100644 .coderabbit.yaml
create mode 100644 e2e/resolve_ts_dependencies.spec.ts
diff --git a/.changeset/strong-wasps-nail.md b/.changeset/strong-wasps-nail.md
new file mode 100644
index 00000000..7e04149b
--- /dev/null
+++ b/.changeset/strong-wasps-nail.md
@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Update dependencies for vector stores and add e2e test to ensure that they work as expected.
diff --git a/.coderabbit.yaml b/.coderabbit.yaml
new file mode 100644
index 00000000..880749b4
--- /dev/null
+++ b/.coderabbit.yaml
@@ -0,0 +1,6 @@
+# coderabbit.yml
+reviews:
+ path_instructions:
+ - path: "templates/**"
+ instructions: |
+ For files under the `templates` folder, do not report 'Missing Dependencies Detected' errors.
diff --git a/e2e/resolve_ts_dependencies.spec.ts b/e2e/resolve_ts_dependencies.spec.ts
new file mode 100644
index 00000000..6b67666f
--- /dev/null
+++ b/e2e/resolve_ts_dependencies.spec.ts
@@ -0,0 +1,97 @@
+import { expect, test } from "@playwright/test";
+import { exec } from "child_process";
+import fs from "fs";
+import path from "path";
+import util from "util";
+import { TemplateFramework, TemplateVectorDB } from "../helpers/types";
+import { createTestDir, runCreateLlama } from "./utils";
+
+const execAsync = util.promisify(exec);
+
+const templateFramework: TemplateFramework = process.env.FRAMEWORK
+ ? (process.env.FRAMEWORK as TemplateFramework)
+ : "nextjs";
+const dataSource: string = process.env.DATASOURCE
+ ? process.env.DATASOURCE
+ : "--example-file";
+
+if (
+ templateFramework == "nextjs" ||
+ templateFramework == "express" // test is only relevant for TS projects
+) {
+ // vectorDBs combinations to test
+ const vectorDbs: TemplateVectorDB[] = [
+ "mongo",
+ "pg",
+ "qdrant",
+ "pinecone",
+ "milvus",
+ "astra",
+ "chroma",
+ "llamacloud",
+ "weaviate",
+ ];
+
+ test.describe("Test resolve TS dependencies", () => {
+ for (const vectorDb of vectorDbs) {
+ const optionDescription = `vectorDb: ${vectorDb}, dataSource: ${dataSource}`;
+
+ test(`options: ${optionDescription}`, async () => {
+ const cwd = await createTestDir();
+
+ const result = await runCreateLlama(
+ cwd,
+ "streaming",
+ templateFramework,
+ dataSource,
+ vectorDb,
+ 3000, // port
+ 8000, // externalPort
+ "none", // postInstallAction
+ undefined, // ui
+ templateFramework === "nextjs" ? "" : "--no-frontend", // appType
+ undefined, // llamaCloudProjectName
+ undefined, // llamaCloudIndexName
+ );
+ const name = result.projectName;
+
+ // Check if the app folder exists
+ const appDir = path.join(cwd, name);
+ const dirExists = fs.existsSync(appDir);
+ expect(dirExists).toBeTruthy();
+
+ // Install dependencies using pnpm
+ try {
+ const { stderr: installStderr } = await execAsync(
+ "pnpm install --prefer-offline",
+ {
+ cwd: appDir,
+ },
+ );
+ expect(installStderr).toBeFalsy();
+ } catch (error) {
+ console.error("Error installing dependencies:", error);
+ throw error;
+ }
+
+ // Run tsc type check and capture the output
+ try {
+ const { stdout, stderr } = await execAsync(
+ "pnpm exec tsc -b --diagnostics",
+ {
+ cwd: appDir,
+ },
+ );
+ // Check if there's any error output
+ expect(stderr).toBeFalsy();
+
+ // Log the stdout for debugging purposes
+ console.log("TypeScript type-check output:", stdout);
+ } catch (error) {
+ console.error("Error running tsc:", error);
+ throw error;
+ }
+ });
+ }
+ });
+}
diff --git a/e2e/utils.ts b/e2e/utils.ts
index afd450b7..7c988617 100644
--- a/e2e/utils.ts
+++ b/e2e/utils.ts
@@ -105,11 +105,11 @@ export async function runCreateLlama(
},
});
appProcess.stderr?.on("data", (data) => {
- console.log(data.toString());
+ console.error(data.toString());
});
appProcess.on("exit", (code) => {
if (code !== 0 && code !== null) {
- throw new Error(`create-llama command was failed!`);
+ throw new Error(`create-llama command failed with exit code ${code}`);
}
});
@@ -121,6 +121,8 @@ export async function runCreateLlama(
port,
externalPort,
);
+ } else if (postInstallAction === "dependencies") {
+ await waitForProcess(appProcess, 1000 * 60); // wait 1 min for dependencies to be resolved
} else {
// wait 10 seconds for create-llama to exit
await waitForProcess(appProcess, 1000 * 10);
diff --git a/helpers/typescript.ts b/helpers/typescript.ts
index f818a1d3..9b4c9ffc 100644
--- a/helpers/typescript.ts
+++ b/helpers/typescript.ts
@@ -180,6 +180,7 @@ export const installTSTemplate = async ({
framework,
ui,
observability,
+ vectorDb,
});
if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
@@ -200,9 +201,16 @@ async function updatePackageJson({
framework,
ui,
observability,
+ vectorDb,
}: Pick<
InstallTemplateArgs,
- "root" | "appName" | "dataSources" | "framework" | "ui" | "observability"
+ | "root"
+ | "appName"
+ | "dataSources"
+ | "framework"
+ | "ui"
+ | "observability"
+ | "vectorDb"
> & {
relativeEngineDestPath: string;
}): Promise<any> {
@@ -249,6 +257,33 @@ async function updatePackageJson({
};
}
+ if (vectorDb === "pg") {
+ packageJson.dependencies = {
+ ...packageJson.dependencies,
+ pg: "^8.12.0",
+ };
+ }
+
+ if (vectorDb === "qdrant") {
+ packageJson.dependencies = {
+ ...packageJson.dependencies,
+ "@qdrant/js-client-rest": "^1.11.0",
+ };
+ }
+ if (vectorDb === "mongo") {
+ packageJson.dependencies = {
+ ...packageJson.dependencies,
+ mongodb: "^6.7.0",
+ };
+ }
+
+ if (vectorDb === "milvus") {
+ packageJson.dependencies = {
+ ...packageJson.dependencies,
+ "@zilliz/milvus2-sdk-node": "^2.4.6",
+ };
+ }
+
if (observability === "traceloop") {
packageJson.dependencies = {
...packageJson.dependencies,
diff --git a/templates/components/engines/typescript/agent/chat.ts b/templates/components/engines/typescript/agent/chat.ts
index ad1d6a00..014a0d08 100644
--- a/templates/components/engines/typescript/agent/chat.ts
+++ b/templates/components/engines/typescript/agent/chat.ts
@@ -1,4 +1,9 @@
-import { BaseToolWithCall, OpenAIAgent, QueryEngineTool } from "llamaindex";
+import {
+ BaseToolWithCall,
+ ChatEngine,
+ OpenAIAgent,
+ QueryEngineTool,
+} from "llamaindex";
import fs from "node:fs/promises";
import path from "node:path";
import { getDataSource } from "./index";
@@ -37,8 +42,10 @@ export async function createChatEngine(documentIds?: string[], params?: any) {
tools.push(...(await createTools(toolConfig)));
}
- return new OpenAIAgent({
+ const agent = new OpenAIAgent({
tools,
systemPrompt: process.env.SYSTEM_PROMPT,
- });
+ }) as unknown as ChatEngine;
+
+ return agent;
}
diff --git a/templates/components/vectordbs/typescript/astra/generate.ts b/templates/components/vectordbs/typescript/astra/generate.ts
index 104745bb..f66657b5 100644
--- a/templates/components/vectordbs/typescript/astra/generate.ts
+++ b/templates/components/vectordbs/typescript/astra/generate.ts
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
-import { AstraDBVectorStore } from "llamaindex/storage/vectorStore/AstraDBVectorStore";
+import { AstraDBVectorStore } from "llamaindex/vector-store/AstraDBVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
@@ -15,13 +15,12 @@ async function loadAndIndex() {
// create vector store and a collection
const collectionName = process.env.ASTRA_DB_COLLECTION!;
const vectorStore = new AstraDBVectorStore();
- await vectorStore.create(collectionName, {
+ await vectorStore.createAndConnect(collectionName, {
vector: {
dimension: parseInt(process.env.EMBEDDING_DIM!),
metric: "cosine",
},
});
- await vectorStore.connect(collectionName);
// create index from documents and store them in Astra
console.log("Start creating embeddings...");
diff --git a/templates/components/vectordbs/typescript/astra/index.ts b/templates/components/vectordbs/typescript/astra/index.ts
index 38c5bbbd..4ee66ade 100644
--- a/templates/components/vectordbs/typescript/astra/index.ts
+++ b/templates/components/vectordbs/typescript/astra/index.ts
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
-import { AstraDBVectorStore } from "llamaindex/storage/vectorStore/AstraDBVectorStore";
+import { AstraDBVectorStore } from "llamaindex/vector-store/AstraDBVectorStore";
import { checkRequiredEnvVars } from "./shared";
export async function getDataSource(params?: any) {
diff --git a/templates/components/vectordbs/typescript/chroma/generate.ts b/templates/components/vectordbs/typescript/chroma/generate.ts
index 83e8ea16..a4abe70d 100644
--- a/templates/components/vectordbs/typescript/chroma/generate.ts
+++ b/templates/components/vectordbs/typescript/chroma/generate.ts
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
-import { ChromaVectorStore } from "llamaindex/storage/vectorStore/ChromaVectorStore";
+import { ChromaVectorStore } from "llamaindex/vector-store/ChromaVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
@@ -16,7 +16,7 @@ async function loadAndIndex() {
const chromaUri = `http://${process.env.CHROMA_HOST}:${process.env.CHROMA_PORT}`;
const vectorStore = new ChromaVectorStore({
- collectionName: process.env.CHROMA_COLLECTION,
+ collectionName: process.env.CHROMA_COLLECTION!,
chromaClientParams: { path: chromaUri },
});
diff --git a/templates/components/vectordbs/typescript/chroma/index.ts b/templates/components/vectordbs/typescript/chroma/index.ts
index fbc7b4bf..7ab5a332 100644
--- a/templates/components/vectordbs/typescript/chroma/index.ts
+++ b/templates/components/vectordbs/typescript/chroma/index.ts
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
-import { ChromaVectorStore } from "llamaindex/storage/vectorStore/ChromaVectorStore";
+import { ChromaVectorStore } from "llamaindex/vector-store/ChromaVectorStore";
import { checkRequiredEnvVars } from "./shared";
export async function getDataSource(params?: any) {
@@ -8,7 +8,7 @@ export async function getDataSource(params?: any) {
const chromaUri = `http://${process.env.CHROMA_HOST}:${process.env.CHROMA_PORT}`;
const store = new ChromaVectorStore({
- collectionName: process.env.CHROMA_COLLECTION,
+ collectionName: process.env.CHROMA_COLLECTION!,
chromaClientParams: { path: chromaUri },
});
diff --git a/templates/components/vectordbs/typescript/llamacloud/queryFilter.ts b/templates/components/vectordbs/typescript/llamacloud/queryFilter.ts
index c3ed6e3e..4df8842f 100644
--- a/templates/components/vectordbs/typescript/llamacloud/queryFilter.ts
+++ b/templates/components/vectordbs/typescript/llamacloud/queryFilter.ts
@@ -4,7 +4,6 @@ export function generateFilters(documentIds: string[]): MetadataFilters {
// public documents don't have the "private" field or it's set to "false"
const publicDocumentsFilter: MetadataFilter = {
key: "private",
- value: null,
operator: "is_empty",
};
diff --git a/templates/components/vectordbs/typescript/milvus/generate.ts b/templates/components/vectordbs/typescript/milvus/generate.ts
index cfdd6839..c36443f3 100644
--- a/templates/components/vectordbs/typescript/milvus/generate.ts
+++ b/templates/components/vectordbs/typescript/milvus/generate.ts
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
-import { MilvusVectorStore } from "llamaindex/storage/vectorStore/MilvusVectorStore";
+import { MilvusVectorStore } from "llamaindex/vector-store/MilvusVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars, getMilvusClient } from "./shared";
diff --git a/templates/components/vectordbs/typescript/milvus/index.ts b/templates/components/vectordbs/typescript/milvus/index.ts
index 91275b11..657f3f37 100644
--- a/templates/components/vectordbs/typescript/milvus/index.ts
+++ b/templates/components/vectordbs/typescript/milvus/index.ts
@@ -1,5 +1,5 @@
import { VectorStoreIndex } from "llamaindex";
-import { MilvusVectorStore } from "llamaindex/storage/vectorStore/MilvusVectorStore";
+import { MilvusVectorStore } from "llamaindex/vector-store/MilvusVectorStore";
import { checkRequiredEnvVars, getMilvusClient } from "./shared";
export async function getDataSource(params?: any) {
diff --git a/templates/components/vectordbs/typescript/mongo/generate.ts b/templates/components/vectordbs/typescript/mongo/generate.ts
index 73ff8592..d0919c28 100644
--- a/templates/components/vectordbs/typescript/mongo/generate.ts
+++ b/templates/components/vectordbs/typescript/mongo/generate.ts
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { storageContextFromDefaults, VectorStoreIndex } from "llamaindex";
-import { MongoDBAtlasVectorSearch } from "llamaindex/storage/vectorStore/MongoDBAtlasVectorStore";
+import { MongoDBAtlasVectorSearch } from "llamaindex/vector-store/MongoDBAtlasVectorStore";
import { MongoClient } from "mongodb";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
diff --git a/templates/components/vectordbs/typescript/mongo/index.ts b/templates/components/vectordbs/typescript/mongo/index.ts
index 75c20fb6..3203b85c 100644
--- a/templates/components/vectordbs/typescript/mongo/index.ts
+++ b/templates/components/vectordbs/typescript/mongo/index.ts
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
-import { MongoDBAtlasVectorSearch } from "llamaindex/storage/vectorStore/MongoDBAtlasVectorStore";
+import { MongoDBAtlasVectorSearch } from "llamaindex/vector-store/MongoDBAtlasVectorStore";
import { MongoClient } from "mongodb";
import { checkRequiredEnvVars, POPULATED_METADATA_FIELDS } from "./shared";
diff --git a/templates/components/vectordbs/typescript/pg/generate.ts b/templates/components/vectordbs/typescript/pg/generate.ts
index 37a0af54..f5664b6f 100644
--- a/templates/components/vectordbs/typescript/pg/generate.ts
+++ b/templates/components/vectordbs/typescript/pg/generate.ts
@@ -1,7 +1,10 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
-import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
-import { PGVectorStore } from "llamaindex/storage/vectorStore/PGVectorStore";
+import {
+ PGVectorStore,
+ VectorStoreIndex,
+ storageContextFromDefaults,
+} from "llamaindex";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import {
diff --git a/templates/components/vectordbs/typescript/pg/index.ts b/templates/components/vectordbs/typescript/pg/index.ts
index 75bcd403..6a4f1370 100644
--- a/templates/components/vectordbs/typescript/pg/index.ts
+++ b/templates/components/vectordbs/typescript/pg/index.ts
@@ -1,6 +1,5 @@
/* eslint-disable turbo/no-undeclared-env-vars */
-import { VectorStoreIndex } from "llamaindex";
-import { PGVectorStore } from "llamaindex/storage/vectorStore/PGVectorStore";
+import { PGVectorStore, VectorStoreIndex } from "llamaindex";
import {
PGVECTOR_SCHEMA,
PGVECTOR_TABLE,
diff --git a/templates/components/vectordbs/typescript/pinecone/generate.ts b/templates/components/vectordbs/typescript/pinecone/generate.ts
index 676ffabb..235817db 100644
--- a/templates/components/vectordbs/typescript/pinecone/generate.ts
+++ b/templates/components/vectordbs/typescript/pinecone/generate.ts
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
-import { PineconeVectorStore } from "llamaindex/storage/vectorStore/PineconeVectorStore";
+import { PineconeVectorStore } from "llamaindex/vector-store/PineconeVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
diff --git a/templates/components/vectordbs/typescript/pinecone/index.ts b/templates/components/vectordbs/typescript/pinecone/index.ts
index 66a22d46..ff4f8b34 100644
--- a/templates/components/vectordbs/typescript/pinecone/index.ts
+++ b/templates/components/vectordbs/typescript/pinecone/index.ts
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
-import { PineconeVectorStore } from "llamaindex/storage/vectorStore/PineconeVectorStore";
+import { PineconeVectorStore } from "llamaindex/vector-store/PineconeVectorStore";
import { checkRequiredEnvVars } from "./shared";
export async function getDataSource(params?: any) {
diff --git a/templates/components/vectordbs/typescript/qdrant/generate.ts b/templates/components/vectordbs/typescript/qdrant/generate.ts
index f71e05de..d3f6bf91 100644
--- a/templates/components/vectordbs/typescript/qdrant/generate.ts
+++ b/templates/components/vectordbs/typescript/qdrant/generate.ts
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
-import { QdrantVectorStore } from "llamaindex/storage/vectorStore/QdrantVectorStore";
+import { QdrantVectorStore } from "llamaindex/vector-store/QdrantVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars, getQdrantClient } from "./shared";
diff --git a/templates/components/vectordbs/typescript/qdrant/index.ts b/templates/components/vectordbs/typescript/qdrant/index.ts
index a9d87ab8..314a94af 100644
--- a/templates/components/vectordbs/typescript/qdrant/index.ts
+++ b/templates/components/vectordbs/typescript/qdrant/index.ts
@@ -1,6 +1,6 @@
import * as dotenv from "dotenv";
import { VectorStoreIndex } from "llamaindex";
-import { QdrantVectorStore } from "llamaindex/storage/vectorStore/QdrantVectorStore";
+import { QdrantVectorStore } from "llamaindex/vector-store/QdrantVectorStore";
import { checkRequiredEnvVars, getQdrantClient } from "./shared";
dotenv.config();
diff --git a/templates/components/vectordbs/typescript/weaviate/index.ts b/templates/components/vectordbs/typescript/weaviate/index.ts
index 27b32b42..047ea029 100644
--- a/templates/components/vectordbs/typescript/weaviate/index.ts
+++ b/templates/components/vectordbs/typescript/weaviate/index.ts
@@ -1,6 +1,6 @@
import * as dotenv from "dotenv";
import { VectorStoreIndex } from "llamaindex";
-import { WeaviateVectorStore } from "llamaindex/storage/vectorStore/WeaviateVectorStore";
+import { WeaviateVectorStore } from "llamaindex/vector-store/WeaviateVectorStore";
import { checkRequiredEnvVars, DEFAULT_INDEX_NAME } from "./shared";
dotenv.config();
diff --git a/templates/types/streaming/express/src/controllers/chat-request.controller.ts b/templates/types/streaming/express/src/controllers/chat-request.controller.ts
index 117713fb..3fbec2b0 100644
--- a/templates/types/streaming/express/src/controllers/chat-request.controller.ts
+++ b/templates/types/streaming/express/src/controllers/chat-request.controller.ts
@@ -35,7 +35,7 @@ export const chatRequest = async (req: Request, res: Response) => {
// Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
// Note: The non-streaming template does not need the Vercel/AI format, we're still using it for consistency with the streaming template
const userMessageContent = convertMessageContent(
- userMessage.content,
+ userMessage.content as string,
data?.imageUrl,
);
--
GitLab