diff --git a/.changeset/three-islands-brake.md b/.changeset/three-islands-brake.md
new file mode 100644
index 0000000000000000000000000000000000000000..308bba0ae023b7df9286104ac7ba2a3b2e75cbab
--- /dev/null
+++ b/.changeset/three-islands-brake.md
@@ -0,0 +1,7 @@
+---
+"llamaindex": patch
+---
+
+refactor: export vector store only in nodejs environment on top level
+
+If you see some missing modules error, please change vector store related imports to `llamaindex/vector-store`
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2e65e98f8e6e9e7b11e37b890b36b8269830eb52..4cfc57f3f5ff93a3e4430a1bcb24ca52e5573d8a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,6 +19,11 @@ jobs:
matrix:
node-version: [18.x, 20.x, 22.x]
name: E2E on Node.js ${{ matrix.node-version }}
+
+ env: POSTGRES_DB=vectordb
+ POSTGRES_USER=testuser
+ POSTGRES_PASSWORD=testpwd
+ POSTGRES_HOST_AUTH_METHOD=trust
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -45,6 +50,16 @@ jobs:
steps:
- uses: actions/checkout@v4
+ - uses: ankane/setup-postgres@v1
+ with:
+ database: llamaindex_node_test
+ dev-files: true
+ - run: |
+ cd /tmp
+ git clone --branch v0.7.0 https://github.com/pgvector/pgvector.git
+ cd pgvector
+ make
+ sudo make install
- uses: pnpm/action-setup@v4
- name: Setup Node.js
uses: actions/setup-node@v4
diff --git a/packages/llamaindex/package.json b/packages/llamaindex/package.json
index d80499e998e5940fd0eaf0653b40259ce17ff33b..2e34b65b2554db7393fa35d597acd40576676742 100644
--- a/packages/llamaindex/package.json
+++ b/packages/llamaindex/package.json
@@ -59,8 +59,6 @@
"openai": "^4.57.0",
"papaparse": "^5.4.1",
"pathe": "^1.1.2",
- "pg": "^8.12.0",
- "pgvector": "^0.2.0",
"portkey-ai": "0.1.16",
"rake-modified": "^1.0.8",
"string-strip-html": "^13.4.8",
@@ -72,11 +70,19 @@
"zod": "^3.23.8"
},
"peerDependencies": {
- "@notionhq/client": "^2.2.15"
+ "@notionhq/client": "^2.2.15",
+ "pg": "^8.12.0",
+ "pgvector": "0.2.0"
},
"peerDependenciesMeta": {
"@notionhq/client": {
"optional": true
+ },
+ "pg": {
+ "optional": true
+ },
+ "pgvector": {
+ "optional": true
}
},
"devDependencies": {
@@ -85,6 +91,8 @@
"@swc/core": "^1.7.22",
"concurrently": "^8.2.2",
"glob": "^11.0.0",
+ "pg": "^8.12.0",
+ "pgvector": "0.2.0",
"typescript": "^5.5.4"
},
"engines": {
diff --git a/packages/llamaindex/src/index.ts b/packages/llamaindex/src/index.ts
index b64397e7ab1182df63370d1ab6e9549c487e5725..0dbfe741112aab117e756e3969d4192745e32d70 100644
--- a/packages/llamaindex/src/index.ts
+++ b/packages/llamaindex/src/index.ts
@@ -17,3 +17,7 @@ export { GeminiVertexSession } from "./llm/gemini/vertex.js";
// Expose AzureDynamicSessionTool for node.js runtime only
export { JinaAIEmbedding } from "./embeddings/JinaAIEmbedding.js";
export { AzureDynamicSessionTool } from "./tools/AzureDynamicSessionTool.node.js";
+
+// Don't export vector store modules for non-node.js runtime on top level,
+// as we cannot guarantee that they will work in other environments
+export * from "./vector-store.js";
diff --git a/packages/llamaindex/src/indices/vectorStore/index.ts b/packages/llamaindex/src/indices/vectorStore/index.ts
index af769810c2c2cd7423a7fdd8490c3e04c6c0f01e..71ec89c1bca028f1ea313b57e4b1a8c6a83f040a 100644
--- a/packages/llamaindex/src/indices/vectorStore/index.ts
+++ b/packages/llamaindex/src/indices/vectorStore/index.ts
@@ -29,16 +29,16 @@ import {
import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
import type { StorageContext } from "../../storage/StorageContext.js";
import { storageContextFromDefaults } from "../../storage/StorageContext.js";
+import type { BaseIndexStore } from "../../storage/indexStore/types.js";
+import type { BaseSynthesizer } from "../../synthesizers/types.js";
+import type { QueryEngine } from "../../types.js";
import type {
MetadataFilters,
VectorStore,
VectorStoreByType,
VectorStoreQueryResult,
-} from "../../storage/index.js";
-import type { BaseIndexStore } from "../../storage/indexStore/types.js";
-import { VectorStoreQueryMode } from "../../storage/vectorStore/types.js";
-import type { BaseSynthesizer } from "../../synthesizers/types.js";
-import type { QueryEngine } from "../../types.js";
+} from "../../vector-store/index.js";
+import { VectorStoreQueryMode } from "../../vector-store/types.js";
import type { BaseIndexInit } from "../BaseIndex.js";
import { BaseIndex } from "../BaseIndex.js";
import { IndexDict, IndexStructType } from "../json-to-index-struct.js";
diff --git a/packages/llamaindex/src/ingestion/IngestionPipeline.ts b/packages/llamaindex/src/ingestion/IngestionPipeline.ts
index c19e42aad7a89c657b1d5e6a36e0ae1b9432f006..21407db571be0f9b5362b7892dacd10be3849276 100644
--- a/packages/llamaindex/src/ingestion/IngestionPipeline.ts
+++ b/packages/llamaindex/src/ingestion/IngestionPipeline.ts
@@ -7,10 +7,7 @@ import {
type Metadata,
} from "@llamaindex/core/schema";
import type { BaseDocumentStore } from "../storage/docStore/types.js";
-import type {
- VectorStore,
- VectorStoreByType,
-} from "../storage/vectorStore/types.js";
+import type { VectorStore, VectorStoreByType } from "../vector-store/types.js";
import { IngestionCache, getTransformationHash } from "./IngestionCache.js";
import {
DocStoreStrategy,
diff --git a/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts b/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts
index 5668551e8a92ab03b7567d2e03ab99396ba5a53a..9ef72f2769af1bf918b91ae303eaced01f32b757 100644
--- a/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts
+++ b/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts
@@ -1,6 +1,6 @@
import { BaseNode, TransformComponent } from "@llamaindex/core/schema";
import type { BaseDocumentStore } from "../../storage/docStore/types.js";
-import type { VectorStore } from "../../storage/vectorStore/types.js";
+import type { VectorStore } from "../../vector-store/types.js";
import { classify } from "./classify.js";
/**
diff --git a/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts b/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts
index 7928f4783c4c693941ea6fe0f6d930e22371e62b..bbf3b2e6dcf2bf2e8c7a0b4164e3539861d989a6 100644
--- a/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts
+++ b/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts
@@ -1,6 +1,6 @@
import { BaseNode, TransformComponent } from "@llamaindex/core/schema";
import type { BaseDocumentStore } from "../../storage/docStore/types.js";
-import type { VectorStore } from "../../storage/vectorStore/types.js";
+import type { VectorStore } from "../../vector-store/types.js";
import { classify } from "./classify.js";
/**
diff --git a/packages/llamaindex/src/ingestion/strategies/index.ts b/packages/llamaindex/src/ingestion/strategies/index.ts
index 00cafe8fc104e9d3b4e8bcdb111b1ef2a8cd6d73..d8b1e2c1b4a691c5e76caa1a1fde6caeee9ea68f 100644
--- a/packages/llamaindex/src/ingestion/strategies/index.ts
+++ b/packages/llamaindex/src/ingestion/strategies/index.ts
@@ -1,6 +1,6 @@
import { TransformComponent } from "@llamaindex/core/schema";
import type { BaseDocumentStore } from "../../storage/docStore/types.js";
-import type { VectorStore } from "../../storage/vectorStore/types.js";
+import type { VectorStore } from "../../vector-store/types.js";
import { DuplicatesStrategy } from "./DuplicatesStrategy.js";
import { UpsertsAndDeleteStrategy } from "./UpsertsAndDeleteStrategy.js";
import { UpsertsStrategy } from "./UpsertsStrategy.js";
diff --git a/packages/llamaindex/src/storage/StorageContext.ts b/packages/llamaindex/src/storage/StorageContext.ts
index b1b4a392772146725ba5913ed1c0a738e574a48c..a3f8fdf53d3753e5e447a5be4d6c7a196f0d60f4 100644
--- a/packages/llamaindex/src/storage/StorageContext.ts
+++ b/packages/llamaindex/src/storage/StorageContext.ts
@@ -5,12 +5,12 @@ import {
import { ModalityType, ObjectType } from "@llamaindex/core/schema";
import { path } from "@llamaindex/env";
import { getImageEmbedModel } from "../internal/settings/image-embed-model.js";
+import { SimpleVectorStore } from "../vector-store/SimpleVectorStore.js";
+import type { VectorStore, VectorStoreByType } from "../vector-store/types.js";
import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";
import type { BaseDocumentStore } from "./docStore/types.js";
import { SimpleIndexStore } from "./indexStore/SimpleIndexStore.js";
import type { BaseIndexStore } from "./indexStore/types.js";
-import { SimpleVectorStore } from "./vectorStore/SimpleVectorStore.js";
-import type { VectorStore, VectorStoreByType } from "./vectorStore/types.js";
export interface StorageContext {
docStore: BaseDocumentStore;
diff --git a/packages/llamaindex/src/storage/index.ts b/packages/llamaindex/src/storage/index.ts
index 27af70716d7566673d22cf0277bd7b7fd0c178bc..5bf08e026a45e38de15902e524c078e7f4c250ce 100644
--- a/packages/llamaindex/src/storage/index.ts
+++ b/packages/llamaindex/src/storage/index.ts
@@ -8,13 +8,3 @@ export * from "./indexStore/types.js";
export { SimpleKVStore } from "./kvStore/SimpleKVStore.js";
export * from "./kvStore/types.js";
export * from "./StorageContext.js";
-export { AstraDBVectorStore } from "./vectorStore/AstraDBVectorStore.js";
-export { ChromaVectorStore } from "./vectorStore/ChromaVectorStore.js";
-export { MilvusVectorStore } from "./vectorStore/MilvusVectorStore.js";
-export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore.js";
-export { PGVectorStore } from "./vectorStore/PGVectorStore.js";
-export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore.js";
-export { QdrantVectorStore } from "./vectorStore/QdrantVectorStore.js";
-export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore.js";
-export * from "./vectorStore/types.js";
-export { WeaviateVectorStore } from "./vectorStore/WeaviateVectorStore.js";
diff --git a/packages/llamaindex/src/vector-store.ts b/packages/llamaindex/src/vector-store.ts
new file mode 100644
index 0000000000000000000000000000000000000000..bc2094949bce673356d614b12cd022b565f75d3c
--- /dev/null
+++ b/packages/llamaindex/src/vector-store.ts
@@ -0,0 +1 @@
+export * from "./vector-store/index.js";
diff --git a/packages/llamaindex/src/storage/vectorStore/AstraDBVectorStore.ts b/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/AstraDBVectorStore.ts
rename to packages/llamaindex/src/vector-store/AstraDBVectorStore.ts
diff --git a/packages/llamaindex/src/storage/vectorStore/ChromaVectorStore.ts b/packages/llamaindex/src/vector-store/ChromaVectorStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/ChromaVectorStore.ts
rename to packages/llamaindex/src/vector-store/ChromaVectorStore.ts
diff --git a/packages/llamaindex/src/storage/vectorStore/MilvusVectorStore.ts b/packages/llamaindex/src/vector-store/MilvusVectorStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/MilvusVectorStore.ts
rename to packages/llamaindex/src/vector-store/MilvusVectorStore.ts
diff --git a/packages/llamaindex/src/storage/vectorStore/MongoDBAtlasVectorStore.ts b/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/MongoDBAtlasVectorStore.ts
rename to packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts
diff --git a/packages/llamaindex/src/storage/vectorStore/PGVectorStore.ts b/packages/llamaindex/src/vector-store/PGVectorStore.ts
similarity index 87%
rename from packages/llamaindex/src/storage/vectorStore/PGVectorStore.ts
rename to packages/llamaindex/src/vector-store/PGVectorStore.ts
index ec4556aeca4e63260e93afb191de70865b4d34b7..ce6cbdd0e55fb805a7f3c9efc066e71041b1bfb8 100644
--- a/packages/llamaindex/src/storage/vectorStore/PGVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/PGVectorStore.ts
@@ -3,10 +3,9 @@ import type pg from "pg";
import {
FilterCondition,
FilterOperator,
- VectorStoreBase,
- type IEmbedModel,
type MetadataFilter,
type MetadataFilterValue,
+ VectorStoreBase,
type VectorStoreNoEmbedModel,
type VectorStoreQuery,
type VectorStoreQueryResult,
@@ -14,12 +13,22 @@ import {
import { escapeLikeString } from "./utils.js";
+import type { BaseEmbedding } from "@llamaindex/core/embeddings";
import type { BaseNode, Metadata } from "@llamaindex/core/schema";
import { Document, MetadataMode } from "@llamaindex/core/schema";
export const PGVECTOR_SCHEMA = "public";
export const PGVECTOR_TABLE = "llamaindex_embedding";
+export type PGVectorStoreConfig = {
+ schemaName?: string | undefined;
+ tableName?: string | undefined;
+ database?: string | undefined;
+ connectionString?: string | undefined;
+ dimensions?: number | undefined;
+ embedModel?: BaseEmbedding | undefined;
+};
+
/**
* Provides support for writing and querying vector data in Postgres.
* Note: Can't be used with data created using the Python version of the vector store (https://docs.llamaindex.ai/en/stable/examples/vector_stores/postgres.html)
@@ -33,10 +42,12 @@ export class PGVectorStore
private collection: string = "";
private schemaName: string = PGVECTOR_SCHEMA;
private tableName: string = PGVECTOR_TABLE;
+
+ private database: string | undefined = undefined;
private connectionString: string | undefined = undefined;
private dimensions: number = 1536;
- private db?: pg.Client;
+ private db?: pg.ClientBase;
/**
* Constructs a new instance of the PGVectorStore
@@ -48,26 +59,27 @@ export class PGVectorStore
* PGPASSWORD=your database password
* PGDATABASE=your database name
* PGPORT=your database port
- *
- * @param {object} config - The configuration settings for the instance.
- * @param {string} config.schemaName - The name of the schema (optional). Defaults to PGVECTOR_SCHEMA.
- * @param {string} config.tableName - The name of the table (optional). Defaults to PGVECTOR_TABLE.
- * @param {string} config.connectionString - The connection string (optional).
- * @param {number} config.dimensions - The dimensions of the embedding model.
*/
- constructor(
- config?: {
- schemaName?: string;
- tableName?: string;
- connectionString?: string;
- dimensions?: number;
- } & Partial<IEmbedModel>,
- ) {
- super(config?.embedModel);
- this.schemaName = config?.schemaName ?? PGVECTOR_SCHEMA;
- this.tableName = config?.tableName ?? PGVECTOR_TABLE;
- this.connectionString = config?.connectionString;
- this.dimensions = config?.dimensions ?? 1536;
+ constructor(configOrClient?: PGVectorStoreConfig | pg.ClientBase) {
+ // We cannot import pg from top level, it might have side effects
+ // so we only check if the config.connect function exists
+ if (
+ configOrClient &&
+ "connect" in configOrClient &&
+ typeof configOrClient.connect === "function"
+ ) {
+ const db = configOrClient as pg.ClientBase;
+ super();
+ this.db = db;
+ } else {
+ const config = configOrClient as PGVectorStoreConfig;
+ super(config?.embedModel);
+ this.schemaName = config?.schemaName ?? PGVECTOR_SCHEMA;
+ this.tableName = config?.tableName ?? PGVECTOR_TABLE;
+ this.database = config?.database;
+ this.connectionString = config?.connectionString;
+ this.dimensions = config?.dimensions ?? 1536;
+ }
}
/**
@@ -92,7 +104,7 @@ export class PGVectorStore
return this.collection;
}
- private async getDb(): Promise<pg.Client> {
+ private async getDb(): Promise<pg.ClientBase> {
if (!this.db) {
try {
const pg = await import("pg");
@@ -102,6 +114,7 @@ export class PGVectorStore
// Create DB connection
// Read connection params from env - see comment block above
const db = new Client({
+ database: this.database,
connectionString: this.connectionString,
});
await db.connect();
@@ -110,9 +123,6 @@ export class PGVectorStore
await db.query("CREATE EXTENSION IF NOT EXISTS vector");
await registerType(db);
- // Check schema, table(s), index(es)
- await this.checkSchema(db);
-
// All good? Keep the connection reference
this.db = db;
} catch (err) {
@@ -121,10 +131,15 @@ export class PGVectorStore
}
}
+ const db = this.db;
+
+ // Check schema, table(s), index(es)
+ await this.checkSchema(db);
+
return Promise.resolve(this.db);
}
- private async checkSchema(db: pg.Client) {
+ private async checkSchema(db: pg.ClientBase) {
await db.query(`CREATE SCHEMA IF NOT EXISTS ${this.schemaName}`);
const tbl = `CREATE TABLE IF NOT EXISTS ${this.schemaName}.${this.tableName}(
@@ -171,26 +186,22 @@ export class PGVectorStore
}
private getDataToInsert(embeddingResults: BaseNode<Metadata>[]) {
- const result = [];
- for (let index = 0; index < embeddingResults.length; index++) {
- const row = embeddingResults[index]!;
-
- const id: any = row.id_.length ? row.id_ : null;
- const meta = row.metadata || {};
- meta.create_date = new Date();
+ return embeddingResults.map((node) => {
+ const id: any = node.id_.length ? node.id_ : null;
+ const meta = node.metadata || {};
+ if (!meta.create_date) {
+ meta.create_date = new Date();
+ }
- const params = [
+ return [
id,
"",
this.collection,
- row.getContent(MetadataMode.EMBED),
+ node.getContent(MetadataMode.NONE),
meta,
- "[" + row.getEmbedding().join(",") + "]",
+ "[" + node.getEmbedding().join(",") + "]",
];
-
- result.push(params);
- }
- return result;
+ });
}
/**
@@ -201,7 +212,7 @@ export class PGVectorStore
*/
async add(embeddingResults: BaseNode<Metadata>[]): Promise<string[]> {
if (embeddingResults.length === 0) {
- console.debug("Empty list sent to PGVectorStore::add");
+ console.warn("Empty list sent to PGVectorStore::add");
return [];
}
diff --git a/packages/llamaindex/src/storage/vectorStore/PineconeVectorStore.ts b/packages/llamaindex/src/vector-store/PineconeVectorStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/PineconeVectorStore.ts
rename to packages/llamaindex/src/vector-store/PineconeVectorStore.ts
diff --git a/packages/llamaindex/src/storage/vectorStore/QdrantVectorStore.ts b/packages/llamaindex/src/vector-store/QdrantVectorStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/QdrantVectorStore.ts
rename to packages/llamaindex/src/vector-store/QdrantVectorStore.ts
diff --git a/packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts
similarity index 98%
rename from packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts
rename to packages/llamaindex/src/vector-store/SimpleVectorStore.ts
index 19765d84f5e6fceaf86e7ba030b2af74414b10ef..184071190299cdbe4cfbed03a1173c189e33d6cf 100644
--- a/packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts
@@ -2,11 +2,8 @@ import type { BaseEmbedding } from "@llamaindex/core/embeddings";
import { DEFAULT_PERSIST_DIR } from "@llamaindex/core/global";
import type { BaseNode } from "@llamaindex/core/schema";
import { fs, path } from "@llamaindex/env";
-import {
- getTopKEmbeddings,
- getTopKMMREmbeddings,
-} from "../../internal/utils.js";
-import { exists } from "../FileSystem.js";
+import { getTopKEmbeddings, getTopKMMREmbeddings } from "../internal/utils.js";
+import { exists } from "../storage/FileSystem.js";
import {
FilterOperator,
VectorStoreBase,
diff --git a/packages/llamaindex/src/storage/vectorStore/WeaviateVectorStore.ts b/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/WeaviateVectorStore.ts
rename to packages/llamaindex/src/vector-store/WeaviateVectorStore.ts
diff --git a/packages/llamaindex/src/vector-store/index.ts b/packages/llamaindex/src/vector-store/index.ts
new file mode 100644
index 0000000000000000000000000000000000000000..f01bed7a5f6b23910571066fb2394466f855c739
--- /dev/null
+++ b/packages/llamaindex/src/vector-store/index.ts
@@ -0,0 +1,10 @@
+export * from "./AstraDBVectorStore.js";
+export * from "./ChromaVectorStore.js";
+export * from "./MilvusVectorStore.js";
+export * from "./MongoDBAtlasVectorStore.js";
+export * from "./PGVectorStore.js";
+export * from "./PineconeVectorStore.js";
+export * from "./QdrantVectorStore.js";
+export * from "./SimpleVectorStore.js";
+export * from "./types.js";
+export * from "./WeaviateVectorStore.js";
diff --git a/packages/llamaindex/src/storage/vectorStore/types.ts b/packages/llamaindex/src/vector-store/types.ts
similarity index 97%
rename from packages/llamaindex/src/storage/vectorStore/types.ts
rename to packages/llamaindex/src/vector-store/types.ts
index 7c726567b03bc575ba2206bd7c577f473eae74a3..217f3c5c08a7ba9a136bf79b7cf693da53bb3e86 100644
--- a/packages/llamaindex/src/storage/vectorStore/types.ts
+++ b/packages/llamaindex/src/vector-store/types.ts
@@ -1,6 +1,6 @@
import type { BaseEmbedding } from "@llamaindex/core/embeddings";
import type { BaseNode, ModalityType } from "@llamaindex/core/schema";
-import { getEmbeddedModel } from "../../internal/settings/EmbedModel.js";
+import { getEmbeddedModel } from "../internal/settings/EmbedModel.js";
export interface VectorStoreQueryResult {
nodes?: BaseNode[];
diff --git a/packages/llamaindex/src/storage/vectorStore/utils.ts b/packages/llamaindex/src/vector-store/utils.ts
similarity index 100%
rename from packages/llamaindex/src/storage/vectorStore/utils.ts
rename to packages/llamaindex/src/vector-store/utils.ts
diff --git a/packages/llamaindex/tests/VectorStore.test.ts b/packages/llamaindex/tests/VectorStore.test.ts
index f44444d7736cefee7bee915c2ef6d1b4337a1ebf..443da2de15c5b81a828f0791ca851ce88a462794 100644
--- a/packages/llamaindex/tests/VectorStore.test.ts
+++ b/packages/llamaindex/tests/VectorStore.test.ts
@@ -2,7 +2,7 @@ import { Document, MetadataMode } from "@llamaindex/core/schema";
import {
metadataDictToNode,
nodeToMetadata,
-} from "llamaindex/storage/vectorStore/utils";
+} from "llamaindex/vector-store/utils";
import { beforeEach, describe, expect, test } from "vitest";
describe("Testing VectorStore utils", () => {
diff --git a/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts b/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts
index b27dfa61a2801a4dc489eeafa27d4de2a9a9f997..965144f965423340b9c681134f8c14d7f367e676 100644
--- a/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts
+++ b/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts
@@ -1,5 +1,5 @@
import type { BaseNode } from "@llamaindex/core/schema";
-import { QdrantVectorStore } from "llamaindex/storage/index";
+import { QdrantVectorStore } from "llamaindex/vector-store";
export class TestableQdrantVectorStore extends QdrantVectorStore {
public nodes: BaseNode[] = [];
diff --git a/packages/llamaindex/tests/vectorStores/MilvusVectorStore.test.ts b/packages/llamaindex/tests/vector-stores/MilvusVectorStore.test.ts
similarity index 100%
rename from packages/llamaindex/tests/vectorStores/MilvusVectorStore.test.ts
rename to packages/llamaindex/tests/vector-stores/MilvusVectorStore.test.ts
diff --git a/packages/llamaindex/tests/vectorStores/QdrantVectorStore.test.ts b/packages/llamaindex/tests/vector-stores/QdrantVectorStore.test.ts
similarity index 98%
rename from packages/llamaindex/tests/vectorStores/QdrantVectorStore.test.ts
rename to packages/llamaindex/tests/vector-stores/QdrantVectorStore.test.ts
index d7d4e9a865f37b6b5e918cc388630269587df70d..569d26479b88898fdf4ce17f7f79a42423eeb54b 100644
--- a/packages/llamaindex/tests/vectorStores/QdrantVectorStore.test.ts
+++ b/packages/llamaindex/tests/vector-stores/QdrantVectorStore.test.ts
@@ -4,7 +4,7 @@ import type { Mocked } from "vitest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { QdrantClient } from "@qdrant/js-client-rest";
-import { VectorStoreQueryMode } from "llamaindex/storage/index";
+import { VectorStoreQueryMode } from "llamaindex/vector-store";
import { TestableQdrantVectorStore } from "../mocks/TestableQdrantVectorStore.js";
vi.mock("@qdrant/js-client-rest");
diff --git a/packages/llamaindex/tests/vectorStores/SimpleVectorStore.test.ts b/packages/llamaindex/tests/vector-stores/SimpleVectorStore.test.ts
similarity index 100%
rename from packages/llamaindex/tests/vectorStores/SimpleVectorStore.test.ts
rename to packages/llamaindex/tests/vector-stores/SimpleVectorStore.test.ts
diff --git a/packages/llamaindex/tests/vector-stores/pg-vector-store.test.ts b/packages/llamaindex/tests/vector-stores/pg-vector-store.test.ts
new file mode 100644
index 0000000000000000000000000000000000000000..82b577e480361d1eae64f985f2a1881ad659e7df
--- /dev/null
+++ b/packages/llamaindex/tests/vector-stores/pg-vector-store.test.ts
@@ -0,0 +1,85 @@
+import { Document } from "llamaindex";
+import { PGVectorStore } from "llamaindex/vector-store/PGVectorStore";
+import pg from "pg";
+import { registerTypes } from "pgvector/pg";
+import { describe, expect, test } from "vitest";
+import { VectorStoreQueryMode } from "../../src/index.js";
+
+describe("pg - init", () => {
+ test("init with client", async () => {
+ const client = new pg.Client({
+ database: "llamaindex_node_test",
+ });
+ await client.connect();
+ await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+ await registerTypes(client);
+ const vectorStore = new PGVectorStore(client);
+ expect(await vectorStore.client()).toBe(client);
+ });
+
+ test("init with pool", async () => {
+ const pool = new pg.Pool({
+ database: "llamaindex_node_test",
+ });
+ await pool.query("CREATE EXTENSION IF NOT EXISTS vector");
+ const client = await pool.connect();
+ await registerTypes(client);
+ const vectorStore = new PGVectorStore(client);
+ expect(await vectorStore.client()).toBe(client);
+ });
+
+ test("init without client", async () => {
+ const vectorStore = new PGVectorStore({
+ database: "llamaindex_node_test",
+ });
+ expect(await vectorStore.client()).toBeDefined();
+ });
+});
+
+describe("pg - save data", () => {
+ test("simple node", async () => {
+ const dimensions = 3;
+ const schemaName =
+ "llamaindex_vector_store_test_" + Math.random().toString(36).substring(7);
+ const nodeId = "5bb16627-f6c0-459c-bb18-71642813ef21";
+ const node = new Document({
+ text: "hello world",
+ id_: nodeId,
+ embedding: [0.1, 0.2, 0.3],
+ });
+ const vectorStore = new PGVectorStore({
+ database: "llamaindex_node_test",
+ dimensions,
+ schemaName,
+ });
+
+ await vectorStore.add([node]);
+
+ {
+ const result = await vectorStore.query({
+ mode: VectorStoreQueryMode.DEFAULT,
+ similarityTopK: 1,
+ queryEmbedding: [1, 2, 3],
+ });
+ const actualJSON = result.nodes![0]!.toJSON();
+ expect(actualJSON).toEqual({
+ ...node.toJSON(),
+ hash: actualJSON.hash,
+ metadata: actualJSON.metadata,
+ });
+ expect(result.ids).toEqual([nodeId]);
+ expect(result.similarities).toEqual([1]);
+ }
+
+ await vectorStore.delete(nodeId);
+
+ {
+ const result = await vectorStore.query({
+ mode: VectorStoreQueryMode.DEFAULT,
+ similarityTopK: 1,
+ queryEmbedding: [1, 2, 3],
+ });
+ expect(result.nodes).toEqual([]);
+ }
+ });
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f46c0c22f1ca94927ebdd7a3742d60e881f7d0d0..9d1561587cb172dbce886698d4664610c9ce3cc0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -167,7 +167,7 @@ importers:
version: link:../packages/llamaindex
mongodb:
specifier: ^6.7.0
- version: 6.8.0(@aws-sdk/credential-providers@3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0)))
+ version: 6.8.0(@aws-sdk/credential-providers@3.637.0)
pathe:
specifier: ^1.1.2
version: 1.1.2
@@ -581,7 +581,7 @@ importers:
version: 2.0.0
mongodb:
specifier: ^6.7.0
- version: 6.8.0(@aws-sdk/credential-providers@3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0)))
+ version: 6.8.0(@aws-sdk/credential-providers@3.637.0)
notion-md-crawler:
specifier: ^1.0.0
version: 1.0.0(encoding@0.1.13)
@@ -594,12 +594,6 @@ importers:
pathe:
specifier: ^1.1.2
version: 1.1.2
- pg:
- specifier: ^8.12.0
- version: 8.12.0
- pgvector:
- specifier: ^0.2.0
- version: 0.2.0
portkey-ai:
specifier: 0.1.16
version: 0.1.16
@@ -643,6 +637,12 @@ importers:
glob:
specifier: ^11.0.0
version: 11.0.0
+ pg:
+ specifier: ^8.12.0
+ version: 8.12.0
+ pgvector:
+ specifier: 0.2.0
+ version: 0.2.0
typescript:
specifier: ^5.5.4
version: 5.5.4
@@ -17747,16 +17747,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- eslint-module-utils@2.8.2(@typescript-eslint/parser@8.3.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0):
- dependencies:
- debug: 3.2.7
- optionalDependencies:
- '@typescript-eslint/parser': 8.3.0(eslint@8.57.0)(typescript@5.5.4)
- eslint: 8.57.0
- eslint-import-resolver-node: 0.3.9
- transitivePeerDependencies:
- - supports-color
-
eslint-plugin-import@2.29.1(@typescript-eslint/parser@8.3.0(eslint@8.57.0)(typescript@5.5.4))(eslint@8.57.0):
dependencies:
array-includes: 3.1.8
@@ -17767,7 +17757,7 @@ snapshots:
doctrine: 2.1.0
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
- eslint-module-utils: 2.8.2(@typescript-eslint/parser@8.3.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0)
+ eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
hasown: 2.0.2
is-core-module: 2.15.1
is-glob: 4.0.3
@@ -20495,7 +20485,7 @@ snapshots:
optionalDependencies:
'@aws-sdk/credential-providers': 3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0))
- mongodb@6.8.0(@aws-sdk/credential-providers@3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0))):
+ mongodb@6.8.0(@aws-sdk/credential-providers@3.637.0):
dependencies:
'@mongodb-js/saslprep': 1.1.7
bson: 6.8.0