From 94566169fb6883beb72bf7a852b2ffeecf7d4c44 Mon Sep 17 00:00:00 2001
From: Thuc Pham <51660321+thucpn@users.noreply.github.com>
Date: Mon, 20 Jan 2025 12:24:44 +0700
Subject: [PATCH] chore: move postgres storage classes to @llamaindex/postgres
(#1597)
---
.changeset/polite-coats-return.md | 8 +
.github/workflows/test.yml | 2 +-
e2e/node/vector-store/pg-vector-store.e2e.ts | 3 +-
examples/vector-store/pg/README.md | 21 +-
examples/vector-store/pg/supabase.ts | 34 +++
packages/core/src/global/settings.ts | 15 ++
.../src/global/settings/embedModel.ts} | 11 +-
.../storage/doc-store/base-document-store.ts | 167 +++++++++++++++
packages/core/src/storage/doc-store/index.ts | 169 +--------------
.../storage/doc-store/kv-document-store.ts} | 40 ++--
packages/core/src/vector-store/index.ts | 107 ++++++++++
packages/llamaindex/package.json | 18 +-
packages/llamaindex/src/Settings.ts | 11 +-
packages/llamaindex/src/index.edge.ts | 5 +-
.../src/indices/vectorStore/index.ts | 2 +-
.../src/ingestion/IngestionPipeline.ts | 2 +-
.../strategies/UpsertsAndDeleteStrategy.ts | 2 +-
.../ingestion/strategies/UpsertsStrategy.ts | 2 +-
.../src/ingestion/strategies/index.ts | 2 +-
.../llamaindex/src/storage/StorageContext.ts | 8 +-
.../AzureCosmosMongovCoreDocumentStore.ts | 2 +-
.../docStore/AzureCosmosNoSqlDocumentStore.ts | 2 +-
.../storage/docStore/SimpleDocumentStore.ts | 2 +-
packages/llamaindex/src/storage/index.ts | 8 +-
.../src/vector-store/AstraDBVectorStore.ts | 4 +-
.../AzureCosmosDBMongoVectorStore.ts | 6 +-
.../AzureCosmosDBNoSqlVectorStore.ts | 2 +-
.../src/vector-store/ChromaVectorStore.ts | 20 +-
.../src/vector-store/MilvusVectorStore.ts | 14 +-
.../vector-store/MongoDBAtlasVectorStore.ts | 8 +-
.../src/vector-store/PineconeVectorStore.ts | 2 +-
.../src/vector-store/QdrantVectorStore.ts | 2 +-
.../src/vector-store/SimpleVectorStore.ts | 6 +-
.../src/vector-store/UpstashVectorStore.ts | 2 +-
.../src/vector-store/WeaviateVectorStore.ts | 6 +-
.../azure/AzureAISearchVectorStore.ts | 4 +-
.../azure/AzureQueryResultSearch.ts | 4 +-
packages/llamaindex/src/vector-store/index.ts | 10 +-
packages/llamaindex/src/vector-store/types.ts | 106 ---------
packages/llamaindex/src/vector-store/utils.ts | 2 +-
.../llamaindex/tests/StorageContext.test.ts | 5 +-
.../TestableAzureCosmosDBNoSqlVectorStore.ts | 2 +-
.../tests/mocks/TestableQdrantVectorStore.ts | 2 +-
.../AzureAISearchVectorStore.test.ts | 2 +-
.../providers/storage/postgres/package.json | 66 ++++++
.../storage/postgres/src}/PGVectorStore.ts | 16 +-
.../postgres/src}/PostgresDocumentStore.ts | 9 +-
.../postgres/src}/PostgresIndexStore.ts | 7 +-
.../storage/postgres/src}/PostgresKVStore.ts | 0
.../providers/storage/postgres/src/index.ts | 4 +
.../providers/storage/postgres/tsconfig.json | 19 ++
pnpm-lock.yaml | 202 +++++++++---------
pnpm-workspace.yaml | 1 +
tsconfig.json | 3 +
54 files changed, 674 insertions(+), 505 deletions(-)
create mode 100644 .changeset/polite-coats-return.md
create mode 100644 examples/vector-store/pg/supabase.ts
rename packages/{llamaindex/src/internal/settings/EmbedModel.ts => core/src/global/settings/embedModel.ts} (67%)
create mode 100644 packages/core/src/storage/doc-store/base-document-store.ts
rename packages/{llamaindex/src/storage/docStore/KVDocumentStore.ts => core/src/storage/doc-store/kv-document-store.ts} (85%)
delete mode 100644 packages/llamaindex/src/vector-store/types.ts
create mode 100644 packages/providers/storage/postgres/package.json
rename packages/{llamaindex/src/vector-store => providers/storage/postgres/src}/PGVectorStore.ts (98%)
rename packages/{llamaindex/src/storage/docStore => providers/storage/postgres/src}/PostgresDocumentStore.ts (80%)
rename packages/{llamaindex/src/storage/indexStore => providers/storage/postgres/src}/PostgresIndexStore.ts (84%)
rename packages/{llamaindex/src/storage/kvStore => providers/storage/postgres/src}/PostgresKVStore.ts (100%)
create mode 100644 packages/providers/storage/postgres/src/index.ts
create mode 100644 packages/providers/storage/postgres/tsconfig.json
diff --git a/.changeset/polite-coats-return.md b/.changeset/polite-coats-return.md
new file mode 100644
index 000000000..2a1186690
--- /dev/null
+++ b/.changeset/polite-coats-return.md
@@ -0,0 +1,8 @@
+---
+"@llamaindex/e2e": patch
+"@llamaindex/core": patch
+"llamaindex": patch
+"pg-vector-store": patch
+---
+
+refactor: @llamaindex/postgres
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4e5162b20..bc6c6ede6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -150,7 +150,7 @@ jobs:
done
- name: Pack provider packages
run: |
- for dir in packages/providers/*; do
+ for dir in packages/providers/* packages/providers/storage/*; do
if [ -d "$dir" ] && [ -f "$dir/package.json" ]; then
echo "Packing $dir"
pnpm pack --pack-destination ${{ runner.temp }} -C $dir
diff --git a/e2e/node/vector-store/pg-vector-store.e2e.ts b/e2e/node/vector-store/pg-vector-store.e2e.ts
index b1725a1a8..8bcb25976 100644
--- a/e2e/node/vector-store/pg-vector-store.e2e.ts
+++ b/e2e/node/vector-store/pg-vector-store.e2e.ts
@@ -1,6 +1,5 @@
import { config } from "dotenv";
-import { Document, VectorStoreQueryMode } from "llamaindex";
-import { PGVectorStore } from "llamaindex/vector-store/PGVectorStore";
+import { Document, PGVectorStore, VectorStoreQueryMode } from "llamaindex";
import assert from "node:assert";
import { test } from "node:test";
import pg from "pg";
diff --git a/examples/vector-store/pg/README.md b/examples/vector-store/pg/README.md
index dc636a4ff..b769196cd 100644
--- a/examples/vector-store/pg/README.md
+++ b/examples/vector-store/pg/README.md
@@ -37,7 +37,7 @@ Read and follow the instructions in the README.md file located one directory up
To import documents and save the embedding vectors to your database:
-> `npx tsx pg-vector-store/load-docs.ts data`
+> `npx tsx vector-store/pg/load-docs.ts data`
where data is the directory containing your input files. Using the `data` directory in the example above will read all of the files in that directory using the LlamaIndexTS default readers for each file type.
@@ -45,6 +45,23 @@ where data is the directory containing your input files. Using the `data` direct
To query using the resulting vector store:
-> `npx tsx pg-vector-store/query.ts`
+> `npx tsx vector-store/pg/query.ts`
The script will prompt for a question, then process and present the answer using the PGVectorStore data and your OpenAI API key. It will continue to prompt until you enter `q`, `quit` or `exit` as the next query.
+
+## Supabase
+
+You can try the supabase example by running:
+
+> `npx tsx vector-store/pg/supabase.ts`
+
+This will use the `POSTGRES_URL` environment variable to connect to your Supabase database.
+Get one from the Supabase project settings page. See more details here: https://supabase.com/docs/guides/database/connecting-to-postgres#direct-connection
+
+## Vercel
+
+You can try the vercel example by running:
+
+> `npx tsx vector-store/pg/vercel.ts`
+
+For more information on Vercel Postgres, see: https://vercel.com/docs/storage/vercel-postgres/sdk
diff --git a/examples/vector-store/pg/supabase.ts b/examples/vector-store/pg/supabase.ts
new file mode 100644
index 000000000..748b48362
--- /dev/null
+++ b/examples/vector-store/pg/supabase.ts
@@ -0,0 +1,34 @@
+import dotenv from "dotenv";
+import {
+ PGVectorStore,
+ SimpleDirectoryReader,
+ storageContextFromDefaults,
+ VectorStoreIndex,
+} from "llamaindex";
+
+dotenv.config();
+
+// Get direct connection string from Supabase and set it as POSTGRES_URL environment variable
+// https://supabase.com/docs/guides/database/connecting-to-postgres#direct-connection
+
+const sourceDir = "../data";
+const connectionString = process.env.POSTGRES_URL;
+
+const rdr = new SimpleDirectoryReader();
+const docs = await rdr.loadData({ directoryPath: sourceDir });
+const pgvs = new PGVectorStore({ clientConfig: { connectionString } });
+pgvs.setCollection(sourceDir);
+
+const ctx = await storageContextFromDefaults({ vectorStore: pgvs });
+
+const index = await VectorStoreIndex.fromDocuments(docs, {
+ storageContext: ctx,
+});
+
+const queryEngine = index.asQueryEngine();
+
+const results = await queryEngine.query({
+ query: "Information about the planet",
+});
+
+console.log(results);
diff --git a/packages/core/src/global/settings.ts b/packages/core/src/global/settings.ts
index 3a3af7ccf..3cb97b880 100644
--- a/packages/core/src/global/settings.ts
+++ b/packages/core/src/global/settings.ts
@@ -1,5 +1,6 @@
import { getEnv } from "@llamaindex/env";
import type { Tokenizer } from "@llamaindex/env/tokenizers";
+import type { BaseEmbedding } from "../embeddings";
import type { LLM } from "../llms";
import {
type CallbackManager,
@@ -12,6 +13,11 @@ import {
setChunkSize,
withChunkSize,
} from "./settings/chunk-size";
+import {
+ getEmbeddedModel,
+ setEmbeddedModel,
+ withEmbeddedModel,
+} from "./settings/embedModel";
import { getLLM, setLLM, withLLM } from "./settings/llm";
import {
getTokenizer,
@@ -29,6 +35,15 @@ export const Settings = {
withLLM<Result>(llm: LLM, fn: () => Result): Result {
return withLLM(llm, fn);
},
+ get embedModel() {
+ return getEmbeddedModel();
+ },
+ set embedModel(embedModel) {
+ setEmbeddedModel(embedModel);
+ },
+ withEmbedModel<Result>(embedModel: BaseEmbedding, fn: () => Result): Result {
+ return withEmbeddedModel(embedModel, fn);
+ },
get tokenizer() {
return getTokenizer();
},
diff --git a/packages/llamaindex/src/internal/settings/EmbedModel.ts b/packages/core/src/global/settings/embedModel.ts
similarity index 67%
rename from packages/llamaindex/src/internal/settings/EmbedModel.ts
rename to packages/core/src/global/settings/embedModel.ts
index d912d20b9..deb80a298 100644
--- a/packages/llamaindex/src/internal/settings/EmbedModel.ts
+++ b/packages/core/src/global/settings/embedModel.ts
@@ -1,15 +1,18 @@
import type { BaseEmbedding } from "@llamaindex/core/embeddings";
import { AsyncLocalStorage } from "@llamaindex/env";
-import { OpenAIEmbedding } from "@llamaindex/openai";
const embeddedModelAsyncLocalStorage = new AsyncLocalStorage<BaseEmbedding>();
let globalEmbeddedModel: BaseEmbedding | null = null;
export function getEmbeddedModel(): BaseEmbedding {
- if (globalEmbeddedModel === null) {
- globalEmbeddedModel = new OpenAIEmbedding();
+ const currentEmbeddedModel =
+ embeddedModelAsyncLocalStorage.getStore() ?? globalEmbeddedModel;
+ if (!currentEmbeddedModel) {
+ throw new Error(
+ "Cannot find Embedding, please set `Settings.embedModel = ...` on the top of your code",
+ );
}
- return embeddedModelAsyncLocalStorage.getStore() ?? globalEmbeddedModel;
+ return currentEmbeddedModel;
}
export function setEmbeddedModel(embeddedModel: BaseEmbedding) {
diff --git a/packages/core/src/storage/doc-store/base-document-store.ts b/packages/core/src/storage/doc-store/base-document-store.ts
new file mode 100644
index 000000000..ffb6725de
--- /dev/null
+++ b/packages/core/src/storage/doc-store/base-document-store.ts
@@ -0,0 +1,167 @@
+import { path } from "@llamaindex/env";
+import {
+ DEFAULT_DOC_STORE_PERSIST_FILENAME,
+ DEFAULT_PERSIST_DIR,
+} from "../../global";
+import type { StoredValue } from "../../schema";
+import { BaseNode, Document, ObjectType, TextNode } from "../../schema";
+
+const TYPE_KEY = "__type__";
+const DATA_KEY = "__data__";
+
+export interface Serializer<T> {
+ toPersistence(data: Record<string, unknown>): T;
+
+ fromPersistence(data: T): Record<string, unknown>;
+}
+
+export const jsonSerializer: Serializer<string> = {
+ toPersistence(data) {
+ return JSON.stringify(data);
+ },
+ fromPersistence(data) {
+ return JSON.parse(data);
+ },
+};
+
+export const noneSerializer: Serializer<Record<string, unknown>> = {
+ toPersistence(data) {
+ return data;
+ },
+ fromPersistence(data) {
+ return data;
+ },
+};
+
+type DocJson<Data> = {
+ [TYPE_KEY]: ObjectType;
+ [DATA_KEY]: Data;
+};
+
+export function isValidDocJson(
+ docJson: StoredValue | null | undefined,
+): docJson is DocJson<unknown> {
+ return (
+ typeof docJson === "object" &&
+ docJson !== null &&
+ docJson[TYPE_KEY] !== undefined &&
+ docJson[DATA_KEY] !== undefined
+ );
+}
+
+export function docToJson(
+ doc: BaseNode,
+ serializer: Serializer<unknown>,
+): DocJson<unknown> {
+ return {
+ [DATA_KEY]: serializer.toPersistence(doc.toJSON()),
+ [TYPE_KEY]: doc.type,
+ };
+}
+
+export function jsonToDoc<Data>(
+ docDict: DocJson<Data>,
+ serializer: Serializer<Data>,
+): BaseNode {
+ const docType = docDict[TYPE_KEY];
+ // fixme: zod type check this
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const dataDict: any = serializer.fromPersistence(docDict[DATA_KEY]);
+ let doc: BaseNode;
+
+ if (docType === ObjectType.DOCUMENT) {
+ doc = new Document({
+ text: dataDict.text,
+ id_: dataDict.id_,
+ embedding: dataDict.embedding,
+ hash: dataDict.hash,
+ metadata: dataDict.metadata,
+ });
+ } else if (docType === ObjectType.TEXT) {
+ doc = new TextNode({
+ text: dataDict.text,
+ id_: dataDict.id_,
+ hash: dataDict.hash,
+ metadata: dataDict.metadata,
+ relationships: dataDict.relationships,
+ });
+ } else {
+ throw new Error(`Unknown doc type: ${docType}`);
+ }
+
+ return doc;
+}
+
+const DEFAULT_PERSIST_PATH = path.join(
+ DEFAULT_PERSIST_DIR,
+ DEFAULT_DOC_STORE_PERSIST_FILENAME,
+);
+
+export interface RefDocInfo {
+ nodeIds: string[];
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ extraInfo: Record<string, any>;
+}
+
+export abstract class BaseDocumentStore {
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ serializer: Serializer<any> = jsonSerializer;
+
+ // Save/load
+ persist(persistPath: string = DEFAULT_PERSIST_PATH): void {
+ // Persist the docstore to a file.
+ }
+
+ // Main interface
+ abstract docs(): Promise<Record<string, BaseNode>>;
+
+ abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): Promise<void>;
+
+ abstract getDocument(
+ docId: string,
+ raiseError: boolean,
+ ): Promise<BaseNode | undefined>;
+
+ abstract deleteDocument(docId: string, raiseError: boolean): Promise<void>;
+
+ abstract documentExists(docId: string): Promise<boolean>;
+
+ // Hash
+ abstract setDocumentHash(docId: string, docHash: string): Promise<void>;
+
+ abstract getDocumentHash(docId: string): Promise<string | undefined>;
+
+ abstract getAllDocumentHashes(): Promise<Record<string, string>>;
+
+ // Ref Docs
+ abstract getAllRefDocInfo(): Promise<Record<string, RefDocInfo> | undefined>;
+
+ abstract getRefDocInfo(refDocId: string): Promise<RefDocInfo | undefined>;
+
+ abstract deleteRefDoc(refDocId: string, raiseError: boolean): Promise<void>;
+
+ // Nodes
+ getNodes(nodeIds: string[], raiseError: boolean = true): Promise<BaseNode[]> {
+ return Promise.all(
+ nodeIds.map((nodeId) => this.getNode(nodeId, raiseError)),
+ );
+ }
+
+ async getNode(nodeId: string, raiseError: boolean = true): Promise<BaseNode> {
+ const doc = await this.getDocument(nodeId, raiseError);
+ if (!(doc instanceof BaseNode)) {
+ throw new Error(`Document ${nodeId} is not a Node.`);
+ }
+ return doc;
+ }
+
+ async getNodeDict(nodeIdDict: {
+ [index: number]: string;
+ }): Promise<Record<number, BaseNode>> {
+ const result: Record<number, BaseNode> = {};
+ for (const index in nodeIdDict) {
+ result[index] = await this.getNode(nodeIdDict[index]!);
+ }
+ return result;
+ }
+}
diff --git a/packages/core/src/storage/doc-store/index.ts b/packages/core/src/storage/doc-store/index.ts
index ffb6725de..bec1bbc40 100644
--- a/packages/core/src/storage/doc-store/index.ts
+++ b/packages/core/src/storage/doc-store/index.ts
@@ -1,167 +1,2 @@
-import { path } from "@llamaindex/env";
-import {
- DEFAULT_DOC_STORE_PERSIST_FILENAME,
- DEFAULT_PERSIST_DIR,
-} from "../../global";
-import type { StoredValue } from "../../schema";
-import { BaseNode, Document, ObjectType, TextNode } from "../../schema";
-
-const TYPE_KEY = "__type__";
-const DATA_KEY = "__data__";
-
-export interface Serializer<T> {
- toPersistence(data: Record<string, unknown>): T;
-
- fromPersistence(data: T): Record<string, unknown>;
-}
-
-export const jsonSerializer: Serializer<string> = {
- toPersistence(data) {
- return JSON.stringify(data);
- },
- fromPersistence(data) {
- return JSON.parse(data);
- },
-};
-
-export const noneSerializer: Serializer<Record<string, unknown>> = {
- toPersistence(data) {
- return data;
- },
- fromPersistence(data) {
- return data;
- },
-};
-
-type DocJson<Data> = {
- [TYPE_KEY]: ObjectType;
- [DATA_KEY]: Data;
-};
-
-export function isValidDocJson(
- docJson: StoredValue | null | undefined,
-): docJson is DocJson<unknown> {
- return (
- typeof docJson === "object" &&
- docJson !== null &&
- docJson[TYPE_KEY] !== undefined &&
- docJson[DATA_KEY] !== undefined
- );
-}
-
-export function docToJson(
- doc: BaseNode,
- serializer: Serializer<unknown>,
-): DocJson<unknown> {
- return {
- [DATA_KEY]: serializer.toPersistence(doc.toJSON()),
- [TYPE_KEY]: doc.type,
- };
-}
-
-export function jsonToDoc<Data>(
- docDict: DocJson<Data>,
- serializer: Serializer<Data>,
-): BaseNode {
- const docType = docDict[TYPE_KEY];
- // fixme: zod type check this
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- const dataDict: any = serializer.fromPersistence(docDict[DATA_KEY]);
- let doc: BaseNode;
-
- if (docType === ObjectType.DOCUMENT) {
- doc = new Document({
- text: dataDict.text,
- id_: dataDict.id_,
- embedding: dataDict.embedding,
- hash: dataDict.hash,
- metadata: dataDict.metadata,
- });
- } else if (docType === ObjectType.TEXT) {
- doc = new TextNode({
- text: dataDict.text,
- id_: dataDict.id_,
- hash: dataDict.hash,
- metadata: dataDict.metadata,
- relationships: dataDict.relationships,
- });
- } else {
- throw new Error(`Unknown doc type: ${docType}`);
- }
-
- return doc;
-}
-
-const DEFAULT_PERSIST_PATH = path.join(
- DEFAULT_PERSIST_DIR,
- DEFAULT_DOC_STORE_PERSIST_FILENAME,
-);
-
-export interface RefDocInfo {
- nodeIds: string[];
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- extraInfo: Record<string, any>;
-}
-
-export abstract class BaseDocumentStore {
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- serializer: Serializer<any> = jsonSerializer;
-
- // Save/load
- persist(persistPath: string = DEFAULT_PERSIST_PATH): void {
- // Persist the docstore to a file.
- }
-
- // Main interface
- abstract docs(): Promise<Record<string, BaseNode>>;
-
- abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): Promise<void>;
-
- abstract getDocument(
- docId: string,
- raiseError: boolean,
- ): Promise<BaseNode | undefined>;
-
- abstract deleteDocument(docId: string, raiseError: boolean): Promise<void>;
-
- abstract documentExists(docId: string): Promise<boolean>;
-
- // Hash
- abstract setDocumentHash(docId: string, docHash: string): Promise<void>;
-
- abstract getDocumentHash(docId: string): Promise<string | undefined>;
-
- abstract getAllDocumentHashes(): Promise<Record<string, string>>;
-
- // Ref Docs
- abstract getAllRefDocInfo(): Promise<Record<string, RefDocInfo> | undefined>;
-
- abstract getRefDocInfo(refDocId: string): Promise<RefDocInfo | undefined>;
-
- abstract deleteRefDoc(refDocId: string, raiseError: boolean): Promise<void>;
-
- // Nodes
- getNodes(nodeIds: string[], raiseError: boolean = true): Promise<BaseNode[]> {
- return Promise.all(
- nodeIds.map((nodeId) => this.getNode(nodeId, raiseError)),
- );
- }
-
- async getNode(nodeId: string, raiseError: boolean = true): Promise<BaseNode> {
- const doc = await this.getDocument(nodeId, raiseError);
- if (!(doc instanceof BaseNode)) {
- throw new Error(`Document ${nodeId} is not a Node.`);
- }
- return doc;
- }
-
- async getNodeDict(nodeIdDict: {
- [index: number]: string;
- }): Promise<Record<number, BaseNode>> {
- const result: Record<number, BaseNode> = {};
- for (const index in nodeIdDict) {
- result[index] = await this.getNode(nodeIdDict[index]!);
- }
- return result;
- }
-}
+export * from "./base-document-store";
+export * from "./kv-document-store";
diff --git a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts b/packages/core/src/storage/doc-store/kv-document-store.ts
similarity index 85%
rename from packages/llamaindex/src/storage/docStore/KVDocumentStore.ts
rename to packages/core/src/storage/doc-store/kv-document-store.ts
index 6657a38a2..ba7dd77a5 100644
--- a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts
+++ b/packages/core/src/storage/doc-store/kv-document-store.ts
@@ -1,15 +1,13 @@
-import { DEFAULT_NAMESPACE } from "@llamaindex/core/global";
-import type { BaseNode } from "@llamaindex/core/schema";
-import { ObjectType } from "@llamaindex/core/schema";
-import type { RefDocInfo } from "@llamaindex/core/storage/doc-store";
+import { DEFAULT_NAMESPACE } from "../../global";
+import { BaseNode, ObjectType, type StoredValue } from "../../schema";
+import type { BaseKVStore } from "../kv-store";
import {
BaseDocumentStore,
docToJson,
isValidDocJson,
jsonToDoc,
-} from "@llamaindex/core/storage/doc-store";
-import type { BaseKVStore } from "@llamaindex/core/storage/kv-store";
-import _ from "lodash";
+ type RefDocInfo,
+} from "./base-document-store";
type DocMetaData = { docHash: string; refDocId?: string };
@@ -68,7 +66,7 @@ export class KVDocumentStore extends BaseDocumentStore {
extraInfo: {},
};
refDocInfo.nodeIds.push(doc.id_);
- if (_.isEmpty(refDocInfo.extraInfo)) {
+ if (Object.keys(refDocInfo.extraInfo).length === 0) {
refDocInfo.extraInfo = {};
}
await this.kvstore.put(
@@ -88,7 +86,7 @@ export class KVDocumentStore extends BaseDocumentStore {
raiseError: boolean = true,
): Promise<BaseNode | undefined> {
const json = await this.kvstore.get(docId, this.nodeCollection);
- if (_.isNil(json)) {
+ if (this.isNil(json)) {
if (raiseError) {
throw new Error(`docId ${docId} not found.`);
} else {
@@ -103,23 +101,23 @@ export class KVDocumentStore extends BaseDocumentStore {
async getRefDocInfo(refDocId: string): Promise<RefDocInfo | undefined> {
const refDocInfo = await this.kvstore.get(refDocId, this.refDocCollection);
- return refDocInfo ? (_.clone(refDocInfo) as RefDocInfo) : undefined;
+ return refDocInfo ? (structuredClone(refDocInfo) as RefDocInfo) : undefined;
}
async getAllRefDocInfo(): Promise<Record<string, RefDocInfo> | undefined> {
const refDocInfos = await this.kvstore.getAll(this.refDocCollection);
- if (_.isNil(refDocInfos)) {
+ if (this.isNil(refDocInfos)) {
return;
}
return refDocInfos as Record<string, RefDocInfo>;
}
async refDocExists(refDocId: string): Promise<boolean> {
- return !_.isNil(await this.getRefDocInfo(refDocId));
+ return !this.isNil(await this.getRefDocInfo(refDocId));
}
async documentExists(docId: string): Promise<boolean> {
- return !_.isNil(await this.kvstore.get(docId, this.nodeCollection));
+ return !this.isNil(await this.kvstore.get(docId, this.nodeCollection));
}
private async removeRefDocNode(docId: string): Promise<void> {
@@ -129,13 +127,13 @@ export class KVDocumentStore extends BaseDocumentStore {
}
const refDocId = metadata.refDocId;
- if (_.isNil(refDocId)) {
+ if (this.isNil(refDocId)) {
return;
}
const refDocInfo = await this.kvstore.get(refDocId, this.refDocCollection);
- if (!_.isNil(refDocInfo)) {
- if (refDocInfo.nodeIds.length > 0) {
+ if (!this.isNil(refDocInfo)) {
+ if (refDocInfo!.nodeIds.length > 0) {
await this.kvstore.put(refDocId, refDocInfo, this.refDocCollection);
}
await this.kvstore.delete(refDocId, this.metadataCollection);
@@ -164,7 +162,7 @@ export class KVDocumentStore extends BaseDocumentStore {
raiseError: boolean = true,
): Promise<void> {
const refDocInfo = await this.getRefDocInfo(refDocId);
- if (_.isNil(refDocInfo)) {
+ if (this.isNil(refDocInfo)) {
if (raiseError) {
throw new Error(`ref_doc_id ${refDocId} not found.`);
} else {
@@ -172,7 +170,7 @@ export class KVDocumentStore extends BaseDocumentStore {
}
}
- for (const docId of refDocInfo.nodeIds) {
+ for (const docId of refDocInfo!.nodeIds) {
await this.deleteDocument(docId, false, false);
}
@@ -187,7 +185,7 @@ export class KVDocumentStore extends BaseDocumentStore {
async getDocumentHash(docId: string): Promise<string | undefined> {
const metadata = await this.kvstore.get(docId, this.metadataCollection);
- return _.get(metadata, "docHash");
+ return metadata?.docHash;
}
async getAllDocumentHashes(): Promise<Record<string, string>> {
@@ -201,4 +199,8 @@ export class KVDocumentStore extends BaseDocumentStore {
}
return hashes;
}
+
+ private isNil(value: RefDocInfo | StoredValue | undefined): boolean {
+ return value === null || value === undefined;
+ }
}
diff --git a/packages/core/src/vector-store/index.ts b/packages/core/src/vector-store/index.ts
index 41542521d..151e89fa7 100644
--- a/packages/core/src/vector-store/index.ts
+++ b/packages/core/src/vector-store/index.ts
@@ -1,3 +1,7 @@
+import type { BaseEmbedding } from "../embeddings/base.js";
+import { Settings } from "../global";
+import type { BaseNode, ModalityType } from "../schema/node.js";
+
/**
* should compatible with npm:pg and npm:postgres
*/
@@ -12,3 +16,106 @@ export interface IsomorphicDB {
close: () => Promise<void>;
onCloseEvent: (listener: () => void) => void;
}
+
+export interface VectorStoreQueryResult {
+ nodes?: BaseNode[];
+ similarities: number[];
+ ids: string[];
+}
+
+export enum VectorStoreQueryMode {
+ DEFAULT = "default",
+ SPARSE = "sparse",
+ HYBRID = "hybrid",
+ // fit learners
+ SVM = "svm",
+ LOGISTIC_REGRESSION = "logistic_regression",
+ LINEAR_REGRESSION = "linear_regression",
+ // maximum marginal relevance
+ MMR = "mmr",
+
+ // for Azure AI Search
+ SEMANTIC_HYBRID = "semantic_hybrid",
+}
+
+export enum FilterOperator {
+ EQ = "==", // default operator (string, number)
+ IN = "in", // In array (string or number)
+ GT = ">", // greater than (number)
+ LT = "<", // less than (number)
+ NE = "!=", // not equal to (string, number)
+ GTE = ">=", // greater than or equal to (number)
+ LTE = "<=", // less than or equal to (number)
+ NIN = "nin", // Not in array (string or number)
+ ANY = "any", // Contains any (array of strings)
+ ALL = "all", // Contains all (array of strings)
+ TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field)
+ CONTAINS = "contains", // metadata array contains value (string or number)
+ IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array)
+}
+
+export enum FilterCondition {
+ AND = "and",
+ OR = "or",
+}
+
+export type MetadataFilterValue = string | number | string[] | number[];
+
+export interface MetadataFilter {
+ key: string;
+ value?: MetadataFilterValue;
+ operator: `${FilterOperator}`; // ==, any, all,...
+}
+
+export interface MetadataFilters {
+ filters: Array<MetadataFilter>;
+ condition?: `${FilterCondition}`; // and, or
+}
+
+export interface MetadataInfo {
+ name: string;
+ type: string;
+ description: string;
+}
+
+export interface VectorStoreInfo {
+ metadataInfo: MetadataInfo[];
+ contentInfo: string;
+}
+
+export interface VectorStoreQuery {
+ queryEmbedding?: number[];
+ similarityTopK: number;
+ docIds?: string[];
+ queryStr?: string;
+ mode: VectorStoreQueryMode;
+ alpha?: number;
+ filters?: MetadataFilters | undefined;
+ mmrThreshold?: number;
+}
+
+// Supported types of vector stores (for each modality)
+export type VectorStoreByType = {
+ [P in ModalityType]?: BaseVectorStore;
+};
+
+export type VectorStoreBaseParams = {
+ embeddingModel?: BaseEmbedding | undefined;
+};
+
+export abstract class BaseVectorStore<Client = unknown> {
+ embedModel: BaseEmbedding;
+ abstract storesText: boolean;
+ isEmbeddingQuery?: boolean;
+ abstract client(): Client;
+ abstract add(embeddingResults: BaseNode[]): Promise<string[]>;
+ abstract delete(refDocId: string, deleteOptions?: object): Promise<void>;
+ abstract query(
+ query: VectorStoreQuery,
+ options?: object,
+ ): Promise<VectorStoreQueryResult>;
+
+ protected constructor(params?: VectorStoreBaseParams) {
+ this.embedModel = params?.embeddingModel ?? Settings.embedModel;
+ }
+}
diff --git a/packages/llamaindex/package.json b/packages/llamaindex/package.json
index 56995fbab..01d250cac 100644
--- a/packages/llamaindex/package.json
+++ b/packages/llamaindex/package.json
@@ -46,13 +46,13 @@
"@llamaindex/readers": "workspace:*",
"@llamaindex/replicate": "workspace:*",
"@llamaindex/vllm": "workspace:*",
+ "@llamaindex/postgres": "workspace:*",
"@mistralai/mistralai": "^1.3.4",
"@mixedbread-ai/sdk": "^2.2.11",
"@pinecone-database/pinecone": "^4.0.0",
"@qdrant/js-client-rest": "^1.11.0",
"@types/lodash": "^4.17.7",
"@types/node": "^22.9.0",
- "@types/pg": "^8.11.8",
"@upstash/vector": "^1.1.5",
"@zilliz/milvus2-sdk-node": "^2.4.6",
"ajv": "^8.17.1",
@@ -74,27 +74,11 @@
"wink-nlp": "^2.3.0",
"zod": "^3.23.8"
},
- "peerDependencies": {
- "pg": "^8.12.0",
- "pgvector": "0.2.0"
- },
- "peerDependenciesMeta": {
- "pg": {
- "optional": true
- },
- "pgvector": {
- "optional": true
- }
- },
"devDependencies": {
"@swc/cli": "^0.5.0",
"@swc/core": "^1.9.2",
- "@vercel/postgres": "^0.10.0",
"concurrently": "^9.1.0",
"glob": "^11.0.0",
- "pg": "^8.12.0",
- "pgvector": "0.2.0",
- "postgres": "^3.4.4",
"typescript": "^5.7.2"
},
"engines": {
diff --git a/packages/llamaindex/src/Settings.ts b/packages/llamaindex/src/Settings.ts
index 21b31e343..89676036b 100644
--- a/packages/llamaindex/src/Settings.ts
+++ b/packages/llamaindex/src/Settings.ts
@@ -13,11 +13,6 @@ import {
} from "@llamaindex/core/node-parser";
import { AsyncLocalStorage } from "@llamaindex/env";
import type { ServiceContext } from "./ServiceContext.js";
-import {
- getEmbeddedModel,
- setEmbeddedModel,
- withEmbeddedModel,
-} from "./internal/settings/EmbedModel.js";
export type PromptConfig = {
llm?: string;
@@ -84,15 +79,15 @@ class GlobalSettings implements Config {
}
get embedModel(): BaseEmbedding {
- return getEmbeddedModel();
+ return CoreSettings.embedModel;
}
set embedModel(embedModel: BaseEmbedding) {
- setEmbeddedModel(embedModel);
+ CoreSettings.embedModel = embedModel;
}
withEmbedModel<Result>(embedModel: BaseEmbedding, fn: () => Result): Result {
- return withEmbeddedModel(embedModel, fn);
+ return CoreSettings.withEmbedModel(embedModel, fn);
}
get nodeParser(): NodeParser {
diff --git a/packages/llamaindex/src/index.edge.ts b/packages/llamaindex/src/index.edge.ts
index 15f3242ae..302c1fb91 100644
--- a/packages/llamaindex/src/index.edge.ts
+++ b/packages/llamaindex/src/index.edge.ts
@@ -1,12 +1,15 @@
//#region initial setup for OpenAI
-import { OpenAI } from "@llamaindex/openai";
+import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
import { Settings } from "./Settings.js";
try {
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
Settings.llm;
+ // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+ Settings.embedModel;
} catch {
Settings.llm = new OpenAI();
+ Settings.embedModel = new OpenAIEmbedding();
}
//#endregion
diff --git a/packages/llamaindex/src/indices/vectorStore/index.ts b/packages/llamaindex/src/indices/vectorStore/index.ts
index 52bafb239..bdbafd62c 100644
--- a/packages/llamaindex/src/indices/vectorStore/index.ts
+++ b/packages/llamaindex/src/indices/vectorStore/index.ts
@@ -19,6 +19,7 @@ import {
} from "@llamaindex/core/schema";
import type { BaseIndexStore } from "@llamaindex/core/storage/index-store";
import { extractText } from "@llamaindex/core/utils";
+import { VectorStoreQueryMode } from "@llamaindex/core/vector-store";
import type { ServiceContext } from "../../ServiceContext.js";
import { nodeParserFromSettingsOrContext } from "../../Settings.js";
import { RetrieverQueryEngine } from "../../engines/query/RetrieverQueryEngine.js";
@@ -38,7 +39,6 @@ import type {
VectorStoreByType,
VectorStoreQueryResult,
} from "../../vector-store/index.js";
-import { VectorStoreQueryMode } from "../../vector-store/types.js";
import type { BaseIndexInit } from "../BaseIndex.js";
import { BaseIndex } from "../BaseIndex.js";
diff --git a/packages/llamaindex/src/ingestion/IngestionPipeline.ts b/packages/llamaindex/src/ingestion/IngestionPipeline.ts
index bf9cb3a0b..96142fc29 100644
--- a/packages/llamaindex/src/ingestion/IngestionPipeline.ts
+++ b/packages/llamaindex/src/ingestion/IngestionPipeline.ts
@@ -10,7 +10,7 @@ import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store";
import type {
BaseVectorStore,
VectorStoreByType,
-} from "../vector-store/types.js";
+} from "@llamaindex/core/vector-store";
import { IngestionCache, getTransformationHash } from "./IngestionCache.js";
import {
DocStoreStrategy,
diff --git a/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts b/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts
index f36dc7cd7..f328c71be 100644
--- a/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts
+++ b/packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts
@@ -1,6 +1,6 @@
import { BaseNode } from "@llamaindex/core/schema";
import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store";
-import type { BaseVectorStore } from "../../vector-store/types.js";
+import type { BaseVectorStore } from "@llamaindex/core/vector-store";
import { classify } from "./classify.js";
import { RollbackableTransformComponent } from "./rollback.js";
diff --git a/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts b/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts
index 370994cd8..7a53e0dcf 100644
--- a/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts
+++ b/packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts
@@ -1,6 +1,6 @@
import { BaseNode } from "@llamaindex/core/schema";
import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store";
-import type { BaseVectorStore } from "../../vector-store/types.js";
+import type { BaseVectorStore } from "@llamaindex/core/vector-store";
import { classify } from "./classify.js";
import { RollbackableTransformComponent } from "./rollback.js";
diff --git a/packages/llamaindex/src/ingestion/strategies/index.ts b/packages/llamaindex/src/ingestion/strategies/index.ts
index a6ba3573f..01e1198cf 100644
--- a/packages/llamaindex/src/ingestion/strategies/index.ts
+++ b/packages/llamaindex/src/ingestion/strategies/index.ts
@@ -1,5 +1,5 @@
import type { BaseDocumentStore } from "@llamaindex/core/storage/doc-store";
-import type { BaseVectorStore } from "../../vector-store/types.js";
+import type { BaseVectorStore } from "@llamaindex/core/vector-store";
import { DuplicatesStrategy } from "./DuplicatesStrategy.js";
import { UpsertsAndDeleteStrategy } from "./UpsertsAndDeleteStrategy.js";
import { UpsertsStrategy } from "./UpsertsStrategy.js";
diff --git a/packages/llamaindex/src/storage/StorageContext.ts b/packages/llamaindex/src/storage/StorageContext.ts
index b6a1cbe68..3eae21ace 100644
--- a/packages/llamaindex/src/storage/StorageContext.ts
+++ b/packages/llamaindex/src/storage/StorageContext.ts
@@ -9,13 +9,13 @@ import {
BaseIndexStore,
SimpleIndexStore,
} from "@llamaindex/core/storage/index-store";
-import { path } from "@llamaindex/env";
-import type { ServiceContext } from "../ServiceContext.js";
-import { SimpleVectorStore } from "../vector-store/SimpleVectorStore.js";
import type {
BaseVectorStore,
VectorStoreByType,
-} from "../vector-store/types.js";
+} from "@llamaindex/core/vector-store";
+import { path } from "@llamaindex/env";
+import type { ServiceContext } from "../ServiceContext.js";
+import { SimpleVectorStore } from "../vector-store/SimpleVectorStore.js";
import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";
export interface StorageContext {
diff --git a/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts b/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts
index f9fbe83ff..e2b165fd7 100644
--- a/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts
+++ b/packages/llamaindex/src/storage/docStore/AzureCosmosMongovCoreDocumentStore.ts
@@ -1,6 +1,6 @@
+import { KVDocumentStore } from "@llamaindex/core/storage/doc-store";
import { MongoClient } from "mongodb";
import { AzureCosmosVCoreKVStore } from "../kvStore/AzureCosmosMongovCoreKVStore.js";
-import { KVDocumentStore } from "./KVDocumentStore.js";
const DEFAULT_DATABASE = "DocumentStoreDB";
const DEFAULT_COLLECTION = "DocumentStoreCollection";
diff --git a/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts b/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts
index abf6bb1e2..caab0e114 100644
--- a/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts
+++ b/packages/llamaindex/src/storage/docStore/AzureCosmosNoSqlDocumentStore.ts
@@ -1,10 +1,10 @@
+import { KVDocumentStore } from "@llamaindex/core/storage/doc-store";
import {
AzureCosmosNoSqlKVStore,
type AadTokenOptions,
type AccountAndKeyOptions,
type ConnectionStringOptions,
} from "../kvStore/AzureCosmosNoSqlKVStore.js";
-import { KVDocumentStore } from "./KVDocumentStore.js";
const DEFAULT_DATABASE = "DocumentStoreDB";
const DEFAULT_CONTAINER = "DocumentStoreContainer";
diff --git a/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts b/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts
index 7a057737c..95482a4b3 100644
--- a/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts
+++ b/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts
@@ -3,13 +3,13 @@ import {
DEFAULT_NAMESPACE,
DEFAULT_PERSIST_DIR,
} from "@llamaindex/core/global";
+import { KVDocumentStore } from "@llamaindex/core/storage/doc-store";
import {
BaseInMemoryKVStore,
SimpleKVStore,
} from "@llamaindex/core/storage/kv-store";
import { path } from "@llamaindex/env";
import _ from "lodash";
-import { KVDocumentStore } from "./KVDocumentStore.js";
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type SaveDict = Record<string, any>;
diff --git a/packages/llamaindex/src/storage/index.ts b/packages/llamaindex/src/storage/index.ts
index 7ffd63453..5ed611e6e 100644
--- a/packages/llamaindex/src/storage/index.ts
+++ b/packages/llamaindex/src/storage/index.ts
@@ -2,17 +2,19 @@ export * from "@llamaindex/core/storage/chat-store";
export * from "@llamaindex/core/storage/doc-store";
export * from "@llamaindex/core/storage/index-store";
export * from "@llamaindex/core/storage/kv-store";
+export {
+ PostgresDocumentStore,
+ PostgresIndexStore,
+ PostgresKVStore,
+} from "@llamaindex/postgres";
export * from "./chatStore/AzureCosmosMongovCoreChatStore.js";
export * from "./chatStore/AzureCosmosNoSqlChatStore.js";
export * from "./docStore/AzureCosmosMongovCoreDocumentStore.js";
export * from "./docStore/AzureCosmosNoSqlDocumentStore.js";
-export { PostgresDocumentStore } from "./docStore/PostgresDocumentStore.js";
export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js";
export * from "./FileSystem.js";
export * from "./indexStore/AzureCosmosMongovCoreIndexStore.js";
export * from "./indexStore/AzureCosmosNoSqlIndexStore.js";
-export { PostgresIndexStore } from "./indexStore/PostgresIndexStore.js";
export * from "./kvStore/AzureCosmosMongovCoreKVStore.js";
export * from "./kvStore/AzureCosmosNoSqlKVStore.js";
-export { PostgresKVStore } from "./kvStore/PostgresKVStore.js";
export * from "./StorageContext.js";
diff --git a/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts b/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts
index 4af05bccf..3f366305f 100644
--- a/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/AstraDBVectorStore.ts
@@ -8,7 +8,6 @@ import {
} from "@datastax/astra-db-ts";
import type { BaseNode } from "@llamaindex/core/schema";
import { MetadataMode } from "@llamaindex/core/schema";
-import { getEnv } from "@llamaindex/env";
import {
BaseVectorStore,
FilterCondition,
@@ -18,7 +17,8 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
+import { getEnv } from "@llamaindex/env";
import {
metadataDictToNode,
nodeToMetadata,
diff --git a/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts b/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts
index afddb59f9..487027326 100644
--- a/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/AzureCosmosDBMongoVectorStore.ts
@@ -1,13 +1,13 @@
import type { BaseNode } from "@llamaindex/core/schema";
import { MetadataMode } from "@llamaindex/core/schema";
-import { getEnv } from "@llamaindex/env";
-import { Collection, Db, MongoClient } from "mongodb";
import {
BaseVectorStore,
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
+import { getEnv } from "@llamaindex/env";
+import { Collection, Db, MongoClient } from "mongodb";
import { metadataDictToNode, nodeToMetadata } from "./utils.js";
/** Azure Cosmos DB for MongoDB vCore Similarity type. */
diff --git a/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts b/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts
index 29d01de50..6118fef75 100644
--- a/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/AzureCosmosDBNoSqlVectorStore.ts
@@ -20,7 +20,7 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
/** Azure Cosmos DB for NoSQL database creation options. */
export type AzureCosmosDBNoSqlCreateDatabaseOptions = Partial<
diff --git a/packages/llamaindex/src/vector-store/ChromaVectorStore.ts b/packages/llamaindex/src/vector-store/ChromaVectorStore.ts
index fbaf2a263..e8b9426f1 100644
--- a/packages/llamaindex/src/vector-store/ChromaVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/ChromaVectorStore.ts
@@ -1,14 +1,5 @@
import type { BaseNode } from "@llamaindex/core/schema";
import { MetadataMode } from "@llamaindex/core/schema";
-import {
- ChromaClient,
- type ChromaClientParams,
- type DeleteParams,
- type QueryRecordsParams,
- type QueryResponse,
- type Where,
- type WhereDocument,
-} from "chromadb";
import {
BaseVectorStore,
FilterCondition,
@@ -18,7 +9,16 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
+import {
+ ChromaClient,
+ type ChromaClientParams,
+ type DeleteParams,
+ type QueryRecordsParams,
+ type QueryResponse,
+ type Where,
+ type WhereDocument,
+} from "chromadb";
import { metadataDictToNode, nodeToMetadata } from "./utils.js";
type ChromaDeleteOptions = {
diff --git a/packages/llamaindex/src/vector-store/MilvusVectorStore.ts b/packages/llamaindex/src/vector-store/MilvusVectorStore.ts
index 1dc475871..97b7029f2 100644
--- a/packages/llamaindex/src/vector-store/MilvusVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/MilvusVectorStore.ts
@@ -1,5 +1,12 @@
import type { ChannelOptions } from "@grpc/grpc-js";
import { BaseNode, MetadataMode, type Metadata } from "@llamaindex/core/schema";
+import {
+ BaseVectorStore,
+ type MetadataFilters,
+ type VectorStoreBaseParams,
+ type VectorStoreQuery,
+ type VectorStoreQueryResult,
+} from "@llamaindex/core/vector-store";
import { getEnv } from "@llamaindex/env";
import {
DataType,
@@ -9,13 +16,6 @@ import {
type RowData,
type SearchSimpleReq,
} from "@zilliz/milvus2-sdk-node";
-import {
- BaseVectorStore,
- type MetadataFilters,
- type VectorStoreBaseParams,
- type VectorStoreQuery,
- type VectorStoreQueryResult,
-} from "./types.js";
import {
metadataDictToNode,
nodeToMetadata,
diff --git a/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts b/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts
index da9cab008..3aaef9dda 100644
--- a/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/MongoDBAtlasVectorStore.ts
@@ -1,9 +1,6 @@
import type { BaseEmbedding } from "@llamaindex/core/embeddings";
import type { BaseNode } from "@llamaindex/core/schema";
import { MetadataMode } from "@llamaindex/core/schema";
-import { getEnv } from "@llamaindex/env";
-import type { BulkWriteOptions, Collection } from "mongodb";
-import { MongoClient } from "mongodb";
import {
BaseVectorStore,
FilterCondition,
@@ -13,7 +10,10 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
+import { getEnv } from "@llamaindex/env";
+import type { BulkWriteOptions, Collection } from "mongodb";
+import { MongoClient } from "mongodb";
import { metadataDictToNode, nodeToMetadata } from "./utils.js";
// define your Atlas Search index. See detail https://www.mongodb.com/docs/atlas/atlas-search/field-types/knn-vector/
diff --git a/packages/llamaindex/src/vector-store/PineconeVectorStore.ts b/packages/llamaindex/src/vector-store/PineconeVectorStore.ts
index e2dca1494..e5cce0bff 100644
--- a/packages/llamaindex/src/vector-store/PineconeVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/PineconeVectorStore.ts
@@ -7,7 +7,7 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
import type { BaseNode, Metadata } from "@llamaindex/core/schema";
import { getEnv } from "@llamaindex/env";
diff --git a/packages/llamaindex/src/vector-store/QdrantVectorStore.ts b/packages/llamaindex/src/vector-store/QdrantVectorStore.ts
index f531adb2b..7a7c083f5 100644
--- a/packages/llamaindex/src/vector-store/QdrantVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/QdrantVectorStore.ts
@@ -7,7 +7,7 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
import type { QdrantClientParams, Schemas } from "@qdrant/js-client-rest";
import { QdrantClient } from "@qdrant/js-client-rest";
diff --git a/packages/llamaindex/src/vector-store/SimpleVectorStore.ts b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts
index b8564e81c..a0ab5b3f5 100644
--- a/packages/llamaindex/src/vector-store/SimpleVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/SimpleVectorStore.ts
@@ -5,8 +5,6 @@ import {
} from "@llamaindex/core/embeddings";
import { DEFAULT_PERSIST_DIR } from "@llamaindex/core/global";
import type { BaseNode } from "@llamaindex/core/schema";
-import { fs, path } from "@llamaindex/env";
-import { exists } from "../storage/FileSystem.js";
import {
BaseVectorStore,
FilterOperator,
@@ -16,7 +14,9 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
+import { fs, path } from "@llamaindex/env";
+import { exists } from "../storage/FileSystem.js";
import {
nodeToMetadata,
parseArrayValue,
diff --git a/packages/llamaindex/src/vector-store/UpstashVectorStore.ts b/packages/llamaindex/src/vector-store/UpstashVectorStore.ts
index f7575dc17..5101617be 100644
--- a/packages/llamaindex/src/vector-store/UpstashVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/UpstashVectorStore.ts
@@ -5,7 +5,7 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
import type { BaseNode, Metadata, TextNode } from "@llamaindex/core/schema";
import { getEnv } from "@llamaindex/env";
diff --git a/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts b/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts
index 66132be97..c85dccaf0 100644
--- a/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/WeaviateVectorStore.ts
@@ -8,8 +8,6 @@ import weaviate, {
type WeaviateNonGenericObject,
} from "weaviate-client";
-import { getEnv } from "@llamaindex/env";
-import type { BaseHybridOptions } from "weaviate-client";
import {
BaseVectorStore,
VectorStoreQueryMode,
@@ -18,7 +16,9 @@ import {
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
+} from "@llamaindex/core/vector-store";
+import { getEnv } from "@llamaindex/env";
+import type { BaseHybridOptions } from "weaviate-client";
import {
metadataDictToNode,
nodeToMetadata,
diff --git a/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts b/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts
index 51cc814db..5dfb22686 100644
--- a/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts
+++ b/packages/llamaindex/src/vector-store/azure/AzureAISearchVectorStore.ts
@@ -27,7 +27,6 @@ import {
ManagedIdentityCredential,
} from "@azure/identity";
import { type BaseNode, MetadataMode } from "@llamaindex/core/schema";
-import { consoleLogger, getEnv } from "@llamaindex/env";
import {
BaseVectorStore,
FilterCondition,
@@ -37,7 +36,8 @@ import {
type VectorStoreQuery,
VectorStoreQueryMode,
type VectorStoreQueryResult,
-} from "../types.js";
+} from "@llamaindex/core/vector-store";
+import { consoleLogger, getEnv } from "@llamaindex/env";
import { metadataDictToNode, nodeToMetadata } from "../utils.js";
import {
AzureAISearchVectorStoreConfig,
diff --git a/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts b/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts
index 1f39cfe01..721e23f6d 100644
--- a/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts
+++ b/packages/llamaindex/src/vector-store/azure/AzureQueryResultSearch.ts
@@ -4,11 +4,11 @@ import {
type VectorizedQuery,
} from "@azure/search-documents";
import {
- TextNode,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "llamaindex";
+} from "@llamaindex/core/vector-store";
+import type { TextNode } from "@llamaindex/core/schema";
import { consoleLogger } from "@llamaindex/env";
import { metadataDictToNode } from "../utils.js";
import {
diff --git a/packages/llamaindex/src/vector-store/index.ts b/packages/llamaindex/src/vector-store/index.ts
index 86554d230..030319e3e 100644
--- a/packages/llamaindex/src/vector-store/index.ts
+++ b/packages/llamaindex/src/vector-store/index.ts
@@ -1,3 +1,11 @@
+export * from "@llamaindex/core/vector-store";
+export {
+ DEFAULT_DIMENSIONS,
+ PGVECTOR_SCHEMA,
+ PGVECTOR_TABLE,
+ PGVectorStore,
+ type PGVectorStoreConfig,
+} from "@llamaindex/postgres";
export * from "./AstraDBVectorStore.js";
export * from "./azure/AzureAISearchVectorStore.js";
export * from "./AzureCosmosDBMongoVectorStore.js";
@@ -5,9 +13,7 @@ export * from "./AzureCosmosDBNoSqlVectorStore.js";
export * from "./ChromaVectorStore.js";
export * from "./MilvusVectorStore.js";
export * from "./MongoDBAtlasVectorStore.js";
-export * from "./PGVectorStore.js";
export * from "./PineconeVectorStore.js";
export * from "./QdrantVectorStore.js";
export * from "./SimpleVectorStore.js";
-export * from "./types.js";
export * from "./WeaviateVectorStore.js";
diff --git a/packages/llamaindex/src/vector-store/types.ts b/packages/llamaindex/src/vector-store/types.ts
deleted file mode 100644
index a5563b284..000000000
--- a/packages/llamaindex/src/vector-store/types.ts
+++ /dev/null
@@ -1,106 +0,0 @@
-import type { BaseEmbedding } from "@llamaindex/core/embeddings";
-import type { BaseNode, ModalityType } from "@llamaindex/core/schema";
-import { getEmbeddedModel } from "../internal/settings/EmbedModel.js";
-
-export interface VectorStoreQueryResult {
- nodes?: BaseNode[];
- similarities: number[];
- ids: string[];
-}
-
-export enum VectorStoreQueryMode {
- DEFAULT = "default",
- SPARSE = "sparse",
- HYBRID = "hybrid",
- // fit learners
- SVM = "svm",
- LOGISTIC_REGRESSION = "logistic_regression",
- LINEAR_REGRESSION = "linear_regression",
- // maximum marginal relevance
- MMR = "mmr",
-
- // for Azure AI Search
- SEMANTIC_HYBRID = "semantic_hybrid",
-}
-
-export enum FilterOperator {
- EQ = "==", // default operator (string, number)
- IN = "in", // In array (string or number)
- GT = ">", // greater than (number)
- LT = "<", // less than (number)
- NE = "!=", // not equal to (string, number)
- GTE = ">=", // greater than or equal to (number)
- LTE = "<=", // less than or equal to (number)
- NIN = "nin", // Not in array (string or number)
- ANY = "any", // Contains any (array of strings)
- ALL = "all", // Contains all (array of strings)
- TEXT_MATCH = "text_match", // full text match (allows you to search for a specific substring, token or phrase within the text field)
- CONTAINS = "contains", // metadata array contains value (string or number)
- IS_EMPTY = "is_empty", // the field is not exist or empty (null or empty array)
-}
-
-export enum FilterCondition {
- AND = "and",
- OR = "or",
-}
-
-export type MetadataFilterValue = string | number | string[] | number[];
-
-export interface MetadataFilter {
- key: string;
- value?: MetadataFilterValue;
- operator: `${FilterOperator}`; // ==, any, all,...
-}
-
-export interface MetadataFilters {
- filters: Array<MetadataFilter>;
- condition?: `${FilterCondition}`; // and, or
-}
-
-export interface MetadataInfo {
- name: string;
- type: string;
- description: string;
-}
-
-export interface VectorStoreInfo {
- metadataInfo: MetadataInfo[];
- contentInfo: string;
-}
-
-export interface VectorStoreQuery {
- queryEmbedding?: number[];
- similarityTopK: number;
- docIds?: string[];
- queryStr?: string;
- mode: VectorStoreQueryMode;
- alpha?: number;
- filters?: MetadataFilters | undefined;
- mmrThreshold?: number;
-}
-
-// Supported types of vector stores (for each modality)
-export type VectorStoreByType = {
- [P in ModalityType]?: BaseVectorStore;
-};
-
-export type VectorStoreBaseParams = {
- embeddingModel?: BaseEmbedding | undefined;
-};
-
-export abstract class BaseVectorStore<Client = unknown> {
- embedModel: BaseEmbedding;
- abstract storesText: boolean;
- isEmbeddingQuery?: boolean;
- abstract client(): Client;
- abstract add(embeddingResults: BaseNode[]): Promise<string[]>;
- abstract delete(refDocId: string, deleteOptions?: object): Promise<void>;
- abstract query(
- query: VectorStoreQuery,
- options?: object,
- ): Promise<VectorStoreQueryResult>;
-
- protected constructor(params?: VectorStoreBaseParams) {
- this.embedModel = params?.embeddingModel ?? getEmbeddedModel();
- }
-}
diff --git a/packages/llamaindex/src/vector-store/utils.ts b/packages/llamaindex/src/vector-store/utils.ts
index 79e02062f..ac50a43e0 100644
--- a/packages/llamaindex/src/vector-store/utils.ts
+++ b/packages/llamaindex/src/vector-store/utils.ts
@@ -1,6 +1,6 @@
import type { BaseNode, Metadata } from "@llamaindex/core/schema";
import { ObjectType, jsonToNode } from "@llamaindex/core/schema";
-import type { MetadataFilterValue } from "./types.js";
+import type { MetadataFilterValue } from "@llamaindex/core/vector-store";
const DEFAULT_TEXT_KEY = "text";
diff --git a/packages/llamaindex/tests/StorageContext.test.ts b/packages/llamaindex/tests/StorageContext.test.ts
index 95dfc7c1e..4b53dfa59 100644
--- a/packages/llamaindex/tests/StorageContext.test.ts
+++ b/packages/llamaindex/tests/StorageContext.test.ts
@@ -1,7 +1,4 @@
-import {
- storageContextFromDefaults,
- type StorageContext,
-} from "llamaindex/storage/StorageContext";
+import { storageContextFromDefaults, type StorageContext } from "llamaindex";
import { existsSync, rmSync } from "node:fs";
import { mkdtemp } from "node:fs/promises";
import { tmpdir } from "node:os";
diff --git a/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts b/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts
index cc028bcec..3d82e753c 100644
--- a/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts
+++ b/packages/llamaindex/tests/mocks/TestableAzureCosmosDBNoSqlVectorStore.ts
@@ -1,6 +1,6 @@
import type { BaseNode } from "@llamaindex/core/schema";
+import { AzureCosmosDBNoSqlVectorStore } from "llamaindex";
import type { Mocked } from "vitest";
-import { AzureCosmosDBNoSqlVectorStore } from "../../src/vector-store.js";
export class TestableAzureCosmosDBNoSqlVectorStore extends AzureCosmosDBNoSqlVectorStore {
public nodes: BaseNode[] = [];
diff --git a/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts b/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts
index 965144f96..23f7416fd 100644
--- a/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts
+++ b/packages/llamaindex/tests/mocks/TestableQdrantVectorStore.ts
@@ -1,5 +1,5 @@
import type { BaseNode } from "@llamaindex/core/schema";
-import { QdrantVectorStore } from "llamaindex/vector-store";
+import { QdrantVectorStore } from "llamaindex";
export class TestableQdrantVectorStore extends QdrantVectorStore {
public nodes: BaseNode[] = [];
diff --git a/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts b/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts
index c9686d53c..d94a333dd 100644
--- a/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts
+++ b/packages/llamaindex/tests/vector-stores/AzureAISearchVectorStore.test.ts
@@ -1,9 +1,9 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { SearchClient, SearchIndexClient } from "@azure/search-documents";
+import { AzureAISearchVectorStore } from "llamaindex";
import { afterEach, beforeEach } from "node:test";
import { describe, expect, it, vi } from "vitest";
-import { AzureAISearchVectorStore } from "../../src/vector-store.js";
// We test only for the initialization of the store, and the search and index clients, will variants of the options provided
const MOCK_ENDPOINT = "https://test-endpoint.com";
diff --git a/packages/providers/storage/postgres/package.json b/packages/providers/storage/postgres/package.json
new file mode 100644
index 000000000..f35b94dae
--- /dev/null
+++ b/packages/providers/storage/postgres/package.json
@@ -0,0 +1,66 @@
+{
+ "name": "@llamaindex/postgres",
+ "description": "PostgreSQL Storage for LlamaIndex",
+ "version": "0.0.29",
+ "type": "module",
+ "main": "./dist/index.cjs",
+ "module": "./dist/index.js",
+ "exports": {
+ ".": {
+ "edge-light": {
+ "types": "./dist/index.edge-light.d.ts",
+ "default": "./dist/index.edge-light.js"
+ },
+ "workerd": {
+ "types": "./dist/index.edge-light.d.ts",
+ "default": "./dist/index.edge-light.js"
+ },
+ "require": {
+ "types": "./dist/index.d.cts",
+ "default": "./dist/index.cjs"
+ },
+ "import": {
+ "types": "./dist/index.d.ts",
+ "default": "./dist/index.js"
+ }
+ }
+ },
+ "files": [
+ "dist"
+ ],
+ "repository": {
+ "type": "git",
+ "url": "https://github.com/run-llama/LlamaIndexTS.git",
+ "directory": "packages/providers/storage/postgres"
+ },
+ "scripts": {
+ "build": "bunchee",
+ "dev": "bunchee --watch"
+ },
+ "devDependencies": {
+ "bunchee": "6.2.0",
+ "@types/pg": "^8.11.8",
+ "@vercel/postgres": "^0.10.0",
+ "pg": "^8.12.0",
+ "pgvector": "0.2.0",
+ "postgres": "^3.4.4"
+ },
+ "dependencies": {
+ "@llamaindex/core": "workspace:*",
+ "@llamaindex/env": "workspace:*",
+ "pg": "^8.11.3",
+ "pg-promise": "^11.5.4"
+ },
+ "peerDependencies": {
+ "pg": "^8.12.0",
+ "pgvector": "0.2.0"
+ },
+ "peerDependenciesMeta": {
+ "pg": {
+ "optional": true
+ },
+ "pgvector": {
+ "optional": true
+ }
+ }
+}
diff --git a/packages/llamaindex/src/vector-store/PGVectorStore.ts b/packages/providers/storage/postgres/src/PGVectorStore.ts
similarity index 98%
rename from packages/llamaindex/src/vector-store/PGVectorStore.ts
rename to packages/providers/storage/postgres/src/PGVectorStore.ts
index 4524a3f3c..a46275044 100644
--- a/packages/llamaindex/src/vector-store/PGVectorStore.ts
+++ b/packages/providers/storage/postgres/src/PGVectorStore.ts
@@ -1,20 +1,18 @@
import type pg from "pg";
-import type { IsomorphicDB } from "@llamaindex/core/vector-store";
-import type { VercelPool } from "@vercel/postgres";
-import type { Sql } from "postgres";
import {
BaseVectorStore,
FilterCondition,
FilterOperator,
+ type IsomorphicDB,
type MetadataFilter,
type MetadataFilterValue,
type VectorStoreBaseParams,
type VectorStoreQuery,
type VectorStoreQueryResult,
-} from "./types.js";
-
-import { escapeLikeString } from "./utils.js";
+} from "@llamaindex/core/vector-store";
+import type { VercelPool } from "@vercel/postgres";
+import type { Sql } from "postgres";
import type { BaseEmbedding } from "@llamaindex/core/embeddings";
import { DEFAULT_COLLECTION } from "@llamaindex/core/global";
@@ -487,7 +485,7 @@ export class PGVectorStore extends BaseVectorStore {
}
if (filter.operator === FilterOperator.TEXT_MATCH) {
- const escapedValue = escapeLikeString(filter.value as string);
+ const escapedValue = this.escapeLikeString(filter.value as string);
return {
clause: `metadata->>'${filter.key}' LIKE $${paramIndex}`,
param: `%${escapedValue}%`,
@@ -592,4 +590,8 @@ export class PGVectorStore extends BaseVectorStore {
persist(persistPath: string): Promise<void> {
return Promise.resolve();
}
+
+ private escapeLikeString(value: string) {
+ return value.replace(/[%_\\]/g, "\\$&");
+ }
}
diff --git a/packages/llamaindex/src/storage/docStore/PostgresDocumentStore.ts b/packages/providers/storage/postgres/src/PostgresDocumentStore.ts
similarity index 80%
rename from packages/llamaindex/src/storage/docStore/PostgresDocumentStore.ts
rename to packages/providers/storage/postgres/src/PostgresDocumentStore.ts
index e15ad6c6c..0ec67354a 100644
--- a/packages/llamaindex/src/storage/docStore/PostgresDocumentStore.ts
+++ b/packages/providers/storage/postgres/src/PostgresDocumentStore.ts
@@ -1,10 +1,9 @@
import { DEFAULT_NAMESPACE } from "@llamaindex/core/global";
-import { noneSerializer } from "@llamaindex/core/storage/doc-store";
import {
- PostgresKVStore,
- type PostgresKVStoreConfig,
-} from "../kvStore/PostgresKVStore.js";
-import { KVDocumentStore } from "./KVDocumentStore.js";
+ KVDocumentStore,
+ noneSerializer,
+} from "@llamaindex/core/storage/doc-store";
+import { PostgresKVStore, type PostgresKVStoreConfig } from "./PostgresKVStore";
const DEFAULT_TABLE_NAME = "llamaindex_doc_store";
diff --git a/packages/llamaindex/src/storage/indexStore/PostgresIndexStore.ts b/packages/providers/storage/postgres/src/PostgresIndexStore.ts
similarity index 84%
rename from packages/llamaindex/src/storage/indexStore/PostgresIndexStore.ts
rename to packages/providers/storage/postgres/src/PostgresIndexStore.ts
index 6e4420743..f925ba56e 100644
--- a/packages/llamaindex/src/storage/indexStore/PostgresIndexStore.ts
+++ b/packages/providers/storage/postgres/src/PostgresIndexStore.ts
@@ -1,9 +1,6 @@
import { DEFAULT_NAMESPACE } from "@llamaindex/core/global";
-import {
- PostgresKVStore,
- type PostgresKVStoreConfig,
-} from "../kvStore/PostgresKVStore.js";
-import { KVIndexStore } from "./KVIndexStore.js";
+import { KVIndexStore } from "@llamaindex/core/storage/index-store";
+import { PostgresKVStore, type PostgresKVStoreConfig } from "./PostgresKVStore";
const DEFAULT_TABLE_NAME = "llamaindex_index_store";
diff --git a/packages/llamaindex/src/storage/kvStore/PostgresKVStore.ts b/packages/providers/storage/postgres/src/PostgresKVStore.ts
similarity index 100%
rename from packages/llamaindex/src/storage/kvStore/PostgresKVStore.ts
rename to packages/providers/storage/postgres/src/PostgresKVStore.ts
diff --git a/packages/providers/storage/postgres/src/index.ts b/packages/providers/storage/postgres/src/index.ts
new file mode 100644
index 000000000..71d1fb05b
--- /dev/null
+++ b/packages/providers/storage/postgres/src/index.ts
@@ -0,0 +1,4 @@
+export * from "./PGVectorStore";
+export * from "./PostgresDocumentStore";
+export * from "./PostgresIndexStore";
+export * from "./PostgresKVStore";
diff --git a/packages/providers/storage/postgres/tsconfig.json b/packages/providers/storage/postgres/tsconfig.json
new file mode 100644
index 000000000..4607e1860
--- /dev/null
+++ b/packages/providers/storage/postgres/tsconfig.json
@@ -0,0 +1,19 @@
+{
+ "extends": "../../../../tsconfig.json",
+ "compilerOptions": {
+ "target": "ESNext",
+ "module": "ESNext",
+ "moduleResolution": "bundler",
+ "outDir": "./lib",
+ "tsBuildInfoFile": "./lib/.tsbuildinfo"
+ },
+ "include": ["./src"],
+ "references": [
+ {
+ "path": "../../../core/tsconfig.json"
+ },
+ {
+ "path": "../../../env/tsconfig.json"
+ }
+ ]
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 6c4ebca28..041d4c3e7 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1001,6 +1001,9 @@ importers:
'@llamaindex/portkey-ai':
specifier: workspace:*
version: link:../providers/portkey-ai
+ '@llamaindex/postgres':
+ specifier: workspace:*
+ version: link:../providers/storage/postgres
'@llamaindex/readers':
specifier: workspace:*
version: link:../readers
@@ -1028,9 +1031,6 @@ importers:
'@types/node':
specifier: ^22.9.0
version: 22.9.0
- '@types/pg':
- specifier: ^8.11.8
- version: 8.11.8
'@upstash/vector':
specifier: ^1.1.5
version: 1.1.5
@@ -1098,24 +1098,12 @@ importers:
'@swc/core':
specifier: ^1.9.2
version: 1.9.2(@swc/helpers@0.5.15)
- '@vercel/postgres':
- specifier: ^0.10.0
- version: 0.10.0
concurrently:
specifier: ^9.1.0
version: 9.1.0
glob:
specifier: ^11.0.0
version: 11.0.0
- pg:
- specifier: ^8.12.0
- version: 8.12.0
- pgvector:
- specifier: 0.2.0
- version: 0.2.0
- postgres:
- specifier: ^3.4.4
- version: 3.4.4
typescript:
specifier: ^5.7.2
version: 5.7.2
@@ -1328,6 +1316,37 @@ importers:
specifier: 6.2.0
version: 6.2.0(typescript@5.7.2)
+ packages/providers/storage/postgres:
+ dependencies:
+ '@llamaindex/core':
+ specifier: workspace:*
+ version: link:../../../core
+ '@llamaindex/env':
+ specifier: workspace:*
+ version: link:../../../env
+ pg:
+ specifier: ^8.11.3
+ version: 8.13.1
+ pg-promise:
+ specifier: ^11.5.4
+ version: 11.10.2(pg-query-stream@4.7.1(pg@8.13.1))
+ devDependencies:
+ '@types/pg':
+ specifier: ^8.11.8
+ version: 8.11.8
+ '@vercel/postgres':
+ specifier: ^0.10.0
+ version: 0.10.0
+ bunchee:
+ specifier: 6.2.0
+ version: 6.2.0(typescript@5.7.2)
+ pgvector:
+ specifier: 0.2.0
+ version: 0.2.0
+ postgres:
+ specifier: ^3.4.4
+ version: 3.4.4
+
packages/providers/vercel:
dependencies:
'@llamaindex/core':
@@ -2054,10 +2073,6 @@ packages:
resolution: {integrity: sha512-IzD+hfqGqFtXymHXm4RzrZW2MsSH2M7RLmZsKaKVi7SUxbeYTUeX+ALk8gVzkM8ykb7EzlDLWCNErKfAa57rYQ==}
engines: {node: '>=18.0.0'}
- '@babel/code-frame@7.22.5':
- resolution: {integrity: sha512-Xmwn266vad+6DAqEB2A6V/CcZVp62BbwVmcOJc2RPuwih1kw02TjQvWVWlcKGbBPd+8/0V5DEkOcizRGYsspYQ==}
- engines: {node: '>=6.9.0'}
-
'@babel/code-frame@7.26.2':
resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==}
engines: {node: '>=6.9.0'}
@@ -2100,10 +2115,6 @@ packages:
resolution: {integrity: sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==}
engines: {node: '>=6.9.0'}
- '@babel/helper-validator-identifier@7.22.5':
- resolution: {integrity: sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==}
- engines: {node: '>=6.9.0'}
-
'@babel/helper-validator-identifier@7.25.9':
resolution: {integrity: sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==}
engines: {node: '>=6.9.0'}
@@ -2116,10 +2127,6 @@ packages:
resolution: {integrity: sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==}
engines: {node: '>=6.9.0'}
- '@babel/highlight@7.22.5':
- resolution: {integrity: sha512-BSKlD1hgnedS5XRnGOljZawtag7H1yPfQp0tdNJCHoH6AZ+Pcm9VvkrK59/Yy593Ypg0zMxH2BxD1VPYUQ7UIw==}
- engines: {node: '>=6.9.0'}
-
'@babel/parser@7.22.7':
resolution: {integrity: sha512-7NF8pOkHP5o2vpmGgNGcfAeCvOYhGLyA3Z4eBQkT1RJlWu47n63bCs93QfJ2hIAFCil7L5P2IWhs1oToVgrL0Q==}
engines: {node: '>=6.0.0'}
@@ -3210,9 +3217,6 @@ packages:
'@jridgewell/sourcemap-codec@1.4.14':
resolution: {integrity: sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==}
- '@jridgewell/sourcemap-codec@1.4.15':
- resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==}
-
'@jridgewell/sourcemap-codec@1.5.0':
resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==}
@@ -5583,6 +5587,10 @@ packages:
engines: {node: '>=16', npm: '>=7'}
hasBin: true
+ assert-options@0.8.2:
+ resolution: {integrity: sha512-XaXoMxY0zuwAb0YuZjxIm8FeWvNq0aWNIbrzHhFjme8Smxw4JlPoyrAKQ6808k5UvQdhvnWqHZCphq5mXd4TDA==}
+ engines: {node: '>=10.0.0'}
+
assertion-error@2.0.1:
resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==}
engines: {node: '>=12'}
@@ -5965,10 +5973,6 @@ packages:
resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==}
engines: {node: '>=18'}
- cli-spinners@2.9.0:
- resolution: {integrity: sha512-4/aL9X3Wh0yiMQlE+eeRhWP6vclO3QRtw1JHKIT0FFUs5FjpFmESqtMvYZ0+lbzBw900b95mS0hohy+qn2VK/g==}
- engines: {node: '>=6'}
-
cli-spinners@2.9.2:
resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==}
engines: {node: '>=6'}
@@ -7171,9 +7175,6 @@ packages:
tailwindcss:
optional: true
- function-bind@1.1.1:
- resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==}
-
function-bind@1.1.2:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
@@ -7266,9 +7267,6 @@ packages:
resolution: {integrity: sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==}
engines: {node: '>= 0.4'}
- get-tsconfig@4.5.0:
- resolution: {integrity: sha512-MjhiaIWCJ1sAU4pIQ5i5OfOuHHxVo1oYeNsWTON7jxYkod8pHocXeh+SSbmu5OZZZK73B6cbJ2XADzXehLyovQ==}
-
get-tsconfig@4.8.1:
resolution: {integrity: sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==}
@@ -7720,9 +7718,6 @@ packages:
resolution: {integrity: sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==}
engines: {node: '>= 0.4'}
- is-core-module@2.12.1:
- resolution: {integrity: sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==}
-
is-core-module@2.16.0:
resolution: {integrity: sha512-urTSINYfAYgcbLb0yDQ6egFm6h3Mo1DcF9EkyXSRjjzdHbsulg01qhwWuXdOoUBuTkbQ80KDboXa0vFJ+BDH+g==}
engines: {node: '>= 0.4'}
@@ -9445,10 +9440,19 @@ packages:
pg-connection-string@2.7.0:
resolution: {integrity: sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==}
+ pg-cursor@2.12.1:
+ resolution: {integrity: sha512-V13tEaA9Oq1w+V6Q3UBIB/blxJrwbbr35/dY54r/86soBJ7xkP236bXaORUTVXUPt9B6Ql2BQu+uwQiuMfRVgg==}
+ peerDependencies:
+ pg: ^8
+
pg-int8@1.0.1:
resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==}
engines: {node: '>=4.0.0'}
+ pg-minify@1.6.5:
+ resolution: {integrity: sha512-u0UE8veaCnMfJmoklqneeBBopOAPG3/6DHqGVHYAhz8DkJXh9dnjPlz25fRxn4e+6XVzdOp7kau63Rp52fZ3WQ==}
+ engines: {node: '>=14.0.0'}
+
pg-numeric@1.0.2:
resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==}
engines: {node: '>=4'}
@@ -9458,9 +9462,20 @@ packages:
peerDependencies:
pg: '>=8.0'
+ pg-promise@11.10.2:
+ resolution: {integrity: sha512-wK4yjxZdfxBmAMcs40q6IsC1SOzdLilc1yNvJqlbOjtm2syayqLDCt1JQ9lhS6yNSgVlGOQZT88yb/SADJmEBw==}
+ engines: {node: '>=14.0'}
+ peerDependencies:
+ pg-query-stream: 4.7.1
+
pg-protocol@1.7.0:
resolution: {integrity: sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==}
+ pg-query-stream@4.7.1:
+ resolution: {integrity: sha512-UMgsgn/pOIYsIifRySp59vwlpTpLADMK9HWJtq5ff0Z3MxBnPMGnCQeaQl5VuL+7ov4F96mSzIRIcz+Duo6OiQ==}
+ peerDependencies:
+ pg: ^8
+
pg-types@2.2.0:
resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==}
engines: {node: '>=4'}
@@ -10157,10 +10172,6 @@ packages:
engines: {node: '>= 0.4'}
hasBin: true
- resolve@1.22.2:
- resolution: {integrity: sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==}
- hasBin: true
-
resolve@1.22.9:
resolution: {integrity: sha512-QxrmX1DzraFIi9PxdG5VkRfRwIgjwyud+z/iBwfRRrVmHc+P9Q7u2lSSpQ6bjr2gy5lrqIiU9vb6iAeGf2400A==}
hasBin: true
@@ -10547,6 +10558,10 @@ packages:
spdx-license-ids@3.0.20:
resolution: {integrity: sha512-jg25NiDV/1fLtSgEgyvVyDunvaNHbuwF9lfNV17gSmPFAlYzdfNBlLtLzXTevwkPj7DhGbmN9VnmJIgLnhvaBw==}
+ spex@3.4.0:
+ resolution: {integrity: sha512-8JeZJ7QlEBnSj1W1fKXgbB2KUPA8k4BxFMf6lZX/c1ZagU/1b9uZWZK0yD6yjfzqAIuTNG4YlRmtMpQiXuohsg==}
+ engines: {node: '>=14.0.0'}
+
split2@4.2.0:
resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
engines: {node: '>= 10.x'}
@@ -13310,7 +13325,7 @@ snapshots:
'@azure/abort-controller@1.1.0':
dependencies:
- tslib: 2.6.0
+ tslib: 2.8.1
'@azure/abort-controller@2.1.2':
dependencies:
@@ -13433,10 +13448,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- '@babel/code-frame@7.22.5':
- dependencies:
- '@babel/highlight': 7.22.5
-
'@babel/code-frame@7.26.2':
dependencies:
'@babel/helper-validator-identifier': 7.25.9
@@ -13503,8 +13514,6 @@ snapshots:
'@babel/helper-string-parser@7.25.9': {}
- '@babel/helper-validator-identifier@7.22.5': {}
-
'@babel/helper-validator-identifier@7.25.9': {}
'@babel/helper-validator-option@7.25.9': {}
@@ -13514,12 +13523,6 @@ snapshots:
'@babel/template': 7.25.9
'@babel/types': 7.26.3
- '@babel/highlight@7.22.5':
- dependencies:
- '@babel/helper-validator-identifier': 7.22.5
- chalk: 2.4.2
- js-tokens: 4.0.0
-
'@babel/parser@7.22.7':
dependencies:
'@babel/types': 7.22.5
@@ -13563,7 +13566,7 @@ snapshots:
'@babel/types@7.22.5':
dependencies:
'@babel/helper-string-parser': 7.22.5
- '@babel/helper-validator-identifier': 7.22.5
+ '@babel/helper-validator-identifier': 7.25.9
to-fast-properties: 2.0.0
'@babel/types@7.26.3':
@@ -14401,13 +14404,13 @@ snapshots:
'@jridgewell/gen-mapping@0.3.3':
dependencies:
'@jridgewell/set-array': 1.1.2
- '@jridgewell/sourcemap-codec': 1.4.15
+ '@jridgewell/sourcemap-codec': 1.5.0
'@jridgewell/trace-mapping': 0.3.18
'@jridgewell/gen-mapping@0.3.8':
dependencies:
'@jridgewell/set-array': 1.2.1
- '@jridgewell/sourcemap-codec': 1.4.15
+ '@jridgewell/sourcemap-codec': 1.5.0
'@jridgewell/trace-mapping': 0.3.25
'@jridgewell/resolve-uri@3.1.0': {}
@@ -14425,8 +14428,6 @@ snapshots:
'@jridgewell/sourcemap-codec@1.4.14': {}
- '@jridgewell/sourcemap-codec@1.4.15': {}
-
'@jridgewell/sourcemap-codec@1.5.0': {}
'@jridgewell/trace-mapping@0.3.18':
@@ -14437,12 +14438,12 @@ snapshots:
'@jridgewell/trace-mapping@0.3.25':
dependencies:
'@jridgewell/resolve-uri': 3.1.1
- '@jridgewell/sourcemap-codec': 1.4.15
+ '@jridgewell/sourcemap-codec': 1.5.0
'@jridgewell/trace-mapping@0.3.9':
dependencies:
'@jridgewell/resolve-uri': 3.1.1
- '@jridgewell/sourcemap-codec': 1.4.15
+ '@jridgewell/sourcemap-codec': 1.5.0
'@js-sdsl/ordered-map@4.4.2': {}
@@ -17103,6 +17104,8 @@ snapshots:
binaryen: 116.0.0-nightly.20240114
long: 5.2.3
+ assert-options@0.8.2: {}
+
assertion-error@2.0.1: {}
ast-module-types@6.0.0: {}
@@ -17347,7 +17350,7 @@ snapshots:
call-bind@1.0.2:
dependencies:
- function-bind: 1.1.1
+ function-bind: 1.1.2
get-intrinsic: 1.2.0
call-bind@1.0.8:
@@ -17521,8 +17524,6 @@ snapshots:
dependencies:
restore-cursor: 5.1.0
- cli-spinners@2.9.0: {}
-
cli-spinners@2.9.2: {}
cli-truncate@4.0.0:
@@ -18374,15 +18375,15 @@ snapshots:
eslint-import-resolver-node@0.3.7:
dependencies:
debug: 3.2.7
- is-core-module: 2.12.1
- resolve: 1.22.2
+ is-core-module: 2.16.1
+ resolve: 1.22.10
transitivePeerDependencies:
- supports-color
eslint-import-resolver-node@0.3.9:
dependencies:
debug: 3.2.7
- is-core-module: 2.16.0
+ is-core-module: 2.16.1
resolve: 1.22.10
transitivePeerDependencies:
- supports-color
@@ -18394,9 +18395,9 @@ snapshots:
eslint: 9.16.0(jiti@2.4.2)
eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5)(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-typescript@3.5.5)(eslint@9.16.0(jiti@2.4.2))
- get-tsconfig: 4.5.0
+ get-tsconfig: 4.8.1
globby: 13.1.4
- is-core-module: 2.12.1
+ is-core-module: 2.16.1
is-glob: 4.0.3
synckit: 0.8.5
transitivePeerDependencies:
@@ -18440,7 +18441,7 @@ snapshots:
eslint-import-resolver-node: 0.3.9
eslint-module-utils: 2.12.0(@typescript-eslint/parser@5.59.2(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.5.5)(eslint@9.16.0(jiti@2.4.2))
hasown: 2.0.2
- is-core-module: 2.16.0
+ is-core-module: 2.16.1
is-glob: 4.0.3
minimatch: 3.1.2
object.fromentries: 2.0.8
@@ -19088,8 +19089,6 @@ snapshots:
- algoliasearch
- supports-color
- function-bind@1.1.1: {}
-
function-bind@1.1.2: {}
function.prototype.name@1.1.5:
@@ -19158,7 +19157,7 @@ snapshots:
get-intrinsic@1.2.0:
dependencies:
- function-bind: 1.1.1
+ function-bind: 1.1.2
has: 1.0.3
has-symbols: 1.0.3
@@ -19205,8 +19204,6 @@ snapshots:
es-errors: 1.3.0
get-intrinsic: 1.2.6
- get-tsconfig@4.5.0: {}
-
get-tsconfig@4.8.1:
dependencies:
resolve-pkg-maps: 1.0.0
@@ -19432,7 +19429,7 @@ snapshots:
has@1.0.3:
dependencies:
- function-bind: 1.1.1
+ function-bind: 1.1.2
hasown@2.0.2:
dependencies:
@@ -19796,10 +19793,6 @@ snapshots:
is-callable@1.2.7: {}
- is-core-module@2.12.1:
- dependencies:
- has: 1.0.3
-
is-core-module@2.16.0:
dependencies:
hasown: 2.0.2
@@ -21920,7 +21913,7 @@ snapshots:
bl: 4.1.0
chalk: 4.1.2
cli-cursor: 3.1.0
- cli-spinners: 2.9.0
+ cli-spinners: 2.9.2
is-interactive: 1.0.0
is-unicode-supported: 0.1.0
log-symbols: 4.1.0
@@ -22012,7 +22005,7 @@ snapshots:
parse-json@5.2.0:
dependencies:
- '@babel/code-frame': 7.22.5
+ '@babel/code-frame': 7.26.2
error-ex: 1.3.2
json-parse-even-better-errors: 2.3.1
lines-and-columns: 1.2.4
@@ -22082,8 +22075,14 @@ snapshots:
pg-connection-string@2.7.0: {}
+ pg-cursor@2.12.1(pg@8.13.1):
+ dependencies:
+ pg: 8.13.1
+
pg-int8@1.0.1: {}
+ pg-minify@1.6.5: {}
+
pg-numeric@1.0.2: {}
pg-pool@3.7.0(pg@8.12.0):
@@ -22094,8 +22093,23 @@ snapshots:
dependencies:
pg: 8.13.1
+ pg-promise@11.10.2(pg-query-stream@4.7.1(pg@8.13.1)):
+ dependencies:
+ assert-options: 0.8.2
+ pg: 8.13.1
+ pg-minify: 1.6.5
+ pg-query-stream: 4.7.1(pg@8.13.1)
+ spex: 3.4.0
+ transitivePeerDependencies:
+ - pg-native
+
pg-protocol@1.7.0: {}
+ pg-query-stream@4.7.1(pg@8.13.1):
+ dependencies:
+ pg: 8.13.1
+ pg-cursor: 2.12.1(pg@8.13.1)
+
pg-types@2.2.0:
dependencies:
pg-int8: 1.0.1
@@ -22960,12 +22974,6 @@ snapshots:
path-parse: 1.0.7
supports-preserve-symlinks-flag: 1.0.0
- resolve@1.22.2:
- dependencies:
- is-core-module: 2.12.1
- path-parse: 1.0.7
- supports-preserve-symlinks-flag: 1.0.0
-
resolve@1.22.9:
dependencies:
is-core-module: 2.16.0
@@ -22974,7 +22982,7 @@ snapshots:
resolve@2.0.0-next.5:
dependencies:
- is-core-module: 2.16.0
+ is-core-module: 2.16.1
path-parse: 1.0.7
supports-preserve-symlinks-flag: 1.0.0
@@ -23428,6 +23436,8 @@ snapshots:
spdx-license-ids@3.0.20: {}
+ spex@3.4.0: {}
+
split2@4.2.0: {}
sprintf-js@1.0.3: {}
@@ -24793,7 +24803,7 @@ snapshots:
miniflare: 3.20241106.0(bufferutil@4.0.8)
nanoid: 3.3.6
path-to-regexp: 6.3.0
- resolve: 1.22.9
+ resolve: 1.22.10
resolve.exports: 2.0.2
selfsigned: 2.1.1
source-map: 0.6.1
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 28ee54f28..235afafd8 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -5,6 +5,7 @@ packages:
- "e2e/examples/*"
- "packages/*"
- "packages/providers/*"
+ - "packages/providers/storage/*"
- "packages/core/tests"
- "packages/llamaindex/tests"
- "packages/autotool/examples/*"
diff --git a/tsconfig.json b/tsconfig.json
index 9f2ad80af..5e20e187d 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -124,6 +124,9 @@
},
{
"path": "./packages/experimental/tsconfig.json"
+ },
+ {
+ "path": "./packages/providers/storage/postgres/tsconfig.json"
}
]
}
--
GitLab