From 2039509811ecf7bfb1839e556debc2e05b5bc4da Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Wed, 13 Dec 2023 17:44:48 +0700
Subject: [PATCH] fix: bugs with persisting and loading image vector stores
---
packages/core/src/embeddings/ClipEmbedding.ts | 2 +-
.../core/src/indices/vectorStore/VectorStoreIndex.ts | 9 +++++++--
packages/core/src/storage/StorageContext.ts | 4 ++++
3 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/packages/core/src/embeddings/ClipEmbedding.ts b/packages/core/src/embeddings/ClipEmbedding.ts
index 989914dc5..d97dad944 100644
--- a/packages/core/src/embeddings/ClipEmbedding.ts
+++ b/packages/core/src/embeddings/ClipEmbedding.ts
@@ -62,7 +62,7 @@ export class ClipEmbedding extends MultiModalEmbedding {
const loadedImage = await readImage(image);
const imageInputs = await (await this.getProcessor())(loadedImage);
const { image_embeds } = await (await this.getVisionModel())(imageInputs);
- return image_embeds.data;
+ return Array.from(image_embeds.data);
}
async getTextEmbedding(text: string): Promise<number[]> {
diff --git a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts
index 6721aaa86..eb66b9c7c 100644
--- a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts
+++ b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts
@@ -66,7 +66,8 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
this.indexStore = init.indexStore;
this.vectorStore = init.vectorStore ?? init.storageContext.vectorStore;
this.embedModel = init.serviceContext.embedModel;
- this.imageVectorStore = init.imageVectorStore;
+ this.imageVectorStore =
+ init.imageVectorStore ?? init.storageContext.imageVectorStore;
if (this.imageVectorStore) {
this.imageEmbedModel = new ClipEmbedding();
}
@@ -219,6 +220,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
static async fromVectorStore(
vectorStore: VectorStore,
serviceContext: ServiceContext,
+ imageVectorStore?: VectorStore,
) {
if (!vectorStore.storesText) {
throw new Error(
@@ -226,7 +228,10 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
);
}
- const storageContext = await storageContextFromDefaults({ vectorStore });
+ const storageContext = await storageContextFromDefaults({
+ vectorStore,
+ imageVectorStore,
+ });
const index = await this.init({
nodes: [],
diff --git a/packages/core/src/storage/StorageContext.ts b/packages/core/src/storage/StorageContext.ts
index 32a5c8e4d..8e557ea95 100644
--- a/packages/core/src/storage/StorageContext.ts
+++ b/packages/core/src/storage/StorageContext.ts
@@ -11,12 +11,14 @@ export interface StorageContext {
docStore: BaseDocumentStore;
indexStore: BaseIndexStore;
vectorStore: VectorStore;
+ imageVectorStore?: VectorStore;
}
type BuilderParams = {
docStore?: BaseDocumentStore;
indexStore?: BaseIndexStore;
vectorStore?: VectorStore;
+ imageVectorStore?: VectorStore;
persistDir?: string;
fs?: GenericFileSystem;
};
@@ -25,6 +27,7 @@ export async function storageContextFromDefaults({
docStore,
indexStore,
vectorStore,
+ imageVectorStore,
persistDir,
fs,
}: BuilderParams): Promise<StorageContext> {
@@ -51,5 +54,6 @@ export async function storageContextFromDefaults({
docStore,
indexStore,
vectorStore,
+ imageVectorStore,
};
}
--
GitLab