diff --git a/packages/core/src/embeddings/ClipEmbedding.ts b/packages/core/src/embeddings/ClipEmbedding.ts index 989914dc5ed2c957eed1cb461a24e7d675fc97bf..d97dad94459346a2e84c8af7229007a2379a474a 100644 --- a/packages/core/src/embeddings/ClipEmbedding.ts +++ b/packages/core/src/embeddings/ClipEmbedding.ts @@ -62,7 +62,7 @@ export class ClipEmbedding extends MultiModalEmbedding { const loadedImage = await readImage(image); const imageInputs = await (await this.getProcessor())(loadedImage); const { image_embeds } = await (await this.getVisionModel())(imageInputs); - return image_embeds.data; + return Array.from(image_embeds.data); } async getTextEmbedding(text: string): Promise<number[]> { diff --git a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts index 6721aaa862da17a9bc3019cec421b2f6c8218300..eb66b9c7c8f79a9e4f23787e23b4594dbc21121e 100644 --- a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts +++ b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts @@ -66,7 +66,8 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> { this.indexStore = init.indexStore; this.vectorStore = init.vectorStore ?? init.storageContext.vectorStore; this.embedModel = init.serviceContext.embedModel; - this.imageVectorStore = init.imageVectorStore; + this.imageVectorStore = + init.imageVectorStore ?? init.storageContext.imageVectorStore; if (this.imageVectorStore) { this.imageEmbedModel = new ClipEmbedding(); } @@ -219,6 +220,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> { static async fromVectorStore( vectorStore: VectorStore, serviceContext: ServiceContext, + imageVectorStore?: VectorStore, ) { if (!vectorStore.storesText) { throw new Error( @@ -226,7 +228,10 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> { ); } - const storageContext = await storageContextFromDefaults({ vectorStore }); + const storageContext = await storageContextFromDefaults({ + vectorStore, + imageVectorStore, + }); const index = await this.init({ nodes: [], diff --git a/packages/core/src/storage/StorageContext.ts b/packages/core/src/storage/StorageContext.ts index 32a5c8e4d0b542c55293a4a380b9604b8330b4ee..8e557ea95f294fe3135948ac75d06ebd26b06487 100644 --- a/packages/core/src/storage/StorageContext.ts +++ b/packages/core/src/storage/StorageContext.ts @@ -11,12 +11,14 @@ export interface StorageContext { docStore: BaseDocumentStore; indexStore: BaseIndexStore; vectorStore: VectorStore; + imageVectorStore?: VectorStore; } type BuilderParams = { docStore?: BaseDocumentStore; indexStore?: BaseIndexStore; vectorStore?: VectorStore; + imageVectorStore?: VectorStore; persistDir?: string; fs?: GenericFileSystem; }; @@ -25,6 +27,7 @@ export async function storageContextFromDefaults({ docStore, indexStore, vectorStore, + imageVectorStore, persistDir, fs, }: BuilderParams): Promise<StorageContext> { @@ -51,5 +54,6 @@ export async function storageContextFromDefaults({ docStore, indexStore, vectorStore, + imageVectorStore, }; }