diff --git a/packages/core/src/indices/BaseIndex.ts b/packages/core/src/indices/BaseIndex.ts index c63b3891924930c1c2a3c4ab1bbe05ea71a20ffa..db04a12853d4c12aa910d68f601dfe71726c4e8f 100644 --- a/packages/core/src/indices/BaseIndex.ts +++ b/packages/core/src/indices/BaseIndex.ts @@ -2,6 +2,7 @@ import { BaseNode, Document, jsonToNode } from "../Node"; import { BaseRetriever } from "../Retriever"; import { ServiceContext } from "../ServiceContext"; import { randomUUID } from "../env"; +import { runTransformations } from "../ingestion"; import { StorageContext } from "../storage/StorageContext"; import { BaseDocumentStore } from "../storage/docStore/types"; import { BaseIndexStore } from "../storage/indexStore/types"; @@ -188,9 +189,10 @@ export abstract class BaseIndex<T> { * @param document */ async insert(document: Document) { - const nodes = this.serviceContext.nodeParser.getNodesFromDocuments([ - document, - ]); + const nodes = await runTransformations( + [document], + [this.serviceContext.nodeParser], + ); await this.insertNodes(nodes); this.docStore.setDocumentHash(document.id_, document.hash); } diff --git a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts index 832d46216b61a391effa00e1d00b8640448b0eb9..2403e1b54f4ac4366f8533f6caadc652e1dd6a67 100644 --- a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts +++ b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts @@ -17,6 +17,7 @@ import { ClipEmbedding, MultiModalEmbedding, } from "../../embeddings"; +import { runTransformations } from "../../ingestion"; import { BaseNodePostprocessor } from "../../postprocessors"; import { BaseIndexStore, @@ -225,8 +226,9 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> { if (args.logProgress) { console.log("Using node parser on documents..."); } - args.nodes = - args.serviceContext.nodeParser.getNodesFromDocuments(documents); + args.nodes = await runTransformations(documents, [ + args.serviceContext.nodeParser, + ]); if (args.logProgress) { console.log("Finished parsing documents."); } diff --git a/packages/core/src/ingestion/IngestionPipeline.ts b/packages/core/src/ingestion/IngestionPipeline.ts index 377b7b67018b100c4a15666cd3394299002d9a91..64e56902334ea1b1871683401484eef1a6c905a7 100644 --- a/packages/core/src/ingestion/IngestionPipeline.ts +++ b/packages/core/src/ingestion/IngestionPipeline.ts @@ -19,7 +19,7 @@ export async function runTransformations( nodesToRun: BaseNode[], transformations: TransformComponent[], transformOptions: any = {}, - { inPlace = true, cache }: TransformRunArgs, + { inPlace = true, cache }: TransformRunArgs = {}, ): Promise<BaseNode[]> { let nodes = nodesToRun; if (!inPlace) {