fix: use pipeline instead of nodeparser (#471)

packages/core/src/indices/BaseIndex.ts

@@ -2,6 +2,7 @@ import { BaseNode, Document, jsonToNode } from "../Node";
 import { BaseRetriever } from "../Retriever";
 import { ServiceContext } from "../ServiceContext";
 import { randomUUID } from "../env";
+import { runTransformations } from "../ingestion";
 import { StorageContext } from "../storage/StorageContext";
 import { BaseDocumentStore } from "../storage/docStore/types";
 import { BaseIndexStore } from "../storage/indexStore/types";
@@ -188,9 +189,10 @@ export abstract class BaseIndex<T> {
    * @param document
    */
   async insert(document: Document) {
-    const nodes = this.serviceContext.nodeParser.getNodesFromDocuments([
-      document,
-    ]);
+    const nodes = await runTransformations(
+      [document],
+      [this.serviceContext.nodeParser],
+    );
     await this.insertNodes(nodes);
     this.docStore.setDocumentHash(document.id_, document.hash);
   }

packages/core/src/indices/vectorStore/VectorStoreIndex.ts

@@ -17,6 +17,7 @@ import {
   ClipEmbedding,
   MultiModalEmbedding,
 } from "../../embeddings";
+import { runTransformations } from "../../ingestion";
 import { BaseNodePostprocessor } from "../../postprocessors";
 import {
   BaseIndexStore,
@@ -225,8 +226,9 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
     if (args.logProgress) {
       console.log("Using node parser on documents...");
     }
-    args.nodes =
-      args.serviceContext.nodeParser.getNodesFromDocuments(documents);
+    args.nodes = await runTransformations(documents, [
+      args.serviceContext.nodeParser,
+    ]);
     if (args.logProgress) {
       console.log("Finished parsing documents.");
     }

packages/core/src/ingestion/IngestionPipeline.ts

@@ -19,7 +19,7 @@ export async function runTransformations(
   nodesToRun: BaseNode[],
   transformations: TransformComponent[],
   transformOptions: any = {},
-  { inPlace = true, cache }: TransformRunArgs,
+  { inPlace = true, cache }: TransformRunArgs = {},
 ): Promise<BaseNode[]> {
   let nodes = nodesToRun;
   if (!inPlace) {