diff --git a/.changeset/three-stingrays-build.md b/.changeset/three-stingrays-build.md new file mode 100644 index 0000000000000000000000000000000000000000..516d535b7d74ccafe20cb66e889297366d3e0778 --- /dev/null +++ b/.changeset/three-stingrays-build.md @@ -0,0 +1,5 @@ +--- +"llamaindex": patch +--- + +fix(cyclic): remove cyclic structures from transform hash diff --git a/packages/core/src/ingestion/IngestionCache.ts b/packages/core/src/ingestion/IngestionCache.ts index 929f3a5d48cbb5ca9565c66665b11a93938d83bd..a98459af8b489341e475ceed5b8c48914bcf55aa 100644 --- a/packages/core/src/ingestion/IngestionCache.ts +++ b/packages/core/src/ingestion/IngestionCache.ts @@ -5,6 +5,25 @@ import { SimpleKVStore } from "../storage/kvStore/SimpleKVStore"; import { BaseKVStore } from "../storage/kvStore/types"; import { TransformComponent } from "./types"; +const transformToJSON = (obj: TransformComponent) => { + let seen: any[] = []; + + const replacer = (key: string, value: any) => { + if (value != null && typeof value == "object") { + if (seen.indexOf(value) >= 0) { + return; + } + seen.push(value); + } + return value; + }; + + // this is a custom replacer function that will allow us to handle circular references + const jsonStr = JSON.stringify(obj, replacer); + + return jsonStr; +}; + export function getTransformationHash( nodes: BaseNode[], transform: TransformComponent, @@ -13,7 +32,8 @@ export function getTransformationHash( .map((node) => node.getContent(MetadataMode.ALL)) .join(""); - const transformString: string = JSON.stringify(transform); + const transformString: string = transformToJSON(transform); + const hash = createSHA256(); hash.update(nodesStr + transformString); return hash.digest(); diff --git a/packages/core/src/tests/ingestion/IngestionCache.test.ts b/packages/core/src/tests/ingestion/IngestionCache.test.ts index 52a27801eeb23cb79a43fb91e7aaf06b3d5bf591..bc4ee1733bd2b4b32484c65673c632f238860dd9 100644 --- a/packages/core/src/tests/ingestion/IngestionCache.test.ts +++ b/packages/core/src/tests/ingestion/IngestionCache.test.ts @@ -71,4 +71,11 @@ describe("getTransformationHash", () => { ); expect(result1).not.toBe(result2); }); + + test("should not break with circular references", () => { + const obj: any = { a: 1, b: 2 }; + obj["circular"] = obj; + const result = getTransformationHash(nodes, obj); + expect(typeof result).toBe("string"); + }); });