From f85c042a948ab3e05c5dd6764eecc5b30f004259 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Thu, 4 Jan 2024 10:05:47 +0700
Subject: [PATCH] refactor: encapsulate node serialization
---
packages/core/src/Node.ts | 19 +++++++++++++++++--
.../indices/vectorStore/VectorStoreIndex.ts | 3 +--
.../core/src/storage/vectorStore/utils.ts | 3 +--
3 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/packages/core/src/Node.ts b/packages/core/src/Node.ts
index 386c37b44..fef42df93 100644
--- a/packages/core/src/Node.ts
+++ b/packages/core/src/Node.ts
@@ -1,3 +1,4 @@
+import _ from "lodash";
import { createHash } from "node:crypto";
import path from "path";
import { v4 as uuidv4 } from "uuid";
@@ -141,12 +142,26 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
}
/**
- * Used with built in JSON.stringify
- * @returns
+ * Called by built in JSON.stringify (see https://javascript.info/json)
+ * Properties are read-only as they are not deep-cloned (not necessary for stringification).
+ * @see toMutableJSON - use to return a mutable JSON instead
*/
toJSON(): Record<string, any> {
return { ...this, type: this.getType() };
}
+
+ clone(): BaseNode {
+ return jsonToNode(this.toMutableJSON()) as BaseNode;
+ }
+
+ /**
+ * Converts the object to a JSON representation.
+ * Properties can be safely modified as a deep clone of the properties are created.
+ * @return {Record<string, any>} - The JSON representation of the object.
+ */
+ toMutableJSON(): Record<string, any> {
+ return _.cloneDeep(this.toJSON());
+ }
}
/**
diff --git a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts
index b588b34ee..278209f2c 100644
--- a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts
+++ b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts
@@ -4,7 +4,6 @@ import {
ImageNode,
MetadataMode,
ObjectType,
- jsonToNode,
splitNodesByType,
} from "../../Node";
import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
@@ -278,7 +277,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
type === ObjectType.INDEX ||
type === ObjectType.IMAGE
) {
- const nodeWithoutEmbedding = jsonToNode(nodes[i].toJSON());
+ const nodeWithoutEmbedding = nodes[i].clone();
nodeWithoutEmbedding.embedding = undefined;
this.indexStruct.addNode(nodeWithoutEmbedding, newIds[i]);
this.docStore.addDocuments([nodeWithoutEmbedding], true);
diff --git a/packages/core/src/storage/vectorStore/utils.ts b/packages/core/src/storage/vectorStore/utils.ts
index cf9e5728e..a20dbf7e6 100644
--- a/packages/core/src/storage/vectorStore/utils.ts
+++ b/packages/core/src/storage/vectorStore/utils.ts
@@ -1,4 +1,3 @@
-import _ from "lodash";
import { BaseNode, jsonToNode, Metadata, ObjectType } from "../../Node";
const DEFAULT_TEXT_KEY = "text";
@@ -17,7 +16,7 @@ export function nodeToMetadata(
textField: string = DEFAULT_TEXT_KEY,
flatMetadata: boolean = false,
): Metadata {
- const { metadata, embedding, ...rest } = _.cloneDeep(node.toJSON());
+ const { metadata, embedding, ...rest } = node.toMutableJSON();
if (flatMetadata) {
validateIsFlat(metadata);
--
GitLab