diff --git a/packages/core/package.json b/packages/core/package.json
index d7a592345fd0307226f5df078fa78c2e7b807de5..ef0dfbf8a7b208bde0ccb2795814010c1b8c9d93 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -5,6 +5,7 @@
"dependencies": {
"@anthropic-ai/sdk": "^0.9.0",
"@notionhq/client": "^2.2.13",
+ "crypto-js": "^4.2.0",
"js-tiktoken": "^1.0.7",
"lodash": "^4.17.21",
"mammoth": "^1.6.0",
@@ -22,6 +23,7 @@
"wink-nlp": "^1.14.3"
},
"devDependencies": {
+ "@types/crypto-js": "^4.2.1",
"@types/lodash": "^4.14.200",
"@types/node": "^18.18.8",
"@types/papaparse": "^5.3.10",
@@ -44,4 +46,4 @@
"build": "tsup src/index.ts --format esm,cjs --dts",
"dev": "tsup src/index.ts --format esm,cjs --dts --watch"
}
-}
+}
\ No newline at end of file
diff --git a/packages/core/src/Node.ts b/packages/core/src/Node.ts
index 25fa75e1fe086526714a70ae11e45410888eab5a..d60f358e4bc364e52d9a15b8ee27adaac0535e12 100644
--- a/packages/core/src/Node.ts
+++ b/packages/core/src/Node.ts
@@ -1,4 +1,4 @@
-import crypto from "crypto"; // TODO Node dependency
+import CryptoJS from "crypto-js";
import { v4 as uuidv4 } from "uuid";
export enum NodeRelationship {
@@ -175,13 +175,13 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
* @returns
*/
generateHash() {
- const hashFunction = crypto.createHash("sha256");
+ const hashFunction = CryptoJS.algo.SHA256.create();
hashFunction.update(`type=${this.getType()}`);
hashFunction.update(
`startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
);
hashFunction.update(this.getContent(MetadataMode.ALL));
- return hashFunction.digest("base64");
+ return hashFunction.finalize().toString(CryptoJS.enc.Base64);
}
getType(): ObjectType {
diff --git a/packages/core/src/tests/Node.test.ts b/packages/core/src/tests/Node.test.ts
new file mode 100644
index 0000000000000000000000000000000000000000..656ad7d7cdf883839921af66db8956cb04c65543
--- /dev/null
+++ b/packages/core/src/tests/Node.test.ts
@@ -0,0 +1,15 @@
+import { TextNode } from "../Node";
+
+describe("TextNode", () => {
+ let node: TextNode;
+
+ beforeEach(() => {
+ node = new TextNode({ text: "Hello World" });
+ });
+
+ describe("generateHash", () => {
+ it("should generate a hash", () => {
+ expect(node.hash).toBe("nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=");
+ });
+ });
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 47d34a3a5d7f5a1fd04b162a5e4506385a2128f2..1dc2addf63b6e5246c38ba05aff6146ef899dbca 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -153,6 +153,9 @@ importers:
'@notionhq/client':
specifier: ^2.2.13
version: 2.2.13
+ crypto-js:
+ specifier: ^4.2.0
+ version: 4.2.0
js-tiktoken:
specifier: ^1.0.7
version: 1.0.7
@@ -199,6 +202,9 @@ importers:
specifier: ^1.14.3
version: 1.14.3
devDependencies:
+ '@types/crypto-js':
+ specifier: ^4.2.1
+ version: 4.2.1
'@types/lodash':
specifier: ^4.14.200
version: 4.14.200
@@ -463,7 +469,7 @@ packages:
engines: {node: '>=6.0.0'}
dependencies:
'@jridgewell/gen-mapping': 0.3.3
- '@jridgewell/trace-mapping': 0.3.19
+ '@jridgewell/trace-mapping': 0.3.20
/@anthropic-ai/sdk@0.9.0:
resolution: {integrity: sha512-qNoNld9luBWNcmCFTIZrsmusCQIdxIxQXyHJ64IDUsrvPvy2lM0kA9+E6bHeeFut463zdGkVz0Ux0U+WDppLGg==}
@@ -568,7 +574,7 @@ packages:
dependencies:
'@babel/types': 7.23.0
'@jridgewell/gen-mapping': 0.3.3
- '@jridgewell/trace-mapping': 0.3.19
+ '@jridgewell/trace-mapping': 0.3.20
jsesc: 2.5.2
/@babel/generator@7.23.3:
@@ -3596,12 +3602,6 @@ packages:
/@jridgewell/sourcemap-codec@1.4.15:
resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==}
- /@jridgewell/trace-mapping@0.3.19:
- resolution: {integrity: sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==}
- dependencies:
- '@jridgewell/resolve-uri': 3.1.1
- '@jridgewell/sourcemap-codec': 1.4.15
-
/@jridgewell/trace-mapping@0.3.20:
resolution: {integrity: sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==}
dependencies:
@@ -4207,6 +4207,10 @@ packages:
'@types/node': 18.18.8
dev: true
+ /@types/crypto-js@4.2.1:
+ resolution: {integrity: sha512-FSPGd9+OcSok3RsM0UZ/9fcvMOXJ1ENE/ZbLfOPlBWj7BgXtEAM8VYfTtT760GiLbQIMoVozwVuisjvsVwqYWw==}
+ dev: true
+
/@types/eslint-scope@3.7.5:
resolution: {integrity: sha512-JNvhIEyxVW6EoMIFIvj93ZOywYFatlpu9deeH6eSx6PE3WHYvHaQtmHmQeNw7aA81bYGBPPQqdtBm6b1SsQMmA==}
dependencies:
@@ -4811,12 +4815,12 @@ packages:
/acorn-walk@8.2.0:
resolution: {integrity: sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==}
engines: {node: '>=0.4.0'}
- dev: false
+ dev: true
/acorn-walk@8.3.0:
resolution: {integrity: sha512-FS7hV565M5l1R08MXqo8odwMTB02C2UqzB17RVgu9EyuYFBqJZ3/ZY97sQD5FewVu1UyDFc1yztUDrAwT0EypA==}
engines: {node: '>=0.4.0'}
- dev: true
+ dev: false
/acorn@8.10.0:
resolution: {integrity: sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==}
@@ -6423,6 +6427,10 @@ packages:
randomfill: 1.0.4
dev: true
+ /crypto-js@4.2.0:
+ resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==}
+ dev: false
+
/crypto-random-string@2.0.0:
resolution: {integrity: sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==}
engines: {node: '>=8'}
@@ -12273,10 +12281,6 @@ packages:
/punycode@1.4.1:
resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==}
- /punycode@2.3.0:
- resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==}
- engines: {node: '>=6'}
-
/punycode@2.3.1:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
@@ -13979,7 +13983,7 @@ packages:
uglify-js:
optional: true
dependencies:
- '@jridgewell/trace-mapping': 0.3.19
+ '@jridgewell/trace-mapping': 0.3.20
jest-worker: 27.5.1
schema-utils: 3.3.0
serialize-javascript: 6.0.1
@@ -14002,7 +14006,7 @@ packages:
uglify-js:
optional: true
dependencies:
- '@jridgewell/trace-mapping': 0.3.19
+ '@jridgewell/trace-mapping': 0.3.20
jest-worker: 27.5.1
schema-utils: 3.3.0
serialize-javascript: 6.0.1
@@ -14228,8 +14232,8 @@ packages:
'@tsconfig/node14': 1.0.3
'@tsconfig/node16': 1.0.4
'@types/node': 18.18.7
- acorn: 8.11.2
- acorn-walk: 8.3.0
+ acorn: 8.10.0
+ acorn-walk: 8.2.0
arg: 4.1.3
create-require: 1.1.1
diff: 4.0.2
@@ -14259,8 +14263,8 @@ packages:
'@tsconfig/node14': 1.0.3
'@tsconfig/node16': 1.0.4
'@types/node': 18.18.8
- acorn: 8.11.2
- acorn-walk: 8.3.0
+ acorn: 8.10.0
+ acorn-walk: 8.2.0
arg: 4.1.3
create-require: 1.1.1
diff: 4.0.2
@@ -14763,7 +14767,7 @@ packages:
/uri-js@4.4.1:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
dependencies:
- punycode: 2.3.0
+ punycode: 2.3.1
/url-loader@4.1.1(file-loader@6.2.0)(webpack@5.88.2):
resolution: {integrity: sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA==}
@@ -15019,8 +15023,8 @@ packages:
hasBin: true
dependencies:
'@discoveryjs/json-ext': 0.5.7
- acorn: 8.10.0
- acorn-walk: 8.2.0
+ acorn: 8.11.2
+ acorn-walk: 8.3.0
commander: 7.2.0
escape-string-regexp: 4.0.0
gzip-size: 6.0.0