diff --git a/packages/core/package.json b/packages/core/package.json
index d7a592345fd0307226f5df078fa78c2e7b807de5..ef0dfbf8a7b208bde0ccb2795814010c1b8c9d93 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -5,6 +5,7 @@
"dependencies": {
"@anthropic-ai/sdk": "^0.9.0",
"@notionhq/client": "^2.2.13",
+ "crypto-js": "^4.2.0",
"js-tiktoken": "^1.0.7",
"lodash": "^4.17.21",
"mammoth": "^1.6.0",
@@ -22,6 +23,7 @@
"wink-nlp": "^1.14.3"
},
"devDependencies": {
+ "@types/crypto-js": "^4.2.1",
"@types/lodash": "^4.14.200",
"@types/node": "^18.18.8",
"@types/papaparse": "^5.3.10",
@@ -44,4 +46,4 @@
"build": "tsup src/index.ts --format esm,cjs --dts",
"dev": "tsup src/index.ts --format esm,cjs --dts --watch"
}
-}
+}
\ No newline at end of file
diff --git a/packages/core/src/Node.ts b/packages/core/src/Node.ts
index 1908e67248a6945774b2e0728583ecd3ac676f8b..f8d45edba610bbdcaa0312a3839a7dd62997dbd7 100644
--- a/packages/core/src/Node.ts
+++ b/packages/core/src/Node.ts
@@ -1,4 +1,4 @@
-import crypto from "crypto"; // TODO Node dependency
+import CryptoJS from "crypto-js";
import { v4 as uuidv4 } from "uuid";
export enum NodeRelationship {
@@ -175,13 +175,13 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
* @returns
*/
generateHash() {
- const hashFunction = crypto.createHash("sha256");
+ const hashFunction = CryptoJS.algo.SHA256.create();
hashFunction.update(`type=${this.getType()}`);
hashFunction.update(
`startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
);
hashFunction.update(this.getContent(MetadataMode.ALL));
- return hashFunction.digest("base64");
+ return hashFunction.finalize().toString(CryptoJS.enc.Base64);
}
getType(): ObjectType {
diff --git a/packages/core/src/tests/Node.test.ts b/packages/core/src/tests/Node.test.ts
new file mode 100644
index 0000000000000000000000000000000000000000..656ad7d7cdf883839921af66db8956cb04c65543
--- /dev/null
+++ b/packages/core/src/tests/Node.test.ts
@@ -0,0 +1,15 @@
+import { TextNode } from "../Node";
+
+describe("TextNode", () => {
+ let node: TextNode;
+
+ beforeEach(() => {
+ node = new TextNode({ text: "Hello World" });
+ });
+
+ describe("generateHash", () => {
+ it("should generate a hash", () => {
+ expect(node.hash).toBe("nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=");
+ });
+ });
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 50f0fdef519c7a04852a02b5cf74acedf6823364..0f6b1dcd7f9f78742bb562346e8918616b4c0485 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -17,7 +17,7 @@ importers:
version: 2.26.2
'@turbo/gen':
specifier: ^1.10.16
- version: 1.10.16(@types/node@20.9.0)(typescript@5.2.2)
+ version: 1.10.16(@types/node@18.18.8)(typescript@5.2.2)
'@types/jest':
specifier: ^29.5.8
version: 29.5.8
@@ -32,7 +32,7 @@ importers:
version: 8.0.3
jest:
specifier: ^29.7.0
- version: 29.7.0(@types/node@20.9.0)
+ version: 29.7.0(@types/node@18.18.8)
lint-staged:
specifier: ^15.1.0
version: 15.1.0
@@ -134,6 +134,9 @@ importers:
'@notionhq/client':
specifier: ^2.2.13
version: 2.2.13
+ crypto-js:
+ specifier: ^4.2.0
+ version: 4.2.0
js-tiktoken:
specifier: ^1.0.7
version: 1.0.7
@@ -180,6 +183,9 @@ importers:
specifier: ^1.14.3
version: 1.14.3
devDependencies:
+ '@types/crypto-js':
+ specifier: ^4.2.1
+ version: 4.2.1
'@types/lodash':
specifier: ^4.14.200
version: 4.14.200
@@ -4067,7 +4073,7 @@ packages:
resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==}
dev: true
- /@turbo/gen@1.10.16(@types/node@20.9.0)(typescript@5.2.2):
+ /@turbo/gen@1.10.16(@types/node@18.18.8)(typescript@5.2.2):
resolution: {integrity: sha512-PzyluADjVuy5OcIi+/aRcD70OElQpRVRDdfZ9fH8G5Fv75lQcNrjd1bBGKmhjSw+g+eTEkXMGnY7s6gsCYjYTQ==}
hasBin: true
dependencies:
@@ -4079,7 +4085,7 @@ packages:
minimatch: 9.0.3
node-plop: 0.26.3
proxy-agent: 6.3.1
- ts-node: 10.9.1(@types/node@20.9.0)(typescript@5.2.2)
+ ts-node: 10.9.1(@types/node@18.18.8)(typescript@5.2.2)
update-check: 1.5.4
validate-npm-package-name: 5.0.0
transitivePeerDependencies:
@@ -4188,6 +4194,10 @@ packages:
'@types/node': 20.9.0
dev: true
+ /@types/crypto-js@4.2.1:
+ resolution: {integrity: sha512-FSPGd9+OcSok3RsM0UZ/9fcvMOXJ1ENE/ZbLfOPlBWj7BgXtEAM8VYfTtT760GiLbQIMoVozwVuisjvsVwqYWw==}
+ dev: true
+
/@types/eslint-scope@3.7.5:
resolution: {integrity: sha512-JNvhIEyxVW6EoMIFIvj93ZOywYFatlpu9deeH6eSx6PE3WHYvHaQtmHmQeNw7aA81bYGBPPQqdtBm6b1SsQMmA==}
dependencies:
@@ -6334,7 +6344,7 @@ packages:
sha.js: 2.4.11
dev: true
- /create-jest@29.7.0(@types/node@20.9.0):
+ /create-jest@29.7.0(@types/node@18.18.8):
resolution: {integrity: sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
hasBin: true
@@ -6343,7 +6353,7 @@ packages:
chalk: 4.1.2
exit: 0.1.2
graceful-fs: 4.2.11
- jest-config: 29.7.0(@types/node@20.9.0)
+ jest-config: 29.7.0(@types/node@18.18.8)
jest-util: 29.7.0
prompts: 2.4.2
transitivePeerDependencies:
@@ -6401,6 +6411,10 @@ packages:
randomfill: 1.0.4
dev: true
+ /crypto-js@4.2.0:
+ resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==}
+ dev: false
+
/crypto-random-string@2.0.0:
resolution: {integrity: sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==}
engines: {node: '>=8'}
@@ -9649,7 +9663,7 @@ packages:
- supports-color
dev: true
- /jest-cli@29.7.0(@types/node@20.9.0):
+ /jest-cli@29.7.0(@types/node@18.18.8):
resolution: {integrity: sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
hasBin: true
@@ -9663,10 +9677,10 @@ packages:
'@jest/test-result': 29.7.0
'@jest/types': 29.6.3
chalk: 4.1.2
- create-jest: 29.7.0(@types/node@20.9.0)
+ create-jest: 29.7.0(@types/node@18.18.8)
exit: 0.1.2
import-local: 3.1.0
- jest-config: 29.7.0(@types/node@20.9.0)
+ jest-config: 29.7.0(@types/node@18.18.8)
jest-util: 29.7.0
jest-validate: 29.7.0
yargs: 17.7.2
@@ -9677,6 +9691,46 @@ packages:
- ts-node
dev: true
+ /jest-config@29.7.0(@types/node@18.18.8):
+ resolution: {integrity: sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==}
+ engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
+ peerDependencies:
+ '@types/node': '*'
+ ts-node: '>=9.0.0'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+ ts-node:
+ optional: true
+ dependencies:
+ '@babel/core': 7.23.3
+ '@jest/test-sequencer': 29.7.0
+ '@jest/types': 29.6.3
+ '@types/node': 18.18.8
+ babel-jest: 29.7.0(@babel/core@7.23.3)
+ chalk: 4.1.2
+ ci-info: 3.9.0
+ deepmerge: 4.3.1
+ glob: 7.2.3
+ graceful-fs: 4.2.11
+ jest-circus: 29.7.0
+ jest-environment-node: 29.7.0
+ jest-get-type: 29.6.3
+ jest-regex-util: 29.6.3
+ jest-resolve: 29.7.0
+ jest-runner: 29.7.0
+ jest-util: 29.7.0
+ jest-validate: 29.7.0
+ micromatch: 4.0.5
+ parse-json: 5.2.0
+ pretty-format: 29.7.0
+ slash: 3.0.0
+ strip-json-comments: 3.1.1
+ transitivePeerDependencies:
+ - babel-plugin-macros
+ - supports-color
+ dev: true
+
/jest-config@29.7.0(@types/node@20.9.0):
resolution: {integrity: sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -10006,7 +10060,7 @@ packages:
merge-stream: 2.0.0
supports-color: 8.1.1
- /jest@29.7.0(@types/node@20.9.0):
+ /jest@29.7.0(@types/node@18.18.8):
resolution: {integrity: sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
hasBin: true
@@ -10019,7 +10073,7 @@ packages:
'@jest/core': 29.7.0
'@jest/types': 29.6.3
import-local: 3.1.0
- jest-cli: 29.7.0(@types/node@20.9.0)
+ jest-cli: 29.7.0(@types/node@18.18.8)
transitivePeerDependencies:
- '@types/node'
- babel-plugin-macros
@@ -14171,7 +14225,7 @@ packages:
'@babel/core': 7.23.3
bs-logger: 0.2.6
fast-json-stable-stringify: 2.1.0
- jest: 29.7.0(@types/node@20.9.0)
+ jest: 29.7.0(@types/node@18.18.8)
jest-util: 29.7.0
json5: 2.2.3
lodash.memoize: 4.1.2
@@ -14212,7 +14266,7 @@ packages:
yn: 3.1.1
dev: true
- /ts-node@10.9.1(@types/node@20.9.0)(typescript@5.2.2):
+ /ts-node@10.9.1(@types/node@18.18.8)(typescript@5.2.2):
resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==}
hasBin: true
peerDependencies:
@@ -14231,7 +14285,7 @@ packages:
'@tsconfig/node12': 1.0.11
'@tsconfig/node14': 1.0.3
'@tsconfig/node16': 1.0.4
- '@types/node': 20.9.0
+ '@types/node': 18.18.8
acorn: 8.11.2
acorn-walk: 8.3.0
arg: 4.1.3