From 7ad30dc660900c5d903ac51f8e02b713a2b5951b Mon Sep 17 00:00:00 2001
From: Alex Yang <himself65@outlook.com>
Date: Wed, 24 Jan 2024 17:09:52 -0600
Subject: [PATCH] chore: abstract `node:path` (#435)
---
packages/core/package.json | 1 +
packages/core/src/Node.ts | 3 +--
packages/core/src/env/index.edge-light.ts | 3 +++
packages/core/src/env/index.ts | 3 ++-
packages/core/src/storage/StorageContext.ts | 3 +--
packages/core/src/storage/docStore/SimpleDocumentStore.ts | 3 +--
packages/core/src/storage/indexStore/SimpleIndexStore.ts | 3 +--
packages/core/src/storage/kvStore/SimpleKVStore.ts | 3 +--
packages/core/src/storage/vectorStore/SimpleVectorStore.ts | 3 +--
pnpm-lock.yaml | 7 +++++++
10 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/packages/core/package.json b/packages/core/package.json
index 0abb2fefe..76ce2a65c 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -21,6 +21,7 @@
"notion-md-crawler": "^0.0.2",
"openai": "^4.20.1",
"papaparse": "^5.4.1",
+ "pathe": "^1.1.2",
"pdfjs-dist": "4.0.269",
"pg": "^8.11.3",
"pgvector": "^0.1.5",
diff --git a/packages/core/src/Node.ts b/packages/core/src/Node.ts
index ba47f3581..538d4a020 100644
--- a/packages/core/src/Node.ts
+++ b/packages/core/src/Node.ts
@@ -1,6 +1,5 @@
import _ from "lodash";
-import path from "node:path";
-import { createSHA256, randomUUID } from "./env";
+import { createSHA256, path, randomUUID } from "./env";
export enum NodeRelationship {
SOURCE = "SOURCE",
diff --git a/packages/core/src/env/index.edge-light.ts b/packages/core/src/env/index.edge-light.ts
index 562710cbc..522fcca3e 100644
--- a/packages/core/src/env/index.edge-light.ts
+++ b/packages/core/src/env/index.edge-light.ts
@@ -1,6 +1,9 @@
import { Sha256 } from "@aws-crypto/sha256-js";
+import pathe from "pathe";
import { CompleteFileSystem, InMemoryFileSystem } from "../storage";
+export { pathe as path };
+
export interface SHA256 {
update(data: string | Uint8Array): void;
// to base64
diff --git a/packages/core/src/env/index.ts b/packages/core/src/env/index.ts
index 6d164da0f..a88c1b3af 100644
--- a/packages/core/src/env/index.ts
+++ b/packages/core/src/env/index.ts
@@ -2,6 +2,7 @@ import { ok } from "node:assert";
import { createHash, randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import { EOL } from "node:os";
+import path from "node:path";
import type { CompleteFileSystem } from "../storage";
import type { SHA256 } from "./index.edge-light";
@@ -19,4 +20,4 @@ export function createSHA256(): SHA256 {
export const defaultFS: CompleteFileSystem = fs;
-export { EOL, ok, randomUUID };
+export { EOL, ok, path, randomUUID };
diff --git a/packages/core/src/storage/StorageContext.ts b/packages/core/src/storage/StorageContext.ts
index 06658ca20..50e973a08 100644
--- a/packages/core/src/storage/StorageContext.ts
+++ b/packages/core/src/storage/StorageContext.ts
@@ -1,5 +1,4 @@
-import path from "path";
-import { defaultFS } from "../env";
+import { defaultFS, path } from "../env";
import { GenericFileSystem } from "./FileSystem";
import { DEFAULT_IMAGE_VECTOR_NAMESPACE, DEFAULT_NAMESPACE } from "./constants";
import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
diff --git a/packages/core/src/storage/docStore/SimpleDocumentStore.ts b/packages/core/src/storage/docStore/SimpleDocumentStore.ts
index 453d76a7c..c05d93424 100644
--- a/packages/core/src/storage/docStore/SimpleDocumentStore.ts
+++ b/packages/core/src/storage/docStore/SimpleDocumentStore.ts
@@ -1,6 +1,5 @@
import _ from "lodash";
-import path from "path";
-import { defaultFS } from "../../env";
+import { defaultFS, path } from "../../env";
import { GenericFileSystem } from "../FileSystem";
import {
DEFAULT_DOC_STORE_PERSIST_FILENAME,
diff --git a/packages/core/src/storage/indexStore/SimpleIndexStore.ts b/packages/core/src/storage/indexStore/SimpleIndexStore.ts
index 4ab7c0530..faa68ac3f 100644
--- a/packages/core/src/storage/indexStore/SimpleIndexStore.ts
+++ b/packages/core/src/storage/indexStore/SimpleIndexStore.ts
@@ -1,5 +1,4 @@
-import path from "path";
-import { defaultFS } from "../../env";
+import { defaultFS, path } from "../../env";
import { GenericFileSystem } from "../FileSystem";
import {
DEFAULT_INDEX_STORE_PERSIST_FILENAME,
diff --git a/packages/core/src/storage/kvStore/SimpleKVStore.ts b/packages/core/src/storage/kvStore/SimpleKVStore.ts
index 6581d3711..30c932deb 100644
--- a/packages/core/src/storage/kvStore/SimpleKVStore.ts
+++ b/packages/core/src/storage/kvStore/SimpleKVStore.ts
@@ -1,6 +1,5 @@
import _ from "lodash";
-import path from "path";
-import { defaultFS } from "../../env";
+import { defaultFS, path } from "../../env";
import { GenericFileSystem, exists } from "../FileSystem";
import { DEFAULT_COLLECTION } from "../constants";
import { BaseKVStore } from "./types";
diff --git a/packages/core/src/storage/vectorStore/SimpleVectorStore.ts b/packages/core/src/storage/vectorStore/SimpleVectorStore.ts
index 5a6679d89..5c9170628 100644
--- a/packages/core/src/storage/vectorStore/SimpleVectorStore.ts
+++ b/packages/core/src/storage/vectorStore/SimpleVectorStore.ts
@@ -1,12 +1,11 @@
import _ from "lodash";
-import path from "path";
import { BaseNode } from "../../Node";
import {
getTopKEmbeddings,
getTopKEmbeddingsLearner,
getTopKMMREmbeddings,
} from "../../embeddings";
-import { defaultFS } from "../../env";
+import { defaultFS, path } from "../../env";
import { GenericFileSystem, exists } from "../FileSystem";
import { DEFAULT_PERSIST_DIR } from "../constants";
import {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 4c31ae497..c68167bd1 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -203,6 +203,9 @@ importers:
papaparse:
specifier: ^5.4.1
version: 5.4.1
+ pathe:
+ specifier: ^1.1.2
+ version: 1.1.2
pdfjs-dist:
specifier: 4.0.269
version: 4.0.269
@@ -12207,6 +12210,10 @@ packages:
dev: false
optional: true
+ /pathe@1.1.2:
+ resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==}
+ dev: false
+
/pdfjs-dist@4.0.269:
resolution: {integrity: sha512-jjWO56tcOjnmPqDf8PmXDeZ781AGvpHMYI3HhNtaFKTRXXPaD1ArSrhVe38/XsrIQJ0onISCND/vuXaWJkiDWw==}
engines: {node: '>=18'}
--
GitLab