From 14cc9ebe596512bea551c614ef2461583c38d8bc Mon Sep 17 00:00:00 2001
From: Alex Yang <himself65@outlook.com>
Date: Thu, 7 Nov 2024 15:01:17 -0800
Subject: [PATCH] chore: move multi-model into single sub module (#1443)
---
.changeset/cool-cows-lick.md | 8 +++++
packages/env/multi-model/package.json | 12 ++++++++
packages/env/package.json | 29 ++++++++++++++++++-
packages/env/src/index.browser.ts | 6 ----
packages/env/src/index.edge-light.ts | 6 ----
packages/env/src/index.ts | 6 ----
packages/env/src/index.workerd.ts | 6 ----
.../multi-model/browser.ts} | 0
.../index.ts => internal/multi-model/node.ts} | 0
.../multi-model/non-node.ts} | 0
.../src/{ => internal}/multi-model/shared.ts | 0
packages/env/src/multi-model.browser.ts | 6 ++++
packages/env/src/multi-model.edge-light.ts | 6 ++++
packages/env/src/multi-model.ts | 6 ++++
packages/env/src/multi-model.workerd.ts | 6 ++++
.../llamaindex/e2e/node/embedding/clip.e2e.ts | 4 +--
packages/providers/clip/src/embedding.ts | 5 +++-
.../providers/huggingface/src/embedding.ts | 5 +++-
packages/providers/huggingface/src/llm.ts | 2 +-
19 files changed, 83 insertions(+), 30 deletions(-)
create mode 100644 .changeset/cool-cows-lick.md
create mode 100644 packages/env/multi-model/package.json
rename packages/env/src/{multi-model/index.browser.ts => internal/multi-model/browser.ts} (100%)
rename packages/env/src/{multi-model/index.ts => internal/multi-model/node.ts} (100%)
rename packages/env/src/{multi-model/index.non-nodejs.ts => internal/multi-model/non-node.ts} (100%)
rename packages/env/src/{ => internal}/multi-model/shared.ts (100%)
create mode 100644 packages/env/src/multi-model.browser.ts
create mode 100644 packages/env/src/multi-model.edge-light.ts
create mode 100644 packages/env/src/multi-model.ts
create mode 100644 packages/env/src/multi-model.workerd.ts
diff --git a/.changeset/cool-cows-lick.md b/.changeset/cool-cows-lick.md
new file mode 100644
index 000000000..4751bf218
--- /dev/null
+++ b/.changeset/cool-cows-lick.md
@@ -0,0 +1,8 @@
+---
+"@llamaindex/env": patch
+"llamaindex": patch
+"@llamaindex/clip": patch
+"@llamaindex/huggingface": patch
+---
+
+chore: move multi-model into single sub module
diff --git a/packages/env/multi-model/package.json b/packages/env/multi-model/package.json
new file mode 100644
index 000000000..35136ddeb
--- /dev/null
+++ b/packages/env/multi-model/package.json
@@ -0,0 +1,12 @@
+{
+ "type": "module",
+ "main": "./dist/index.cjs",
+ "module": "./dist/index.js",
+ "types": "./dist/index.d.ts",
+ "exports": {
+ "browser": "./dist/index.browser.js",
+ "edge-light": "./dist/index.edge-light.js",
+ "workerd": "./dist/index.workerd.js"
+ },
+ "private": true
+}
diff --git a/packages/env/package.json b/packages/env/package.json
index 16d43caf0..c11889f79 100644
--- a/packages/env/package.json
+++ b/packages/env/package.json
@@ -28,7 +28,7 @@
"types": "./dist/index.d.ts",
"import": "./dist/index.js",
"require": "./dist/index.cjs",
- "default": "./dist/index.cjs"
+ "default": "./dist/index.js"
},
"workerd": {
"types": "./dist/index.workerd.d.ts",
@@ -50,9 +50,36 @@
"types": "./dist/index.d.cts",
"default": "./dist/index.cjs"
}
+ },
+ "./multi-model": {
+ "workerd": {
+ "types": "./multi-model/dist/multi-model.workerd.d.ts",
+ "default": "./multi-model/dist/multi-model.workerd.js"
+ },
+ "edge-light": {
+ "types": "./multi-model/dist/multi-model.edge-light.d.ts",
+ "default": "./multi-model/dist/multi-model.edge-light.js"
+ },
+ "browser": {
+ "types": "./multi-model/dist/multi-model.browser.d.ts",
+ "default": "./multi-model/dist/multi-model.browser.js"
+ },
+ "import": {
+ "types": "./multi-model/dist/multi-model.d.ts",
+ "default": "./multi-model/dist/multi-model.js"
+ },
+ "require": {
+ "types": "./multi-model/dist/multi-model.d.cts",
+ "default": "./multi-model/dist/multi-model.cjs"
+ },
+ "default": {
+ "types": "./multi-model/dist/multi-model.d.ts",
+ "default": "./multi-model/dist/multi-model.js"
+ }
}
},
"files": [
+ "multi-model",
"dist",
"CHANGELOG.md",
"!**/*.tsbuildinfo"
diff --git a/packages/env/src/index.browser.ts b/packages/env/src/index.browser.ts
index cd1747ffe..9d55f99ed 100644
--- a/packages/env/src/index.browser.ts
+++ b/packages/env/src/index.browser.ts
@@ -6,12 +6,6 @@
import "./global-check.js";
export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
- loadTransformers,
- setTransformers,
- type LoadTransformerEvent,
- type OnLoad,
-} from "./multi-model/index.browser.js";
export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";
export * from "./web-polyfill.js";
diff --git a/packages/env/src/index.edge-light.ts b/packages/env/src/index.edge-light.ts
index 7facc4e44..f7e49cd31 100644
--- a/packages/env/src/index.edge-light.ts
+++ b/packages/env/src/index.edge-light.ts
@@ -5,12 +5,6 @@
*/
import "./global-check.js";
export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
- loadTransformers,
- setTransformers,
- type LoadTransformerEvent,
- type OnLoad,
-} from "./multi-model/index.non-nodejs.js";
export * from "./node-polyfill.js";
export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";
diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts
index a1233b171..cb7e77e64 100644
--- a/packages/env/src/index.ts
+++ b/packages/env/src/index.ts
@@ -35,12 +35,6 @@ export function createSHA256(): SHA256 {
}
export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
- loadTransformers,
- setTransformers,
- type LoadTransformerEvent,
- type OnLoad,
-} from "./multi-model/index.js";
export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
export {
AsyncLocalStorage,
diff --git a/packages/env/src/index.workerd.ts b/packages/env/src/index.workerd.ts
index 171fb7d37..1eacb2860 100644
--- a/packages/env/src/index.workerd.ts
+++ b/packages/env/src/index.workerd.ts
@@ -16,10 +16,4 @@ export function getEnv(name: string): string | undefined {
}
export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
-export {
- loadTransformers,
- setTransformers,
- type LoadTransformerEvent,
- type OnLoad,
-} from "./multi-model/index.non-nodejs.js";
export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
diff --git a/packages/env/src/multi-model/index.browser.ts b/packages/env/src/internal/multi-model/browser.ts
similarity index 100%
rename from packages/env/src/multi-model/index.browser.ts
rename to packages/env/src/internal/multi-model/browser.ts
diff --git a/packages/env/src/multi-model/index.ts b/packages/env/src/internal/multi-model/node.ts
similarity index 100%
rename from packages/env/src/multi-model/index.ts
rename to packages/env/src/internal/multi-model/node.ts
diff --git a/packages/env/src/multi-model/index.non-nodejs.ts b/packages/env/src/internal/multi-model/non-node.ts
similarity index 100%
rename from packages/env/src/multi-model/index.non-nodejs.ts
rename to packages/env/src/internal/multi-model/non-node.ts
diff --git a/packages/env/src/multi-model/shared.ts b/packages/env/src/internal/multi-model/shared.ts
similarity index 100%
rename from packages/env/src/multi-model/shared.ts
rename to packages/env/src/internal/multi-model/shared.ts
diff --git a/packages/env/src/multi-model.browser.ts b/packages/env/src/multi-model.browser.ts
new file mode 100644
index 000000000..d37b0793e
--- /dev/null
+++ b/packages/env/src/multi-model.browser.ts
@@ -0,0 +1,6 @@
+export {
+ loadTransformers,
+ setTransformers,
+ type LoadTransformerEvent,
+ type OnLoad,
+} from "./internal/multi-model/browser.js";
diff --git a/packages/env/src/multi-model.edge-light.ts b/packages/env/src/multi-model.edge-light.ts
new file mode 100644
index 000000000..b38c78b7f
--- /dev/null
+++ b/packages/env/src/multi-model.edge-light.ts
@@ -0,0 +1,6 @@
+export {
+ loadTransformers,
+ setTransformers,
+ type LoadTransformerEvent,
+ type OnLoad,
+} from "./internal/multi-model/non-node.js";
diff --git a/packages/env/src/multi-model.ts b/packages/env/src/multi-model.ts
new file mode 100644
index 000000000..c0a9dbb98
--- /dev/null
+++ b/packages/env/src/multi-model.ts
@@ -0,0 +1,6 @@
+export {
+ loadTransformers,
+ setTransformers,
+ type LoadTransformerEvent,
+ type OnLoad,
+} from "./internal/multi-model/node.js";
diff --git a/packages/env/src/multi-model.workerd.ts b/packages/env/src/multi-model.workerd.ts
new file mode 100644
index 000000000..b38c78b7f
--- /dev/null
+++ b/packages/env/src/multi-model.workerd.ts
@@ -0,0 +1,6 @@
+export {
+ loadTransformers,
+ setTransformers,
+ type LoadTransformerEvent,
+ type OnLoad,
+} from "./internal/multi-model/non-node.js";
diff --git a/packages/llamaindex/e2e/node/embedding/clip.e2e.ts b/packages/llamaindex/e2e/node/embedding/clip.e2e.ts
index d0cf2e707..e21c9fe16 100644
--- a/packages/llamaindex/e2e/node/embedding/clip.e2e.ts
+++ b/packages/llamaindex/e2e/node/embedding/clip.e2e.ts
@@ -1,5 +1,5 @@
-import type { LoadTransformerEvent } from "@llamaindex/env";
-import { setTransformers } from "@llamaindex/env";
+import type { LoadTransformerEvent } from "@llamaindex/env/multi-model";
+import { setTransformers } from "@llamaindex/env/multi-model";
import { ClipEmbedding, ImageNode, Settings } from "llamaindex";
import assert from "node:assert";
import { type Mock, test } from "node:test";
diff --git a/packages/providers/clip/src/embedding.ts b/packages/providers/clip/src/embedding.ts
index cf29eff21..4889ef697 100644
--- a/packages/providers/clip/src/embedding.ts
+++ b/packages/providers/clip/src/embedding.ts
@@ -2,7 +2,10 @@ import { MultiModalEmbedding } from "@llamaindex/core/embeddings";
import type { ImageType } from "@llamaindex/core/schema";
// only import type, to avoid bundling error
import { Settings } from "@llamaindex/core/global";
-import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env";
+import {
+ type LoadTransformerEvent,
+ loadTransformers,
+} from "@llamaindex/env/multi-model";
import type {
CLIPTextModelWithProjection,
CLIPVisionModelWithProjection,
diff --git a/packages/providers/huggingface/src/embedding.ts b/packages/providers/huggingface/src/embedding.ts
index 4cf8aa157..3e741b36f 100644
--- a/packages/providers/huggingface/src/embedding.ts
+++ b/packages/providers/huggingface/src/embedding.ts
@@ -1,6 +1,9 @@
import { BaseEmbedding } from "@llamaindex/core/embeddings";
import { Settings } from "@llamaindex/core/global";
-import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env";
+import {
+ type LoadTransformerEvent,
+ loadTransformers,
+} from "@llamaindex/env/multi-model";
import type { pipeline } from "@xenova/transformers";
import { HuggingFaceEmbeddingModelType } from "./shared";
diff --git a/packages/providers/huggingface/src/llm.ts b/packages/providers/huggingface/src/llm.ts
index 82d8e3c32..4a5b7ca85 100644
--- a/packages/providers/huggingface/src/llm.ts
+++ b/packages/providers/huggingface/src/llm.ts
@@ -9,7 +9,7 @@ import {
type LLMChatParamsStreaming,
type LLMMetadata,
} from "@llamaindex/core/llms";
-import { loadTransformers } from "@llamaindex/env";
+import { loadTransformers } from "@llamaindex/env/multi-model";
import type {
PreTrainedModel,
PreTrainedTokenizer,
--
GitLab