diff --git a/.changeset/cool-cows-lick.md b/.changeset/cool-cows-lick.md new file mode 100644 index 0000000000000000000000000000000000000000..4751bf218035fd5e3d7fafc90aa3c89bfc4f7120 --- /dev/null +++ b/.changeset/cool-cows-lick.md @@ -0,0 +1,8 @@ +--- +"@llamaindex/env": patch +"llamaindex": patch +"@llamaindex/clip": patch +"@llamaindex/huggingface": patch +--- + +chore: move multi-model into single sub module diff --git a/packages/env/multi-model/package.json b/packages/env/multi-model/package.json new file mode 100644 index 0000000000000000000000000000000000000000..35136ddebdb2434db5ab14331da648d6fc423167 --- /dev/null +++ b/packages/env/multi-model/package.json @@ -0,0 +1,12 @@ +{ + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + "browser": "./dist/index.browser.js", + "edge-light": "./dist/index.edge-light.js", + "workerd": "./dist/index.workerd.js" + }, + "private": true +} diff --git a/packages/env/package.json b/packages/env/package.json index 16d43caf02b70f9849b75cdf5dd5b36193ccfbf6..c11889f799e3e5baed43280eabc4bfa8fabfc84b 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -28,7 +28,7 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js", "require": "./dist/index.cjs", - "default": "./dist/index.cjs" + "default": "./dist/index.js" }, "workerd": { "types": "./dist/index.workerd.d.ts", @@ -50,9 +50,36 @@ "types": "./dist/index.d.cts", "default": "./dist/index.cjs" } + }, + "./multi-model": { + "workerd": { + "types": "./multi-model/dist/multi-model.workerd.d.ts", + "default": "./multi-model/dist/multi-model.workerd.js" + }, + "edge-light": { + "types": "./multi-model/dist/multi-model.edge-light.d.ts", + "default": "./multi-model/dist/multi-model.edge-light.js" + }, + "browser": { + "types": "./multi-model/dist/multi-model.browser.d.ts", + "default": "./multi-model/dist/multi-model.browser.js" + }, + "import": { + "types": "./multi-model/dist/multi-model.d.ts", + "default": "./multi-model/dist/multi-model.js" + }, + "require": { + "types": "./multi-model/dist/multi-model.d.cts", + "default": "./multi-model/dist/multi-model.cjs" + }, + "default": { + "types": "./multi-model/dist/multi-model.d.ts", + "default": "./multi-model/dist/multi-model.js" + } } }, "files": [ + "multi-model", "dist", "CHANGELOG.md", "!**/*.tsbuildinfo" diff --git a/packages/env/src/index.browser.ts b/packages/env/src/index.browser.ts index cd1747ffe0cd4098ae94b8342f065980fbe7671e..9d55f99eddb65e412e49f81e9a11ef98c15cab66 100644 --- a/packages/env/src/index.browser.ts +++ b/packages/env/src/index.browser.ts @@ -6,12 +6,6 @@ import "./global-check.js"; export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.browser.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js"; export { NotSupportCurrentRuntimeClass } from "./utils/shared.js"; export * from "./web-polyfill.js"; diff --git a/packages/env/src/index.edge-light.ts b/packages/env/src/index.edge-light.ts index 7facc4e44e8f70c7e91011b5df97f2cd45d7f846..f7e49cd319412891886d5c1fd489d5bd982e766e 100644 --- a/packages/env/src/index.edge-light.ts +++ b/packages/env/src/index.edge-light.ts @@ -5,12 +5,6 @@ */ import "./global-check.js"; export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.non-nodejs.js"; export * from "./node-polyfill.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js"; export { NotSupportCurrentRuntimeClass } from "./utils/shared.js"; diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index a1233b171be2b6bcbf9faac01c90b5330c09cd6f..cb7e77e64fd1d8d81ad1552956dee7e50b428f2a 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -35,12 +35,6 @@ export function createSHA256(): SHA256 { } export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js"; export { AsyncLocalStorage, diff --git a/packages/env/src/index.workerd.ts b/packages/env/src/index.workerd.ts index 171fb7d37495cccd3396427f1d617dd678748cc3..1eacb2860c387372e9aabb2ded70d6d6476b02db 100644 --- a/packages/env/src/index.workerd.ts +++ b/packages/env/src/index.workerd.ts @@ -16,10 +16,4 @@ export function getEnv(name: string): string | undefined { } export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js"; -export { - loadTransformers, - setTransformers, - type LoadTransformerEvent, - type OnLoad, -} from "./multi-model/index.non-nodejs.js"; export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js"; diff --git a/packages/env/src/multi-model/index.browser.ts b/packages/env/src/internal/multi-model/browser.ts similarity index 100% rename from packages/env/src/multi-model/index.browser.ts rename to packages/env/src/internal/multi-model/browser.ts diff --git a/packages/env/src/multi-model/index.ts b/packages/env/src/internal/multi-model/node.ts similarity index 100% rename from packages/env/src/multi-model/index.ts rename to packages/env/src/internal/multi-model/node.ts diff --git a/packages/env/src/multi-model/index.non-nodejs.ts b/packages/env/src/internal/multi-model/non-node.ts similarity index 100% rename from packages/env/src/multi-model/index.non-nodejs.ts rename to packages/env/src/internal/multi-model/non-node.ts diff --git a/packages/env/src/multi-model/shared.ts b/packages/env/src/internal/multi-model/shared.ts similarity index 100% rename from packages/env/src/multi-model/shared.ts rename to packages/env/src/internal/multi-model/shared.ts diff --git a/packages/env/src/multi-model.browser.ts b/packages/env/src/multi-model.browser.ts new file mode 100644 index 0000000000000000000000000000000000000000..d37b0793ec47415bebced2183460ff1a356baaa1 --- /dev/null +++ b/packages/env/src/multi-model.browser.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/browser.js"; diff --git a/packages/env/src/multi-model.edge-light.ts b/packages/env/src/multi-model.edge-light.ts new file mode 100644 index 0000000000000000000000000000000000000000..b38c78b7f1e43c9dfeb199ab2dd4795ff991e288 --- /dev/null +++ b/packages/env/src/multi-model.edge-light.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/non-node.js"; diff --git a/packages/env/src/multi-model.ts b/packages/env/src/multi-model.ts new file mode 100644 index 0000000000000000000000000000000000000000..c0a9dbb9896605d582093e1d5ba2a87ee0c42e0c --- /dev/null +++ b/packages/env/src/multi-model.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/node.js"; diff --git a/packages/env/src/multi-model.workerd.ts b/packages/env/src/multi-model.workerd.ts new file mode 100644 index 0000000000000000000000000000000000000000..b38c78b7f1e43c9dfeb199ab2dd4795ff991e288 --- /dev/null +++ b/packages/env/src/multi-model.workerd.ts @@ -0,0 +1,6 @@ +export { + loadTransformers, + setTransformers, + type LoadTransformerEvent, + type OnLoad, +} from "./internal/multi-model/non-node.js"; diff --git a/packages/llamaindex/e2e/node/embedding/clip.e2e.ts b/packages/llamaindex/e2e/node/embedding/clip.e2e.ts index d0cf2e7072a07e2a85dd9a84ac6886e6e85f07b4..e21c9fe16cd093dc177a65e8feb020be993e1699 100644 --- a/packages/llamaindex/e2e/node/embedding/clip.e2e.ts +++ b/packages/llamaindex/e2e/node/embedding/clip.e2e.ts @@ -1,5 +1,5 @@ -import type { LoadTransformerEvent } from "@llamaindex/env"; -import { setTransformers } from "@llamaindex/env"; +import type { LoadTransformerEvent } from "@llamaindex/env/multi-model"; +import { setTransformers } from "@llamaindex/env/multi-model"; import { ClipEmbedding, ImageNode, Settings } from "llamaindex"; import assert from "node:assert"; import { type Mock, test } from "node:test"; diff --git a/packages/providers/clip/src/embedding.ts b/packages/providers/clip/src/embedding.ts index cf29eff21370853c0c62a0de20ea427dea9d5599..4889ef69784ac5f76903a1ad8c6efe62c0fdc506 100644 --- a/packages/providers/clip/src/embedding.ts +++ b/packages/providers/clip/src/embedding.ts @@ -2,7 +2,10 @@ import { MultiModalEmbedding } from "@llamaindex/core/embeddings"; import type { ImageType } from "@llamaindex/core/schema"; // only import type, to avoid bundling error import { Settings } from "@llamaindex/core/global"; -import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env"; +import { + type LoadTransformerEvent, + loadTransformers, +} from "@llamaindex/env/multi-model"; import type { CLIPTextModelWithProjection, CLIPVisionModelWithProjection, diff --git a/packages/providers/huggingface/src/embedding.ts b/packages/providers/huggingface/src/embedding.ts index 4cf8aa157c0e4b27e9239ec955f3339e2eb6e0e0..3e741b36f50b265d7780aabfe1789420691686ec 100644 --- a/packages/providers/huggingface/src/embedding.ts +++ b/packages/providers/huggingface/src/embedding.ts @@ -1,6 +1,9 @@ import { BaseEmbedding } from "@llamaindex/core/embeddings"; import { Settings } from "@llamaindex/core/global"; -import { type LoadTransformerEvent, loadTransformers } from "@llamaindex/env"; +import { + type LoadTransformerEvent, + loadTransformers, +} from "@llamaindex/env/multi-model"; import type { pipeline } from "@xenova/transformers"; import { HuggingFaceEmbeddingModelType } from "./shared"; diff --git a/packages/providers/huggingface/src/llm.ts b/packages/providers/huggingface/src/llm.ts index 82d8e3c3229140e96233a3cdaa664d12ce050cfa..4a5b7ca8584ef44fd4fc82416162778a23bf8129 100644 --- a/packages/providers/huggingface/src/llm.ts +++ b/packages/providers/huggingface/src/llm.ts @@ -9,7 +9,7 @@ import { type LLMChatParamsStreaming, type LLMMetadata, } from "@llamaindex/core/llms"; -import { loadTransformers } from "@llamaindex/env"; +import { loadTransformers } from "@llamaindex/env/multi-model"; import type { PreTrainedModel, PreTrainedTokenizer,