From 4fc001c8deae1ac620214b8b2cbde1ba3cd3e0d5 Mon Sep 17 00:00:00 2001
From: Alex Yang <himself65@outlook.com>
Date: Sun, 10 Nov 2024 20:14:44 -0800
Subject: [PATCH] chore: bump `@huggingface/transformers` (#1459)
---
.changeset/pink-socks-kneel.md | 10 ++
.../nextjs-node-runtime/src/actions/openai.ts | 1 -
e2e/node/embedding/clip.e2e.ts | 2 +-
e2e/package.json | 3 +-
examples/vectorIndexLocal.ts | 1 -
packages/env/package.json | 6 +-
.../env/src/internal/multi-model/browser.ts | 6 +-
packages/env/src/internal/multi-model/node.ts | 2 +-
.../env/src/internal/multi-model/non-node.ts | 8 +-
.../env/src/internal/multi-model/shared.ts | 8 +-
packages/llamaindex/package.json | 2 +-
packages/llamaindex/src/next.ts | 2 +-
packages/providers/clip/package.json | 5 +-
packages/providers/clip/src/embedding.ts | 12 +-
packages/providers/huggingface/package.json | 5 +-
.../providers/huggingface/src/embedding.ts | 26 ++-
packages/providers/huggingface/src/index.ts | 5 +-
packages/providers/huggingface/src/llm.ts | 15 +-
pnpm-lock.yaml | 149 ++++++++++++++----
19 files changed, 196 insertions(+), 72 deletions(-)
create mode 100644 .changeset/pink-socks-kneel.md
diff --git a/.changeset/pink-socks-kneel.md b/.changeset/pink-socks-kneel.md
new file mode 100644
index 000000000..3c7ead08b
--- /dev/null
+++ b/.changeset/pink-socks-kneel.md
@@ -0,0 +1,10 @@
+---
+"@llamaindex/env": patch
+"llamaindex": patch
+"@llamaindex/clip": patch
+"@llamaindex/huggingface": patch
+---
+
+chore: bump `@huggingface/transformers`
+
+Upgrade to v3, please read https://github.com/huggingface/transformers.js/releases/tag/3.0.0 for more information.
diff --git a/e2e/examples/nextjs-node-runtime/src/actions/openai.ts b/e2e/examples/nextjs-node-runtime/src/actions/openai.ts
index a5643bb92..28d9fe911 100644
--- a/e2e/examples/nextjs-node-runtime/src/actions/openai.ts
+++ b/e2e/examples/nextjs-node-runtime/src/actions/openai.ts
@@ -15,7 +15,6 @@ Settings.llm = new OpenAI({
});
Settings.embedModel = new HuggingFaceEmbedding({
modelType: "BAAI/bge-small-en-v1.5",
- quantized: false,
});
Settings.callbackManager.on("llm-tool-call", (event) => {
console.log(event.detail);
diff --git a/e2e/node/embedding/clip.e2e.ts b/e2e/node/embedding/clip.e2e.ts
index e21c9fe16..3b6a8c2a7 100644
--- a/e2e/node/embedding/clip.e2e.ts
+++ b/e2e/node/embedding/clip.e2e.ts
@@ -64,7 +64,7 @@ await test("clip embedding", async (t) => {
});
await t.test("custom transformer", async () => {
- const transformers = await import("@xenova/transformers");
+ const transformers = await import("@huggingface/transformers");
const getter = test.mock.fn((t, k, r) => {
return Reflect.get(t, k, r);
});
diff --git a/e2e/package.json b/e2e/package.json
index 65a797cb3..55808e1e2 100644
--- a/e2e/package.json
+++ b/e2e/package.json
@@ -12,10 +12,11 @@
"@faker-js/faker": "^9.2.0",
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*",
+ "@llamaindex/ollama": "workspace:*",
"@llamaindex/openai": "workspace:*",
"@types/node": "^22.9.0",
"@types/pg": "^8.11.8",
- "@xenova/transformers": "^2.17.2",
+ "@huggingface/transformers": "^3.0.2",
"consola": "^3.2.3",
"dotenv": "^16.4.5",
"llamaindex": "workspace:*",
diff --git a/examples/vectorIndexLocal.ts b/examples/vectorIndexLocal.ts
index 859f21e0a..9d19e68ce 100644
--- a/examples/vectorIndexLocal.ts
+++ b/examples/vectorIndexLocal.ts
@@ -14,7 +14,6 @@ Settings.llm = new Ollama({
Settings.embedModel = new HuggingFaceEmbedding({
modelType: "BAAI/bge-small-en-v1.5",
- quantized: false,
});
async function main() {
diff --git a/packages/env/package.json b/packages/env/package.json
index fc39cf6fd..433d40f52 100644
--- a/packages/env/package.json
+++ b/packages/env/package.json
@@ -124,7 +124,7 @@
"devDependencies": {
"@types/node": "^22.9.0",
"@types/readable-stream": "^4.0.15",
- "@xenova/transformers": "^2.17.2",
+ "@huggingface/transformers": "^3.0.2",
"bunchee": "5.6.1",
"gpt-tokenizer": "^2.6.0",
"pathe": "^1.1.2",
@@ -132,7 +132,7 @@
},
"peerDependencies": {
"@aws-crypto/sha256-js": "^5.2.0",
- "@xenova/transformers": "^2.17.2",
+ "@huggingface/transformers": "^3.0.2",
"gpt-tokenizer": "^2.5.0",
"js-tiktoken": "^1.0.12",
"pathe": "^1.1.2"
@@ -141,7 +141,7 @@
"@aws-crypto/sha256-js": {
"optional": true
},
- "@xenova/transformers": {
+ "@huggingface/transformers": {
"optional": true
},
"pathe": {
diff --git a/packages/env/src/internal/multi-model/browser.ts b/packages/env/src/internal/multi-model/browser.ts
index 7f46c0472..f527eb8fb 100644
--- a/packages/env/src/internal/multi-model/browser.ts
+++ b/packages/env/src/internal/multi-model/browser.ts
@@ -8,8 +8,10 @@ export {
export async function loadTransformers(onLoad: OnLoad) {
if (getTransformers() === null) {
setTransformers(
- // @ts-expect-error no type
- await import("https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2"),
+ await import(
+ // @ts-expect-error no type
+ "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.2"
+ ),
);
} else {
return getTransformers()!;
diff --git a/packages/env/src/internal/multi-model/node.ts b/packages/env/src/internal/multi-model/node.ts
index 2b3430060..b653cf616 100644
--- a/packages/env/src/internal/multi-model/node.ts
+++ b/packages/env/src/internal/multi-model/node.ts
@@ -8,7 +8,7 @@ export {
export async function loadTransformers(onLoad: OnLoad) {
if (getTransformers() === null) {
- setTransformers(await import("@xenova/transformers"));
+ setTransformers(await import("@huggingface/transformers"));
} else {
return getTransformers()!;
}
diff --git a/packages/env/src/internal/multi-model/non-node.ts b/packages/env/src/internal/multi-model/non-node.ts
index 6dba2e0c7..a8a320036 100644
--- a/packages/env/src/internal/multi-model/non-node.ts
+++ b/packages/env/src/internal/multi-model/non-node.ts
@@ -9,7 +9,7 @@ export async function loadTransformers(onLoad: OnLoad) {
if (getTransformers() === null) {
/**
* If you see this warning, it means that the current environment does not support the transformer.
- * because "@xeonva/transformers" highly depends on Node.js APIs.
+ * because "@huggingface/transformers" highly depends on Node.js APIs.
*
* One possible solution is to fix their implementation to make it work in the non-Node.js environment,
* but it's not worth the effort because Edge Runtime and Cloudflare Workers are not the for heavy Machine Learning task.
@@ -17,14 +17,14 @@ export async function loadTransformers(onLoad: OnLoad) {
* Or you can provide an RPC server that runs the transformer in a Node.js environment.
* Or you just run the code in a Node.js environment.
*
- * Refs: https://github.com/xenova/transformers.js/issues/309
+ * Refs: https://github.com/huggingface/transformers.js/issues/309
*/
console.warn(
- '"@xenova/transformers" is not officially supported in this environment, some features may not work as expected.',
+ '"@huggingface/transformers" is not officially supported in this environment, some features may not work as expected.',
);
setTransformers(
// @ts-expect-error no type
- await import("@xenova/transformers/dist/transformers"),
+ await import("@huggingface/transformers/dist/transformers.js"),
);
} else {
return getTransformers()!;
diff --git a/packages/env/src/internal/multi-model/shared.ts b/packages/env/src/internal/multi-model/shared.ts
index 7247a1a8c..93a0b1047 100644
--- a/packages/env/src/internal/multi-model/shared.ts
+++ b/packages/env/src/internal/multi-model/shared.ts
@@ -1,17 +1,17 @@
-let transformer: typeof import("@xenova/transformers") | null = null;
+let transformer: typeof import("@huggingface/transformers") | null = null;
export function getTransformers() {
return transformer;
}
-export function setTransformers(t: typeof import("@xenova/transformers")) {
+export function setTransformers(t: typeof import("@huggingface/transformers")) {
transformer = t;
}
export type OnLoad = (
- transformer: typeof import("@xenova/transformers"),
+ transformer: typeof import("@huggingface/transformers"),
) => void;
export type LoadTransformerEvent = {
- transformer: typeof import("@xenova/transformers");
+ transformer: typeof import("@huggingface/transformers");
};
diff --git a/packages/llamaindex/package.json b/packages/llamaindex/package.json
index e513c0fa9..b35e7ec40 100644
--- a/packages/llamaindex/package.json
+++ b/packages/llamaindex/package.json
@@ -88,7 +88,7 @@
"@swc/cli": "^0.5.0",
"@swc/core": "^1.7.22",
"@vercel/postgres": "^0.10.0",
- "@xenova/transformers": "^2.17.2",
+ "@huggingface/transformers": "^3.0.2",
"concurrently": "^9.1.0",
"glob": "^11.0.0",
"pg": "^8.12.0",
diff --git a/packages/llamaindex/src/next.ts b/packages/llamaindex/src/next.ts
index 41b440a1f..155e180ad 100644
--- a/packages/llamaindex/src/next.ts
+++ b/packages/llamaindex/src/next.ts
@@ -22,7 +22,7 @@ export default function withLlamaIndex(config: any) {
config.experimental.serverComponentsExternalPackages =
config.experimental.serverComponentsExternalPackages ?? [];
config.experimental.serverComponentsExternalPackages.push(
- "@xenova/transformers",
+ "@huggingface/transformers",
);
const userWebpack = config.webpack;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
diff --git a/packages/providers/clip/package.json b/packages/providers/clip/package.json
index 4ac76d3ab..6276d76aa 100644
--- a/packages/providers/clip/package.json
+++ b/packages/providers/clip/package.json
@@ -45,6 +45,9 @@
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*",
"@llamaindex/openai": "workspace:*",
- "@xenova/transformers": "^2.17.2"
+ "@huggingface/transformers": "^3.0.2"
+ },
+ "peerDependencies": {
+ "@huggingface/transformers": "^3.0.2"
}
}
diff --git a/packages/providers/clip/src/embedding.ts b/packages/providers/clip/src/embedding.ts
index 4889ef697..789507581 100644
--- a/packages/providers/clip/src/embedding.ts
+++ b/packages/providers/clip/src/embedding.ts
@@ -1,17 +1,17 @@
import { MultiModalEmbedding } from "@llamaindex/core/embeddings";
import type { ImageType } from "@llamaindex/core/schema";
// only import type, to avoid bundling error
-import { Settings } from "@llamaindex/core/global";
-import {
- type LoadTransformerEvent,
- loadTransformers,
-} from "@llamaindex/env/multi-model";
import type {
CLIPTextModelWithProjection,
CLIPVisionModelWithProjection,
PreTrainedTokenizer,
Processor,
-} from "@xenova/transformers";
+} from "@huggingface/transformers";
+import { Settings } from "@llamaindex/core/global";
+import {
+ type LoadTransformerEvent,
+ loadTransformers,
+} from "@llamaindex/env/multi-model";
import { ClipEmbeddingModelType } from "./shared";
declare module "@llamaindex/core/global" {
diff --git a/packages/providers/huggingface/package.json b/packages/providers/huggingface/package.json
index b6082c8fd..86bc69897 100644
--- a/packages/providers/huggingface/package.json
+++ b/packages/providers/huggingface/package.json
@@ -46,6 +46,9 @@
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*",
"@llamaindex/openai": "workspace:*",
- "@xenova/transformers": "^2.17.2"
+ "@huggingface/transformers": "^3.0.2"
+ },
+ "peerDependencies": {
+ "@huggingface/transformers": "^3.0.2"
}
}
diff --git a/packages/providers/huggingface/src/embedding.ts b/packages/providers/huggingface/src/embedding.ts
index 3e741b36f..0af90266c 100644
--- a/packages/providers/huggingface/src/embedding.ts
+++ b/packages/providers/huggingface/src/embedding.ts
@@ -1,10 +1,10 @@
+import type { pipeline } from "@huggingface/transformers";
import { BaseEmbedding } from "@llamaindex/core/embeddings";
import { Settings } from "@llamaindex/core/global";
import {
type LoadTransformerEvent,
loadTransformers,
} from "@llamaindex/env/multi-model";
-import type { pipeline } from "@xenova/transformers";
import { HuggingFaceEmbeddingModelType } from "./shared";
declare module "@llamaindex/core/global" {
@@ -13,6 +13,11 @@ declare module "@llamaindex/core/global" {
}
}
+export type HuggingFaceEmbeddingParams = {
+ modelType?: string;
+ modelOptions?: Parameters<typeof pipeline<"feature-extraction">>[2];
+};
+
/**
* Uses feature extraction from '@xenova/transformers' to generate embeddings.
* Per default the model [XENOVA_ALL_MINILM_L6_V2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) is used.
@@ -28,15 +33,20 @@ declare module "@llamaindex/core/global" {
*/
export class HuggingFaceEmbedding extends BaseEmbedding {
modelType: string = HuggingFaceEmbeddingModelType.XENOVA_ALL_MINILM_L6_V2;
- quantized: boolean = true;
+ modelOptions: Parameters<typeof pipeline<"feature-extraction">>[2] = {};
private extractor: Awaited<
ReturnType<typeof pipeline<"feature-extraction">>
> | null = null;
- constructor(init?: Partial<HuggingFaceEmbedding>) {
+ constructor(params: HuggingFaceEmbeddingParams = {}) {
super();
- Object.assign(this, init);
+ if (params.modelType) {
+ this.modelType = params.modelType;
+ }
+ if (params.modelOptions) {
+ this.modelOptions = params.modelOptions;
+ }
}
async getExtractor() {
@@ -50,9 +60,11 @@ export class HuggingFaceEmbedding extends BaseEmbedding {
true,
);
});
- this.extractor = await pipeline("feature-extraction", this.modelType, {
- quantized: this.quantized,
- });
+ this.extractor = await pipeline(
+ "feature-extraction",
+ this.modelType,
+ this.modelOptions,
+ );
}
return this.extractor;
}
diff --git a/packages/providers/huggingface/src/index.ts b/packages/providers/huggingface/src/index.ts
index eca5865de..06b31c9a9 100644
--- a/packages/providers/huggingface/src/index.ts
+++ b/packages/providers/huggingface/src/index.ts
@@ -1,4 +1,7 @@
-export { HuggingFaceEmbedding } from "./embedding";
+export {
+ HuggingFaceEmbedding,
+ type HuggingFaceEmbeddingParams,
+} from "./embedding";
export { HuggingFaceLLM, type HFLLMConfig } from "./llm";
export {
HuggingFaceEmbeddingModelType,
diff --git a/packages/providers/huggingface/src/llm.ts b/packages/providers/huggingface/src/llm.ts
index 4a5b7ca85..4af3fd49b 100644
--- a/packages/providers/huggingface/src/llm.ts
+++ b/packages/providers/huggingface/src/llm.ts
@@ -1,3 +1,8 @@
+import type {
+ PreTrainedModel,
+ PreTrainedTokenizer,
+ Tensor,
+} from "@huggingface/transformers";
import { wrapLLMEvent } from "@llamaindex/core/decorator";
import { Settings } from "@llamaindex/core/global";
import "@llamaindex/core/llms";
@@ -10,11 +15,6 @@ import {
type LLMMetadata,
} from "@llamaindex/core/llms";
import { loadTransformers } from "@llamaindex/env/multi-model";
-import type {
- PreTrainedModel,
- PreTrainedTokenizer,
- Tensor,
-} from "@xenova/transformers";
import { DEFAULT_PARAMS } from "./shared";
const DEFAULT_HUGGINGFACE_MODEL = "stabilityai/stablelm-tuned-alpha-3b";
@@ -122,7 +122,10 @@ export class HuggingFaceLLM extends BaseLLM {
// TODO: the input for model.generate should be updated when using @xenova/transformers v3
// We should add `stopping_criteria` also when it's supported in v3
// See: https://github.com/xenova/transformers.js/blob/3260640b192b3e06a10a1f4dc004b1254fdf1b80/src/models.js#L1248C9-L1248C27
- const outputs = await model.generate(inputs, this.metadata);
+ const outputs = (await model.generate({
+ inputs,
+ ...this.metadata,
+ })) as Tensor;
const outputText = tokenizer.batch_decode(outputs, {
skip_special_tokens: false,
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1dfaa1ba6..1feea40fc 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -340,12 +340,18 @@ importers:
'@faker-js/faker':
specifier: ^9.2.0
version: 9.2.0
+ '@huggingface/transformers':
+ specifier: ^3.0.2
+ version: 3.0.2
'@llamaindex/core':
specifier: workspace:*
version: link:../packages/core
'@llamaindex/env':
specifier: workspace:*
version: link:../packages/env
+ '@llamaindex/ollama':
+ specifier: workspace:*
+ version: link:../packages/providers/ollama
'@llamaindex/openai':
specifier: workspace:*
version: link:../packages/providers/openai
@@ -355,9 +361,6 @@ importers:
'@types/pg':
specifier: ^8.11.8
version: 8.11.10
- '@xenova/transformers':
- specifier: ^2.17.2
- version: 2.17.2
consola:
specifier: ^3.2.3
version: 3.2.3
@@ -890,15 +893,15 @@ importers:
specifier: ^1.0.12
version: 1.0.15
devDependencies:
+ '@huggingface/transformers':
+ specifier: ^3.0.2
+ version: 3.0.2
'@types/node':
specifier: ^22.9.0
version: 22.9.0
'@types/readable-stream':
specifier: ^4.0.15
version: 4.0.18
- '@xenova/transformers':
- specifier: ^2.17.2
- version: 2.17.2
bunchee:
specifier: 5.6.1
version: 5.6.1(typescript@5.6.3)
@@ -1105,6 +1108,9 @@ importers:
specifier: ^3.23.8
version: 3.23.8
devDependencies:
+ '@huggingface/transformers':
+ specifier: ^3.0.2
+ version: 3.0.2
'@swc/cli':
specifier: ^0.5.0
version: 0.5.0(@swc/core@1.7.42(@swc/helpers@0.5.13))(chokidar@3.6.0)
@@ -1114,9 +1120,6 @@ importers:
'@vercel/postgres':
specifier: ^0.10.0
version: 0.10.0
- '@xenova/transformers':
- specifier: ^2.17.2
- version: 2.17.2
concurrently:
specifier: ^9.1.0
version: 9.1.0
@@ -1203,6 +1206,9 @@ importers:
packages/providers/clip:
dependencies:
+ '@huggingface/transformers':
+ specifier: ^3.0.2
+ version: 3.0.2
'@llamaindex/core':
specifier: workspace:*
version: link:../../core
@@ -1212,9 +1218,6 @@ importers:
'@llamaindex/openai':
specifier: workspace:*
version: link:../openai
- '@xenova/transformers':
- specifier: ^2.17.2
- version: 2.17.2
devDependencies:
bunchee:
specifier: 5.6.1
@@ -1257,6 +1260,9 @@ importers:
'@huggingface/inference':
specifier: ^2.8.1
version: 2.8.1
+ '@huggingface/transformers':
+ specifier: ^3.0.2
+ version: 3.0.2
'@llamaindex/core':
specifier: workspace:*
version: link:../../core
@@ -1266,9 +1272,6 @@ importers:
'@llamaindex/openai':
specifier: workspace:*
version: link:../openai
- '@xenova/transformers':
- specifier: ^2.17.2
- version: 2.17.2
devDependencies:
bunchee:
specifier: 5.6.1
@@ -3527,13 +3530,16 @@ packages:
resolution: {integrity: sha512-9KsiorsdIK8+7VmlamAT7Uh90zxAhC/SeKaKc80v58JhtPYuwaJpmR/ST7XAUxrHAFqHTCoTH5aJnJDwSL6xIQ==}
engines: {node: '>=18'}
- '@huggingface/jinja@0.2.2':
- resolution: {integrity: sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==}
+ '@huggingface/jinja@0.3.2':
+ resolution: {integrity: sha512-F2FvuIc+w1blGsaqJI/OErRbWH6bVJDCBI8Rm5D86yZ2wlwrGERsfIaru7XUv9eYC3DMP3ixDRRtF0h6d8AZcQ==}
engines: {node: '>=18'}
'@huggingface/tasks@0.12.22':
resolution: {integrity: sha512-zpoCUJPUUQ9924PGMXZx8fYpcBnsMV7c+6EUZ4edVZmw28sW9yTo+RJAXrbcLquVsYrYxNgkKYc3odnM0/swfg==}
+ '@huggingface/transformers@3.0.2':
+ resolution: {integrity: sha512-lTyS81eQazMea5UCehDGFMfdcNRZyei7XQLH5X6j4AhA/18Ka0+5qPgMxUxuZLU4xkv60aY2KNz9Yzthv6WVJg==}
+
'@humanfs/core@0.19.1':
resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==}
engines: {node: '>=18.18.0'}
@@ -3688,6 +3694,10 @@ packages:
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
engines: {node: '>=12'}
+ '@isaacs/fs-minipass@4.0.1':
+ resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==}
+ engines: {node: '>=18.0.0'}
+
'@jest/schemas@29.6.3':
resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -5729,9 +5739,6 @@ packages:
peerDependencies:
react: ^16.8.0 || ^17.0.0 || ^18.0.0
- '@xenova/transformers@2.17.2':
- resolution: {integrity: sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ==}
-
'@xmldom/xmldom@0.8.10':
resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==}
engines: {node: '>=10.0.0'}
@@ -6366,6 +6373,10 @@ packages:
resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==}
engines: {node: '>=10'}
+ chownr@3.0.0:
+ resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==}
+ engines: {node: '>=18'}
+
chromadb-default-embed@2.13.2:
resolution: {integrity: sha512-mhqo5rLjkF2KkxAV0WS82vNIXWpVMzvz5y5ayIB2FxcebUbEBNlcRh6XSSqYChWMfJ9us1ZzLQU8RXqsy3sKaA==}
@@ -9634,6 +9645,10 @@ packages:
resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==}
engines: {node: '>= 8'}
+ minizlib@3.0.1:
+ resolution: {integrity: sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==}
+ engines: {node: '>= 18'}
+
mkdirp-classic@0.5.3:
resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
@@ -9646,6 +9661,11 @@ packages:
engines: {node: '>=10'}
hasBin: true
+ mkdirp@3.0.1:
+ resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==}
+ engines: {node: '>=10'}
+ hasBin: true
+
mlly@1.7.2:
resolution: {integrity: sha512-tN3dvVHYVz4DhSXinXIk7u9syPYaJvio118uomkovAtWBT+RdbP6Lfh/5Lvo519YMmwBafwlh20IPTXIStscpA==}
@@ -10034,13 +10054,26 @@ packages:
onnxruntime-common@1.14.0:
resolution: {integrity: sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==}
+ onnxruntime-common@1.19.2:
+ resolution: {integrity: sha512-a4R7wYEVFbZBlp0BfhpbFWqe4opCor3KM+5Wm22Az3NGDcQMiU2hfG/0MfnBs+1ZrlSGmlgWeMcXQkDk1UFb8Q==}
+
+ onnxruntime-common@1.20.0-dev.20241016-2b8fc5529b:
+ resolution: {integrity: sha512-KZK8b6zCYGZFjd4ANze0pqBnqnFTS3GIVeclQpa2qseDpXrCQJfkWBixRcrZShNhm3LpFOZ8qJYFC5/qsJK9WQ==}
+
onnxruntime-node@1.14.0:
resolution: {integrity: sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==}
os: [win32, darwin, linux]
+ onnxruntime-node@1.19.2:
+ resolution: {integrity: sha512-9eHMP/HKbbeUcqte1JYzaaRC8JPn7ojWeCeoyShO86TOR97OCyIyAIOGX3V95ErjslVhJRXY8Em/caIUc0hm1Q==}
+ os: [win32, darwin, linux]
+
onnxruntime-web@1.14.0:
resolution: {integrity: sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==}
+ onnxruntime-web@1.21.0-dev.20241024-d9ca84ef96:
+ resolution: {integrity: sha512-ANSQfMALvCviN3Y4tvTViKofKToV1WUb2r2VjZVCi3uUBPaK15oNJyIxhsNyEckBr/Num3JmSXlkHOD8HfVzSQ==}
+
open@8.4.2:
resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==}
engines: {node: '>=12'}
@@ -11397,6 +11430,10 @@ packages:
deprecated: Rimraf versions prior to v4 are no longer supported
hasBin: true
+ rimraf@5.0.10:
+ resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==}
+ hasBin: true
+
rimraf@6.0.1:
resolution: {integrity: sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==}
engines: {node: 20 || >=22}
@@ -12077,6 +12114,10 @@ packages:
resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==}
engines: {node: '>=10'}
+ tar@7.4.3:
+ resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}
+ engines: {node: '>=18'}
+
term-size@2.2.1:
resolution: {integrity: sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==}
engines: {node: '>=8'}
@@ -13052,6 +13093,10 @@ packages:
yallist@4.0.0:
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
+ yallist@5.0.0:
+ resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==}
+ engines: {node: '>=18'}
+
yaml@1.10.2:
resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
engines: {node: '>= 6'}
@@ -16271,12 +16316,19 @@ snapshots:
'@huggingface/jinja@0.1.3': {}
- '@huggingface/jinja@0.2.2': {}
+ '@huggingface/jinja@0.3.2': {}
'@huggingface/tasks@0.12.22':
dependencies:
'@huggingface/gguf': 0.1.12
+ '@huggingface/transformers@3.0.2':
+ dependencies:
+ '@huggingface/jinja': 0.3.2
+ onnxruntime-node: 1.19.2
+ onnxruntime-web: 1.21.0-dev.20241024-d9ca84ef96
+ sharp: 0.33.5
+
'@humanfs/core@0.19.1': {}
'@humanfs/node@0.16.6':
@@ -16404,6 +16456,10 @@ snapshots:
wrap-ansi: 8.1.0
wrap-ansi-cjs: wrap-ansi@7.0.0
+ '@isaacs/fs-minipass@4.0.1':
+ dependencies:
+ minipass: 7.1.2
+
'@jest/schemas@29.6.3':
dependencies:
'@sinclair/typebox': 0.27.8
@@ -18733,14 +18789,6 @@ snapshots:
'@types/react': 18.3.12
react: 18.3.1
- '@xenova/transformers@2.17.2':
- dependencies:
- '@huggingface/jinja': 0.2.2
- onnxruntime-web: 1.14.0
- sharp: 0.32.6
- optionalDependencies:
- onnxruntime-node: 1.14.0
-
'@xmldom/xmldom@0.8.10': {}
'@xtuc/ieee754@1.2.0': {}
@@ -19503,6 +19551,8 @@ snapshots:
chownr@2.0.0: {}
+ chownr@3.0.0: {}
+
chromadb-default-embed@2.13.2:
dependencies:
'@huggingface/jinja': 0.1.3
@@ -23815,6 +23865,11 @@ snapshots:
minipass: 3.3.6
yallist: 4.0.0
+ minizlib@3.0.1:
+ dependencies:
+ minipass: 7.1.2
+ rimraf: 5.0.10
+
mkdirp-classic@0.5.3: {}
mkdirp@0.5.6:
@@ -23823,6 +23878,8 @@ snapshots:
mkdirp@1.0.4: {}
+ mkdirp@3.0.1: {}
+
mlly@1.7.2:
dependencies:
acorn: 8.14.0
@@ -24251,11 +24308,20 @@ snapshots:
onnxruntime-common@1.14.0: {}
+ onnxruntime-common@1.19.2: {}
+
+ onnxruntime-common@1.20.0-dev.20241016-2b8fc5529b: {}
+
onnxruntime-node@1.14.0:
dependencies:
onnxruntime-common: 1.14.0
optional: true
+ onnxruntime-node@1.19.2:
+ dependencies:
+ onnxruntime-common: 1.19.2
+ tar: 7.4.3
+
onnxruntime-web@1.14.0:
dependencies:
flatbuffers: 1.12.0
@@ -24265,6 +24331,15 @@ snapshots:
onnxruntime-common: 1.14.0
platform: 1.3.6
+ onnxruntime-web@1.21.0-dev.20241024-d9ca84ef96:
+ dependencies:
+ flatbuffers: 1.12.0
+ guid-typescript: 1.0.9
+ long: 5.2.3
+ onnxruntime-common: 1.20.0-dev.20241016-2b8fc5529b
+ platform: 1.3.6
+ protobufjs: 7.2.6
+
open@8.4.2:
dependencies:
define-lazy-prop: 2.0.0
@@ -25827,6 +25902,10 @@ snapshots:
dependencies:
glob: 7.2.3
+ rimraf@5.0.10:
+ dependencies:
+ glob: 10.4.5
+
rimraf@6.0.1:
dependencies:
glob: 11.0.0
@@ -26131,7 +26210,6 @@ snapshots:
'@img/sharp-wasm32': 0.33.5
'@img/sharp-win32-ia32': 0.33.5
'@img/sharp-win32-x64': 0.33.5
- optional: true
shebang-command@1.2.0:
dependencies:
@@ -26667,6 +26745,15 @@ snapshots:
mkdirp: 1.0.4
yallist: 4.0.0
+ tar@7.4.3:
+ dependencies:
+ '@isaacs/fs-minipass': 4.0.1
+ chownr: 3.0.0
+ minipass: 7.1.2
+ minizlib: 3.0.1
+ mkdirp: 3.0.1
+ yallist: 5.0.0
+
term-size@2.2.1: {}
terser-webpack-plugin@5.3.10(@swc/core@1.7.42(@swc/helpers@0.5.13))(webpack@5.96.1(@swc/core@1.7.42(@swc/helpers@0.5.13))):
@@ -27813,6 +27900,8 @@ snapshots:
yallist@4.0.0: {}
+ yallist@5.0.0: {}
+
yaml@1.10.2: {}
yaml@2.5.0: {}
--
GitLab