From 446dc85bdd058edd6258ea6a70abca4facd34181 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Tue, 21 Nov 2023 13:42:40 +0700
Subject: [PATCH] fix: usage of transformers.js as CJS
---
apps/clip/{clip_test.js => clip_test.ts} | 4 ++--
apps/clip/package.json | 8 ++++++--
packages/core/src/Embedding.ts | 21 +++++++++++----------
pnpm-lock.yaml | 7 +++++++
4 files changed, 26 insertions(+), 14 deletions(-)
rename apps/clip/{clip_test.js => clip_test.ts} (94%)
diff --git a/apps/clip/clip_test.js b/apps/clip/clip_test.ts
similarity index 94%
rename from apps/clip/clip_test.js
rename to apps/clip/clip_test.ts
index 386eae08e..5f2da74a1 100644
--- a/apps/clip/clip_test.js
+++ b/apps/clip/clip_test.ts
@@ -19,12 +19,12 @@ async function main() {
const sim1 = similarity(
textEmbedding1,
imageEmbedding,
- SimilarityType.COSINE,
+ SimilarityType.DEFAULT,
);
const sim2 = similarity(
textEmbedding2,
imageEmbedding,
- SimilarityType.COSINE,
+ SimilarityType.DEFAULT,
);
console.log(`Similarity between "${text1}" and the image is ${sim1}`);
diff --git a/apps/clip/package.json b/apps/clip/package.json
index a4530e12d..99cec3903 100644
--- a/apps/clip/package.json
+++ b/apps/clip/package.json
@@ -2,12 +2,16 @@
"version": "0.0.1",
"private": true,
"name": "clip-test",
- "type": "module",
"dependencies": {
"dotenv": "^16.3.1",
"llamaindex": "workspace:*"
},
+ "devDependencies": {
+ "@types/node": "^18",
+ "ts-node": "^10.9.1"
+ },
"scripts": {
- "lint": "eslint ."
+ "lint": "eslint .",
+ "start": "ts-node ./clip_test.ts"
}
}
\ No newline at end of file
diff --git a/packages/core/src/Embedding.ts b/packages/core/src/Embedding.ts
index 78db4b937..ad6650251 100644
--- a/packages/core/src/Embedding.ts
+++ b/packages/core/src/Embedding.ts
@@ -1,13 +1,5 @@
-import { ClientOptions as OpenAIClientOptions } from "openai";
-
-import {
- AutoProcessor,
- AutoTokenizer,
- CLIPTextModelWithProjection,
- CLIPVisionModelWithProjection,
- RawImage,
-} from "@xenova/transformers";
import _ from "lodash";
+import { ClientOptions as OpenAIClientOptions } from "openai";
import { DEFAULT_SIMILARITY_TOP_K } from "./constants";
import {
AzureOpenAIConfig,
@@ -308,6 +300,7 @@ export class OpenAIEmbedding extends BaseEmbedding {
export type ImageType = string | Blob | URL;
async function readImage(input: ImageType) {
+ const { RawImage } = await import("@xenova/transformers");
if (input instanceof Blob) {
return await RawImage.fromBlob(input);
} else if (_.isString(input) || input instanceof URL) {
@@ -320,7 +313,7 @@ async function readImage(input: ImageType) {
/*
* Base class for Multi Modal embeddings.
*/
-abstract class MultiModalEmbedding extends BaseEmbedding {
+export abstract class MultiModalEmbedding extends BaseEmbedding {
abstract getImageEmbedding(images: ImageType): Promise<number[]>;
async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
@@ -347,6 +340,7 @@ export class ClipEmbedding extends MultiModalEmbedding {
async getTokenizer() {
if (!this.tokenizer) {
+ const { AutoTokenizer } = await import("@xenova/transformers");
this.tokenizer = await AutoTokenizer.from_pretrained(this.modelType);
}
return this.tokenizer;
@@ -354,6 +348,7 @@ export class ClipEmbedding extends MultiModalEmbedding {
async getProcessor() {
if (!this.processor) {
+ const { AutoProcessor } = await import("@xenova/transformers");
this.processor = await AutoProcessor.from_pretrained(this.modelType);
}
return this.processor;
@@ -361,6 +356,9 @@ export class ClipEmbedding extends MultiModalEmbedding {
async getVisionModel() {
if (!this.visionModel) {
+ const { CLIPVisionModelWithProjection } = await import(
+ "@xenova/transformers"
+ );
this.visionModel = await CLIPVisionModelWithProjection.from_pretrained(
this.modelType,
);
@@ -371,6 +369,9 @@ export class ClipEmbedding extends MultiModalEmbedding {
async getTextModel() {
if (!this.textModel) {
+ const { CLIPTextModelWithProjection } = await import(
+ "@xenova/transformers"
+ );
this.textModel = await CLIPTextModelWithProjection.from_pretrained(
this.modelType,
);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 95795125d..d556d651f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -57,6 +57,13 @@ importers:
llamaindex:
specifier: workspace:*
version: link:../../packages/core
+ devDependencies:
+ '@types/node':
+ specifier: ^18
+ version: 18.18.8
+ ts-node:
+ specifier: ^10.9.1
+ version: 10.9.1(@types/node@18.18.8)(typescript@5.2.2)
apps/docs:
dependencies:
--
GitLab