From 5c5f4c1c84c7a2acc51309d865ab81afbaae4747 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Mon, 1 Apr 2024 13:52:07 +0800
Subject: [PATCH] Revert "feat: support calculate llama 2 tokens (#676)"
This reverts commit 041acd11fe913593516ba4782684dc5f5b7b6f5f.
---
.changeset/khaki-books-type.md | 6 ------
packages/core/package.json | 5 ++---
packages/core/src/llm/LLM.ts | 4 +---
packages/edge/package.json | 5 ++---
pnpm-lock.yaml | 10 ----------
5 files changed, 5 insertions(+), 25 deletions(-)
delete mode 100644 .changeset/khaki-books-type.md
diff --git a/.changeset/khaki-books-type.md b/.changeset/khaki-books-type.md
deleted file mode 100644
index 4954897a9..000000000
--- a/.changeset/khaki-books-type.md
+++ /dev/null
@@ -1,6 +0,0 @@
----
-"llamaindex": patch
-"@llamaindex/edge": patch
----
-
-feat: support calculate Llama2 model tokens size
diff --git a/packages/core/package.json b/packages/core/package.json
index 7f83c8c69..1261452d1 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -25,7 +25,6 @@
"chromadb": "~1.7.3",
"cohere-ai": "^7.7.5",
"js-tiktoken": "^1.0.10",
- "llama-tokenizer-js": "^1.2.1",
"lodash": "^4.17.21",
"magic-bytes.js": "^1.10.0",
"mammoth": "^1.6.0",
@@ -42,8 +41,8 @@
"rake-modified": "^1.0.8",
"replicate": "^0.25.2",
"string-strip-html": "^13.4.6",
- "wikipedia": "^2.1.2",
- "wink-nlp": "^1.14.3"
+ "wink-nlp": "^1.14.3",
+ "wikipedia": "^2.1.2"
},
"devDependencies": {
"@swc/cli": "^0.3.9",
diff --git a/packages/core/src/llm/LLM.ts b/packages/core/src/llm/LLM.ts
index 2f6c69d5a..6970cab33 100644
--- a/packages/core/src/llm/LLM.ts
+++ b/packages/core/src/llm/LLM.ts
@@ -8,7 +8,6 @@ import {
type StreamCallbackResponse,
} from "../callbacks/CallbackManager.js";
-import llamaTokenizer from "llama-tokenizer-js";
import type { ChatCompletionMessageParam } from "openai/resources/index.js";
import type { LLMOptions } from "portkey-ai";
import { Tokenizers, globalsHelper } from "../GlobalsHelper.js";
@@ -416,8 +415,7 @@ export class LlamaDeuce extends BaseLLM {
}
tokens(messages: ChatMessage[]): number {
- return llamaTokenizer.encode(this.mapMessagesToPrompt(messages).prompt)
- .length;
+ throw new Error("Method not implemented.");
}
get metadata() {
diff --git a/packages/edge/package.json b/packages/edge/package.json
index afba58f29..8edf4ba89 100644
--- a/packages/edge/package.json
+++ b/packages/edge/package.json
@@ -24,7 +24,6 @@
"chromadb": "~1.7.3",
"cohere-ai": "^7.7.5",
"js-tiktoken": "^1.0.10",
- "llama-tokenizer-js": "^1.2.1",
"lodash": "^4.17.21",
"magic-bytes.js": "^1.10.0",
"mammoth": "^1.6.0",
@@ -41,8 +40,8 @@
"rake-modified": "^1.0.8",
"replicate": "^0.25.2",
"string-strip-html": "^13.4.6",
- "wikipedia": "^2.1.2",
- "wink-nlp": "^1.14.3"
+ "wink-nlp": "^1.14.3",
+ "wikipedia": "^2.1.2"
},
"engines": {
"node": ">=18.0.0"
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 9b6a7a666..4b69ec290 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -228,9 +228,6 @@ importers:
js-tiktoken:
specifier: ^1.0.10
version: 1.0.10
- llama-tokenizer-js:
- specifier: ^1.2.1
- version: 1.2.1
lodash:
specifier: ^4.17.21
version: 4.17.21
@@ -376,9 +373,6 @@ importers:
js-tiktoken:
specifier: ^1.0.10
version: 1.0.10
- llama-tokenizer-js:
- specifier: ^1.2.1
- version: 1.2.1
lodash:
specifier: ^4.17.21
version: 4.17.21
@@ -9809,10 +9803,6 @@ packages:
wrap-ansi: 9.0.0
dev: true
- /llama-tokenizer-js@1.2.1:
- resolution: {integrity: sha512-SEVVc++cXR0D0Wv30AzMVWzPCAKM701vZYU31h5lCTIn4k5cfZpJ070YDcb2nPq2Ts3xgu44L19wIrq1z/XjXQ==}
- dev: false
-
/load-yaml-file@0.2.0:
resolution: {integrity: sha512-OfCBkGEw4nN6JLtgRidPX6QxjBQGQf72q3si2uvqyFEMbycSFFHwAZeXx6cJgFM9wmLrf9zBwCP3Ivqa+LLZPw==}
engines: {node: '>=6'}
--
GitLab