diff --git a/.changeset/khaki-books-type.md b/.changeset/khaki-books-type.md
deleted file mode 100644
index 4954897a979ae90f03013b8e4da2b2df6a67f99a..0000000000000000000000000000000000000000
--- a/.changeset/khaki-books-type.md
+++ /dev/null
@@ -1,6 +0,0 @@
----
-"llamaindex": patch
-"@llamaindex/edge": patch
----
-
-feat: support calculate Llama2 model tokens size
diff --git a/packages/core/package.json b/packages/core/package.json
index 7f83c8c6972edc791aa544b55c3f5a249ac1664c..1261452d19cd25953882203804cea0e5288f79a7 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -25,7 +25,6 @@
"chromadb": "~1.7.3",
"cohere-ai": "^7.7.5",
"js-tiktoken": "^1.0.10",
- "llama-tokenizer-js": "^1.2.1",
"lodash": "^4.17.21",
"magic-bytes.js": "^1.10.0",
"mammoth": "^1.6.0",
@@ -42,8 +41,8 @@
"rake-modified": "^1.0.8",
"replicate": "^0.25.2",
"string-strip-html": "^13.4.6",
- "wikipedia": "^2.1.2",
- "wink-nlp": "^1.14.3"
+ "wink-nlp": "^1.14.3",
+ "wikipedia": "^2.1.2"
},
"devDependencies": {
"@swc/cli": "^0.3.9",
diff --git a/packages/core/src/llm/LLM.ts b/packages/core/src/llm/LLM.ts
index 2f6c69d5a3712440a2efd4f70e9de9eee2d4ac42..6970cab3319455dc4b47fc0ec2007587a5c7f98d 100644
--- a/packages/core/src/llm/LLM.ts
+++ b/packages/core/src/llm/LLM.ts
@@ -8,7 +8,6 @@ import {
type StreamCallbackResponse,
} from "../callbacks/CallbackManager.js";
-import llamaTokenizer from "llama-tokenizer-js";
import type { ChatCompletionMessageParam } from "openai/resources/index.js";
import type { LLMOptions } from "portkey-ai";
import { Tokenizers, globalsHelper } from "../GlobalsHelper.js";
@@ -416,8 +415,7 @@ export class LlamaDeuce extends BaseLLM {
}
tokens(messages: ChatMessage[]): number {
- return llamaTokenizer.encode(this.mapMessagesToPrompt(messages).prompt)
- .length;
+ throw new Error("Method not implemented.");
}
get metadata() {
diff --git a/packages/edge/package.json b/packages/edge/package.json
index afba58f2912428e534e1975f4d970915d0f40aa7..8edf4ba89dc38a100a7b5b448b632a68ea458de7 100644
--- a/packages/edge/package.json
+++ b/packages/edge/package.json
@@ -24,7 +24,6 @@
"chromadb": "~1.7.3",
"cohere-ai": "^7.7.5",
"js-tiktoken": "^1.0.10",
- "llama-tokenizer-js": "^1.2.1",
"lodash": "^4.17.21",
"magic-bytes.js": "^1.10.0",
"mammoth": "^1.6.0",
@@ -41,8 +40,8 @@
"rake-modified": "^1.0.8",
"replicate": "^0.25.2",
"string-strip-html": "^13.4.6",
- "wikipedia": "^2.1.2",
- "wink-nlp": "^1.14.3"
+ "wink-nlp": "^1.14.3",
+ "wikipedia": "^2.1.2"
},
"engines": {
"node": ">=18.0.0"
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 9b6a7a6669aaf3f081566d9e7eaeacf4b720a06e..4b69ec290f33e90ad56049f36ee9b96cb330bed9 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -228,9 +228,6 @@ importers:
js-tiktoken:
specifier: ^1.0.10
version: 1.0.10
- llama-tokenizer-js:
- specifier: ^1.2.1
- version: 1.2.1
lodash:
specifier: ^4.17.21
version: 4.17.21
@@ -376,9 +373,6 @@ importers:
js-tiktoken:
specifier: ^1.0.10
version: 1.0.10
- llama-tokenizer-js:
- specifier: ^1.2.1
- version: 1.2.1
lodash:
specifier: ^4.17.21
version: 4.17.21
@@ -9809,10 +9803,6 @@ packages:
wrap-ansi: 9.0.0
dev: true
- /llama-tokenizer-js@1.2.1:
- resolution: {integrity: sha512-SEVVc++cXR0D0Wv30AzMVWzPCAKM701vZYU31h5lCTIn4k5cfZpJ070YDcb2nPq2Ts3xgu44L19wIrq1z/XjXQ==}
- dev: false
-
/load-yaml-file@0.2.0:
resolution: {integrity: sha512-OfCBkGEw4nN6JLtgRidPX6QxjBQGQf72q3si2uvqyFEMbycSFFHwAZeXx6cJgFM9wmLrf9zBwCP3Ivqa+LLZPw==}
engines: {node: '>=6'}