fix: use tiktoken instead of tiktoken/lite (#967)

.changeset/odd-insects-vanish.md

@@ -2,4 +2,4 @@
 "@llamaindex/env": patch
 ---
 
-Fix TypeError: Module "..." needs an import attribute of "type: json"
+Use tiktoken instead of tiktoken/lite and disable WASM tiktoken for non-Node environments

packages/env/src/index.workerd.ts

@@ -13,4 +13,4 @@ export function getEnv(name: string): string | undefined {
   return INTERNAL_ENV[name];
 }
 
-export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
+export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/tokenizers/node.ts

 // Note: This is using th WASM implementation of tiktoken which is 60x faster
-import cl100k_base from "tiktoken/encoders/cl100k_base.json" with { type: "json" };
-import { Tiktoken } from "tiktoken/lite";
 import type { Tokenizer } from "./types.js";
 import { Tokenizers } from "./types.js";
 
+import { get_encoding } from "tiktoken";
+
 class TokenizerSingleton {
   private defaultTokenizer: Tokenizer;
 
   constructor() {
-    const encoding = new Tiktoken(
-      cl100k_base.bpe_ranks,
-      cl100k_base.special_tokens,
-      cl100k_base.pat_str,
-    );
+    const encoding = get_encoding("cl100k_base");
 
     this.defaultTokenizer = {
       encode: (text: string) => {