From d2b2722a30cfaebf06699cb154206ddc4e534f6f Mon Sep 17 00:00:00 2001
From: Alex Yang <himself65@outlook.com>
Date: Tue, 26 Nov 2024 15:42:20 -0800
Subject: [PATCH] fix: switch tokenizer in cloudflare (#1525)
---
.changeset/loud-gifts-wink.md | 5 +++++
packages/env/src/tokenizers.workerd.ts | 2 +-
2 files changed, 6 insertions(+), 1 deletion(-)
create mode 100644 .changeset/loud-gifts-wink.md
diff --git a/.changeset/loud-gifts-wink.md b/.changeset/loud-gifts-wink.md
new file mode 100644
index 000000000..e6218583c
--- /dev/null
+++ b/.changeset/loud-gifts-wink.md
@@ -0,0 +1,5 @@
+---
+"@llamaindex/env": patch
+---
+
+fix: switch tokenizer in cloudflare
diff --git a/packages/env/src/tokenizers.workerd.ts b/packages/env/src/tokenizers.workerd.ts
index de80cda92..a1ac4a758 100644
--- a/packages/env/src/tokenizers.workerd.ts
+++ b/packages/env/src/tokenizers.workerd.ts
@@ -2,4 +2,4 @@ export {
Tokenizers,
tokenizers,
type Tokenizer,
-} from "./internal/tokenizers/js.js";
+} from "./internal/tokenizers/node.js";
--
GitLab