From 22ff486fbe1ecf82ddf5da0376878e746e6664e0 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Wed, 28 Aug 2024 10:39:14 +0700
Subject: [PATCH] fix: Add tiktoken WASM to withLlamaIndex (#1134)
Co-authored-by: Thuc Pham <51660321+thucpn@users.noreply.github.com>
---
.changeset/late-ties-design.md | 5 +++++
apps/docs/docs/getting_started/environments.md | 13 ++++++++++---
packages/llamaindex/src/next.ts | 7 +++++++
3 files changed, 22 insertions(+), 3 deletions(-)
create mode 100644 .changeset/late-ties-design.md
diff --git a/.changeset/late-ties-design.md b/.changeset/late-ties-design.md
new file mode 100644
index 000000000..8a4894da5
--- /dev/null
+++ b/.changeset/late-ties-design.md
@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add tiktoken WASM to withLlamaIndex
diff --git a/apps/docs/docs/getting_started/environments.md b/apps/docs/docs/getting_started/environments.md
index b86c28b37..eac528375 100644
--- a/apps/docs/docs/getting_started/environments.md
+++ b/apps/docs/docs/getting_started/environments.md
@@ -6,10 +6,17 @@ sidebar_position: 2
We support Node.JS versions 18, 20 and 22, with experimental support for Deno, Bun and Vercel Edge functions.
-## NextJS App Router
+## NextJS
-If you're using NextJS App Router route handlers/serverless functions, you'll need to use the NodeJS mode:
+If you're using NextJS you'll need to add `withLlamaIndex` to your `next.config.js` file. This will add the necessary configuration for included 3rd-party libraries to your build:
```js
-export const runtime = "nodejs"; // default
+// next.config.js
+const withLlamaIndex = require("llamaindex/next");
+
+module.exports = withLlamaIndex({
+ // your next.js config
+});
```
+
+For details, check the latest [withLlamaIndex](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/llamaindex/src/next.ts) implementation.
diff --git a/packages/llamaindex/src/next.ts b/packages/llamaindex/src/next.ts
index 26d31735c..3ff8d6064 100644
--- a/packages/llamaindex/src/next.ts
+++ b/packages/llamaindex/src/next.ts
@@ -17,6 +17,13 @@
*/
export default function withLlamaIndex(config: any) {
config.experimental = config.experimental ?? {};
+ // copy tiktoken WASM files to the NextJS build
+ config.experimental.outputFileTracingIncludes =
+ config.experimental.outputFileTracingIncludes ?? {};
+ config.experimental.outputFileTracingIncludes["/**/*"] = [
+ "./node_modules/tiktoken/*.wasm",
+ ];
+ // needed for transformers, see https://huggingface.co/docs/transformers.js/en/tutorials/next#step-2-install-and-configure-transformersjs
config.experimental.serverComponentsExternalPackages =
config.experimental.serverComponentsExternalPackages ?? [];
config.experimental.serverComponentsExternalPackages.push(
--
GitLab