diff --git a/.changeset/warm-papayas-wait.md b/.changeset/warm-papayas-wait.md
new file mode 100644
index 0000000000000000000000000000000000000000..d8bd813b198d6bcdf047eaebcb8b4e33f45a04a9
--- /dev/null
+++ b/.changeset/warm-papayas-wait.md
@@ -0,0 +1,10 @@
+---
+"llamaindex": patch
+"@llamaindex/core-e2e": patch
+"@llamaindex/next-agent-test": patch
+"@llamaindex/nextjs-edge-runtime-test": patch
+---
+
+fix: import `@xenova/transformers`
+
+For now, if you use llamaindex in next.js, you need to add a plugin from `llamaindex/next` to ensure some module resolutions are correct.
diff --git a/README.md b/README.md
index b00b4ff832fc0c974305b8459e0dce07c3841490..f01298d843c6bd001128f048f77eac4901b532e4 100644
--- a/README.md
+++ b/README.md
@@ -78,6 +78,17 @@ node --import tsx ./main.ts
### Next.js
+First, you will need to add a llamaindex plugin to your Next.js project.
+
+```js
+// next.config.js
+const withLlamaIndex = require("llamaindex/next");
+
+module.exports = withLlamaIndex({
+ // your next.js config
+});
+```
+
You can combine `ai` with `llamaindex` in Next.js with RSC (React Server Components).
```tsx
diff --git a/packages/core/e2e/examples/nextjs-agent/next.config.mjs b/packages/core/e2e/examples/nextjs-agent/next.config.mjs
index 4678774e6d606704bce1897a5dab960cd798bf66..894e9a1dc436f268a91a2e602489580e24b432cc 100644
--- a/packages/core/e2e/examples/nextjs-agent/next.config.mjs
+++ b/packages/core/e2e/examples/nextjs-agent/next.config.mjs
@@ -1,4 +1,6 @@
/** @type {import('next').NextConfig} */
const nextConfig = {};
-export default nextConfig;
+import withLlamaIndex from "llamaindex/next";
+
+export default withLlamaIndex(nextConfig);
diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs b/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs
index 4678774e6d606704bce1897a5dab960cd798bf66..894e9a1dc436f268a91a2e602489580e24b432cc 100644
--- a/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs
+++ b/packages/core/e2e/examples/nextjs-edge-runtime/next.config.mjs
@@ -1,4 +1,6 @@
/** @type {import('next').NextConfig} */
const nextConfig = {};
-export default nextConfig;
+import withLlamaIndex from "llamaindex/next";
+
+export default withLlamaIndex(nextConfig);
diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.module.css b/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.module.css
deleted file mode 100644
index d979f776c714e3f28aad6b24c75cad808e79e60b..0000000000000000000000000000000000000000
--- a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.module.css
+++ /dev/null
@@ -1,232 +0,0 @@
-.main {
- display: flex;
- flex-direction: column;
- justify-content: space-between;
- align-items: center;
- padding: 6rem;
- min-height: 100vh;
-}
-
-.description {
- display: inherit;
- justify-content: inherit;
- align-items: inherit;
- font-size: 0.85rem;
- max-width: var(--max-width);
- width: 100%;
- z-index: 2;
- font-family: var(--font-mono);
-}
-
-.description a {
- display: flex;
- justify-content: center;
- align-items: center;
- gap: 0.5rem;
-}
-
-.description p {
- position: relative;
- margin: 0;
- padding: 1rem;
- background-color: rgba(var(--callout-rgb), 0.5);
- border: 1px solid rgba(var(--callout-border-rgb), 0.3);
- border-radius: var(--border-radius);
-}
-
-.code {
- font-weight: 700;
- font-family: var(--font-mono);
-}
-
-.grid {
- display: grid;
- grid-template-columns: repeat(4, minmax(25%, auto));
- max-width: 100%;
- width: var(--max-width);
-}
-
-.card {
- padding: 1rem 1.2rem;
- border-radius: var(--border-radius);
- background: rgba(var(--card-rgb), 0);
- border: 1px solid rgba(var(--card-border-rgb), 0);
- transition:
- background 200ms,
- border 200ms;
-}
-
-.card span {
- display: inline-block;
- transition: transform 200ms;
-}
-
-.card h2 {
- font-weight: 600;
- margin-bottom: 0.7rem;
-}
-
-.card p {
- margin: 0;
- opacity: 0.6;
- font-size: 0.9rem;
- line-height: 1.5;
- max-width: 30ch;
- text-wrap: balance;
-}
-
-.center {
- display: flex;
- justify-content: center;
- align-items: center;
- position: relative;
- padding: 4rem 0;
-}
-
-.center::before {
- background: var(--secondary-glow);
- border-radius: 50%;
- width: 480px;
- height: 360px;
- margin-left: -400px;
-}
-
-.center::after {
- background: var(--primary-glow);
- width: 240px;
- height: 180px;
- z-index: -1;
-}
-
-.center::before,
-.center::after {
- content: "";
- left: 50%;
- position: absolute;
- filter: blur(45px);
- transform: translateZ(0);
-}
-
-.logo {
- position: relative;
-}
-/* Enable hover only on non-touch devices */
-@media (hover: hover) and (pointer: fine) {
- .card:hover {
- background: rgba(var(--card-rgb), 0.1);
- border: 1px solid rgba(var(--card-border-rgb), 0.15);
- }
-
- .card:hover span {
- transform: translateX(4px);
- }
-}
-
-@media (prefers-reduced-motion) {
- .card:hover span {
- transform: none;
- }
-}
-
-/* Mobile */
-@media (max-width: 700px) {
- .content {
- padding: 4rem;
- }
-
- .grid {
- grid-template-columns: 1fr;
- margin-bottom: 120px;
- max-width: 320px;
- text-align: center;
- }
-
- .card {
- padding: 1rem 2.5rem;
- }
-
- .card h2 {
- margin-bottom: 0.5rem;
- }
-
- .center {
- padding: 8rem 0 6rem;
- }
-
- .center::before {
- transform: none;
- height: 300px;
- }
-
- .description {
- font-size: 0.8rem;
- }
-
- .description a {
- padding: 1rem;
- }
-
- .description p,
- .description div {
- display: flex;
- justify-content: center;
- position: fixed;
- width: 100%;
- }
-
- .description p {
- align-items: center;
- inset: 0 0 auto;
- padding: 2rem 1rem 1.4rem;
- border-radius: 0;
- border: none;
- border-bottom: 1px solid rgba(var(--callout-border-rgb), 0.25);
- background: linear-gradient(
- to bottom,
- rgba(var(--background-start-rgb), 1),
- rgba(var(--callout-rgb), 0.5)
- );
- background-clip: padding-box;
- backdrop-filter: blur(24px);
- }
-
- .description div {
- align-items: flex-end;
- pointer-events: none;
- inset: auto 0 0;
- padding: 2rem;
- height: 200px;
- background: linear-gradient(
- to bottom,
- transparent 0%,
- rgb(var(--background-end-rgb)) 40%
- );
- z-index: 1;
- }
-}
-
-/* Tablet and Smaller Desktop */
-@media (min-width: 701px) and (max-width: 1120px) {
- .grid {
- grid-template-columns: repeat(2, 50%);
- }
-}
-
-@media (prefers-color-scheme: dark) {
- .vercelLogo {
- filter: invert(1);
- }
-
- .logo {
- filter: invert(1) drop-shadow(0 0 0.3rem #ffffff70);
- }
-}
-
-@keyframes rotate {
- from {
- transform: rotate(360deg);
- }
- to {
- transform: rotate(0deg);
- }
-}
diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx b/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx
index 264289621114b7734d30342f46d1f1e3d76439f8..6765b53440d3c90837ed3a26a85c3de47a1ac70a 100644
--- a/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx
+++ b/packages/core/e2e/examples/nextjs-edge-runtime/src/app/page.tsx
@@ -1,98 +1,20 @@
-import Image from "next/image";
-import "../utils/llm";
-import styles from "./page.module.css";
+import { tokenizerResultPromise } from "@/utils/llm";
+import { use } from "react";
export const runtime = "edge";
export default function Home() {
+ const result = use(tokenizerResultPromise);
return (
- <main className={styles.main}>
- <div className={styles.description}>
- <p>
- Get started by editing
- <code className={styles.code}>src/app/page.tsx</code>
- </p>
+ <main>
+ <div>
+ <h1>Next.js Edge Runtime</h1>
<div>
- <a
- href="https://vercel.com?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
- target="_blank"
- rel="noopener noreferrer"
- >
- By{" "}
- <Image
- src="/vercel.svg"
- alt="Vercel Logo"
- className={styles.vercelLogo}
- width={100}
- height={24}
- priority
- />
- </a>
+ {result.map((value, index) => (
+ <span key={index}>{value}</span>
+ ))}
</div>
</div>
-
- <div className={styles.center}>
- <Image
- className={styles.logo}
- src="/next.svg"
- alt="Next.js Logo"
- width={180}
- height={37}
- priority
- />
- </div>
-
- <div className={styles.grid}>
- <a
- href="https://nextjs.org/docs?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
- className={styles.card}
- target="_blank"
- rel="noopener noreferrer"
- >
- <h2>
- Docs <span>-></span>
- </h2>
- <p>Find in-depth information about Next.js features and API.</p>
- </a>
-
- <a
- href="https://nextjs.org/learn?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
- className={styles.card}
- target="_blank"
- rel="noopener noreferrer"
- >
- <h2>
- Learn <span>-></span>
- </h2>
- <p>Learn about Next.js in an interactive course with quizzes!</p>
- </a>
-
- <a
- href="https://vercel.com/templates?framework=next.js&utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
- className={styles.card}
- target="_blank"
- rel="noopener noreferrer"
- >
- <h2>
- Templates <span>-></span>
- </h2>
- <p>Explore starter templates for Next.js.</p>
- </a>
-
- <a
- href="https://vercel.com/new?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
- className={styles.card}
- target="_blank"
- rel="noopener noreferrer"
- >
- <h2>
- Deploy <span>-></span>
- </h2>
- <p>
- Instantly deploy your Next.js site to a shareable URL with Vercel.
- </p>
- </a>
- </div>
</main>
);
}
diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts b/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts
index 29635d9c6872a2990f574727512d440c3cbf9b48..0396636617b20cb64a6e360c4da748d76599733b 100644
--- a/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts
+++ b/packages/core/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts
@@ -1,9 +1,23 @@
-"use server";
// test runtime
import "llamaindex";
+import { ClipEmbedding } from "llamaindex/embeddings/ClipEmbedding";
import "llamaindex/readers/SimpleDirectoryReader";
// @ts-expect-error
if (typeof EdgeRuntime !== "string") {
throw new Error("Expected run in EdgeRuntime");
}
+
+export const tokenizerResultPromise = new Promise<number[]>(
+ (resolve, reject) => {
+ const embedding = new ClipEmbedding();
+ //#region make sure @xenova/transformers is working in edge runtime
+ embedding
+ .getTokenizer()
+ .then((tokenizer) => {
+ resolve(tokenizer.encode("hello world"));
+ })
+ .catch(reject);
+ //#endregion
+ },
+);
diff --git a/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json b/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json
index b5e6c4d437da8b5dc65a08225d6b485d1d045c78..0f4b49ede388d540f2f140be228ac4cb02e19517 100644
--- a/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json
+++ b/packages/core/e2e/examples/nextjs-edge-runtime/tsconfig.json
@@ -1,5 +1,6 @@
{
"compilerOptions": {
+ "target": "ESNext",
"lib": ["dom", "dom.iterable", "esnext"],
"outDir": "./dist",
"allowJs": true,
diff --git a/packages/core/src/embeddings/ClipEmbedding.ts b/packages/core/src/embeddings/ClipEmbedding.ts
index 5cd6f1bc39abd1b892d084fd4009180141cffbfc..f4f1082916320c96ac8ec0599fb40174ce864f4e 100644
--- a/packages/core/src/embeddings/ClipEmbedding.ts
+++ b/packages/core/src/embeddings/ClipEmbedding.ts
@@ -1,12 +1,17 @@
import _ from "lodash";
import type { ImageType } from "../Node.js";
+import { lazyLoadTransformers } from "../internal/deps/transformers.js";
import { MultiModalEmbedding } from "./MultiModalEmbedding.js";
+// only import type, to avoid bundling error
+import type {
+ CLIPTextModelWithProjection,
+ CLIPVisionModelWithProjection,
+ PreTrainedTokenizer,
+ Processor,
+} from "@xenova/transformers";
async function readImage(input: ImageType) {
- const { RawImage } = await import(
- /* webpackIgnore: true */
- "@xenova/transformers"
- );
+ const { RawImage } = await lazyLoadTransformers();
if (input instanceof Blob) {
return await RawImage.fromBlob(input);
} else if (_.isString(input) || input instanceof URL) {
@@ -25,39 +30,30 @@ export class ClipEmbedding extends MultiModalEmbedding {
modelType: ClipEmbeddingModelType =
ClipEmbeddingModelType.XENOVA_CLIP_VIT_BASE_PATCH16;
- private tokenizer: any;
- private processor: any;
- private visionModel: any;
- private textModel: any;
+ private tokenizer: PreTrainedTokenizer | null = null;
+ private processor: Processor | null = null;
+ private visionModel: CLIPVisionModelWithProjection | null = null;
+ private textModel: CLIPTextModelWithProjection | null = null;
async getTokenizer() {
+ const { AutoTokenizer } = await lazyLoadTransformers();
if (!this.tokenizer) {
- const { AutoTokenizer } = await import(
- /* webpackIgnore: true */
- "@xenova/transformers"
- );
this.tokenizer = await AutoTokenizer.from_pretrained(this.modelType);
}
return this.tokenizer;
}
async getProcessor() {
+ const { AutoProcessor } = await lazyLoadTransformers();
if (!this.processor) {
- const { AutoProcessor } = await import(
- /* webpackIgnore: true */
- "@xenova/transformers"
- );
this.processor = await AutoProcessor.from_pretrained(this.modelType);
}
return this.processor;
}
async getVisionModel() {
+ const { CLIPVisionModelWithProjection } = await lazyLoadTransformers();
if (!this.visionModel) {
- const { CLIPVisionModelWithProjection } = await import(
- /* webpackIgnore: true */
- "@xenova/transformers"
- );
this.visionModel = await CLIPVisionModelWithProjection.from_pretrained(
this.modelType,
);
@@ -67,11 +63,8 @@ export class ClipEmbedding extends MultiModalEmbedding {
}
async getTextModel() {
+ const { CLIPTextModelWithProjection } = await lazyLoadTransformers();
if (!this.textModel) {
- const { CLIPTextModelWithProjection } = await import(
- /* webpackIgnore: true */
- "@xenova/transformers"
- );
this.textModel = await CLIPTextModelWithProjection.from_pretrained(
this.modelType,
);
diff --git a/packages/core/src/embeddings/HuggingFaceEmbedding.ts b/packages/core/src/embeddings/HuggingFaceEmbedding.ts
index 73e6cb2e374ef563d063e9499b4656b06e3fcf18..d889aaeeb1d62ad3f6713d12fc0f79fafaa375cd 100644
--- a/packages/core/src/embeddings/HuggingFaceEmbedding.ts
+++ b/packages/core/src/embeddings/HuggingFaceEmbedding.ts
@@ -1,3 +1,4 @@
+import { lazyLoadTransformers } from "../internal/deps/transformers.js";
import { BaseEmbedding } from "./types.js";
export enum HuggingFaceEmbeddingModelType {
@@ -31,7 +32,7 @@ export class HuggingFaceEmbedding extends BaseEmbedding {
async getExtractor() {
if (!this.extractor) {
- const { pipeline } = await import("@xenova/transformers");
+ const { pipeline } = await lazyLoadTransformers();
this.extractor = await pipeline("feature-extraction", this.modelType, {
quantized: this.quantized,
});
diff --git a/packages/core/src/internal/deps/transformers.ts b/packages/core/src/internal/deps/transformers.ts
new file mode 100644
index 0000000000000000000000000000000000000000..96ebceaa47322e79de22fc4f8e6ccac7fa9c448e
--- /dev/null
+++ b/packages/core/src/internal/deps/transformers.ts
@@ -0,0 +1,15 @@
+let transformer: typeof import("@xenova/transformers") | null = null;
+
+export async function lazyLoadTransformers() {
+ if (!transformer) {
+ transformer = await import("@xenova/transformers");
+ }
+
+ // @ts-expect-error
+ if (typeof EdgeRuntime === "string") {
+ // there is no local file system in the edge runtime
+ transformer.env.allowLocalModels = false;
+ }
+ // fixme: handle cloudflare workers case here?
+ return transformer;
+}
diff --git a/packages/core/src/next.ts b/packages/core/src/next.ts
new file mode 100644
index 0000000000000000000000000000000000000000..8097a0ea73824e6aa8bf4fc1c95d15cda0ffa6ba
--- /dev/null
+++ b/packages/core/src/next.ts
@@ -0,0 +1,36 @@
+/**
+ * This is a Next.js configuration file that is used to customize the build process.
+ *
+ * @example
+ * ```js
+ * // next.config.js
+ * const withLlamaIndex = require("llamaindex/next")
+ *
+ * module.exports = withLlamaIndex({
+ * // Your Next.js configuration
+ * })
+ * ```
+ *
+ * This is only for Next.js projects, do not export this function on top-level.
+ *
+ * @module
+ */
+export default function withLlamaIndex(config: any) {
+ const userWebpack = config.webpack;
+ //#region hack for `@xenova/transformers`
+ // Ignore node-specific modules when bundling for the browser
+ // See https://webpack.js.org/configuration/resolve/#resolvealias
+ config.webpack = function (webpackConfig: any) {
+ if (userWebpack) {
+ webpackConfig = userWebpack(webpackConfig);
+ }
+ webpackConfig.resolve.alias = {
+ ...webpackConfig.resolve.alias,
+ sharp$: false,
+ "onnxruntime-node$": false,
+ };
+ return webpackConfig;
+ };
+ //#endregion
+ return config;
+}