diff --git a/apps/next/package.json b/apps/next/package.json
index 6a37184fb19d87d1ac79042d15643c49f8485577..917c05f604f38395ddb38e822e648ca50bd53134 100644
--- a/apps/next/package.json
+++ b/apps/next/package.json
@@ -7,7 +7,7 @@
"dev": "next dev",
"start": "next start",
"postdev": "fumadocs-mdx",
- "postbuild": "fumadocs-mdx",
+ "postbuild": "fumadocs-mdx && tsx scripts/post-build.mts",
"build:docs": "node ./scripts/generate-docs.mjs"
},
"dependencies": {
@@ -53,13 +53,21 @@
"zod": "^3.23.8"
},
"devDependencies": {
+ "@next/env": "^15.0.1",
"@types/mdx": "^2.0.13",
"@types/node": "22.7.8",
"@types/react": "^18.3.12",
"@types/react-dom": "^18.3.1",
"autoprefixer": "^10.4.20",
+ "fast-glob": "^3.3.2",
+ "gray-matter": "^4.0.3",
"postcss": "^8.4.47",
+ "remark": "^15.0.1",
+ "remark-gfm": "^4.0.0",
+ "remark-mdx": "^3.1.0",
+ "remark-stringify": "^11.0.0",
"tailwindcss": "^3.4.14",
+ "tsx": "^4.19.0",
"typescript": "^5.6.3"
}
}
diff --git a/apps/next/scripts/post-build.mts b/apps/next/scripts/post-build.mts
new file mode 100644
index 0000000000000000000000000000000000000000..72367ee02b4d003523e4e43e366439d7437029ac
--- /dev/null
+++ b/apps/next/scripts/post-build.mts
@@ -0,0 +1,7 @@
+import env from "@next/env";
+
+import { updateLlamaCloud } from "./update-llamacloud.mjs";
+
+env.loadEnvConfig(process.cwd());
+
+await updateLlamaCloud();
diff --git a/apps/next/scripts/update-llamacloud.mts b/apps/next/scripts/update-llamacloud.mts
new file mode 100644
index 0000000000000000000000000000000000000000..9c995c32f9b3bfb6cf0cf13cbe0115d6193c9e1b
--- /dev/null
+++ b/apps/next/scripts/update-llamacloud.mts
@@ -0,0 +1,110 @@
+import { PipelinesService } from "@llamaindex/cloud/api";
+import fg from "fast-glob";
+import {
+ fileGenerator,
+ remarkDocGen,
+ remarkInstall,
+ typescriptGenerator,
+} from "fumadocs-docgen";
+import matter from "gray-matter";
+import * as fs from "node:fs/promises";
+import path, { relative } from "node:path";
+import { fileURLToPath } from "node:url";
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import remarkMdx from "remark-mdx";
+import remarkStringify from "remark-stringify";
+
+const baseDir = fileURLToPath(new URL("../src/content", import.meta.url));
+
+async function processContent(content: string): Promise<string> {
+ const file = await remark()
+ .use(remarkMdx)
+ .use(remarkGfm)
+ .use(remarkDocGen, { generators: [typescriptGenerator(), fileGenerator()] })
+ .use(remarkInstall, { persist: { id: "package-manager" } })
+ .use(remarkStringify)
+ .process(content);
+
+ return String(file);
+}
+
+export async function updateLlamaCloud(): Promise<void> {
+ // eslint-disable-next-line turbo/no-undeclared-env-vars
+ const apiKey = process.env.LLAMA_CLOUD_API_KEY;
+ // eslint-disable-next-line turbo/no-undeclared-env-vars
+ const index = process.env.LLAMA_CLOUD_PIPELINE_ID;
+
+ if (!apiKey || !index) {
+ console.log("no api key for LlamaCloud found, skipping");
+ return;
+ }
+
+ const files = await fg([
+ "./src/content/docs/**/*.mdx",
+ "!./src/content/docs/cloud/api/**/*",
+ ]);
+
+ const records: {
+ id: string;
+ title: string;
+ description: string;
+ content: string;
+ category: string | undefined;
+ }[] = [];
+
+ console.log("processing documents for AI");
+ const scan = files.map(async (file) => {
+ const fileContent = await fs.readFile(file);
+ const { content, data } = matter(fileContent.toString());
+
+ const dir = path.dirname(file).split(path.sep).at(3);
+ const category = {
+ cloud: "LlamaCloud",
+ llamaindex: "LlamaIndex.TS",
+ }[dir ?? ""];
+
+ if (data._mdx?.mirror) {
+ return;
+ }
+
+ const processed = await processContent(content);
+ const id = relative(baseDir, file);
+ records.push({
+ id,
+ title: data.title as string,
+ description: data.description as string,
+ content: processed,
+ category,
+ });
+ });
+
+ await Promise.all(scan);
+
+ console.log(`added ${records.length} records`);
+
+ await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
+ {
+ baseUrl: "https://api.cloud.llamaindex.ai/",
+ body: records.map((record) => ({
+ id: record.id,
+ metadata: {
+ title: record.title,
+ description: record.description,
+ documentUrl: record.id,
+ category: record.category,
+ },
+ text: record.content,
+ })),
+ path: {
+ pipeline_id: index,
+ },
+ throwOnError: true,
+ headers: {
+ Authorization: `Bearer ${apiKey}`,
+ },
+ },
+ );
+
+ console.log("done");
+}
diff --git a/apps/next/source.config.ts b/apps/next/source.config.ts
index 346eb3c7aee3dc4aad154691ec1e4cb1ea83e029..b03a6bac6ef3e80da7dcf001f2ed92f1e4fbbc7d 100644
--- a/apps/next/source.config.ts
+++ b/apps/next/source.config.ts
@@ -1,15 +1,10 @@
-import { PipelinesService } from "@llamaindex/cloud/api";
import { rehypeCodeDefaultOptions } from "fumadocs-core/mdx-plugins";
import { fileGenerator, remarkDocGen, remarkInstall } from "fumadocs-docgen";
import { defineConfig, defineDocs } from "fumadocs-mdx/config";
import { transformerTwoslash } from "fumadocs-twoslash";
-import { relative } from "node:path";
-import { fileURLToPath } from "node:url";
import rehypeKatex from "rehype-katex";
import remarkMath from "remark-math";
-const baseDir = fileURLToPath(new URL("../src/content", import.meta.url));
-
export const { docs, meta } = defineDocs({
dir: "./src/content/docs",
});
@@ -49,68 +44,6 @@ export default defineConfig({
remarkMath,
[remarkInstall, { persist: { id: "package-manager" } }],
[remarkDocGen, { generators: [fileGenerator()] }],
- () => {
- return (_, file, next) => {
- const metadata = file.data.frontmatter as Record<string, unknown>;
- const title = metadata.title as string;
- const description = metadata.description as string;
- let content: string;
- if (file.value instanceof Uint8Array) {
- content = file.value.toString();
- } else {
- content = file.value;
- }
- if (file.path.includes("content/docs/cloud/api")) {
- // skip cloud api docs
- return next();
- }
- // eslint-disable-next-line turbo/no-undeclared-env-vars
- if (process.env.NODE_ENV === "development") {
- // skip development
- return next();
- }
- if (!title || !description) {
- throw new Error(`Missing title or description in ${file.path}`);
- }
- const id = relative(baseDir, file.path);
-
- if (
- // eslint-disable-next-line turbo/no-undeclared-env-vars
- process.env.LLAMA_CLOUD_UPSERT_PIPELINE_DOCUMENTS === "true" &&
- // eslint-disable-next-line turbo/no-undeclared-env-vars
- process.env.LLAMA_CLOUD_PIPELINE_ID !== undefined
- ) {
- PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
- {
- baseUrl: "https://api.cloud.llamaindex.ai/",
- body: [
- {
- metadata: {
- title,
- description,
- documentUrl: id,
- },
- text: content,
- id,
- },
- ],
- path: {
- // eslint-disable-next-line turbo/no-undeclared-env-vars
- pipeline_id: process.env.LLAMA_CLOUD_PIPELINE_ID,
- },
- throwOnError: true,
- headers: {
- // eslint-disable-next-line turbo/no-undeclared-env-vars
- Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
- },
- },
- ).catch((error) => {
- console.error(error);
- });
- }
- return next();
- };
- },
],
rehypePlugins: (v) => [rehypeKatex, ...v],
},
diff --git a/apps/next/tsconfig.json b/apps/next/tsconfig.json
index 4b48dac84795dad219da6474ddc49fd8c818d4ed..2ba48c5759c413a96f4bd8d24fd4add5751c28d9 100644
--- a/apps/next/tsconfig.json
+++ b/apps/next/tsconfig.json
@@ -24,6 +24,12 @@
}
]
},
- "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+ "include": [
+ "next-env.d.ts",
+ "**/*.ts",
+ "**/*.tsx",
+ ".next/types/**/*.ts",
+ "**/*.mts"
+ ],
"exclude": ["node_modules"]
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index af5535d9b987796f72eb3cbe8e1d40efc095d0f0..a2dcb075081977f0c088756f586375f27c328b0d 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -245,6 +245,9 @@ importers:
specifier: ^3.23.8
version: 3.23.8
devDependencies:
+ '@next/env':
+ specifier: ^15.0.1
+ version: 15.0.1
'@types/mdx':
specifier: ^2.0.13
version: 2.0.13
@@ -260,12 +263,33 @@ importers:
autoprefixer:
specifier: ^10.4.20
version: 10.4.20(postcss@8.4.47)
+ fast-glob:
+ specifier: ^3.3.2
+ version: 3.3.2
+ gray-matter:
+ specifier: ^4.0.3
+ version: 4.0.3
postcss:
specifier: ^8.4.47
version: 8.4.47
+ remark:
+ specifier: ^15.0.1
+ version: 15.0.1
+ remark-gfm:
+ specifier: ^4.0.0
+ version: 4.0.0
+ remark-mdx:
+ specifier: ^3.1.0
+ version: 3.1.0
+ remark-stringify:
+ specifier: ^11.0.0
+ version: 11.0.0
tailwindcss:
specifier: ^3.4.14
version: 3.4.14
+ tsx:
+ specifier: ^4.19.0
+ version: 4.19.0
typescript:
specifier: ^5.6.3
version: 5.6.3
@@ -11489,8 +11513,8 @@ packages:
remark-math@6.0.0:
resolution: {integrity: sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA==}
- remark-mdx@3.0.1:
- resolution: {integrity: sha512-3Pz3yPQ5Rht2pM5R+0J2MrGoBSrzf+tJG94N+t/ilfdh8YLyyKYtidAYwTveB20BoHAcwIopOUqhcmh2F7hGYA==}
+ remark-mdx@3.1.0:
+ resolution: {integrity: sha512-Ngl/H3YXyBV9RcRNdlYsZujAmhsxwzxpDzpDEhFBVAGthS4GDgnctpDjgFl/ULx5UEDzqtW1cyBSNKqYYrqLBA==}
remark-parse@11.0.0:
resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==}
@@ -17878,7 +17902,7 @@ snapshots:
recma-jsx: 1.0.0(acorn@8.13.0)
recma-stringify: 1.0.0
rehype-recma: 1.0.0
- remark-mdx: 3.0.1
+ remark-mdx: 3.1.0
remark-parse: 11.0.0
remark-rehype: 11.1.1
source-map: 0.7.4
@@ -22111,7 +22135,7 @@ snapshots:
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.3)
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
- eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0)
+ eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0)
eslint-plugin-jsx-a11y: 6.9.0(eslint@8.57.0)
eslint-plugin-react: 7.35.0(eslint@8.57.0)
@@ -22140,13 +22164,13 @@ snapshots:
transitivePeerDependencies:
- supports-color
- eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0):
+ eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
dependencies:
'@nolyfill/is-core-module': 1.0.39
debug: 4.3.7
enhanced-resolve: 5.17.1
eslint: 8.57.0
- eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+ eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
fast-glob: 3.3.2
get-tsconfig: 4.8.1
is-bun-module: 1.1.0
@@ -22159,14 +22183,14 @@ snapshots:
- eslint-import-resolver-webpack
- supports-color
- eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
+ eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
dependencies:
debug: 3.2.7
optionalDependencies:
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.3)
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
- eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0)
+ eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
transitivePeerDependencies:
- supports-color
@@ -22180,7 +22204,7 @@ snapshots:
doctrine: 2.1.0
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
- eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+ eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
hasown: 2.0.2
is-core-module: 2.15.1
is-glob: 4.0.3
@@ -26986,7 +27010,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
- remark-mdx@3.0.1:
+ remark-mdx@3.1.0:
dependencies:
mdast-util-mdx: 3.0.0
micromark-extension-mdxjs: 3.0.0