Skip to content
Snippets Groups Projects
Unverified Commit 0fd78d54 authored by Alex Yang's avatar Alex Yang Committed by GitHub
Browse files

docs(next): fix update llamacloud (#1409)

parent 5c4f4a8d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
apps/next/package.json
+ 9
− 1
View file @ 0fd78d54
......@@ -7,7 +7,7 @@
"dev": "next dev",
"start": "next start",
"postdev": "fumadocs-mdx",
"postbuild": "fumadocs-mdx",
"postbuild": "fumadocs-mdx && tsx scripts/post-build.mts",
"build:docs": "node ./scripts/generate-docs.mjs"
},
"dependencies": {
......@@ -53,13 +53,21 @@
"zod": "^3.23.8"
},
"devDependencies": {
"@next/env": "^15.0.1",
"@types/mdx": "^2.0.13",
"@types/node": "22.7.8",
"@types/react": "^18.3.12",
"@types/react-dom": "^18.3.1",
"autoprefixer": "^10.4.20",
"fast-glob": "^3.3.2",
"gray-matter": "^4.0.3",
"postcss": "^8.4.47",
"remark": "^15.0.1",
"remark-gfm": "^4.0.0",
"remark-mdx": "^3.1.0",
"remark-stringify": "^11.0.0",
"tailwindcss": "^3.4.14",
"tsx": "^4.19.0",
"typescript": "^5.6.3"
}
}
apps/next/scripts/post-build.mts 0 → 100644
+ 7
− 0
View file @ 0fd78d54
import env from "@next/env";
import { updateLlamaCloud } from "./update-llamacloud.mjs";
env.loadEnvConfig(process.cwd());
await updateLlamaCloud();
apps/next/scripts/update-llamacloud.mts 0 → 100644
+ 110
− 0
View file @ 0fd78d54
import { PipelinesService } from "@llamaindex/cloud/api";
import fg from "fast-glob";
import {
fileGenerator,
remarkDocGen,
remarkInstall,
typescriptGenerator,
} from "fumadocs-docgen";
import matter from "gray-matter";
import * as fs from "node:fs/promises";
import path, { relative } from "node:path";
import { fileURLToPath } from "node:url";
import { remark } from "remark";
import remarkGfm from "remark-gfm";
import remarkMdx from "remark-mdx";
import remarkStringify from "remark-stringify";
const baseDir = fileURLToPath(new URL("../src/content", import.meta.url));
async function processContent(content: string): Promise<string> {
const file = await remark()
.use(remarkMdx)
.use(remarkGfm)
.use(remarkDocGen, { generators: [typescriptGenerator(), fileGenerator()] })
.use(remarkInstall, { persist: { id: "package-manager" } })
.use(remarkStringify)
.process(content);
return String(file);
}
export async function updateLlamaCloud(): Promise<void> {
// eslint-disable-next-line turbo/no-undeclared-env-vars
const apiKey = process.env.LLAMA_CLOUD_API_KEY;
// eslint-disable-next-line turbo/no-undeclared-env-vars
const index = process.env.LLAMA_CLOUD_PIPELINE_ID;
if (!apiKey || !index) {
console.log("no api key for LlamaCloud found, skipping");
return;
}
const files = await fg([
"./src/content/docs/**/*.mdx",
"!./src/content/docs/cloud/api/**/*",
]);
const records: {
id: string;
title: string;
description: string;
content: string;
category: string | undefined;
}[] = [];
console.log("processing documents for AI");
const scan = files.map(async (file) => {
const fileContent = await fs.readFile(file);
const { content, data } = matter(fileContent.toString());
const dir = path.dirname(file).split(path.sep).at(3);
const category = {
cloud: "LlamaCloud",
llamaindex: "LlamaIndex.TS",
}[dir ?? ""];
if (data._mdx?.mirror) {
return;
}
const processed = await processContent(content);
const id = relative(baseDir, file);
records.push({
id,
title: data.title as string,
description: data.description as string,
content: processed,
category,
});
});
await Promise.all(scan);
console.log(`added ${records.length} records`);
await PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
{
baseUrl: "https://api.cloud.llamaindex.ai/",
body: records.map((record) => ({
id: record.id,
metadata: {
title: record.title,
description: record.description,
documentUrl: record.id,
category: record.category,
},
text: record.content,
})),
path: {
pipeline_id: index,
},
throwOnError: true,
headers: {
Authorization: `Bearer ${apiKey}`,
},
},
);
console.log("done");
}
apps/next/source.config.ts
+ 0
− 67
View file @ 0fd78d54
import { PipelinesService } from "@llamaindex/cloud/api";
import { rehypeCodeDefaultOptions } from "fumadocs-core/mdx-plugins";
import { fileGenerator, remarkDocGen, remarkInstall } from "fumadocs-docgen";
import { defineConfig, defineDocs } from "fumadocs-mdx/config";
import { transformerTwoslash } from "fumadocs-twoslash";
import { relative } from "node:path";
import { fileURLToPath } from "node:url";
import rehypeKatex from "rehype-katex";
import remarkMath from "remark-math";
const baseDir = fileURLToPath(new URL("../src/content", import.meta.url));
export const { docs, meta } = defineDocs({
dir: "./src/content/docs",
});
......@@ -49,68 +44,6 @@ export default defineConfig({
remarkMath,
[remarkInstall, { persist: { id: "package-manager" } }],
[remarkDocGen, { generators: [fileGenerator()] }],
() => {
return (_, file, next) => {
const metadata = file.data.frontmatter as Record<string, unknown>;
const title = metadata.title as string;
const description = metadata.description as string;
let content: string;
if (file.value instanceof Uint8Array) {
content = file.value.toString();
} else {
content = file.value;
}
if (file.path.includes("content/docs/cloud/api")) {
// skip cloud api docs
return next();
}
// eslint-disable-next-line turbo/no-undeclared-env-vars
if (process.env.NODE_ENV === "development") {
// skip development
return next();
}
if (!title || !description) {
throw new Error(`Missing title or description in ${file.path}`);
}
const id = relative(baseDir, file.path);
if (
// eslint-disable-next-line turbo/no-undeclared-env-vars
process.env.LLAMA_CLOUD_UPSERT_PIPELINE_DOCUMENTS === "true" &&
// eslint-disable-next-line turbo/no-undeclared-env-vars
process.env.LLAMA_CLOUD_PIPELINE_ID !== undefined
) {
PipelinesService.upsertBatchPipelineDocumentsApiV1PipelinesPipelineIdDocumentsPut(
{
baseUrl: "https://api.cloud.llamaindex.ai/",
body: [
{
metadata: {
title,
description,
documentUrl: id,
},
text: content,
id,
},
],
path: {
// eslint-disable-next-line turbo/no-undeclared-env-vars
pipeline_id: process.env.LLAMA_CLOUD_PIPELINE_ID,
},
throwOnError: true,
headers: {
// eslint-disable-next-line turbo/no-undeclared-env-vars
Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
},
},
).catch((error) => {
console.error(error);
});
}
return next();
};
},
],
rehypePlugins: (v) => [rehypeKatex, ...v],
},
......
apps/next/tsconfig.json
+ 7
− 1
View file @ 0fd78d54
......@@ -24,6 +24,12 @@
}
]
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx",
".next/types/**/*.ts",
"**/*.mts"
],
"exclude": ["node_modules"]
}
pnpm-lock.yaml
+ 34
− 10
View file @ 0fd78d54
......@@ -245,6 +245,9 @@ importers:
specifier: ^3.23.8
version: 3.23.8
devDependencies:
'@next/env':
specifier: ^15.0.1
version: 15.0.1
'@types/mdx':
specifier: ^2.0.13
version: 2.0.13
......@@ -260,12 +263,33 @@ importers:
autoprefixer:
specifier: ^10.4.20
version: 10.4.20(postcss@8.4.47)
fast-glob:
specifier: ^3.3.2
version: 3.3.2
gray-matter:
specifier: ^4.0.3
version: 4.0.3
postcss:
specifier: ^8.4.47
version: 8.4.47
remark:
specifier: ^15.0.1
version: 15.0.1
remark-gfm:
specifier: ^4.0.0
version: 4.0.0
remark-mdx:
specifier: ^3.1.0
version: 3.1.0
remark-stringify:
specifier: ^11.0.0
version: 11.0.0
tailwindcss:
specifier: ^3.4.14
version: 3.4.14
tsx:
specifier: ^4.19.0
version: 4.19.0
typescript:
specifier: ^5.6.3
version: 5.6.3
......@@ -11489,8 +11513,8 @@ packages:
remark-math@6.0.0:
resolution: {integrity: sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA==}
 
remark-mdx@3.0.1:
resolution: {integrity: sha512-3Pz3yPQ5Rht2pM5R+0J2MrGoBSrzf+tJG94N+t/ilfdh8YLyyKYtidAYwTveB20BoHAcwIopOUqhcmh2F7hGYA==}
remark-mdx@3.1.0:
resolution: {integrity: sha512-Ngl/H3YXyBV9RcRNdlYsZujAmhsxwzxpDzpDEhFBVAGthS4GDgnctpDjgFl/ULx5UEDzqtW1cyBSNKqYYrqLBA==}
 
remark-parse@11.0.0:
resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==}
......@@ -17878,7 +17902,7 @@ snapshots:
recma-jsx: 1.0.0(acorn@8.13.0)
recma-stringify: 1.0.0
rehype-recma: 1.0.0
remark-mdx: 3.0.1
remark-mdx: 3.1.0
remark-parse: 11.0.0
remark-rehype: 11.1.1
source-map: 0.7.4
......@@ -22111,7 +22135,7 @@ snapshots:
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.3)
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0)
eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0)
eslint-plugin-jsx-a11y: 6.9.0(eslint@8.57.0)
eslint-plugin-react: 7.35.0(eslint@8.57.0)
......@@ -22140,13 +22164,13 @@ snapshots:
transitivePeerDependencies:
- supports-color
 
eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0):
eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
dependencies:
'@nolyfill/is-core-module': 1.0.39
debug: 4.3.7
enhanced-resolve: 5.17.1
eslint: 8.57.0
eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
fast-glob: 3.3.2
get-tsconfig: 4.8.1
is-bun-module: 1.1.0
......@@ -22159,14 +22183,14 @@ snapshots:
- eslint-import-resolver-webpack
- supports-color
 
eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
dependencies:
debug: 3.2.7
optionalDependencies:
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.3)
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0)
eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
transitivePeerDependencies:
- supports-color
 
......@@ -22180,7 +22204,7 @@ snapshots:
doctrine: 2.1.0
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
hasown: 2.0.2
is-core-module: 2.15.1
is-glob: 4.0.3
......@@ -26986,7 +27010,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
 
remark-mdx@3.0.1:
remark-mdx@3.1.0:
dependencies:
mdast-util-mdx: 3.0.0
micromark-extension-mdxjs: 3.0.0
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

:feet::heart: Strive to be the person your dogs believe you are :heart::feet: