From ba1cb996cf4b39c037db1064fb3b29852b884670 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Fri, 29 Dec 2023 14:10:01 +0700
Subject: [PATCH] refactor: separate python and typescript generators
---
packages/create-llama/templates/index.ts | 328 +-----------------
packages/create-llama/templates/python.ts | 114 ++++++
packages/create-llama/templates/typescript.ts | 217 ++++++++++++
3 files changed, 334 insertions(+), 325 deletions(-)
create mode 100644 packages/create-llama/templates/python.ts
create mode 100644 packages/create-llama/templates/typescript.ts
diff --git a/packages/create-llama/templates/index.ts b/packages/create-llama/templates/index.ts
index 4508b24b1..973a65b85 100644
--- a/packages/create-llama/templates/index.ts
+++ b/packages/create-llama/templates/index.ts
@@ -1,22 +1,21 @@
-import { parse, stringify } from "smol-toml";
import { copy } from "../helpers/copy";
import { callPackageManager } from "../helpers/install";
import fs from "fs/promises";
-import os from "os";
import path from "path";
-import { bold, cyan } from "picocolors";
-import { version } from "../../core/package.json";
+import { cyan } from "picocolors";
import { COMMUNITY_OWNER, COMMUNITY_REPO } from "../helpers/constant";
import { PackageManager } from "../helpers/get-pkg-manager";
import { downloadAndExtractRepo } from "../helpers/repo";
+import { installPythonTemplate } from "./python";
import {
InstallTemplateArgs,
TemplateEngine,
TemplateFramework,
TemplateVectorDB,
} from "./types";
+import { installTSTemplate } from "./typescript";
const createEnvLocalFile = async (
root: string,
@@ -103,327 +102,6 @@ const copyTestData = async (
}
};
-const rename = (name: string) => {
- switch (name) {
- case "gitignore":
- case "eslintrc.json": {
- return `.${name}`;
- }
- // README.md is ignored by webpack-asset-relocator-loader used by ncc:
- // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
- case "README-template.md": {
- return "README.md";
- }
- default: {
- return name;
- }
- }
-};
-
-/**
- * Install a LlamaIndex internal template to a given `root` directory.
- */
-const installTSTemplate = async ({
- appName,
- root,
- packageManager,
- isOnline,
- template,
- framework,
- engine,
- ui,
- eslint,
- customApiPath,
- forBackend,
- model,
- vectorDb,
-}: InstallTemplateArgs) => {
- console.log(bold(`Using ${packageManager}.`));
-
- /**
- * Copy the template files to the target directory.
- */
- console.log("\nInitializing project with template:", template, "\n");
- const templatePath = path.join(__dirname, "types", template, framework);
- const copySource = ["**"];
- if (!eslint) copySource.push("!eslintrc.json");
-
- await copy(copySource, root, {
- parents: true,
- cwd: templatePath,
- rename,
- });
-
- /**
- * If the backend is next.js, rename next.config.app.js to next.config.js
- * If not, rename next.config.static.js to next.config.js
- */
- if (framework == "nextjs" && forBackend === "nextjs") {
- const nextConfigAppPath = path.join(root, "next.config.app.js");
- const nextConfigPath = path.join(root, "next.config.js");
- await fs.rename(nextConfigAppPath, nextConfigPath);
- // delete next.config.static.js
- const nextConfigStaticPath = path.join(root, "next.config.static.js");
- await fs.rm(nextConfigStaticPath);
- } else if (framework == "nextjs" && typeof forBackend === "undefined") {
- const nextConfigStaticPath = path.join(root, "next.config.static.js");
- const nextConfigPath = path.join(root, "next.config.js");
- await fs.rename(nextConfigStaticPath, nextConfigPath);
- // delete next.config.app.js
- const nextConfigAppPath = path.join(root, "next.config.app.js");
- await fs.rm(nextConfigAppPath);
- }
-
- /**
- * Copy the selected chat engine files to the target directory and reference it.
- */
- let relativeEngineDestPath;
- const compPath = path.join(__dirname, "components");
- if (engine && (framework === "express" || framework === "nextjs")) {
- console.log("\nUsing chat engine:", engine, "\n");
-
- let vectorDBFolder: string = engine;
-
- if (engine !== "simple" && vectorDb) {
- console.log("\nUsing vector DB:", vectorDb, "\n");
- vectorDBFolder = vectorDb;
- }
-
- const VectorDBPath = path.join(
- compPath,
- "vectordbs",
- "typescript",
- vectorDBFolder,
- );
- relativeEngineDestPath =
- framework === "nextjs"
- ? path.join("app", "api", "chat")
- : path.join("src", "controllers");
- await copy("**", path.join(root, relativeEngineDestPath, "engine"), {
- parents: true,
- cwd: VectorDBPath,
- });
- }
-
- /**
- * Copy the selected UI files to the target directory and reference it.
- */
- if (framework === "nextjs" && ui !== "shadcn") {
- console.log("\nUsing UI:", ui, "\n");
- const uiPath = path.join(compPath, "ui", ui);
- const destUiPath = path.join(root, "app", "components", "ui");
- // remove the default ui folder
- await fs.rm(destUiPath, { recursive: true });
- // copy the selected ui folder
- await copy("**", destUiPath, {
- parents: true,
- cwd: uiPath,
- rename,
- });
- }
-
- /**
- * Update the package.json scripts.
- */
- const packageJsonFile = path.join(root, "package.json");
- const packageJson: any = JSON.parse(
- await fs.readFile(packageJsonFile, "utf8"),
- );
- packageJson.name = appName;
- packageJson.version = "0.1.0";
-
- packageJson.dependencies = {
- ...packageJson.dependencies,
- llamaindex: version,
- };
-
- if (framework === "nextjs" && customApiPath) {
- console.log(
- "\nUsing external API with custom API path:",
- customApiPath,
- "\n",
- );
- // remove the default api folder
- const apiPath = path.join(root, "app", "api");
- await fs.rm(apiPath, { recursive: true });
- // modify the dev script to use the custom api path
- packageJson.scripts = {
- ...packageJson.scripts,
- dev: `cross-env NEXT_PUBLIC_CHAT_API=${customApiPath} next dev`,
- };
- }
-
- if (engine === "context" && relativeEngineDestPath) {
- // add generate script if using context engine
- packageJson.scripts = {
- ...packageJson.scripts,
- generate: `node ${path.join(
- relativeEngineDestPath,
- "engine",
- "generate.mjs",
- )}`,
- };
- }
-
- if (framework === "nextjs" && ui === "html") {
- // remove shadcn dependencies if html ui is selected
- packageJson.dependencies = {
- ...packageJson.dependencies,
- "tailwind-merge": undefined,
- "@radix-ui/react-slot": undefined,
- "class-variance-authority": undefined,
- clsx: undefined,
- "lucide-react": undefined,
- remark: undefined,
- "remark-code-import": undefined,
- "remark-gfm": undefined,
- "remark-math": undefined,
- "react-markdown": undefined,
- "react-syntax-highlighter": undefined,
- };
-
- packageJson.devDependencies = {
- ...packageJson.devDependencies,
- "@types/react-syntax-highlighter": undefined,
- };
- }
-
- if (!eslint) {
- // Remove packages starting with "eslint" from devDependencies
- packageJson.devDependencies = Object.fromEntries(
- Object.entries(packageJson.devDependencies).filter(
- ([key]) => !key.startsWith("eslint"),
- ),
- );
- }
- await fs.writeFile(
- packageJsonFile,
- JSON.stringify(packageJson, null, 2) + os.EOL,
- );
-
- console.log("\nInstalling dependencies:");
- for (const dependency in packageJson.dependencies)
- console.log(`- ${cyan(dependency)}`);
-
- console.log("\nInstalling devDependencies:");
- for (const dependency in packageJson.devDependencies)
- console.log(`- ${cyan(dependency)}`);
-
- console.log();
-
- await callPackageManager(packageManager, isOnline);
-};
-
-interface IDependencyItem {
- name: string;
- version: string;
-}
-
-const getPythonAddOnDependencies = (vectorDb?: TemplateVectorDB) => {
- const addOnDependencies: IDependencyItem[] = [];
-
- switch (vectorDb) {
- case "mongo": {
- addOnDependencies.push({
- name: "pymongo",
- version: "^4.6.1",
- });
- break;
- }
- }
-
- return addOnDependencies;
-};
-
-const preparePythonDependencies = async (
- root: string,
- addOnDependencies: IDependencyItem[],
-) => {
- if (addOnDependencies.length === 0) return;
-
- const FILENAME = "pyproject.toml";
- try {
- // Parse toml file
- const file = path.join(root, FILENAME);
- const fileContent = await fs.readFile(file, "utf8");
- const fileParsed = parse(fileContent);
-
- // Modify toml dependencies
- const tool = fileParsed.tool as any;
- const dependencies = tool.poetry.dependencies as any;
- for (const dependency of addOnDependencies) {
- dependencies[dependency.name] = dependency.version;
- }
-
- // Write toml file
- const newFileContent = stringify(fileParsed);
- await fs.writeFile(file, newFileContent);
-
- const dependenciesString = addOnDependencies.map((d) => d.name).join(", ");
- console.log(`\nAdded ${dependenciesString} to ${cyan(FILENAME)}\n`);
- } catch (error) {
- console.log(
- `Error when preparing ${FILENAME} file for Python template\n`,
- error,
- );
- console.log(error);
- }
-};
-
-const installPythonTemplate = async ({
- root,
- template,
- framework,
- engine,
- vectorDb,
-}: Pick<
- InstallTemplateArgs,
- "root" | "framework" | "template" | "engine" | "vectorDb"
->) => {
- console.log("\nInitializing Python project with template:", template, "\n");
- const templatePath = path.join(__dirname, "types", template, framework);
- await copy("**", root, {
- parents: true,
- cwd: templatePath,
- rename(name) {
- switch (name) {
- case "gitignore": {
- return `.${name}`;
- }
- // README.md is ignored by webpack-asset-relocator-loader used by ncc:
- // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
- case "README-template.md": {
- return "README.md";
- }
- default: {
- return name;
- }
- }
- },
- });
-
- if (engine === "context") {
- const compPath = path.join(__dirname, "components");
- const VectorDBPath = path.join(
- compPath,
- "vectordbs",
- "python",
- vectorDb || "none",
- );
- await copy("**", path.join(root, "app", "engine"), {
- parents: true,
- cwd: VectorDBPath,
- });
- }
-
- const addOnDependencies = getPythonAddOnDependencies(vectorDb);
- await preparePythonDependencies(root, addOnDependencies);
-
- console.log(
- "\nPython project, dependencies won't be installed automatically.\n",
- );
-};
-
const installCommunityProject = async ({
root,
communityProjectPath,
diff --git a/packages/create-llama/templates/python.ts b/packages/create-llama/templates/python.ts
new file mode 100644
index 000000000..f845615c7
--- /dev/null
+++ b/packages/create-llama/templates/python.ts
@@ -0,0 +1,114 @@
+import fs from "fs/promises";
+import path from "path";
+import { cyan } from "picocolors";
+import { parse, stringify } from "smol-toml";
+import { copy } from "../helpers/copy";
+import { InstallTemplateArgs, TemplateVectorDB } from "./types";
+
+interface IDependencyItem {
+ name: string;
+ version: string;
+}
+
+const getPythonAddOnDependencies = (vectorDb?: TemplateVectorDB) => {
+ const addOnDependencies: IDependencyItem[] = [];
+
+ switch (vectorDb) {
+ case "mongo": {
+ addOnDependencies.push({
+ name: "pymongo",
+ version: "^4.6.1",
+ });
+ break;
+ }
+ }
+
+ return addOnDependencies;
+};
+const preparePythonDependencies = async (
+ root: string,
+ addOnDependencies: IDependencyItem[],
+) => {
+ if (addOnDependencies.length === 0) return;
+
+ const FILENAME = "pyproject.toml";
+ try {
+ // Parse toml file
+ const file = path.join(root, FILENAME);
+ const fileContent = await fs.readFile(file, "utf8");
+ const fileParsed = parse(fileContent);
+
+ // Modify toml dependencies
+ const tool = fileParsed.tool as any;
+ const dependencies = tool.poetry.dependencies as any;
+ for (const dependency of addOnDependencies) {
+ dependencies[dependency.name] = dependency.version;
+ }
+
+ // Write toml file
+ const newFileContent = stringify(fileParsed);
+ await fs.writeFile(file, newFileContent);
+
+ const dependenciesString = addOnDependencies.map((d) => d.name).join(", ");
+ console.log(`\nAdded ${dependenciesString} to ${cyan(FILENAME)}\n`);
+ } catch (error) {
+ console.log(
+ `Error when preparing ${FILENAME} file for Python template\n`,
+ error,
+ );
+ console.log(error);
+ }
+};
+export const installPythonTemplate = async ({
+ root,
+ template,
+ framework,
+ engine,
+ vectorDb,
+}: Pick<
+ InstallTemplateArgs,
+ "root" | "framework" | "template" | "engine" | "vectorDb"
+>) => {
+ console.log("\nInitializing Python project with template:", template, "\n");
+ const templatePath = path.join(__dirname, "types", template, framework);
+ await copy("**", root, {
+ parents: true,
+ cwd: templatePath,
+ rename(name) {
+ switch (name) {
+ case "gitignore": {
+ return `.${name}`;
+ }
+ // README.md is ignored by webpack-asset-relocator-loader used by ncc:
+ // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
+ case "README-template.md": {
+ return "README.md";
+ }
+ default: {
+ return name;
+ }
+ }
+ },
+ });
+
+ if (engine === "context") {
+ const compPath = path.join(__dirname, "components");
+ const VectorDBPath = path.join(
+ compPath,
+ "vectordbs",
+ "python",
+ vectorDb || "none",
+ );
+ await copy("**", path.join(root, "app", "engine"), {
+ parents: true,
+ cwd: VectorDBPath,
+ });
+ }
+
+ const addOnDependencies = getPythonAddOnDependencies(vectorDb);
+ await preparePythonDependencies(root, addOnDependencies);
+
+ console.log(
+ "\nPython project, dependencies won't be installed automatically.\n",
+ );
+};
diff --git a/packages/create-llama/templates/typescript.ts b/packages/create-llama/templates/typescript.ts
new file mode 100644
index 000000000..df3c0b15b
--- /dev/null
+++ b/packages/create-llama/templates/typescript.ts
@@ -0,0 +1,217 @@
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { bold, cyan } from "picocolors";
+import { version } from "../../core/package.json";
+import { copy } from "../helpers/copy";
+import { callPackageManager } from "../helpers/install";
+import { InstallTemplateArgs } from "./types";
+
+const rename = (name: string) => {
+ switch (name) {
+ case "gitignore":
+ case "eslintrc.json": {
+ return `.${name}`;
+ }
+ // README.md is ignored by webpack-asset-relocator-loader used by ncc:
+ // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
+ case "README-template.md": {
+ return "README.md";
+ }
+ default: {
+ return name;
+ }
+ }
+};
+/**
+ * Install a LlamaIndex internal template to a given `root` directory.
+ */
+export const installTSTemplate = async ({
+ appName,
+ root,
+ packageManager,
+ isOnline,
+ template,
+ framework,
+ engine,
+ ui,
+ eslint,
+ customApiPath,
+ forBackend,
+ vectorDb,
+}: InstallTemplateArgs) => {
+ console.log(bold(`Using ${packageManager}.`));
+
+ /**
+ * Copy the template files to the target directory.
+ */
+ console.log("\nInitializing project with template:", template, "\n");
+ const templatePath = path.join(__dirname, "types", template, framework);
+ const copySource = ["**"];
+ if (!eslint) copySource.push("!eslintrc.json");
+
+ await copy(copySource, root, {
+ parents: true,
+ cwd: templatePath,
+ rename,
+ });
+
+ /**
+ * If the backend is next.js, rename next.config.app.js to next.config.js
+ * If not, rename next.config.static.js to next.config.js
+ */
+ if (framework == "nextjs" && forBackend === "nextjs") {
+ const nextConfigAppPath = path.join(root, "next.config.app.js");
+ const nextConfigPath = path.join(root, "next.config.js");
+ await fs.rename(nextConfigAppPath, nextConfigPath);
+ // delete next.config.static.js
+ const nextConfigStaticPath = path.join(root, "next.config.static.js");
+ await fs.rm(nextConfigStaticPath);
+ } else if (framework == "nextjs" && typeof forBackend === "undefined") {
+ const nextConfigStaticPath = path.join(root, "next.config.static.js");
+ const nextConfigPath = path.join(root, "next.config.js");
+ await fs.rename(nextConfigStaticPath, nextConfigPath);
+ // delete next.config.app.js
+ const nextConfigAppPath = path.join(root, "next.config.app.js");
+ await fs.rm(nextConfigAppPath);
+ }
+
+ /**
+ * Copy the selected chat engine files to the target directory and reference it.
+ */
+ let relativeEngineDestPath;
+ const compPath = path.join(__dirname, "components");
+ if (engine && (framework === "express" || framework === "nextjs")) {
+ console.log("\nUsing chat engine:", engine, "\n");
+
+ let vectorDBFolder: string = engine;
+
+ if (engine !== "simple" && vectorDb) {
+ console.log("\nUsing vector DB:", vectorDb, "\n");
+ vectorDBFolder = vectorDb;
+ }
+
+ const VectorDBPath = path.join(
+ compPath,
+ "vectordbs",
+ "typescript",
+ vectorDBFolder,
+ );
+ relativeEngineDestPath =
+ framework === "nextjs"
+ ? path.join("app", "api", "chat")
+ : path.join("src", "controllers");
+ await copy("**", path.join(root, relativeEngineDestPath, "engine"), {
+ parents: true,
+ cwd: VectorDBPath,
+ });
+ }
+
+ /**
+ * Copy the selected UI files to the target directory and reference it.
+ */
+ if (framework === "nextjs" && ui !== "shadcn") {
+ console.log("\nUsing UI:", ui, "\n");
+ const uiPath = path.join(compPath, "ui", ui);
+ const destUiPath = path.join(root, "app", "components", "ui");
+ // remove the default ui folder
+ await fs.rm(destUiPath, { recursive: true });
+ // copy the selected ui folder
+ await copy("**", destUiPath, {
+ parents: true,
+ cwd: uiPath,
+ rename,
+ });
+ }
+
+ /**
+ * Update the package.json scripts.
+ */
+ const packageJsonFile = path.join(root, "package.json");
+ const packageJson: any = JSON.parse(
+ await fs.readFile(packageJsonFile, "utf8"),
+ );
+ packageJson.name = appName;
+ packageJson.version = "0.1.0";
+
+ packageJson.dependencies = {
+ ...packageJson.dependencies,
+ llamaindex: version,
+ };
+
+ if (framework === "nextjs" && customApiPath) {
+ console.log(
+ "\nUsing external API with custom API path:",
+ customApiPath,
+ "\n",
+ );
+ // remove the default api folder
+ const apiPath = path.join(root, "app", "api");
+ await fs.rm(apiPath, { recursive: true });
+ // modify the dev script to use the custom api path
+ packageJson.scripts = {
+ ...packageJson.scripts,
+ dev: `cross-env NEXT_PUBLIC_CHAT_API=${customApiPath} next dev`,
+ };
+ }
+
+ if (engine === "context" && relativeEngineDestPath) {
+ // add generate script if using context engine
+ packageJson.scripts = {
+ ...packageJson.scripts,
+ generate: `node ${path.join(
+ relativeEngineDestPath,
+ "engine",
+ "generate.mjs",
+ )}`,
+ };
+ }
+
+ if (framework === "nextjs" && ui === "html") {
+ // remove shadcn dependencies if html ui is selected
+ packageJson.dependencies = {
+ ...packageJson.dependencies,
+ "tailwind-merge": undefined,
+ "@radix-ui/react-slot": undefined,
+ "class-variance-authority": undefined,
+ clsx: undefined,
+ "lucide-react": undefined,
+ remark: undefined,
+ "remark-code-import": undefined,
+ "remark-gfm": undefined,
+ "remark-math": undefined,
+ "react-markdown": undefined,
+ "react-syntax-highlighter": undefined,
+ };
+
+ packageJson.devDependencies = {
+ ...packageJson.devDependencies,
+ "@types/react-syntax-highlighter": undefined,
+ };
+ }
+
+ if (!eslint) {
+ // Remove packages starting with "eslint" from devDependencies
+ packageJson.devDependencies = Object.fromEntries(
+ Object.entries(packageJson.devDependencies).filter(
+ ([key]) => !key.startsWith("eslint"),
+ ),
+ );
+ }
+ await fs.writeFile(
+ packageJsonFile,
+ JSON.stringify(packageJson, null, 2) + os.EOL,
+ );
+
+ console.log("\nInstalling dependencies:");
+ for (const dependency in packageJson.dependencies)
+ console.log(`- ${cyan(dependency)}`);
+
+ console.log("\nInstalling devDependencies:");
+ for (const dependency in packageJson.devDependencies)
+ console.log(`- ${cyan(dependency)}`);
+
+ console.log();
+
+ await callPackageManager(packageManager, isOnline);
+};
--
GitLab