Skip to content
Snippets Groups Projects
Commit ba1cb996 authored by Marcus Schiesser's avatar Marcus Schiesser
Browse files

refactor: separate python and typescript generators

parent 41f41d65
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
packages/create-llama/templates/index.ts
+ 3
− 325
View file @ ba1cb996
import { parse, stringify } from "smol-toml";
import { copy } from "../helpers/copy"; import { copy } from "../helpers/copy";
import { callPackageManager } from "../helpers/install"; import { callPackageManager } from "../helpers/install";
import fs from "fs/promises"; import fs from "fs/promises";
import os from "os";
import path from "path"; import path from "path";
import { bold, cyan } from "picocolors"; import { cyan } from "picocolors";
import { version } from "../../core/package.json";
import { COMMUNITY_OWNER, COMMUNITY_REPO } from "../helpers/constant"; import { COMMUNITY_OWNER, COMMUNITY_REPO } from "../helpers/constant";
import { PackageManager } from "../helpers/get-pkg-manager"; import { PackageManager } from "../helpers/get-pkg-manager";
import { downloadAndExtractRepo } from "../helpers/repo"; import { downloadAndExtractRepo } from "../helpers/repo";
import { installPythonTemplate } from "./python";
import { import {
InstallTemplateArgs, InstallTemplateArgs,
TemplateEngine, TemplateEngine,
TemplateFramework, TemplateFramework,
TemplateVectorDB, TemplateVectorDB,
} from "./types"; } from "./types";
import { installTSTemplate } from "./typescript";
const createEnvLocalFile = async ( const createEnvLocalFile = async (
root: string, root: string,
...@@ -103,327 +102,6 @@ const copyTestData = async ( ...@@ -103,327 +102,6 @@ const copyTestData = async (
} }
}; };
const rename = (name: string) => {
switch (name) {
case "gitignore":
case "eslintrc.json": {
return `.${name}`;
}
// README.md is ignored by webpack-asset-relocator-loader used by ncc:
// https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
case "README-template.md": {
return "README.md";
}
default: {
return name;
}
}
};
/**
* Install a LlamaIndex internal template to a given `root` directory.
*/
const installTSTemplate = async ({
appName,
root,
packageManager,
isOnline,
template,
framework,
engine,
ui,
eslint,
customApiPath,
forBackend,
model,
vectorDb,
}: InstallTemplateArgs) => {
console.log(bold(`Using ${packageManager}.`));
/**
* Copy the template files to the target directory.
*/
console.log("\nInitializing project with template:", template, "\n");
const templatePath = path.join(__dirname, "types", template, framework);
const copySource = ["**"];
if (!eslint) copySource.push("!eslintrc.json");
await copy(copySource, root, {
parents: true,
cwd: templatePath,
rename,
});
/**
* If the backend is next.js, rename next.config.app.js to next.config.js
* If not, rename next.config.static.js to next.config.js
*/
if (framework == "nextjs" && forBackend === "nextjs") {
const nextConfigAppPath = path.join(root, "next.config.app.js");
const nextConfigPath = path.join(root, "next.config.js");
await fs.rename(nextConfigAppPath, nextConfigPath);
// delete next.config.static.js
const nextConfigStaticPath = path.join(root, "next.config.static.js");
await fs.rm(nextConfigStaticPath);
} else if (framework == "nextjs" && typeof forBackend === "undefined") {
const nextConfigStaticPath = path.join(root, "next.config.static.js");
const nextConfigPath = path.join(root, "next.config.js");
await fs.rename(nextConfigStaticPath, nextConfigPath);
// delete next.config.app.js
const nextConfigAppPath = path.join(root, "next.config.app.js");
await fs.rm(nextConfigAppPath);
}
/**
* Copy the selected chat engine files to the target directory and reference it.
*/
let relativeEngineDestPath;
const compPath = path.join(__dirname, "components");
if (engine && (framework === "express" || framework === "nextjs")) {
console.log("\nUsing chat engine:", engine, "\n");
let vectorDBFolder: string = engine;
if (engine !== "simple" && vectorDb) {
console.log("\nUsing vector DB:", vectorDb, "\n");
vectorDBFolder = vectorDb;
}
const VectorDBPath = path.join(
compPath,
"vectordbs",
"typescript",
vectorDBFolder,
);
relativeEngineDestPath =
framework === "nextjs"
? path.join("app", "api", "chat")
: path.join("src", "controllers");
await copy("**", path.join(root, relativeEngineDestPath, "engine"), {
parents: true,
cwd: VectorDBPath,
});
}
/**
* Copy the selected UI files to the target directory and reference it.
*/
if (framework === "nextjs" && ui !== "shadcn") {
console.log("\nUsing UI:", ui, "\n");
const uiPath = path.join(compPath, "ui", ui);
const destUiPath = path.join(root, "app", "components", "ui");
// remove the default ui folder
await fs.rm(destUiPath, { recursive: true });
// copy the selected ui folder
await copy("**", destUiPath, {
parents: true,
cwd: uiPath,
rename,
});
}
/**
* Update the package.json scripts.
*/
const packageJsonFile = path.join(root, "package.json");
const packageJson: any = JSON.parse(
await fs.readFile(packageJsonFile, "utf8"),
);
packageJson.name = appName;
packageJson.version = "0.1.0";
packageJson.dependencies = {
...packageJson.dependencies,
llamaindex: version,
};
if (framework === "nextjs" && customApiPath) {
console.log(
"\nUsing external API with custom API path:",
customApiPath,
"\n",
);
// remove the default api folder
const apiPath = path.join(root, "app", "api");
await fs.rm(apiPath, { recursive: true });
// modify the dev script to use the custom api path
packageJson.scripts = {
...packageJson.scripts,
dev: `cross-env NEXT_PUBLIC_CHAT_API=${customApiPath} next dev`,
};
}
if (engine === "context" && relativeEngineDestPath) {
// add generate script if using context engine
packageJson.scripts = {
...packageJson.scripts,
generate: `node ${path.join(
relativeEngineDestPath,
"engine",
"generate.mjs",
)}`,
};
}
if (framework === "nextjs" && ui === "html") {
// remove shadcn dependencies if html ui is selected
packageJson.dependencies = {
...packageJson.dependencies,
"tailwind-merge": undefined,
"@radix-ui/react-slot": undefined,
"class-variance-authority": undefined,
clsx: undefined,
"lucide-react": undefined,
remark: undefined,
"remark-code-import": undefined,
"remark-gfm": undefined,
"remark-math": undefined,
"react-markdown": undefined,
"react-syntax-highlighter": undefined,
};
packageJson.devDependencies = {
...packageJson.devDependencies,
"@types/react-syntax-highlighter": undefined,
};
}
if (!eslint) {
// Remove packages starting with "eslint" from devDependencies
packageJson.devDependencies = Object.fromEntries(
Object.entries(packageJson.devDependencies).filter(
([key]) => !key.startsWith("eslint"),
),
);
}
await fs.writeFile(
packageJsonFile,
JSON.stringify(packageJson, null, 2) + os.EOL,
);
console.log("\nInstalling dependencies:");
for (const dependency in packageJson.dependencies)
console.log(`- ${cyan(dependency)}`);
console.log("\nInstalling devDependencies:");
for (const dependency in packageJson.devDependencies)
console.log(`- ${cyan(dependency)}`);
console.log();
await callPackageManager(packageManager, isOnline);
};
interface IDependencyItem {
name: string;
version: string;
}
const getPythonAddOnDependencies = (vectorDb?: TemplateVectorDB) => {
const addOnDependencies: IDependencyItem[] = [];
switch (vectorDb) {
case "mongo": {
addOnDependencies.push({
name: "pymongo",
version: "^4.6.1",
});
break;
}
}
return addOnDependencies;
};
const preparePythonDependencies = async (
root: string,
addOnDependencies: IDependencyItem[],
) => {
if (addOnDependencies.length === 0) return;
const FILENAME = "pyproject.toml";
try {
// Parse toml file
const file = path.join(root, FILENAME);
const fileContent = await fs.readFile(file, "utf8");
const fileParsed = parse(fileContent);
// Modify toml dependencies
const tool = fileParsed.tool as any;
const dependencies = tool.poetry.dependencies as any;
for (const dependency of addOnDependencies) {
dependencies[dependency.name] = dependency.version;
}
// Write toml file
const newFileContent = stringify(fileParsed);
await fs.writeFile(file, newFileContent);
const dependenciesString = addOnDependencies.map((d) => d.name).join(", ");
console.log(`\nAdded ${dependenciesString} to ${cyan(FILENAME)}\n`);
} catch (error) {
console.log(
`Error when preparing ${FILENAME} file for Python template\n`,
error,
);
console.log(error);
}
};
const installPythonTemplate = async ({
root,
template,
framework,
engine,
vectorDb,
}: Pick<
InstallTemplateArgs,
"root" | "framework" | "template" | "engine" | "vectorDb"
>) => {
console.log("\nInitializing Python project with template:", template, "\n");
const templatePath = path.join(__dirname, "types", template, framework);
await copy("**", root, {
parents: true,
cwd: templatePath,
rename(name) {
switch (name) {
case "gitignore": {
return `.${name}`;
}
// README.md is ignored by webpack-asset-relocator-loader used by ncc:
// https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
case "README-template.md": {
return "README.md";
}
default: {
return name;
}
}
},
});
if (engine === "context") {
const compPath = path.join(__dirname, "components");
const VectorDBPath = path.join(
compPath,
"vectordbs",
"python",
vectorDb || "none",
);
await copy("**", path.join(root, "app", "engine"), {
parents: true,
cwd: VectorDBPath,
});
}
const addOnDependencies = getPythonAddOnDependencies(vectorDb);
await preparePythonDependencies(root, addOnDependencies);
console.log(
"\nPython project, dependencies won't be installed automatically.\n",
);
};
const installCommunityProject = async ({ const installCommunityProject = async ({
root, root,
communityProjectPath, communityProjectPath,
......
packages/create-llama/templates/python.ts 0 → 100644
+ 114
− 0
View file @ ba1cb996
import fs from "fs/promises";
import path from "path";
import { cyan } from "picocolors";
import { parse, stringify } from "smol-toml";
import { copy } from "../helpers/copy";
import { InstallTemplateArgs, TemplateVectorDB } from "./types";
interface IDependencyItem {
name: string;
version: string;
}
const getPythonAddOnDependencies = (vectorDb?: TemplateVectorDB) => {
const addOnDependencies: IDependencyItem[] = [];
switch (vectorDb) {
case "mongo": {
addOnDependencies.push({
name: "pymongo",
version: "^4.6.1",
});
break;
}
}
return addOnDependencies;
};
const preparePythonDependencies = async (
root: string,
addOnDependencies: IDependencyItem[],
) => {
if (addOnDependencies.length === 0) return;
const FILENAME = "pyproject.toml";
try {
// Parse toml file
const file = path.join(root, FILENAME);
const fileContent = await fs.readFile(file, "utf8");
const fileParsed = parse(fileContent);
// Modify toml dependencies
const tool = fileParsed.tool as any;
const dependencies = tool.poetry.dependencies as any;
for (const dependency of addOnDependencies) {
dependencies[dependency.name] = dependency.version;
}
// Write toml file
const newFileContent = stringify(fileParsed);
await fs.writeFile(file, newFileContent);
const dependenciesString = addOnDependencies.map((d) => d.name).join(", ");
console.log(`\nAdded ${dependenciesString} to ${cyan(FILENAME)}\n`);
} catch (error) {
console.log(
`Error when preparing ${FILENAME} file for Python template\n`,
error,
);
console.log(error);
}
};
export const installPythonTemplate = async ({
root,
template,
framework,
engine,
vectorDb,
}: Pick<
InstallTemplateArgs,
"root" | "framework" | "template" | "engine" | "vectorDb"
>) => {
console.log("\nInitializing Python project with template:", template, "\n");
const templatePath = path.join(__dirname, "types", template, framework);
await copy("**", root, {
parents: true,
cwd: templatePath,
rename(name) {
switch (name) {
case "gitignore": {
return `.${name}`;
}
// README.md is ignored by webpack-asset-relocator-loader used by ncc:
// https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
case "README-template.md": {
return "README.md";
}
default: {
return name;
}
}
},
});
if (engine === "context") {
const compPath = path.join(__dirname, "components");
const VectorDBPath = path.join(
compPath,
"vectordbs",
"python",
vectorDb || "none",
);
await copy("**", path.join(root, "app", "engine"), {
parents: true,
cwd: VectorDBPath,
});
}
const addOnDependencies = getPythonAddOnDependencies(vectorDb);
await preparePythonDependencies(root, addOnDependencies);
console.log(
"\nPython project, dependencies won't be installed automatically.\n",
);
};
packages/create-llama/templates/typescript.ts 0 → 100644
+ 217
− 0
View file @ ba1cb996
import fs from "fs/promises";
import os from "os";
import path from "path";
import { bold, cyan } from "picocolors";
import { version } from "../../core/package.json";
import { copy } from "../helpers/copy";
import { callPackageManager } from "../helpers/install";
import { InstallTemplateArgs } from "./types";
const rename = (name: string) => {
switch (name) {
case "gitignore":
case "eslintrc.json": {
return `.${name}`;
}
// README.md is ignored by webpack-asset-relocator-loader used by ncc:
// https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
case "README-template.md": {
return "README.md";
}
default: {
return name;
}
}
};
/**
* Install a LlamaIndex internal template to a given `root` directory.
*/
export const installTSTemplate = async ({
appName,
root,
packageManager,
isOnline,
template,
framework,
engine,
ui,
eslint,
customApiPath,
forBackend,
vectorDb,
}: InstallTemplateArgs) => {
console.log(bold(`Using ${packageManager}.`));
/**
* Copy the template files to the target directory.
*/
console.log("\nInitializing project with template:", template, "\n");
const templatePath = path.join(__dirname, "types", template, framework);
const copySource = ["**"];
if (!eslint) copySource.push("!eslintrc.json");
await copy(copySource, root, {
parents: true,
cwd: templatePath,
rename,
});
/**
* If the backend is next.js, rename next.config.app.js to next.config.js
* If not, rename next.config.static.js to next.config.js
*/
if (framework == "nextjs" && forBackend === "nextjs") {
const nextConfigAppPath = path.join(root, "next.config.app.js");
const nextConfigPath = path.join(root, "next.config.js");
await fs.rename(nextConfigAppPath, nextConfigPath);
// delete next.config.static.js
const nextConfigStaticPath = path.join(root, "next.config.static.js");
await fs.rm(nextConfigStaticPath);
} else if (framework == "nextjs" && typeof forBackend === "undefined") {
const nextConfigStaticPath = path.join(root, "next.config.static.js");
const nextConfigPath = path.join(root, "next.config.js");
await fs.rename(nextConfigStaticPath, nextConfigPath);
// delete next.config.app.js
const nextConfigAppPath = path.join(root, "next.config.app.js");
await fs.rm(nextConfigAppPath);
}
/**
* Copy the selected chat engine files to the target directory and reference it.
*/
let relativeEngineDestPath;
const compPath = path.join(__dirname, "components");
if (engine && (framework === "express" || framework === "nextjs")) {
console.log("\nUsing chat engine:", engine, "\n");
let vectorDBFolder: string = engine;
if (engine !== "simple" && vectorDb) {
console.log("\nUsing vector DB:", vectorDb, "\n");
vectorDBFolder = vectorDb;
}
const VectorDBPath = path.join(
compPath,
"vectordbs",
"typescript",
vectorDBFolder,
);
relativeEngineDestPath =
framework === "nextjs"
? path.join("app", "api", "chat")
: path.join("src", "controllers");
await copy("**", path.join(root, relativeEngineDestPath, "engine"), {
parents: true,
cwd: VectorDBPath,
});
}
/**
* Copy the selected UI files to the target directory and reference it.
*/
if (framework === "nextjs" && ui !== "shadcn") {
console.log("\nUsing UI:", ui, "\n");
const uiPath = path.join(compPath, "ui", ui);
const destUiPath = path.join(root, "app", "components", "ui");
// remove the default ui folder
await fs.rm(destUiPath, { recursive: true });
// copy the selected ui folder
await copy("**", destUiPath, {
parents: true,
cwd: uiPath,
rename,
});
}
/**
* Update the package.json scripts.
*/
const packageJsonFile = path.join(root, "package.json");
const packageJson: any = JSON.parse(
await fs.readFile(packageJsonFile, "utf8"),
);
packageJson.name = appName;
packageJson.version = "0.1.0";
packageJson.dependencies = {
...packageJson.dependencies,
llamaindex: version,
};
if (framework === "nextjs" && customApiPath) {
console.log(
"\nUsing external API with custom API path:",
customApiPath,
"\n",
);
// remove the default api folder
const apiPath = path.join(root, "app", "api");
await fs.rm(apiPath, { recursive: true });
// modify the dev script to use the custom api path
packageJson.scripts = {
...packageJson.scripts,
dev: `cross-env NEXT_PUBLIC_CHAT_API=${customApiPath} next dev`,
};
}
if (engine === "context" && relativeEngineDestPath) {
// add generate script if using context engine
packageJson.scripts = {
...packageJson.scripts,
generate: `node ${path.join(
relativeEngineDestPath,
"engine",
"generate.mjs",
)}`,
};
}
if (framework === "nextjs" && ui === "html") {
// remove shadcn dependencies if html ui is selected
packageJson.dependencies = {
...packageJson.dependencies,
"tailwind-merge": undefined,
"@radix-ui/react-slot": undefined,
"class-variance-authority": undefined,
clsx: undefined,
"lucide-react": undefined,
remark: undefined,
"remark-code-import": undefined,
"remark-gfm": undefined,
"remark-math": undefined,
"react-markdown": undefined,
"react-syntax-highlighter": undefined,
};
packageJson.devDependencies = {
...packageJson.devDependencies,
"@types/react-syntax-highlighter": undefined,
};
}
if (!eslint) {
// Remove packages starting with "eslint" from devDependencies
packageJson.devDependencies = Object.fromEntries(
Object.entries(packageJson.devDependencies).filter(
([key]) => !key.startsWith("eslint"),
),
);
}
await fs.writeFile(
packageJsonFile,
JSON.stringify(packageJson, null, 2) + os.EOL,
);
console.log("\nInstalling dependencies:");
for (const dependency in packageJson.dependencies)
console.log(`- ${cyan(dependency)}`);
console.log("\nInstalling devDependencies:");
for (const dependency in packageJson.devDependencies)
console.log(`- ${cyan(dependency)}`);
console.log();
await callPackageManager(packageManager, isOnline);
};
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

:feet::heart: Strive to be the person your dogs believe you are :heart::feet: