diff --git a/helpers/constant.ts b/helpers/constant.ts index 702d1d7fdf326d22ab4a0314309939d3037c39e2..64f01be676867ca0b2cf781a50aea162dec8ac4d 100644 --- a/helpers/constant.ts +++ b/helpers/constant.ts @@ -1,6 +1,6 @@ export const COMMUNITY_OWNER = "run-llama"; export const COMMUNITY_REPO = "create_llama_projects"; export const LLAMA_PACK_OWNER = "run-llama"; -export const LLAMA_PACK_REPO = "llama-hub"; -export const LLAMA_HUB_FOLDER_PATH = `${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/main/llama_hub`; -export const LLAMA_PACK_CONFIG_PATH = `${LLAMA_HUB_FOLDER_PATH}/llama_packs/library.json`; +export const LLAMA_PACK_REPO = "llama_index"; +export const LLAMA_PACK_FOLDER = "llama-index-packs"; +export const LLAMA_PACK_FOLDER_PATH = `${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/main/${LLAMA_PACK_FOLDER}`; diff --git a/helpers/llama-pack.ts b/helpers/llama-pack.ts index adc0966afcec22a9797a367b819db72fe728d713..16cd801fa3e24cbfc666eb85c00b389dc6f61d3c 100644 --- a/helpers/llama-pack.ts +++ b/helpers/llama-pack.ts @@ -1,29 +1,71 @@ import fs from "fs/promises"; +import got from "got"; import path from "path"; -import { LLAMA_HUB_FOLDER_PATH, LLAMA_PACK_CONFIG_PATH } from "./constant"; +import { parse } from "smol-toml"; +import { + LLAMA_PACK_FOLDER, + LLAMA_PACK_FOLDER_PATH, + LLAMA_PACK_OWNER, + LLAMA_PACK_REPO, +} from "./constant"; import { copy } from "./copy"; import { templatesDir } from "./dir"; -import { installPythonDependencies } from "./python"; +import { addDependencies, installPythonDependencies } from "./python"; import { getRepoRawContent } from "./repo"; import { InstallTemplateArgs } from "./types"; +const getLlamaPackFolderSHA = async () => { + const url = `https://api.github.com/repos/${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/contents`; + const response = await got(url, { + responseType: "json", + }); + const data = response.body as any[]; + const llamaPackFolder = data.find((item) => item.name === LLAMA_PACK_FOLDER); + return llamaPackFolder.sha; +}; + +const getLLamaPackFolderTree = async ( + sha: string, +): Promise< + Array<{ + path: string; + }> +> => { + const url = `https://api.github.com/repos/${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/git/trees/${sha}?recursive=1`; + const response = await got(url, { + responseType: "json", + }); + return (response.body as any).tree; +}; + export async function getAvailableLlamapackOptions(): Promise< { name: string; folderPath: string; - example: boolean | undefined; }[] > { - const libraryJsonRaw = await getRepoRawContent(LLAMA_PACK_CONFIG_PATH); - const libraryJson = JSON.parse(libraryJsonRaw); - const llamapackKeys = Object.keys(libraryJson); - return llamapackKeys - .map((key) => ({ - name: key, - folderPath: libraryJson[key].id, - example: libraryJson[key].example, - })) - .filter((item) => !!item.example); + const EXAMPLE_RELATIVE_PATH = "/examples/example.py"; + const PACK_FOLDER_SUBFIX = "llama-index-packs"; + + const llamaPackFolderSHA = await getLlamaPackFolderSHA(); + const llamaPackTree = await getLLamaPackFolderTree(llamaPackFolderSHA); + + // Return options that have example files + const exampleFiles = llamaPackTree.filter((item) => + item.path.endsWith(EXAMPLE_RELATIVE_PATH), + ); + const options = exampleFiles.map((file) => { + const packFolder = file.path.substring( + 0, + file.path.indexOf(EXAMPLE_RELATIVE_PATH), + ); + const packName = packFolder.substring(PACK_FOLDER_SUBFIX.length + 1); + return { + name: packName, + folderPath: packFolder, + }; + }); + return options; } const copyLlamapackEmptyProject = async ({ @@ -55,8 +97,10 @@ const installLlamapackExample = async ({ }: Pick<InstallTemplateArgs, "root" | "llamapack">) => { const exampleFileName = "example.py"; const readmeFileName = "README.md"; - const exampleFilePath = `${LLAMA_HUB_FOLDER_PATH}/${llamapack}/${exampleFileName}`; - const readmeFilePath = `${LLAMA_HUB_FOLDER_PATH}/${llamapack}/${readmeFileName}`; + const projectTomlFileName = "pyproject.toml"; + const exampleFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/examples/${exampleFileName}`; + const readmeFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/${readmeFileName}`; + const projectTomlFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/${projectTomlFileName}`; // Download example.py from llamapack and save to root const exampleContent = await getRepoRawContent(exampleFilePath); @@ -74,6 +118,19 @@ const installLlamapackExample = async ({ `${readmeContent}\n${readmeTemplateContent}`, ); await fs.unlink(path.join(root, "README-template.md")); + + // Download pyproject.toml from llamapack, parse it to get package name and version, + // then add it as a dependency to current toml file in the project + const projectTomlContent = await getRepoRawContent(projectTomlFilePath); + const fileParsed = parse(projectTomlContent) as any; + const packageName = fileParsed.tool.poetry.name; + const packageVersion = fileParsed.tool.poetry.version; + await addDependencies(root, [ + { + name: packageName, + version: packageVersion, + }, + ]); }; export const installLlamapackProject = async ({ diff --git a/templates/components/sample-projects/llamapack/pyproject.toml b/templates/components/sample-projects/llamapack/pyproject.toml index 5e1934ca92f1e703c480931b102eef7ca8a4b09f..4bd28bd809d189b8f6d505459c0ab14a2a2f0b7f 100644 --- a/templates/components/sample-projects/llamapack/pyproject.toml +++ b/templates/components/sample-projects/llamapack/pyproject.toml @@ -7,7 +7,8 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.11,<3.12" -llama-index = "^0.9.19" +llama-index = "^0.10.6" +llama-index-readers-file = "^0.1.3" python-dotenv = "^1.0.0"