feat(create-llama-pack): generate llama pack project from llama index (#549)

helpers/constant.ts

 export const COMMUNITY_OWNER = "run-llama";
 export const COMMUNITY_REPO = "create_llama_projects";
 export const LLAMA_PACK_OWNER = "run-llama";
-export const LLAMA_PACK_REPO = "llama-hub";
-export const LLAMA_HUB_FOLDER_PATH = `${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/main/llama_hub`;
-export const LLAMA_PACK_CONFIG_PATH = `${LLAMA_HUB_FOLDER_PATH}/llama_packs/library.json`;
+export const LLAMA_PACK_REPO = "llama_index";
+export const LLAMA_PACK_FOLDER = "llama-index-packs";
+export const LLAMA_PACK_FOLDER_PATH = `${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/main/${LLAMA_PACK_FOLDER}`;

helpers/llama-pack.ts

 import fs from "fs/promises";
+import got from "got";
 import path from "path";
-import { LLAMA_HUB_FOLDER_PATH, LLAMA_PACK_CONFIG_PATH } from "./constant";
+import { parse } from "smol-toml";
+import {
+  LLAMA_PACK_FOLDER,
+  LLAMA_PACK_FOLDER_PATH,
+  LLAMA_PACK_OWNER,
+  LLAMA_PACK_REPO,
+} from "./constant";
 import { copy } from "./copy";
 import { templatesDir } from "./dir";
-import { installPythonDependencies } from "./python";
+import { addDependencies, installPythonDependencies } from "./python";
 import { getRepoRawContent } from "./repo";
 import { InstallTemplateArgs } from "./types";
 
+const getLlamaPackFolderSHA = async () => {
+  const url = `https://api.github.com/repos/${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/contents`;
+  const response = await got(url, {
+    responseType: "json",
+  });
+  const data = response.body as any[];
+  const llamaPackFolder = data.find((item) => item.name === LLAMA_PACK_FOLDER);
+  return llamaPackFolder.sha;
+};
+
+const getLLamaPackFolderTree = async (
+  sha: string,
+): Promise<
+  Array<{
+    path: string;
+  }>
+> => {
+  const url = `https://api.github.com/repos/${LLAMA_PACK_OWNER}/${LLAMA_PACK_REPO}/git/trees/${sha}?recursive=1`;
+  const response = await got(url, {
+    responseType: "json",
+  });
+  return (response.body as any).tree;
+};
+
 export async function getAvailableLlamapackOptions(): Promise<
   {
     name: string;
     folderPath: string;
-    example: boolean | undefined;
   }[]
 > {
-  const libraryJsonRaw = await getRepoRawContent(LLAMA_PACK_CONFIG_PATH);
-  const libraryJson = JSON.parse(libraryJsonRaw);
-  const llamapackKeys = Object.keys(libraryJson);
-  return llamapackKeys
-    .map((key) => ({
-      name: key,
-      folderPath: libraryJson[key].id,
-      example: libraryJson[key].example,
-    }))
-    .filter((item) => !!item.example);
+  const EXAMPLE_RELATIVE_PATH = "/examples/example.py";
+  const PACK_FOLDER_SUBFIX = "llama-index-packs";
+
+  const llamaPackFolderSHA = await getLlamaPackFolderSHA();
+  const llamaPackTree = await getLLamaPackFolderTree(llamaPackFolderSHA);
+
+  // Return options that have example files
+  const exampleFiles = llamaPackTree.filter((item) =>
+    item.path.endsWith(EXAMPLE_RELATIVE_PATH),
+  );
+  const options = exampleFiles.map((file) => {
+    const packFolder = file.path.substring(
+      0,
+      file.path.indexOf(EXAMPLE_RELATIVE_PATH),
+    );
+    const packName = packFolder.substring(PACK_FOLDER_SUBFIX.length + 1);
+    return {
+      name: packName,
+      folderPath: packFolder,
+    };
+  });
+  return options;
 }
 
 const copyLlamapackEmptyProject = async ({
@@ -55,8 +97,10 @@ const installLlamapackExample = async ({
 }: Pick<InstallTemplateArgs, "root" | "llamapack">) => {
   const exampleFileName = "example.py";
   const readmeFileName = "README.md";
-  const exampleFilePath = `${LLAMA_HUB_FOLDER_PATH}/${llamapack}/${exampleFileName}`;
-  const readmeFilePath = `${LLAMA_HUB_FOLDER_PATH}/${llamapack}/${readmeFileName}`;
+  const projectTomlFileName = "pyproject.toml";
+  const exampleFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/examples/${exampleFileName}`;
+  const readmeFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/${readmeFileName}`;
+  const projectTomlFilePath = `${LLAMA_PACK_FOLDER_PATH}/${llamapack}/${projectTomlFileName}`;
 
   // Download example.py from llamapack and save to root
   const exampleContent = await getRepoRawContent(exampleFilePath);
@@ -74,6 +118,19 @@ const installLlamapackExample = async ({
     `${readmeContent}\n${readmeTemplateContent}`,
   );
   await fs.unlink(path.join(root, "README-template.md"));
+
+  // Download pyproject.toml from llamapack, parse it to get package name and version,
+  // then add it as a dependency to current toml file in the project
+  const projectTomlContent = await getRepoRawContent(projectTomlFilePath);
+  const fileParsed = parse(projectTomlContent) as any;
+  const packageName = fileParsed.tool.poetry.name;
+  const packageVersion = fileParsed.tool.poetry.version;
+  await addDependencies(root, [
+    {
+      name: packageName,
+      version: packageVersion,
+    },
+  ]);
 };
 
 export const installLlamapackProject = async ({

templates/components/sample-projects/llamapack/pyproject.toml

@@ -7,7 +7,8 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.11,<3.12"
-llama-index = "^0.9.19"
+llama-index = "^0.10.6"
+llama-index-readers-file = "^0.1.3"
 python-dotenv = "^1.0.0"