fix: remove redundant template engine variable and fixed llamaparse flag (#22)

create-app.ts

@@ -26,7 +26,6 @@ export type InstallAppArgs = Omit<
 export async function createApp({
   template,
   framework,
-  engine,
   ui,
   appPath,
   packageManager,
@@ -76,7 +75,6 @@ export async function createApp({
     root,
     template,
     framework,
-    engine,
     ui,
     packageManager,
     isOnline,

e2e/basic.spec.ts

@@ -4,7 +4,6 @@ import { ChildProcess } from "child_process";
 import fs from "fs";
 import path from "path";
 import type {
-  TemplateEngine,
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateType,
@@ -18,7 +17,7 @@ const templateFrameworks: TemplateFramework[] = [
   "express",
   "fastapi",
 ];
-const templateEngines: TemplateEngine[] = ["simple", "context"];
+const dataSources: string[] = ["--no-files", "--example-file"];
 const templateUIs: TemplateUI[] = ["shadcn", "html"];
 const templatePostInstallActions: TemplatePostInstallAction[] = [
   "none",
@@ -27,12 +26,12 @@ const templatePostInstallActions: TemplatePostInstallAction[] = [
 
 for (const templateType of templateTypes) {
   for (const templateFramework of templateFrameworks) {
-    for (const templateEngine of templateEngines) {
+    for (const dataSource of dataSources) {
       for (const templateUI of templateUIs) {
         for (const templatePostInstallAction of templatePostInstallActions) {
           const appType: AppType =
             templateFramework === "nextjs" ? "" : "--frontend";
-          test.describe(`try create-llama ${templateType} ${templateFramework} ${templateEngine} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+          test.describe(`try create-llama ${templateType} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
             let port: number;
             let externalPort: number;
             let cwd: string;
@@ -49,7 +48,7 @@ for (const templateType of templateTypes) {
                 cwd,
                 templateType,
                 templateFramework,
-                templateEngine,
+                dataSource,
                 templateUI,
                 vectorDb,
                 appType,

e2e/utils.ts

@@ -4,7 +4,6 @@ import { mkdir } from "node:fs/promises";
 import * as path from "path";
 import waitPort from "wait-port";
 import {
-  TemplateEngine,
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateType,
@@ -67,7 +66,7 @@ export async function runCreateLlama(
   cwd: string,
   templateType: TemplateType,
   templateFramework: TemplateFramework,
-  templateEngine: TemplateEngine,
+  dataSource: string,
   templateUI: TemplateUI,
   vectorDb: TemplateVectorDB,
   appType: AppType,
@@ -75,10 +74,13 @@ export async function runCreateLlama(
   externalPort: number,
   postInstallAction: TemplatePostInstallAction,
 ): Promise<CreateLlamaResult> {
+  if (!process.env.OPENAI_API_KEY) {
+    throw new Error("Setting OPENAI_API_KEY is mandatory to run tests");
+  }
   const name = [
     templateType,
     templateFramework,
-    templateEngine,
+    dataSource,
     templateUI,
     appType,
   ].join("-");
@@ -89,8 +91,7 @@ export async function runCreateLlama(
     templateType,
     "--framework",
     templateFramework,
-    "--engine",
-    templateEngine,
+    dataSource,
     "--ui",
     templateUI,
     "--vector-db",
@@ -100,7 +101,7 @@ export async function runCreateLlama(
     "--embedding-model",
     EMBEDDING_MODEL,
     "--open-ai-key",
-    process.env.OPENAI_API_KEY || "testKey",
+    process.env.OPENAI_API_KEY,
     appType,
     "--eslint",
     "--use-pnpm",

helpers/datasources.ts

+import { TemplateDataSource } from "./types";
+
+// Example file has an empty config
+export const EXAMPLE_FILE: TemplateDataSource = {
+  type: "file",
+  config: {},
+};
+
+export function getDataSources(
+  files?: string,
+  exampleFile?: boolean,
+): TemplateDataSource[] | undefined {
+  let dataSources: TemplateDataSource[] | undefined = undefined;
+  if (files) {
+    // If user specified files option, then the program should use context engine
+    dataSources = files.split(",").map((filePath) => ({
+      type: "file",
+      config: {
+        path: filePath,
+      },
+    }));
+  }
+  if (exampleFile) {
+    dataSources = [...(dataSources ? dataSources : []), EXAMPLE_FILE];
+  }
+  return dataSources;
+}

helpers/index.ts

@@ -137,7 +137,7 @@ export const installTemplate = async (
       port: props.externalPort,
     });
 
-    if (props.engine === "context") {
+    if (props.dataSources.length > 0) {
       console.log("\nGenerating context data...\n");
       await copyContextData(props.root, props.dataSources);
       if (

helpers/python.ts

@@ -177,7 +177,6 @@ export const installPythonTemplate = async ({
   root,
   template,
   framework,
-  engine,
   vectorDb,
   dataSources,
   tools,
@@ -188,7 +187,6 @@ export const installPythonTemplate = async ({
   | "root"
   | "framework"
   | "template"
-  | "engine"
   | "vectorDb"
   | "dataSources"
   | "tools"
@@ -217,7 +215,7 @@ export const installPythonTemplate = async ({
     },
   });
 
-  if (engine === "context") {
+  if (dataSources.length > 0) {
     const enginePath = path.join(root, "app", "engine");
     const compPath = path.join(templatesDir, "components");
 
@@ -257,46 +255,44 @@ export const installPythonTemplate = async ({
       });
     }
 
-    if (dataSources.length > 0) {
-      const loaderConfigs: Record<string, any> = {};
-      const loaderPath = path.join(enginePath, "loaders");
+    const loaderConfigs: Record<string, any> = {};
+    const loaderPath = path.join(enginePath, "loaders");
 
-      // Copy loaders to enginePath
-      await copy("**", loaderPath, {
-        parents: true,
-        cwd: path.join(compPath, "loaders", "python"),
-      });
+    // Copy loaders to enginePath
+    await copy("**", loaderPath, {
+      parents: true,
+      cwd: path.join(compPath, "loaders", "python"),
+    });
 
-      // Generate loaders config
-      // Web loader config
-      if (dataSources.some((ds) => ds.type === "web")) {
-        const webLoaderConfig = dataSources
-          .filter((ds) => ds.type === "web")
-          .map((ds) => {
-            const dsConfig = ds.config as WebSourceConfig;
-            return {
-              base_url: dsConfig.baseUrl,
-              prefix: dsConfig.prefix,
-              depth: dsConfig.depth,
-            };
-          });
-        loaderConfigs["web"] = webLoaderConfig;
-      }
-      // File loader config
-      if (dataSources.some((ds) => ds.type === "file")) {
-        loaderConfigs["file"] = {
-          use_llama_parse: useLlamaParse,
-        };
-      }
-      // Write loaders config
-      if (Object.keys(loaderConfigs).length > 0) {
-        const loaderConfigPath = path.join(root, "config/loaders.json");
-        await fs.mkdir(path.join(root, "config"), { recursive: true });
-        await fs.writeFile(
-          loaderConfigPath,
-          JSON.stringify(loaderConfigs, null, 2),
-        );
-      }
+    // Generate loaders config
+    // Web loader config
+    if (dataSources.some((ds) => ds.type === "web")) {
+      const webLoaderConfig = dataSources
+        .filter((ds) => ds.type === "web")
+        .map((ds) => {
+          const dsConfig = ds.config as WebSourceConfig;
+          return {
+            base_url: dsConfig.baseUrl,
+            prefix: dsConfig.prefix,
+            depth: dsConfig.depth,
+          };
+        });
+      loaderConfigs["web"] = webLoaderConfig;
+    }
+    // File loader config
+    if (dataSources.some((ds) => ds.type === "file")) {
+      loaderConfigs["file"] = {
+        use_llama_parse: useLlamaParse,
+      };
+    }
+    // Write loaders config
+    if (Object.keys(loaderConfigs).length > 0) {
+      const loaderConfigPath = path.join(root, "config/loaders.json");
+      await fs.mkdir(path.join(root, "config"), { recursive: true });
+      await fs.writeFile(
+        loaderConfigPath,
+        JSON.stringify(loaderConfigs, null, 2),
+      );
     }
   }
 

helpers/types.ts

@@ -3,7 +3,6 @@ import { Tool } from "./tools";
 
 export type TemplateType = "streaming" | "community" | "llamapack";
 export type TemplateFramework = "nextjs" | "express" | "fastapi";
-export type TemplateEngine = "simple" | "context";
 export type TemplateUI = "html" | "shadcn";
 export type TemplateVectorDB = "none" | "mongo" | "pg" | "pinecone" | "milvus";
 export type TemplatePostInstallAction =
@@ -43,7 +42,6 @@ export interface InstallTemplateArgs {
   isOnline: boolean;
   template: TemplateType;
   framework: TemplateFramework;
-  engine: TemplateEngine;
   ui: TemplateUI;
   dataSources: TemplateDataSource[];
   eslint: boolean;

helpers/typescript.ts

@@ -56,7 +56,6 @@ export const installTSTemplate = async ({
   isOnline,
   template,
   framework,
-  engine,
   ui,
   eslint,
   customApiPath,
@@ -142,50 +141,41 @@ export const installTSTemplate = async ({
   /**
    * Copy the selected chat engine files to the target directory and reference it.
    */
-  let relativeEngineDestPath;
   const compPath = path.join(templatesDir, "components");
-  if (engine && (framework === "express" || framework === "nextjs")) {
-    console.log("\nUsing chat engine:", engine, "\n");
+  const relativeEngineDestPath =
+    framework === "nextjs"
+      ? path.join("app", "api", "chat")
+      : path.join("src", "controllers");
+  const enginePath = path.join(root, relativeEngineDestPath, "engine");
 
-    let vectorDBFolder: string = engine;
-
-    if (engine !== "simple" && vectorDb) {
+  if (dataSources.length === 0) {
+    // use simple hat engine if user neither select tools nor a data source
+    console.log("\nUsing simple chat engine\n");
+  } else {
+    if (vectorDb) {
+      // copy vector db component
       console.log("\nUsing vector DB:", vectorDb, "\n");
-      vectorDBFolder = vectorDb;
-    }
-
-    relativeEngineDestPath =
-      framework === "nextjs"
-        ? path.join("app", "api", "chat")
-        : path.join("src", "controllers");
-
-    const enginePath = path.join(root, relativeEngineDestPath, "engine");
-
-    // copy vector db component
-    const vectorDBPath = path.join(
-      compPath,
-      "vectordbs",
-      "typescript",
-      vectorDBFolder,
-    );
-    await copy("**", enginePath, {
-      parents: true,
-      cwd: vectorDBPath,
-    });
-
-    // copy loader component
-    const dataSourceType = dataSources[0]?.type;
-    if (dataSourceType) {
-      let loaderFolder: string;
-      loaderFolder = useLlamaParse ? "llama_parse" : dataSourceType;
+      const vectorDBPath = path.join(
+        compPath,
+        "vectordbs",
+        "typescript",
+        vectorDb,
+      );
       await copy("**", enginePath, {
         parents: true,
-        cwd: path.join(compPath, "loaders", "typescript", loaderFolder),
+        cwd: vectorDBPath,
       });
     }
-
-    // copy tools component
+    // copy loader component (TS only supports llama_parse and file for now)
+    let loaderFolder: string;
+    loaderFolder = useLlamaParse ? "llama_parse" : "file";
+    await copy("**", enginePath, {
+      parents: true,
+      cwd: path.join(compPath, "loaders", "typescript", loaderFolder),
+    });
     if (tools?.length) {
+      // use agent chat engine if user selects tools
+      console.log("\nUsing agent chat engine\n");
       await copy("**", enginePath, {
         parents: true,
         cwd: path.join(compPath, "engines", "typescript", "agent"),
@@ -201,7 +191,9 @@ export const installTSTemplate = async ({
         configFilePath,
         JSON.stringify(configContent, null, 2),
       );
-    } else if (engine !== "simple") {
+    } else {
+      // use context chat engine if user does not select tools
+      console.log("\nUsing context chat engine\n");
       await copy("**", enginePath, {
         parents: true,
         cwd: path.join(compPath, "engines", "typescript", "chat"),
@@ -248,7 +240,7 @@ export const installTSTemplate = async ({
     // modify the dev script to use the custom api path
   }
 
-  if (engine === "context" && relativeEngineDestPath) {
+  if (dataSources.length > 0 && relativeEngineDestPath) {
     // add generate script if using context engine
     packageJson.scripts = {
       ...packageJson.scripts,

index.ts

@@ -9,6 +9,7 @@ import prompts from "prompts";
 import terminalLink from "terminal-link";
 import checkForUpdate from "update-check";
 import { createApp } from "./create-app";
+import { getDataSources } from "./helpers/datasources";
 import { getPkgManager } from "./helpers/get-pkg-manager";
 import { isFolderEmpty } from "./helpers/is-folder-empty";
 import { runApp } from "./helpers/run-app";
@@ -71,13 +72,6 @@ const program = new Commander.Command(packageJson.name)
     `
 
   Select a template to bootstrap the application with.
-`,
-  )
-  .option(
-    "--engine <engine>",
-    `
-
-  Select a chat engine to bootstrap the application with.
 `,
   )
   .option(
@@ -92,6 +86,13 @@ const program = new Commander.Command(packageJson.name)
     `
   
     Specify the path to a local file or folder for chatting.
+`,
+  )
+  .option(
+    "--example-file",
+    `
+
+  Select to use an example PDF as data source.
 `,
   )
   .option(
@@ -164,7 +165,7 @@ const program = new Commander.Command(packageJson.name)
 `,
   )
   .option(
-    "--llama-parse",
+    "--use-llama-parse",
     `
     Enable LlamaParse.
 `,
@@ -199,7 +200,12 @@ if (process.argv.includes("--tools")) {
   }
 }
 if (process.argv.includes("--no-llama-parse")) {
-  program.llamaParse = false;
+  program.useLlamaParse = false;
+}
+if (process.argv.includes("--no-files")) {
+  program.dataSources = [];
+} else {
+  program.dataSources = getDataSources(program.files, program.exampleFile);
 }
 
 const packageManager = !!program.useNpm
@@ -287,7 +293,6 @@ async function run(): Promise<void> {
   await createApp({
     template: program.template,
     framework: program.framework,
-    engine: program.engine,
     ui: program.ui,
     appPath: resolvedProjectPath,
     packageManager,

package.json

@@ -25,6 +25,7 @@
     "format:write": "prettier --ignore-unknown --write .",
     "dev": "ncc build ./index.ts -w -o dist/",
     "build": "bash ./scripts/build.sh",
+    "build:ncc": "pnpm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
     "lint": "eslint . --ignore-pattern dist --ignore-pattern e2e/cache",
     "e2e": "playwright test",
     "prepare": "husky",

questions.ts

@@ -13,6 +13,7 @@ import {
   TemplateFramework,
 } from "./helpers";
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
+import { EXAMPLE_FILE } from "./helpers/datasources";
 import { templatesDir } from "./helpers/dir";
 import { getAvailableLlamapackOptions } from "./helpers/llama-pack";
 import { getProjectOptions } from "./helpers/repo";
@@ -24,7 +25,6 @@ export type QuestionArgs = Omit<
   InstallAppArgs,
   "appPath" | "packageManager"
 > & {
-  files?: string;
   listServerModels?: boolean;
 };
 const supportedContextFileTypes = [
@@ -70,7 +70,6 @@ if ($dialogResult -eq [System.Windows.Forms.DialogResult]::OK)
 const defaults: QuestionArgs = {
   template: "streaming",
   framework: "nextjs",
-  engine: "simple",
   ui: "html",
   eslint: true,
   frontend: false,
@@ -621,25 +620,12 @@ export const askQuestions = async (
     }
   }
 
-  if (program.files) {
-    // If user specified files option, then the program should use context engine
-    program.engine = "context";
-    program.files.split(",").forEach((filePath) => {
-      program.dataSources.push({
-        type: "file",
-        config: {
-          path: filePath,
-        },
-      });
-    });
-  }
-
-  if (!program.engine) {
+  if (!program.dataSources) {
     if (ciInfo.isCI) {
-      program.engine = getPrefOrDefault("engine");
       program.dataSources = getPrefOrDefault("dataSources");
     } else {
       program.dataSources = [];
+      // continue asking user for data sources if none are initially provided
       while (true) {
         const { selectedSource } = await prompts(
           {
@@ -658,22 +644,12 @@ export const askQuestions = async (
           handlers,
         );
 
-        if (selectedSource === "no") {
+        if (selectedSource === "no" || selectedSource === "none") {
+          // user doesn't want another data source or any data source
           break;
         }
-
-        if (selectedSource === "none") {
-          // Selected simple chat
-          program.dataSources = [];
-          // Stop asking for another data source
-          break;
-        }
-
         if (selectedSource === "exampleFile") {
-          program.dataSources.push({
-            type: "file",
-            config: {},
-          });
+          program.dataSources.push(EXAMPLE_FILE);
         } else if (selectedSource === "file" || selectedSource === "folder") {
           // Select local data source
           const selectedPaths = await selectLocalContextData(selectedSource);
@@ -720,31 +696,13 @@ export const askQuestions = async (
           });
         }
       }
-
-      if (program.dataSources.length === 0) {
-        program.engine = "simple";
-      } else {
-        program.engine = "context";
-      }
-    }
-  } else if (!program.dataSources) {
-    // Handle a case when engine is specified but dataSource is not
-    if (program.engine === "context") {
-      program.dataSources = [
-        {
-          type: "file",
-          config: {},
-        },
-      ];
-    } else if (program.engine === "simple") {
-      program.dataSources = [];
     }
   }
 
   // Asking for LlamaParse if user selected file or folder data source
   if (
     program.dataSources.some((ds) => ds.type === "file") &&
-    !program.useLlamaParse
+    program.useLlamaParse === undefined
   ) {
     if (ciInfo.isCI) {
       program.useLlamaParse = getPrefOrDefault("useLlamaParse");
@@ -780,7 +738,7 @@ export const askQuestions = async (
     }
   }
 
-  if (program.engine !== "simple" && !program.vectorDb) {
+  if (program.dataSources.length > 0 && !program.vectorDb) {
     if (ciInfo.isCI) {
       program.vectorDb = getPrefOrDefault("vectorDb");
     } else {
@@ -799,7 +757,8 @@ export const askQuestions = async (
     }
   }
 
-  if (!program.tools && program.engine === "context") {
+  // TODO: allow tools also without datasources
+  if (!program.tools && program.dataSources.length > 0) {
     if (ciInfo.isCI) {
       program.tools = getPrefOrDefault("tools");
     } else {
@@ -845,10 +804,4 @@ export const askQuestions = async (
   }
 
   await askPostInstallAction();
-
-  // TODO: consider using zod to validate the input (doesn't work like this as not every option is required)
-  // templateUISchema.parse(program.ui);
-  // templateEngineSchema.parse(program.engine);
-  // templateFrameworkSchema.parse(program.framework);
-  // templateTypeSchema.parse(program.template);``
 };

scripts/build.sh

 #!/usr/bin/env bash
 
 # build dist/index.js file
-npm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register
+pnpm run build:ncc
 
 # add shebang to the top of dist/index.js
 # XXX: Windows needs a space after `node` to work correctly