diff --git a/packages/create-llama/e2e/basic.spec.ts b/packages/create-llama/e2e/basic.spec.ts
index 36b8d793f6687fa21e826900eea365ac0b2cb815..dbaeb4b9b718f5770e58b919ec1ef44b6a602bb3 100644
--- a/packages/create-llama/e2e/basic.spec.ts
+++ b/packages/create-llama/e2e/basic.spec.ts
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { expect, test } from "@playwright/test";
-import { exec } from "child_process";
+import { ChildProcess, exec } from "child_process";
import { execSync } from "node:child_process";
import crypto from "node:crypto";
import { mkdir } from "node:fs/promises";
@@ -21,109 +21,149 @@ test.beforeEach(async () => {
await mkdir(cwd, { recursive: true });
});
-const templateTypes: TemplateType[] = ["streaming", "simple"];
-const templateFrameworks: TemplateFramework[] = ["nextjs", "express"];
-const templateEngines: TemplateEngine[] = ["simple", "context"];
-const templateUIs: TemplateUI[] = ["shadcn", "html"];
+type AppType = "--frontend" | "--no-frontend" | "";
+
+const templateTypes: TemplateType[] = ["streaming"];
+const templateFrameworks: TemplateFramework[] = ["express"];
+const templateEngines: TemplateEngine[] = ["simple"];
+const templateUIs: TemplateUI[] = ["html"];
+const MODEL = "gpt-3.5-turbo";
for (const templateType of templateTypes) {
for (const templateFramework of templateFrameworks) {
for (const templateEngine of templateEngines) {
for (const templateUI of templateUIs) {
- const shouldGenerateFrontendEnum =
+ const appType: AppType =
templateFramework === "express" || templateFramework === "fastapi"
- ? ["--frontend", "--no-frontend"]
- : [""];
- for (const shouldGenerateFrontend of shouldGenerateFrontendEnum) {
- if (templateEngine === "context") {
- // we don't test context templates because it needs OPEN_AI_KEY
- continue;
- }
- test(`try create-llama ${templateType} ${templateFramework} ${templateEngine} ${templateUI} ${shouldGenerateFrontend}`, async ({
- page,
- }) => {
- const createLlama = fileURLToPath(
- new URL("../dist/index.js", import.meta.url),
- );
+ ? "--frontend"
+ : "";
+ if (templateEngine === "context") {
+ // we don't test context templates because it needs OPEN_AI_KEY
+ continue;
+ }
+ test(`try create-llama ${templateType} ${templateFramework} ${templateEngine} ${templateUI} ${appType}`, async ({
+ page,
+ }) => {
+ const name = runCreateLlama(
+ templateType,
+ templateFramework,
+ templateEngine,
+ templateUI,
+ appType,
+ );
- const name = [
- templateType,
- templateFramework,
- templateEngine,
- templateUI,
- shouldGenerateFrontend,
- ].join("-");
- const command = [
- "node",
- createLlama,
- name,
- "--template",
- templateType,
- "--framework",
- templateFramework,
- "--engine",
- templateEngine,
- "--ui",
- templateUI,
- "--open-ai-key",
- process.env.OPEN_AI_KEY || "",
- shouldGenerateFrontend,
- "--eslint",
- ].join(" ");
- console.log(`running command '${command}' in ${cwd}`);
- execSync(command, {
- stdio: "inherit",
- cwd,
- });
+ const port = Math.floor(Math.random() * 10000) + 10000;
+ const cps = await runApp(name, appType, port);
+
+ // test frontend
+ await page.goto(`http://localhost:${port}`);
+ await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
+ // TODO: test backend using curl (would need OpenAI key)
+ // clean processes
+ cps.forEach((cp) => cp.kill());
+ });
+ }
+ }
+ }
+}
- const port = Math.floor(Math.random() * 10000) + 10000;
+async function runApp(
+ name: string,
+ appType: AppType,
+ port: number,
+): Promise<ChildProcess[]> {
+ const cps = [];
- if (
- shouldGenerateFrontend === "--frontend" &&
- templateFramework === "express"
- ) {
- execSync("npm install", {
- stdio: "inherit",
- cwd: `${cwd}/${name}/frontend`,
- });
- execSync("npm install", {
- stdio: "inherit",
- cwd: `${cwd}/${name}/backend`,
- });
- } else {
- execSync("npm install", {
- stdio: "inherit",
- cwd: `${cwd}/${name}`,
- });
- }
+ try {
+ switch (appType) {
+ case "--no-frontend":
+ cps.push(
+ await createProcess("npm run dev", `${cwd}/${name}/backend`, port),
+ );
+ break;
+ case "--frontend":
+ cps.push(
+ await createProcess(
+ "npm run dev",
+ `${cwd}/${name}/backend`,
+ port + 1,
+ ),
+ );
+ cps.push(
+ await createProcess("npm run dev", `${cwd}/${name}/frontend`, port),
+ );
+ break;
+ default:
+ cps.push(await createProcess("npm run dev", `${cwd}/${name}`, port));
+ break;
+ }
+ } catch (e) {
+ cps.forEach((cp) => cp.kill());
+ throw e;
+ }
+ return cps;
+}
- if (shouldGenerateFrontend === "--no-frontend") {
- return;
- }
+async function createProcess(command: string, cwd: string, port: number) {
+ const cp = exec(command, {
+ cwd,
+ env: {
+ ...process.env,
+ PORT: `${port}`,
+ },
+ });
+ if (!cp) throw new Error(`Can't start process ${command} in ${cwd}`);
- const cp = exec("npm run dev", {
- cwd:
- shouldGenerateFrontend === "--frontend"
- ? `${cwd}/${name}/frontend`
- : `${cwd}/${name}`,
- env: {
- ...process.env,
- PORT: `${port}`,
- },
- });
+ await waitPort({
+ host: "localhost",
+ port,
+ timeout: 1000 * 60,
+ });
+ return cp;
+}
- await waitPort({
- host: "localhost",
- port,
- timeout: 1000 * 60,
- });
+function runCreateLlama(
+ templateType: string,
+ templateFramework: string,
+ templateEngine: string,
+ templateUI: string,
+ appType: AppType,
+) {
+ const createLlama = fileURLToPath(
+ new URL("../dist/index.js", import.meta.url),
+ );
- await page.goto(`http://localhost:${port}`);
- await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
- cp.kill();
- });
- }
- }
- }
- }
+ const name = [
+ templateType,
+ templateFramework,
+ templateEngine,
+ templateUI,
+ appType,
+ ].join("-");
+ const command = [
+ "node",
+ createLlama,
+ name,
+ "--template",
+ templateType,
+ "--framework",
+ templateFramework,
+ "--engine",
+ templateEngine,
+ "--ui",
+ templateUI,
+ "--model",
+ MODEL,
+ "--open-ai-key",
+ "testKey", // TODO: pass in OPEN_AI_KEY from CI env if needed for tests
+ appType,
+ "--eslint",
+ "--use-npm",
+ ].join(" ");
+ console.log(`running command '${command}' in ${cwd}`);
+ execSync(command, {
+ stdio: "inherit",
+ cwd,
+ });
+ return name;
}
diff --git a/packages/create-llama/package.json b/packages/create-llama/package.json
index f0079ecbc156fd6175317c46de6bf7d96bfc1c23..3d77e5f2a539c10e79df5bc7c0ba14e00ebf5d17 100644
--- a/packages/create-llama/package.json
+++ b/packages/create-llama/package.json
@@ -24,7 +24,7 @@
"dev": "ncc build ./index.ts -w -o dist/",
"build": "ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
"lint": "eslint . --ignore-pattern dist",
- "e2e": "playwright test",
+ "e2e": "playwright test --reporter=list",
"prepublishOnly": "cd ../../ && turbo run build"
},
"devDependencies": {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 079de736ae46d5bb6af79677bb4773e12547620a..33f5341147c4bc71214789168b9adbd9de3126b5 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -240,6 +240,9 @@ importers:
packages/create-llama:
devDependencies:
+ '@playwright/test':
+ specifier: ^1.40.0
+ version: 1.40.0
'@types/async-retry':
specifier: 1.4.2
version: 1.4.2
@@ -306,6 +309,9 @@ importers:
validate-npm-package-name:
specifier: 3.0.0
version: 3.0.0
+ wait-port:
+ specifier: ^1.1.0
+ version: 1.1.0
packages/eslint-config-custom:
dependencies:
@@ -3903,6 +3909,14 @@ packages:
tslib: 2.6.1
dev: false
+ /@playwright/test@1.40.0:
+ resolution: {integrity: sha512-PdW+kn4eV99iP5gxWNSDQCbhMaDVej+RXL5xr6t04nbKLCBwYtA046t7ofoczHOm8u6c+45hpDKQVZqtqwkeQg==}
+ engines: {node: '>=16'}
+ hasBin: true
+ dependencies:
+ playwright: 1.40.0
+ dev: true
+
/@polka/url@1.0.0-next.23:
resolution: {integrity: sha512-C16M+IYz0rgRhWZdCmK+h58JMv8vijAA61gmz2rspCSwKwzBebpdcsiUmwrtJRdphuY30i6BSLEOP8ppbNLyLg==}
dev: false
@@ -6326,6 +6340,11 @@ packages:
engines: {node: '>= 12'}
dev: false
+ /commander@9.5.0:
+ resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==}
+ engines: {node: ^12.20.0 || >=14}
+ dev: true
+
/commondir@1.0.1:
resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==}
dev: false
@@ -8559,6 +8578,14 @@ packages:
/fs.realpath@1.0.0:
resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
+ /fsevents@2.3.2:
+ resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
+ engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
+ os: [darwin]
+ requiresBuild: true
+ dev: true
+ optional: true
+
/fsevents@2.3.3:
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@@ -12147,6 +12174,22 @@ packages:
resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
dev: false
+ /playwright-core@1.40.0:
+ resolution: {integrity: sha512-fvKewVJpGeca8t0ipM56jkVSU6Eo0RmFvQ/MaCQNDYm+sdvKkMBBWTE1FdeMqIdumRaXXjZChWHvIzCGM/tA/Q==}
+ engines: {node: '>=16'}
+ hasBin: true
+ dev: true
+
+ /playwright@1.40.0:
+ resolution: {integrity: sha512-gyHAgQjiDf1m34Xpwzaqb76KgfzYrhK7iih+2IzcOCoZWr/8ZqmdBw+t0RU85ZmfJMgtgAiNtBQ/KS2325INXw==}
+ engines: {node: '>=16'}
+ hasBin: true
+ dependencies:
+ playwright-core: 1.40.0
+ optionalDependencies:
+ fsevents: 2.3.2
+ dev: true
+
/portkey-ai@0.1.16:
resolution: {integrity: sha512-EY4FRp6PZSD75Q1o1qc08DfPNTG9FnkUPN3Z1/lEvaq9iFpSO5UekcagUZaKSVhao311qjBjns+kF0rS9ht7iA==}
dependencies:
@@ -15655,6 +15698,18 @@ packages:
- debug
dev: false
+ /wait-port@1.1.0:
+ resolution: {integrity: sha512-3e04qkoN3LxTMLakdqeWth8nih8usyg+sf1Bgdf9wwUkp05iuK1eSY/QpLvscT/+F/gA89+LpUmmgBtesbqI2Q==}
+ engines: {node: '>=10'}
+ hasBin: true
+ dependencies:
+ chalk: 4.1.2
+ commander: 9.5.0
+ debug: 4.3.4
+ transitivePeerDependencies:
+ - supports-color
+ dev: true
+
/walker@1.0.8:
resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==}
dependencies: