reorganize e2e tests (split Python and TS) (#329)

---------
Co-authored-by: leehuwuj <leehuwuj@gmail.com>

.github/workflows/e2e.yml

@@ -9,8 +9,75 @@ env:
   POETRY_VERSION: "1.6.1"
 
 jobs:
-  e2e:
-    name: create-llama
+  e2e-python:
+    name: python
+    timeout-minutes: 60
+    strategy:
+      fail-fast: true
+      matrix:
+        node-version: [20]
+        python-version: ["3.11"]
+        os: [macos-latest, windows-latest, ubuntu-22.04]
+        frameworks: ["fastapi"]
+        datasources: ["--no-files", "--example-file"]
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+
+      - uses: pnpm/action-setup@v3
+
+      - name: Setup Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Install Playwright Browsers
+        run: pnpm exec playwright install --with-deps
+        working-directory: .
+
+      - name: Build create-llama
+        run: pnpm run build
+        working-directory: .
+
+      - name: Install
+        run: pnpm run pack-install
+        working-directory: .
+
+      - name: Run Playwright tests for Python
+        run: pnpm run e2e:python
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
+          FRAMEWORK: ${{ matrix.frameworks }}
+          DATASOURCE: ${{ matrix.datasources }}
+        working-directory: .
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report-python
+          path: ./playwright-report/
+          retention-days: 30
+
+  e2e-typescript:
+    name: typescript
     timeout-minutes: 60
     strategy:
       fail-fast: true
@@ -18,7 +85,7 @@ jobs:
         node-version: [18, 20]
         python-version: ["3.11"]
         os: [macos-latest, windows-latest, ubuntu-22.04]
-        frameworks: ["nextjs", "express", "fastapi"]
+        frameworks: ["nextjs", "express"]
         datasources: ["--no-files", "--example-file"]
     defaults:
       run:
@@ -60,8 +127,8 @@ jobs:
         run: pnpm run pack-install
         working-directory: .
 
-      - name: Run Playwright tests
-        run: pnpm run e2e
+      - name: Run Playwright tests for TypeScript
+        run: pnpm run e2e:typescript
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
@@ -72,6 +139,6 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: playwright-report
+          name: playwright-report-typescript
           path: ./playwright-report/
           retention-days: 30

e2e/resolve_python_dependencies.spec.ts → e2e/python/resolve_dependencies.spec.ts

@@ -3,8 +3,8 @@ import { exec } from "child_process";
 import fs from "fs";
 import path from "path";
 import util from "util";
-import { TemplateFramework, TemplateVectorDB } from "../helpers/types";
-import { createTestDir, runCreateLlama } from "./utils";
+import { TemplateFramework, TemplateVectorDB } from "../../helpers/types";
+import { createTestDir, runCreateLlama } from "../utils";
 
 const execAsync = util.promisify(exec);
 
@@ -16,8 +16,6 @@ const dataSource: string = process.env.DATASOURCE
   : "--example-file";
 
 if (
-  templateFramework == "fastapi" && // test is only relevant for fastapi
-  process.version.startsWith("v20.") && // XXX: Only run for Node.js version 20 (CI matrix will trigger other versions)
   dataSource === "--example-file" // XXX: this test provides its own data source - only trigger it on one data source (usually the CI matrix will trigger multiple data sources)
 ) {
   // vectorDBs, tools, and data source combinations to test
@@ -56,7 +54,7 @@ if (
             const result = await runCreateLlama({
               cwd,
               templateType: "streaming",
-              templateFramework: "fastapi",
+              templateFramework,
               dataSource,
               vectorDb,
               port: 3000, // port

e2e/resolve_ts_dependencies.spec.ts

-import { expect, test } from "@playwright/test";
-import { exec } from "child_process";
-import fs from "fs";
-import path from "path";
-import util from "util";
-import { TemplateFramework, TemplateVectorDB } from "../helpers/types";
-import { createTestDir, runCreateLlama } from "./utils";
-
-const execAsync = util.promisify(exec);
-
-const templateFramework: TemplateFramework = process.env.FRAMEWORK
-  ? (process.env.FRAMEWORK as TemplateFramework)
-  : "nextjs";
-const dataSource: string = process.env.DATASOURCE
-  ? process.env.DATASOURCE
-  : "--example-file";
-
-if (
-  templateFramework == "nextjs" ||
-  templateFramework == "express" // test is only relevant for TS projects
-) {
-  const llamaParseOptions = [true, false];
-  // vectorDBs combinations to test
-  const vectorDbs: TemplateVectorDB[] = [
-    "mongo",
-    "pg",
-    "qdrant",
-    "pinecone",
-    "milvus",
-    "astra",
-    "chroma",
-    "llamacloud",
-    "weaviate",
-  ];
-
-  test.describe("Test resolve TS dependencies", () => {
-    for (const llamaParseOpt of llamaParseOptions) {
-      for (const vectorDb of vectorDbs) {
-        const optionDescription = `vectorDb: ${vectorDb}, dataSource: ${dataSource}, llamaParse: ${llamaParseOpt}`;
-
-        test(`options: ${optionDescription}`, async () => {
-          const cwd = await createTestDir();
-
-          const result = await runCreateLlama({
-            cwd: cwd,
-            templateType: "streaming",
-            templateFramework: templateFramework,
-            dataSource: dataSource,
-            vectorDb: vectorDb,
-            port: 3000,
-            externalPort: 8000,
-            postInstallAction: "none",
-            templateUI: undefined,
-            appType: templateFramework === "nextjs" ? "" : "--no-frontend",
-            llamaCloudProjectName: undefined,
-            llamaCloudIndexName: undefined,
-            tools: undefined,
-            useLlamaParse: llamaParseOpt,
-          });
-          const name = result.projectName;
-
-          // Check if the app folder exists
-          const appDir = path.join(cwd, name);
-          const dirExists = fs.existsSync(appDir);
-          expect(dirExists).toBeTruthy();
-
-          // Install dependencies using pnpm
-          try {
-            const { stderr: installStderr } = await execAsync(
-              "pnpm install --prefer-offline",
-              {
-                cwd: appDir,
-              },
-            );
-            expect(installStderr).toBeFalsy();
-          } catch (error) {
-            console.error("Error installing dependencies:", error);
-            throw error;
-          }
-
-          // Run tsc type check and capture the output
-          try {
-            const { stdout, stderr } = await execAsync(
-              "pnpm exec tsc -b --diagnostics",
-              {
-                cwd: appDir,
-              },
-            );
-            // Check if there's any error output
-            expect(stderr).toBeFalsy();
-
-            // Log the stdout for debugging purposes
-            console.log("TypeScript type-check output:", stdout);
-          } catch (error) {
-            console.error("Error running tsc:", error);
-            throw error;
-          }
-        });
-      }
-    }
-  });
-}

e2e/extractor_template.spec.ts → e2e/shared/extractor_template.spec.ts

@@ -3,8 +3,8 @@ import { expect, test } from "@playwright/test";
 import { ChildProcess } from "child_process";
 import fs from "fs";
 import path from "path";
-import { TemplateFramework } from "../helpers";
-import { createTestDir, runCreateLlama } from "./utils";
+import { TemplateFramework } from "../../helpers";
+import { createTestDir, runCreateLlama } from "../utils";
 
 const templateFramework: TemplateFramework = process.env.FRAMEWORK
   ? (process.env.FRAMEWORK as TemplateFramework)
@@ -16,9 +16,8 @@ const dataSource: string = process.env.DATASOURCE
 // The extractor template currently only works with FastAPI and files (and not on Windows)
 if (
   process.platform !== "win32" &&
-  templateFramework !== "nextjs" &&
-  templateFramework !== "express" &&
-  dataSource !== "--no-files"
+  templateFramework === "fastapi" &&
+  dataSource === "--example-file"
 ) {
   test.describe("Test extractor template", async () => {
     let frontendPort: number;

e2e/multiagent_template.spec.ts → e2e/shared/multiagent_template.spec.ts

@@ -7,8 +7,8 @@ import type {
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateUI,
-} from "../helpers";
-import { createTestDir, runCreateLlama, type AppType } from "./utils";
+} from "../../helpers";
+import { createTestDir, runCreateLlama, type AppType } from "../utils";
 
 const templateFramework: TemplateFramework = process.env.FRAMEWORK
   ? (process.env.FRAMEWORK as TemplateFramework)

e2e/streaming_template.spec.ts → e2e/shared/streaming_template.spec.ts

@@ -7,8 +7,8 @@ import type {
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateUI,
-} from "../helpers";
-import { createTestDir, runCreateLlama, type AppType } from "./utils";
+} from "../../helpers";
+import { createTestDir, runCreateLlama, type AppType } from "../utils";
 
 const templateFramework: TemplateFramework = process.env.FRAMEWORK
   ? (process.env.FRAMEWORK as TemplateFramework)

e2e/typescript/resolve_dependencies.spec.ts

+import { expect, test } from "@playwright/test";
+import { exec } from "child_process";
+import fs from "fs";
+import path from "path";
+import util from "util";
+import { TemplateFramework, TemplateVectorDB } from "../../helpers/types";
+import { createTestDir, runCreateLlama } from "../utils";
+
+const execAsync = util.promisify(exec);
+
+const templateFramework: TemplateFramework = process.env.FRAMEWORK
+  ? (process.env.FRAMEWORK as TemplateFramework)
+  : "nextjs";
+const dataSource: string = process.env.DATASOURCE
+  ? process.env.DATASOURCE
+  : "--example-file";
+
+// vectorDBs combinations to test
+const vectorDbs: TemplateVectorDB[] = [
+  "mongo",
+  "pg",
+  "qdrant",
+  "pinecone",
+  "milvus",
+  "astra",
+  "chroma",
+  "llamacloud",
+  "weaviate",
+];
+
+test.describe("Test resolve TS dependencies", () => {
+  // Test vector DBs without LlamaParse
+  for (const vectorDb of vectorDbs) {
+    const optionDescription = `vectorDb: ${vectorDb}, dataSource: ${dataSource}`;
+
+    test(`Vector DB test - ${optionDescription}`, async () => {
+      await runTest(vectorDb, false);
+    });
+  }
+
+  // Test LlamaParse with vectorDB 'none'
+  test(`LlamaParse test - vectorDb: none, dataSource: ${dataSource}, llamaParse: true`, async () => {
+    await runTest("none", true);
+  });
+
+  async function runTest(
+    vectorDb: TemplateVectorDB | "none",
+    useLlamaParse: boolean,
+  ) {
+    const cwd = await createTestDir();
+
+    const result = await runCreateLlama({
+      cwd: cwd,
+      templateType: "streaming",
+      templateFramework: templateFramework,
+      dataSource: dataSource,
+      vectorDb: vectorDb,
+      port: 3000,
+      externalPort: 8000,
+      postInstallAction: "none",
+      templateUI: undefined,
+      appType: templateFramework === "nextjs" ? "" : "--no-frontend",
+      llamaCloudProjectName: undefined,
+      llamaCloudIndexName: undefined,
+      tools: undefined,
+      useLlamaParse: useLlamaParse,
+    });
+    const name = result.projectName;
+
+    // Check if the app folder exists
+    const appDir = path.join(cwd, name);
+    const dirExists = fs.existsSync(appDir);
+    expect(dirExists).toBeTruthy();
+
+    // Install dependencies using pnpm
+    try {
+      const { stderr: installStderr } = await execAsync(
+        "pnpm install --prefer-offline",
+        {
+          cwd: appDir,
+        },
+      );
+    } catch (error) {
+      console.error("Error installing dependencies:", error);
+      throw error;
+    }
+
+    // Run tsc type check and capture the output
+    try {
+      const { stdout, stderr } = await execAsync(
+        "pnpm exec tsc -b --diagnostics",
+        {
+          cwd: appDir,
+        },
+      );
+      // Check if there's any error output
+      expect(stderr).toBeFalsy();
+
+      // Log the stdout for debugging purposes
+      console.log("TypeScript type-check output:", stdout);
+    } catch (error) {
+      console.error("Error running tsc:", error);
+      throw error;
+    }
+  }
+});

package.json

@@ -25,6 +25,8 @@
     "clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock",
     "dev": "ncc build ./index.ts -w -o dist/",
     "e2e": "playwright test",
+    "e2e:python": "playwright test e2e/shared e2e/python",
+    "e2e:typescript": "playwright test e2e/shared e2e/typescript",
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
     "lint": "eslint . --ignore-pattern dist --ignore-pattern e2e/cache",

templates/types/streaming/express/src/controllers/chat-upload.controller.ts

@@ -14,5 +14,11 @@ export const chatUpload = async (req: Request, res: Response) => {
     });
   }
   const index = await getDataSource(params);
+  if (!index) {
+    return res.status(500).json({
+      error:
+        "StorageContext is empty - call 'npm run generate' to generate the storage first",
+    });
+  }
   return res.status(200).json(await uploadDocument(index, filename, base64));
 };