From bde3daae084e52c9ee7758fe08996d5bd118fd09 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Tue, 1 Oct 2024 11:50:21 +0700
Subject: [PATCH] reorganize e2e tests (split Python and TS) (#329)
---------
Co-authored-by: leehuwuj <leehuwuj@gmail.com>
---
.github/workflows/e2e.yml | 79 ++++++++++++-
.../resolve_dependencies.spec.ts} | 8 +-
e2e/resolve_ts_dependencies.spec.ts | 102 -----------------
e2e/{ => shared}/extractor_template.spec.ts | 9 +-
e2e/{ => shared}/multiagent_template.spec.ts | 4 +-
e2e/{ => shared}/streaming_template.spec.ts | 4 +-
e2e/typescript/resolve_dependencies.spec.ts | 106 ++++++++++++++++++
package.json | 2 +
.../src/controllers/chat-upload.controller.ts | 6 +
9 files changed, 198 insertions(+), 122 deletions(-)
rename e2e/{resolve_python_dependencies.spec.ts => python/resolve_dependencies.spec.ts} (92%)
delete mode 100644 e2e/resolve_ts_dependencies.spec.ts
rename e2e/{ => shared}/extractor_template.spec.ts (89%)
rename e2e/{ => shared}/multiagent_template.spec.ts (96%)
rename e2e/{ => shared}/streaming_template.spec.ts (97%)
create mode 100644 e2e/typescript/resolve_dependencies.spec.ts
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 332b025b..d99890a6 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -9,8 +9,75 @@ env:
POETRY_VERSION: "1.6.1"
jobs:
- e2e:
- name: create-llama
+ e2e-python:
+ name: python
+ timeout-minutes: 60
+ strategy:
+ fail-fast: true
+ matrix:
+ node-version: [20]
+ python-version: ["3.11"]
+ os: [macos-latest, windows-latest, ubuntu-22.04]
+ frameworks: ["fastapi"]
+ datasources: ["--no-files", "--example-file"]
+ defaults:
+ run:
+ shell: bash
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ version: ${{ env.POETRY_VERSION }}
+
+ - uses: pnpm/action-setup@v3
+
+ - name: Setup Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v4
+ with:
+ node-version: ${{ matrix.node-version }}
+ cache: "pnpm"
+
+ - name: Install dependencies
+ run: pnpm install
+
+ - name: Install Playwright Browsers
+ run: pnpm exec playwright install --with-deps
+ working-directory: .
+
+ - name: Build create-llama
+ run: pnpm run build
+ working-directory: .
+
+ - name: Install
+ run: pnpm run pack-install
+ working-directory: .
+
+ - name: Run Playwright tests for Python
+ run: pnpm run e2e:python
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
+ FRAMEWORK: ${{ matrix.frameworks }}
+ DATASOURCE: ${{ matrix.datasources }}
+ working-directory: .
+
+ - uses: actions/upload-artifact@v3
+ if: always()
+ with:
+ name: playwright-report-python
+ path: ./playwright-report/
+ retention-days: 30
+
+ e2e-typescript:
+ name: typescript
timeout-minutes: 60
strategy:
fail-fast: true
@@ -18,7 +85,7 @@ jobs:
node-version: [18, 20]
python-version: ["3.11"]
os: [macos-latest, windows-latest, ubuntu-22.04]
- frameworks: ["nextjs", "express", "fastapi"]
+ frameworks: ["nextjs", "express"]
datasources: ["--no-files", "--example-file"]
defaults:
run:
@@ -60,8 +127,8 @@ jobs:
run: pnpm run pack-install
working-directory: .
- - name: Run Playwright tests
- run: pnpm run e2e
+ - name: Run Playwright tests for TypeScript
+ run: pnpm run e2e:typescript
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
@@ -72,6 +139,6 @@ jobs:
- uses: actions/upload-artifact@v3
if: always()
with:
- name: playwright-report
+ name: playwright-report-typescript
path: ./playwright-report/
retention-days: 30
diff --git a/e2e/resolve_python_dependencies.spec.ts b/e2e/python/resolve_dependencies.spec.ts
similarity index 92%
rename from e2e/resolve_python_dependencies.spec.ts
rename to e2e/python/resolve_dependencies.spec.ts
index b678a107..96864ac0 100644
--- a/e2e/resolve_python_dependencies.spec.ts
+++ b/e2e/python/resolve_dependencies.spec.ts
@@ -3,8 +3,8 @@ import { exec } from "child_process";
import fs from "fs";
import path from "path";
import util from "util";
-import { TemplateFramework, TemplateVectorDB } from "../helpers/types";
-import { createTestDir, runCreateLlama } from "./utils";
+import { TemplateFramework, TemplateVectorDB } from "../../helpers/types";
+import { createTestDir, runCreateLlama } from "../utils";
const execAsync = util.promisify(exec);
@@ -16,8 +16,6 @@ const dataSource: string = process.env.DATASOURCE
: "--example-file";
if (
- templateFramework == "fastapi" && // test is only relevant for fastapi
- process.version.startsWith("v20.") && // XXX: Only run for Node.js version 20 (CI matrix will trigger other versions)
dataSource === "--example-file" // XXX: this test provides its own data source - only trigger it on one data source (usually the CI matrix will trigger multiple data sources)
) {
// vectorDBs, tools, and data source combinations to test
@@ -56,7 +54,7 @@ if (
const result = await runCreateLlama({
cwd,
templateType: "streaming",
- templateFramework: "fastapi",
+ templateFramework,
dataSource,
vectorDb,
port: 3000, // port
diff --git a/e2e/resolve_ts_dependencies.spec.ts b/e2e/resolve_ts_dependencies.spec.ts
deleted file mode 100644
index 7e775303..00000000
--- a/e2e/resolve_ts_dependencies.spec.ts
+++ /dev/null
@@ -1,102 +0,0 @@
-import { expect, test } from "@playwright/test";
-import { exec } from "child_process";
-import fs from "fs";
-import path from "path";
-import util from "util";
-import { TemplateFramework, TemplateVectorDB } from "../helpers/types";
-import { createTestDir, runCreateLlama } from "./utils";
-
-const execAsync = util.promisify(exec);
-
-const templateFramework: TemplateFramework = process.env.FRAMEWORK
- ? (process.env.FRAMEWORK as TemplateFramework)
- : "nextjs";
-const dataSource: string = process.env.DATASOURCE
- ? process.env.DATASOURCE
- : "--example-file";
-
-if (
- templateFramework == "nextjs" ||
- templateFramework == "express" // test is only relevant for TS projects
-) {
- const llamaParseOptions = [true, false];
- // vectorDBs combinations to test
- const vectorDbs: TemplateVectorDB[] = [
- "mongo",
- "pg",
- "qdrant",
- "pinecone",
- "milvus",
- "astra",
- "chroma",
- "llamacloud",
- "weaviate",
- ];
-
- test.describe("Test resolve TS dependencies", () => {
- for (const llamaParseOpt of llamaParseOptions) {
- for (const vectorDb of vectorDbs) {
- const optionDescription = `vectorDb: ${vectorDb}, dataSource: ${dataSource}, llamaParse: ${llamaParseOpt}`;
-
- test(`options: ${optionDescription}`, async () => {
- const cwd = await createTestDir();
-
- const result = await runCreateLlama({
- cwd: cwd,
- templateType: "streaming",
- templateFramework: templateFramework,
- dataSource: dataSource,
- vectorDb: vectorDb,
- port: 3000,
- externalPort: 8000,
- postInstallAction: "none",
- templateUI: undefined,
- appType: templateFramework === "nextjs" ? "" : "--no-frontend",
- llamaCloudProjectName: undefined,
- llamaCloudIndexName: undefined,
- tools: undefined,
- useLlamaParse: llamaParseOpt,
- });
- const name = result.projectName;
-
- // Check if the app folder exists
- const appDir = path.join(cwd, name);
- const dirExists = fs.existsSync(appDir);
- expect(dirExists).toBeTruthy();
-
- // Install dependencies using pnpm
- try {
- const { stderr: installStderr } = await execAsync(
- "pnpm install --prefer-offline",
- {
- cwd: appDir,
- },
- );
- expect(installStderr).toBeFalsy();
- } catch (error) {
- console.error("Error installing dependencies:", error);
- throw error;
- }
-
- // Run tsc type check and capture the output
- try {
- const { stdout, stderr } = await execAsync(
- "pnpm exec tsc -b --diagnostics",
- {
- cwd: appDir,
- },
- );
- // Check if there's any error output
- expect(stderr).toBeFalsy();
-
- // Log the stdout for debugging purposes
- console.log("TypeScript type-check output:", stdout);
- } catch (error) {
- console.error("Error running tsc:", error);
- throw error;
- }
- });
- }
- }
- });
-}
diff --git a/e2e/extractor_template.spec.ts b/e2e/shared/extractor_template.spec.ts
similarity index 89%
rename from e2e/extractor_template.spec.ts
rename to e2e/shared/extractor_template.spec.ts
index 0818e7c7..643fc6dd 100644
--- a/e2e/extractor_template.spec.ts
+++ b/e2e/shared/extractor_template.spec.ts
@@ -3,8 +3,8 @@ import { expect, test } from "@playwright/test";
import { ChildProcess } from "child_process";
import fs from "fs";
import path from "path";
-import { TemplateFramework } from "../helpers";
-import { createTestDir, runCreateLlama } from "./utils";
+import { TemplateFramework } from "../../helpers";
+import { createTestDir, runCreateLlama } from "../utils";
const templateFramework: TemplateFramework = process.env.FRAMEWORK
? (process.env.FRAMEWORK as TemplateFramework)
@@ -16,9 +16,8 @@ const dataSource: string = process.env.DATASOURCE
// The extractor template currently only works with FastAPI and files (and not on Windows)
if (
process.platform !== "win32" &&
- templateFramework !== "nextjs" &&
- templateFramework !== "express" &&
- dataSource !== "--no-files"
+ templateFramework === "fastapi" &&
+ dataSource === "--example-file"
) {
test.describe("Test extractor template", async () => {
let frontendPort: number;
diff --git a/e2e/multiagent_template.spec.ts b/e2e/shared/multiagent_template.spec.ts
similarity index 96%
rename from e2e/multiagent_template.spec.ts
rename to e2e/shared/multiagent_template.spec.ts
index 061c8a47..d76d320c 100644
--- a/e2e/multiagent_template.spec.ts
+++ b/e2e/shared/multiagent_template.spec.ts
@@ -7,8 +7,8 @@ import type {
TemplateFramework,
TemplatePostInstallAction,
TemplateUI,
-} from "../helpers";
-import { createTestDir, runCreateLlama, type AppType } from "./utils";
+} from "../../helpers";
+import { createTestDir, runCreateLlama, type AppType } from "../utils";
const templateFramework: TemplateFramework = process.env.FRAMEWORK
? (process.env.FRAMEWORK as TemplateFramework)
diff --git a/e2e/streaming_template.spec.ts b/e2e/shared/streaming_template.spec.ts
similarity index 97%
rename from e2e/streaming_template.spec.ts
rename to e2e/shared/streaming_template.spec.ts
index 53eb2318..74c7eb4e 100644
--- a/e2e/streaming_template.spec.ts
+++ b/e2e/shared/streaming_template.spec.ts
@@ -7,8 +7,8 @@ import type {
TemplateFramework,
TemplatePostInstallAction,
TemplateUI,
-} from "../helpers";
-import { createTestDir, runCreateLlama, type AppType } from "./utils";
+} from "../../helpers";
+import { createTestDir, runCreateLlama, type AppType } from "../utils";
const templateFramework: TemplateFramework = process.env.FRAMEWORK
? (process.env.FRAMEWORK as TemplateFramework)
diff --git a/e2e/typescript/resolve_dependencies.spec.ts b/e2e/typescript/resolve_dependencies.spec.ts
new file mode 100644
index 00000000..9ae8aa7a
--- /dev/null
+++ b/e2e/typescript/resolve_dependencies.spec.ts
@@ -0,0 +1,106 @@
+import { expect, test } from "@playwright/test";
+import { exec } from "child_process";
+import fs from "fs";
+import path from "path";
+import util from "util";
+import { TemplateFramework, TemplateVectorDB } from "../../helpers/types";
+import { createTestDir, runCreateLlama } from "../utils";
+
+const execAsync = util.promisify(exec);
+
+const templateFramework: TemplateFramework = process.env.FRAMEWORK
+ ? (process.env.FRAMEWORK as TemplateFramework)
+ : "nextjs";
+const dataSource: string = process.env.DATASOURCE
+ ? process.env.DATASOURCE
+ : "--example-file";
+
+// vectorDBs combinations to test
+const vectorDbs: TemplateVectorDB[] = [
+ "mongo",
+ "pg",
+ "qdrant",
+ "pinecone",
+ "milvus",
+ "astra",
+ "chroma",
+ "llamacloud",
+ "weaviate",
+];
+
+test.describe("Test resolve TS dependencies", () => {
+ // Test vector DBs without LlamaParse
+ for (const vectorDb of vectorDbs) {
+ const optionDescription = `vectorDb: ${vectorDb}, dataSource: ${dataSource}`;
+
+ test(`Vector DB test - ${optionDescription}`, async () => {
+ await runTest(vectorDb, false);
+ });
+ }
+
+ // Test LlamaParse with vectorDB 'none'
+ test(`LlamaParse test - vectorDb: none, dataSource: ${dataSource}, llamaParse: true`, async () => {
+ await runTest("none", true);
+ });
+
+ async function runTest(
+ vectorDb: TemplateVectorDB | "none",
+ useLlamaParse: boolean,
+ ) {
+ const cwd = await createTestDir();
+
+ const result = await runCreateLlama({
+ cwd: cwd,
+ templateType: "streaming",
+ templateFramework: templateFramework,
+ dataSource: dataSource,
+ vectorDb: vectorDb,
+ port: 3000,
+ externalPort: 8000,
+ postInstallAction: "none",
+ templateUI: undefined,
+ appType: templateFramework === "nextjs" ? "" : "--no-frontend",
+ llamaCloudProjectName: undefined,
+ llamaCloudIndexName: undefined,
+ tools: undefined,
+ useLlamaParse: useLlamaParse,
+ });
+ const name = result.projectName;
+
+ // Check if the app folder exists
+ const appDir = path.join(cwd, name);
+ const dirExists = fs.existsSync(appDir);
+ expect(dirExists).toBeTruthy();
+
+ // Install dependencies using pnpm
+ try {
+ const { stderr: installStderr } = await execAsync(
+ "pnpm install --prefer-offline",
+ {
+ cwd: appDir,
+ },
+ );
+ } catch (error) {
+ console.error("Error installing dependencies:", error);
+ throw error;
+ }
+
+ // Run tsc type check and capture the output
+ try {
+ const { stdout, stderr } = await execAsync(
+ "pnpm exec tsc -b --diagnostics",
+ {
+ cwd: appDir,
+ },
+ );
+ // Check if there's any error output
+ expect(stderr).toBeFalsy();
+
+ // Log the stdout for debugging purposes
+ console.log("TypeScript type-check output:", stdout);
+ } catch (error) {
+ console.error("Error running tsc:", error);
+ throw error;
+ }
+ }
+});
diff --git a/package.json b/package.json
index 7491f095..3a98d6f8 100644
--- a/package.json
+++ b/package.json
@@ -25,6 +25,8 @@
"clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock",
"dev": "ncc build ./index.ts -w -o dist/",
"e2e": "playwright test",
+ "e2e:python": "playwright test e2e/shared e2e/python",
+ "e2e:typescript": "playwright test e2e/shared e2e/typescript",
"format": "prettier --ignore-unknown --cache --check .",
"format:write": "prettier --ignore-unknown --write .",
"lint": "eslint . --ignore-pattern dist --ignore-pattern e2e/cache",
diff --git a/templates/types/streaming/express/src/controllers/chat-upload.controller.ts b/templates/types/streaming/express/src/controllers/chat-upload.controller.ts
index 0bd5b431..70dfa2fe 100644
--- a/templates/types/streaming/express/src/controllers/chat-upload.controller.ts
+++ b/templates/types/streaming/express/src/controllers/chat-upload.controller.ts
@@ -14,5 +14,11 @@ export const chatUpload = async (req: Request, res: Response) => {
});
}
const index = await getDataSource(params);
+ if (!index) {
+ return res.status(500).json({
+ error:
+ "StorageContext is empty - call 'npm run generate' to generate the storage first",
+ });
+ }
return res.status(200).json(await uploadDocument(index, filename, base64));
};
--
GitLab