✅ test: add BDD test framework and initial tests with Playwright and Cucumber (#9843)

* try with bdd test * update * update * add workspace * update * fix * ci * ci * fix * update * update * update parallel * update config * ⚡️ perf: increase e2e timeout to 120 seconds Co-authored-by: Arvin Xu <arvinxx@users.noreply.github.com> * update config * more parallel * fix parallel * fix tests * refactor to improve performance * fix * fix * fix * refactor with tsx --------- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: Arvin Xu <arvinxx@users.noreply.github.com>
2026-03-26 13:19:34 +07:00 · 2025-10-23 02:15:24 +08:00
parent 0dc112436b
commit b6f1fc4a14
23 changed files with 673 additions and 133 deletions
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -0,0 +1,143 @@
+# E2E Tests for LobeChat
+
+This directory contains end-to-end (E2E) tests for LobeChat using Cucumber (BDD) and Playwright.
+
+## Directory Structure
+
+````
+e2e/
+├── src/               # Source files
+│   ├── features/      # Gherkin feature files
+│   │   └── discover/  # Discover page tests
+│   ├── steps/         # Step definitions
+│   │   ├── common/    # Reusable step definitions
+│   │   └── discover/  # Discover-specific steps
+│   └── support/       # Test support files
+│       └── world.ts   # Custom World context
+├── reports/           # Test reports (generated)
+├── cucumber.config.js # Cucumber configuration
+├── tsconfig.json      # TypeScript configuration
+└── package.json       # Dependencies and scripts
+
+## Prerequisites
+
+- Node.js 20, 22, or >=24
+- Dev server running on `http://localhost:3010` (or set `BASE_URL` env var)
+
+## Installation
+
+Install dependencies:
+
+```bash
+cd e2e
+pnpm install
+````
+
+Install Playwright browsers:
+
+```bash
+npx playwright install chromium
+```
+
+## Running Tests
+
+Run all tests:
+
+```bash
+npm test
+```
+
+Run tests in headed mode (see browser):
+
+```bash
+npm run test:headed
+```
+
+Run only smoke tests:
+
+```bash
+npm run test:smoke
+```
+
+Run discover tests:
+
+```bash
+npm run test:discover
+```
+
+## Environment Variables
+
+- `BASE_URL`: Base URL for the application (default: `http://localhost:3010`)
+- `PORT`: Port number (default: `3010`)
+- `HEADLESS`: Run browser in headless mode (default: `true`, set to `false` to see browser)
+
+Example:
+
+```bash
+HEADLESS=false BASE_URL=http://localhost:3000 npm run test:smoke
+```
+
+## Writing Tests
+
+### Feature Files
+
+Feature files are written in Gherkin syntax and placed in the `src/features/` directory:
+
+```gherkin
+@discover @smoke
+Feature: Discover Smoke Tests
+  Critical path tests to ensure the discover module is functional
+
+  @DISCOVER-SMOKE-001 @P0
+  Scenario: Load discover assistant list page
+    Given I navigate to "/discover/assistant"
+    Then the page should load without errors
+    And I should see the page body
+    And I should see the search bar
+    And I should see assistant cards
+```
+
+### Step Definitions
+
+Step definitions are TypeScript files in the `src/steps/` directory that implement the steps from feature files:
+
+```typescript
+import { Given, Then } from '@cucumber/cucumber';
+import { expect } from '@playwright/test';
+
+import { CustomWorld } from '../../support/world';
+
+Given('I navigate to {string}', async function (this: CustomWorld, path: string) {
+  await this.page.goto(path);
+  await this.page.waitForLoadState('domcontentloaded');
+});
+```
+
+## Test Reports
+
+After running tests, HTML and JSON reports are generated in the `reports/` directory:
+
+- `reports/cucumber-report.html` - Human-readable HTML report
+- `reports/cucumber-report.json` - Machine-readable JSON report
+
+## Troubleshooting
+
+### Browser not found
+
+If you see errors about missing browser executables:
+
+```bash
+npx playwright install chromium
+```
+
+### Port already in use
+
+Make sure the dev server is running on the expected port (3010 by default), or set `PORT` or `BASE_URL` environment variable.
+
+### Test timeout
+
+Increase timeout in `cucumber.config.js` or `src/steps/hooks.ts`:
+
+```typescript
+setDefaultTimeout(120000); // 2 minutes
+```
--- a/e2e/cucumber.config.js
+++ b/e2e/cucumber.config.js
@@ -0,0 +1,20 @@
+/**
+ * @type {import('@cucumber/cucumber').IConfiguration}
+ */
+export default {
+  format: [
+    'progress-bar',
+    'html:reports/cucumber-report.html',
+    'json:reports/cucumber-report.json',
+  ],
+  formatOptions: {
+    snippetInterface: 'async-await',
+  },
+  parallel: process.env.CI ? 1 : 4,
+  paths: ['src/features/**/*.feature'],
+  publishQuiet: true,
+  require: ['src/steps/**/*.ts', 'src/support/**/*.ts'],
+  requireModule: ['tsx/cjs'],
+  retry: 0,
+  timeout: 120_000,
+};
--- a/e2e/package.json
+++ b/e2e/package.json
@@ -0,0 +1,24 @@
+{
+  "name": "@lobechat/e2e-tests",
+  "version": "0.1.0",
+  "private": true,
+  "description": "E2E tests for LobeChat using Cucumber and Playwright",
+  "scripts": {
+    "test": "cucumber-js --config cucumber.config.js",
+    "test:discover": "cucumber-js --config cucumber.config.js src/features/discover/",
+    "test:headed": "HEADLESS=false cucumber-js --config cucumber.config.js",
+    "test:routes": "cucumber-js --config cucumber.config.js --tags '@routes'",
+    "test:routes:ci": "cucumber-js --config cucumber.config.js --tags '@routes and not @ci-skip'",
+    "test:smoke": "cucumber-js --config cucumber.config.js --tags '@smoke'"
+  },
+  "dependencies": {
+    "@cucumber/cucumber": "^12.2.0",
+    "@playwright/test": "^1.56.1",
+    "playwright": "^1.56.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.5",
+    "tsx": "^4.20.6",
+    "typescript": "^5.7.3"
+  }
+}
--- a/e2e/routes.spec.ts
+++ b/e2e/routes.spec.ts
@@ -1,73 +0,0 @@
-import { expect, test } from '@playwright/test';
-
-// 覆盖核心可访问路径（含重定向来源）
-const baseRoutes: string[] = [
-  '/',
-  '/chat',
-  '/discover',
-  '/image',
-  '/files',
-  '/repos', // next.config.ts -> /files
-  '/changelog',
-];
-
-// settings 路由改为通过 query 参数控制 active tab
-// 参考 SettingsTabs: about, agent, common, hotkey, llm, provider, proxy, storage, system-agent, tts
-const settingsTabs = [
-  'common',
-  'llm',
-  'provider',
-  'about',
-  'hotkey',
-  'proxy',
-  'storage',
-  'tts',
-  'system-agent',
-  'agent',
-];
-
-const routes: string[] = [...baseRoutes, ...settingsTabs.map((key) => `/settings?active=${key}`)];
-
-// CI 环境下跳过容易不稳定或受特性开关影响的路由
-const ciSkipPaths = new Set<string>([
-  '/image',
-  '/changelog',
-  '/settings?active=common',
-  '/settings?active=llm',
-]);
-
-// @ts-ignore
-async function assertNoPageErrors(page: Parameters<typeof test>[0]['page']) {
-  const pageErrors: Error[] = [];
-  const consoleErrors: string[] = [];
-
-  page.on('pageerror', (err: Error) => pageErrors.push(err));
-  page.on('console', (msg: any) => {
-    if (msg.type() === 'error') consoleErrors.push(msg.text());
-  });
-
-  // 仅校验页面级错误，忽略控制台 error 以提升稳定性
-  expect
-    .soft(pageErrors, `page errors: ${pageErrors.map((e) => e.message).join('\n')}`)
-    .toHaveLength(0);
-}
-
-test.describe('Smoke: core routes', () => {
-  for (const path of routes) {
-    test(`should open ${path} without error`, async ({ page }) => {
-      if (process.env.CI && ciSkipPaths.has(path)) test.skip(true, 'skip flaky route on CI');
-      const response = await page.goto(path, { waitUntil: 'commit' });
-      // 2xx 或 3xx 视为可接受（允许中间件/重定向）
-      const status = response?.status() ?? 0;
-      expect(status, `unexpected status for ${path}: ${status}`).toBeLessThan(400);
-
-      // 一般错误标题防御
-      await expect(page).not.toHaveTitle(/not found|error/i);
-
-      // body 可见
-      await expect(page.locator('body')).toBeVisible();
-
-      await assertNoPageErrors(page);
-    });
-  }
-});
--- a/e2e/src/features/discover/smoke.feature
+++ b/e2e/src/features/discover/smoke.feature
@@ -0,0 +1,11 @@
+@discover @smoke
+Feature: Discover Smoke Tests
+  Critical path tests to ensure the discover module is functional
+
+  @DISCOVER-SMOKE-001 @P0
+  Scenario: Load discover assistant list page
+    Given I navigate to "/discover/assistant"
+    Then the page should load without errors
+    And I should see the page body
+    And I should see the search bar
+    And I should see assistant cards
--- a/e2e/src/features/routes/core-routes.feature
+++ b/e2e/src/features/routes/core-routes.feature
@@ -0,0 +1,43 @@
+@routes @smoke
+Feature: Core Routes Accessibility
+  As a user
+  I want all core application routes to be accessible
+  So that I can navigate the application without errors
+
+  Background:
+    Given the application is running
+
+  @ROUTES-001 @P0
+  Scenario Outline: Access core routes without errors
+    When I navigate to "<route>"
+    Then the response status should be less than 400
+    And the page should load without errors
+    And I should see the page body
+    And the page title should not contain "error" or "not found"
+
+    Examples:
+      | route      |
+      | /          |
+      | /chat      |
+      | /discover  |
+      | /files     |
+      | /repos     |
+
+  @ROUTES-002 @P0
+  Scenario Outline: Access settings routes without errors
+    When I navigate to "/settings?active=<tab>"
+    Then the response status should be less than 400
+    And the page should load without errors
+    And I should see the page body
+    And the page title should not contain "error" or "not found"
+
+    Examples:
+      | tab          |
+      | about        |
+      | agent        |
+      | hotkey       |
+      | provider     |
+      | proxy        |
+      | storage      |
+      | system-agent |
+      | tts          |
--- a/e2e/src/steps/common/navigation.steps.ts
+++ b/e2e/src/steps/common/navigation.steps.ts
@@ -0,0 +1,36 @@
+import { Given, Then } from '@cucumber/cucumber';
+import { expect } from '@playwright/test';
+
+import { CustomWorld } from '../../support/world';
+
+// ============================================
+// Given Steps (Preconditions)
+// ============================================
+
+Given('I navigate to {string}', async function (this: CustomWorld, path: string) {
+  const response = await this.page.goto(path, { waitUntil: 'commit' });
+  this.testContext.lastResponse = response;
+  await this.page.waitForLoadState('domcontentloaded');
+});
+
+// ============================================
+// Then Steps (Assertions)
+// ============================================
+
+Then('the page should load without errors', async function (this: CustomWorld) {
+  // Check for no JavaScript errors
+  expect(this.testContext.jsErrors).toHaveLength(0);
+
+  // Check page didn't navigate to error page
+  const url = this.page.url();
+  expect(url).not.toMatch(/\/404|\/error|not-found/i);
+
+  // Check no error title
+  const title = await this.page.title();
+  expect(title).not.toMatch(/not found|error/i);
+});
+
+Then('I should see the page body', async function (this: CustomWorld) {
+  const body = this.page.locator('body');
+  await expect(body).toBeVisible();
+});
--- a/e2e/src/steps/discover/smoke.steps.ts
+++ b/e2e/src/steps/discover/smoke.steps.ts
@@ -0,0 +1,34 @@
+import { Then } from '@cucumber/cucumber';
+import { expect } from '@playwright/test';
+
+import { CustomWorld } from '../../support/world';
+
+// ============================================
+// Then Steps (Assertions)
+// ============================================
+
+Then('I should see the search bar', async function (this: CustomWorld) {
+  // Wait for network to be idle to ensure Suspense components are loaded
+  await this.page.waitForLoadState('networkidle', { timeout: 120_000 });
+
+  // The SearchBar component from @lobehub/ui may not pass through data-testid
+  // Try to find the input element within the search component
+  const searchBar = this.page.locator('input[type="text"]').first();
+  await expect(searchBar).toBeVisible({ timeout: 120_000 });
+});
+
+Then('I should see assistant cards', async function (this: CustomWorld) {
+  // Wait for content to load
+  await this.page.waitForLoadState('networkidle', { timeout: 120_000 });
+
+  // After migrating to SPA (react-router), links use relative paths like /assistant/:id
+  // Look for assistant items by data-testid instead of href
+  const assistantItems = this.page.locator('[data-testid="assistant-item"]');
+
+  // Wait for at least one item to be visible
+  await expect(assistantItems.first()).toBeVisible({ timeout: 120_000 });
+
+  // Check we have multiple items
+  const count = await assistantItems.count();
+  expect(count).toBeGreaterThan(0);
+});
--- a/e2e/src/steps/hooks.ts
+++ b/e2e/src/steps/hooks.ts
@@ -0,0 +1,69 @@
+import { After, AfterAll, Before, BeforeAll, Status, setDefaultTimeout } from '@cucumber/cucumber';
+
+import { startWebServer, stopWebServer } from '../support/webServer';
+import { CustomWorld } from '../support/world';
+
+// Set default timeout for all steps to 120 seconds
+setDefaultTimeout(120_000);
+
+BeforeAll({ timeout: 120_000 }, async function () {
+  console.log('🚀 Starting E2E test suite...');
+
+  const PORT = process.env.PORT ? Number(process.env.PORT) : 3010;
+  const BASE_URL = process.env.BASE_URL || `http://localhost:${PORT}`;
+
+  console.log(`Base URL: ${BASE_URL}`);
+
+  // Start web server if not using external BASE_URL
+  if (!process.env.BASE_URL) {
+    await startWebServer({
+      command: 'npm run dev',
+      port: PORT,
+      reuseExistingServer: !process.env.CI,
+    });
+  }
+});
+
+Before(async function (this: CustomWorld, { pickle }) {
+  await this.init();
+
+  const testId = pickle.tags.find((tag) => tag.name.startsWith('@DISCOVER-'));
+  console.log(`\n📝 Running: ${pickle.name}${testId ? ` (${testId.name.replace('@', '')})` : ''}`);
+});
+
+After(async function (this: CustomWorld, { pickle, result }) {
+  const testId = pickle.tags
+    .find((tag) => tag.name.startsWith('@DISCOVER-'))
+    ?.name.replace('@', '');
+
+  if (result?.status === Status.FAILED) {
+    const screenshot = await this.takeScreenshot(`${testId || 'failure'}-${Date.now()}`);
+    this.attach(screenshot, 'image/png');
+
+    const html = await this.page.content();
+    this.attach(html, 'text/html');
+
+    if (this.testContext.jsErrors.length > 0) {
+      const errors = this.testContext.jsErrors.map((e) => e.message).join('\n');
+      this.attach(`JavaScript Errors:\n${errors}`, 'text/plain');
+    }
+
+    console.log(`❌ Failed: ${pickle.name}`);
+    if (result.message) {
+      console.log(`   Error: ${result.message}`);
+    }
+  } else if (result?.status === Status.PASSED) {
+    console.log(`✅ Passed: ${pickle.name}`);
+  }
+
+  await this.cleanup();
+});
+
+AfterAll(async function () {
+  console.log('\n🏁 Test suite completed');
+
+  // Stop web server if we started it
+  if (!process.env.BASE_URL && process.env.CI) {
+    await stopWebServer();
+  }
+});
--- a/e2e/src/steps/routes/routes.steps.ts
+++ b/e2e/src/steps/routes/routes.steps.ts
@@ -0,0 +1,41 @@
+import { Given, Then } from '@cucumber/cucumber';
+import { expect } from '@playwright/test';
+
+import { CustomWorld } from '../../support/world';
+
+// ============================================
+// Given Steps (Preconditions)
+// ============================================
+
+Given('the application is running', async function (this: CustomWorld) {
+  // This is a placeholder step to indicate that the app should be running
+  // The actual server startup is handled outside the test (in CI or locally)
+  // We just verify we can reach the base URL
+  const response = await this.page.goto('/');
+  expect(response).toBeTruthy();
+  // Store the response for later assertions
+  this.testContext.lastResponse = response;
+});
+
+// ============================================
+// Then Steps (Assertions)
+// ============================================
+
+Then(
+  'the response status should be less than {int}',
+  async function (this: CustomWorld, maxStatus: number) {
+    const status = this.testContext.lastResponse?.status() ?? 0;
+    expect(status, `Expected status < ${maxStatus}, but got ${status}`).toBeLessThan(maxStatus);
+  },
+);
+
+Then(
+  'the page title should not contain {string} or {string}',
+  async function (this: CustomWorld, text1: string, text2: string) {
+    const title = await this.page.title();
+    const regex = new RegExp(`${text1}|${text2}`, 'i');
+    expect(title, `Page title "${title}" should not contain "${text1}" or "${text2}"`).not.toMatch(
+      regex,
+    );
+  },
+);
--- a/e2e/src/support/webServer.ts
+++ b/e2e/src/support/webServer.ts
@@ -0,0 +1,96 @@
+import { type ChildProcess, exec } from 'node:child_process';
+import { resolve } from 'node:path';
+
+let serverProcess: ChildProcess | null = null;
+let serverStartPromise: Promise<void> | null = null;
+
+interface WebServerOptions {
+  command: string;
+  env?: Record<string, string>;
+  port: number;
+  reuseExistingServer?: boolean;
+  timeout?: number;
+}
+
+async function isServerRunning(port: number): Promise<boolean> {
+  try {
+    const response = await fetch(`http://localhost:${port}/chat`, {
+      method: 'HEAD',
+    });
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+
+export async function startWebServer(options: WebServerOptions): Promise<void> {
+  const { command, port, timeout = 120_000, env = {}, reuseExistingServer = true } = options;
+
+  // If server is already being started by another worker, wait for it
+  if (serverStartPromise) {
+    console.log(`⏳ Waiting for server to start (started by another worker)...`);
+    return serverStartPromise;
+  }
+
+  // Check if server is already running
+  if (reuseExistingServer && (await isServerRunning(port))) {
+    console.log(`✅ Reusing existing server on port ${port}`);
+    return;
+  }
+
+  // Create a promise for the server startup and store it
+  serverStartPromise = (async () => {
+    console.log(`🚀 Starting web server: ${command}`);
+
+    // Get the project root directory (parent of e2e folder)
+    const projectRoot = resolve(__dirname, '../../..');
+
+    // Start the server process
+    serverProcess = exec(command, {
+      cwd: projectRoot,
+      env: {
+        ...process.env,
+        ENABLE_AUTH_PROTECTION: '0',
+        ENABLE_OIDC: '0',
+        NEXT_PUBLIC_ENABLE_CLERK_AUTH: '0',
+        NEXT_PUBLIC_ENABLE_NEXT_AUTH: '0',
+        NODE_OPTIONS: '--max-old-space-size=6144',
+        PORT: String(port),
+        ...env,
+      },
+    });
+
+    // Forward server output to console for debugging
+    serverProcess.stdout?.on('data', (data) => {
+      console.log(`[server] ${data}`);
+    });
+
+    serverProcess.stderr?.on('data', (data) => {
+      console.error(`[server] ${data}`);
+    });
+
+    // Wait for server to be ready
+    const startTime = Date.now();
+    while (!(await isServerRunning(port))) {
+      if (Date.now() - startTime > timeout) {
+        throw new Error(`Server failed to start within ${timeout}ms`);
+      }
+      await new Promise((resolve) => {
+        setTimeout(resolve, 1000);
+      });
+    }
+
+    console.log(`✅ Web server is ready on port ${port}`);
+  })();
+
+  return serverStartPromise;
+}
+
+export async function stopWebServer(): Promise<void> {
+  if (serverProcess) {
+    console.log('🛑 Stopping web server...');
+    serverProcess.kill();
+    serverProcess = null;
+    serverStartPromise = null;
+  }
+}
--- a/e2e/src/support/world.ts
+++ b/e2e/src/support/world.ts
@@ -0,0 +1,76 @@
+import { IWorldOptions, World, setWorldConstructor } from '@cucumber/cucumber';
+import { Browser, BrowserContext, Page, Response, chromium } from '@playwright/test';
+
+export interface TestContext {
+  [key: string]: any;
+  consoleErrors: string[];
+  jsErrors: Error[];
+  lastResponse?: Response | null;
+  previousUrl?: string;
+}
+
+export class CustomWorld extends World {
+  browser!: Browser;
+  browserContext!: BrowserContext;
+  page!: Page;
+  testContext: TestContext;
+
+  constructor(options: IWorldOptions) {
+    super(options);
+    this.testContext = {
+      consoleErrors: [],
+      jsErrors: [],
+    };
+  }
+
+  // Getter for easier access
+  get context(): TestContext {
+    return this.testContext;
+  }
+
+  async init() {
+    const PORT = process.env.PORT ? Number(process.env.PORT) : 3010;
+    const BASE_URL = process.env.BASE_URL || `http://localhost:${PORT}`;
+
+    this.browser = await chromium.launch({
+      headless: process.env.HEADLESS !== 'false',
+    });
+
+    this.browserContext = await this.browser.newContext({
+      baseURL: BASE_URL,
+      viewport: { height: 720, width: 1280 },
+    });
+
+    // Set expect timeout for assertions (e.g., toBeVisible, toHaveText)
+    this.browserContext.setDefaultTimeout(120_000);
+
+    this.page = await this.browserContext.newPage();
+
+    // Set up error listeners
+    this.page.on('pageerror', (error) => {
+      this.testContext.jsErrors.push(error);
+      console.error('Page error:', error.message);
+    });
+
+    this.page.on('console', (msg) => {
+      if (msg.type() === 'error') {
+        this.testContext.consoleErrors.push(msg.text());
+      }
+    });
+
+    this.page.setDefaultTimeout(120_000);
+  }
+
+  async cleanup() {
+    await this.page?.close();
+    await this.browserContext?.close();
+    await this.browser?.close();
+  }
+
+  async takeScreenshot(name: string): Promise<Buffer> {
+    console.log(name);
+    return await this.page.screenshot({ fullPage: true });
+  }
+}
+
+setWorldConstructor(CustomWorld);
--- a/e2e/tsconfig.json
+++ b/e2e/tsconfig.json
@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "target": "ES2020",
+    "lib": ["ES2020"],
+    "types": ["node", "@cucumber/cucumber", "@playwright/test"],
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["../*"]
+    }
+  },
+  "exclude": ["node_modules", "reports"],
+  "extends": "../tsconfig.json",
+  "include": ["src/**/*", "*.js"]
+}