diff --git a/docs/help/testing.md b/docs/help/testing.md index 6cf3ae14246..7db025fcae0 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -79,6 +79,15 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): - `pnpm test` also passes `--isolate=false` at the wrapper level. - Opt back into Vitest file isolation with `OPENCLAW_TEST_ISOLATE=1 pnpm test`. - `OPENCLAW_TEST_NO_ISOLATE=0` or `OPENCLAW_TEST_NO_ISOLATE=false` also force isolated runs. +- Fast-local iteration note: + - `pnpm test:changed` runs the wrapper with `--changed origin/main`. + - The base Vitest config marks the wrapper manifests/config files as `forceRerunTriggers` so changed-mode reruns stay correct when scheduler inputs change. + - Use `OPENCLAW_VITEST_FS_MODULE_CACHE=1` for repeated local reruns on a stable branch when transform cost dominates. +- Perf-debug note: + - `pnpm test:perf:imports` enables Vitest import-duration reporting plus import-breakdown output. + - `pnpm test:perf:imports:changed` scopes the same profiling view to files changed since `origin/main`. + - `pnpm test:perf:profile:main` writes a main-thread CPU profile for Vitest/Vite startup and transform overhead. + - `pnpm test:perf:profile:runner` writes runner CPU+heap profiles for the unit suite with file parallelism disabled. ### E2E (gateway smoke) diff --git a/docs/reference/test.md b/docs/reference/test.md index 81f2c674344..c0a65ab60c6 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -11,11 +11,17 @@ title: "Tests" - `pnpm test:force`: Kills any lingering gateway process holding the default control port, then runs the full Vitest suite with an isolated gateway port so server tests don’t collide with a running instance. Use this when a prior gateway run left port 18789 occupied. - `pnpm test:coverage`: Runs the unit suite with V8 coverage (via `vitest.unit.config.ts`). Global thresholds are 70% lines/branches/functions/statements. Coverage excludes integration-heavy entrypoints (CLI wiring, gateway/telegram bridges, webchat static server) to keep the target focused on unit-testable logic. +- `pnpm test:coverage:changed`: Runs unit coverage only for files changed since `origin/main`. +- `pnpm test:changed`: runs the wrapper with `--changed origin/main`. The base Vitest config treats the wrapper manifests/config files as `forceRerunTriggers` so scheduler changes still rerun broadly when needed. - `pnpm test`: runs the full wrapper. It keeps only a small behavioral override manifest in git, then uses a checked-in timing snapshot to peel the heaviest measured unit files into dedicated lanes. - Unit files default to `threads` in the wrapper; keep fork-only exceptions documented in `test/fixtures/test-parallel.behavior.json`. - `pnpm test:channels` now defaults to `threads` via `vitest.channels.config.ts`; the March 22, 2026 direct full-suite control run passed clean without channel-specific fork exceptions. - `pnpm test:extensions` now defaults to `threads` via `vitest.extensions.config.ts`; the March 22, 2026 direct full-suite control run passed clean without extension-specific fork exceptions. - `pnpm test:extensions`: runs extension/plugin suites. +- `pnpm test:perf:imports`: enables Vitest import-duration + import-breakdown reporting for the wrapper. +- `pnpm test:perf:imports:changed`: same import profiling, but only for files changed since `origin/main`. +- `pnpm test:perf:profile:main`: writes a CPU profile for the Vitest main thread (`.artifacts/vitest-main-profile`). +- `pnpm test:perf:profile:runner`: writes CPU + heap profiles for the unit runner (`.artifacts/vitest-runner-profile`). - `pnpm test:perf:update-timings`: refreshes the checked-in slow-file timing snapshot used by `scripts/test-parallel.mjs`. - Gateway integration: opt-in via `OPENCLAW_TEST_INCLUDE_GATEWAY=1 pnpm test` or `pnpm test:gateway`. - `pnpm test:e2e`: Runs gateway end-to-end smoke tests (multi-instance WS/HTTP/node pairing). Defaults to `forks` + adaptive workers in `vitest.e2e.config.ts`; tune with `OPENCLAW_E2E_WORKERS=` and set `OPENCLAW_E2E_VERBOSE=1` for verbose logs. @@ -33,6 +39,7 @@ For local PR land/gate checks, run: If `pnpm test` flakes on a loaded host, rerun once before treating it as a regression, then isolate with `pnpm vitest run `. For memory-constrained hosts, use: - `OPENCLAW_TEST_PROFILE=low OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test` +- `OPENCLAW_VITEST_FS_MODULE_CACHE=1 pnpm test:changed` ## Model latency bench (local keys) diff --git a/package.json b/package.json index ac233643122..d013112764f 100644 --- a/package.json +++ b/package.json @@ -691,11 +691,13 @@ "test:all": "pnpm lint && pnpm build && pnpm test && pnpm test:e2e && pnpm test:live && pnpm test:docker:all", "test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts", "test:build:singleton": "node scripts/test-built-plugin-singleton.mjs", + "test:changed": "pnpm test -- --changed origin/main", "test:channels": "vitest run --config vitest.channels.config.ts", "test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins", "test:contracts:channels": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/channels/plugins/contracts", "test:contracts:plugins": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/plugins/contracts", "test:coverage": "vitest run --config vitest.unit.config.ts --coverage", + "test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main", "test:docker:all": "pnpm test:docker:live-models && pnpm test:docker:live-gateway && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup", "test:docker:cleanup": "bash scripts/test-cleanup-docker.sh", "test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh", @@ -726,6 +728,10 @@ "test:perf:budget": "node scripts/test-perf-budget.mjs", "test:perf:find-thread-candidates": "node scripts/test-find-thread-candidates.mjs", "test:perf:hotspots": "node scripts/test-hotspots.mjs", + "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 pnpm test", + "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 pnpm test -- --changed origin/main", + "test:perf:profile:main": "node --cpu-prof --cpu-prof-dir=.artifacts/vitest-main-profile ./node_modules/vitest/vitest.mjs run --config vitest.unit.config.ts --no-file-parallelism", + "test:perf:profile:runner": "vitest run --config vitest.unit.config.ts --no-file-parallelism --execArgv=--cpu-prof --execArgv=--cpu-prof-dir=.artifacts/vitest-runner-profile --execArgv=--heap-prof --execArgv=--heap-prof-dir=.artifacts/vitest-runner-profile", "test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs", "test:perf:update-timings": "node scripts/test-update-timings.mjs", "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts", diff --git a/test/vitest-performance-config.test.ts b/test/vitest-performance-config.test.ts new file mode 100644 index 00000000000..6f6fdd07ad3 --- /dev/null +++ b/test/vitest-performance-config.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { loadVitestExperimentalConfig } from "../vitest.performance-config.ts"; + +describe("loadVitestExperimentalConfig", () => { + it("returns an empty object when no perf flags are enabled", () => { + expect(loadVitestExperimentalConfig({})).toEqual({}); + }); + + it("enables the filesystem module cache explicitly", () => { + expect( + loadVitestExperimentalConfig({ + OPENCLAW_VITEST_FS_MODULE_CACHE: "1", + }), + ).toEqual({ + experimental: { + fsModuleCache: true, + }, + }); + }); + + it("enables import timing output and import breakdown reporting", () => { + expect( + loadVitestExperimentalConfig({ + OPENCLAW_VITEST_IMPORT_DURATIONS: "true", + OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN: "1", + }), + ).toEqual({ + experimental: { + importDurations: { print: true }, + printImportBreakdown: true, + }, + }); + }); +}); diff --git a/test/vitest-scoped-config.test.ts b/test/vitest-scoped-config.test.ts index 8713f0939dd..7f6310d508c 100644 --- a/test/vitest-scoped-config.test.ts +++ b/test/vitest-scoped-config.test.ts @@ -20,6 +20,13 @@ describe("createScopedVitestConfig", () => { const config = createScopedVitestConfig(["src/example.test.ts"]); expect(config.test?.isolate).toBe(false); }); + + it("passes through a scoped root dir when provided", () => { + const config = createScopedVitestConfig(["src/example.test.ts"], { + dir: "src", + }); + expect(config.test?.dir).toBe("src"); + }); }); describe("scoped vitest configs", () => { diff --git a/vitest.config.ts b/vitest.config.ts index 2341006a3df..558b588a3a9 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,12 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; import { pluginSdkSubpaths } from "./scripts/lib/plugin-sdk-entries.mjs"; +import { + behaviorManifestPath, + unitMemoryHotspotManifestPath, + unitTimingManifestPath, +} from "./scripts/test-runner-manifest.mjs"; +import { loadVitestExperimentalConfig } from "./vitest.performance-config.ts"; const repoRoot = path.dirname(fileURLToPath(import.meta.url)); const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true"; @@ -37,6 +43,27 @@ export default defineConfig({ unstubGlobals: true, pool: "forks", maxWorkers: isCI ? ciWorkers : localWorkers, + forceRerunTriggers: [ + "package.json", + "pnpm-lock.yaml", + "test/setup.ts", + "scripts/test-parallel.mjs", + "scripts/test-runner-manifest.mjs", + "vitest.channel-paths.mjs", + "vitest.channels.config.ts", + "vitest.config.ts", + "vitest.e2e.config.ts", + "vitest.extensions.config.ts", + "vitest.gateway.config.ts", + "vitest.live.config.ts", + "vitest.performance-config.ts", + "vitest.scoped-config.ts", + "vitest.unit.config.ts", + "vitest.unit-paths.mjs", + behaviorManifestPath, + unitTimingManifestPath, + unitMemoryHotspotManifestPath, + ], include: [ "src/**/*.test.ts", "extensions/**/*.test.ts", @@ -154,5 +181,6 @@ export default defineConfig({ "src/infra/tailscale.ts", ], }, + ...loadVitestExperimentalConfig(), }, }); diff --git a/vitest.extensions.config.ts b/vitest.extensions.config.ts index 187e8e4ef25..71c4dad761a 100644 --- a/vitest.extensions.config.ts +++ b/vitest.extensions.config.ts @@ -23,6 +23,7 @@ export function loadIncludePatternsFromEnv( export default createScopedVitestConfig( loadIncludePatternsFromEnv() ?? ["extensions/**/*.test.ts"], { + dir: "extensions", pool: "threads", // Channel implementations live under extensions/ but are tested by // vitest.channels.config.ts (pnpm test:channels) which provides diff --git a/vitest.gateway.config.ts b/vitest.gateway.config.ts index b8f85a89bca..4e7d2fc20f5 100644 --- a/vitest.gateway.config.ts +++ b/vitest.gateway.config.ts @@ -1,3 +1,5 @@ import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; -export default createScopedVitestConfig(["src/gateway/**/*.test.ts"]); +export default createScopedVitestConfig(["src/gateway/**/*.test.ts"], { + dir: "src/gateway", +}); diff --git a/vitest.performance-config.ts b/vitest.performance-config.ts new file mode 100644 index 00000000000..07f35946258 --- /dev/null +++ b/vitest.performance-config.ts @@ -0,0 +1,32 @@ +type EnvMap = Record; + +const isEnabled = (value: string | undefined): boolean => { + const normalized = value?.trim().toLowerCase(); + return normalized === "1" || normalized === "true"; +}; + +export function loadVitestExperimentalConfig(env: EnvMap = process.env): { + experimental?: { + fsModuleCache?: true; + importDurations?: { print: true }; + printImportBreakdown?: true; + }; +} { + const experimental: { + fsModuleCache?: true; + importDurations?: { print: true }; + printImportBreakdown?: true; + } = {}; + + if (isEnabled(env.OPENCLAW_VITEST_FS_MODULE_CACHE)) { + experimental.fsModuleCache = true; + } + if (isEnabled(env.OPENCLAW_VITEST_IMPORT_DURATIONS)) { + experimental.importDurations = { print: true }; + } + if (isEnabled(env.OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN)) { + experimental.printImportBreakdown = true; + } + + return Object.keys(experimental).length > 0 ? { experimental } : {}; +} diff --git a/vitest.scoped-config.ts b/vitest.scoped-config.ts index a34ad5e73c4..f15afe2863f 100644 --- a/vitest.scoped-config.ts +++ b/vitest.scoped-config.ts @@ -13,11 +13,12 @@ export function resolveVitestIsolation( export function createScopedVitestConfig( include: string[], - options?: { exclude?: string[]; pool?: "threads" | "forks" }, + options?: { dir?: string; exclude?: string[]; pool?: "threads" | "forks" }, ) { const base = baseConfig as unknown as Record; const baseTest = - (baseConfig as { test?: { exclude?: string[]; pool?: "threads" | "forks" } }).test ?? {}; + (baseConfig as { test?: { dir?: string; exclude?: string[]; pool?: "threads" | "forks" } }) + .test ?? {}; const exclude = [...(baseTest.exclude ?? []), ...(options?.exclude ?? [])]; return defineConfig({ @@ -25,6 +26,7 @@ export function createScopedVitestConfig( test: { ...baseTest, isolate: resolveVitestIsolation(), + ...(options?.dir ? { dir: options.dir } : {}), include, exclude, ...(options?.pool ? { pool: options.pool } : {}),