perf: add vitest test perf workflows

This commit is contained in:
Peter Steinberger
2026-03-23 04:40:45 +00:00
parent 1c60e00a34
commit 7909236bd1
10 changed files with 131 additions and 3 deletions

View File

@@ -79,6 +79,15 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost):
- `pnpm test` also passes `--isolate=false` at the wrapper level.
- Opt back into Vitest file isolation with `OPENCLAW_TEST_ISOLATE=1 pnpm test`.
- `OPENCLAW_TEST_NO_ISOLATE=0` or `OPENCLAW_TEST_NO_ISOLATE=false` also force isolated runs.
- Fast-local iteration note:
- `pnpm test:changed` runs the wrapper with `--changed origin/main`.
- The base Vitest config marks the wrapper manifests/config files as `forceRerunTriggers` so changed-mode reruns stay correct when scheduler inputs change.
- Use `OPENCLAW_VITEST_FS_MODULE_CACHE=1` for repeated local reruns on a stable branch when transform cost dominates.
- Perf-debug note:
- `pnpm test:perf:imports` enables Vitest import-duration reporting plus import-breakdown output.
- `pnpm test:perf:imports:changed` scopes the same profiling view to files changed since `origin/main`.
- `pnpm test:perf:profile:main` writes a main-thread CPU profile for Vitest/Vite startup and transform overhead.
- `pnpm test:perf:profile:runner` writes runner CPU+heap profiles for the unit suite with file parallelism disabled.
### E2E (gateway smoke)

View File

@@ -11,11 +11,17 @@ title: "Tests"
- `pnpm test:force`: Kills any lingering gateway process holding the default control port, then runs the full Vitest suite with an isolated gateway port so server tests dont collide with a running instance. Use this when a prior gateway run left port 18789 occupied.
- `pnpm test:coverage`: Runs the unit suite with V8 coverage (via `vitest.unit.config.ts`). Global thresholds are 70% lines/branches/functions/statements. Coverage excludes integration-heavy entrypoints (CLI wiring, gateway/telegram bridges, webchat static server) to keep the target focused on unit-testable logic.
- `pnpm test:coverage:changed`: Runs unit coverage only for files changed since `origin/main`.
- `pnpm test:changed`: runs the wrapper with `--changed origin/main`. The base Vitest config treats the wrapper manifests/config files as `forceRerunTriggers` so scheduler changes still rerun broadly when needed.
- `pnpm test`: runs the full wrapper. It keeps only a small behavioral override manifest in git, then uses a checked-in timing snapshot to peel the heaviest measured unit files into dedicated lanes.
- Unit files default to `threads` in the wrapper; keep fork-only exceptions documented in `test/fixtures/test-parallel.behavior.json`.
- `pnpm test:channels` now defaults to `threads` via `vitest.channels.config.ts`; the March 22, 2026 direct full-suite control run passed clean without channel-specific fork exceptions.
- `pnpm test:extensions` now defaults to `threads` via `vitest.extensions.config.ts`; the March 22, 2026 direct full-suite control run passed clean without extension-specific fork exceptions.
- `pnpm test:extensions`: runs extension/plugin suites.
- `pnpm test:perf:imports`: enables Vitest import-duration + import-breakdown reporting for the wrapper.
- `pnpm test:perf:imports:changed`: same import profiling, but only for files changed since `origin/main`.
- `pnpm test:perf:profile:main`: writes a CPU profile for the Vitest main thread (`.artifacts/vitest-main-profile`).
- `pnpm test:perf:profile:runner`: writes CPU + heap profiles for the unit runner (`.artifacts/vitest-runner-profile`).
- `pnpm test:perf:update-timings`: refreshes the checked-in slow-file timing snapshot used by `scripts/test-parallel.mjs`.
- Gateway integration: opt-in via `OPENCLAW_TEST_INCLUDE_GATEWAY=1 pnpm test` or `pnpm test:gateway`.
- `pnpm test:e2e`: Runs gateway end-to-end smoke tests (multi-instance WS/HTTP/node pairing). Defaults to `forks` + adaptive workers in `vitest.e2e.config.ts`; tune with `OPENCLAW_E2E_WORKERS=<n>` and set `OPENCLAW_E2E_VERBOSE=1` for verbose logs.
@@ -33,6 +39,7 @@ For local PR land/gate checks, run:
If `pnpm test` flakes on a loaded host, rerun once before treating it as a regression, then isolate with `pnpm vitest run <path/to/test>`. For memory-constrained hosts, use:
- `OPENCLAW_TEST_PROFILE=low OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test`
- `OPENCLAW_VITEST_FS_MODULE_CACHE=1 pnpm test:changed`
## Model latency bench (local keys)

View File

@@ -691,11 +691,13 @@
"test:all": "pnpm lint && pnpm build && pnpm test && pnpm test:e2e && pnpm test:live && pnpm test:docker:all",
"test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts",
"test:build:singleton": "node scripts/test-built-plugin-singleton.mjs",
"test:changed": "pnpm test -- --changed origin/main",
"test:channels": "vitest run --config vitest.channels.config.ts",
"test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins",
"test:contracts:channels": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/channels/plugins/contracts",
"test:contracts:plugins": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/plugins/contracts",
"test:coverage": "vitest run --config vitest.unit.config.ts --coverage",
"test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main",
"test:docker:all": "pnpm test:docker:live-models && pnpm test:docker:live-gateway && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup",
"test:docker:cleanup": "bash scripts/test-cleanup-docker.sh",
"test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh",
@@ -726,6 +728,10 @@
"test:perf:budget": "node scripts/test-perf-budget.mjs",
"test:perf:find-thread-candidates": "node scripts/test-find-thread-candidates.mjs",
"test:perf:hotspots": "node scripts/test-hotspots.mjs",
"test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 pnpm test",
"test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 pnpm test -- --changed origin/main",
"test:perf:profile:main": "node --cpu-prof --cpu-prof-dir=.artifacts/vitest-main-profile ./node_modules/vitest/vitest.mjs run --config vitest.unit.config.ts --no-file-parallelism",
"test:perf:profile:runner": "vitest run --config vitest.unit.config.ts --no-file-parallelism --execArgv=--cpu-prof --execArgv=--cpu-prof-dir=.artifacts/vitest-runner-profile --execArgv=--heap-prof --execArgv=--heap-prof-dir=.artifacts/vitest-runner-profile",
"test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs",
"test:perf:update-timings": "node scripts/test-update-timings.mjs",
"test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",

View File

@@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import { loadVitestExperimentalConfig } from "../vitest.performance-config.ts";
describe("loadVitestExperimentalConfig", () => {
it("returns an empty object when no perf flags are enabled", () => {
expect(loadVitestExperimentalConfig({})).toEqual({});
});
it("enables the filesystem module cache explicitly", () => {
expect(
loadVitestExperimentalConfig({
OPENCLAW_VITEST_FS_MODULE_CACHE: "1",
}),
).toEqual({
experimental: {
fsModuleCache: true,
},
});
});
it("enables import timing output and import breakdown reporting", () => {
expect(
loadVitestExperimentalConfig({
OPENCLAW_VITEST_IMPORT_DURATIONS: "true",
OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN: "1",
}),
).toEqual({
experimental: {
importDurations: { print: true },
printImportBreakdown: true,
},
});
});
});

View File

@@ -20,6 +20,13 @@ describe("createScopedVitestConfig", () => {
const config = createScopedVitestConfig(["src/example.test.ts"]);
expect(config.test?.isolate).toBe(false);
});
it("passes through a scoped root dir when provided", () => {
const config = createScopedVitestConfig(["src/example.test.ts"], {
dir: "src",
});
expect(config.test?.dir).toBe("src");
});
});
describe("scoped vitest configs", () => {

View File

@@ -3,6 +3,12 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { defineConfig } from "vitest/config";
import { pluginSdkSubpaths } from "./scripts/lib/plugin-sdk-entries.mjs";
import {
behaviorManifestPath,
unitMemoryHotspotManifestPath,
unitTimingManifestPath,
} from "./scripts/test-runner-manifest.mjs";
import { loadVitestExperimentalConfig } from "./vitest.performance-config.ts";
const repoRoot = path.dirname(fileURLToPath(import.meta.url));
const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true";
@@ -37,6 +43,27 @@ export default defineConfig({
unstubGlobals: true,
pool: "forks",
maxWorkers: isCI ? ciWorkers : localWorkers,
forceRerunTriggers: [
"package.json",
"pnpm-lock.yaml",
"test/setup.ts",
"scripts/test-parallel.mjs",
"scripts/test-runner-manifest.mjs",
"vitest.channel-paths.mjs",
"vitest.channels.config.ts",
"vitest.config.ts",
"vitest.e2e.config.ts",
"vitest.extensions.config.ts",
"vitest.gateway.config.ts",
"vitest.live.config.ts",
"vitest.performance-config.ts",
"vitest.scoped-config.ts",
"vitest.unit.config.ts",
"vitest.unit-paths.mjs",
behaviorManifestPath,
unitTimingManifestPath,
unitMemoryHotspotManifestPath,
],
include: [
"src/**/*.test.ts",
"extensions/**/*.test.ts",
@@ -154,5 +181,6 @@ export default defineConfig({
"src/infra/tailscale.ts",
],
},
...loadVitestExperimentalConfig(),
},
});

View File

@@ -23,6 +23,7 @@ export function loadIncludePatternsFromEnv(
export default createScopedVitestConfig(
loadIncludePatternsFromEnv() ?? ["extensions/**/*.test.ts"],
{
dir: "extensions",
pool: "threads",
// Channel implementations live under extensions/ but are tested by
// vitest.channels.config.ts (pnpm test:channels) which provides

View File

@@ -1,3 +1,5 @@
import { createScopedVitestConfig } from "./vitest.scoped-config.ts";
export default createScopedVitestConfig(["src/gateway/**/*.test.ts"]);
export default createScopedVitestConfig(["src/gateway/**/*.test.ts"], {
dir: "src/gateway",
});

View File

@@ -0,0 +1,32 @@
type EnvMap = Record<string, string | undefined>;
const isEnabled = (value: string | undefined): boolean => {
const normalized = value?.trim().toLowerCase();
return normalized === "1" || normalized === "true";
};
export function loadVitestExperimentalConfig(env: EnvMap = process.env): {
experimental?: {
fsModuleCache?: true;
importDurations?: { print: true };
printImportBreakdown?: true;
};
} {
const experimental: {
fsModuleCache?: true;
importDurations?: { print: true };
printImportBreakdown?: true;
} = {};
if (isEnabled(env.OPENCLAW_VITEST_FS_MODULE_CACHE)) {
experimental.fsModuleCache = true;
}
if (isEnabled(env.OPENCLAW_VITEST_IMPORT_DURATIONS)) {
experimental.importDurations = { print: true };
}
if (isEnabled(env.OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN)) {
experimental.printImportBreakdown = true;
}
return Object.keys(experimental).length > 0 ? { experimental } : {};
}

View File

@@ -13,11 +13,12 @@ export function resolveVitestIsolation(
export function createScopedVitestConfig(
include: string[],
options?: { exclude?: string[]; pool?: "threads" | "forks" },
options?: { dir?: string; exclude?: string[]; pool?: "threads" | "forks" },
) {
const base = baseConfig as unknown as Record<string, unknown>;
const baseTest =
(baseConfig as { test?: { exclude?: string[]; pool?: "threads" | "forks" } }).test ?? {};
(baseConfig as { test?: { dir?: string; exclude?: string[]; pool?: "threads" | "forks" } })
.test ?? {};
const exclude = [...(baseTest.exclude ?? []), ...(options?.exclude ?? [])];
return defineConfig({
@@ -25,6 +26,7 @@ export function createScopedVitestConfig(
test: {
...baseTest,
isolate: resolveVitestIsolation(),
...(options?.dir ? { dir: options.dir } : {}),
include,
exclude,
...(options?.pool ? { pool: options.pool } : {}),