mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-27 09:21:35 +07:00
perf: add vitest test perf workflows
This commit is contained in:
@@ -79,6 +79,15 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost):
|
||||
- `pnpm test` also passes `--isolate=false` at the wrapper level.
|
||||
- Opt back into Vitest file isolation with `OPENCLAW_TEST_ISOLATE=1 pnpm test`.
|
||||
- `OPENCLAW_TEST_NO_ISOLATE=0` or `OPENCLAW_TEST_NO_ISOLATE=false` also force isolated runs.
|
||||
- Fast-local iteration note:
|
||||
- `pnpm test:changed` runs the wrapper with `--changed origin/main`.
|
||||
- The base Vitest config marks the wrapper manifests/config files as `forceRerunTriggers` so changed-mode reruns stay correct when scheduler inputs change.
|
||||
- Use `OPENCLAW_VITEST_FS_MODULE_CACHE=1` for repeated local reruns on a stable branch when transform cost dominates.
|
||||
- Perf-debug note:
|
||||
- `pnpm test:perf:imports` enables Vitest import-duration reporting plus import-breakdown output.
|
||||
- `pnpm test:perf:imports:changed` scopes the same profiling view to files changed since `origin/main`.
|
||||
- `pnpm test:perf:profile:main` writes a main-thread CPU profile for Vitest/Vite startup and transform overhead.
|
||||
- `pnpm test:perf:profile:runner` writes runner CPU+heap profiles for the unit suite with file parallelism disabled.
|
||||
|
||||
### E2E (gateway smoke)
|
||||
|
||||
|
||||
@@ -11,11 +11,17 @@ title: "Tests"
|
||||
|
||||
- `pnpm test:force`: Kills any lingering gateway process holding the default control port, then runs the full Vitest suite with an isolated gateway port so server tests don’t collide with a running instance. Use this when a prior gateway run left port 18789 occupied.
|
||||
- `pnpm test:coverage`: Runs the unit suite with V8 coverage (via `vitest.unit.config.ts`). Global thresholds are 70% lines/branches/functions/statements. Coverage excludes integration-heavy entrypoints (CLI wiring, gateway/telegram bridges, webchat static server) to keep the target focused on unit-testable logic.
|
||||
- `pnpm test:coverage:changed`: Runs unit coverage only for files changed since `origin/main`.
|
||||
- `pnpm test:changed`: runs the wrapper with `--changed origin/main`. The base Vitest config treats the wrapper manifests/config files as `forceRerunTriggers` so scheduler changes still rerun broadly when needed.
|
||||
- `pnpm test`: runs the full wrapper. It keeps only a small behavioral override manifest in git, then uses a checked-in timing snapshot to peel the heaviest measured unit files into dedicated lanes.
|
||||
- Unit files default to `threads` in the wrapper; keep fork-only exceptions documented in `test/fixtures/test-parallel.behavior.json`.
|
||||
- `pnpm test:channels` now defaults to `threads` via `vitest.channels.config.ts`; the March 22, 2026 direct full-suite control run passed clean without channel-specific fork exceptions.
|
||||
- `pnpm test:extensions` now defaults to `threads` via `vitest.extensions.config.ts`; the March 22, 2026 direct full-suite control run passed clean without extension-specific fork exceptions.
|
||||
- `pnpm test:extensions`: runs extension/plugin suites.
|
||||
- `pnpm test:perf:imports`: enables Vitest import-duration + import-breakdown reporting for the wrapper.
|
||||
- `pnpm test:perf:imports:changed`: same import profiling, but only for files changed since `origin/main`.
|
||||
- `pnpm test:perf:profile:main`: writes a CPU profile for the Vitest main thread (`.artifacts/vitest-main-profile`).
|
||||
- `pnpm test:perf:profile:runner`: writes CPU + heap profiles for the unit runner (`.artifacts/vitest-runner-profile`).
|
||||
- `pnpm test:perf:update-timings`: refreshes the checked-in slow-file timing snapshot used by `scripts/test-parallel.mjs`.
|
||||
- Gateway integration: opt-in via `OPENCLAW_TEST_INCLUDE_GATEWAY=1 pnpm test` or `pnpm test:gateway`.
|
||||
- `pnpm test:e2e`: Runs gateway end-to-end smoke tests (multi-instance WS/HTTP/node pairing). Defaults to `forks` + adaptive workers in `vitest.e2e.config.ts`; tune with `OPENCLAW_E2E_WORKERS=<n>` and set `OPENCLAW_E2E_VERBOSE=1` for verbose logs.
|
||||
@@ -33,6 +39,7 @@ For local PR land/gate checks, run:
|
||||
If `pnpm test` flakes on a loaded host, rerun once before treating it as a regression, then isolate with `pnpm vitest run <path/to/test>`. For memory-constrained hosts, use:
|
||||
|
||||
- `OPENCLAW_TEST_PROFILE=low OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test`
|
||||
- `OPENCLAW_VITEST_FS_MODULE_CACHE=1 pnpm test:changed`
|
||||
|
||||
## Model latency bench (local keys)
|
||||
|
||||
|
||||
@@ -691,11 +691,13 @@
|
||||
"test:all": "pnpm lint && pnpm build && pnpm test && pnpm test:e2e && pnpm test:live && pnpm test:docker:all",
|
||||
"test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts",
|
||||
"test:build:singleton": "node scripts/test-built-plugin-singleton.mjs",
|
||||
"test:changed": "pnpm test -- --changed origin/main",
|
||||
"test:channels": "vitest run --config vitest.channels.config.ts",
|
||||
"test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins",
|
||||
"test:contracts:channels": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/channels/plugins/contracts",
|
||||
"test:contracts:plugins": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/plugins/contracts",
|
||||
"test:coverage": "vitest run --config vitest.unit.config.ts --coverage",
|
||||
"test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main",
|
||||
"test:docker:all": "pnpm test:docker:live-models && pnpm test:docker:live-gateway && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup",
|
||||
"test:docker:cleanup": "bash scripts/test-cleanup-docker.sh",
|
||||
"test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh",
|
||||
@@ -726,6 +728,10 @@
|
||||
"test:perf:budget": "node scripts/test-perf-budget.mjs",
|
||||
"test:perf:find-thread-candidates": "node scripts/test-find-thread-candidates.mjs",
|
||||
"test:perf:hotspots": "node scripts/test-hotspots.mjs",
|
||||
"test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 pnpm test",
|
||||
"test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 pnpm test -- --changed origin/main",
|
||||
"test:perf:profile:main": "node --cpu-prof --cpu-prof-dir=.artifacts/vitest-main-profile ./node_modules/vitest/vitest.mjs run --config vitest.unit.config.ts --no-file-parallelism",
|
||||
"test:perf:profile:runner": "vitest run --config vitest.unit.config.ts --no-file-parallelism --execArgv=--cpu-prof --execArgv=--cpu-prof-dir=.artifacts/vitest-runner-profile --execArgv=--heap-prof --execArgv=--heap-prof-dir=.artifacts/vitest-runner-profile",
|
||||
"test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs",
|
||||
"test:perf:update-timings": "node scripts/test-update-timings.mjs",
|
||||
"test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",
|
||||
|
||||
34
test/vitest-performance-config.test.ts
Normal file
34
test/vitest-performance-config.test.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { loadVitestExperimentalConfig } from "../vitest.performance-config.ts";
|
||||
|
||||
describe("loadVitestExperimentalConfig", () => {
|
||||
it("returns an empty object when no perf flags are enabled", () => {
|
||||
expect(loadVitestExperimentalConfig({})).toEqual({});
|
||||
});
|
||||
|
||||
it("enables the filesystem module cache explicitly", () => {
|
||||
expect(
|
||||
loadVitestExperimentalConfig({
|
||||
OPENCLAW_VITEST_FS_MODULE_CACHE: "1",
|
||||
}),
|
||||
).toEqual({
|
||||
experimental: {
|
||||
fsModuleCache: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("enables import timing output and import breakdown reporting", () => {
|
||||
expect(
|
||||
loadVitestExperimentalConfig({
|
||||
OPENCLAW_VITEST_IMPORT_DURATIONS: "true",
|
||||
OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN: "1",
|
||||
}),
|
||||
).toEqual({
|
||||
experimental: {
|
||||
importDurations: { print: true },
|
||||
printImportBreakdown: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,13 @@ describe("createScopedVitestConfig", () => {
|
||||
const config = createScopedVitestConfig(["src/example.test.ts"]);
|
||||
expect(config.test?.isolate).toBe(false);
|
||||
});
|
||||
|
||||
it("passes through a scoped root dir when provided", () => {
|
||||
const config = createScopedVitestConfig(["src/example.test.ts"], {
|
||||
dir: "src",
|
||||
});
|
||||
expect(config.test?.dir).toBe("src");
|
||||
});
|
||||
});
|
||||
|
||||
describe("scoped vitest configs", () => {
|
||||
|
||||
@@ -3,6 +3,12 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { defineConfig } from "vitest/config";
|
||||
import { pluginSdkSubpaths } from "./scripts/lib/plugin-sdk-entries.mjs";
|
||||
import {
|
||||
behaviorManifestPath,
|
||||
unitMemoryHotspotManifestPath,
|
||||
unitTimingManifestPath,
|
||||
} from "./scripts/test-runner-manifest.mjs";
|
||||
import { loadVitestExperimentalConfig } from "./vitest.performance-config.ts";
|
||||
|
||||
const repoRoot = path.dirname(fileURLToPath(import.meta.url));
|
||||
const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true";
|
||||
@@ -37,6 +43,27 @@ export default defineConfig({
|
||||
unstubGlobals: true,
|
||||
pool: "forks",
|
||||
maxWorkers: isCI ? ciWorkers : localWorkers,
|
||||
forceRerunTriggers: [
|
||||
"package.json",
|
||||
"pnpm-lock.yaml",
|
||||
"test/setup.ts",
|
||||
"scripts/test-parallel.mjs",
|
||||
"scripts/test-runner-manifest.mjs",
|
||||
"vitest.channel-paths.mjs",
|
||||
"vitest.channels.config.ts",
|
||||
"vitest.config.ts",
|
||||
"vitest.e2e.config.ts",
|
||||
"vitest.extensions.config.ts",
|
||||
"vitest.gateway.config.ts",
|
||||
"vitest.live.config.ts",
|
||||
"vitest.performance-config.ts",
|
||||
"vitest.scoped-config.ts",
|
||||
"vitest.unit.config.ts",
|
||||
"vitest.unit-paths.mjs",
|
||||
behaviorManifestPath,
|
||||
unitTimingManifestPath,
|
||||
unitMemoryHotspotManifestPath,
|
||||
],
|
||||
include: [
|
||||
"src/**/*.test.ts",
|
||||
"extensions/**/*.test.ts",
|
||||
@@ -154,5 +181,6 @@ export default defineConfig({
|
||||
"src/infra/tailscale.ts",
|
||||
],
|
||||
},
|
||||
...loadVitestExperimentalConfig(),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@ export function loadIncludePatternsFromEnv(
|
||||
export default createScopedVitestConfig(
|
||||
loadIncludePatternsFromEnv() ?? ["extensions/**/*.test.ts"],
|
||||
{
|
||||
dir: "extensions",
|
||||
pool: "threads",
|
||||
// Channel implementations live under extensions/ but are tested by
|
||||
// vitest.channels.config.ts (pnpm test:channels) which provides
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createScopedVitestConfig } from "./vitest.scoped-config.ts";
|
||||
|
||||
export default createScopedVitestConfig(["src/gateway/**/*.test.ts"]);
|
||||
export default createScopedVitestConfig(["src/gateway/**/*.test.ts"], {
|
||||
dir: "src/gateway",
|
||||
});
|
||||
|
||||
32
vitest.performance-config.ts
Normal file
32
vitest.performance-config.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
type EnvMap = Record<string, string | undefined>;
|
||||
|
||||
const isEnabled = (value: string | undefined): boolean => {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
return normalized === "1" || normalized === "true";
|
||||
};
|
||||
|
||||
export function loadVitestExperimentalConfig(env: EnvMap = process.env): {
|
||||
experimental?: {
|
||||
fsModuleCache?: true;
|
||||
importDurations?: { print: true };
|
||||
printImportBreakdown?: true;
|
||||
};
|
||||
} {
|
||||
const experimental: {
|
||||
fsModuleCache?: true;
|
||||
importDurations?: { print: true };
|
||||
printImportBreakdown?: true;
|
||||
} = {};
|
||||
|
||||
if (isEnabled(env.OPENCLAW_VITEST_FS_MODULE_CACHE)) {
|
||||
experimental.fsModuleCache = true;
|
||||
}
|
||||
if (isEnabled(env.OPENCLAW_VITEST_IMPORT_DURATIONS)) {
|
||||
experimental.importDurations = { print: true };
|
||||
}
|
||||
if (isEnabled(env.OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN)) {
|
||||
experimental.printImportBreakdown = true;
|
||||
}
|
||||
|
||||
return Object.keys(experimental).length > 0 ? { experimental } : {};
|
||||
}
|
||||
@@ -13,11 +13,12 @@ export function resolveVitestIsolation(
|
||||
|
||||
export function createScopedVitestConfig(
|
||||
include: string[],
|
||||
options?: { exclude?: string[]; pool?: "threads" | "forks" },
|
||||
options?: { dir?: string; exclude?: string[]; pool?: "threads" | "forks" },
|
||||
) {
|
||||
const base = baseConfig as unknown as Record<string, unknown>;
|
||||
const baseTest =
|
||||
(baseConfig as { test?: { exclude?: string[]; pool?: "threads" | "forks" } }).test ?? {};
|
||||
(baseConfig as { test?: { dir?: string; exclude?: string[]; pool?: "threads" | "forks" } })
|
||||
.test ?? {};
|
||||
const exclude = [...(baseTest.exclude ?? []), ...(options?.exclude ?? [])];
|
||||
|
||||
return defineConfig({
|
||||
@@ -25,6 +26,7 @@ export function createScopedVitestConfig(
|
||||
test: {
|
||||
...baseTest,
|
||||
isolate: resolveVitestIsolation(),
|
||||
...(options?.dir ? { dir: options.dir } : {}),
|
||||
include,
|
||||
exclude,
|
||||
...(options?.pool ? { pool: options.pool } : {}),
|
||||
|
||||
Reference in New Issue
Block a user