test: introduce planner-backed test runner, stabilize local builds (#54650)

* test: stabilize ci and local vitest workers

* test: introduce planner-backed test runner

* test: address planner review follow-ups

* test: derive planner budgets from host capabilities

* test: restore planner filter helper import

* test: align planner explain output with execution

* test: keep low profile as serial alias

* test: restrict explicit planner file targets

* test: clean planner exits and pnpm launch

* test: tighten wrapper flag validation

* ci: gate heavy fanout on check

* test: key shard assignments by unit identity

* ci(bun): shard vitest lanes further

* test: restore ci overlap and stabilize planner tests

* test: relax planner output worker assertions

* test: reset plugin runtime state in optional tools suite

* ci: split macos node and swift jobs

* test: honor no-isolate top-level concurrency budgets

* ci: fix macos swift format lint

* test: cap max-profile top-level concurrency

* ci: shard macos node checks

* ci: use four macos node shards

* test: normalize explain targets before classification
This commit is contained in:
Tak Hoffman
2026-03-25 18:11:58 -05:00
committed by GitHub
parent 764394c78b
commit ab37d8810d
19 changed files with 3243 additions and 1721 deletions

View File

@@ -1,4 +1,6 @@
import { execFileSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
@@ -12,6 +14,13 @@ import {
resolveTestRunExitCode,
} from "../../scripts/test-parallel-utils.mjs";
const clearPlannerShardEnv = (env) => {
const nextEnv = { ...env };
delete nextEnv.OPENCLAW_TEST_SHARDS;
delete nextEnv.OPENCLAW_TEST_SHARD_INDEX;
return nextEnv;
};
describe("scripts/test-parallel fatal output guard", () => {
it("fails a zero exit when V8 reports an out-of-memory fatal", () => {
const output = [
@@ -114,11 +123,10 @@ describe("scripts/test-parallel memory trace parsing", () => {
describe("scripts/test-parallel lane planning", () => {
it("keeps serial profile on split unit lanes instead of one giant unit worker", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
const output = execFileSync("node", ["scripts/test-parallel.mjs"], {
const output = execFileSync("node", ["scripts/test-parallel.mjs", "--plan"], {
cwd: repoRoot,
env: {
...process.env,
OPENCLAW_TEST_LIST_LANES: "1",
...clearPlannerShardEnv(process.env),
OPENCLAW_TEST_PROFILE: "serial",
},
encoding: "utf8",
@@ -130,12 +138,11 @@ describe("scripts/test-parallel lane planning", () => {
it("recycles default local unit-fast runs into bounded batches", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
const output = execFileSync("node", ["scripts/test-parallel.mjs"], {
const output = execFileSync("node", ["scripts/test-parallel.mjs", "--plan"], {
cwd: repoRoot,
env: {
...process.env,
...clearPlannerShardEnv(process.env),
CI: "",
OPENCLAW_TEST_LIST_LANES: "1",
OPENCLAW_TEST_UNIT_FAST_LANES: "1",
OPENCLAW_TEST_UNIT_FAST_BATCH_TARGET_MS: "1",
},
@@ -150,13 +157,15 @@ describe("scripts/test-parallel lane planning", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
const output = execFileSync(
"node",
["scripts/test-parallel.mjs", "src/auto-reply/reply/followup-runner.test.ts"],
[
"scripts/test-parallel.mjs",
"--plan",
"--files",
"src/auto-reply/reply/followup-runner.test.ts",
],
{
cwd: repoRoot,
env: {
...process.env,
OPENCLAW_TEST_LIST_LANES: "1",
},
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
},
);
@@ -164,4 +173,165 @@ describe("scripts/test-parallel lane planning", () => {
expect(output).toContain("base-pinned-followup-runner");
expect(output).not.toContain("base-followup-runner");
});
it("reports capability-derived output for mid-memory local macOS hosts", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
const output = execFileSync(
"node",
["scripts/test-parallel.mjs", "--plan", "--surface", "unit", "--surface", "extensions"],
{
cwd: repoRoot,
env: {
...clearPlannerShardEnv(process.env),
CI: "",
GITHUB_ACTIONS: "",
RUNNER_OS: "macOS",
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
OPENCLAW_TEST_LOAD_AWARE: "0",
},
encoding: "utf8",
},
);
expect(output).toContain("mode=local intent=normal memoryBand=mid");
expect(output).toContain("unit-fast filters=all maxWorkers=");
expect(output).toContain("extensions filters=all maxWorkers=");
});
it("explains targeted file ownership and execution policy", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
const output = execFileSync(
"node",
["scripts/test-parallel.mjs", "--explain", "src/auto-reply/reply/followup-runner.test.ts"],
{
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
},
);
expect(output).toContain("surface=base");
expect(output).toContain("reasons=base-surface,base-pinned-manifest");
expect(output).toContain("pool=forks");
});
it("passes through vitest --mode values that are not wrapper runtime overrides", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
const output = execFileSync(
"node",
[
"scripts/test-parallel.mjs",
"--plan",
"--mode",
"development",
"src/infra/outbound/deliver.test.ts",
],
{
cwd: repoRoot,
env: {
...clearPlannerShardEnv(process.env),
CI: "",
GITHUB_ACTIONS: "",
RUNNER_OS: "Linux",
OPENCLAW_TEST_HOST_CPU_COUNT: "16",
OPENCLAW_TEST_HOST_MEMORY_GIB: "128",
},
encoding: "utf8",
},
);
expect(output).toContain("mode=local intent=normal memoryBand=high");
expect(output).toContain("unit-deliver-isolated filters=1");
});
it("rejects removed machine-name profiles", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
expect(() =>
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--profile", "macmini"], {
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
}),
).toThrowError(/Unsupported test profile "macmini"/u);
});
it("rejects unknown explicit surface names", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
expect(() =>
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--surface", "channel"], {
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
}),
).toThrowError(/Unsupported --surface value\(s\): channel/u);
});
it("rejects wrapper --files values that look like options", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
expect(() =>
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--files", "--config"], {
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
}),
).toThrowError(/Invalid --files value/u);
});
it("rejects missing --profile values", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
expect(() =>
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--profile"], {
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
}),
).toThrowError(/Invalid --profile value/u);
});
it("rejects missing --surface values", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
expect(() =>
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--surface"], {
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
}),
).toThrowError(/Invalid --surface value/u);
});
it("rejects missing --explain values", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
expect(() =>
execFileSync("node", ["scripts/test-parallel.mjs", "--explain"], {
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
}),
).toThrowError(/Invalid --explain value/u);
});
it("rejects explicit existing files that are not known test files", () => {
const repoRoot = path.resolve(import.meta.dirname, "../..");
const tempFilePath = path.join(os.tmpdir(), `openclaw-non-test-${Date.now()}.ts`);
fs.writeFileSync(tempFilePath, "export const notATest = true;\n", "utf8");
try {
expect(() =>
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--files", tempFilePath], {
cwd: repoRoot,
env: clearPlannerShardEnv(process.env),
encoding: "utf8",
}),
).toThrowError(/is not a known test file/u);
} finally {
fs.rmSync(tempFilePath, { force: true });
}
});
});

View File

@@ -0,0 +1,287 @@
import fs from "node:fs";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
createExecutionArtifacts,
resolvePnpmCommandInvocation,
} from "../../scripts/test-planner/executor.mjs";
import { buildExecutionPlan, explainExecutionTarget } from "../../scripts/test-planner/planner.mjs";
describe("test planner", () => {
it("builds a capability-aware plan for mid-memory local runs", () => {
const artifacts = createExecutionArtifacts({
RUNNER_OS: "macOS",
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
OPENCLAW_TEST_LOAD_AWARE: "0",
});
const plan = buildExecutionPlan(
{
profile: null,
mode: "local",
surfaces: ["unit", "extensions"],
passthroughArgs: [],
},
{
env: {
RUNNER_OS: "macOS",
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
OPENCLAW_TEST_LOAD_AWARE: "0",
},
platform: "darwin",
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
},
);
expect(plan.runtimeCapabilities.runtimeProfileName).toBe("local-darwin");
expect(plan.runtimeCapabilities.memoryBand).toBe("mid");
expect(plan.executionBudget.unitSharedWorkers).toBe(4);
expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(8);
expect(plan.executionBudget.topLevelParallelLimitIsolated).toBe(3);
expect(plan.selectedUnits.some((unit) => unit.id.startsWith("unit-fast"))).toBe(true);
expect(plan.selectedUnits.some((unit) => unit.id.startsWith("extensions"))).toBe(true);
expect(plan.topLevelParallelLimit).toBe(8);
artifacts.cleanupTempArtifacts();
});
it("scales down mid-tier local concurrency under saturated load", () => {
const artifacts = createExecutionArtifacts({
RUNNER_OS: "Linux",
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
});
const plan = buildExecutionPlan(
{
profile: null,
mode: "local",
surfaces: ["unit", "extensions"],
passthroughArgs: [],
},
{
env: {
RUNNER_OS: "Linux",
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
},
platform: "linux",
loadAverage: [11.5, 11.5, 11.5],
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
},
);
expect(plan.runtimeCapabilities.memoryBand).toBe("mid");
expect(plan.runtimeCapabilities.loadBand).toBe("saturated");
expect(plan.executionBudget.unitSharedWorkers).toBe(2);
expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(4);
expect(plan.executionBudget.topLevelParallelLimitIsolated).toBe(1);
expect(plan.topLevelParallelLimit).toBe(4);
expect(plan.deferredRunConcurrency).toBe(1);
artifacts.cleanupTempArtifacts();
});
it("honors the max-profile top-level no-isolate cap without adding extra lanes", () => {
const artifacts = createExecutionArtifacts({
RUNNER_OS: "Linux",
OPENCLAW_TEST_HOST_CPU_COUNT: "16",
OPENCLAW_TEST_HOST_MEMORY_GIB: "128",
OPENCLAW_TEST_LOAD_AWARE: "0",
OPENCLAW_TEST_PROFILE: "max",
});
const plan = buildExecutionPlan(
{
profile: "max",
mode: "local",
surfaces: ["unit", "extensions"],
passthroughArgs: [],
},
{
env: {
RUNNER_OS: "Linux",
OPENCLAW_TEST_HOST_CPU_COUNT: "16",
OPENCLAW_TEST_HOST_MEMORY_GIB: "128",
OPENCLAW_TEST_LOAD_AWARE: "0",
OPENCLAW_TEST_PROFILE: "max",
},
platform: "linux",
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
},
);
expect(plan.runtimeCapabilities.intentProfile).toBe("max");
expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(8);
expect(plan.topLevelParallelLimit).toBe(8);
artifacts.cleanupTempArtifacts();
});
it("splits mixed targeted file selections across surfaces", () => {
const artifacts = createExecutionArtifacts({});
const plan = buildExecutionPlan(
{
mode: "local",
surfaces: [],
passthroughArgs: [
"src/auto-reply/reply/followup-runner.test.ts",
"extensions/discord/src/monitor/message-handler.preflight.acp-bindings.test.ts",
],
},
{
env: {},
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
},
);
expect(plan.targetedUnits).toHaveLength(2);
expect(
plan.targetedUnits
.map((unit) => unit.surface)
.toSorted((left, right) => left.localeCompare(right)),
).toEqual(["base", "channels"]);
artifacts.cleanupTempArtifacts();
});
it("explains runtime truth using the same catalog and worker policy", () => {
const explanation = explainExecutionTarget(
{
mode: "local",
fileFilters: ["src/auto-reply/reply/followup-runner.test.ts"],
},
{
env: {},
},
);
expect(explanation.surface).toBe("base");
expect(explanation.pool).toBe("forks");
expect(explanation.reasons).toContain("base-pinned-manifest");
expect(explanation.intentProfile).toBe("normal");
});
it("uses hotspot-backed memory isolation when explaining unit tests", () => {
const explanation = explainExecutionTarget(
{
mode: "local",
fileFilters: ["src/infra/outbound/targets.channel-resolution.test.ts"],
},
{
env: {
OPENCLAW_TEST_LOAD_AWARE: "0",
},
},
);
expect(explanation.isolate).toBe(true);
expect(explanation.reasons).toContain("unit-memory-isolated");
});
it("normalizes absolute explain targets before classification", () => {
const relativeExplanation = explainExecutionTarget(
{
mode: "local",
fileFilters: ["src/infra/outbound/targets.channel-resolution.test.ts"],
},
{
env: {
OPENCLAW_TEST_LOAD_AWARE: "0",
},
},
);
const absoluteExplanation = explainExecutionTarget(
{
mode: "local",
fileFilters: [
path.join(process.cwd(), "src/infra/outbound/targets.channel-resolution.test.ts"),
],
},
{
env: {
OPENCLAW_TEST_LOAD_AWARE: "0",
},
},
);
expect(absoluteExplanation.file).toBe(relativeExplanation.file);
expect(absoluteExplanation.surface).toBe(relativeExplanation.surface);
expect(absoluteExplanation.pool).toBe(relativeExplanation.pool);
expect(absoluteExplanation.isolate).toBe(relativeExplanation.isolate);
expect(absoluteExplanation.reasons).toEqual(relativeExplanation.reasons);
});
it("does not leak default-plan shard assignments into targeted units with the same id", () => {
const artifacts = createExecutionArtifacts({});
const plan = buildExecutionPlan(
{
mode: "local",
fileFilters: ["src/cli/qr-dashboard.integration.test.ts"],
passthroughArgs: [],
},
{
env: {
OPENCLAW_TEST_SHARDS: "4",
OPENCLAW_TEST_SHARD_INDEX: "2",
OPENCLAW_TEST_LOAD_AWARE: "0",
},
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
},
);
const targetedUnit = plan.targetedUnits.at(0);
const defaultUnitWithSameId = plan.allUnits.find((unit) => unit.id === targetedUnit?.id);
expect(targetedUnit).toBeTruthy();
expect(defaultUnitWithSameId).toBeTruthy();
expect(defaultUnitWithSameId).not.toBe(targetedUnit);
expect(plan.topLevelSingleShardAssignments.get(targetedUnit)).toBeUndefined();
expect(plan.topLevelSingleShardAssignments.get(defaultUnitWithSameId)).toBeDefined();
artifacts.cleanupTempArtifacts();
});
it("removes planner temp artifacts when cleanup runs after planning", () => {
const artifacts = createExecutionArtifacts({});
buildExecutionPlan(
{
mode: "local",
surfaces: ["unit"],
passthroughArgs: [],
},
{
env: {},
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
},
);
const artifactDir = artifacts.ensureTempArtifactDir();
expect(fs.existsSync(artifactDir)).toBe(true);
artifacts.cleanupTempArtifacts();
expect(fs.existsSync(artifactDir)).toBe(false);
});
});
describe("resolvePnpmCommandInvocation", () => {
it("prefers the parent pnpm CLI path when npm_execpath points to pnpm", () => {
expect(
resolvePnpmCommandInvocation({
npmExecPath: "/opt/homebrew/lib/node_modules/corepack/dist/pnpm.cjs",
nodeExecPath: "/usr/local/bin/node",
platform: "linux",
}),
).toEqual({
command: "/usr/local/bin/node",
args: ["/opt/homebrew/lib/node_modules/corepack/dist/pnpm.cjs"],
});
});
it("falls back to cmd.exe mediation on Windows when npm_execpath is unavailable", () => {
expect(
resolvePnpmCommandInvocation({
npmExecPath: "",
platform: "win32",
comSpec: "C:\\Windows\\System32\\cmd.exe",
}),
).toEqual({
command: "C:\\Windows\\System32\\cmd.exe",
args: ["/d", "/s", "/c", "pnpm.cmd"],
});
});
});