mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-27 09:21:35 +07:00
test: introduce planner-backed test runner, stabilize local builds (#54650)
* test: stabilize ci and local vitest workers * test: introduce planner-backed test runner * test: address planner review follow-ups * test: derive planner budgets from host capabilities * test: restore planner filter helper import * test: align planner explain output with execution * test: keep low profile as serial alias * test: restrict explicit planner file targets * test: clean planner exits and pnpm launch * test: tighten wrapper flag validation * ci: gate heavy fanout on check * test: key shard assignments by unit identity * ci(bun): shard vitest lanes further * test: restore ci overlap and stabilize planner tests * test: relax planner output worker assertions * test: reset plugin runtime state in optional tools suite * ci: split macos node and swift jobs * test: honor no-isolate top-level concurrency budgets * ci: fix macos swift format lint * test: cap max-profile top-level concurrency * ci: shard macos node checks * ci: use four macos node shards * test: normalize explain targets before classification
This commit is contained in:
Tak Hoffman
14
.github/workflows/ci-bun.yml
vendored
14
.github/workflows/ci-bun.yml
vendored
@@ -45,11 +45,17 @@ jobs:
|
||||
matrix:
|
||||
include:
|
||||
- shard_index: 1
|
||||
shard_count: 2
|
||||
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 1/2
|
||||
shard_count: 4
|
||||
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 1/4
|
||||
- shard_index: 2
|
||||
shard_count: 2
|
||||
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 2/2
|
||||
shard_count: 4
|
||||
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 2/4
|
||||
- shard_index: 3
|
||||
shard_count: 4
|
||||
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 3/4
|
||||
- shard_index: 4
|
||||
shard_count: 4
|
||||
command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 4/4
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
167
.github/workflows/ci.yml
vendored
167
.github/workflows/ci.yml
vendored
@@ -201,9 +201,9 @@ jobs:
|
||||
- name: Audit production dependencies
|
||||
run: pre-commit run --config "${PRE_COMMIT_CONFIG_PATH:-.pre-commit-config.yaml}" --all-files pnpm-audit-prod
|
||||
|
||||
# Fanout: downstream lanes branch from preflight outputs instead of waiting
|
||||
# on unrelated Linux checks.
|
||||
# Build dist once for Node-relevant changes and share it with downstream jobs.
|
||||
# Keep this overlapping with the fast correctness lanes so green PRs get heavy
|
||||
# test/build feedback sooner instead of waiting behind a full `check` pass.
|
||||
build-artifacts:
|
||||
needs: [scope]
|
||||
if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true'
|
||||
@@ -280,7 +280,7 @@ jobs:
|
||||
|
||||
checks-fast:
|
||||
needs: [scope]
|
||||
if: always() && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true'
|
||||
if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true'
|
||||
runs-on: blacksmith-16vcpu-ubuntu-2404
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
@@ -820,13 +820,9 @@ jobs:
|
||||
- name: Run ${{ matrix.task }} (${{ matrix.runtime }})
|
||||
run: ${{ matrix.command }}
|
||||
|
||||
# Consolidated macOS job: runs TS tests + Swift lint/build/test sequentially
|
||||
# on a single runner. GitHub limits macOS concurrent jobs to 5 per org;
|
||||
# running 4 separate jobs per PR (as before) starved the queue. One job
|
||||
# per PR allows 5 PRs to run macOS checks simultaneously.
|
||||
macos:
|
||||
needs: [scope]
|
||||
if: github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true'
|
||||
macos-node-1:
|
||||
needs: [scope, build-artifacts]
|
||||
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
@@ -841,16 +837,157 @@ jobs:
|
||||
with:
|
||||
install-bun: "false"
|
||||
|
||||
- name: Build dist (macOS)
|
||||
run: pnpm build
|
||||
- name: Download dist artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: dist-build
|
||||
path: dist/
|
||||
|
||||
# --- Run all checks sequentially (fast gates first) ---
|
||||
- name: TS tests (macOS)
|
||||
- name: Download A2UI bundle artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: canvas-a2ui-bundle
|
||||
path: src/canvas-host/a2ui/
|
||||
|
||||
- name: Configure test shard (macOS 1/4)
|
||||
run: |
|
||||
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
|
||||
echo "OPENCLAW_TEST_SHARD_INDEX=1" >> "$GITHUB_ENV"
|
||||
|
||||
- name: TS tests (macOS 1/4)
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
run: pnpm test
|
||||
|
||||
# --- Xcode/Swift setup ---
|
||||
macos-node-2:
|
||||
needs: [scope, build-artifacts]
|
||||
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
install-bun: "false"
|
||||
|
||||
- name: Download dist artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: dist-build
|
||||
path: dist/
|
||||
|
||||
- name: Download A2UI bundle artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: canvas-a2ui-bundle
|
||||
path: src/canvas-host/a2ui/
|
||||
|
||||
- name: Configure test shard (macOS 2/4)
|
||||
run: |
|
||||
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
|
||||
echo "OPENCLAW_TEST_SHARD_INDEX=2" >> "$GITHUB_ENV"
|
||||
|
||||
- name: TS tests (macOS 2/4)
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
run: pnpm test
|
||||
|
||||
macos-node-3:
|
||||
needs: [scope, build-artifacts]
|
||||
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
install-bun: "false"
|
||||
|
||||
- name: Download dist artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: dist-build
|
||||
path: dist/
|
||||
|
||||
- name: Download A2UI bundle artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: canvas-a2ui-bundle
|
||||
path: src/canvas-host/a2ui/
|
||||
|
||||
- name: Configure test shard (macOS 3/4)
|
||||
run: |
|
||||
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
|
||||
echo "OPENCLAW_TEST_SHARD_INDEX=3" >> "$GITHUB_ENV"
|
||||
|
||||
- name: TS tests (macOS 3/4)
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
run: pnpm test
|
||||
|
||||
macos-node-4:
|
||||
needs: [scope, build-artifacts]
|
||||
if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success'
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
install-bun: "false"
|
||||
|
||||
- name: Download dist artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: dist-build
|
||||
path: dist/
|
||||
|
||||
- name: Download A2UI bundle artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: canvas-a2ui-bundle
|
||||
path: src/canvas-host/a2ui/
|
||||
|
||||
- name: Configure test shard (macOS 4/4)
|
||||
run: |
|
||||
echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV"
|
||||
echo "OPENCLAW_TEST_SHARD_INDEX=4" >> "$GITHUB_ENV"
|
||||
|
||||
- name: TS tests (macOS 4/4)
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
run: pnpm test
|
||||
|
||||
macos-swift:
|
||||
needs: [scope]
|
||||
if: github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true'
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
|
||||
- name: Select Xcode 26.1
|
||||
run: |
|
||||
sudo xcode-select -s /Applications/Xcode_26.1.app
|
||||
|
||||
@@ -120,7 +120,7 @@
|
||||
- For targeted/local debugging, keep using the wrapper: `pnpm test -- <path-or-filter> [vitest args...]` (for example `pnpm test -- src/commands/onboard-search.test.ts -t "shows registered plugin providers"`); do not default to raw `pnpm vitest run ...` because it bypasses wrapper config/profile/pool routing.
|
||||
- Do not set test workers above 16; tried already.
|
||||
- Keep Vitest on `forks` only. Do not introduce or reintroduce any non-`forks` Vitest pool or alternate execution mode in configs, wrapper scripts, or default test commands without explicit approval in this chat. This includes `threads`, `vmThreads`, `vmForks`, and any future/nonstandard pool variant.
|
||||
- If local Vitest runs cause memory pressure (common on non-Mac-Studio hosts), use `OPENCLAW_TEST_PROFILE=low OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test` for land/gate runs.
|
||||
- If local Vitest runs cause memory pressure, the wrapper now derives budgets from host capabilities (CPU, memory band, current load). For a conservative explicit override during land/gate runs, use `OPENCLAW_TEST_PROFILE=serial OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test`.
|
||||
- Live tests (real keys): `OPENCLAW_LIVE_TEST=1 pnpm test:live` (OpenClaw-only) or `LIVE=1 pnpm test:live` (includes provider live tests). Docker: `pnpm test:docker:live-models`, `pnpm test:docker:live-gateway`. Onboarding Docker E2E: `pnpm test:docker:onboard`.
|
||||
- Full kit + what’s covered: `docs/help/testing.md`.
|
||||
- Changelog: user-facing changes only; no internal/meta notes (version alignment, appcast reminders, release process).
|
||||
|
||||
@@ -493,8 +493,12 @@ enum OpenClawConfigFile {
|
||||
return
|
||||
}
|
||||
|
||||
let backup = self.readConfigFingerprint(at: configURL.deletingLastPathComponent().appendingPathComponent("\(configURL.lastPathComponent).bak"))
|
||||
let clobberedPath = self.persistClobberedSnapshot(data: data, configURL: configURL, observedAt: observedAt)
|
||||
let backup = self.readConfigFingerprint(
|
||||
at: configURL.deletingLastPathComponent().appendingPathComponent("\(configURL.lastPathComponent).bak"))
|
||||
let clobberedPath = self.persistClobberedSnapshot(
|
||||
data: data,
|
||||
configURL: configURL,
|
||||
observedAt: observedAt)
|
||||
self.logger.warning("config observe anomaly (\(suspicious.joined(separator: ", "))) at \(configURL.path)")
|
||||
self.appendConfigObserveAudit([
|
||||
"phase": "read",
|
||||
|
||||
@@ -259,9 +259,12 @@ private struct SkillRow: View {
|
||||
guard let raw = self.skill.homepage?.trimmingCharacters(in: .whitespacesAndNewlines) else {
|
||||
return nil
|
||||
}
|
||||
guard !raw.isEmpty, let url = URL(string: raw),
|
||||
let scheme = url.scheme?.lowercased(),
|
||||
scheme == "http" || scheme == "https" else {
|
||||
guard
|
||||
!raw.isEmpty,
|
||||
let url = URL(string: raw),
|
||||
let scheme = url.scheme?.lowercased(),
|
||||
scheme == "http" || scheme == "https"
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return url
|
||||
@@ -481,9 +484,12 @@ private struct EnvEditorView: View {
|
||||
guard let raw = self.editor.homepage?.trimmingCharacters(in: .whitespacesAndNewlines) else {
|
||||
return nil
|
||||
}
|
||||
guard !raw.isEmpty, let url = URL(string: raw),
|
||||
let scheme = url.scheme?.lowercased(),
|
||||
scheme == "http" || scheme == "https" else {
|
||||
guard
|
||||
!raw.isEmpty,
|
||||
let url = URL(string: raw),
|
||||
let scheme = url.scheme?.lowercased(),
|
||||
scheme == "http" || scheme == "https"
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return url
|
||||
|
||||
10
package.json
10
package.json
@@ -701,10 +701,10 @@
|
||||
"test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts",
|
||||
"test:build:singleton": "node scripts/test-built-plugin-singleton.mjs",
|
||||
"test:changed": "pnpm test -- --changed origin/main",
|
||||
"test:channels": "OPENCLAW_TEST_SKIP_DEFAULT=1 OPENCLAW_TEST_INCLUDE_CHANNELS=1 node scripts/test-parallel.mjs",
|
||||
"test:channels": "node scripts/test-parallel.mjs --surface channels",
|
||||
"test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins",
|
||||
"test:contracts:channels": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/channels/plugins/contracts",
|
||||
"test:contracts:plugins": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/plugins/contracts",
|
||||
"test:contracts:channels": "OPENCLAW_TEST_PROFILE=serial pnpm test -- src/channels/plugins/contracts",
|
||||
"test:contracts:plugins": "OPENCLAW_TEST_PROFILE=serial pnpm test -- src/plugins/contracts",
|
||||
"test:coverage": "vitest run --config vitest.unit.config.ts --coverage",
|
||||
"test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main",
|
||||
"test:docker:all": "pnpm test:docker:live-models && pnpm test:docker:live-gateway && pnpm test:docker:openwebui && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup",
|
||||
@@ -720,7 +720,7 @@
|
||||
"test:e2e": "vitest run --config vitest.e2e.config.ts",
|
||||
"test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts",
|
||||
"test:extension": "node scripts/test-extension.mjs",
|
||||
"test:extensions": "OPENCLAW_TEST_SKIP_DEFAULT=1 OPENCLAW_TEST_INCLUDE_EXTENSIONS=1 node scripts/test-parallel.mjs",
|
||||
"test:extensions": "node scripts/test-parallel.mjs --surface extensions",
|
||||
"test:extensions:memory": "node scripts/profile-extension-memory.mjs",
|
||||
"test:fast": "vitest run --config vitest.unit.config.ts",
|
||||
"test:force": "node --import tsx scripts/test-force.ts",
|
||||
@@ -731,7 +731,6 @@
|
||||
"test:install:e2e:openai": "OPENCLAW_E2E_MODELS=openai bash scripts/test-install-sh-e2e-docker.sh",
|
||||
"test:install:smoke": "bash scripts/test-install-sh-docker.sh",
|
||||
"test:live": "OPENCLAW_LIVE_TEST=1 vitest run --config vitest.live.config.ts",
|
||||
"test:macmini": "OPENCLAW_TEST_PROFILE=macmini node scripts/test-parallel.mjs",
|
||||
"test:parallels:linux": "bash scripts/e2e/parallels-linux-smoke.sh",
|
||||
"test:parallels:macos": "bash scripts/e2e/parallels-macos-smoke.sh",
|
||||
"test:parallels:npm-update": "bash scripts/e2e/parallels-npm-update-smoke.sh",
|
||||
@@ -745,6 +744,7 @@
|
||||
"test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs",
|
||||
"test:perf:update-timings": "node scripts/test-update-timings.mjs",
|
||||
"test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",
|
||||
"test:serial": "node scripts/test-parallel.mjs --profile serial",
|
||||
"test:startup:memory": "node scripts/check-cli-startup-memory.mjs",
|
||||
"test:ui": "pnpm lint:ui:no-raw-window-open && pnpm --dir ui test",
|
||||
"test:voicecall:closedloop": "vitest run extensions/voice-call/src/manager.test.ts extensions/voice-call/src/media-stream.test.ts src/plugins/voice-call.plugin.test.ts --maxWorkers=1",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
187
scripts/test-planner/catalog.mjs
Normal file
187
scripts/test-planner/catalog.mjs
Normal file
@@ -0,0 +1,187 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { channelTestPrefixes } from "../../vitest.channel-paths.mjs";
|
||||
import { isUnitConfigTestFile } from "../../vitest.unit-paths.mjs";
|
||||
import { dedupeFilesPreserveOrder, loadTestRunnerBehavior } from "../test-runner-manifest.mjs";
|
||||
|
||||
const baseConfigPrefixes = ["src/agents/", "src/auto-reply/", "src/commands/", "test/", "ui/"];
|
||||
|
||||
export const normalizeRepoPath = (value) => value.split(path.sep).join("/");
|
||||
|
||||
const toRepoRelativePath = (value) => {
|
||||
const relativePath = normalizeRepoPath(path.relative(process.cwd(), path.resolve(value)));
|
||||
return relativePath.startsWith("../") || relativePath === ".." ? null : relativePath;
|
||||
};
|
||||
|
||||
const walkTestFiles = (rootDir) => {
|
||||
if (!fs.existsSync(rootDir)) {
|
||||
return [];
|
||||
}
|
||||
const entries = fs.readdirSync(rootDir, { withFileTypes: true });
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(rootDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...walkTestFiles(fullPath));
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
fullPath.endsWith(".test.ts") ||
|
||||
fullPath.endsWith(".live.test.ts") ||
|
||||
fullPath.endsWith(".e2e.test.ts")
|
||||
) {
|
||||
files.push(normalizeRepoPath(fullPath));
|
||||
}
|
||||
}
|
||||
return files;
|
||||
};
|
||||
|
||||
export function loadTestCatalog() {
|
||||
const behaviorManifest = loadTestRunnerBehavior();
|
||||
const existingFiles = (entries) =>
|
||||
entries.map((entry) => entry.file).filter((file) => fs.existsSync(file));
|
||||
const existingUnitConfigFiles = (entries) => existingFiles(entries).filter(isUnitConfigTestFile);
|
||||
const baseThreadPinnedFiles = existingFiles(behaviorManifest.base?.threadPinned ?? []);
|
||||
const channelIsolatedManifestFiles = existingFiles(behaviorManifest.channels?.isolated ?? []);
|
||||
const channelIsolatedPrefixes = behaviorManifest.channels?.isolatedPrefixes ?? [];
|
||||
const extensionForkIsolatedFiles = existingFiles(behaviorManifest.extensions?.isolated ?? []);
|
||||
const unitForkIsolatedFiles = existingUnitConfigFiles(behaviorManifest.unit.isolated);
|
||||
const unitThreadPinnedFiles = existingUnitConfigFiles(behaviorManifest.unit.threadPinned);
|
||||
const unitBehaviorOverrideSet = new Set([...unitForkIsolatedFiles, ...unitThreadPinnedFiles]);
|
||||
const allKnownTestFiles = [
|
||||
...new Set([
|
||||
...walkTestFiles("src"),
|
||||
...walkTestFiles("extensions"),
|
||||
...walkTestFiles("test"),
|
||||
...walkTestFiles(path.join("ui", "src", "ui")),
|
||||
]),
|
||||
];
|
||||
const channelIsolatedFiles = dedupeFilesPreserveOrder([
|
||||
...channelIsolatedManifestFiles,
|
||||
...allKnownTestFiles.filter((file) =>
|
||||
channelIsolatedPrefixes.some((prefix) => file.startsWith(prefix)),
|
||||
),
|
||||
]);
|
||||
const channelIsolatedFileSet = new Set(channelIsolatedFiles);
|
||||
const extensionForkIsolatedFileSet = new Set(extensionForkIsolatedFiles);
|
||||
const baseThreadPinnedFileSet = new Set(baseThreadPinnedFiles);
|
||||
const unitThreadPinnedFileSet = new Set(unitThreadPinnedFiles);
|
||||
const unitForkIsolatedFileSet = new Set(unitForkIsolatedFiles);
|
||||
|
||||
const classifyTestFile = (fileFilter, options = {}) => {
|
||||
const normalizedFile = normalizeRepoPath(fileFilter);
|
||||
const reasons = [];
|
||||
const isolated =
|
||||
options.unitMemoryIsolatedFiles?.includes(normalizedFile) ||
|
||||
unitForkIsolatedFileSet.has(normalizedFile) ||
|
||||
extensionForkIsolatedFileSet.has(normalizedFile) ||
|
||||
channelIsolatedFileSet.has(normalizedFile);
|
||||
if (options.unitMemoryIsolatedFiles?.includes(normalizedFile)) {
|
||||
reasons.push("unit-memory-isolated");
|
||||
}
|
||||
if (unitForkIsolatedFileSet.has(normalizedFile)) {
|
||||
reasons.push("unit-isolated-manifest");
|
||||
}
|
||||
if (extensionForkIsolatedFileSet.has(normalizedFile)) {
|
||||
reasons.push("extensions-isolated-manifest");
|
||||
}
|
||||
if (channelIsolatedFileSet.has(normalizedFile)) {
|
||||
reasons.push("channels-isolated-rule");
|
||||
}
|
||||
|
||||
let surface = "base";
|
||||
if (isUnitConfigTestFile(normalizedFile)) {
|
||||
surface = "unit";
|
||||
} else if (normalizedFile.endsWith(".live.test.ts")) {
|
||||
surface = "live";
|
||||
} else if (normalizedFile.endsWith(".e2e.test.ts")) {
|
||||
surface = "e2e";
|
||||
} else if (channelTestPrefixes.some((prefix) => normalizedFile.startsWith(prefix))) {
|
||||
surface = "channels";
|
||||
} else if (normalizedFile.startsWith("extensions/")) {
|
||||
surface = "extensions";
|
||||
} else if (normalizedFile.startsWith("src/gateway/")) {
|
||||
surface = "gateway";
|
||||
} else if (baseConfigPrefixes.some((prefix) => normalizedFile.startsWith(prefix))) {
|
||||
surface = "base";
|
||||
} else if (normalizedFile.startsWith("src/")) {
|
||||
surface = "unit";
|
||||
}
|
||||
if (surface === "unit") {
|
||||
reasons.push("unit-surface");
|
||||
} else if (surface !== "base") {
|
||||
reasons.push(`${surface}-surface`);
|
||||
} else {
|
||||
reasons.push("base-surface");
|
||||
}
|
||||
|
||||
const legacyBasePinned = baseThreadPinnedFileSet.has(normalizedFile);
|
||||
if (legacyBasePinned) {
|
||||
reasons.push("base-pinned-manifest");
|
||||
}
|
||||
if (unitThreadPinnedFileSet.has(normalizedFile)) {
|
||||
reasons.push("unit-pinned-manifest");
|
||||
}
|
||||
|
||||
return {
|
||||
file: normalizedFile,
|
||||
surface,
|
||||
isolated,
|
||||
legacyBasePinned,
|
||||
reasons,
|
||||
};
|
||||
};
|
||||
|
||||
const resolveFilterMatches = (fileFilter) => {
|
||||
const normalizedFilter = normalizeRepoPath(fileFilter);
|
||||
const repoRelativeFilter = toRepoRelativePath(fileFilter);
|
||||
if (fs.existsSync(fileFilter)) {
|
||||
const stats = fs.statSync(fileFilter);
|
||||
if (stats.isFile()) {
|
||||
if (repoRelativeFilter && allKnownTestFiles.includes(repoRelativeFilter)) {
|
||||
return [repoRelativeFilter];
|
||||
}
|
||||
throw new Error(`Explicit path ${fileFilter} is not a known test file.`);
|
||||
}
|
||||
if (stats.isDirectory()) {
|
||||
if (!repoRelativeFilter) {
|
||||
throw new Error(`Explicit path ${fileFilter} is outside the repo test roots.`);
|
||||
}
|
||||
const prefix = repoRelativeFilter.endsWith("/")
|
||||
? repoRelativeFilter
|
||||
: `${repoRelativeFilter}/`;
|
||||
const matches = allKnownTestFiles.filter((file) => file.startsWith(prefix));
|
||||
if (matches.length === 0) {
|
||||
throw new Error(`Explicit path ${fileFilter} does not contain known test files.`);
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
}
|
||||
if (/[*?[\]{}]/.test(normalizedFilter)) {
|
||||
return allKnownTestFiles.filter((file) => path.matchesGlob(file, normalizedFilter));
|
||||
}
|
||||
return allKnownTestFiles.filter((file) => file.includes(normalizedFilter));
|
||||
};
|
||||
|
||||
return {
|
||||
allKnownTestFiles,
|
||||
allKnownUnitFiles: allKnownTestFiles.filter((file) => isUnitConfigTestFile(file)),
|
||||
baseThreadPinnedFiles,
|
||||
channelIsolatedFiles,
|
||||
channelIsolatedFileSet,
|
||||
channelTestPrefixes,
|
||||
extensionForkIsolatedFiles,
|
||||
extensionForkIsolatedFileSet,
|
||||
unitBehaviorOverrideSet,
|
||||
unitForkIsolatedFiles,
|
||||
unitThreadPinnedFiles,
|
||||
baseThreadPinnedFileSet,
|
||||
classifyTestFile,
|
||||
resolveFilterMatches,
|
||||
};
|
||||
}
|
||||
|
||||
export const testSurfaces = ["unit", "extensions", "channels", "gateway", "live", "e2e", "base"];
|
||||
668
scripts/test-planner/executor.mjs
Normal file
668
scripts/test-planner/executor.mjs
Normal file
@@ -0,0 +1,668 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
getProcessTreeRecords,
|
||||
parseCompletedTestFileLines,
|
||||
sampleProcessTreeRssKb,
|
||||
} from "../test-parallel-memory.mjs";
|
||||
import {
|
||||
appendCapturedOutput,
|
||||
formatCapturedOutputTail,
|
||||
hasFatalTestRunOutput,
|
||||
resolveTestRunExitCode,
|
||||
} from "../test-parallel-utils.mjs";
|
||||
import { countExplicitEntryFilters, getExplicitEntryFilters } from "./vitest-args.mjs";
|
||||
|
||||
export function resolvePnpmCommandInvocation(options = {}) {
|
||||
const npmExecPath = typeof options.npmExecPath === "string" ? options.npmExecPath.trim() : "";
|
||||
if (npmExecPath && path.isAbsolute(npmExecPath)) {
|
||||
const npmExecBase = path.basename(npmExecPath).toLowerCase();
|
||||
if (npmExecBase.startsWith("pnpm")) {
|
||||
return {
|
||||
command: options.nodeExecPath || process.execPath,
|
||||
args: [npmExecPath],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (options.platform === "win32") {
|
||||
return {
|
||||
command: options.comSpec || "cmd.exe",
|
||||
args: ["/d", "/s", "/c", "pnpm.cmd"],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
command: "pnpm",
|
||||
args: [],
|
||||
};
|
||||
}
|
||||
|
||||
const sanitizeArtifactName = (value) => {
|
||||
const normalized = value
|
||||
.trim()
|
||||
.replace(/[^a-z0-9._-]+/giu, "-")
|
||||
.replace(/^-+|-+$/gu, "");
|
||||
return normalized || "artifact";
|
||||
};
|
||||
|
||||
const DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB = 4096;
|
||||
const WARNING_SUPPRESSION_FLAGS = [
|
||||
"--disable-warning=ExperimentalWarning",
|
||||
"--disable-warning=DEP0040",
|
||||
"--disable-warning=DEP0060",
|
||||
"--disable-warning=MaxListenersExceededWarning",
|
||||
];
|
||||
|
||||
const formatElapsedMs = (elapsedMs) =>
|
||||
elapsedMs >= 1000 ? `${(elapsedMs / 1000).toFixed(1)}s` : `${Math.round(elapsedMs)}ms`;
|
||||
const formatMemoryKb = (rssKb) =>
|
||||
rssKb >= 1024 ** 2
|
||||
? `${(rssKb / 1024 ** 2).toFixed(2)}GiB`
|
||||
: rssKb >= 1024
|
||||
? `${(rssKb / 1024).toFixed(1)}MiB`
|
||||
: `${rssKb}KiB`;
|
||||
const formatMemoryDeltaKb = (rssKb) =>
|
||||
`${rssKb >= 0 ? "+" : "-"}${formatMemoryKb(Math.abs(rssKb))}`;
|
||||
|
||||
export function createExecutionArtifacts(env = process.env) {
|
||||
let tempArtifactDir = null;
|
||||
const ensureTempArtifactDir = () => {
|
||||
if (tempArtifactDir === null) {
|
||||
tempArtifactDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-test-parallel-"));
|
||||
}
|
||||
return tempArtifactDir;
|
||||
};
|
||||
const writeTempJsonArtifact = (name, value) => {
|
||||
const filePath = path.join(ensureTempArtifactDir(), `${sanitizeArtifactName(name)}.json`);
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8");
|
||||
return filePath;
|
||||
};
|
||||
const cleanupTempArtifacts = () => {
|
||||
if (tempArtifactDir === null) {
|
||||
return;
|
||||
}
|
||||
if (env.OPENCLAW_TEST_KEEP_TEMP_ARTIFACTS === "1") {
|
||||
console.error(`[test-parallel] keeping temp artifacts at ${tempArtifactDir}`);
|
||||
return;
|
||||
}
|
||||
fs.rmSync(tempArtifactDir, { recursive: true, force: true });
|
||||
tempArtifactDir = null;
|
||||
};
|
||||
return { ensureTempArtifactDir, writeTempJsonArtifact, cleanupTempArtifacts };
|
||||
}
|
||||
|
||||
const ensureNodeOptionFlag = (nodeOptions, flagPrefix, nextValue) =>
|
||||
nodeOptions.includes(flagPrefix) ? nodeOptions : `${nodeOptions} ${nextValue}`.trim();
|
||||
|
||||
const isNodeLikeProcess = (command) => /(?:^|\/)node(?:$|\.exe$)/iu.test(command);
|
||||
|
||||
const getShardLabel = (args) => {
|
||||
const shardIndex = args.findIndex((arg) => arg === "--shard");
|
||||
if (shardIndex < 0) {
|
||||
return "";
|
||||
}
|
||||
return typeof args[shardIndex + 1] === "string" ? args[shardIndex + 1] : "";
|
||||
};
|
||||
|
||||
export function formatPlanOutput(plan) {
|
||||
return [
|
||||
`runtime=${plan.runtimeCapabilities.runtimeProfileName} mode=${plan.runtimeCapabilities.mode} intent=${plan.runtimeCapabilities.intentProfile} memoryBand=${plan.runtimeCapabilities.memoryBand} loadBand=${plan.runtimeCapabilities.loadBand} vitestMaxWorkers=${String(plan.executionBudget.vitestMaxWorkers ?? "default")} topLevelParallel=${plan.topLevelParallelEnabled ? String(plan.topLevelParallelLimit) : "off"}`,
|
||||
...plan.selectedUnits.map(
|
||||
(unit) =>
|
||||
`${unit.id} filters=${String(countExplicitEntryFilters(unit.args) ?? "all")} maxWorkers=${String(
|
||||
unit.maxWorkers ?? "default",
|
||||
)} surface=${unit.surface} isolate=${unit.isolate ? "yes" : "no"} pool=${unit.pool}`,
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function formatExplanation(explanation) {
|
||||
return [
|
||||
`file=${explanation.file}`,
|
||||
`runtime=${explanation.runtimeProfile} intent=${explanation.intentProfile} memoryBand=${explanation.memoryBand} loadBand=${explanation.loadBand}`,
|
||||
`surface=${explanation.surface}`,
|
||||
`isolate=${explanation.isolate ? "yes" : "no"}`,
|
||||
`pool=${explanation.pool}`,
|
||||
`maxWorkers=${String(explanation.maxWorkers ?? "default")}`,
|
||||
`reasons=${explanation.reasons.join(",")}`,
|
||||
`command=${explanation.args.join(" ")}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function executePlan(plan, options = {}) {
|
||||
const env = options.env ?? process.env;
|
||||
const artifacts = options.artifacts ?? createExecutionArtifacts(env);
|
||||
const pnpmInvocation = resolvePnpmCommandInvocation({
|
||||
npmExecPath: env.npm_execpath,
|
||||
nodeExecPath: process.execPath,
|
||||
platform: process.platform,
|
||||
comSpec: env.ComSpec,
|
||||
});
|
||||
const children = new Set();
|
||||
const windowsCiArgs = plan.runtimeCapabilities.isWindowsCi
|
||||
? ["--dangerouslyIgnoreUnhandledErrors"]
|
||||
: [];
|
||||
const silentArgs = env.OPENCLAW_TEST_SHOW_PASSED_LOGS === "1" ? [] : ["--silent=passed-only"];
|
||||
const rawMemoryTrace = env.OPENCLAW_TEST_MEMORY_TRACE?.trim().toLowerCase();
|
||||
const memoryTraceEnabled =
|
||||
process.platform !== "win32" &&
|
||||
(rawMemoryTrace === "1" ||
|
||||
rawMemoryTrace === "true" ||
|
||||
(rawMemoryTrace !== "0" && rawMemoryTrace !== "false" && plan.runtimeCapabilities.isCI));
|
||||
const parseEnvNumber = (name, fallback) => {
|
||||
const parsed = Number.parseInt(env[name] ?? "", 10);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
};
|
||||
const memoryTracePollMs = Math.max(
|
||||
250,
|
||||
parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_POLL_MS", 1000),
|
||||
);
|
||||
const memoryTraceTopCount = Math.max(
|
||||
1,
|
||||
parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_TOP_COUNT", 6),
|
||||
);
|
||||
const requestedHeapSnapshotIntervalMs = Math.max(
|
||||
0,
|
||||
parseEnvNumber("OPENCLAW_TEST_HEAPSNAPSHOT_INTERVAL_MS", 0),
|
||||
);
|
||||
const heapSnapshotMinIntervalMs = 1000;
|
||||
const heapSnapshotIntervalMs =
|
||||
requestedHeapSnapshotIntervalMs > 0
|
||||
? Math.max(heapSnapshotMinIntervalMs, requestedHeapSnapshotIntervalMs)
|
||||
: 0;
|
||||
const heapSnapshotEnabled =
|
||||
process.platform !== "win32" && heapSnapshotIntervalMs >= heapSnapshotMinIntervalMs;
|
||||
const heapSnapshotSignal = env.OPENCLAW_TEST_HEAPSNAPSHOT_SIGNAL?.trim() || "SIGUSR2";
|
||||
const heapSnapshotBaseDir = heapSnapshotEnabled
|
||||
? path.resolve(
|
||||
env.OPENCLAW_TEST_HEAPSNAPSHOT_DIR?.trim() ||
|
||||
path.join(os.tmpdir(), `openclaw-heapsnapshots-${Date.now()}`),
|
||||
)
|
||||
: null;
|
||||
const maxOldSpaceSizeMb = (() => {
|
||||
const raw = env.OPENCLAW_TEST_MAX_OLD_SPACE_SIZE_MB ?? "";
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (Number.isFinite(parsed) && parsed > 0) {
|
||||
return parsed;
|
||||
}
|
||||
if (plan.runtimeCapabilities.isCI && !plan.runtimeCapabilities.isWindows) {
|
||||
return DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB;
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
const shutdown = (signal) => {
|
||||
for (const child of children) {
|
||||
child.kill(signal);
|
||||
}
|
||||
};
|
||||
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||
process.on("exit", artifacts.cleanupTempArtifacts);
|
||||
|
||||
const runOnce = (unit, extraArgs = []) =>
|
||||
new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
const entryArgs = unit.args;
|
||||
const explicitEntryFilters = getExplicitEntryFilters(entryArgs);
|
||||
const args = unit.maxWorkers
|
||||
? [
|
||||
...entryArgs,
|
||||
"--maxWorkers",
|
||||
String(unit.maxWorkers),
|
||||
...silentArgs,
|
||||
...windowsCiArgs,
|
||||
...extraArgs,
|
||||
]
|
||||
: [...entryArgs, ...silentArgs, ...windowsCiArgs, ...extraArgs];
|
||||
const spawnArgs = [...pnpmInvocation.args, ...args];
|
||||
const shardLabel = getShardLabel(extraArgs);
|
||||
const artifactStem = [
|
||||
sanitizeArtifactName(unit.id),
|
||||
shardLabel ? `shard-${sanitizeArtifactName(shardLabel)}` : "",
|
||||
String(startedAt),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("-");
|
||||
const laneLogPath = path.join(artifacts.ensureTempArtifactDir(), `${artifactStem}.log`);
|
||||
const laneLogStream = fs.createWriteStream(laneLogPath, { flags: "w" });
|
||||
laneLogStream.write(`[test-parallel] entry=${unit.id}\n`);
|
||||
laneLogStream.write(`[test-parallel] cwd=${process.cwd()}\n`);
|
||||
laneLogStream.write(
|
||||
`[test-parallel] command=${[pnpmInvocation.command, ...spawnArgs].join(" ")}\n\n`,
|
||||
);
|
||||
console.log(
|
||||
`[test-parallel] start ${unit.id} workers=${unit.maxWorkers ?? "default"} filters=${String(
|
||||
countExplicitEntryFilters(entryArgs) ?? "all",
|
||||
)}`,
|
||||
);
|
||||
const nodeOptions = env.NODE_OPTIONS ?? "";
|
||||
const nextNodeOptions = WARNING_SUPPRESSION_FLAGS.reduce(
|
||||
(acc, flag) => (acc.includes(flag) ? acc : `${acc} ${flag}`.trim()),
|
||||
nodeOptions,
|
||||
);
|
||||
const heapSnapshotDir =
|
||||
heapSnapshotBaseDir === null ? null : path.join(heapSnapshotBaseDir, unit.id);
|
||||
let resolvedNodeOptions =
|
||||
maxOldSpaceSizeMb && !nextNodeOptions.includes("--max-old-space-size=")
|
||||
? `${nextNodeOptions} --max-old-space-size=${maxOldSpaceSizeMb}`.trim()
|
||||
: nextNodeOptions;
|
||||
if (heapSnapshotEnabled && heapSnapshotDir) {
|
||||
try {
|
||||
fs.mkdirSync(heapSnapshotDir, { recursive: true });
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[test-parallel] failed to create heap snapshot dir ${heapSnapshotDir}: ${String(err)}`,
|
||||
);
|
||||
resolve(1);
|
||||
return;
|
||||
}
|
||||
resolvedNodeOptions = ensureNodeOptionFlag(
|
||||
resolvedNodeOptions,
|
||||
"--diagnostic-dir=",
|
||||
`--diagnostic-dir=${heapSnapshotDir}`,
|
||||
);
|
||||
resolvedNodeOptions = ensureNodeOptionFlag(
|
||||
resolvedNodeOptions,
|
||||
"--heapsnapshot-signal=",
|
||||
`--heapsnapshot-signal=${heapSnapshotSignal}`,
|
||||
);
|
||||
}
|
||||
let output = "";
|
||||
let fatalSeen = false;
|
||||
let childError = null;
|
||||
let child;
|
||||
let pendingLine = "";
|
||||
let memoryPollTimer = null;
|
||||
let heapSnapshotTimer = null;
|
||||
const memoryFileRecords = [];
|
||||
let initialTreeSample = null;
|
||||
let latestTreeSample = null;
|
||||
let peakTreeSample = null;
|
||||
let heapSnapshotSequence = 0;
|
||||
const updatePeakTreeSample = (sample, reason) => {
|
||||
if (!sample) {
|
||||
return;
|
||||
}
|
||||
if (!peakTreeSample || sample.rssKb > peakTreeSample.rssKb) {
|
||||
peakTreeSample = { ...sample, reason };
|
||||
}
|
||||
};
|
||||
const triggerHeapSnapshot = (reason) => {
|
||||
if (!heapSnapshotEnabled || !child?.pid || !heapSnapshotDir) {
|
||||
return;
|
||||
}
|
||||
const records = getProcessTreeRecords(child.pid) ?? [];
|
||||
const targetPids = records
|
||||
.filter((record) => record.pid !== process.pid && isNodeLikeProcess(record.command))
|
||||
.map((record) => record.pid);
|
||||
if (targetPids.length === 0) {
|
||||
return;
|
||||
}
|
||||
heapSnapshotSequence += 1;
|
||||
let signaledCount = 0;
|
||||
for (const pid of targetPids) {
|
||||
try {
|
||||
process.kill(pid, heapSnapshotSignal);
|
||||
signaledCount += 1;
|
||||
} catch {}
|
||||
}
|
||||
if (signaledCount > 0) {
|
||||
console.log(
|
||||
`[test-parallel][heap] ${unit.id} seq=${String(heapSnapshotSequence)} reason=${reason} signaled=${String(
|
||||
signaledCount,
|
||||
)}/${String(targetPids.length)} dir=${heapSnapshotDir}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
const captureTreeSample = (reason) => {
|
||||
if (!memoryTraceEnabled || !child?.pid) {
|
||||
return null;
|
||||
}
|
||||
const sample = sampleProcessTreeRssKb(child.pid);
|
||||
if (!sample) {
|
||||
return null;
|
||||
}
|
||||
latestTreeSample = sample;
|
||||
if (!initialTreeSample) {
|
||||
initialTreeSample = sample;
|
||||
}
|
||||
updatePeakTreeSample(sample, reason);
|
||||
return sample;
|
||||
};
|
||||
const logMemoryTraceForText = (text) => {
|
||||
if (!memoryTraceEnabled) {
|
||||
return;
|
||||
}
|
||||
const combined = `${pendingLine}${text}`;
|
||||
const lines = combined.split(/\r?\n/u);
|
||||
pendingLine = lines.pop() ?? "";
|
||||
const completedFiles = parseCompletedTestFileLines(lines.join("\n"));
|
||||
for (const completedFile of completedFiles) {
|
||||
const sample = captureTreeSample(completedFile.file);
|
||||
if (!sample) {
|
||||
continue;
|
||||
}
|
||||
const previousRssKb =
|
||||
memoryFileRecords.length > 0
|
||||
? (memoryFileRecords.at(-1)?.rssKb ?? initialTreeSample?.rssKb ?? sample.rssKb)
|
||||
: (initialTreeSample?.rssKb ?? sample.rssKb);
|
||||
const deltaKb = sample.rssKb - previousRssKb;
|
||||
const record = {
|
||||
...completedFile,
|
||||
rssKb: sample.rssKb,
|
||||
processCount: sample.processCount,
|
||||
deltaKb,
|
||||
};
|
||||
memoryFileRecords.push(record);
|
||||
console.log(
|
||||
`[test-parallel][mem] ${unit.id} file=${record.file} rss=${formatMemoryKb(
|
||||
record.rssKb,
|
||||
)} delta=${formatMemoryDeltaKb(record.deltaKb)} peak=${formatMemoryKb(
|
||||
peakTreeSample?.rssKb ?? record.rssKb,
|
||||
)} procs=${record.processCount}${record.durationMs ? ` duration=${formatElapsedMs(record.durationMs)}` : ""}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
const logMemoryTraceSummary = () => {
|
||||
if (!memoryTraceEnabled) {
|
||||
return;
|
||||
}
|
||||
captureTreeSample("close");
|
||||
const fallbackRecord =
|
||||
memoryFileRecords.length === 0 &&
|
||||
explicitEntryFilters.length === 1 &&
|
||||
latestTreeSample &&
|
||||
initialTreeSample
|
||||
? [
|
||||
{
|
||||
file: explicitEntryFilters[0],
|
||||
deltaKb: latestTreeSample.rssKb - initialTreeSample.rssKb,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
const totalDeltaKb =
|
||||
initialTreeSample && latestTreeSample
|
||||
? latestTreeSample.rssKb - initialTreeSample.rssKb
|
||||
: 0;
|
||||
const topGrowthFiles = [...memoryFileRecords, ...fallbackRecord]
|
||||
.filter((record) => record.deltaKb > 0 && typeof record.file === "string")
|
||||
.toSorted((left, right) => right.deltaKb - left.deltaKb)
|
||||
.slice(0, memoryTraceTopCount)
|
||||
.map((record) => `${record.file}:${formatMemoryDeltaKb(record.deltaKb)}`);
|
||||
console.log(
|
||||
`[test-parallel][mem] summary ${unit.id} files=${memoryFileRecords.length} peak=${formatMemoryKb(
|
||||
peakTreeSample?.rssKb ?? 0,
|
||||
)} totalDelta=${formatMemoryDeltaKb(totalDeltaKb)} peakAt=${
|
||||
peakTreeSample?.reason ?? "n/a"
|
||||
} top=${topGrowthFiles.length > 0 ? topGrowthFiles.join(", ") : "none"}`,
|
||||
);
|
||||
};
|
||||
try {
|
||||
child = spawn(pnpmInvocation.command, spawnArgs, {
|
||||
stdio: ["inherit", "pipe", "pipe"],
|
||||
env: {
|
||||
...env,
|
||||
...unit.env,
|
||||
VITEST_GROUP: unit.id,
|
||||
NODE_OPTIONS: resolvedNodeOptions,
|
||||
},
|
||||
shell: false,
|
||||
});
|
||||
captureTreeSample("spawn");
|
||||
if (memoryTraceEnabled) {
|
||||
memoryPollTimer = setInterval(() => {
|
||||
captureTreeSample("poll");
|
||||
}, memoryTracePollMs);
|
||||
}
|
||||
if (heapSnapshotEnabled) {
|
||||
heapSnapshotTimer = setInterval(() => {
|
||||
triggerHeapSnapshot("interval");
|
||||
}, heapSnapshotIntervalMs);
|
||||
}
|
||||
} catch (err) {
|
||||
laneLogStream.end();
|
||||
console.error(`[test-parallel] spawn failed: ${String(err)}`);
|
||||
resolve(1);
|
||||
return;
|
||||
}
|
||||
children.add(child);
|
||||
child.stdout?.on("data", (chunk) => {
|
||||
const text = chunk.toString();
|
||||
fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
|
||||
output = appendCapturedOutput(output, text);
|
||||
laneLogStream.write(text);
|
||||
logMemoryTraceForText(text);
|
||||
process.stdout.write(chunk);
|
||||
});
|
||||
child.stderr?.on("data", (chunk) => {
|
||||
const text = chunk.toString();
|
||||
fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
|
||||
output = appendCapturedOutput(output, text);
|
||||
laneLogStream.write(text);
|
||||
logMemoryTraceForText(text);
|
||||
process.stderr.write(chunk);
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
childError = err;
|
||||
laneLogStream.write(`\n[test-parallel] child error: ${String(err)}\n`);
|
||||
console.error(`[test-parallel] child error: ${String(err)}`);
|
||||
});
|
||||
child.on("close", (code, signal) => {
|
||||
if (memoryPollTimer) {
|
||||
clearInterval(memoryPollTimer);
|
||||
}
|
||||
if (heapSnapshotTimer) {
|
||||
clearInterval(heapSnapshotTimer);
|
||||
}
|
||||
children.delete(child);
|
||||
const resolvedCode = resolveTestRunExitCode({
|
||||
code,
|
||||
signal,
|
||||
output,
|
||||
fatalSeen,
|
||||
childError,
|
||||
});
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
logMemoryTraceSummary();
|
||||
if (resolvedCode !== 0) {
|
||||
const failureTail = formatCapturedOutputTail(output);
|
||||
const failureArtifactPath = artifacts.writeTempJsonArtifact(`${artifactStem}-failure`, {
|
||||
entry: unit.id,
|
||||
command: [pnpmInvocation.command, ...spawnArgs],
|
||||
elapsedMs,
|
||||
error: childError ? String(childError) : null,
|
||||
exitCode: resolvedCode,
|
||||
fatalSeen,
|
||||
logPath: laneLogPath,
|
||||
outputTail: failureTail,
|
||||
signal: signal ?? null,
|
||||
});
|
||||
if (failureTail) {
|
||||
console.error(`[test-parallel] failure tail ${unit.id}\n${failureTail}`);
|
||||
}
|
||||
console.error(
|
||||
`[test-parallel] failure artifacts ${unit.id} log=${laneLogPath} meta=${failureArtifactPath}`,
|
||||
);
|
||||
}
|
||||
laneLogStream.write(
|
||||
`\n[test-parallel] done ${unit.id} code=${String(resolvedCode)} signal=${
|
||||
signal ?? "none"
|
||||
} elapsed=${formatElapsedMs(elapsedMs)}\n`,
|
||||
);
|
||||
laneLogStream.end();
|
||||
console.log(
|
||||
`[test-parallel] done ${unit.id} code=${String(resolvedCode)} elapsed=${formatElapsedMs(elapsedMs)}`,
|
||||
);
|
||||
resolve(resolvedCode);
|
||||
});
|
||||
});
|
||||
|
||||
const runUnit = async (unit, extraArgs = []) => {
|
||||
if (unit.fixedShardIndex !== undefined) {
|
||||
if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== unit.fixedShardIndex) {
|
||||
return 0;
|
||||
}
|
||||
return runOnce(unit, extraArgs);
|
||||
}
|
||||
const explicitFilterCount = countExplicitEntryFilters(unit.args);
|
||||
const topLevelAssignedShard = plan.topLevelSingleShardAssignments.get(unit);
|
||||
if (topLevelAssignedShard !== undefined) {
|
||||
if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== topLevelAssignedShard) {
|
||||
return 0;
|
||||
}
|
||||
return runOnce(unit, extraArgs);
|
||||
}
|
||||
const effectiveShardCount =
|
||||
explicitFilterCount === null
|
||||
? plan.shardCount
|
||||
: Math.min(plan.shardCount, Math.max(1, explicitFilterCount - 1));
|
||||
if (effectiveShardCount <= 1) {
|
||||
if (plan.shardIndexOverride !== null && plan.shardIndexOverride > effectiveShardCount) {
|
||||
return 0;
|
||||
}
|
||||
return runOnce(unit, extraArgs);
|
||||
}
|
||||
if (plan.shardIndexOverride !== null) {
|
||||
if (plan.shardIndexOverride > effectiveShardCount) {
|
||||
return 0;
|
||||
}
|
||||
return runOnce(unit, [
|
||||
"--shard",
|
||||
`${plan.shardIndexOverride}/${effectiveShardCount}`,
|
||||
...extraArgs,
|
||||
]);
|
||||
}
|
||||
for (let shardIndex = 1; shardIndex <= effectiveShardCount; shardIndex += 1) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const code = await runOnce(unit, [
|
||||
"--shard",
|
||||
`${shardIndex}/${effectiveShardCount}`,
|
||||
...extraArgs,
|
||||
]);
|
||||
if (code !== 0) {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
|
||||
const runUnitsWithLimit = async (units, extraArgs = [], concurrency = 1) => {
|
||||
if (units.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const normalizedConcurrency = Math.max(1, Math.floor(concurrency));
|
||||
if (normalizedConcurrency <= 1) {
|
||||
for (const unit of units) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const code = await runUnit(unit, extraArgs);
|
||||
if (code !== 0) {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
let nextIndex = 0;
|
||||
let firstFailure;
|
||||
const worker = async () => {
|
||||
while (firstFailure === undefined) {
|
||||
const unitIndex = nextIndex;
|
||||
nextIndex += 1;
|
||||
if (unitIndex >= units.length) {
|
||||
return;
|
||||
}
|
||||
const code = await runUnit(units[unitIndex], extraArgs);
|
||||
if (code !== 0 && firstFailure === undefined) {
|
||||
firstFailure = code;
|
||||
}
|
||||
}
|
||||
};
|
||||
const workerCount = Math.min(normalizedConcurrency, units.length);
|
||||
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
||||
return firstFailure;
|
||||
};
|
||||
|
||||
const runUnits = async (units, extraArgs = []) => {
|
||||
if (plan.topLevelParallelEnabled) {
|
||||
return runUnitsWithLimit(units, extraArgs, plan.topLevelParallelLimit);
|
||||
}
|
||||
return runUnitsWithLimit(units, extraArgs);
|
||||
};
|
||||
|
||||
if (plan.passthroughMetadataOnly) {
|
||||
return runOnce(
|
||||
{
|
||||
id: "vitest-meta",
|
||||
args: ["vitest", "run"],
|
||||
maxWorkers: null,
|
||||
},
|
||||
plan.passthroughOptionArgs,
|
||||
);
|
||||
}
|
||||
|
||||
if (plan.targetedUnits.length > 0) {
|
||||
if (plan.passthroughRequiresSingleRun && plan.targetedUnits.length > 1) {
|
||||
console.error(
|
||||
"[test-parallel] The provided Vitest args require a single run, but the selected test filters span multiple wrapper configs. Run one target/config at a time.",
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
const failedTargetedParallel = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs);
|
||||
if (failedTargetedParallel !== undefined) {
|
||||
return failedTargetedParallel;
|
||||
}
|
||||
for (const unit of plan.serialUnits) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const code = await runUnit(unit, plan.passthroughOptionArgs);
|
||||
if (code !== 0) {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (plan.passthroughRequiresSingleRun && plan.passthroughOptionArgs.length > 0) {
|
||||
console.error(
|
||||
"[test-parallel] The provided Vitest args require a single run. Use the dedicated npm script for that workflow (for example `pnpm test:coverage`) or target a single test file/filter.",
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (plan.serialPrefixUnits.length > 0) {
|
||||
const failedSerialPrefix = await runUnitsWithLimit(
|
||||
plan.serialPrefixUnits,
|
||||
plan.passthroughOptionArgs,
|
||||
1,
|
||||
);
|
||||
if (failedSerialPrefix !== undefined) {
|
||||
return failedSerialPrefix;
|
||||
}
|
||||
const failedDeferredParallel = plan.deferredRunConcurrency
|
||||
? await runUnitsWithLimit(
|
||||
plan.deferredParallelUnits,
|
||||
plan.passthroughOptionArgs,
|
||||
plan.deferredRunConcurrency,
|
||||
)
|
||||
: await runUnits(plan.deferredParallelUnits, plan.passthroughOptionArgs);
|
||||
if (failedDeferredParallel !== undefined) {
|
||||
return failedDeferredParallel;
|
||||
}
|
||||
} else {
|
||||
const failedParallel = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs);
|
||||
if (failedParallel !== undefined) {
|
||||
return failedParallel;
|
||||
}
|
||||
}
|
||||
|
||||
for (const unit of plan.serialUnits) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const code = await runUnit(unit, plan.passthroughOptionArgs);
|
||||
if (code !== 0) {
|
||||
return code;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
1023
scripts/test-planner/planner.mjs
Normal file
1023
scripts/test-planner/planner.mjs
Normal file
File diff suppressed because it is too large
Load Diff
348
scripts/test-planner/runtime-profile.mjs
Normal file
348
scripts/test-planner/runtime-profile.mjs
Normal file
@@ -0,0 +1,348 @@
|
||||
import os from "node:os";
|
||||
|
||||
export const TEST_PROFILES = new Set(["normal", "serial", "max"]);
|
||||
|
||||
export const parsePositiveInt = (value) => {
|
||||
const parsed = Number.parseInt(value ?? "", 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||||
};
|
||||
|
||||
export const resolveVitestMode = (env = process.env, explicitMode = null) => {
|
||||
if (explicitMode === "ci" || explicitMode === "local") {
|
||||
return explicitMode;
|
||||
}
|
||||
return env.CI === "true" || env.GITHUB_ACTIONS === "true" ? "ci" : "local";
|
||||
};
|
||||
|
||||
const clamp = (value, min, max) => Math.max(min, Math.min(max, value));
|
||||
|
||||
const parseProfile = (rawProfile) => {
|
||||
if (!rawProfile) {
|
||||
return "normal";
|
||||
}
|
||||
const normalized = rawProfile.trim().toLowerCase();
|
||||
if (normalized === "low") {
|
||||
return "serial";
|
||||
}
|
||||
if (!TEST_PROFILES.has(normalized)) {
|
||||
throw new Error(
|
||||
`Unsupported test profile "${normalized}". Supported profiles: normal, serial, max.`,
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
};
|
||||
|
||||
const resolveLoadRatio = (env, cpuCount, platform, loadAverage) => {
|
||||
const loadAwareDisabledRaw = env.OPENCLAW_TEST_LOAD_AWARE?.trim().toLowerCase();
|
||||
const loadAwareDisabled = loadAwareDisabledRaw === "0" || loadAwareDisabledRaw === "false";
|
||||
if (loadAwareDisabled || platform === "win32" || cpuCount <= 0) {
|
||||
return 0;
|
||||
}
|
||||
const source = Array.isArray(loadAverage) ? loadAverage : os.loadavg();
|
||||
return source.length > 0 ? source[0] / cpuCount : 0;
|
||||
};
|
||||
|
||||
const resolveMemoryBand = (memoryGiB) => {
|
||||
if (memoryGiB < 24) {
|
||||
return "constrained";
|
||||
}
|
||||
if (memoryGiB < 48) {
|
||||
return "moderate";
|
||||
}
|
||||
if (memoryGiB < 96) {
|
||||
return "mid";
|
||||
}
|
||||
return "high";
|
||||
};
|
||||
|
||||
const resolveLoadBand = (isLoadAware, loadRatio) => {
|
||||
if (!isLoadAware) {
|
||||
return "normal";
|
||||
}
|
||||
if (loadRatio < 0.5) {
|
||||
return "idle";
|
||||
}
|
||||
if (loadRatio < 0.9) {
|
||||
return "normal";
|
||||
}
|
||||
if (loadRatio < 1.1) {
|
||||
return "busy";
|
||||
}
|
||||
return "saturated";
|
||||
};
|
||||
|
||||
const scaleForLoad = (value, loadBand) => {
|
||||
if (value === null || value === undefined) {
|
||||
return value;
|
||||
}
|
||||
const scale = loadBand === "busy" ? 0.75 : loadBand === "saturated" ? 0.5 : 1;
|
||||
return Math.max(1, Math.floor(value * scale));
|
||||
};
|
||||
|
||||
const scaleConcurrencyForLoad = (value, loadBand) => {
|
||||
if (value === null || value === undefined) {
|
||||
return value;
|
||||
}
|
||||
const scale = loadBand === "busy" ? 0.8 : loadBand === "saturated" ? 0.5 : 1;
|
||||
return Math.max(1, Math.floor(value * scale));
|
||||
};
|
||||
|
||||
const LOCAL_MEMORY_BUDGETS = {
|
||||
constrained: {
|
||||
vitestCap: 2,
|
||||
unitShared: 2,
|
||||
unitIsolated: 1,
|
||||
unitHeavy: 1,
|
||||
extensions: 1,
|
||||
gateway: 1,
|
||||
topLevelNoIsolate: 4,
|
||||
topLevelIsolated: 2,
|
||||
deferred: 1,
|
||||
heavyFileLimit: 36,
|
||||
heavyLaneCount: 3,
|
||||
memoryHeavyFileLimit: 8,
|
||||
unitFastBatchTargetMs: 10_000,
|
||||
},
|
||||
moderate: {
|
||||
vitestCap: 3,
|
||||
unitShared: 3,
|
||||
unitIsolated: 1,
|
||||
unitHeavy: 1,
|
||||
extensions: 2,
|
||||
gateway: 1,
|
||||
topLevelNoIsolate: 6,
|
||||
topLevelIsolated: 2,
|
||||
deferred: 1,
|
||||
heavyFileLimit: 48,
|
||||
heavyLaneCount: 4,
|
||||
memoryHeavyFileLimit: 12,
|
||||
unitFastBatchTargetMs: 15_000,
|
||||
},
|
||||
mid: {
|
||||
vitestCap: 4,
|
||||
unitShared: 4,
|
||||
unitIsolated: 1,
|
||||
unitHeavy: 1,
|
||||
extensions: 3,
|
||||
gateway: 1,
|
||||
topLevelNoIsolate: 8,
|
||||
topLevelIsolated: 3,
|
||||
deferred: 2,
|
||||
heavyFileLimit: 60,
|
||||
heavyLaneCount: 4,
|
||||
memoryHeavyFileLimit: 16,
|
||||
unitFastBatchTargetMs: 0,
|
||||
},
|
||||
high: {
|
||||
vitestCap: 6,
|
||||
unitShared: 6,
|
||||
unitIsolated: 2,
|
||||
unitHeavy: 2,
|
||||
extensions: 4,
|
||||
gateway: 3,
|
||||
topLevelNoIsolate: 12,
|
||||
topLevelIsolated: 4,
|
||||
deferred: 3,
|
||||
heavyFileLimit: 80,
|
||||
heavyLaneCount: 5,
|
||||
memoryHeavyFileLimit: 16,
|
||||
unitFastBatchTargetMs: 45_000,
|
||||
},
|
||||
};
|
||||
|
||||
const withIntentBudgetAdjustments = (budget, intentProfile, cpuCount) => {
|
||||
if (intentProfile === "serial") {
|
||||
return {
|
||||
...budget,
|
||||
vitestMaxWorkers: 1,
|
||||
unitSharedWorkers: 1,
|
||||
unitIsolatedWorkers: 1,
|
||||
unitHeavyWorkers: 1,
|
||||
extensionWorkers: 1,
|
||||
gatewayWorkers: 1,
|
||||
topLevelParallelEnabled: false,
|
||||
topLevelParallelLimit: 1,
|
||||
topLevelParallelLimitNoIsolate: 1,
|
||||
topLevelParallelLimitIsolated: 1,
|
||||
deferredRunConcurrency: 1,
|
||||
};
|
||||
}
|
||||
|
||||
if (intentProfile === "max") {
|
||||
const maxTopLevelParallelLimit = clamp(
|
||||
Math.max(budget.topLevelParallelLimitNoIsolate ?? budget.topLevelParallelLimit ?? 1, 5),
|
||||
1,
|
||||
8,
|
||||
);
|
||||
return {
|
||||
...budget,
|
||||
vitestMaxWorkers: clamp(Math.max(budget.vitestMaxWorkers, Math.min(8, cpuCount)), 1, 16),
|
||||
unitSharedWorkers: clamp(Math.max(budget.unitSharedWorkers, Math.min(8, cpuCount)), 1, 16),
|
||||
unitIsolatedWorkers: clamp(Math.max(budget.unitIsolatedWorkers, Math.min(4, cpuCount)), 1, 4),
|
||||
unitHeavyWorkers: clamp(Math.max(budget.unitHeavyWorkers, Math.min(4, cpuCount)), 1, 4),
|
||||
extensionWorkers: clamp(Math.max(budget.extensionWorkers, Math.min(6, cpuCount)), 1, 6),
|
||||
gatewayWorkers: clamp(Math.max(budget.gatewayWorkers, Math.min(2, cpuCount)), 1, 6),
|
||||
topLevelParallelEnabled: true,
|
||||
topLevelParallelLimit: maxTopLevelParallelLimit,
|
||||
topLevelParallelLimitNoIsolate: maxTopLevelParallelLimit,
|
||||
topLevelParallelLimitIsolated: clamp(
|
||||
Math.max(budget.topLevelParallelLimitIsolated ?? budget.topLevelParallelLimit ?? 1, 4),
|
||||
1,
|
||||
8,
|
||||
),
|
||||
deferredRunConcurrency: Math.max(budget.deferredRunConcurrency ?? 1, 3),
|
||||
};
|
||||
}
|
||||
|
||||
return budget;
|
||||
};
|
||||
|
||||
export function resolveRuntimeCapabilities(env = process.env, options = {}) {
|
||||
const mode = resolveVitestMode(env, options.mode ?? null);
|
||||
const isCI = mode === "ci";
|
||||
const platform = options.platform ?? process.platform;
|
||||
const runnerOs = env.RUNNER_OS ?? "";
|
||||
const isMacOS = platform === "darwin" || runnerOs === "macOS";
|
||||
const isWindows = platform === "win32" || runnerOs === "Windows";
|
||||
const isWindowsCi = isCI && isWindows;
|
||||
const hostCpuCount =
|
||||
parsePositiveInt(env.OPENCLAW_TEST_HOST_CPU_COUNT) ?? options.cpuCount ?? os.cpus().length;
|
||||
const totalMemoryBytes = options.totalMemoryBytes ?? os.totalmem();
|
||||
const hostMemoryGiB =
|
||||
parsePositiveInt(env.OPENCLAW_TEST_HOST_MEMORY_GIB) ?? Math.floor(totalMemoryBytes / 1024 ** 3);
|
||||
const nodeMajor = Number.parseInt(
|
||||
(options.nodeVersion ?? process.versions.node).split(".")[0] ?? "",
|
||||
10,
|
||||
);
|
||||
const intentProfile = parseProfile(options.profile ?? env.OPENCLAW_TEST_PROFILE ?? "normal");
|
||||
const loadRatio = !isCI ? resolveLoadRatio(env, hostCpuCount, platform, options.loadAverage) : 0;
|
||||
const loadAware = !isCI && platform !== "win32";
|
||||
const memoryBand = resolveMemoryBand(hostMemoryGiB);
|
||||
const loadBand = resolveLoadBand(loadAware, loadRatio);
|
||||
const runtimeProfileName = isCI
|
||||
? isWindows
|
||||
? "ci-windows"
|
||||
: isMacOS
|
||||
? "ci-macos"
|
||||
: "ci-linux"
|
||||
: isWindows
|
||||
? "local-windows"
|
||||
: isMacOS
|
||||
? "local-darwin"
|
||||
: "local-linux";
|
||||
|
||||
return {
|
||||
mode,
|
||||
runtimeProfileName,
|
||||
isCI,
|
||||
isMacOS,
|
||||
isWindows,
|
||||
isWindowsCi,
|
||||
platform,
|
||||
hostCpuCount,
|
||||
hostMemoryGiB,
|
||||
nodeMajor,
|
||||
intentProfile,
|
||||
memoryBand,
|
||||
loadAware,
|
||||
loadRatio,
|
||||
loadBand,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveExecutionBudget(runtimeCapabilities) {
|
||||
const runtime = runtimeCapabilities;
|
||||
const cpuCount = clamp(runtime.hostCpuCount, 1, 16);
|
||||
|
||||
if (runtime.isCI) {
|
||||
const macCiWorkers = runtime.isMacOS ? 1 : null;
|
||||
return {
|
||||
vitestMaxWorkers: runtime.isWindows ? 2 : runtime.isMacOS ? 1 : 3,
|
||||
unitSharedWorkers: macCiWorkers,
|
||||
unitIsolatedWorkers: macCiWorkers,
|
||||
unitHeavyWorkers: macCiWorkers,
|
||||
extensionWorkers: macCiWorkers,
|
||||
gatewayWorkers: macCiWorkers,
|
||||
topLevelParallelEnabled: runtime.intentProfile !== "serial" && !runtime.isWindows,
|
||||
topLevelParallelLimit: runtime.isWindows ? 2 : 4,
|
||||
topLevelParallelLimitNoIsolate: runtime.isWindows ? 2 : 4,
|
||||
topLevelParallelLimitIsolated: runtime.isWindows ? 2 : 4,
|
||||
deferredRunConcurrency: null,
|
||||
heavyUnitFileLimit: 64,
|
||||
heavyUnitLaneCount: 4,
|
||||
memoryHeavyUnitFileLimit: 64,
|
||||
unitFastLaneCount: runtime.isWindows ? 1 : 3,
|
||||
unitFastBatchTargetMs: runtime.isWindows ? 0 : 45_000,
|
||||
channelsBatchTargetMs: runtime.isWindows ? 0 : 30_000,
|
||||
extensionsBatchTargetMs: runtime.isWindows ? 0 : 30_000,
|
||||
};
|
||||
}
|
||||
|
||||
const bandBudget = LOCAL_MEMORY_BUDGETS[runtime.memoryBand];
|
||||
const baseBudget = {
|
||||
vitestMaxWorkers: Math.min(cpuCount, bandBudget.vitestCap),
|
||||
unitSharedWorkers: Math.min(cpuCount, bandBudget.unitShared),
|
||||
unitIsolatedWorkers: Math.min(cpuCount, bandBudget.unitIsolated),
|
||||
unitHeavyWorkers: Math.min(cpuCount, bandBudget.unitHeavy),
|
||||
extensionWorkers: Math.min(cpuCount, bandBudget.extensions),
|
||||
gatewayWorkers: Math.min(cpuCount, bandBudget.gateway),
|
||||
topLevelParallelEnabled: runtime.nodeMajor < 25,
|
||||
topLevelParallelLimit: Math.min(cpuCount, bandBudget.topLevelIsolated),
|
||||
topLevelParallelLimitNoIsolate: Math.min(cpuCount, bandBudget.topLevelNoIsolate),
|
||||
topLevelParallelLimitIsolated: Math.min(cpuCount, bandBudget.topLevelIsolated),
|
||||
deferredRunConcurrency: bandBudget.deferred,
|
||||
heavyUnitFileLimit: bandBudget.heavyFileLimit,
|
||||
heavyUnitLaneCount: bandBudget.heavyLaneCount,
|
||||
memoryHeavyUnitFileLimit: bandBudget.memoryHeavyFileLimit,
|
||||
unitFastLaneCount: 1,
|
||||
unitFastBatchTargetMs: bandBudget.unitFastBatchTargetMs,
|
||||
channelsBatchTargetMs: 0,
|
||||
extensionsBatchTargetMs: 0,
|
||||
};
|
||||
|
||||
const loadAdjustedBudget = {
|
||||
...baseBudget,
|
||||
vitestMaxWorkers: scaleForLoad(baseBudget.vitestMaxWorkers, runtime.loadBand),
|
||||
unitSharedWorkers: scaleForLoad(baseBudget.unitSharedWorkers, runtime.loadBand),
|
||||
unitHeavyWorkers: scaleForLoad(baseBudget.unitHeavyWorkers, runtime.loadBand),
|
||||
extensionWorkers: scaleForLoad(baseBudget.extensionWorkers, runtime.loadBand),
|
||||
gatewayWorkers: scaleForLoad(baseBudget.gatewayWorkers, runtime.loadBand),
|
||||
topLevelParallelLimit: scaleConcurrencyForLoad(
|
||||
baseBudget.topLevelParallelLimit,
|
||||
runtime.loadBand,
|
||||
),
|
||||
topLevelParallelLimitNoIsolate: scaleConcurrencyForLoad(
|
||||
baseBudget.topLevelParallelLimitNoIsolate,
|
||||
runtime.loadBand,
|
||||
),
|
||||
topLevelParallelLimitIsolated: scaleConcurrencyForLoad(
|
||||
baseBudget.topLevelParallelLimitIsolated,
|
||||
runtime.loadBand,
|
||||
),
|
||||
deferredRunConcurrency:
|
||||
runtime.loadBand === "busy"
|
||||
? Math.max(1, (baseBudget.deferredRunConcurrency ?? 1) - 1)
|
||||
: runtime.loadBand === "saturated"
|
||||
? 1
|
||||
: baseBudget.deferredRunConcurrency,
|
||||
};
|
||||
|
||||
return withIntentBudgetAdjustments(loadAdjustedBudget, runtime.intentProfile, cpuCount);
|
||||
}
|
||||
|
||||
export function resolveLocalVitestMaxWorkers(env = process.env, options = {}) {
|
||||
const explicit = parsePositiveInt(env.OPENCLAW_VITEST_MAX_WORKERS);
|
||||
if (explicit !== null) {
|
||||
return explicit;
|
||||
}
|
||||
|
||||
const runtimeCapabilities = resolveRuntimeCapabilities(env, {
|
||||
cpuCount: options.cpuCount,
|
||||
totalMemoryBytes: options.totalMemoryBytes,
|
||||
platform: options.platform,
|
||||
mode: "local",
|
||||
loadAverage: options.loadAverage,
|
||||
profile: options.profile,
|
||||
});
|
||||
return resolveExecutionBudget(runtimeCapabilities).vitestMaxWorkers;
|
||||
}
|
||||
74
scripts/test-planner/vitest-args.mjs
Normal file
74
scripts/test-planner/vitest-args.mjs
Normal file
@@ -0,0 +1,74 @@
|
||||
export const OPTION_TAKES_VALUE = new Set([
|
||||
"-t",
|
||||
"-c",
|
||||
"-r",
|
||||
"--testNamePattern",
|
||||
"--config",
|
||||
"--root",
|
||||
"--dir",
|
||||
"--reporter",
|
||||
"--outputFile",
|
||||
"--pool",
|
||||
"--execArgv",
|
||||
"--vmMemoryLimit",
|
||||
"--maxWorkers",
|
||||
"--environment",
|
||||
"--shard",
|
||||
"--changed",
|
||||
"--sequence",
|
||||
"--inspect",
|
||||
"--inspectBrk",
|
||||
"--testTimeout",
|
||||
"--hookTimeout",
|
||||
"--bail",
|
||||
"--retry",
|
||||
"--diff",
|
||||
"--exclude",
|
||||
"--project",
|
||||
"--slowTestThreshold",
|
||||
"--teardownTimeout",
|
||||
"--attachmentsDir",
|
||||
"--mode",
|
||||
"--api",
|
||||
"--browser",
|
||||
"--maxConcurrency",
|
||||
"--mergeReports",
|
||||
"--configLoader",
|
||||
"--experimental",
|
||||
]);
|
||||
|
||||
export const SINGLE_RUN_ONLY_FLAGS = new Set(["--coverage", "--outputFile", "--mergeReports"]);
|
||||
|
||||
export const parsePassthroughArgs = (args = []) => {
|
||||
const fileFilters = [];
|
||||
const optionArgs = [];
|
||||
let consumeNextAsOptionValue = false;
|
||||
|
||||
for (const arg of args) {
|
||||
if (consumeNextAsOptionValue) {
|
||||
optionArgs.push(arg);
|
||||
consumeNextAsOptionValue = false;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--") {
|
||||
optionArgs.push(arg);
|
||||
continue;
|
||||
}
|
||||
if (typeof arg === "string" && arg.startsWith("-")) {
|
||||
optionArgs.push(arg);
|
||||
consumeNextAsOptionValue = !arg.includes("=") && OPTION_TAKES_VALUE.has(arg);
|
||||
continue;
|
||||
}
|
||||
fileFilters.push(arg);
|
||||
}
|
||||
|
||||
return { fileFilters, optionArgs };
|
||||
};
|
||||
|
||||
export const countExplicitEntryFilters = (entryArgs) => {
|
||||
const { fileFilters } = parsePassthroughArgs(entryArgs.slice(2));
|
||||
return fileFilters.length > 0 ? fileFilters.length : null;
|
||||
};
|
||||
|
||||
export const getExplicitEntryFilters = (entryArgs) =>
|
||||
parsePassthroughArgs(entryArgs.slice(2)).fileFilters;
|
||||
@@ -14,6 +14,7 @@ vi.mock("./loader.js", () => ({
|
||||
}));
|
||||
|
||||
let resolvePluginTools: typeof import("./tools.js").resolvePluginTools;
|
||||
let resetPluginRuntimeStateForTest: typeof import("./runtime.js").resetPluginRuntimeStateForTest;
|
||||
|
||||
function makeTool(name: string) {
|
||||
return {
|
||||
@@ -94,6 +95,8 @@ describe("resolvePluginTools optional tools", () => {
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
loadOpenClawPluginsMock.mockClear();
|
||||
({ resetPluginRuntimeStateForTest } = await import("./runtime.js"));
|
||||
resetPluginRuntimeStateForTest();
|
||||
({ resolvePluginTools } = await import("./tools.js"));
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
@@ -12,6 +14,13 @@ import {
|
||||
resolveTestRunExitCode,
|
||||
} from "../../scripts/test-parallel-utils.mjs";
|
||||
|
||||
const clearPlannerShardEnv = (env) => {
|
||||
const nextEnv = { ...env };
|
||||
delete nextEnv.OPENCLAW_TEST_SHARDS;
|
||||
delete nextEnv.OPENCLAW_TEST_SHARD_INDEX;
|
||||
return nextEnv;
|
||||
};
|
||||
|
||||
describe("scripts/test-parallel fatal output guard", () => {
|
||||
it("fails a zero exit when V8 reports an out-of-memory fatal", () => {
|
||||
const output = [
|
||||
@@ -114,11 +123,10 @@ describe("scripts/test-parallel memory trace parsing", () => {
|
||||
describe("scripts/test-parallel lane planning", () => {
|
||||
it("keeps serial profile on split unit lanes instead of one giant unit worker", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const output = execFileSync("node", ["scripts/test-parallel.mjs"], {
|
||||
const output = execFileSync("node", ["scripts/test-parallel.mjs", "--plan"], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_TEST_LIST_LANES: "1",
|
||||
...clearPlannerShardEnv(process.env),
|
||||
OPENCLAW_TEST_PROFILE: "serial",
|
||||
},
|
||||
encoding: "utf8",
|
||||
@@ -130,12 +138,11 @@ describe("scripts/test-parallel lane planning", () => {
|
||||
|
||||
it("recycles default local unit-fast runs into bounded batches", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const output = execFileSync("node", ["scripts/test-parallel.mjs"], {
|
||||
const output = execFileSync("node", ["scripts/test-parallel.mjs", "--plan"], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
...clearPlannerShardEnv(process.env),
|
||||
CI: "",
|
||||
OPENCLAW_TEST_LIST_LANES: "1",
|
||||
OPENCLAW_TEST_UNIT_FAST_LANES: "1",
|
||||
OPENCLAW_TEST_UNIT_FAST_BATCH_TARGET_MS: "1",
|
||||
},
|
||||
@@ -150,13 +157,15 @@ describe("scripts/test-parallel lane planning", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const output = execFileSync(
|
||||
"node",
|
||||
["scripts/test-parallel.mjs", "src/auto-reply/reply/followup-runner.test.ts"],
|
||||
[
|
||||
"scripts/test-parallel.mjs",
|
||||
"--plan",
|
||||
"--files",
|
||||
"src/auto-reply/reply/followup-runner.test.ts",
|
||||
],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_TEST_LIST_LANES: "1",
|
||||
},
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
@@ -164,4 +173,165 @@ describe("scripts/test-parallel lane planning", () => {
|
||||
expect(output).toContain("base-pinned-followup-runner");
|
||||
expect(output).not.toContain("base-followup-runner");
|
||||
});
|
||||
|
||||
it("reports capability-derived output for mid-memory local macOS hosts", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const output = execFileSync(
|
||||
"node",
|
||||
["scripts/test-parallel.mjs", "--plan", "--surface", "unit", "--surface", "extensions"],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...clearPlannerShardEnv(process.env),
|
||||
CI: "",
|
||||
GITHUB_ACTIONS: "",
|
||||
RUNNER_OS: "macOS",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
},
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(output).toContain("mode=local intent=normal memoryBand=mid");
|
||||
expect(output).toContain("unit-fast filters=all maxWorkers=");
|
||||
expect(output).toContain("extensions filters=all maxWorkers=");
|
||||
});
|
||||
|
||||
it("explains targeted file ownership and execution policy", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const output = execFileSync(
|
||||
"node",
|
||||
["scripts/test-parallel.mjs", "--explain", "src/auto-reply/reply/followup-runner.test.ts"],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(output).toContain("surface=base");
|
||||
expect(output).toContain("reasons=base-surface,base-pinned-manifest");
|
||||
expect(output).toContain("pool=forks");
|
||||
});
|
||||
|
||||
it("passes through vitest --mode values that are not wrapper runtime overrides", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const output = execFileSync(
|
||||
"node",
|
||||
[
|
||||
"scripts/test-parallel.mjs",
|
||||
"--plan",
|
||||
"--mode",
|
||||
"development",
|
||||
"src/infra/outbound/deliver.test.ts",
|
||||
],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...clearPlannerShardEnv(process.env),
|
||||
CI: "",
|
||||
GITHUB_ACTIONS: "",
|
||||
RUNNER_OS: "Linux",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "16",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "128",
|
||||
},
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
|
||||
expect(output).toContain("mode=local intent=normal memoryBand=high");
|
||||
expect(output).toContain("unit-deliver-isolated filters=1");
|
||||
});
|
||||
|
||||
it("rejects removed machine-name profiles", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--profile", "macmini"], {
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrowError(/Unsupported test profile "macmini"/u);
|
||||
});
|
||||
|
||||
it("rejects unknown explicit surface names", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--surface", "channel"], {
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrowError(/Unsupported --surface value\(s\): channel/u);
|
||||
});
|
||||
|
||||
it("rejects wrapper --files values that look like options", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--files", "--config"], {
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrowError(/Invalid --files value/u);
|
||||
});
|
||||
|
||||
it("rejects missing --profile values", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--profile"], {
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrowError(/Invalid --profile value/u);
|
||||
});
|
||||
|
||||
it("rejects missing --surface values", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--surface"], {
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrowError(/Invalid --surface value/u);
|
||||
});
|
||||
|
||||
it("rejects missing --explain values", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/test-parallel.mjs", "--explain"], {
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrowError(/Invalid --explain value/u);
|
||||
});
|
||||
|
||||
it("rejects explicit existing files that are not known test files", () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const tempFilePath = path.join(os.tmpdir(), `openclaw-non-test-${Date.now()}.ts`);
|
||||
fs.writeFileSync(tempFilePath, "export const notATest = true;\n", "utf8");
|
||||
|
||||
try {
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--files", tempFilePath], {
|
||||
cwd: repoRoot,
|
||||
env: clearPlannerShardEnv(process.env),
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrowError(/is not a known test file/u);
|
||||
} finally {
|
||||
fs.rmSync(tempFilePath, { force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
287
test/scripts/test-planner.test.ts
Normal file
287
test/scripts/test-planner.test.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createExecutionArtifacts,
|
||||
resolvePnpmCommandInvocation,
|
||||
} from "../../scripts/test-planner/executor.mjs";
|
||||
import { buildExecutionPlan, explainExecutionTarget } from "../../scripts/test-planner/planner.mjs";
|
||||
|
||||
describe("test planner", () => {
|
||||
it("builds a capability-aware plan for mid-memory local runs", () => {
|
||||
const artifacts = createExecutionArtifacts({
|
||||
RUNNER_OS: "macOS",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
});
|
||||
const plan = buildExecutionPlan(
|
||||
{
|
||||
profile: null,
|
||||
mode: "local",
|
||||
surfaces: ["unit", "extensions"],
|
||||
passthroughArgs: [],
|
||||
},
|
||||
{
|
||||
env: {
|
||||
RUNNER_OS: "macOS",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
},
|
||||
platform: "darwin",
|
||||
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
|
||||
},
|
||||
);
|
||||
|
||||
expect(plan.runtimeCapabilities.runtimeProfileName).toBe("local-darwin");
|
||||
expect(plan.runtimeCapabilities.memoryBand).toBe("mid");
|
||||
expect(plan.executionBudget.unitSharedWorkers).toBe(4);
|
||||
expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(8);
|
||||
expect(plan.executionBudget.topLevelParallelLimitIsolated).toBe(3);
|
||||
expect(plan.selectedUnits.some((unit) => unit.id.startsWith("unit-fast"))).toBe(true);
|
||||
expect(plan.selectedUnits.some((unit) => unit.id.startsWith("extensions"))).toBe(true);
|
||||
expect(plan.topLevelParallelLimit).toBe(8);
|
||||
artifacts.cleanupTempArtifacts();
|
||||
});
|
||||
|
||||
it("scales down mid-tier local concurrency under saturated load", () => {
|
||||
const artifacts = createExecutionArtifacts({
|
||||
RUNNER_OS: "Linux",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
|
||||
});
|
||||
const plan = buildExecutionPlan(
|
||||
{
|
||||
profile: null,
|
||||
mode: "local",
|
||||
surfaces: ["unit", "extensions"],
|
||||
passthroughArgs: [],
|
||||
},
|
||||
{
|
||||
env: {
|
||||
RUNNER_OS: "Linux",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "10",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "64",
|
||||
},
|
||||
platform: "linux",
|
||||
loadAverage: [11.5, 11.5, 11.5],
|
||||
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
|
||||
},
|
||||
);
|
||||
|
||||
expect(plan.runtimeCapabilities.memoryBand).toBe("mid");
|
||||
expect(plan.runtimeCapabilities.loadBand).toBe("saturated");
|
||||
expect(plan.executionBudget.unitSharedWorkers).toBe(2);
|
||||
expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(4);
|
||||
expect(plan.executionBudget.topLevelParallelLimitIsolated).toBe(1);
|
||||
expect(plan.topLevelParallelLimit).toBe(4);
|
||||
expect(plan.deferredRunConcurrency).toBe(1);
|
||||
artifacts.cleanupTempArtifacts();
|
||||
});
|
||||
|
||||
it("honors the max-profile top-level no-isolate cap without adding extra lanes", () => {
|
||||
const artifacts = createExecutionArtifacts({
|
||||
RUNNER_OS: "Linux",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "16",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "128",
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
OPENCLAW_TEST_PROFILE: "max",
|
||||
});
|
||||
const plan = buildExecutionPlan(
|
||||
{
|
||||
profile: "max",
|
||||
mode: "local",
|
||||
surfaces: ["unit", "extensions"],
|
||||
passthroughArgs: [],
|
||||
},
|
||||
{
|
||||
env: {
|
||||
RUNNER_OS: "Linux",
|
||||
OPENCLAW_TEST_HOST_CPU_COUNT: "16",
|
||||
OPENCLAW_TEST_HOST_MEMORY_GIB: "128",
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
OPENCLAW_TEST_PROFILE: "max",
|
||||
},
|
||||
platform: "linux",
|
||||
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
|
||||
},
|
||||
);
|
||||
|
||||
expect(plan.runtimeCapabilities.intentProfile).toBe("max");
|
||||
expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(8);
|
||||
expect(plan.topLevelParallelLimit).toBe(8);
|
||||
artifacts.cleanupTempArtifacts();
|
||||
});
|
||||
|
||||
it("splits mixed targeted file selections across surfaces", () => {
|
||||
const artifacts = createExecutionArtifacts({});
|
||||
const plan = buildExecutionPlan(
|
||||
{
|
||||
mode: "local",
|
||||
surfaces: [],
|
||||
passthroughArgs: [
|
||||
"src/auto-reply/reply/followup-runner.test.ts",
|
||||
"extensions/discord/src/monitor/message-handler.preflight.acp-bindings.test.ts",
|
||||
],
|
||||
},
|
||||
{
|
||||
env: {},
|
||||
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
|
||||
},
|
||||
);
|
||||
|
||||
expect(plan.targetedUnits).toHaveLength(2);
|
||||
expect(
|
||||
plan.targetedUnits
|
||||
.map((unit) => unit.surface)
|
||||
.toSorted((left, right) => left.localeCompare(right)),
|
||||
).toEqual(["base", "channels"]);
|
||||
artifacts.cleanupTempArtifacts();
|
||||
});
|
||||
|
||||
it("explains runtime truth using the same catalog and worker policy", () => {
|
||||
const explanation = explainExecutionTarget(
|
||||
{
|
||||
mode: "local",
|
||||
fileFilters: ["src/auto-reply/reply/followup-runner.test.ts"],
|
||||
},
|
||||
{
|
||||
env: {},
|
||||
},
|
||||
);
|
||||
|
||||
expect(explanation.surface).toBe("base");
|
||||
expect(explanation.pool).toBe("forks");
|
||||
expect(explanation.reasons).toContain("base-pinned-manifest");
|
||||
expect(explanation.intentProfile).toBe("normal");
|
||||
});
|
||||
|
||||
it("uses hotspot-backed memory isolation when explaining unit tests", () => {
|
||||
const explanation = explainExecutionTarget(
|
||||
{
|
||||
mode: "local",
|
||||
fileFilters: ["src/infra/outbound/targets.channel-resolution.test.ts"],
|
||||
},
|
||||
{
|
||||
env: {
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(explanation.isolate).toBe(true);
|
||||
expect(explanation.reasons).toContain("unit-memory-isolated");
|
||||
});
|
||||
|
||||
it("normalizes absolute explain targets before classification", () => {
|
||||
const relativeExplanation = explainExecutionTarget(
|
||||
{
|
||||
mode: "local",
|
||||
fileFilters: ["src/infra/outbound/targets.channel-resolution.test.ts"],
|
||||
},
|
||||
{
|
||||
env: {
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
},
|
||||
},
|
||||
);
|
||||
const absoluteExplanation = explainExecutionTarget(
|
||||
{
|
||||
mode: "local",
|
||||
fileFilters: [
|
||||
path.join(process.cwd(), "src/infra/outbound/targets.channel-resolution.test.ts"),
|
||||
],
|
||||
},
|
||||
{
|
||||
env: {
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(absoluteExplanation.file).toBe(relativeExplanation.file);
|
||||
expect(absoluteExplanation.surface).toBe(relativeExplanation.surface);
|
||||
expect(absoluteExplanation.pool).toBe(relativeExplanation.pool);
|
||||
expect(absoluteExplanation.isolate).toBe(relativeExplanation.isolate);
|
||||
expect(absoluteExplanation.reasons).toEqual(relativeExplanation.reasons);
|
||||
});
|
||||
|
||||
it("does not leak default-plan shard assignments into targeted units with the same id", () => {
|
||||
const artifacts = createExecutionArtifacts({});
|
||||
const plan = buildExecutionPlan(
|
||||
{
|
||||
mode: "local",
|
||||
fileFilters: ["src/cli/qr-dashboard.integration.test.ts"],
|
||||
passthroughArgs: [],
|
||||
},
|
||||
{
|
||||
env: {
|
||||
OPENCLAW_TEST_SHARDS: "4",
|
||||
OPENCLAW_TEST_SHARD_INDEX: "2",
|
||||
OPENCLAW_TEST_LOAD_AWARE: "0",
|
||||
},
|
||||
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
|
||||
},
|
||||
);
|
||||
|
||||
const targetedUnit = plan.targetedUnits.at(0);
|
||||
const defaultUnitWithSameId = plan.allUnits.find((unit) => unit.id === targetedUnit?.id);
|
||||
|
||||
expect(targetedUnit).toBeTruthy();
|
||||
expect(defaultUnitWithSameId).toBeTruthy();
|
||||
expect(defaultUnitWithSameId).not.toBe(targetedUnit);
|
||||
expect(plan.topLevelSingleShardAssignments.get(targetedUnit)).toBeUndefined();
|
||||
expect(plan.topLevelSingleShardAssignments.get(defaultUnitWithSameId)).toBeDefined();
|
||||
|
||||
artifacts.cleanupTempArtifacts();
|
||||
});
|
||||
|
||||
it("removes planner temp artifacts when cleanup runs after planning", () => {
|
||||
const artifacts = createExecutionArtifacts({});
|
||||
buildExecutionPlan(
|
||||
{
|
||||
mode: "local",
|
||||
surfaces: ["unit"],
|
||||
passthroughArgs: [],
|
||||
},
|
||||
{
|
||||
env: {},
|
||||
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
|
||||
},
|
||||
);
|
||||
|
||||
const artifactDir = artifacts.ensureTempArtifactDir();
|
||||
expect(fs.existsSync(artifactDir)).toBe(true);
|
||||
artifacts.cleanupTempArtifacts();
|
||||
expect(fs.existsSync(artifactDir)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolvePnpmCommandInvocation", () => {
|
||||
it("prefers the parent pnpm CLI path when npm_execpath points to pnpm", () => {
|
||||
expect(
|
||||
resolvePnpmCommandInvocation({
|
||||
npmExecPath: "/opt/homebrew/lib/node_modules/corepack/dist/pnpm.cjs",
|
||||
nodeExecPath: "/usr/local/bin/node",
|
||||
platform: "linux",
|
||||
}),
|
||||
).toEqual({
|
||||
command: "/usr/local/bin/node",
|
||||
args: ["/opt/homebrew/lib/node_modules/corepack/dist/pnpm.cjs"],
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to cmd.exe mediation on Windows when npm_execpath is unavailable", () => {
|
||||
expect(
|
||||
resolvePnpmCommandInvocation({
|
||||
npmExecPath: "",
|
||||
platform: "win32",
|
||||
comSpec: "C:\\Windows\\System32\\cmd.exe",
|
||||
}),
|
||||
).toEqual({
|
||||
command: "C:\\Windows\\System32\\cmd.exe",
|
||||
args: ["/d", "/s", "/c", "pnpm.cmd"],
|
||||
});
|
||||
});
|
||||
});
|
||||
156
test/vitest-config.test.ts
Normal file
156
test/vitest-config.test.ts
Normal file
@@ -0,0 +1,156 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveExecutionBudget,
|
||||
resolveRuntimeCapabilities,
|
||||
} from "../scripts/test-planner/runtime-profile.mjs";
|
||||
import { resolveLocalVitestMaxWorkers } from "../vitest.config.ts";
|
||||
|
||||
describe("resolveLocalVitestMaxWorkers", () => {
|
||||
it("derives a mid-tier local cap for 64 GiB hosts", () => {
|
||||
expect(
|
||||
resolveLocalVitestMaxWorkers(
|
||||
{
|
||||
RUNNER_OS: "macOS",
|
||||
},
|
||||
{
|
||||
cpuCount: 10,
|
||||
totalMemoryBytes: 64 * 1024 ** 3,
|
||||
platform: "darwin",
|
||||
loadAverage: [0.1, 0.1, 0.1],
|
||||
},
|
||||
),
|
||||
).toBe(4);
|
||||
});
|
||||
|
||||
it("lets OPENCLAW_VITEST_MAX_WORKERS override the inferred cap", () => {
|
||||
expect(
|
||||
resolveLocalVitestMaxWorkers(
|
||||
{
|
||||
OPENCLAW_VITEST_MAX_WORKERS: "2",
|
||||
},
|
||||
{
|
||||
cpuCount: 10,
|
||||
totalMemoryBytes: 128 * 1024 ** 3,
|
||||
platform: "darwin",
|
||||
},
|
||||
),
|
||||
).toBe(2);
|
||||
});
|
||||
|
||||
it("maps the legacy low profile to serial intent for compatibility", () => {
|
||||
const runtime = resolveRuntimeCapabilities(
|
||||
{
|
||||
OPENCLAW_TEST_PROFILE: "low",
|
||||
RUNNER_OS: "Linux",
|
||||
},
|
||||
{
|
||||
cpuCount: 8,
|
||||
totalMemoryBytes: 32 * 1024 ** 3,
|
||||
platform: "linux",
|
||||
mode: "local",
|
||||
},
|
||||
);
|
||||
|
||||
expect(runtime.intentProfile).toBe("serial");
|
||||
});
|
||||
|
||||
it("classifies 64 GiB local macOS hosts as mid-memory capabilities", () => {
|
||||
const runtime = resolveRuntimeCapabilities(
|
||||
{
|
||||
RUNNER_OS: "macOS",
|
||||
},
|
||||
{
|
||||
cpuCount: 10,
|
||||
totalMemoryBytes: 64 * 1024 ** 3,
|
||||
platform: "darwin",
|
||||
mode: "local",
|
||||
loadAverage: [0.2, 0.2, 0.2],
|
||||
},
|
||||
);
|
||||
|
||||
expect(runtime.runtimeProfileName).toBe("local-darwin");
|
||||
expect(runtime.memoryBand).toBe("mid");
|
||||
expect(runtime.loadBand).toBe("idle");
|
||||
});
|
||||
|
||||
it("does not classify 64 GiB non-macOS hosts as constrained locals", () => {
|
||||
const runtime = resolveRuntimeCapabilities(
|
||||
{
|
||||
RUNNER_OS: "Linux",
|
||||
},
|
||||
{
|
||||
cpuCount: 16,
|
||||
totalMemoryBytes: 64 * 1024 ** 3,
|
||||
platform: "linux",
|
||||
mode: "local",
|
||||
loadAverage: [0.2, 0.2, 0.2],
|
||||
},
|
||||
);
|
||||
|
||||
expect(runtime.memoryBand).toBe("mid");
|
||||
expect(runtime.runtimeProfileName).toBe("local-linux");
|
||||
});
|
||||
|
||||
it("reduces local budgets when the host is busy", () => {
|
||||
const runtime = resolveRuntimeCapabilities(
|
||||
{
|
||||
RUNNER_OS: "Linux",
|
||||
},
|
||||
{
|
||||
cpuCount: 10,
|
||||
totalMemoryBytes: 16 * 1024 ** 3,
|
||||
platform: "linux",
|
||||
mode: "local",
|
||||
loadAverage: [9.5, 9.5, 9.5],
|
||||
},
|
||||
);
|
||||
const budget = resolveExecutionBudget(runtime);
|
||||
|
||||
expect(runtime.memoryBand).toBe("constrained");
|
||||
expect(runtime.loadBand).toBe("busy");
|
||||
expect(budget.vitestMaxWorkers).toBe(1);
|
||||
expect(budget.topLevelParallelLimit).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps 64 GiB hosts mid-tier but scales them down under saturation", () => {
|
||||
const runtime = resolveRuntimeCapabilities(
|
||||
{
|
||||
RUNNER_OS: "Linux",
|
||||
},
|
||||
{
|
||||
cpuCount: 10,
|
||||
totalMemoryBytes: 64 * 1024 ** 3,
|
||||
platform: "linux",
|
||||
mode: "local",
|
||||
loadAverage: [11.5, 11.5, 11.5],
|
||||
},
|
||||
);
|
||||
const budget = resolveExecutionBudget(runtime);
|
||||
|
||||
expect(runtime.memoryBand).toBe("mid");
|
||||
expect(runtime.loadBand).toBe("saturated");
|
||||
expect(budget.vitestMaxWorkers).toBe(2);
|
||||
expect(budget.deferredRunConcurrency).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps CI windows policy constrained independently of host load", () => {
|
||||
const runtime = resolveRuntimeCapabilities(
|
||||
{
|
||||
CI: "true",
|
||||
RUNNER_OS: "Windows",
|
||||
},
|
||||
{
|
||||
cpuCount: 32,
|
||||
totalMemoryBytes: 128 * 1024 ** 3,
|
||||
platform: "win32",
|
||||
mode: "ci",
|
||||
loadAverage: [0, 0, 0],
|
||||
},
|
||||
);
|
||||
const budget = resolveExecutionBudget(runtime);
|
||||
|
||||
expect(runtime.runtimeProfileName).toBe("ci-windows");
|
||||
expect(budget.vitestMaxWorkers).toBe(2);
|
||||
expect(budget.topLevelParallelLimit).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,6 @@ export function loadIncludePatternsFromEnv(
|
||||
export function createChannelsVitestConfig(env?: Record<string, string | undefined>) {
|
||||
return createScopedVitestConfig(loadIncludePatternsFromEnv(env) ?? channelTestInclude, {
|
||||
env,
|
||||
pool: "forks",
|
||||
exclude: ["src/gateway/**"],
|
||||
passWithNoTests: true,
|
||||
});
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { defineConfig } from "vitest/config";
|
||||
import { pluginSdkSubpaths } from "./scripts/lib/plugin-sdk-entries.mjs";
|
||||
import { resolveLocalVitestMaxWorkers } from "./scripts/test-planner/runtime-profile.mjs";
|
||||
import {
|
||||
behaviorManifestPath,
|
||||
unitMemoryHotspotManifestPath,
|
||||
@@ -10,10 +10,12 @@ import {
|
||||
} from "./scripts/test-runner-manifest.mjs";
|
||||
import { loadVitestExperimentalConfig } from "./vitest.performance-config.ts";
|
||||
|
||||
export { resolveLocalVitestMaxWorkers };
|
||||
|
||||
const repoRoot = path.dirname(fileURLToPath(import.meta.url));
|
||||
const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true";
|
||||
const isWindows = process.platform === "win32";
|
||||
const localWorkers = Math.max(4, Math.min(16, os.cpus().length));
|
||||
const localWorkers = resolveLocalVitestMaxWorkers();
|
||||
const ciWorkers = isWindows ? 2 : 3;
|
||||
export default defineConfig({
|
||||
resolve: {
|
||||
@@ -48,6 +50,10 @@ export default defineConfig({
|
||||
"pnpm-lock.yaml",
|
||||
"test/setup.ts",
|
||||
"scripts/test-parallel.mjs",
|
||||
"scripts/test-planner/catalog.mjs",
|
||||
"scripts/test-planner/executor.mjs",
|
||||
"scripts/test-planner/planner.mjs",
|
||||
"scripts/test-planner/runtime-profile.mjs",
|
||||
"scripts/test-runner-manifest.mjs",
|
||||
"vitest.channel-paths.mjs",
|
||||
"vitest.channels.config.ts",
|
||||
|
||||
@@ -14,7 +14,6 @@ export function createExtensionsVitestConfig(
|
||||
return createScopedVitestConfig(loadIncludePatternsFromEnv(env) ?? ["extensions/**/*.test.ts"], {
|
||||
dir: "extensions",
|
||||
env,
|
||||
pool: "forks",
|
||||
passWithNoTests: true,
|
||||
// Channel implementations live under extensions/ but are tested by
|
||||
// vitest.channels.config.ts (pnpm test:channels) which provides
|
||||
|
||||
Reference in New Issue
Block a user