From 7605eb4c20d9489bbd70e7800a004ef0acb80987 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Mon, 23 Mar 2026 16:26:56 +0100 Subject: [PATCH] Packaging: include optional bundled plugins in npm packs --- .github/workflows/ci.yml | 9 +-- .github/workflows/macos-release.yml | 2 +- .github/workflows/openclaw-npm-release.yml | 2 +- package.json | 3 +- scripts/build-npm-pack.mjs | 27 +++++++++ scripts/lib/optional-bundled-clusters.d.mts | 5 ++ scripts/lib/optional-bundled-clusters.d.ts | 5 ++ scripts/lib/optional-bundled-clusters.mjs | 21 +++++++ scripts/openclaw-npm-release-check.ts | 65 +++++++++++++-------- scripts/release-check.ts | 19 +++++- test/openclaw-npm-release-check.test.ts | 9 +++ test/release-check.test.ts | 14 ++++- 12 files changed, 145 insertions(+), 36 deletions(-) create mode 100644 scripts/build-npm-pack.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7e5c97d1a9..a07e2b1b827 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -212,11 +212,8 @@ jobs: install-bun: "false" use-sticky-disk: "false" - - name: Download dist artifact - uses: actions/download-artifact@v8 - with: - name: dist-build - path: dist/ + - name: Build npm pack dist + run: pnpm build:npm-pack - name: Check release contents run: pnpm release:check @@ -260,7 +257,7 @@ jobs: node_version: "22.x" cache_key_suffix: "node22" command: | - pnpm build + pnpm build:npm-pack node openclaw.mjs --help node openclaw.mjs status --json --timeout 1 pnpm test:build:singleton diff --git a/.github/workflows/macos-release.yml b/.github/workflows/macos-release.yml index 91ae67ccbd7..03ad16e39f3 100644 --- a/.github/workflows/macos-release.yml +++ b/.github/workflows/macos-release.yml @@ -78,7 +78,7 @@ jobs: run: pnpm check - name: Build - run: pnpm build + run: pnpm build:npm-pack - name: Build Control UI run: node scripts/ui.js build diff --git a/.github/workflows/openclaw-npm-release.yml b/.github/workflows/openclaw-npm-release.yml index 644231ccceb..7cd4c10586d 100644 --- a/.github/workflows/openclaw-npm-release.yml +++ b/.github/workflows/openclaw-npm-release.yml @@ -81,7 +81,7 @@ jobs: run: pnpm check - name: Build - run: pnpm build + run: pnpm build:npm-pack - name: Verify release contents run: pnpm release:check diff --git a/package.json b/package.json index 506b541cb4e..e4943627351 100644 --- a/package.json +++ b/package.json @@ -588,6 +588,7 @@ "android:test:third-party": "cd apps/android && ./gradlew :app:testThirdPartyDebugUnitTest", "build": "pnpm canvas:a2ui:bundle && node scripts/tsdown-build.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && pnpm build:plugin-sdk:dts && node --import tsx scripts/write-plugin-sdk-entry-dts.ts && node --import tsx scripts/canvas-a2ui-copy.ts && node --import tsx scripts/copy-hook-metadata.ts && node --import tsx scripts/copy-export-html-templates.ts && node --import tsx scripts/write-build-info.ts && node --import tsx scripts/write-cli-startup-metadata.ts && node --import tsx scripts/write-cli-compat.ts", "build:docker": "node scripts/tsdown-build.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && node --import tsx scripts/canvas-a2ui-copy.ts && node --import tsx scripts/copy-hook-metadata.ts && node --import tsx scripts/copy-export-html-templates.ts && node --import tsx scripts/write-build-info.ts && node --import tsx scripts/write-cli-startup-metadata.ts && node --import tsx scripts/write-cli-compat.ts", + "build:npm-pack": "node scripts/build-npm-pack.mjs", "build:plugin-sdk:dts": "tsc -p tsconfig.plugin-sdk.dts.json", "build:strict-smoke": "pnpm canvas:a2ui:bundle && node scripts/tsdown-build.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && pnpm build:plugin-sdk:dts", "canvas:a2ui:bundle": "bash scripts/bundle-a2ui.sh", @@ -676,7 +677,7 @@ "plugin-sdk:check-exports": "node scripts/sync-plugin-sdk-exports.mjs --check", "plugin-sdk:sync-exports": "node scripts/sync-plugin-sdk-exports.mjs", "plugins:sync": "node --import tsx scripts/sync-plugin-versions.ts", - "prepack": "pnpm build && pnpm ui:build", + "prepack": "pnpm build:npm-pack && pnpm ui:build", "prepare": "command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1 && git config core.hooksPath git-hooks || exit 0", "protocol:check": "pnpm protocol:gen && pnpm protocol:gen:swift && git diff --exit-code -- dist/protocol.schema.json apps/macos/Sources/OpenClawProtocol/GatewayModels.swift apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift", "protocol:gen": "node --import tsx scripts/protocol-gen.ts", diff --git a/scripts/build-npm-pack.mjs b/scripts/build-npm-pack.mjs new file mode 100644 index 00000000000..415b1c9ae50 --- /dev/null +++ b/scripts/build-npm-pack.mjs @@ -0,0 +1,27 @@ +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +const PNPM_COMMAND = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; + +function main() { + execFileSync(PNPM_COMMAND, ["build"], { + stdio: "inherit", + env: { + ...process.env, + OPENCLAW_INCLUDE_OPTIONAL_BUNDLED: "1", + }, + }); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + main(); + } catch (error) { + const status = + typeof error === "object" && error !== null && "status" in error ? error.status : undefined; + if (typeof status === "number") { + process.exit(status); + } + throw error; + } +} diff --git a/scripts/lib/optional-bundled-clusters.d.mts b/scripts/lib/optional-bundled-clusters.d.mts index 425e241ced7..21adb9f45a0 100644 --- a/scripts/lib/optional-bundled-clusters.d.mts +++ b/scripts/lib/optional-bundled-clusters.d.mts @@ -4,3 +4,8 @@ export const OPTIONAL_BUNDLED_BUILD_ENV: "OPENCLAW_INCLUDE_OPTIONAL_BUNDLED"; export function isOptionalBundledCluster(cluster: string): boolean; export function shouldIncludeOptionalBundledClusters(env?: NodeJS.ProcessEnv): boolean; export function shouldBuildBundledCluster(cluster: string, env?: NodeJS.ProcessEnv): boolean; +export function resolveOptionalBundledClusterRequiredPackPaths(repoRoot?: string): string[]; +export function collectMissingOptionalBundledClusterPackPaths( + paths: Iterable, + repoRoot?: string, +): string[]; diff --git a/scripts/lib/optional-bundled-clusters.d.ts b/scripts/lib/optional-bundled-clusters.d.ts index 425e241ced7..21adb9f45a0 100644 --- a/scripts/lib/optional-bundled-clusters.d.ts +++ b/scripts/lib/optional-bundled-clusters.d.ts @@ -4,3 +4,8 @@ export const OPTIONAL_BUNDLED_BUILD_ENV: "OPENCLAW_INCLUDE_OPTIONAL_BUNDLED"; export function isOptionalBundledCluster(cluster: string): boolean; export function shouldIncludeOptionalBundledClusters(env?: NodeJS.ProcessEnv): boolean; export function shouldBuildBundledCluster(cluster: string, env?: NodeJS.ProcessEnv): boolean; +export function resolveOptionalBundledClusterRequiredPackPaths(repoRoot?: string): string[]; +export function collectMissingOptionalBundledClusterPackPaths( + paths: Iterable, + repoRoot?: string, +): string[]; diff --git a/scripts/lib/optional-bundled-clusters.mjs b/scripts/lib/optional-bundled-clusters.mjs index 53ca72009b6..512ef3ecf69 100644 --- a/scripts/lib/optional-bundled-clusters.mjs +++ b/scripts/lib/optional-bundled-clusters.mjs @@ -1,3 +1,6 @@ +import fs from "node:fs"; +import path from "node:path"; + export const optionalBundledClusters = [ "acpx", "diagnostics-otel", @@ -29,3 +32,21 @@ export function shouldIncludeOptionalBundledClusters(env = process.env) { export function shouldBuildBundledCluster(cluster, env = process.env) { return shouldIncludeOptionalBundledClusters(env) || !isOptionalBundledCluster(cluster); } + +export function resolveOptionalBundledClusterRequiredPackPaths(repoRoot = process.cwd()) { + return optionalBundledClusters + .filter((cluster) => + fs.existsSync(path.join(repoRoot, "extensions", cluster, "openclaw.plugin.json")), + ) + .flatMap((cluster) => [ + `dist/extensions/${cluster}/openclaw.plugin.json`, + `dist/extensions/${cluster}/package.json`, + ]); +} + +export function collectMissingOptionalBundledClusterPackPaths(paths, repoRoot = process.cwd()) { + const presentPaths = new Set(paths); + return resolveOptionalBundledClusterRequiredPackPaths(repoRoot).filter( + (path) => !presentPaths.has(path), + ); +} diff --git a/scripts/openclaw-npm-release-check.ts b/scripts/openclaw-npm-release-check.ts index 33e2dda8624..f2bd877c380 100644 --- a/scripts/openclaw-npm-release-check.ts +++ b/scripts/openclaw-npm-release-check.ts @@ -3,6 +3,7 @@ import { execFileSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; +import { resolveOptionalBundledClusterRequiredPackPaths } from "./lib/optional-bundled-clusters.mjs"; type PackageJson = { name?: string; @@ -39,7 +40,6 @@ const BETA_VERSION_REGEX = const CORRECTION_TAG_REGEX = /^(?\d{4}\.[1-9]\d?\.[1-9]\d?)-(?[1-9]\d*)$/; const EXPECTED_REPOSITORY_URL = "https://github.com/openclaw/openclaw"; const MAX_CALVER_DISTANCE_DAYS = 2; -const REQUIRED_PACKED_PATHS = ["dist/control-ui/index.html"]; const NPM_COMMAND = process.platform === "win32" ? "npm.cmd" : "npm"; function normalizeRepoUrl(value: unknown): string { @@ -289,13 +289,6 @@ function loadPackageJson(): PackageJson { } function runNpmCommand(args: string[]): string { - const npmExecPath = process.env.npm_execpath; - if (typeof npmExecPath === "string" && npmExecPath.length > 0) { - return execFileSync(process.execPath, [npmExecPath, ...args], { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - }); - } return execFileSync(NPM_COMMAND, args, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], @@ -312,15 +305,15 @@ type NpmPackResult = { }; type ExecFailure = Error & { - stderr?: string | Buffer; - stdout?: string | Buffer; + stderr?: string | Uint8Array; + stdout?: string | Uint8Array; }; -function toTrimmedUtf8(value: string | Buffer | undefined): string { +function toTrimmedUtf8(value: string | Uint8Array | undefined): string { if (typeof value === "string") { return value.trim(); } - if (Buffer.isBuffer(value)) { + if (value instanceof Uint8Array) { return value.toString("utf8").trim(); } return ""; @@ -343,6 +336,41 @@ function describeExecFailure(error: unknown): string { return details.join(" | "); } +function parseTrailingJson(text: string): unknown { + const candidates: number[] = []; + for (let index = text.indexOf("["); index !== -1; index = text.indexOf("[", index + 1)) { + candidates.push(index); + } + for (let i = candidates.length - 1; i >= 0; i -= 1) { + const start = candidates[i]; + try { + return JSON.parse(text.slice(start)); + } catch { + // Keep scanning earlier `[` boundaries until we find the actual payload. + } + } + throw new Error("no trailing JSON payload found"); +} + +export function collectPackedTarballPathErrors(paths: Iterable): string[] { + const packedPaths = new Set(paths); + const errors: string[] = []; + const requiredPackedPaths = [ + "dist/control-ui/index.html", + ...resolveOptionalBundledClusterRequiredPackPaths(), + ]; + + for (const requiredPath of requiredPackedPaths) { + if (!packedPaths.has(requiredPath)) { + errors.push( + `npm package is missing required path "${requiredPath}". Build the package with \`pnpm build:npm-pack\` and include UI assets before publish.`, + ); + } + } + + return errors; +} + function collectPackedTarballErrors(): string[] { const errors: string[] = []; let stdout = ""; @@ -358,7 +386,7 @@ function collectPackedTarballErrors(): string[] { let parsed: unknown; try { - parsed = JSON.parse(stdout); + parsed = parseTrailingJson(stdout); } catch { errors.push("Failed to parse JSON output from `npm pack --json --dry-run`."); return errors; @@ -376,16 +404,7 @@ function collectPackedTarballErrors(): string[] { .map((entry) => entry.path) .filter((path): path is string => typeof path === "string" && path.length > 0), ); - - for (const requiredPath of REQUIRED_PACKED_PATHS) { - if (!packedPaths.has(requiredPath)) { - errors.push( - `npm package is missing required path "${requiredPath}". Ensure UI assets are built and included before publish.`, - ); - } - } - - return errors; + return collectPackedTarballPathErrors(packedPaths); } function main(): number { diff --git a/scripts/release-check.ts b/scripts/release-check.ts index f7f36373a49..f15969e9c5b 100755 --- a/scripts/release-check.ts +++ b/scripts/release-check.ts @@ -9,6 +9,7 @@ import { type BundledExtension, type ExtensionPackageJson as PackageJson, } from "./lib/bundled-extension-manifest.ts"; +import { collectMissingOptionalBundledClusterPackPaths } from "./lib/optional-bundled-clusters.mjs"; import { listPluginSdkDistArtifacts } from "./lib/plugin-sdk-entries.mjs"; import { sparkleBuildFloorsFromShortVersion, type SparkleBuildFloors } from "./sparkle-build.ts"; @@ -27,13 +28,18 @@ const requiredPathGroups = [ ]; const forbiddenPrefixes = ["dist-runtime/", "dist/OpenClaw.app/"]; // 2026.3.12 ballooned to ~213.6 MiB unpacked and correlated with low-memory -// startup/doctor OOM reports. Keep enough headroom for the current pack while -// failing fast if duplicate/shim content sneaks back into the release artifact. -const npmPackUnpackedSizeBudgetBytes = 160 * 1024 * 1024; +// startup/doctor OOM reports. The npm release now intentionally includes the +// optional bundled plugin set, so keep headroom above the current ~167.7 MiB +// pack while still failing fast if duplicate/shim content sneaks back in. +const npmPackUnpackedSizeBudgetBytes = 180 * 1024 * 1024; const appcastPath = resolve("appcast.xml"); const laneBuildMin = 1_000_000_000; const laneFloorAdoptionDateKey = 20260227; +export function collectMissingBundledPluginPackPaths(paths: Iterable): string[] { + return collectMissingOptionalBundledClusterPackPaths(paths); +} + function collectBundledExtensions(): BundledExtension[] { const extensionsDir = resolve("extensions"); const entries = readdirSync(extensionsDir, { withFileTypes: true }).filter((entry) => @@ -295,6 +301,7 @@ async function main() { const results = runPackDry(); const files = results.flatMap((entry) => entry.files ?? []); const paths = new Set(files.map((file) => file.path)); + const missingBundledPluginPaths = collectMissingBundledPluginPackPaths(paths); const missing = requiredPathGroups .flatMap((group) => { @@ -303,6 +310,7 @@ async function main() { } return paths.has(group) ? [] : [group]; }) + .concat(missingBundledPluginPaths) .toSorted(); const forbidden = collectForbiddenPackPaths(paths); const sizeErrors = collectPackUnpackedSizeErrors(results); @@ -313,6 +321,11 @@ async function main() { for (const path of missing) { console.error(` - ${path}`); } + if (missingBundledPluginPaths.length > 0) { + console.error( + "release-check: rebuild pack artifacts with `pnpm build:npm-pack` so optional bundled plugins are included.", + ); + } } if (forbidden.length > 0) { console.error("release-check: forbidden files in npm pack:"); diff --git a/test/openclaw-npm-release-check.test.ts b/test/openclaw-npm-release-check.test.ts index 6ce0d35cfdb..9e75d14cb78 100644 --- a/test/openclaw-npm-release-check.test.ts +++ b/test/openclaw-npm-release-check.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + collectPackedTarballPathErrors, collectReleasePackageMetadataErrors, collectReleaseTagErrors, parseReleaseTagVersion, @@ -132,3 +133,11 @@ describe("collectReleasePackageMetadataErrors", () => { ).toContain('package.json peerDependenciesMeta["node-llama-cpp"].optional must be true.'); }); }); + +describe("collectPackedTarballPathErrors", () => { + it("requires optional bundled plugin paths in the npm tarball", () => { + expect(collectPackedTarballPathErrors(["dist/control-ui/index.html"])).toContain( + 'npm package is missing required path "dist/extensions/acpx/openclaw.plugin.json". Build the package with `pnpm build:npm-pack` and include UI assets before publish.', + ); + }); +}); diff --git a/test/release-check.test.ts b/test/release-check.test.ts index 09f19d2babb..a35e63223da 100644 --- a/test/release-check.test.ts +++ b/test/release-check.test.ts @@ -3,6 +3,7 @@ import { collectAppcastSparkleVersionErrors, collectBundledExtensionManifestErrors, collectForbiddenPackPaths, + collectMissingBundledPluginPackPaths, collectPackUnpackedSizeErrors, } from "../scripts/release-check.ts"; @@ -115,6 +116,17 @@ describe("collectForbiddenPackPaths", () => { }); }); +describe("collectMissingBundledPluginPackPaths", () => { + it("requires optional bundled plugin metadata in npm packs", () => { + expect( + collectMissingBundledPluginPackPaths([ + "dist/extensions/discord/openclaw.plugin.json", + "dist/extensions/discord/package.json", + ]), + ).toContain("dist/extensions/acpx/openclaw.plugin.json"); + }); +}); + describe("collectPackUnpackedSizeErrors", () => { it("accepts pack results within the unpacked size budget", () => { expect( @@ -126,7 +138,7 @@ describe("collectPackUnpackedSizeErrors", () => { expect( collectPackUnpackedSizeErrors([makePackResult("openclaw-2026.3.12.tgz", 224_002_564)]), ).toEqual([ - "openclaw-2026.3.12.tgz unpackedSize 224002564 bytes (213.6 MiB) exceeds budget 167772160 bytes (160.0 MiB). Investigate duplicate channel shims, copied extension trees, or other accidental pack bloat before release.", + "openclaw-2026.3.12.tgz unpackedSize 224002564 bytes (213.6 MiB) exceeds budget 188743680 bytes (180.0 MiB). Investigate duplicate channel shims, copied extension trees, or other accidental pack bloat before release.", ]); });