perf: generate bundled plugin metadata for cold startup

This commit is contained in:
Peter Steinberger
2026-03-22 21:09:38 +00:00
parent 3ca7922dfe
commit 171b24c5c5
8 changed files with 4564 additions and 9 deletions

View File

@@ -571,7 +571,8 @@
"build:plugin-sdk:dts": "tsc -p tsconfig.plugin-sdk.dts.json",
"build:strict-smoke": "pnpm canvas:a2ui:bundle && node scripts/tsdown-build.mjs && node scripts/runtime-postbuild.mjs && pnpm build:plugin-sdk:dts",
"canvas:a2ui:bundle": "bash scripts/bundle-a2ui.sh",
"check": "pnpm check:host-env-policy:swift && pnpm check:bundled-provider-auth-env-vars && pnpm format:check && pnpm tsgo && pnpm plugin-sdk:check-exports && pnpm lint && pnpm lint:tmp:no-random-messaging && pnpm lint:tmp:channel-agnostic-boundaries && pnpm lint:tmp:no-raw-channel-fetch && pnpm lint:agent:ingress-owner && pnpm lint:plugins:no-register-http-handler && pnpm lint:plugins:no-monolithic-plugin-sdk-entry-imports && pnpm lint:plugins:no-extension-src-imports && pnpm lint:plugins:no-extension-test-core-imports && pnpm lint:plugins:no-extension-imports && pnpm lint:plugins:plugin-sdk-subpaths-exported && pnpm lint:extensions:no-src-outside-plugin-sdk && pnpm lint:extensions:no-plugin-sdk-internal && pnpm lint:extensions:no-relative-outside-package && pnpm lint:web-search-provider-boundaries && pnpm lint:webhook:no-low-level-body-read && pnpm lint:auth:no-pairing-store-group && pnpm lint:auth:pairing-account-scope",
"check": "pnpm check:host-env-policy:swift && pnpm check:bundled-plugin-metadata && pnpm check:bundled-provider-auth-env-vars && pnpm format:check && pnpm tsgo && pnpm plugin-sdk:check-exports && pnpm lint && pnpm lint:tmp:no-random-messaging && pnpm lint:tmp:channel-agnostic-boundaries && pnpm lint:tmp:no-raw-channel-fetch && pnpm lint:agent:ingress-owner && pnpm lint:plugins:no-register-http-handler && pnpm lint:plugins:no-monolithic-plugin-sdk-entry-imports && pnpm lint:plugins:no-extension-src-imports && pnpm lint:plugins:no-extension-test-core-imports && pnpm lint:plugins:no-extension-imports && pnpm lint:plugins:plugin-sdk-subpaths-exported && pnpm lint:extensions:no-src-outside-plugin-sdk && pnpm lint:extensions:no-plugin-sdk-internal && pnpm lint:extensions:no-relative-outside-package && pnpm lint:web-search-provider-boundaries && pnpm lint:webhook:no-low-level-body-read && pnpm lint:auth:no-pairing-store-group && pnpm lint:auth:pairing-account-scope",
"check:bundled-plugin-metadata": "node scripts/generate-bundled-plugin-metadata.mjs --check",
"check:bundled-provider-auth-env-vars": "node scripts/generate-bundled-provider-auth-env-vars.mjs --check",
"check:docs": "pnpm format:docs:check && pnpm lint:docs && pnpm docs:check-i18n-glossary && pnpm docs:check-links",
"check:host-env-policy:swift": "node scripts/generate-host-env-security-policy-swift.mjs --check",

View File

@@ -0,0 +1,32 @@
export type BundledPluginPathPair = {
source: string;
built: string;
};
export type BundledPluginMetadataEntry = {
dirName: string;
idHint: string;
source: BundledPluginPathPair;
setupSource?: BundledPluginPathPair;
packageName?: string;
packageVersion?: string;
packageDescription?: string;
packageManifest?: Record<string, unknown>;
manifest: Record<string, unknown>;
};
export function collectBundledPluginMetadata(params?: {
repoRoot?: string;
}): BundledPluginMetadataEntry[];
export function renderBundledPluginMetadataModule(entries: BundledPluginMetadataEntry[]): string;
export function writeBundledPluginMetadataModule(params?: {
repoRoot?: string;
outputPath?: string;
check?: boolean;
}): {
changed: boolean;
wrote: boolean;
outputPath: string;
};

View File

@@ -0,0 +1,273 @@
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { writeTextFileIfChanged } from "./runtime-postbuild-shared.mjs";
const GENERATED_BY = "scripts/generate-bundled-plugin-metadata.mjs";
const DEFAULT_OUTPUT_PATH = "src/plugins/bundled-plugin-metadata.generated.ts";
const MANIFEST_KEY = "openclaw";
const FORMATTER_CWD = path.resolve(import.meta.dirname, "..");
const CANONICAL_PACKAGE_ID_ALIASES = {
"elevenlabs-speech": "elevenlabs",
"microsoft-speech": "microsoft",
"ollama-provider": "ollama",
"sglang-provider": "sglang",
"vllm-provider": "vllm",
};
function readIfExists(filePath) {
try {
return fs.readFileSync(filePath, "utf8");
} catch {
return null;
}
}
function rewriteEntryToBuiltPath(entry) {
if (typeof entry !== "string" || entry.trim().length === 0) {
return undefined;
}
const normalized = entry.replace(/^\.\//u, "");
return normalized.replace(/\.[^.]+$/u, ".js");
}
function deriveIdHint({ filePath, packageName, hasMultipleExtensions }) {
const base = path.basename(filePath, path.extname(filePath));
const rawPackageName = packageName?.trim();
if (!rawPackageName) {
return base;
}
const unscoped = rawPackageName.includes("/")
? (rawPackageName.split("/").pop() ?? rawPackageName)
: rawPackageName;
const canonicalPackageId = CANONICAL_PACKAGE_ID_ALIASES[unscoped] ?? unscoped;
const normalizedPackageId =
canonicalPackageId.endsWith("-provider") && canonicalPackageId.length > "-provider".length
? canonicalPackageId.slice(0, -"-provider".length)
: canonicalPackageId;
if (!hasMultipleExtensions) {
return normalizedPackageId;
}
return `${normalizedPackageId}/${base}`;
}
function normalizeStringList(values) {
if (!Array.isArray(values)) {
return undefined;
}
const normalized = values.map((value) => String(value).trim()).filter(Boolean);
return normalized.length > 0 ? normalized : undefined;
}
function normalizeObject(value) {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return undefined;
}
return value;
}
function normalizePackageManifest(raw) {
const packageManifest = normalizeObject(raw?.[MANIFEST_KEY]);
if (!packageManifest) {
return undefined;
}
const normalized = {
...(Array.isArray(packageManifest.extensions)
? { extensions: packageManifest.extensions.map((entry) => String(entry).trim()) }
: {}),
...(typeof packageManifest.setupEntry === "string"
? { setupEntry: packageManifest.setupEntry.trim() }
: {}),
...(normalizeObject(packageManifest.channel) ? { channel: packageManifest.channel } : {}),
...(normalizeObject(packageManifest.install) ? { install: packageManifest.install } : {}),
...(normalizeObject(packageManifest.startup) ? { startup: packageManifest.startup } : {}),
};
return Object.keys(normalized).length > 0 ? normalized : undefined;
}
function normalizePluginManifest(raw) {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
return null;
}
if (typeof raw.id !== "string" || !raw.id.trim()) {
return null;
}
if (
!raw.configSchema ||
typeof raw.configSchema !== "object" ||
Array.isArray(raw.configSchema)
) {
return null;
}
return {
id: raw.id.trim(),
configSchema: raw.configSchema,
...(raw.enabledByDefault === true ? { enabledByDefault: true } : {}),
...(typeof raw.kind === "string" ? { kind: raw.kind.trim() } : {}),
...(normalizeStringList(raw.channels) ? { channels: normalizeStringList(raw.channels) } : {}),
...(normalizeStringList(raw.providers)
? { providers: normalizeStringList(raw.providers) }
: {}),
...(normalizeObject(raw.providerAuthEnvVars)
? { providerAuthEnvVars: raw.providerAuthEnvVars }
: {}),
...(Array.isArray(raw.providerAuthChoices)
? { providerAuthChoices: raw.providerAuthChoices }
: {}),
...(normalizeStringList(raw.skills) ? { skills: normalizeStringList(raw.skills) } : {}),
...(typeof raw.name === "string" ? { name: raw.name.trim() } : {}),
...(typeof raw.description === "string" ? { description: raw.description.trim() } : {}),
...(typeof raw.version === "string" ? { version: raw.version.trim() } : {}),
...(normalizeObject(raw.uiHints) ? { uiHints: raw.uiHints } : {}),
};
}
function formatTypeScriptModule(source, { outputPath }) {
const formatter = spawnSync(
process.platform === "win32" ? "pnpm.cmd" : "pnpm",
["exec", "oxfmt", "--stdin-filepath", outputPath],
{
cwd: FORMATTER_CWD,
input: source,
encoding: "utf8",
},
);
if (formatter.status !== 0) {
const details =
formatter.stderr?.trim() || formatter.stdout?.trim() || "unknown formatter failure";
throw new Error(`failed to format generated bundled plugin metadata: ${details}`);
}
return formatter.stdout;
}
export function collectBundledPluginMetadata(params = {}) {
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
const extensionsRoot = path.join(repoRoot, "extensions");
if (!fs.existsSync(extensionsRoot)) {
return [];
}
const entries = [];
for (const dirent of fs.readdirSync(extensionsRoot, { withFileTypes: true })) {
if (!dirent.isDirectory()) {
continue;
}
const pluginDir = path.join(extensionsRoot, dirent.name);
const manifestPath = path.join(pluginDir, "openclaw.plugin.json");
const packageJsonPath = path.join(pluginDir, "package.json");
if (!fs.existsSync(manifestPath) || !fs.existsSync(packageJsonPath)) {
continue;
}
const manifest = normalizePluginManifest(JSON.parse(fs.readFileSync(manifestPath, "utf8")));
if (!manifest) {
continue;
}
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
const packageManifest = normalizePackageManifest(packageJson);
const extensions = Array.isArray(packageManifest?.extensions)
? packageManifest.extensions.filter((entry) => typeof entry === "string" && entry.trim())
: [];
if (extensions.length === 0) {
continue;
}
const sourceEntry = extensions[0];
const builtEntry = rewriteEntryToBuiltPath(sourceEntry);
if (!builtEntry) {
continue;
}
const setupEntry =
typeof packageManifest?.setupEntry === "string" &&
packageManifest.setupEntry.trim().length > 0
? {
source: packageManifest.setupEntry.trim(),
built: rewriteEntryToBuiltPath(packageManifest.setupEntry.trim()),
}
: undefined;
entries.push({
dirName: dirent.name,
idHint: deriveIdHint({
filePath: sourceEntry,
packageName: typeof packageJson.name === "string" ? packageJson.name : undefined,
hasMultipleExtensions: extensions.length > 1,
}),
source: {
source: sourceEntry,
built: builtEntry,
},
...(setupEntry?.built
? { setupSource: { source: setupEntry.source, built: setupEntry.built } }
: {}),
...(typeof packageJson.name === "string" ? { packageName: packageJson.name.trim() } : {}),
...(typeof packageJson.version === "string"
? { packageVersion: packageJson.version.trim() }
: {}),
...(typeof packageJson.description === "string"
? { packageDescription: packageJson.description.trim() }
: {}),
...(packageManifest ? { packageManifest } : {}),
manifest,
});
}
return entries.toSorted((left, right) => left.dirName.localeCompare(right.dirName));
}
export function renderBundledPluginMetadataModule(entries) {
return `// Auto-generated by ${GENERATED_BY}. Do not edit directly.
export const GENERATED_BUNDLED_PLUGIN_METADATA = ${JSON.stringify(entries, null, 2)} as const;
`;
}
export function writeBundledPluginMetadataModule(params = {}) {
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
const outputPath = path.resolve(repoRoot, params.outputPath ?? DEFAULT_OUTPUT_PATH);
const next = formatTypeScriptModule(
renderBundledPluginMetadataModule(collectBundledPluginMetadata({ repoRoot })),
{ outputPath },
);
const current = readIfExists(outputPath);
const changed = current !== next;
if (params.check) {
return {
changed,
wrote: false,
outputPath,
};
}
return {
changed,
wrote: writeTextFileIfChanged(outputPath, next),
outputPath,
};
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
const result = writeBundledPluginMetadataModule({
check: process.argv.includes("--check"),
});
if (result.changed) {
if (process.argv.includes("--check")) {
console.error(
`[bundled-plugin-metadata] stale generated output at ${path.relative(process.cwd(), result.outputPath)}`,
);
process.exitCode = 1;
} else {
console.log(
`[bundled-plugin-metadata] wrote ${path.relative(process.cwd(), result.outputPath)}`,
);
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,106 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
collectBundledPluginMetadata,
writeBundledPluginMetadataModule,
} from "../../scripts/generate-bundled-plugin-metadata.mjs";
import {
BUNDLED_PLUGIN_METADATA,
resolveBundledPluginGeneratedPath,
} from "./bundled-plugin-metadata.js";
const repoRoot = path.resolve(import.meta.dirname, "../..");
const tempDirs: string[] = [];
function writeJson(filePath: string, value: unknown): void {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
}
afterEach(() => {
for (const dir of tempDirs.splice(0, tempDirs.length)) {
fs.rmSync(dir, { recursive: true, force: true });
}
});
describe("bundled plugin metadata", () => {
it("matches the generated metadata snapshot", () => {
expect(BUNDLED_PLUGIN_METADATA).toEqual(collectBundledPluginMetadata({ repoRoot }));
});
it("captures setup-entry metadata for bundled channel plugins", () => {
const discord = BUNDLED_PLUGIN_METADATA.find((entry) => entry.dirName === "discord");
expect(discord?.source).toEqual({ source: "./index.ts", built: "index.js" });
expect(discord?.setupSource).toEqual({ source: "./setup-entry.ts", built: "setup-entry.js" });
expect(discord?.manifest.id).toBe("discord");
});
it("prefers built generated paths when present and falls back to source paths", () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bundled-plugin-metadata-"));
tempDirs.push(tempRoot);
fs.mkdirSync(path.join(tempRoot, "plugin"), { recursive: true });
fs.writeFileSync(path.join(tempRoot, "plugin", "index.ts"), "export {};\n", "utf8");
expect(
resolveBundledPluginGeneratedPath(tempRoot, {
source: "plugin/index.ts",
built: "plugin/index.js",
}),
).toBe(path.join(tempRoot, "plugin", "index.ts"));
fs.writeFileSync(path.join(tempRoot, "plugin", "index.js"), "export {};\n", "utf8");
expect(
resolveBundledPluginGeneratedPath(tempRoot, {
source: "plugin/index.ts",
built: "plugin/index.js",
}),
).toBe(path.join(tempRoot, "plugin", "index.js"));
});
it("supports check mode for stale generated artifacts", () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bundled-plugin-generated-"));
tempDirs.push(tempRoot);
writeJson(path.join(tempRoot, "extensions", "alpha", "package.json"), {
name: "@openclaw/alpha",
version: "0.0.1",
openclaw: {
extensions: ["./index.ts"],
},
});
writeJson(path.join(tempRoot, "extensions", "alpha", "openclaw.plugin.json"), {
id: "alpha",
configSchema: { type: "object" },
});
const initial = writeBundledPluginMetadataModule({
repoRoot: tempRoot,
outputPath: "src/plugins/bundled-plugin-metadata.generated.ts",
});
expect(initial.wrote).toBe(true);
const current = writeBundledPluginMetadataModule({
repoRoot: tempRoot,
outputPath: "src/plugins/bundled-plugin-metadata.generated.ts",
check: true,
});
expect(current.changed).toBe(false);
expect(current.wrote).toBe(false);
fs.writeFileSync(
path.join(tempRoot, "src/plugins/bundled-plugin-metadata.generated.ts"),
"// stale\n",
"utf8",
);
const stale = writeBundledPluginMetadataModule({
repoRoot: tempRoot,
outputPath: "src/plugins/bundled-plugin-metadata.generated.ts",
check: true,
});
expect(stale.changed).toBe(true);
expect(stale.wrote).toBe(false);
});
});

View File

@@ -0,0 +1,44 @@
import fs from "node:fs";
import path from "node:path";
import { GENERATED_BUNDLED_PLUGIN_METADATA } from "./bundled-plugin-metadata.generated.js";
import type { PluginManifest, OpenClawPackageManifest } from "./manifest.js";
type GeneratedBundledPluginPathPair = {
source: string;
built: string;
};
export type GeneratedBundledPluginMetadata = {
dirName: string;
idHint: string;
source: GeneratedBundledPluginPathPair;
setupSource?: GeneratedBundledPluginPathPair;
packageName?: string;
packageVersion?: string;
packageDescription?: string;
packageManifest?: OpenClawPackageManifest;
manifest: PluginManifest;
};
export const BUNDLED_PLUGIN_METADATA =
GENERATED_BUNDLED_PLUGIN_METADATA as unknown as readonly GeneratedBundledPluginMetadata[];
export function resolveBundledPluginGeneratedPath(
rootDir: string,
entry: GeneratedBundledPluginPathPair | undefined,
): string | null {
if (!entry) {
return null;
}
const candidates = [entry.built, entry.source]
.filter(
(candidate): candidate is string => typeof candidate === "string" && candidate.length > 0,
)
.map((candidate) => path.resolve(rootDir, candidate));
for (const candidate of candidates) {
if (fs.existsSync(candidate)) {
return candidate;
}
}
return null;
}

View File

@@ -3,9 +3,14 @@ import path from "node:path";
import { openBoundaryFileSync } from "../infra/boundary-file-read.js";
import { resolveUserPath } from "../utils.js";
import { detectBundleManifestFormat, loadBundleManifest } from "./bundle-manifest.js";
import {
BUNDLED_PLUGIN_METADATA,
resolveBundledPluginGeneratedPath,
} from "./bundled-plugin-metadata.js";
import {
DEFAULT_PLUGIN_ENTRY_CANDIDATES,
getPackageManifestMetadata,
type PluginManifest,
resolvePackageExtensionEntries,
type OpenClawPackageManifest,
type PackageManifest,
@@ -38,6 +43,8 @@ export type PluginCandidate = {
packageDescription?: string;
packageDir?: string;
packageManifest?: OpenClawPackageManifest;
bundledManifest?: PluginManifest;
bundledManifestPath?: string;
};
export type PluginDiscoveryResult = {
@@ -372,6 +379,8 @@ function addCandidate(params: {
workspaceDir?: string;
manifest?: PackageManifest | null;
packageDir?: string;
bundledManifest?: PluginManifest;
bundledManifestPath?: string;
}) {
const resolved = path.resolve(params.source);
if (params.seen.has(resolved)) {
@@ -405,6 +414,8 @@ function addCandidate(params: {
packageDescription: manifest?.description?.trim() || undefined,
packageDir: params.packageDir,
packageManifest: getPackageManifestMetadata(manifest ?? undefined),
bundledManifest: params.bundledManifest,
bundledManifestPath: params.bundledManifestPath,
});
}
@@ -485,6 +496,7 @@ function discoverInDirectory(params: {
candidates: PluginCandidate[];
diagnostics: PluginDiagnostic[];
seen: Set<string>;
skipDirectories?: Set<string>;
}) {
if (!fs.existsSync(params.dir)) {
return;
@@ -522,6 +534,9 @@ function discoverInDirectory(params: {
if (!entry.isDirectory()) {
continue;
}
if (params.skipDirectories?.has(entry.name)) {
continue;
}
if (shouldIgnoreScannedDirectory(entry.name)) {
continue;
}
@@ -754,6 +769,54 @@ function discoverFromPath(params: {
}
}
function discoverBundledMetadataInDirectory(params: {
dir: string;
ownershipUid?: number | null;
candidates: PluginCandidate[];
diagnostics: PluginDiagnostic[];
seen: Set<string>;
}) {
if (!fs.existsSync(params.dir)) {
return null;
}
const coveredDirectories = new Set<string>();
for (const entry of BUNDLED_PLUGIN_METADATA) {
const rootDir = path.join(params.dir, entry.dirName);
if (!fs.existsSync(rootDir)) {
continue;
}
coveredDirectories.add(entry.dirName);
const source = resolveBundledPluginGeneratedPath(rootDir, entry.source);
if (!source) {
continue;
}
const setupSource = resolveBundledPluginGeneratedPath(rootDir, entry.setupSource);
addCandidate({
candidates: params.candidates,
diagnostics: params.diagnostics,
seen: params.seen,
idHint: entry.idHint,
source,
...(setupSource ? { setupSource } : {}),
rootDir,
origin: "bundled",
ownershipUid: params.ownershipUid,
manifest: {
...(entry.packageName ? { name: entry.packageName } : {}),
...(entry.packageVersion ? { version: entry.packageVersion } : {}),
...(entry.packageDescription ? { description: entry.packageDescription } : {}),
...(entry.packageManifest ? { openclaw: entry.packageManifest } : {}),
},
packageDir: rootDir,
bundledManifest: entry.manifest,
bundledManifestPath: path.join(rootDir, "openclaw.plugin.json"),
});
}
return coveredDirectories;
}
export function discoverOpenClawPlugins(params: {
workspaceDir?: string;
extraPaths?: string[];
@@ -816,6 +879,13 @@ export function discoverOpenClawPlugins(params: {
}
if (roots.stock) {
const coveredBundledDirectories = discoverBundledMetadataInDirectory({
dir: roots.stock,
ownershipUid: params.ownershipUid,
candidates,
diagnostics,
seen,
});
discoverInDirectory({
dir: roots.stock,
origin: "bundled",
@@ -823,6 +893,7 @@ export function discoverOpenClawPlugins(params: {
candidates,
diagnostics,
seen,
...(coveredBundledDirectories ? { skipDirectories: coveredBundledDirectories } : {}),
});
}

View File

@@ -335,14 +335,23 @@ export function loadPluginManifestRegistry(
for (const candidate of candidates) {
const rejectHardlinks = candidate.origin !== "bundled";
const isBundleRecord = (candidate.format ?? "openclaw") === "bundle";
const manifestRes =
isBundleRecord && candidate.bundleFormat
? loadBundleManifest({
rootDir: candidate.rootDir,
bundleFormat: candidate.bundleFormat,
rejectHardlinks,
})
: loadPluginManifest(candidate.rootDir, rejectHardlinks);
const manifestRes:
| ReturnType<typeof loadPluginManifest>
| ReturnType<typeof loadBundleManifest>
| { ok: true; manifest: PluginManifest; manifestPath: string } =
candidate.origin === "bundled" && candidate.bundledManifest && candidate.bundledManifestPath
? {
ok: true,
manifest: candidate.bundledManifest,
manifestPath: candidate.bundledManifestPath,
}
: isBundleRecord && candidate.bundleFormat
? loadBundleManifest({
rootDir: candidate.rootDir,
bundleFormat: candidate.bundleFormat,
rejectHardlinks,
})
: loadPluginManifest(candidate.rootDir, rejectHardlinks);
if (!manifestRes.ok) {
diagnostics.push({
level: "error",