1 +import { spawn, spawnSync } from "node:child_process";
2 +import { readdir } from "node:fs/promises";
3 +import { dirname, join, resolve } from "node:path";
4 +import { fileURLToPath } from "node:url";
6 +// End-to-end metric: process spawn until get_state confirms RPC readiness after
7 +// extension factories and startup handlers. Go benchmark output supports benchstat.
8 +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
9 +const rounds = Number(process.env.EXTENSION_STARTUP_BENCH_ROUNDS ?? 20);
10 +const warmups = Number(process.env.EXTENSION_STARTUP_BENCH_WARMUPS ?? 3);
11 +const timeoutMs = Number(
12 + process.env.EXTENSION_STARTUP_BENCH_TIMEOUT_MS ?? 15_000,
15 +for (const [name, value] of [
16 + ["EXTENSION_STARTUP_BENCH_ROUNDS", rounds],
17 + ["EXTENSION_STARTUP_BENCH_WARMUPS", warmups],
18 + ["EXTENSION_STARTUP_BENCH_TIMEOUT_MS", timeoutMs],
21 + !Number.isSafeInteger(value) ||
22 + value < (name.endsWith("WARMUPS") ? 0 : 1)
24 + throw new Error(`${name} must be a valid benchmark count`);
29 + await readdir(join(root, "extensions"), {
30 + withFileTypes: true,
33 + .filter((entry) => entry.isDirectory())
34 + .map((entry) => join(root, "extensions", entry.name, "index.ts"))
43 + "--no-context-files",
45 + "--no-prompt-templates",
47 + ...extensions.flatMap((extension) => ["--extension", extension]),
50 +async function measureStartup() {
51 + const started = process.hrtime.bigint();
52 + const child = spawn("pi", args, {
54 + env: { ...process.env, PI_OFFLINE: "1" },
55 + stdio: ["pipe", "pipe", "pipe"],
60 + let settled = false;
62 + return new Promise((resolveMeasurement, rejectMeasurement) => {
63 + const timeout = setTimeout(() => {
64 + child.kill("SIGKILL");
66 + new Error(`Pi did not become RPC-ready within ${timeoutMs} ms`),
70 + const finish = (error) => {
71 + if (settled) return;
73 + clearTimeout(timeout);
75 + child.kill("SIGKILL");
76 + rejectMeasurement(error);
78 + resolveMeasurement(elapsed);
82 + child.on("error", (error) => finish(error));
83 + child.stderr.setEncoding("utf8");
84 + child.stderr.on("data", (chunk) => {
87 + child.stdout.setEncoding("utf8");
88 + child.stdout.on("data", (chunk) => {
91 + const newline = stdout.indexOf("\n");
92 + if (newline < 0) break;
93 + const line = stdout.slice(0, newline);
94 + stdout = stdout.slice(newline + 1);
97 + event = JSON.parse(line.endsWith("\r") ? line.slice(0, -1) : line);
99 + finish(new Error(`Invalid Pi RPC output: ${error.message}`));
102 + if (event.type === "response" && event.command === "get_state") {
103 + if (!event.success) {
104 + finish(new Error(`Pi get_state failed: ${event.error}`));
107 + elapsed = process.hrtime.bigint() - started;
112 + child.on("exit", (code, signal) => {
113 + if (settled) return;
114 + if (elapsed !== undefined) {
120 + `Pi exited before RPC readiness (code=${code}, signal=${signal})${stderr ? `: ${stderr.trim()}` : ""}`,
124 + child.stdin.end('{"type":"get_state"}\n');
128 +for (let index = 0; index < warmups; index++) await measureStartup();
130 +const piVersion = spawnSync("pi", ["--version"], { encoding: "utf8" });
131 +if (piVersion.status !== 0) {
132 + throw new Error(`Unable to read Pi version: ${piVersion.stderr.trim()}`);
135 +console.log(`goos: ${process.platform}`);
136 +console.log(`goarch: ${process.arch}`);
137 +console.log("pkg: pi-ext/startup");
138 +console.log(`node: ${process.version}`);
139 +console.log(`pi: ${piVersion.stdout.trim()}`);
140 +console.log(`extensions: ${extensions.length}`);
141 +for (let round = 0; round < rounds; round++) {
142 + const elapsed = await measureStartup();
143 + console.log(`BenchmarkExtensionStartup\t1\t${elapsed} ns/op`);