repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

bench(startup): add repeatable extension timing

dc9381c554e64dd27d75db9d670f81869ec6f5fb

Oliver Jan Krylow <oliver@bugabinga.net> ·

SSH signature · not checked

Browse filesCompare with HEAD

Changed files (2)

mise.toml

Modified
148 148 node .pi/skills/modernize/scripts/audit.bench.mjs{% for arg in usage.args %} "{{ arg }}"{% endfor %}
151 +[tasks.extension-startup-bench]
152 +description = "Benchmark configured extension startup to RPC readiness"
153 +depends = ["install"]
154 +run = "node scripts/extension-startup.bench.mjs"
156 +[tasks.extension-startup-benchstat]
157 +description = "Compare extension startup benchmark results"
158 +depends = ["install"]
159 +usage = '''
160 +arg "<baseline>"
161 +arg "<candidate>"
162 +'''
163 +run = 'benchstat "{{ usage.baseline }}" "{{ usage.candidate }}"'
151 165 [tasks.e2e]
152 166 description = "Run all extension browser E2E tests"
153 167 depends = ["install"]

scripts/extension-startup.bench.mjs

Added
new file mode 100644
1 +import { spawn, spawnSync } from "node:child_process";
2 +import { readdir } from "node:fs/promises";
3 +import { dirname, join, resolve } from "node:path";
4 +import { fileURLToPath } from "node:url";
6 +// End-to-end metric: process spawn until get_state confirms RPC readiness after
7 +// extension factories and startup handlers. Go benchmark output supports benchstat.
8 +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
9 +const rounds = Number(process.env.EXTENSION_STARTUP_BENCH_ROUNDS ?? 20);
10 +const warmups = Number(process.env.EXTENSION_STARTUP_BENCH_WARMUPS ?? 3);
11 +const timeoutMs = Number(
12 + process.env.EXTENSION_STARTUP_BENCH_TIMEOUT_MS ?? 15_000,
13 +);
15 +for (const [name, value] of [
16 + ["EXTENSION_STARTUP_BENCH_ROUNDS", rounds],
17 + ["EXTENSION_STARTUP_BENCH_WARMUPS", warmups],
18 + ["EXTENSION_STARTUP_BENCH_TIMEOUT_MS", timeoutMs],
19 +]) {
20 + if (
21 + !Number.isSafeInteger(value) ||
22 + value < (name.endsWith("WARMUPS") ? 0 : 1)
23 + ) {
24 + throw new Error(`${name} must be a valid benchmark count`);
25 + }
28 +const extensions = (
29 + await readdir(join(root, "extensions"), {
30 + withFileTypes: true,
31 + })
33 + .filter((entry) => entry.isDirectory())
34 + .map((entry) => join(root, "extensions", entry.name, "index.ts"))
35 + .sort();
37 +const args = [
38 + "--mode",
39 + "rpc",
40 + "--no-session",
41 + "--approve",
42 + "--no-extensions",
43 + "--no-context-files",
44 + "--no-skills",
45 + "--no-prompt-templates",
46 + "--no-themes",
47 + ...extensions.flatMap((extension) => ["--extension", extension]),
48 +];
50 +async function measureStartup() {
51 + const started = process.hrtime.bigint();
52 + const child = spawn("pi", args, {
53 + cwd: root,
54 + env: { ...process.env, PI_OFFLINE: "1" },
55 + stdio: ["pipe", "pipe", "pipe"],
56 + });
57 + let stdout = "";
58 + let stderr = "";
59 + let elapsed;
60 + let settled = false;
62 + return new Promise((resolveMeasurement, rejectMeasurement) => {
63 + const timeout = setTimeout(() => {
64 + child.kill("SIGKILL");
65 + rejectMeasurement(
66 + new Error(`Pi did not become RPC-ready within ${timeoutMs} ms`),
67 + );
68 + }, timeoutMs);
70 + const finish = (error) => {
71 + if (settled) return;
72 + settled = true;
73 + clearTimeout(timeout);
74 + if (error) {
75 + child.kill("SIGKILL");
76 + rejectMeasurement(error);
77 + } else {
78 + resolveMeasurement(elapsed);
79 + }
80 + };
82 + child.on("error", (error) => finish(error));
83 + child.stderr.setEncoding("utf8");
84 + child.stderr.on("data", (chunk) => {
85 + stderr += chunk;
86 + });
87 + child.stdout.setEncoding("utf8");
88 + child.stdout.on("data", (chunk) => {
89 + stdout += chunk;
90 + for (;;) {
91 + const newline = stdout.indexOf("\n");
92 + if (newline < 0) break;
93 + const line = stdout.slice(0, newline);
94 + stdout = stdout.slice(newline + 1);
95 + let event;
96 + try {
97 + event = JSON.parse(line.endsWith("\r") ? line.slice(0, -1) : line);
98 + } catch (error) {
99 + finish(new Error(`Invalid Pi RPC output: ${error.message}`));
100 + return;
102 + if (event.type === "response" && event.command === "get_state") {
103 + if (!event.success) {
104 + finish(new Error(`Pi get_state failed: ${event.error}`));
105 + return;
107 + elapsed = process.hrtime.bigint() - started;
108 + return;
111 + });
112 + child.on("exit", (code, signal) => {
113 + if (settled) return;
114 + if (elapsed !== undefined) {
115 + finish();
116 + return;
118 + finish(
119 + new Error(
120 + `Pi exited before RPC readiness (code=${code}, signal=${signal})${stderr ? `: ${stderr.trim()}` : ""}`,
121 + ),
122 + );
123 + });
124 + child.stdin.end('{"type":"get_state"}\n');
125 + });
128 +for (let index = 0; index < warmups; index++) await measureStartup();
130 +const piVersion = spawnSync("pi", ["--version"], { encoding: "utf8" });
131 +if (piVersion.status !== 0) {
132 + throw new Error(`Unable to read Pi version: ${piVersion.stderr.trim()}`);
135 +console.log(`goos: ${process.platform}`);
136 +console.log(`goarch: ${process.arch}`);
137 +console.log("pkg: pi-ext/startup");
138 +console.log(`node: ${process.version}`);
139 +console.log(`pi: ${piVersion.stdout.trim()}`);
140 +console.log(`extensions: ${extensions.length}`);
141 +for (let round = 0; round < rounds; round++) {
142 + const elapsed = await measureStartup();
143 + console.log(`BenchmarkExtensionStartup\t1\t${elapsed} ns/op`);