repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

refactor(tune): remove shell dependency

31526e9c2dca87e3a910d3a4db648280a6dddf30

Oliver Krylow <o.krylow@isp-insoft.de> ·

unsigned

Browse filesCompare with HEAD

Changed files (10)

CHANGELOG.md

Modified
2 2 # Changelog
4 4 ## Unreleased
6 -- Add a Tune cross-platform runner design that removes Bash/WSL and shell command strings, runs `.pi/tune/benchmark.mjs` directly with Node on every platform, preserves existing experiment behavior, and uses direct Git calls for discard
6 +- Make Tune shell-free across platforms by running `.pi/tune/benchmark.mjs` with Pi's Node executable and using checked direct Git calls for discard
7 7 - Remove the `mode`, `shame`, `ssh-tools`, and `terminal-signals` extensions
8 8 - Add an `/overkill aftermath` design for improving one confirmed skill from all past Pi sessions, with checkable task mining, deterministic checks plus LLM rubric fallback, hash-split validation, strict no-tie acceptance, resumable staged results, and no automatic adoption or scheduling
9 +- Fix Overkill Aftermath persistence on Windows by retrying transient `EPERM` atomic file replacements
9 10 - Fix npm audit's moderate `protobufjs` vulnerability by updating the root lockfile entry to 7.6.5
10 11 - Add a read-only `modernize` prompt and skill for auditing repositories against the latest Pi changelog and APIs, using resolved Pi package versions to scope the migration window
11 12 - Fix extension settings loading to use Pi's public settings API

docs/super/plans/2026-07-16-tune-cross-platform-runner.md

Modified
17 -### Task 1: Establish Regression Harness and No-Bash RED
17 +## Task 1: Establish Regression Harness and No-Bash RED
19 19 **Files:**
76 76 Expected: static checking fails because Tune still contains both Bash invocations.
78 -### Task 2: Replace Bash-Based Git Discard
78 +## Task 2: Replace Bash-Based Git Discard
80 80 **Files:**
203 203 Expected: all focused tests pass.
205 -### Task 3: Characterize and Replace the Benchmark Runner
205 +## Task 3: Characterize and Replace the Benchmark Runner
207 207 **Files:**
338 338 Expected: all Tune tests pass, including timeout and abort with no descendant marker.
340 -### Task 4: Use Pi's Node Runtime for Checks and Hooks
340 +## Task 4: Use Pi's Node Runtime for Checks and Hooks
342 342 **Files:**
392 392 Expected: focused tests pass and the fake PATH node is unused.
394 -### Task 5: Documentation and Repository Verification
394 +## Task 5: Documentation and Repository Verification
396 396 **Files:**
425 425 git commit -m "docs(tune): document shell-free runner"
428 -### Task 6: Release Gate
428 +## Task 6: Release Gate
430 430 **Files:**

extensions/tune/CHANGELOG.md

Modified
1 1 # Changelog
3 +## Unreleased
5 +- Remove the `run_experiment.command` parameter and execute `.pi/tune/benchmark.mjs` directly with Pi's Node runtime
6 +- Remove Bash/WSL dependencies from benchmark execution and Git discard
7 +- Preserve `.pi/tune` through checked, direct Git discard calls
3 9 ## 0.0.3
5 11 - Rename package, slug, command, and state paths from autoresearch to tune

extensions/tune/README.md

Modified
19 19 Tools:
21 21 - `init_experiment` — configure session name, metric, unit, and direction
22 -- `run_experiment` — run a timed benchmark command, capture output, parse
23 - `METRIC name=value` lines, and run checks when configured
22 +- `run_experiment` — run `.pi/tune/benchmark.mjs` with Pi's Node executable,
23 + capture output, parse `METRIC name=value` lines, and run checks when configured
24 24 - `log_experiment` — record result and keep or discard VCS changes
26 26 Settings live under `tune` in Pi settings (`~/.pi/agent/settings.json` or
85 85 `run_experiment` parses finite numeric metrics and rejects unsafe metric names
86 86 such as `__proto__` and `constructor`.
88 -Benchmark commands are guarded. `node .pi/tune/benchmark.mjs`, `node
89 -benchmark.mjs`, simple environment prefixes, and `nice node benchmark.mjs` are
90 -accepted. Chained commands are rejected.
88 +`run_experiment` takes no command.
89 +Tune requires no Bash, Git Bash, PowerShell, or WSL for experiments or VCS handling.
90 +Put environment setup and subprocess orchestration in `benchmark.mjs` with Node APIs.
92 92 If `.pi/tune/checks.mjs` exists, it runs after every passing benchmark. Check
93 93 failures block `keep`.
94 +Benchmark, checks, and hook modules use the same Node executable running Pi.
95 96 VCS behavior is native per repo:

extensions/tune/__tests__/harness.test.ts

Modified
1 +import * as fs from "node:fs";
1 2 import path from "node:path";
2 3 import {
3 4 createTestSession,
4 5 type TestSession,
5 6 } from "@marcfargas/pi-test-harness";
6 7 import { afterEach, describe, expect, it } from "vitest";
8 +import { __test } from "../index.ts";
8 10 describe("tune pi-test-harness", () => {
9 11 let t: TestSession | undefined;
13 + async function activeTune() {
14 + const extensionPath = path.resolve(import.meta.dirname, "../index.ts");
15 + t = await createTestSession({ extensions: [extensionPath] });
16 + fs.mkdirSync(path.join(t.cwd, ".pi", "tune"), { recursive: true });
17 + fs.writeFileSync(
18 + path.join(t.cwd, ".pi", "tune", "session.jsonl"),
19 + '{"type":"config","name":"runtime","metricName":"score","metricUnit":"","bestDirection":"lower"}\n',
20 + );
21 + __test.acquireTuneLock(t.cwd, t.session.sessionManager.getSessionId());
22 + await t.session.extensionRunner.emit({
23 + type: "session_tree",
24 + newLeafId: null,
25 + oldLeafId: null,
26 + });
27 + return t.session.extensionRunner.extensions[0];
28 + }
11 30 afterEach(() => {
12 31 t?.dispose();
13 32 t = undefined;
20 39 expect(t.session.extensionRunner.extensions).toHaveLength(1);
21 40 expect(t.session.extensionRunner.extensions[0].path).toBe(extensionPath);
43 + it("exposes and runs the fixed benchmark without a command", async () => {
44 + const extension = await activeTune();
45 + const tool = extension.tools.get("run_experiment").definition;
46 + expect(tool.parameters.properties).not.toHaveProperty("command");
47 + fs.writeFileSync(
48 + path.join(t.cwd, ".pi", "tune", "benchmark.mjs"),
49 + 'console.error("stderr"); console.log("METRIC score=42"); console.log("METRIC bytes_kb=7");\n',
50 + );
52 + const result = await tool.execute(
53 + "run",
54 + {},
55 + new AbortController().signal,
56 + undefined,
57 + t.session.extensionRunner.createContext(),
58 + );
60 + expect(result.details).toMatchObject({
61 + exitCode: 0,
62 + passed: true,
63 + parsedPrimary: 42,
64 + parsedMetrics: { score: 42, bytes_kb: 7 },
65 + });
66 + expect(result.content[0].text).toContain("stderr");
67 + });
69 + it("fails before launch when benchmark.mjs is missing", async () => {
70 + const extension = await activeTune();
71 + const result = await extension.tools
72 + .get("run_experiment")
73 + .definition.execute(
74 + "run",
75 + {},
76 + new AbortController().signal,
77 + undefined,
78 + t.session.extensionRunner.createContext(),
79 + );
80 + expect(result.content[0].text).toContain("Create .pi/tune/benchmark.mjs");
81 + });
83 + it("runs checks with Pi's Node executable", async () => {
84 + const extension = await activeTune();
85 + const tuneDir = path.join(t.cwd, ".pi", "tune");
86 + fs.writeFileSync(path.join(tuneDir, "benchmark.mjs"), "process.exit(0);\n");
87 + fs.writeFileSync(
88 + path.join(tuneDir, "checks.mjs"),
89 + "console.log(process.execPath);\n",
90 + );
91 + const result = await extension.tools
92 + .get("run_experiment")
93 + .definition.execute(
94 + "run",
95 + {},
96 + new AbortController().signal,
97 + undefined,
98 + t.session.extensionRunner.createContext(),
99 + );
100 + expect(result.details).toMatchObject({ checksPass: true });
101 + expect(result.content[0].text).toContain("Checks passed");
102 + });
104 + it("times out the fixed benchmark", async () => {
105 + const extension = await activeTune();
106 + fs.writeFileSync(
107 + path.join(t.cwd, ".pi", "tune", "benchmark.mjs"),
108 + "setInterval(() => {}, 1000);\n",
109 + );
110 + const result = await extension.tools
111 + .get("run_experiment")
112 + .definition.execute(
113 + "run",
114 + { timeout_seconds: 0.05 },
115 + new AbortController().signal,
116 + undefined,
117 + t.session.extensionRunner.createContext(),
118 + );
119 + expect(result.details).toMatchObject({ timedOut: true, passed: false });
120 + expect(result.content[0].text).toContain("TIMEOUT");
121 + });

extensions/tune/__tests__/remaining.test.ts

Modified
96 96 expect(__test.formatNum(42.5, "ms")).toBe("42.50ms");
97 97 expect(__test.formatNum(0, "")).toBe("0");
100 - it("isBenchmarkCommand accepts/rejects correctly", () => {
101 - expect(__test.isBenchmarkCommand("node .pi/tune/benchmark.mjs")).toBe(true);
102 - expect(__test.isBenchmarkCommand("node benchmark.mjs")).toBe(true);
103 - expect(__test.isBenchmarkCommand("pnpm test")).toBe(false);
104 - expect(__test.isBenchmarkCommand("nice node benchmark.mjs")).toBe(true);
105 - });
108 101 // ── 3. SHORTCUTS ──────────────────────────────────────────────────────

extensions/tune/__tests__/unit.test.ts

Modified
164 -// ── Benchmark command guard ───────────────────────────────────────────────
166 -describe("isBenchmarkCommand", () => {
167 - const { isBenchmarkCommand } = __test;
169 - it("accepts node benchmark.mjs", () => {
170 - expect(isBenchmarkCommand("node .pi/tune/benchmark.mjs")).toBe(true);
171 - });
173 - it("accepts node with path to benchmark.mjs", () => {
174 - expect(isBenchmarkCommand("node benchmark.mjs")).toBe(true);
175 - });
177 - it("rejects arbitrary commands", () => {
178 - expect(isBenchmarkCommand("pnpm test")).toBe(false);
179 - });
181 - it("strips env prefix", () => {
182 - expect(isBenchmarkCommand("NODE_ENV=test node benchmark.mjs")).toBe(true);
183 - });
185 - it("rejects chained commands", () => {
186 - expect(isBenchmarkCommand("evil.py; node benchmark.mjs")).toBe(false);
187 - expect(isBenchmarkCommand("node benchmark.mjs && echo pwned")).toBe(false);
188 - });
189 -});
191 164 // ── VCS backend ──────────────────────────────────────────────────────────
193 166 describe("vcs backend", () => {
249 222 ["jj", "new"],
226 + it("uses direct Git calls on discard", async () => {
227 + const calls: Array<[string, string[]]> = [];
228 + const exec = async (cmd: string, args: string[]) => {
229 + calls.push([cmd, args]);
230 + return { code: cmd === "jj" ? 1 : 0, stdout: "", stderr: "" };
231 + };
233 + await __test.discardVcsChanges(exec, "/repo", "discard");
235 + expect(calls).toEqual([
236 + ["jj", ["root"]],
238 + "git",
240 + "checkout",
241 + "--",
242 + ".",
243 + ":(exclude,glob).pi/tune/**",
244 + ":(exclude,glob).pi/tune/*",
245 + ],
246 + ],
247 + ["git", ["clean", "-fd", "-e", ".pi/tune", "-e", ".pi/tune/**"]],
248 + ]);
249 + });
251 + it("does not clean after checkout failure", async () => {
252 + const calls: string[] = [];
253 + const exec = async (cmd: string, args: string[]) => {
254 + calls.push(`${cmd} ${args[0]}`);
255 + if (cmd === "jj") return { code: 1, stdout: "", stderr: "" };
256 + return { code: 1, stdout: "", stderr: "checkout failed" };
257 + };
259 + await expect(
260 + __test.discardVcsChanges(exec, "/repo", "discard"),
261 + ).rejects.toThrow("git checkout failed: checkout failed");
262 + expect(calls).toEqual(["jj root", "git checkout"]);
263 + });
254 266 // ── Tune tool activation ──────────────────────────────────────────

extensions/tune/hooks.ts

Modified
104 104 const t0 = Date.now();
105 105 return new Promise<HookResult>((resolve) => {
106 - const child = spawn("node", [script], {
106 + const child = spawn(process.execPath, [script], {
107 107 cwd: payload.cwd,
108 108 timeout: TIMEOUT_MS,

extensions/tune/index.ts

Modified
157 157 // ── Tool parameter schemas ────────────────────────────────────────────────
159 159 const RunParams = Type.Object({
160 - command: Type.String({
161 - description: "Shell command to run (e.g. 'node .pi/tune/benchmark.mjs')",
162 - }),
163 160 timeout_seconds: Type.Optional(
164 161 Type.Number({ description: "Kill after this many seconds (default: 600)" }),
287 284 // ── Misc helpers ──────────────────────────────────────────────────────────
289 -function killTree(pid: number): void {
286 +const TASKKILL_PATH =
287 + process.platform === "win32"
288 + ? path.join(
289 + process.env.SystemRoot ?? "C:\\Windows",
290 + "System32",
291 + "taskkill.exe",
293 + : null;
295 +async function killTree(
296 + pid: number,
297 + platform = process.platform,
298 + taskkillPath = TASKKILL_PATH,
299 +): Promise<void> {
300 + if (platform === "win32" && taskkillPath && fs.existsSync(taskkillPath)) {
301 + const killed = await new Promise<boolean>((resolve) => {
302 + const killer = spawn(taskkillPath, ["/F", "/T", "/PID", String(pid)], {
303 + stdio: "ignore",
304 + windowsHide: true,
305 + });
306 + killer.once("error", () => resolve(false));
307 + killer.once("close", (code) => resolve(code === 0));
308 + });
309 + if (killed) return;
291 - process.kill(-pid, "SIGTERM");
312 + process.kill(platform === "win32" ? pid : -pid, "SIGTERM");
292 313 } catch {
294 315 process.kill(pid, "SIGTERM");
594 -// ── Benchmark command guard ───────────────────────────────────────────────
596 -function isBenchmarkCommand(command: string): boolean {
597 - let cmd = command.trim();
598 - cmd = cmd.replace(/^(?:\w+=\S*\s+)+/, "");
599 - let prev: string;
600 - do {
601 - prev = cmd;
602 - cmd = cmd.replace(/^(?:env|time|nice|nohup)(?:\s+-\S+(?:\s+\d+)?)*\s+/, "");
603 - } while (cmd !== prev);
605 - return /^(?:node)\s+(?:.*\/)?benchmark\.mjs\s*$/.test(cmd);
608 615 type VcsKind = "jj" | "git";
609 616 interface ExecResultLike {
610 617 code: number | null;
734 741 return { kind, text: `\n📝 jj: abandoned change (${status})` };
737 - const revertScript = `
738 - git checkout -- . ':(exclude,glob).pi/tune/**' ':(exclude,glob).pi/tune/*'
739 - git clean -fd -e '.pi/tune' -e '.pi/tune/**' 2>/dev/null
740 - `;
741 - await exec("bash", ["-c", revertScript], { cwd, timeout: 10000 });
744 + const execOpts = { cwd, timeout: 10000 };
745 + const checkout = await exec(
746 + "git",
748 + "checkout",
749 + "--",
750 + ".",
751 + ":(exclude,glob).pi/tune/**",
752 + ":(exclude,glob).pi/tune/*",
753 + ],
754 + execOpts,
755 + );
756 + if (checkout.code !== 0) {
757 + throw new Error(
758 + `git checkout failed: ${(checkout.stdout + checkout.stderr).trim().slice(0, 200)}`,
759 + );
761 + const clean = await exec(
762 + "git",
763 + ["clean", "-fd", "-e", ".pi/tune", "-e", ".pi/tune/**"],
764 + execOpts,
765 + );
766 + if (clean.code !== 0) {
767 + throw new Error(
768 + `git clean failed: ${(clean.stdout + clean.stderr).trim().slice(0, 200)}`,
769 + );
742 771 return {
744 773 text: `\n📝 Git: reverted changes (${status}) — .pi/tune/ preserved`,
1173 1202 export const __test = {
1174 1203 parseMetricLines,
1175 1204 formatNum,
1176 - isBenchmarkCommand,
1177 1205 dashboardTemplatePath,
1178 1206 tuneArgumentCompletions,
1179 1207 tuneCommandArgumentSuggestions,
1714 1742 // ── Lifecycle hooks ─────────────────────────────────────────────────
1744 + let warnedMissingTaskkill = false;
1716 1745 pi.on("session_start", async (_e, ctx) => {
1717 1746 settingsCwd = ctx.cwd;
1747 + if (
1748 + TASKKILL_PATH &&
1749 + !warnedMissingTaskkill &&
1750 + !fs.existsSync(TASKKILL_PATH)
1751 + ) {
1752 + warnedMissingTaskkill = true;
1753 + ctx.ui.notify(
1754 + "taskkill.exe unavailable; Tune timeouts will kill only the benchmark process",
1755 + "info",
1756 + );
1718 1758 reconstructState(ctx);
1719 1759 setTuneToolsActive(pi, getRuntime(ctx).tuneMode);
1985 2025 pi.registerTool({
1986 2026 name: "run_experiment",
1987 2027 label: "Run Experiment",
1988 - description: `Run a shell command as an experiment. Times wall-clock duration, captures output, detects pass/fail. Output truncated to last ${EXPERIMENT_MAX_LINES} lines or ${EXPERIMENT_MAX_BYTES / 1024}KB.`,
1989 - promptSnippet: "Run a timed experiment command",
2028 + description: `Run .pi/tune/benchmark.mjs as an experiment. Times wall-clock duration, captures output, detects pass/fail. Output truncated to last ${EXPERIMENT_MAX_LINES} lines or ${EXPERIMENT_MAX_BYTES / 1024}KB.`,
2029 + promptSnippet: "Run the timed benchmark.mjs experiment",
1990 2030 promptGuidelines: [
1991 - "Use run_experiment instead of bash for experiment commands.",
2031 + "Edit .pi/tune/benchmark.mjs, then call run_experiment without a command.",
1992 2032 "After run_experiment, always call log_experiment to record the result.",
1993 2033 "If the benchmark script outputs METRIC lines, run_experiment parses them automatically.",
2029 - // Guard: if benchmark.mjs exists, only allow running it
2030 2069 const benchmarkPath = tuneScriptPath(cwd);
2031 - if (fs.existsSync(benchmarkPath) && !isBenchmarkCommand(params.command)) {
2070 + if (!fs.existsSync(benchmarkPath)) {
2032 2071 return {
2033 2072 content: [
2035 2074 type: "text",
2036 - text: `❌ benchmark.mjs exists — you must run it instead of a custom command.\n\nFound: ${benchmarkPath}\nYour command: ${params.command}\n\nUse: run_experiment({ command: "node .pi/tune/benchmark.mjs" })`,
2075 + text: "❌ Create .pi/tune/benchmark.mjs before run_experiment.",
2039 - details: {
2040 - command: params.command,
2041 - exitCode: null,
2042 - durationSeconds: 0,
2043 - passed: false,
2044 - crashed: true,
2045 - timedOut: false,
2046 - tailOutput: "",
2047 - checksPass: null,
2048 - checksTimedOut: false,
2049 - checksOutput: "",
2050 - checksDuration: 0,
2051 - } as RunDetails,
2078 + details: {},
2082 + const benchmarkDisplay = "node .pi/tune/benchmark.mjs";
2055 2083 runtime.runningExperiment = {
2056 2084 startedAt: Date.now(),
2057 - command: params.command,
2085 + command: benchmarkDisplay,
2059 2087 updateWidget(ctx);
2060 2088 if (overlayTui) overlayTui.requestRender();
2078 2106 }>((resolve, reject) => {
2079 2107 let processTimedOut = false;
2081 - const child = spawn("bash", ["-c", params.command], {
2109 + const child = spawn(process.execPath, [benchmarkPath], {
2083 - detached: true,
2111 + detached: process.platform !== "win32",
2112 + windowsHide: true,
2084 2113 stdio: ["ignore", "pipe", "pipe"],
2149 2178 if (child.stdout) child.stdout.on("data", handleData);
2150 2179 if (child.stderr) child.stderr.on("data", handleData);
2181 + let termination: Promise<void> | null = null;
2182 + const terminateOnce = (): Promise<void> => {
2183 + if (termination) return termination;
2184 + termination = child.pid
2185 + ? killTree(child.pid)
2186 + : new Promise((resolve) => {
2187 + child.once("spawn", () => {
2188 + if (child.pid) void killTree(child.pid).then(resolve);
2189 + else resolve();
2190 + });
2191 + child.once("error", () => resolve());
2192 + });
2193 + return termination;
2194 + };
2152 2195 let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
2153 2196 if (timeout > 0) {
2154 2197 timeoutHandle = setTimeout(() => {
2155 2198 processTimedOut = true;
2156 - if (child.pid) killTree(child.pid);
2199 + void terminateOnce();
2157 2200 }, timeout);
2160 - const onAbort = () => {
2161 - if (child.pid) killTree(child.pid);
2162 - else {
2163 - child.kill();
2164 - child.once("spawn", () => {
2165 - if (child.pid) killTree(child.pid);
2166 - });
2168 - };
2203 + const onAbort = () => void terminateOnce();
2169 2204 if (signal) {
2170 2205 if (signal.aborted) onAbort();
2171 2206 else signal.addEventListener("abort", onAbort, { once: true });
2179 2214 reject(err);
2182 - child.on("close", (code) => {
2217 + child.on("close", async (code) => {
2183 2218 clearInterval(timerInterval);
2184 2219 if (timeoutHandle) clearTimeout(timeoutHandle);
2185 2220 if (signal) signal.removeEventListener("abort", onAbort);
2221 + if (termination) await termination;
2186 2222 if (tempFileStream) tempFileStream.end();
2188 2224 if (signal?.aborted) {
2221 2257 Math.max(0, params.checks_timeout_seconds ?? 300) * 1000;
2222 2258 const ct0 = Date.now();
2224 - const checksResult = await pi.exec("node", [checksPath], {
2260 + const checksResult = await pi.exec(process.execPath, [checksPath], {
2225 2261 signal,
2226 2262 timeout: checksTimeout,
2274 2310 const parsedPrimary = parsedMetricMap.get(state.metricName) ?? null;
2276 2312 const details: RunDetails = {
2277 - command: params.command,
2313 + command: benchmarkDisplay,
2278 2314 exitCode,
2279 2315 durationSeconds,
2280 2316 passed,
2361 2397 renderCall(args, theme) {
2362 2398 let text = theme.fg("toolTitle", theme.bold("run_experiment "));
2363 - text += theme.fg("muted", args.command);
2399 + text += theme.fg("muted", ".pi/tune/benchmark.mjs");
2364 2400 if (args.timeout_seconds !== undefined)
2365 2401 text += theme.fg(

scripts/test_extensions_static_check.mjs

Modified
171 +const tuneIndex = readFileSync(join(ROOT, "extensions/tune/index.ts"), "utf8");
172 +assert(!tuneIndex.includes('spawn("bash"'));
173 +assert(!tuneIndex.includes('exec("bash"'));
171 175 assert.deepEqual(rootPkg().pi, {
172 176 extensions: ["extensions/*/index.ts"],
173 177 skills: [