repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

perf(rtk): benchmark and cache command rewrites

b0ccec2a5bc991a21976bb81fc7979bba2220bce

Oliver Jan Krylow <oliver@bugabinga.net> ·

SSH signature · not checked

Browse filesCompare with HEAD

Changed files (8)

CHANGELOG.md

Modified
2 2 # Changelog
4 4 ## Unreleased
6 +- Remove RTK `find -path` compatibility probing and guard; `find` rewrites now apply directly
6 7 - Harden dynamic-extension validator protocol handling and share workflow preparation across command and tool runs
7 8 - Identify Ultra child assignments by workflow, phase, and agent; attribute interpolated prior-agent output to its immediate producers, preserve provenance through fanout transformations, and invalidate checkpoints when attribution changes

extensions/rtk/README.md

Modified
39 39 The bash tool widget is the command indicator.
40 40 When RTK applies, Pi shows the rewritten `rtk ...` command.
41 -After session start, Pi checks RTK in the background and warns whether `find` rewrites are disabled or the compatibility guard can be removed.
41 +After session start, Pi checks RTK in the background and warns if it is unavailable or too old.
42 42 The first `bash` call waits for that shared check when it is still running.
44 44 ## Behavior
48 48 1. registers the `bash` hook without blocking extension load
49 49 2. probes `rtk --version` after session start, or on the first `bash` call
50 50 3. warns once and leaves commands unchanged if `rtk` is missing or too old
51 -4. probes `rtk find -path` against a temporary nested marker and warns with the result
52 -5. ignores non-bash tools
53 -6. ignores empty, already-`rtk`, and disabled commands
54 -7. calls `rtk rewrite <command>` with a 2s timeout
55 -8. applies exit `0` and `3` rewrites
56 -9. rejects every `rtk find` rewrite while the path probe fails
57 -10. locally maps missing `journalctl`, `systemctl status`, `hyprctl clients`, and `ps` filters to `rtk summary`
58 -11. leaves other exit `1`, exit `2`, timeouts, errors, and empty output untouched
51 +4. ignores non-bash tools
52 +5. ignores empty, already-`rtk`, and disabled commands
53 +6. calls `rtk rewrite <command>` with a 2s timeout
54 +7. applies exit `0` and `3` rewrites
55 +8. caches exact successful rewrites for the current session
56 +9. locally maps missing `journalctl`, `systemctl status`, `hyprctl clients`, and `ps` filters to `rtk summary`
57 +10. leaves other exit `1`, exit `2`, timeouts, errors, and empty output untouched
59 +## Performance
61 +`rtk rewrite` starts an external process, so repeated exact commands reuse deterministic successful results within one session.
62 +Session transitions clear the cache, and no-match or transient failures remain retryable.
63 +`mise run //extensions/rtk:bench` compares repeated commands against unique-command controls through Pi's test harness and installed RTK executable.
64 +`mise run //extensions/rtk:profile` records a Node CPU profile under `.research/rtk`.
65 +`mise run //extensions/rtk:benchstat <baseline> <candidate>` performs statistical comparison.
60 67 No prompt injection.
61 68 No status widget.

extensions/rtk/__bench__/rewrite.bench.ts

Added
new file mode 100644
1 +import { createTestSession } from "@marcfargas/pi-test-harness";
2 +import rtk from "../index.ts";
4 +const ROUNDS = Number(process.env.RTK_BENCH_ROUNDS ?? 10);
5 +const ITERATIONS = Number(process.env.RTK_BENCH_ITERATIONS ?? 1);
6 +let sink = 0;
8 +function event(command: string, id: string) {
9 + return {
10 + type: "tool_call" as const,
11 + toolName: "bash" as const,
12 + toolCallId: id,
13 + input: { command },
14 + };
17 +async function main(): Promise<void> {
18 + if (!Number.isInteger(ROUNDS) || ROUNDS < 1)
19 + throw new Error("RTK_BENCH_ROUNDS must be a positive integer");
20 + if (!Number.isInteger(ITERATIONS) || ITERATIONS < 1)
21 + throw new Error("RTK_BENCH_ITERATIONS must be a positive integer");
23 + const testSession = await createTestSession({ extensionFactories: [rtk] });
24 + const runner = testSession.session.extensionRunner;
25 + const rewrite = async (command: string, id: string) => {
26 + const call = event(command, id);
27 + await runner.emitToolCall(call);
28 + if (!call.input.command.startsWith("rtk "))
29 + throw new Error(`RTK did not rewrite: ${command}`);
30 + sink ^= call.input.command.length;
31 + };
33 + try {
34 + await rewrite("git status", "warmup");
35 + console.log(`goos: ${process.platform}`);
36 + console.log(`goarch: ${process.arch}`);
37 + console.log("pkg: pi-ext/rtk");
38 + for (const name of ["RepeatedRewrite", "UniqueRewrite"] as const) {
39 + for (let round = 0; round < ROUNDS; round++) {
40 + const started = process.hrtime.bigint();
41 + for (let index = 0; index < ITERATIONS; index++) {
42 + const command =
43 + name === "RepeatedRewrite"
44 + ? "git status"
45 + : `git status # rtk-bench-${round}-${index}`;
46 + await rewrite(command, `${name}-${round}-${index}`);
47 + }
48 + const elapsed = Number(process.hrtime.bigint() - started);
49 + console.log(
50 + `Benchmark${name}\t${ITERATIONS}\t${(elapsed / ITERATIONS).toFixed(0)} ns/op`,
51 + );
52 + }
53 + }
54 + } finally {
55 + await runner.emit({ type: "session_shutdown", reason: "quit" });
56 + testSession.dispose();
57 + }
60 +await main();
61 +if (sink === 42) console.error(sink);

extensions/rtk/__tests__/harness.test.ts

Modified
79 79 expect(event.input.command).toBe("rtk summary ps aux");
82 - it("rejects find rewrites when the capability probe fails", async () => {
82 + it("rewrites find commands without a compatibility probe", async () => {
83 83 const event = {
84 84 type: "tool_call",
85 85 toolName: "bash",
87 87 input: { command: "find . -name AGENTS.md" },
89 89 await t?.session.extensionRunner.emitToolCall(event as never);
90 - expect(event.input.command).toBe("find . -name AGENTS.md");
90 + expect(event.input.command).toBe("rtk find . -name AGENTS.md");

extensions/rtk/__tests__/rtk.test.ts

Modified
2 2 import { describe, expect, test, vi } from "vitest";
3 3 import rtkExtension, { __test } from "../index.ts";
5 5 const {
6 - isFindRewrite,
7 6 localFallbackRewrite,
8 7 parseRtkVersion,
9 8 shouldSkipCommand,
62 61 expect(shouldSkipCommand("rtk git status")).toBe(true);
63 62 expect(shouldSkipCommand("env RTK_DISABLED=1 git status")).toBe(true);
64 63 expect(shouldSkipCommand("git status")).toBe(false);
65 - expect(isFindRewrite("rtk find . -name AGENTS.md")).toBe(true);
66 - expect(isFindRewrite("rtk git status && rtk find . -name AGENTS.md")).toBe(
67 - true,
68 - );
69 - expect(isFindRewrite("rtk git status")).toBe(false);
72 66 test("preserves registered command prefixes around rewrites", async () => {
145 139 const fx = await loadWithRewrite(1);
146 140 expect(fx.pi.on).toHaveBeenCalledWith("tool_call", expect.any(Function));
147 141 await emit(fx, "git status");
148 - expect(fx.notes).toEqual([
149 - ["rtk find rewrites disabled: -path semantics unsupported", "warning"],
150 - ]);
142 + expect(fx.notes).toEqual([]);
153 145 test("rewrites via rtk and falls back only on no-match", async () => {
172 164 expect(await emit(fx as never, "ps aux")).toBe("rtk summary ps aux");
175 - test("rejects find rewrites when rtk find lacks path semantics", async () => {
167 + test("rewrites find commands without a compatibility probe", async () => {
176 168 const fx = fakePi(async (_command, args) => {
177 169 if (args[0] === "--version") return { code: 0, stdout: "rtk 0.42.4" };
178 - if (args[0] === "find") {
179 - return {
180 - code: 0,
181 - stdout: "0 for '*nested/rtk-find-path-capability-marker'",
182 - };
183 - }
184 - return { code: 3, stdout: "rtk find . -name AGENTS.md" };
185 - });
186 - await rtkExtension(fx.pi as never, fx.ctx as never);
187 - expect(await emit(fx as never, "find . -name AGENTS.md")).toBe(
188 - "find . -name AGENTS.md",
189 - );
190 - expect(fx.notes).toEqual([
191 - ["rtk find rewrites disabled: -path semantics unsupported", "warning"],
192 - ]);
193 - });
195 - test("allows find rewrites when the path capability probe passes", async () => {
196 - const fx = fakePi(async (_command, args) => {
197 - if (args[0] === "--version") return { code: 0, stdout: "rtk 0.42.4" };
198 - if (args[0] === "find") {
199 - return { code: 0, stdout: "nested/rtk-find-path-capability-marker" };
200 - }
201 170 return { code: 3, stdout: "rtk find . -name AGENTS.md" };
203 172 await rtkExtension(fx.pi as never, fx.ctx as never);
204 173 expect(await emit(fx as never, "find . -name AGENTS.md")).toBe(
205 174 "rtk find . -name AGENTS.md",
207 - expect(fx.notes).toEqual([
208 - ["rtk find -path works: remove the compatibility guard", "warning"],
209 - ]);
176 + expect(fx.notes).toEqual([]);
177 + });
179 + test("caches exact rewrites for the current session", async () => {
180 + const fx = await loadWithRewrite(0, "rtk git status\n");
181 + expect(await emit(fx, "git status")).toBe("rtk git status");
182 + expect(await emit(fx, "git status")).toBe("rtk git status");
183 + expect(fx.pi.exec).toHaveBeenCalledTimes(2);
185 + fx.handlers.session_start?.[0]?.({}, fx.ctx);
186 + expect(await emit(fx, "git status")).toBe("rtk git status");
187 + expect(fx.pi.exec).toHaveBeenCalledTimes(4);
212 190 test("leaves non-bash and skipped commands untouched", async () => {

extensions/rtk/index.ts

Modified
1 1 import { constants } from "node:fs";
2 -import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
3 -import { tmpdir } from "node:os";
2 +import { access } from "node:fs/promises";
4 3 import { delimiter, join } from "node:path";
5 4 import type {
6 5 ExecOptions,
6 + ExecResult,
7 7 ExtensionAPI,
8 8 ExtensionContext,
9 9 } from "@earendil-works/pi-coding-agent";
17 17 type VersionTuple = [number, number, number];
19 -type ExecResult = {
20 - code?: number | null;
21 - stdout?: string;
22 - stderr?: string;
23 - killed?: boolean;
24 -};
26 19 type RtkStatus =
27 20 | { ok: true }
28 21 | { ok: false; reason: "missing" | "old"; version?: string };
63 56 return /[|;<>`]|\$\(|&&|\|\|/u.test(command);
66 -export function isFindRewrite(command: string): boolean {
67 - return command === "rtk find" || command.includes("rtk find ");
70 59 export function localFallbackRewrite(command: string): string | undefined {
71 60 const trimmed = command.trim();
72 61 if (hasShellOperators(trimmed)) return undefined;
115 104 ): Promise<ExecResult> {
116 105 const command = await rtkCommand();
117 106 if (process.platform === "win32" && /\.(?:cmd|bat)$/iu.test(command)) {
118 - return (await pi.exec(
107 + return await pi.exec(
119 108 process.env.ComSpec || process.env.COMSPEC || "cmd.exe",
120 109 ["/d", "/c", "call", command, ...args],
121 110 options,
122 - )) as ExecResult;
111 + );
124 - return (await pi.exec(command, args, options)) as ExecResult;
113 + return await pi.exec(command, args, options);
127 116 function execCode(error: unknown): number | null {
175 -async function supportsFindPath(pi: ExtensionAPI): Promise<boolean> {
176 - const marker = "rtk-find-path-capability-marker";
177 - let root: string | undefined;
178 - try {
179 - root = await mkdtemp(join(tmpdir(), "pi-rtk-find-"));
180 - const nested = join(root, "nested");
181 - await mkdir(nested);
182 - await writeFile(join(nested, marker), "");
183 - const result = await execRtk(
184 - pi,
185 - ["find", root, "-path", `*nested/${marker}`],
186 - { timeout: TIMEOUT_MS },
187 - );
188 - const output = result.stdout ?? "";
189 - return (
190 - result.code === 0 &&
191 - !result.killed &&
192 - output.includes(marker) &&
193 - !/^\s*0 for /u.test(output)
194 - );
195 - } catch {
196 - return false;
197 - } finally {
198 - if (root) await rm(root, { force: true, recursive: true }).catch(() => {});
199 - }
202 164 function notifyUser(
203 165 pi: ExtensionAPI,
204 166 ctx: ExtensionContext | undefined,
241 203 export const __test = {
242 - isFindRewrite,
243 204 localFallbackRewrite,
244 205 parseRtkVersion,
245 206 shouldSkipCommand,
249 210 export default function rtk(pi: ExtensionAPI) {
250 211 let generation = 0;
251 212 let announced = false;
213 + let rewriteCache = new Map<string, RewriteOutcome>();
252 214 let initialization:
254 216 generation: number;
255 - promise: Promise<{
256 - status: RtkStatus;
257 - findPathSupported: boolean;
258 - }>;
217 + promise: Promise<RtkStatus>;
260 219 | undefined;
262 221 const ensureInitialized = async (ctx: ExtensionContext) => {
263 222 if (initialization?.generation !== generation) {
264 223 const ticket = generation;
265 - const promise = (async () => {
266 - const status = await rtkStatus(pi);
267 - return {
268 - status,
269 - findPathSupported: status.ok ? await supportsFindPath(pi) : false,
270 - };
271 - })();
224 + const promise = rtkStatus(pi);
272 225 initialization = { generation: ticket, promise };
276 229 const support = await initialization.promise;
277 230 if (ticket !== generation) return undefined;
278 231 if (!announced) {
279 - announced = support.status.ok
280 - ? notifyUser(
281 - pi,
282 - ctx,
283 - support.findPathSupported
284 - ? "rtk find -path works: remove the compatibility guard"
285 - : "rtk find rewrites disabled: -path semantics unsupported",
286 - )
287 - : warnUnavailable(support.status, pi, ctx);
232 + announced = support.ok || warnUnavailable(support, pi, ctx);
289 234 return support;
292 - pi.on("session_start", () => {
237 + const reset = () => {
293 238 generation++;
294 239 announced = false;
295 240 initialization = undefined;
296 - });
297 - pi.on("session_shutdown", () => {
298 - generation++;
299 - announced = false;
300 - initialization = undefined;
301 - });
241 + rewriteCache = new Map();
242 + };
243 + pi.on("session_start", reset);
244 + pi.on("session_shutdown", reset);
302 245 pi.on("tool_call", async (event, ctx) => {
303 246 if (!isToolCallEventType("bash", event)) return;
304 247 const support = await ensureInitialized(ctx);
305 - if (!support?.status.ok) return;
248 + if (!support?.ok) return;
306 249 const framed = splitCommandPrefix(event.input.command);
307 250 if (shouldSkipCommand(framed.command)) return;
308 - const outcome = await rewriteCommand(pi, framed.command, ctx.signal);
251 + const cache = rewriteCache;
252 + let outcome = cache.get(framed.command);
253 + if (!outcome) {
254 + outcome = await rewriteCommand(pi, framed.command, ctx.signal);
255 + if (outcome.kind === "rewrite") cache.set(framed.command, outcome);
309 257 const rewritten =
310 258 outcome.kind === "rewrite"
311 259 ? outcome.command
312 260 : outcome.kind === "fallback"
313 261 ? localFallbackRewrite(framed.command)
314 262 : undefined;
315 - if (rewritten && (support.findPathSupported || !isFindRewrite(rewritten)))
316 - event.input.command = framed.prefix + rewritten;
263 + if (rewritten) event.input.command = framed.prefix + rewritten;

extensions/rtk/mise.toml

Modified
1 +[tasks.bench]
2 +description = "Benchmark RTK rewrite hook"
3 +depends = ["//:install"]
4 +run = "node --no-warnings --experimental-strip-types __bench__/rewrite.bench.ts"
6 +[tasks.profile]
7 +description = "Profile RTK rewrite hook"
8 +depends = ["//:install"]
9 +run = [
10 + '''node -e "require('node:fs').mkdirSync('../../.research/rtk', { recursive: true })"''',
11 + "node --no-warnings --experimental-strip-types --cpu-prof --cpu-prof-dir ../../.research/rtk --cpu-prof-name rtk.cpuprofile __bench__/rewrite.bench.ts",
14 +[tasks.benchstat]
15 +description = "Compare RTK benchmark results"
16 +depends = ["//:install"]
17 +usage = '''
18 +arg "<baseline>"
19 +arg "<candidate>"
20 +'''
21 +run = 'benchstat "{{ usage.baseline }}" "{{ usage.candidate }}"'
1 23 [tasks.lint]
2 24 description = "Lint rtk extension"
3 25 depends = ["//:install"]

mise.android.toml

Modified
1 1 [settings]
2 -enable_tools = []
2 +enable_tools = ["go:golang.org/x/perf/cmd/benchstat"]
4 4 [tasks.e2e]
5 5 description = "Run browser E2E through Debian PRoot"