repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

Split Overkill into a lightweight command/renderer shell and lazy implementation; import fell 85% Win32 and 75% WSL, 177 tests passed

63d2b32823e98519fb52262c59d2d4955af09132

Oliver Krylow <o.krylow@isp-insoft.de> ·

SSH signature · not checked

Browse filesCompare with HEAD

Result: {"status":"keep","overkill_import_ms":57,"elapsed_s":182.8,"win32_start_reload_ms":8795.272,"win32_extension_tax_ms":5401.166,"win32_startup_ms":6797.067,"win32_reload_ms":1998.204,"wsl_start_reload_ms":1320.399,"wsl_extension_tax_ms":565.383,"wsl_overkill_import_ms":20}

Changed files (7)

extensions/overkill/__tests__/acceptance.test.ts

Modified
19 19 type ExplicitSuiteAcceptanceSeam,
20 20 installExplicitSuiteAcceptanceSeam,
21 21 installModeAcceptanceSeam,
22 -} from "../index.ts";
22 +} from "../implementation.ts";
24 24 const extensionPath = fileURLToPath(new URL("../index.ts", import.meta.url));
25 25 const temporaryPaths: string[] = [];

extensions/overkill/__tests__/aftermath-acceptance.test.ts

Modified
24 24 type AftermathAcceptanceSeam,
25 25 installAftermathAcceptanceSeam,
26 26 installModeAcceptanceSeam,
27 -} from "../index.ts";
27 +} from "../implementation.ts";
28 28 import {
29 29 assignSplits,
30 30 type FrozenTask,

extensions/overkill/__tests__/aftermath-commands.test.ts

Modified
1 1 import { SessionManager } from "@earendil-works/pi-coding-agent";
2 2 import { afterEach, expect, it, vi } from "vitest";
3 -import overkill, { harvestAftermath } from "../index.ts";
3 +import overkill, { harvestAftermath } from "../implementation.ts";
5 5 afterEach(() => vi.restoreAllMocks());

extensions/overkill/__tests__/commands.test.ts

Modified
7 7 chooseLoadedSkill,
8 8 parseArguments,
9 9 unfinishedRunWarning,
10 -} from "../index.ts";
10 +} from "../implementation.ts";
11 11 import { createRunDirectory, persistManifest } from "../lib/run-state.ts";
13 13 type Command = {

extensions/overkill/entry-renderer.ts

Added
new file mode 100644
1 +import { join } from "node:path";
2 +import {
3 + type ExtensionAPI,
4 + getAgentDir,
5 + keyHint,
6 + renderDiff,
7 +} from "@earendil-works/pi-coding-agent";
8 +import { Text } from "@earendil-works/pi-tui";
9 +import {
10 + buildReviewDiff,
11 + buildReviewDiffPreview,
12 + formatProgress,
13 + sanitizeTerminalText,
14 +} from "./lib/modes.ts";
15 +import {
16 + type RunEntry,
17 + readFrozenReviewArtifactsSync,
18 + validateRunEntry,
19 +} from "./lib/run-state.ts";
21 +export function registerOverkillEntryRenderer(pi: ExtensionAPI): void {
22 + pi.registerEntryRenderer<RunEntry>(
23 + "overkill-run",
24 + (entry, options, theme) => {
25 + let data: RunEntry;
26 + try {
27 + data = validateRunEntry(entry.data);
28 + } catch {
29 + return undefined;
30 + }
31 + const summary = theme.fg("accent", formatProgress(data));
32 + if (!options.expanded) {
33 + const details = [
34 + `${summary} ${theme.fg("dim", keyHint("app.tools.expand", "to expand"))}`,
35 + ];
36 + if (data.terminal === "staged")
37 + try {
38 + const artifacts = readFrozenReviewArtifactsSync(
39 + join(getAgentDir(), "overkill-runs", data.runId),
40 + );
41 + if (artifacts.stagedPath && artifacts.staged)
42 + details.push(
43 + renderDiff(
44 + buildReviewDiffPreview(
45 + buildReviewDiff(
46 + artifacts.sourcePath,
47 + artifacts.stagedPath,
48 + artifacts.source,
49 + artifacts.staged,
50 + ),
51 + ),
52 + ),
53 + );
54 + } catch {
55 + // The durable run summary remains useful without review artifacts.
56 + }
57 + return new Text(details.join("\n"), 0, 0);
58 + }
59 + const details = [summary];
60 + if (data.stage === "preflight") return new Text(summary, 0, 0);
61 + try {
62 + const artifacts = readFrozenReviewArtifactsSync(
63 + join(getAgentDir(), "overkill-runs", data.runId),
64 + );
65 + details.push(
66 + theme.fg(
67 + "dim",
68 + `Source: ${sanitizeTerminalText(artifacts.sourcePath)}`,
69 + ),
70 + );
71 + if (artifacts.reportPath)
72 + details.push(
73 + theme.fg(
74 + "dim",
75 + `Report: ${sanitizeTerminalText(artifacts.reportPath)}`,
76 + ),
77 + );
78 + if (artifacts.stagedPath && artifacts.staged) {
79 + details.push(
80 + theme.fg(
81 + "dim",
82 + `Staged: ${sanitizeTerminalText(artifacts.stagedPath)}`,
83 + ),
84 + );
85 + details.push(
86 + renderDiff(
87 + buildReviewDiff(
88 + artifacts.sourcePath,
89 + artifacts.stagedPath,
90 + artifacts.source,
91 + artifacts.staged,
92 + ),
93 + ),
94 + );
95 + }
96 + } catch {
97 + details.push(theme.fg("warning", "Review artifacts unavailable."));
98 + }
99 + return new Text(details.join("\n"), 0, 0);
100 + },
101 + );

extensions/overkill/implementation.ts

Added
new file mode 100644
1 +import { createHash } from "node:crypto";
2 +import { writeSync } from "node:fs";
3 +import { readdir, readFile, realpath } from "node:fs/promises";
4 +import {
5 + basename,
6 + dirname,
7 + extname,
8 + isAbsolute,
9 + join,
10 + relative,
11 + resolve,
12 +} from "node:path";
13 +import {
14 + type ExtensionAPI,
15 + type ExtensionCommandContext,
16 + type ExtensionContext,
17 + getAgentDir,
18 + SessionManager,
19 +} from "@earendil-works/pi-coding-agent";
20 +import { Key, matchesKey, SelectList } from "@earendil-works/pi-tui";
21 +import { registerOverkillEntryRenderer } from "./entry-renderer.ts";
22 +import {
23 + buildDigest,
24 + type Digest,
25 + rankSkillEvidence,
26 +} from "./lib/aftermath-harvest.ts";
27 +import {
28 + assignSplits,
29 + createMiningSession,
30 + estimateMiningCalls,
31 + mineTaskProposals,
32 + type StructuredOutputSdkSeam,
33 + validateAndFreezeTasks,
34 +} from "./lib/aftermath-mining.ts";
35 +import {
36 + createGeneratedTaskScorer,
37 + createRubricJudgeSession,
38 + generatedTaskPrompt,
39 + rubricCacheKey,
40 + runRubricJudge,
41 +} from "./lib/aftermath-scoring.ts";
42 +import {
43 + type AftermathManifest,
44 + createAftermathDirectory,
45 + persistAftermathCost,
46 + persistAftermathManifest,
47 + persistFrozenTasks,
48 + persistHarvest,
49 + persistJudgeCache,
50 + persistMiningCheckpoint,
51 + persistMiningOutput,
52 + readAftermathCost,
53 + readAftermathManifest,
54 + readFrozenTasks,
55 + readHarvest,
56 + readJudgeCache,
57 + readMiningCheckpoint,
58 + readMiningOutput,
59 +} from "./lib/aftermath-state.ts";
60 +import {
61 + executeChecker,
62 + fingerprintSource,
63 + fingerprintSuite,
64 +} from "./lib/checker.ts";
65 +import {
66 + getValidatedSuiteItems,
67 + parseSourceSkill,
68 + resolveLoadedSkill,
69 + type ValidatedItem,
70 + validateTrustedSuite,
71 +} from "./lib/contracts.ts";
72 +import {
73 + buildConfirmation,
74 + buildReviewDiff,
75 + formatProgress,
76 + formatResultPaths,
77 + formatScore,
78 + sanitizeTerminalText,
79 + upperBoundCalls,
80 +} from "./lib/modes.ts";
81 +import {
82 + createOptimizerSession,
83 + defaultOptimizerSdk,
84 + OptimizerCancelledError,
85 + type OptimizerSdkSeam,
86 + runOptimizerStep,
87 +} from "./lib/optimizer.ts";
88 +import { parsePatch } from "./lib/patches.ts";
89 +import { inspectSuiteRun, prepareSuiteRun } from "./lib/preflight.ts";
90 +import {
91 + createIsolatedTargetSession,
92 + defaultTargetSdk,
93 + RolloutCancelledError,
94 + runSplitRollout,
95 + runValidationBaseline,
96 + type TargetSdkSeam,
97 +} from "./lib/rollouts.ts";
98 +import {
99 + discardLinkedRun,
100 + persistFinalReport,
101 + persistManifest,
102 + persistTrainingCheckpoint,
103 + type RunEntry,
104 + type RunManifest,
105 + readFrozenReviewArtifacts,
106 + readManifest,
107 + readScoreCache,
108 + readTrainingCheckpoint,
109 + recoverRunLink,
110 + selectNewestAftermathRun,
111 + selectNewestSuiteRun,
112 + validateRunEntry,
113 + writePrivateText,
114 +} from "./lib/run-state.ts";
115 +import {
116 + type AttemptedPatch,
117 + createTrainingState,
118 + type DurableTrainingState,
119 + runDurableTrainingSchedule,
120 +} from "./lib/training.ts";
122 +const SUBCOMMANDS = [
123 + "aftermath",
124 + "suite",
125 + "status",
126 + "resume",
127 + "review",
128 + "discard",
129 + "cancel",
130 +] as const;
132 +type ActiveRun = {
133 + kind: "suite" | "aftermath";
134 + runId: string;
135 + sessionId: string;
136 + controller: AbortController;
137 + suppressOutput: boolean;
138 + completed?: Promise<void>;
139 + cancelRequested?: boolean;
140 + interrupt?: () => Promise<void>;
141 + removeCancellationListeners?: () => void;
144 +let activeRun: ActiveRun | undefined;
145 +let startPending = false;
146 +let pendingSessionId: string | undefined;
147 +let pendingCancelled = false;
149 +export type ExplicitSuiteAcceptanceSeam = {
150 + targetSdk: TargetSdkSeam;
151 + optimizerSdk: OptimizerSdkSeam;
154 +export type AftermathSessionApi = {
155 + listAll: (
156 + sessionDir?: string,
157 + ) => Promise<Array<{ path: string; id: string }>>;
158 + open: (
159 + path: string,
160 + sessionDir?: string,
161 + ) => {
162 + getBranch: () => unknown[];
163 + getSessionId: () => string;
164 + getCwd: () => string;
165 + };
168 +export type AftermathAcceptanceSeam = ExplicitSuiteAcceptanceSeam & {
169 + sessionApi: AftermathSessionApi;
170 + structuredOutputSdk: StructuredOutputSdkSeam;
171 + /** Lets acceptance fakes request cancellation through the registered command. */
172 + bindCommand?: (handler: (args: string) => Promise<void>) => void;
175 +export type ModeAcceptanceSeam = {
176 + write: (fd: number, text: string) => void;
179 +const explicitSuiteAcceptanceSeam = Symbol.for(
180 + "@bugabinga/pi-ext-overkill/explicit-suite-acceptance-seam",
182 +const aftermathAcceptanceSeam = Symbol.for(
183 + "@bugabinga/pi-ext-overkill/aftermath-acceptance-seam",
185 +const modeAcceptanceSeam = Symbol.for(
186 + "@bugabinga/pi-ext-overkill/mode-acceptance-seam",
189 +function installedExplicitSuiteAcceptanceSeam():
190 + | ExplicitSuiteAcceptanceSeam
191 + | undefined {
192 + return (globalThis as Record<PropertyKey, unknown>)[
193 + explicitSuiteAcceptanceSeam
194 + ] as ExplicitSuiteAcceptanceSeam | undefined;
197 +function installedAftermathAcceptanceSeam():
198 + | AftermathAcceptanceSeam
199 + | undefined {
200 + return (globalThis as Record<PropertyKey, unknown>)[
201 + aftermathAcceptanceSeam
202 + ] as AftermathAcceptanceSeam | undefined;
205 +function installedModeAcceptanceSeam(): ModeAcceptanceSeam | undefined {
206 + return (globalThis as Record<PropertyKey, unknown>)[modeAcceptanceSeam] as
207 + | ModeAcceptanceSeam
208 + | undefined;
211 +/** Installs deterministic session boundaries while retaining command orchestration. */
212 +export function installExplicitSuiteAcceptanceSeam(
213 + seam: ExplicitSuiteAcceptanceSeam,
214 +): () => void {
215 + const globals = globalThis as Record<PropertyKey, unknown>;
216 + const previous = globals[explicitSuiteAcceptanceSeam];
217 + globals[explicitSuiteAcceptanceSeam] = seam;
218 + return () => {
219 + if (globals[explicitSuiteAcceptanceSeam] !== seam) return;
220 + if (previous === undefined) delete globals[explicitSuiteAcceptanceSeam];
221 + else globals[explicitSuiteAcceptanceSeam] = previous;
222 + };
225 +/** Captures direct mode output while preserving the fd selected by production code. */
226 +export function installModeAcceptanceSeam(
227 + seam: ModeAcceptanceSeam,
228 +): () => void {
229 + const globals = globalThis as Record<PropertyKey, unknown>;
230 + const previous = globals[modeAcceptanceSeam];
231 + globals[modeAcceptanceSeam] = seam;
232 + return () => {
233 + if (globals[modeAcceptanceSeam] !== seam) return;
234 + if (previous === undefined) delete globals[modeAcceptanceSeam];
235 + else globals[modeAcceptanceSeam] = previous;
236 + };
239 +/** Installs deterministic constructors at Aftermath's production boundaries. */
240 +export function installAftermathAcceptanceSeam(
241 + seam: AftermathAcceptanceSeam,
242 +): () => void {
243 + const globals = globalThis as Record<PropertyKey, unknown>;
244 + const previous = globals[aftermathAcceptanceSeam];
245 + globals[aftermathAcceptanceSeam] = seam;
246 + return () => {
247 + if (globals[aftermathAcceptanceSeam] !== seam) return;
248 + if (previous === undefined) delete globals[aftermathAcceptanceSeam];
249 + else globals[aftermathAcceptanceSeam] = previous;
250 + };
253 +export function buildSuiteAuthoringPrompt(
254 + sourcePath: string,
255 + suitePath: string | undefined,
256 + projectRoot: string,
257 +): string {
258 + const sourceName = basename(sourcePath);
259 + const skillName =
260 + sourceName === "SKILL.md"
261 + ? basename(dirname(sourcePath))
262 + : basename(sourceName, extname(sourceName));
263 + const destination = suitePath
264 + ? `Use ${JSON.stringify(suitePath)} as the suite directory.`
265 + : `Choose a suite directory inside ${JSON.stringify(projectRoot)}. Prefer an established project evaluator location; otherwise use the project-local scratch directory ${JSON.stringify(join(projectRoot, ".overkill", "suites", skillName))}.`;
266 + return `Create a trusted explicit evaluator suite for the loaded Pi skill at ${JSON.stringify(sourcePath)}.
267 +${destination}
269 +Read the complete skill and only its directly referenced files needed to understand behavior.
270 +The suite contract below is authoritative; do not inspect Overkill implementation or unrelated project documentation.
271 +Inspect the destination once immediately before writing; if evaluator files already exist or appear concurrently modified, stop and ask rather than overwriting them.
272 +Do not design locks, atomic reservations, or repeated concurrency probes.
273 +After that single inspection, write the four files immediately without narrating a plan.
274 +Create exactly this layout and nothing else inside the suite directory:
275 +- train/items.json
276 +- val/items.json
277 +- test/items.json
278 +- check.mjs
280 +Create a small first suite: exactly 3 train items, 2 validation items, and 2 test items, with globally unique nonempty string ids and nonempty prompts no larger than 16 KiB.
281 +Add checker-owned expected fields when useful.
282 +Use train cases for optimization, validation cases for candidate selection, and held-out test cases for final evaluation; keep scenarios distinct across splits.
283 +Prioritize the skill's highest-risk and most representative successful and failure workflows; report omitted coverage instead of adding more cases.
284 +Keep each item concise and use a small shared expectation vocabulary such as exact, required substrings, forbidden substrings, or simple patterns.
285 +Do not design a domain-specific testing framework.
286 +The checker must read one JSON object from stdin, inspect item, response, trajectory, and calledTools, then write exactly one JSON object with score in [0,1], Boolean passed, and optional feedback.
287 +Write all four files before refining them, and keep check.mjs under 120 lines.
288 +Use only Node.js standard library in check.mjs.
289 +Keep checks bounded, cross-platform, and independent of network, credentials, mutable external state, and the source skill's exact wording.
290 +Do not weaken trust-boundary, security, data-loss, or required human-confirmation behavior.
291 +Validate the JSON and checker locally with one passing and one failing input.
292 +Do not start Overkill automatically.
293 +Summarize coverage, limitations, and the exact /overkill <skill-path> <suite-dir> command the user should run after reviewing check.mjs.`;
296 +export function parseArguments(input: string): string[] {
297 + const parts: string[] = [];
298 + let index = 0;
299 + while (index < input.length) {
300 + while (/\s/u.test(input[index] ?? "")) index += 1;
301 + if (index >= input.length) break;
302 + let token = "";
303 + while (index < input.length && !/\s/u.test(input[index] ?? "")) {
304 + const quote = input[index];
305 + if (quote !== '"' && quote !== "'") {
306 + token += quote;
307 + index += 1;
308 + continue;
310 + index += 1;
311 + let closed = false;
312 + while (index < input.length) {
313 + const character = input[index];
314 + if (character === quote) {
315 + closed = true;
316 + index += 1;
317 + break;
319 + if (character === "\\" && input[index + 1] === quote) {
320 + token += quote;
321 + index += 2;
322 + } else {
323 + token += character;
324 + index += 1;
327 + if (!closed) throw new Error("unterminated quoted argument");
329 + parts.push(token);
331 + return parts;
334 +async function canonicalProspectivePath(path: string): Promise<string> {
335 + let existing = resolve(path);
336 + const missing: string[] = [];
337 + while (true) {
338 + try {
339 + return resolve(await realpath(existing), ...missing);
340 + } catch (error) {
341 + if (
342 + !(error instanceof Error) ||
343 + !("code" in error) ||
344 + error.code !== "ENOENT"
346 + throw error;
348 + const parent = dirname(existing);
349 + if (parent === existing) throw new Error("path has no existing ancestor");
350 + missing.unshift(basename(existing));
351 + existing = parent;
355 +function runEntry(
356 + run: {
357 + manifest: {
358 + runId: string;
359 + kind: "suite" | "aftermath";
360 + stage: string;
361 + terminal: RunEntry["terminal"];
362 + source: { path: string };
363 + };
364 + },
365 + progress: Pick<
366 + RunEntry,
367 + "completed" | "total" | "currentScore" | "bestScore"
368 + > = {},
369 + artifacts: Pick<RunEntry, "reportPath" | "stagedPath"> = {
370 + reportPath: null,
371 + stagedPath: null,
372 + },
373 +): RunEntry {
374 + return {
375 + version: 1,
376 + runId: run.manifest.runId,
377 + kind: run.manifest.kind,
378 + stage: run.manifest.stage,
379 + terminal: run.manifest.terminal,
380 + sourcePath: run.manifest.source.path,
381 + ...artifacts,
382 + ...progress,
383 + };
386 +async function modelHasAuth(
387 + ctx: ExtensionCommandContext,
388 + provider = ctx.model?.provider,
389 + id = ctx.model?.id,
390 +): Promise<boolean> {
391 + if (!provider || !id) return false;
392 + const available = await ctx.modelRegistry.getAvailable();
393 + return available.some(
394 + (model) => model.provider === provider && model.id === id,
395 + );
398 +function selectedRun(ctx: ExtensionCommandContext) {
399 + return selectNewestSuiteRun(ctx.sessionManager.getBranch());
402 +function selectedAftermathRun(ctx: ExtensionCommandContext) {
403 + return selectNewestAftermathRun(ctx.sessionManager.getBranch());
406 +export async function unfinishedRunWarning(
407 + sourcePath: string | undefined,
408 + suitePath?: string,
409 + agentDir = getAgentDir(),
410 +): Promise<string | undefined> {
411 + const root = join(agentDir, "overkill-runs");
412 + const entries = await readdir(root, { withFileTypes: true }).catch(() => []);
413 + const runIds: string[] = [];
414 + for (const entry of entries) {
415 + if (!entry.isDirectory()) continue;
416 + const manifest = await readManifest(join(root, entry.name)).catch(
417 + () => undefined,
418 + );
419 + if (
420 + manifest?.terminal === null &&
421 + ((sourcePath !== undefined && manifest.source.path === sourcePath) ||
422 + (suitePath !== undefined && manifest.suite?.path === suitePath))
424 + runIds.push(manifest.runId);
426 + if (runIds.length === 0) return undefined;
427 + return `${runIds.length} unfinished Overkill run${runIds.length === 1 ? "" : "s"} already target this skill or suite (${runIds.slice(0, 3).join(", ")}${runIds.length > 3 ? ", …" : ""}). They may be active in other Pi processes; parallel runs duplicate model cost, and suite edits can invalidate running evaluations.`;
430 +function statusMessage(run: RunEntry): string {
431 + if (
432 + run.kind === "suite" &&
433 + run.stage === "complete" &&
434 + run.terminal === "no_improvement" &&
435 + run.bestScore === 1
437 + return `Overkill: complete | completed: ${run.completed ?? 0}${run.total === undefined ? "" : `/${run.total}`} | outcome: baseline perfect; optimization skipped`;
438 + const fields = [`Overkill: ${run.stage}`];
439 + if (run.completed !== undefined)
440 + fields.push(
441 + `completed: ${run.completed}${run.total === undefined ? "" : `/${run.total}`}`,
442 + );
443 + if (run.currentScore !== undefined)
444 + fields.push(`current score: ${run.currentScore}`);
445 + if (run.bestScore !== undefined) fields.push(`best score: ${run.bestScore}`);
446 + fields.push(`terminal: ${run.terminal ?? "active"}`);
447 + if (run.stagedPath) fields.push(`staged: ${run.stagedPath}`);
448 + return fields.join(" | ");
451 +function commandMode(
452 + ctx: ExtensionCommandContext,
453 +): "tui" | "rpc" | "json" | "print" {
454 + return ctx.mode ?? "tui";
457 +function writePrint(line: string): void {
458 + const text = `${line}\n`;
459 + const seam = installedModeAcceptanceSeam();
460 + if (seam) seam.write(1, text);
461 + else writeSync(1, text);
464 +function appendProgressEntry(pi: ExtensionAPI, entry: RunEntry): void {
465 + pi.appendEntry("overkill-run", entry);
468 +function showProgress(ctx: ExtensionCommandContext, text: string): void {
469 + const mode = commandMode(ctx);
470 + if (mode === "rpc") ctx.ui.setStatus("overkill", text);
471 + if (mode === "tui")
472 + ctx.ui.setWidget("overkill-progress", [text], {
473 + placement: "belowEditor",
474 + });
477 +function appendProgress(
478 + pi: ExtensionAPI,
479 + ctx: ExtensionCommandContext,
480 + entry: RunEntry,
481 +): void {
482 + const mode = commandMode(ctx);
483 + const durable =
484 + entry.terminal !== null ||
485 + entry.stage === "preflight" ||
486 + entry.stage === "interrupted" ||
487 + (entry.kind === "aftermath" &&
488 + entry.stage === "mining" &&
489 + entry.completed === undefined);
490 + if (mode !== "tui" || durable) appendProgressEntry(pi, entry);
491 + const progress = formatProgress(entry);
492 + showProgress(ctx, progress);
493 + if (mode === "print") writePrint(progress);
496 +function clearProgress(ctx: ExtensionCommandContext): void {
497 + if (commandMode(ctx) === "rpc") ctx.ui.setStatus("overkill", undefined);
498 + if (commandMode(ctx) === "tui")
499 + ctx.ui.setWidget("overkill-progress", undefined);
502 +function requestCancellation(active: ActiveRun): void {
503 + if (active.cancelRequested) return;
504 + active.cancelRequested = true;
505 + active.controller.abort();
508 +function installCancellationListeners(
509 + active: ActiveRun,
510 + ctx: ExtensionCommandContext,
511 +): void {
512 + const mode = commandMode(ctx);
513 + const cleanup: Array<() => void> = [];
514 + if (mode === "tui")
515 + cleanup.push(
516 + ctx.ui.onTerminalInput((data) => {
517 + if (!matchesKey(data, Key.escape)) return undefined;
518 + requestCancellation(active);
519 + return { consume: true };
520 + }),
521 + );
522 + if (mode === "json" || mode === "print") {
523 + const onSignal = () => requestCancellation(active);
524 + process.once("SIGINT", onSignal);
525 + cleanup.push(() => process.removeListener("SIGINT", onSignal));
527 + active.removeCancellationListeners = () => {
528 + for (const remove of cleanup.splice(0)) remove();
529 + };
532 +async function withActiveRun(
533 + ctx: ExtensionCommandContext,
534 + operation: (active: ActiveRun) => Promise<void>,
535 + kind: ActiveRun["kind"] = "suite",
536 +): Promise<void> {
537 + if (activeRun) throw new Error("an Overkill rollout is already active");
538 + const active: ActiveRun = {
539 + kind,
540 + runId: "pending",
541 + sessionId: ctx.sessionManager.getSessionId(),
542 + controller: new AbortController(),
543 + suppressOutput: false,
544 + };
545 + activeRun = active;
546 + installCancellationListeners(active, ctx);
547 + const completed = (async () => {
548 + try {
549 + await operation(active);
550 + } catch (error) {
551 + if (active.controller.signal.aborted) {
552 + await active.interrupt?.();
553 + throw new RolloutCancelledError();
555 + try {
556 + await active.interrupt?.();
557 + } catch {
558 + // Preserve the original infrastructure failure.
560 + throw error;
561 + } finally {
562 + active.removeCancellationListeners?.();
563 + clearProgress(ctx);
564 + if (activeRun === active) activeRun = undefined;
566 + })();
567 + active.completed = completed;
568 + return await completed;
571 +async function checkpointInterrupted(
572 + directory: string,
573 + manifest: RunManifest,
574 +): Promise<void> {
575 + const training = await readTrainingCheckpoint(directory);
576 + if (training)
577 + await persistTrainingCheckpoint(directory, {
578 + ...training,
579 + stage: "interrupted",
580 + });
581 + manifest.stage = "interrupted";
582 + manifest.terminal = null;
583 + await persistManifest(directory, manifest);
586 +function throwIfCancelled(active: ActiveRun): void {
587 + if (active.controller.signal.aborted) throw new RolloutCancelledError();
590 +type CoreTaskAdapter = {
591 + items: (split: "train" | "validation" | "test") => readonly ValidatedItem[];
592 + scoreResponse: (options: {
593 + item: ValidatedItem;
594 + response: string;
595 + input: string;
596 + signal?: AbortSignal;
597 + }) => Promise<{ score: number; passed: boolean; feedback?: string }>;
598 + onTargetCall?: () => void | Promise<void>;
601 +async function executeBaseline(options: {
602 + pi: ExtensionAPI;
603 + ctx: ExtensionCommandContext;
604 + directory: string;
605 + manifest: RunManifest;
606 + source: ReturnType<typeof parseSourceSkill>;
607 + loadedSkill: Awaited<ReturnType<typeof resolveLoadedSkill>>;
608 + suite?: Awaited<ReturnType<typeof validateTrustedSuite>>;
609 + targetModel: unknown;
610 + taskAdapter?: CoreTaskAdapter;
611 + reportExtras?: () => Record<string, unknown>;
612 + deferCompletionNotice?: boolean;
613 +}): Promise<void> {
614 + const { ctx, manifest } = options;
615 + const itemsFor = (split: "train" | "validation" | "test") =>
616 + options.taskAdapter
617 + ? options.taskAdapter.items(split)
618 + : getValidatedSuiteItems(
619 + options.suite ??
620 + (() => {
621 + throw new Error("suite tasks are unavailable");
622 + })(),
623 + split === "validation" ? "val" : split,
624 + );
625 + const scorer = options.taskAdapter
626 + ? { scoreResponse: options.taskAdapter.scoreResponse }
627 + : { executeChecker };
628 + const active = activeRun;
629 + if (!active || active.sessionId !== ctx.sessionManager.getSessionId())
630 + throw new Error("Overkill run ownership is unavailable");
631 + active.runId = manifest.runId;
632 + active.interrupt = async () =>
633 + await checkpointInterrupted(options.directory, manifest);
634 + throwIfCancelled(active);
635 + const controller = active.controller;
636 + const acceptanceSeam =
637 + manifest.kind === "aftermath"
638 + ? installedAftermathAcceptanceSeam()
639 + : installedExplicitSuiteAcceptanceSeam();
640 + const recordTargetAttempt = async () => {
641 + if (options.taskAdapter?.onTargetCall) {
642 + await options.taskAdapter.onTargetCall();
643 + return;
645 + if (!options.suite || !manifest.suite)
646 + throw new Error("suite call contract is unavailable");
647 + const maximum = upperBoundCalls(options.suite.splitCounts).target;
648 + if (manifest.calls.target >= maximum)
649 + throw new Error("target call maximum exceeded");
650 + if (
651 + fingerprintSource(await readFile(manifest.source.path)) !==
652 + manifest.source.fingerprint
654 + throw new Error("source fingerprint changed");
655 + if (
656 + (await fingerprintSuite(options.suite.suitePath)) !==
657 + manifest.suite.fingerprint
659 + throw new Error("suite fingerprint changed");
660 + manifest.calls.target += 1;
661 + await persistManifest(options.directory, manifest);
662 + };
663 + let progress: Pick<
664 + RunEntry,
665 + "completed" | "total" | "currentScore" | "bestScore"
666 + > = {};
667 + const operation = (async () => {
668 + manifest.stage = "baseline";
669 + await persistManifest(options.directory, manifest);
670 + try {
671 + const result = await runValidationBaseline(
673 + directory: options.directory,
674 + cwd: ctx.cwd,
675 + agentDir: getAgentDir(),
676 + sourcePath: manifest.source.path,
677 + sourceFingerprint: manifest.source.fingerprint,
678 + source: options.source,
679 + loadedSkill: options.loadedSkill,
680 + model: manifest.model,
681 + suite: options.suite,
682 + items: itemsFor("validation"),
683 + signal: controller.signal,
684 + onTargetCall: recordTargetAttempt,
685 + onCheckerCall: async () => {
686 + if (options.taskAdapter) return;
687 + manifest.calls.checker += 1;
688 + await persistManifest(options.directory, manifest);
689 + },
690 + onProgress: async (next) => {
691 + progress = {
692 + completed: next.completed,
693 + total: next.total,
694 + ...(next.mean === undefined
695 + ? {}
696 + : { currentScore: next.mean, bestScore: next.mean }),
697 + };
698 + if (!active.suppressOutput)
699 + appendProgress(options.pi, ctx, runEntry({ manifest }, progress));
700 + },
701 + },
703 + createTarget: () =>
704 + createIsolatedTargetSession(
706 + cwd: ctx.cwd,
707 + agentDir: getAgentDir(),
708 + sourcePath: manifest.source.path,
709 + loadedSkill: options.loadedSkill,
710 + candidatePath: join(options.directory, "baseline-snapshot.md"),
711 + model: manifest.model,
712 + targetModel: options.targetModel,
713 + },
714 + acceptanceSeam?.targetSdk ?? defaultTargetSdk,
715 + ),
716 + ...scorer,
717 + },
718 + );
719 + manifest.stage = "baseline_complete";
720 + manifest.scores = {
721 + ...manifest.scores,
722 + baseline: result.mean,
723 + current: result.mean,
724 + best: result.mean,
725 + };
726 + await persistManifest(options.directory, manifest);
727 + if (controller.signal.aborted) throw new RolloutCancelledError();
728 + if (!active.suppressOutput)
729 + appendProgress(options.pi, ctx, runEntry({ manifest }, progress));
730 + if (result.mean === 1) {
731 + const reportPath = await persistFinalReport(options.directory, {
732 + version: 1,
733 + baselineScore: result.mean,
734 + bestScore: result.mean,
735 + accepted: 0,
736 + optimizerCalls: 0,
737 + targetCalls: manifest.calls.target,
738 + checkerCalls: manifest.calls.checker,
739 + terminal: "perfect",
740 + ...options.reportExtras?.(),
741 + });
742 + if (controller.signal.aborted) throw new RolloutCancelledError();
743 + manifest.stage = "complete";
744 + manifest.terminal = "no_improvement";
745 + manifest.artifacts.report = "report.json";
746 + await persistManifest(options.directory, manifest);
747 + if (!active.suppressOutput) {
748 + const entry = runEntry({ manifest }, progress, {
749 + reportPath,
750 + stagedPath: null,
751 + });
752 + appendProgress(options.pi, ctx, entry);
753 + const paths = formatResultPaths({
754 + sourcePath: manifest.source.path,
755 + reportPath,
756 + stagedPath: null,
757 + });
758 + if (commandMode(ctx) === "print")
759 + writePrint(
760 + `Baseline already scores 1; no strictly better candidate exists, so optimization was skipped.\n${paths}`,
761 + );
762 + else if (
763 + commandMode(ctx) !== "json" &&
764 + !options.deferCompletionNotice
766 + ctx.ui.notify(
767 + "Baseline already scores 1; no strictly better candidate exists, so optimization was skipped.",
768 + "info",
769 + );
771 + return;
773 + const baselineCache = await readScoreCache(options.directory);
774 + if (!baselineCache)
775 + throw new Error("validation baseline cache is unavailable");
776 + const sourceHash = createHash("sha256")
777 + .update(options.source.body, "utf8")
778 + .digest("hex");
779 + const saved = await readTrainingCheckpoint(options.directory);
780 + const readSnapshotBody = async (hash: string, filename: string) => {
781 + if (hash === sourceHash) return options.source.body;
782 + const snapshot = await readFile(
783 + join(options.directory, filename),
784 + "utf8",
785 + );
786 + if (!snapshot.startsWith(options.source.frontmatter))
787 + throw new Error("candidate snapshot frontmatter is unsafe");
788 + const body = snapshot.slice(options.source.frontmatter.length);
789 + if (createHash("sha256").update(body, "utf8").digest("hex") !== hash)
790 + throw new Error("candidate snapshot identity is unsafe");
791 + return body;
792 + };
793 + const readAttempt = async (filename: string): Promise<AttemptedPatch> => {
794 + const value: unknown = JSON.parse(
795 + await readFile(join(options.directory, filename), "utf8"),
796 + );
797 + if (typeof value !== "object" || value === null || Array.isArray(value))
798 + throw new Error("attempt artifact is malformed");
799 + const artifact = value as Record<string, unknown>;
800 + if (!Array.isArray(artifact.reports))
801 + throw new Error("attempt artifact reports are malformed");
802 + return {
803 + patch: parsePatch(artifact.patch),
804 + reports: artifact.reports as AttemptedPatch["reports"],
805 + ...(typeof artifact.validationScore === "number"
806 + ? { validationScore: artifact.validationScore }
807 + : {}),
808 + };
809 + };
810 + let latestTraining: DurableTrainingState;
811 + if (saved) {
812 + const attempts = await Promise.all(
813 + saved.attemptedPatchFiles.map(readAttempt),
814 + );
815 + const pendingArtifact = saved.pendingPatch
816 + ? await readAttempt(saved.pendingPatch.file)
817 + : undefined;
818 + latestTraining = {
819 + nextStep: saved.nextStep,
820 + body: await readSnapshotBody(
821 + saved.currentBodyHash,
822 + saved.currentSnapshot,
823 + ),
824 + bestBody: await readSnapshotBody(
825 + saved.bestBodyHash,
826 + saved.bestSnapshot,
827 + ),
828 + baselineScore: saved.baselineScore,
829 + currentScore: saved.currentScore,
830 + bestScore: saved.bestScore,
831 + accepted: saved.accepted,
832 + optimizerCalls: saved.optimizerCalls,
833 + consecutiveRejections: saved.consecutiveRejections,
834 + rejections: saved.rejections.map(({ edits, ...rejection }) => ({
835 + ...rejection,
836 + ...(edits === undefined
837 + ? {}
838 + : { edits: parsePatch({ edits }).edits }),
839 + })),
840 + attempts,
841 + ...(pendingArtifact && saved.pendingPatch
842 + ? {
843 + pendingPatch: {
844 + step: saved.pendingPatch.step,
845 + body: await readSnapshotBody(
846 + saved.pendingPatch.bodyHash,
847 + saved.currentSnapshot,
848 + ),
849 + patch: pendingArtifact.patch,
850 + },
852 + : {}),
853 + ...(saved.terminal === undefined ? {} : { terminal: saved.terminal }),
854 + ...(saved.finalTest === undefined
855 + ? {}
856 + : {
857 + finalTest: {
858 + body: await readSnapshotBody(
859 + saved.finalTest.bodyHash,
860 + saved.bestSnapshot,
861 + ),
862 + completedItemIds: saved.finalTest.completedItemIds,
863 + total: saved.finalTest.total,
864 + ...(saved.finalTest.score === undefined
865 + ? {}
866 + : { score: saved.finalTest.score }),
867 + },
868 + ...(saved.finalTest.score === undefined
869 + ? {}
870 + : { finalTestScore: saved.finalTest.score }),
871 + }),
872 + ...(saved.pendingOptimizer === undefined
873 + ? {}
874 + : { pendingOptimizer: saved.pendingOptimizer }),
875 + };
876 + } else
877 + latestTraining = createTrainingState({
878 + body: options.source.body,
879 + baseline: baselineCache.scores,
880 + });
881 + const checkpointTraining = async (state: DurableTrainingState) => {
882 + latestTraining = state;
883 + if (state.pendingPatch)
884 + await writePrivateText(
885 + options.directory,
886 + `attempt-${state.pendingPatch.step}.json`,
887 + `${JSON.stringify({ patch: state.pendingPatch.patch, reports: [] })}\n`,
888 + );
889 + for (const [index, attempt] of state.attempts.entries())
890 + await writePrivateText(
891 + options.directory,
892 + `attempt-${index + 1}.json`,
893 + `${JSON.stringify(attempt)}\n`,
894 + );
895 + const snapshotName = (body: string) => {
896 + const hash = createHash("sha256").update(body, "utf8").digest("hex");
897 + return hash === sourceHash
898 + ? "baseline-snapshot.md"
899 + : `candidate-${hash}.md`;
900 + };
901 + const stage =
902 + state.terminal &&
903 + state.accepted > 0 &&
904 + state.finalTestScore === undefined
905 + ? "final_test"
906 + : "training";
907 + await persistTrainingCheckpoint(options.directory, {
908 + version: 1,
909 + stage,
910 + nextStep: state.nextStep,
911 + currentBodyHash: createHash("sha256")
912 + .update(state.body, "utf8")
913 + .digest("hex"),
914 + bestBodyHash: createHash("sha256")
915 + .update(state.bestBody, "utf8")
916 + .digest("hex"),
917 + currentSnapshot: snapshotName(state.body),
918 + bestSnapshot: snapshotName(state.bestBody),
919 + ...(state.terminal === undefined ? {} : { terminal: state.terminal }),
920 + baselineScore: state.baselineScore,
921 + currentScore: state.currentScore,
922 + bestScore: state.bestScore,
923 + accepted: state.accepted,
924 + optimizerCalls: state.optimizerCalls,
925 + consecutiveRejections: state.consecutiveRejections,
926 + rejections: state.rejections.map(
927 + ({ reason, delta, reports, edits }) => ({
928 + reason,
929 + delta,
930 + ...(reports === undefined ? {} : { reports }),
931 + ...(edits === undefined ? {} : { edits }),
932 + }),
933 + ),
934 + attemptedPatchFiles: state.attempts.map(
935 + (_, index) => `attempt-${index + 1}.json`,
936 + ),
937 + ...(state.pendingPatch
938 + ? {
939 + pendingPatch: {
940 + step: state.pendingPatch.step,
941 + bodyHash: createHash("sha256")
942 + .update(state.pendingPatch.body, "utf8")
943 + .digest("hex"),
944 + file: `attempt-${state.pendingPatch.step}.json`,
945 + },
947 + : {}),
948 + ...(state.pendingOptimizer === undefined
949 + ? {}
950 + : { pendingOptimizer: state.pendingOptimizer }),
951 + ...(state.finalTest
952 + ? {
953 + finalTest: {
954 + bodyHash: createHash("sha256")
955 + .update(state.finalTest.body, "utf8")
956 + .digest("hex"),
957 + completedItemIds: state.finalTest.completedItemIds,
958 + total: state.finalTest.total,
959 + ...(state.finalTest.score === undefined
960 + ? {}
961 + : { score: state.finalTest.score }),
962 + },
964 + : {}),
965 + });
966 + manifest.stage = stage;
967 + manifest.scores = {
968 + ...manifest.scores,
969 + current: state.currentScore,
970 + best: state.bestScore,
971 + ...(state.finalTestScore === undefined
972 + ? {}
973 + : { test: state.finalTestScore }),
974 + };
975 + manifest.calls.optimizer = state.optimizerCalls;
976 + await persistManifest(options.directory, manifest);
977 + if (!active.suppressOutput)
978 + appendProgress(
979 + options.pi,
980 + ctx,
981 + runEntry(
982 + { manifest },
984 + ...(state.finalTest
985 + ? {
986 + completed: state.finalTest.completedItemIds.length,
987 + total: state.finalTest.total,
989 + : {}),
990 + currentScore: state.currentScore,
991 + bestScore: state.bestScore,
992 + },
993 + ),
994 + );
995 + };
996 + const training = await runDurableTrainingSchedule(latestTraining, {
997 + evaluate: async (body, split, onProgress) =>
998 + await splitScores(split, body, onProgress),
999 + optimize: async (input, onCall, onProgress) =>
1000 + await runOptimizerStep(
1001 + input,
1003 + createSession: () =>
1004 + createOptimizerSession(
1006 + cwd: ctx.cwd,
1007 + agentDir: getAgentDir(),
1008 + model: manifest.model,
1009 + targetModel: options.targetModel,
1010 + },
1011 + acceptanceSeam?.optimizerSdk ?? defaultOptimizerSdk,
1012 + ),
1013 + },
1014 + controller.signal,
1015 + latestTraining.pendingOptimizer,
1016 + onCall,
1017 + onProgress,
1018 + ),
1019 + checkpoint: checkpointTraining,
1020 + testItemIds: async () => itemsFor("test").map((item) => item.id),
1021 + signal: controller.signal,
1022 + });
1023 + if (controller.signal.aborted) throw new RolloutCancelledError();
1024 + const reportPath = await persistFinalReport(options.directory, {
1025 + version: 1,
1026 + baselineScore: training.baselineScore,
1027 + bestScore: training.bestScore,
1028 + accepted: training.accepted,
1029 + optimizerCalls: training.optimizerCalls,
1030 + targetCalls: manifest.calls.target,
1031 + checkerCalls: manifest.calls.checker,
1032 + terminal: training.terminal,
1033 + rejections: training.rejections,
1034 + attempts: training.attempts,
1035 + ...(training.testScore === undefined
1036 + ? {}
1037 + : { testScore: training.testScore }),
1038 + ...options.reportExtras?.(),
1039 + });
1040 + const stagedPath =
1041 + training.accepted > 0
1042 + ? await writePrivateText(
1043 + options.directory,
1044 + "staged-skill.md",
1045 + `${options.source.frontmatter}${training.bestBody}`,
1047 + : null;
1048 + await checkpointTraining(latestTraining);
1049 + if (controller.signal.aborted) throw new RolloutCancelledError();
1050 + const completeCheckpoint = await readTrainingCheckpoint(
1051 + options.directory,
1052 + );
1053 + if (!completeCheckpoint)
1054 + throw new Error("training checkpoint is unavailable");
1055 + await persistTrainingCheckpoint(options.directory, {
1056 + ...completeCheckpoint,
1057 + stage: "complete",
1058 + reportFile: "report.json",
1059 + ...(stagedPath === null ? {} : { stagedFile: "staged-skill.md" }),
1060 + });
1061 + manifest.stage = "complete";
1062 + manifest.terminal = training.accepted > 0 ? "staged" : "no_improvement";
1063 + manifest.scores = {
1064 + ...manifest.scores,
1065 + best: training.bestScore,
1066 + ...(training.testScore === undefined
1067 + ? {}
1068 + : { test: training.testScore }),
1069 + };
1070 + manifest.calls.optimizer = training.optimizerCalls;
1071 + manifest.artifacts.report = "report.json";
1072 + manifest.artifacts.staged =
1073 + stagedPath === null ? null : "staged-skill.md";
1074 + await persistManifest(options.directory, manifest);
1075 + if (controller.signal.aborted) throw new RolloutCancelledError();
1076 + if (!active.suppressOutput) {
1077 + const entry = runEntry(
1078 + { manifest },
1079 + { currentScore: training.bestScore, bestScore: training.bestScore },
1080 + { reportPath, stagedPath },
1081 + );
1082 + appendProgress(options.pi, ctx, entry);
1083 + const paths = formatResultPaths({
1084 + sourcePath: manifest.source.path,
1085 + reportPath,
1086 + stagedPath,
1087 + });
1088 + if (commandMode(ctx) === "print") writePrint(paths);
1089 + else if (commandMode(ctx) !== "json" && !options.deferCompletionNotice)
1090 + ctx.ui.notify(paths, "info");
1093 + async function splitScores(
1094 + split: "train" | "validation" | "test",
1095 + body: string,
1096 + onProgress?: (progress: {
1097 + completed: number;
1098 + total: number;
1099 + }) => Promise<void>,
1100 + ) {
1101 + const hash = createHash("sha256").update(body, "utf8").digest("hex");
1102 + const rollout = await runSplitRollout(
1104 + directory: options.directory,
1105 + cwd: ctx.cwd,
1106 + agentDir: getAgentDir(),
1107 + sourcePath: manifest.source.path,
1108 + sourceFingerprint: manifest.source.fingerprint,
1109 + source: options.source,
1110 + loadedSkill: options.loadedSkill,
1111 + model: manifest.model,
1112 + suite: options.suite,
1113 + items: itemsFor(split),
1114 + body,
1115 + split,
1116 + signal: controller.signal,
1117 + onProgress,
1118 + onTargetCall: recordTargetAttempt,
1119 + onCheckerCall: async () => {
1120 + if (options.taskAdapter) return;
1121 + manifest.calls.checker += 1;
1122 + await persistManifest(options.directory, manifest);
1123 + },
1124 + },
1126 + createTarget: () =>
1127 + createIsolatedTargetSession(
1129 + cwd: ctx.cwd,
1130 + agentDir: getAgentDir(),
1131 + sourcePath: manifest.source.path,
1132 + loadedSkill: options.loadedSkill,
1133 + candidatePath: join(
1134 + options.directory,
1135 + `candidate-${hash}.md`,
1136 + ),
1137 + model: manifest.model,
1138 + targetModel: options.targetModel,
1139 + },
1140 + acceptanceSeam?.targetSdk ?? defaultTargetSdk,
1141 + ),
1142 + ...scorer,
1143 + },
1144 + );
1145 + return rollout.scores;
1147 + } catch (error) {
1148 + if (
1149 + error instanceof RolloutCancelledError ||
1150 + error instanceof OptimizerCancelledError ||
1151 + controller.signal.aborted
1152 + ) {
1153 + await checkpointInterrupted(options.directory, manifest);
1154 + const entry = runEntry({ manifest }, progress);
1155 + if (active.suppressOutput) appendProgressEntry(options.pi, entry);
1156 + else appendProgress(options.pi, ctx, entry);
1157 + throw new RolloutCancelledError();
1159 + throw error;
1161 + })();
1162 + return await operation;
1165 +export async function harvestAftermath(options: {
1166 + ctx: ExtensionCommandContext;
1167 + knownSkillPaths: ReadonlyMap<string, string>;
1168 + sessionApi?: AftermathSessionApi;
1169 +}): Promise<{ digests: Digest[]; total: number; unreadable: number }> {
1170 + const api =
1171 + options.sessionApi ??
1172 + (SessionManager as unknown as NonNullable<typeof options.sessionApi>);
1173 + const sessionDir = options.ctx.sessionManager.getSessionDir();
1174 + const listed = await api.listAll(sessionDir);
1175 + const snapshots = new Map<string, Digest>();
1176 + const unreadableIds = new Set<string>();
1177 + for (const listedSession of listed) {
1178 + try {
1179 + const session = api.open(listedSession.path, sessionDir);
1180 + const id = session.getSessionId();
1181 + snapshots.set(
1182 + id,
1183 + buildDigest({
1184 + sessionId: id,
1185 + cwd: session.getCwd(),
1186 + branch: session.getBranch(),
1187 + knownSkillPaths: options.knownSkillPaths,
1188 + }),
1189 + );
1190 + } catch {
1191 + unreadableIds.add(listedSession.id);
1194 + const current = options.ctx.sessionManager;
1195 + const currentId = current.getSessionId();
1196 + snapshots.set(
1197 + currentId,
1198 + buildDigest({
1199 + sessionId: currentId,
1200 + cwd: options.ctx.cwd,
1201 + branch: current.getBranch(),
1202 + knownSkillPaths: options.knownSkillPaths,
1203 + }),
1204 + );
1205 + unreadableIds.delete(currentId);
1206 + const currentWasListed = listed.some((session) => session.id === currentId);
1207 + return {
1208 + digests: [...snapshots.values()].sort((left, right) =>
1209 + left.sessionId.localeCompare(right.sessionId),
1210 + ),
1211 + total: listed.length + (currentWasListed ? 0 : 1),
1212 + unreadable: unreadableIds.size,
1213 + };
1216 +type LoadedSkillTarget = { name: string; path: string };
1218 +function loadedSkills(pi: ExtensionAPI): LoadedSkillTarget[] {
1219 + return pi
1220 + .getCommands()
1221 + .flatMap((command) => {
1222 + if (
1223 + command.source !== "skill" ||
1224 + typeof command.sourceInfo?.path !== "string"
1226 + return [];
1227 + return [
1229 + name: command.name.replace(/^skill:/, ""),
1230 + path: command.sourceInfo.path,
1231 + },
1232 + ];
1233 + })
1234 + .sort(
1235 + (left, right) =>
1236 + left.name.localeCompare(right.name) ||
1237 + left.path.localeCompare(right.path),
1238 + );
1241 +export async function chooseLoadedSkill(
1242 + ctx: ExtensionCommandContext,
1243 + skills: readonly LoadedSkillTarget[],
1244 + canonicalPaths: ReadonlyMap<string, string>,
1245 + purpose: string,
1246 +): Promise<LoadedSkillTarget | undefined> {
1247 + if (commandMode(ctx) !== "tui" || skills.length <= 8) {
1248 + const labels = skills.map(
1249 + (skill) =>
1250 + `${skill.name} — ${canonicalPaths.get(skill.path) ?? skill.path}`,
1251 + );
1252 + const choice = await ctx.ui.select(
1253 + `Choose exactly one ${purpose} skill.`,
1254 + labels,
1255 + );
1256 + return skills[labels.indexOf(choice ?? "")];
1259 + const path = await ctx.ui.custom<string | null>(
1260 + (tui, theme, _keybindings, done) => {
1261 + const list = new SelectList(
1262 + skills.map((skill) => {
1263 + const canonical = canonicalPaths.get(skill.path) ?? skill.path;
1264 + const parent = dirname(canonical);
1265 + return {
1266 + value: skill.path,
1267 + label: skill.name,
1268 + description: join(
1269 + "…",
1270 + basename(dirname(parent)),
1271 + basename(parent),
1272 + basename(canonical),
1273 + ),
1274 + };
1275 + }),
1276 + Math.max(1, Math.min(8, tui.terminal.rows - 3)),
1278 + selectedPrefix: (text) => theme.fg("accent", text),
1279 + selectedText: (text) => theme.fg("accent", text),
1280 + description: (text) => theme.fg("muted", text),
1281 + scrollInfo: (text) => theme.fg("dim", text),
1282 + noMatch: (text) => theme.fg("warning", text),
1283 + },
1284 + );
1285 + list.onSelect = (item) => done(item.value);
1286 + list.onCancel = () => done(null);
1287 + return {
1288 + render: (width: number) => [
1289 + theme.fg("accent", `Choose one ${purpose} skill`),
1290 + ...list.render(width),
1291 + theme.fg("dim", "type filter · arrows move · enter · esc"),
1292 + ],
1293 + invalidate: () => list.invalidate(),
1294 + handleInput: (data: string) => {
1295 + list.handleInput(data);
1296 + tui.requestRender();
1297 + },
1298 + };
1299 + },
1300 + );
1301 + return skills.find((skill) => skill.path === path);
1304 +async function runAftermath(
1305 + pi: ExtensionAPI,
1306 + ctx: ExtensionCommandContext,
1307 + parts: string[],
1308 +): Promise<void> {
1309 + const active = activeRun;
1310 + if (active?.kind !== "aftermath")
1311 + throw new Error("Aftermath run ownership is unavailable");
1312 + throwIfCancelled(active);
1313 + const explicitPath = parts[0];
1314 + if (parts.length > 1)
1315 + throw new Error("Usage: /overkill aftermath [skill-path]");
1316 + const interrupted = selectedAftermathRun(ctx);
1317 + if (interrupted) {
1318 + const directory = join(
1319 + getAgentDir(),
1320 + "overkill-runs",
1321 + interrupted.data.runId,
1322 + );
1323 + const manifest = await readAftermathManifest(directory).catch(
1324 + () => undefined,
1325 + );
1326 + if (manifest && !manifest.terminal) {
1327 + const instructions =
1328 + "An incomplete Aftermath run is linked here; use /overkill aftermath resume, discard, or cancel.";
1329 + if (explicitPath) throw new Error(instructions);
1330 + if (ctx.hasUI) {
1331 + const choice = await ctx.ui.select("Resume incomplete Aftermath run?", [
1332 + "resume",
1333 + "discard",
1334 + "cancel",
1335 + ]);
1336 + if (choice === "resume") {
1337 + await resumeAftermath(pi, ctx, interrupted);
1338 + return;
1340 + if (choice === "discard") {
1341 + await discardLinkedRun({
1342 + runRoot: join(getAgentDir(), "overkill-runs"),
1343 + runDirectory: directory,
1344 + selected: interrupted,
1345 + branch: ctx.sessionManager.getBranch(),
1346 + sessionId: ctx.sessionManager.getSessionId(),
1347 + allowNonterminal: true,
1348 + isRunActive: (runId) => activeRun?.runId === runId,
1349 + });
1350 + appendProgress(pi, ctx, {
1351 + ...interrupted.data,
1352 + stage: "discarded",
1353 + terminal: "discarded",
1354 + });
1356 + return;
1358 + throw new Error(instructions);
1361 + if (
1362 + (commandMode(ctx) === "json" || commandMode(ctx) === "print") &&
1363 + !explicitPath
1365 + throw new Error(
1366 + "JSON and print Aftermath runs require an explicit skill path",
1367 + );
1368 + const skills = loadedSkills(pi);
1369 + if (skills.length === 0) {
1370 + ctx.ui.notify("No loaded Pi skills are available for Aftermath.", "info");
1371 + return;
1373 + const knownPaths = new Map<string, string>();
1374 + const canonicalSkillPaths = new Map<string, string>();
1375 + for (const skill of skills) {
1376 + try {
1377 + const canonical = await realpath(skill.path);
1378 + knownPaths.set(canonical, skill.name);
1379 + canonicalSkillPaths.set(skill.path, canonical);
1380 + } catch {}
1382 + const canonicalExplicit = explicitPath
1383 + ? await realpath(resolve(ctx.cwd, explicitPath)).catch(() => undefined)
1384 + : undefined;
1385 + let selected = explicitPath
1386 + ? skills.find(
1387 + (skill) =>
1388 + canonicalExplicit !== undefined &&
1389 + canonicalSkillPaths.get(skill.path) === canonicalExplicit,
1391 + : undefined;
1392 + if (explicitPath && !selected)
1393 + throw new Error("explicit Aftermath skill path is not a loaded skill");
1394 + if (!selected && ctx.hasUI)
1395 + selected = await chooseLoadedSkill(
1396 + ctx,
1397 + skills,
1398 + canonicalSkillPaths,
1399 + "Aftermath",
1400 + );
1401 + if (!selected) throw new Error("one confirmed Aftermath skill is required");
1402 + throwIfCancelled(active);
1403 + const sourcePath =
1404 + canonicalSkillPaths.get(selected.path) ?? (await realpath(selected.path));
1405 + const parallelWarning = await unfinishedRunWarning(sourcePath);
1406 + const harvested = await harvestAftermath({
1407 + ctx,
1408 + knownSkillPaths: knownPaths,
1409 + sessionApi: installedAftermathAcceptanceSeam()?.sessionApi,
1410 + });
1411 + throwIfCancelled(active);
1412 + const evidence = rankSkillEvidence(
1413 + harvested.digests,
1414 + new Set(skills.map((skill) => skill.name)),
1415 + );
1416 + const calls = estimateMiningCalls(harvested.digests.length);
1417 + const confirmation = [
1418 + `Readable sessions: ${harvested.digests.length}; unreadable: ${harvested.unreadable}.`,
1419 + "Each digest is capped at 16 KiB, persisted, and sent to the model without redaction.",
1420 + `Expected mining/aggregation calls: ${calls.mining}/${calls.aggregation}; malformed structured output retries once per call.`,
1421 + `Maximum mining/aggregation calls with retries: ${calls.mining * 2}/${calls.aggregation * 2}.`,
1422 + "Maximum replay calls after mining: 116 target, 116 rubric-judge, 12 optimizer.",
1423 + ...(parallelWarning ? [`Warning: ${parallelWarning}`] : []),
1424 + "Overkill will not write the source skill.",
1425 + ].join("\n");
1426 + if (
1427 + ctx.hasUI &&
1428 + !(await ctx.ui.confirm("Persist and mine Aftermath digests?", confirmation))
1430 + return;
1431 + throwIfCancelled(active);
1432 + const frozenModel = ctx.model;
1433 + const targetModel =
1434 + frozenModel && ctx.modelRegistry.find(frozenModel.provider, frozenModel.id);
1435 + if (!frozenModel || !targetModel || !(await modelHasAuth(ctx)))
1436 + throw new Error("selected model authentication is unavailable");
1437 + const sourceBytes = await readFile(sourcePath);
1438 + throwIfCancelled(active);
1439 + parseSourceSkill(sourceBytes);
1440 + await resolveLoadedSkill(
1441 + sourcePath,
1442 + pi.getCommands(),
1443 + ctx.isProjectTrusted(),
1444 + ctx.cwd,
1445 + );
1446 + throwIfCancelled(active);
1448 + const created = await createAftermathDirectory(getAgentDir());
1449 + active.runId = created.runId;
1450 + const manifest: AftermathManifest = {
1451 + version: 1 as const,
1452 + runId: created.runId,
1453 + kind: "aftermath" as const,
1454 + source: {
1455 + path: sourcePath,
1456 + fingerprint: fingerprintSource(sourceBytes),
1457 + },
1458 + model: {
1459 + provider: frozenModel.provider,
1460 + id: frozenModel.id,
1461 + thinking: pi.getThinkingLevel(),
1462 + },
1463 + sessionId: ctx.sessionManager.getSessionId(),
1464 + linkEntryId: null,
1465 + stage: "harvest",
1466 + miningPromptVersion: 2,
1467 + terminal: null,
1468 + scores: { baseline: null, current: null, best: null, test: null },
1469 + calls: { target: 0, checker: 0, optimizer: 0, judge: 0 },
1470 + artifacts: {
1471 + baselineSnapshot: "baseline-snapshot.md",
1472 + report: null,
1473 + staged: null,
1474 + },
1475 + target: { name: selected.name, path: sourcePath },
1476 + };
1477 + active.interrupt = async () =>
1478 + await persistAftermathManifest(created.directory, {
1479 + ...manifest,
1480 + stage: "interrupted",
1481 + });
1482 + await writePrivateText(
1483 + created.directory,
1484 + "baseline-snapshot.md",
1485 + sourceBytes,
1486 + );
1487 + const persisted = await persistHarvest(created.directory, {
1488 + digests: harvested.digests,
1489 + unreadable: harvested.unreadable,
1490 + total: harvested.total,
1491 + evidence: evidence.counts,
1492 + });
1493 + manifest.stage = "mining";
1494 + await persistAftermathManifest(created.directory, {
1495 + ...manifest,
1496 + harvestFingerprint: persisted.fingerprint,
1497 + });
1498 + appendProgress(pi, ctx, runEntry({ manifest }));
1499 + const link = ctx.sessionManager.getBranch().at(-1);
1500 + if (!link || typeof link.id !== "string")
1501 + throw new Error("could not persist Aftermath branch link");
1502 + manifest.linkEntryId = link.id;
1503 + await persistAftermathManifest(created.directory, {
1504 + ...manifest,
1505 + harvestFingerprint: persisted.fingerprint,
1506 + });
1507 + await mineAftermath({
1508 + pi,
1509 + ctx,
1510 + directory: created.directory,
1511 + manifest,
1512 + digests: harvested.digests,
1513 + targetModel,
1514 + harvestFingerprint: persisted.fingerprint,
1515 + });
1519 +async function offerAftermathCompletion(
1520 + pi: ExtensionAPI,
1521 + ctx: ExtensionCommandContext,
1522 + directory: string,
1523 +): Promise<void> {
1524 + if (!ctx.hasUI || (commandMode(ctx) !== "tui" && commandMode(ctx) !== "rpc"))
1525 + return;
1526 + const artifacts = await readFrozenReviewArtifacts(directory);
1527 + const manifest = await readAftermathManifest(directory);
1528 + const staged = artifacts.stagedPath !== null && artifacts.staged !== null;
1529 + const report = artifacts.reportPath
1530 + ? `Report: ${sanitizeTerminalText(artifacts.reportPath)}`
1531 + : "";
1532 + let summary: string;
1533 + if (staged) {
1534 + const scores = [
1535 + manifest.scores.baseline === null
1536 + ? null
1537 + : `baseline ${formatScore(manifest.scores.baseline)}`,
1538 + manifest.scores.best === null
1539 + ? null
1540 + : `validation ${formatScore(manifest.scores.best)}`,
1541 + manifest.scores.test === null
1542 + ? null
1543 + : `held-out ${formatScore(manifest.scores.test)}`,
1544 + ].filter((value): value is string => value !== null);
1545 + summary = `Aftermath complete — candidate staged.\n${scores.join(" → ")}\n${report}`;
1546 + } else if (manifest.scores.baseline === null) {
1547 + const frozen = await readFrozenTasks(directory);
1548 + summary = `Aftermath complete — no candidate.\nOnly ${frozen.tasks.length}/3 recurring tasks; replay was skipped.\n${report}`;
1549 + } else if (manifest.scores.baseline === 1) {
1550 + summary = `Aftermath complete — no candidate needed.\nBaseline already scores 1; no improvement is possible.\n${report}`;
1551 + } else {
1552 + summary = `Aftermath complete — no candidate.\nBest validation ${formatScore(manifest.scores.best ?? manifest.scores.baseline)} did not beat baseline ${formatScore(manifest.scores.baseline)}; no diff was staged.\n${report}`;
1554 + const choice = await ctx.ui.select(
1555 + summary,
1556 + staged ? ["review diff", "done", "discard"] : ["done", "discard report"],
1557 + );
1558 + if (choice === "review diff") {
1559 + const paths = [
1560 + artifacts.reportPath
1561 + ? `Report: ${sanitizeTerminalText(artifacts.reportPath)}`
1562 + : null,
1563 + artifacts.stagedPath
1564 + ? `Staged: ${sanitizeTerminalText(artifacts.stagedPath)}`
1565 + : null,
1567 + .filter((value): value is string => value !== null)
1568 + .join("\n");
1569 + ctx.ui.notify(
1570 + commandMode(ctx) === "tui" && artifacts.stagedPath && artifacts.staged
1571 + ? `${paths}\n${buildReviewDiff(
1572 + artifacts.sourcePath,
1573 + artifacts.stagedPath,
1574 + artifacts.source,
1575 + artifacts.staged,
1576 + )}`
1577 + : paths,
1578 + "info",
1579 + );
1580 + return;
1582 + if (choice === "done" || choice === undefined) return;
1583 + if (choice === "discard" || choice === "discard report") {
1584 + const selected = selectedAftermathRun(ctx);
1585 + if (!selected)
1586 + throw new Error("Aftermath run is no longer linked to this branch");
1587 + await discardLinkedRun({
1588 + runRoot: join(getAgentDir(), "overkill-runs"),
1589 + runDirectory: directory,
1590 + selected,
1591 + branch: ctx.sessionManager.getBranch(),
1592 + sessionId: ctx.sessionManager.getSessionId(),
1593 + });
1594 + appendProgress(pi, ctx, {
1595 + ...selected.data,
1596 + stage: "discarded",
1597 + terminal: "discarded",
1598 + });
1602 +async function mineAftermath(options: {
1603 + pi: ExtensionAPI;
1604 + ctx: ExtensionCommandContext;
1605 + directory: string;
1606 + manifest: AftermathManifest;
1607 + digests: Digest[];
1608 + targetModel: unknown;
1609 + harvestFingerprint: string;
1610 + completedChunks?: ReadonlyMap<number, unknown[]>;
1611 + completedLevels?: ReadonlyMap<number, unknown[][]>;
1612 + attemptedMiningCalls?: number;
1613 + frozen?: Awaited<ReturnType<typeof readFrozenTasks>>;
1614 +}): Promise<void> {
1615 + const active = activeRun;
1616 + if (active?.kind !== "aftermath")
1617 + throw new Error("Aftermath run ownership is unavailable");
1618 + const { ctx, manifest } = options;
1619 + if (manifest.miningPromptVersion !== 2)
1620 + throw new Error(
1621 + "Aftermath run predates target-relevant mining; discard it and start a new run",
1622 + );
1623 + const frozenSource = await readFile(
1624 + join(options.directory, "baseline-snapshot.md"),
1625 + );
1626 + if (fingerprintSource(frozenSource) !== manifest.source.fingerprint)
1627 + throw new Error("frozen source fingerprint changed; replay is unsafe");
1628 + const source = parseSourceSkill(frozenSource);
1629 + if (source.name !== manifest.target.name)
1630 + throw new Error("frozen source name changed; replay is unsafe");
1631 + manifest.stage = "mining";
1632 + active.interrupt = async () => {
1633 + const interrupted = {
1634 + ...manifest,
1635 + stage: "interrupted",
1636 + harvestFingerprint: options.harvestFingerprint,
1637 + };
1638 + await persistAftermathManifest(options.directory, interrupted);
1639 + const entry = runEntry({ manifest: interrupted });
1640 + if (active.suppressOutput) appendProgressEntry(options.pi, entry);
1641 + else appendProgress(options.pi, ctx, entry);
1642 + };
1643 + const chunks = new Set(options.completedChunks?.keys());
1644 + const levels = new Set<string>();
1645 + let attemptedMiningCalls = options.attemptedMiningCalls ?? 0;
1646 + for (const [level, outputs] of options.completedLevels ?? [])
1647 + for (const index of outputs.keys()) levels.add(`${level}:${index}`);
1648 + const miningTotal = estimateMiningCalls(options.digests.length).total;
1649 + const updateMiningProgress = (
1650 + phase?: "mining" | "aggregation",
1651 + attempt?: number,
1652 + maximumAttempts?: number,
1653 + ) => {
1654 + const completed = chunks.size + levels.size;
1655 + const progress = formatProgress({
1656 + stage: "mining",
1657 + completed,
1658 + total: miningTotal,
1659 + });
1660 + showProgress(
1661 + ctx,
1662 + phase
1663 + ? `${progress} · ${phase} call ${Math.min(completed + 1, miningTotal)} running${attempt && attempt > 1 ? ` · retry ${attempt}/${maximumAttempts}` : ""}`
1664 + : progress,
1665 + );
1666 + };
1667 + const persistCheckpoint = async () =>
1668 + await persistMiningCheckpoint(options.directory, {
1669 + attemptedCalls: attemptedMiningCalls,
1670 + completedChunks: [...chunks],
1671 + completedLevels: [...levels].map((item) => {
1672 + const [levelText, indexText] = item.split(":");
1673 + return { level: Number(levelText), index: Number(indexText) };
1674 + }),
1675 + });
1676 + const frozen =
1677 + options.frozen ??
1678 + (await (async () => {
1679 + updateMiningProgress();
1680 + const proposals = await mineTaskProposals({
1681 + digests: options.digests,
1682 + target: { name: source.name, description: source.description },
1683 + createSession: () =>
1684 + createMiningSession(
1686 + cwd: ctx.cwd,
1687 + agentDir: getAgentDir(),
1688 + model: manifest.model,
1689 + targetModel: options.targetModel,
1690 + },
1691 + installedAftermathAcceptanceSeam()?.structuredOutputSdk,
1692 + ),
1693 + signal: active.controller.signal,
1694 + completedChunks: options.completedChunks,
1695 + completedLevels: options.completedLevels,
1696 + onCall: async (phase, attempt, maximumAttempts) => {
1697 + attemptedMiningCalls += 1;
1698 + await persistCheckpoint();
1699 + updateMiningProgress(phase, attempt, maximumAttempts);
1700 + },
1701 + onChunk: async (index, output) => {
1702 + await persistMiningOutput(
1703 + options.directory,
1704 + `miner-chunk-${index}.json`,
1705 + output,
1706 + );
1707 + chunks.add(index);
1708 + await persistCheckpoint();
1709 + updateMiningProgress();
1710 + },
1711 + onLevel: async (level, index, output) => {
1712 + await persistMiningOutput(
1713 + options.directory,
1714 + `aggregate-${level}-${index}.json`,
1715 + output,
1716 + );
1717 + levels.add(`${level}:${index}`);
1718 + await persistCheckpoint();
1719 + updateMiningProgress();
1720 + },
1721 + });
1722 + throwIfCancelled(active);
1723 + const tasks = validateAndFreezeTasks({
1724 + proposals,
1725 + harvestedSessionIds: new Set(
1726 + options.digests.map((digest) => digest.sessionId),
1727 + ),
1728 + });
1729 + const splits = assignSplits(tasks);
1730 + const persisted = await persistFrozenTasks(
1731 + options.directory,
1732 + tasks,
1733 + splits,
1734 + );
1735 + return { ...persisted, tasks, splits };
1736 + })());
1737 + const { tasks, splits } = frozen;
1738 + if (
1739 + manifest.taskFingerprint !== undefined &&
1740 + manifest.taskFingerprint !== frozen.fingerprint
1742 + throw new Error("frozen task fingerprint changed; resume is unsafe");
1743 + manifest.harvestFingerprint = options.harvestFingerprint;
1744 + manifest.taskFingerprint = frozen.fingerprint;
1745 + const callBounds = upperBoundCalls({
1746 + train: splits.train.length,
1747 + val: splits.validation.length,
1748 + test: splits.test.length,
1749 + });
1750 + const taskById = new Map(tasks.map((task) => [task.id, task]));
1751 + const rubricCount = (split: "train" | "validation" | "test") =>
1752 + splits[split].filter((id) => taskById.get(id)?.rubric !== undefined).length;
1753 + const judgeMaximum =
1754 + rubricCount("validation") +
1755 + 3 * (rubricCount("train") + rubricCount("validation")) +
1756 + rubricCount("test");
1757 + manifest.scorers = tasks.map((task) => ({
1758 + id: task.id,
1759 + kind: task.checks ? ("deterministic" as const) : ("rubric" as const),
1760 + }));
1761 + manifest.splits = splits;
1762 + let costs = (await readAftermathCost(options.directory)) ?? {
1763 + targetMaximum: tasks.length < 3 ? 0 : callBounds.target,
1764 + judgeMaximum: tasks.length < 3 ? 0 : judgeMaximum,
1765 + targetConsumed: 0,
1766 + judgeConsumed: 0,
1767 + };
1768 + if (
1769 + costs.targetMaximum !== (tasks.length < 3 ? 0 : callBounds.target) ||
1770 + costs.judgeMaximum !== (tasks.length < 3 ? 0 : judgeMaximum)
1772 + throw new Error("Aftermath replay call maximum changed; resume is unsafe");
1773 + manifest.calls.target = costs.targetConsumed;
1774 + manifest.calls.judge = costs.judgeConsumed;
1775 + await persistAftermathCost(options.directory, costs);
1776 + const callNotice = `Replay budget: ≤${costs.targetMaximum} target + ${costs.judgeMaximum} judge calls; malformed judge retries consume this budget.`;
1777 + if (commandMode(ctx) === "print") writePrint(callNotice);
1778 + else if (commandMode(ctx) !== "json") ctx.ui.notify(callNotice, "info");
1779 + const allowCall = (kind: "target" | "judge") => {
1780 + const maximum =
1781 + kind === "target" ? costs.targetMaximum : costs.judgeMaximum;
1782 + const consumed =
1783 + kind === "target" ? costs.targetConsumed : costs.judgeConsumed;
1784 + if (consumed >= maximum)
1785 + throw new Error(`Aftermath ${kind} call maximum exceeded`);
1786 + };
1787 + const recordCall = async (kind: "target" | "judge") => {
1788 + allowCall(kind);
1789 + if (
1790 + kind === "target" &&
1791 + fingerprintSource(await readFile(manifest.source.path)) !==
1792 + manifest.source.fingerprint
1794 + throw new Error("source fingerprint changed; replay is unsafe");
1795 + costs = {
1796 + ...costs,
1797 + ...(kind === "target"
1798 + ? { targetConsumed: costs.targetConsumed + 1 }
1799 + : { judgeConsumed: costs.judgeConsumed + 1 }),
1800 + };
1801 + await persistAftermathCost(options.directory, costs);
1802 + manifest.calls[kind] =
1803 + kind === "target" ? costs.targetConsumed : costs.judgeConsumed;
1804 + await persistAftermathManifest(options.directory, manifest);
1805 + };
1806 + throwIfCancelled(active);
1807 + if (tasks.length < 3) {
1808 + const reportPath = await persistFinalReport(options.directory, {
1809 + version: 1,
1810 + kind: "aftermath",
1811 + taskFingerprint: frozen.fingerprint,
1812 + taskIds: tasks.map((task) => task.id),
1813 + splits,
1814 + targetCallMaximum: costs.targetMaximum,
1815 + judgeCallMaximum: costs.judgeMaximum,
1816 + targetCalls: costs.targetConsumed,
1817 + judgeCalls: costs.judgeConsumed,
1818 + terminal: "insufficient_tasks",
1819 + });
1820 + manifest.stage = "complete";
1821 + manifest.terminal = "no_improvement";
1822 + manifest.artifacts.report = "report.json";
1823 + await persistAftermathManifest(options.directory, manifest);
1824 + const entry = runEntry(
1825 + { manifest },
1826 + { completed: tasks.length, total: 3 },
1827 + { reportPath, stagedPath: null },
1828 + );
1829 + appendProgress(options.pi, ctx, entry);
1830 + if (commandMode(ctx) === "print")
1831 + writePrint(
1832 + `Aftermath complete — no candidate.\nOnly ${tasks.length}/3 recurring tasks; replay was skipped.\nReport: ${sanitizeTerminalText(reportPath)}`,
1833 + );
1834 + await offerAftermathCompletion(options.pi, ctx, options.directory);
1835 + return;
1837 + await persistAftermathManifest(options.directory, manifest);
1838 + const currentSource = await readFile(manifest.source.path);
1839 + if (fingerprintSource(currentSource) !== manifest.source.fingerprint)
1840 + throw new Error("source fingerprint changed; replay is unsafe");
1841 + const loadedSkill = await resolveLoadedSkill(
1842 + manifest.source.path,
1843 + options.pi.getCommands(),
1844 + ctx.isProjectTrusted(),
1845 + ctx.cwd,
1846 + );
1847 + const byId = new Map(tasks.map((task) => [task.id, task]));
1848 + const taskItems = (split: "train" | "validation" | "test") =>
1849 + splits[split].map((id) => {
1850 + const task = byId.get(id);
1851 + if (!task) throw new Error("frozen split task is unavailable");
1852 + return { id, prompt: generatedTaskPrompt(task) };
1853 + });
1854 + const judgeCache = await readJudgeCache(options.directory);
1855 + const scorer = createGeneratedTaskScorer(tasks, {
1856 + judge: async (task, response, signal) => {
1857 + const key = rubricCacheKey(task.id, response);
1858 + const cached = judgeCache[key];
1859 + if (cached) return cached;
1860 + const result = await runRubricJudge(
1861 + task,
1862 + response,
1864 + createSession: () =>
1865 + createRubricJudgeSession(
1867 + cwd: ctx.cwd,
1868 + agentDir: getAgentDir(),
1869 + model: manifest.model,
1870 + targetModel: options.targetModel,
1871 + },
1872 + installedAftermathAcceptanceSeam()?.structuredOutputSdk,
1873 + ),
1874 + },
1875 + signal,
1876 + async () => await recordCall("judge"),
1877 + );
1878 + judgeCache[result.cacheKey] = {
1879 + score: result.score,
1880 + passed: result.passed,
1881 + rationale: result.rationale,
1882 + };
1883 + await persistJudgeCache(options.directory, judgeCache);
1884 + return result;
1885 + },
1886 + });
1887 + await executeBaseline({
1888 + pi: options.pi,
1889 + ctx,
1890 + directory: options.directory,
1891 + manifest,
1892 + source,
1893 + loadedSkill,
1894 + targetModel: options.targetModel,
1895 + taskAdapter: {
1896 + items: taskItems,
1897 + scoreResponse: async ({ item, response, signal }) =>
1898 + await scorer({ id: item.id, response, signal }),
1899 + onTargetCall: async () => await recordCall("target"),
1900 + },
1901 + reportExtras: () => ({
1902 + taskFingerprint: frozen.fingerprint,
1903 + targetCallMaximum: costs.targetMaximum,
1904 + judgeCallMaximum: costs.judgeMaximum,
1905 + targetCalls: costs.targetConsumed,
1906 + judgeCalls: costs.judgeConsumed,
1907 + }),
1908 + deferCompletionNotice: true,
1909 + });
1910 + await offerAftermathCompletion(options.pi, ctx, options.directory);
1913 +async function resumeAftermath(
1914 + pi: ExtensionAPI,
1915 + ctx: ExtensionCommandContext,
1916 + selected: NonNullable<ReturnType<typeof selectedAftermathRun>>,
1917 +): Promise<void> {
1918 + const active = activeRun;
1919 + if (active?.kind !== "aftermath")
1920 + throw new Error("Aftermath run ownership is unavailable");
1921 + const directory = join(getAgentDir(), "overkill-runs", selected.data.runId);
1922 + let manifest = await readAftermathManifest(directory);
1923 + if (manifest.terminal)
1924 + throw new Error("only incomplete Aftermath runs can resume");
1925 + if (
1926 + manifest.runId !== selected.data.runId ||
1927 + manifest.sessionId !== ctx.sessionManager.getSessionId()
1929 + throw new Error("run is not linked to this active session branch");
1930 + manifest = (await recoverRunLink({
1931 + directory,
1932 + manifest,
1933 + selected,
1934 + branch: ctx.sessionManager.getBranch(),
1935 + })) as AftermathManifest;
1936 + const sourcePath = await realpath(manifest.source.path);
1937 + const source = await readFile(sourcePath);
1938 + if (fingerprintSource(source) !== manifest.source.fingerprint)
1939 + throw new Error("source fingerprint changed; resume is unsafe");
1940 + const targetModel = ctx.modelRegistry.find(
1941 + manifest.model.provider,
1942 + manifest.model.id,
1943 + );
1944 + if (
1945 + !targetModel ||
1946 + !(await modelHasAuth(ctx, manifest.model.provider, manifest.model.id))
1948 + throw new Error("recorded model authentication is unavailable");
1949 + const harvest = await readHarvest(directory);
1950 + if (harvest.fingerprint !== manifest.harvestFingerprint)
1951 + throw new Error("harvest fingerprint changed; resume is unsafe");
1952 + const frozen = await readFrozenTasks(directory).catch((error) => {
1953 + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
1954 + throw error;
1955 + });
1956 + const checkpoint = await readMiningCheckpoint(directory);
1957 + const chunks = new Map<number, unknown[]>();
1958 + const levels = new Map<number, unknown[][]>();
1959 + for (const index of checkpoint?.completedChunks ?? [])
1960 + chunks.set(
1961 + index,
1962 + await readMiningOutput(directory, `miner-chunk-${index}.json`),
1963 + );
1964 + for (const { level, index } of checkpoint?.completedLevels ?? []) {
1965 + const outputs = levels.get(level) ?? [];
1966 + outputs[index] = await readMiningOutput(
1967 + directory,
1968 + `aggregate-${level}-${index}.json`,
1969 + );
1970 + levels.set(level, outputs);
1972 + active.runId = manifest.runId;
1973 + await mineAftermath({
1974 + pi,
1975 + ctx,
1976 + directory,
1977 + manifest,
1978 + digests: harvest.digests,
1979 + targetModel,
1980 + harvestFingerprint: harvest.fingerprint,
1981 + completedChunks: chunks,
1982 + completedLevels: levels,
1983 + attemptedMiningCalls: checkpoint?.attemptedCalls ?? 0,
1984 + frozen,
1985 + });
1988 +async function resumeBaseline(
1989 + pi: ExtensionAPI,
1990 + ctx: ExtensionCommandContext,
1991 + selected: NonNullable<ReturnType<typeof selectedRun>>,
1992 +): Promise<void> {
1993 + const directory = join(getAgentDir(), "overkill-runs", selected.data.runId);
1994 + const active = activeRun;
1995 + if (!active || active.sessionId !== ctx.sessionManager.getSessionId())
1996 + throw new Error("Overkill run ownership is unavailable");
1997 + active.runId = selected.data.runId;
1998 + active.interrupt = async () =>
1999 + await checkpointInterrupted(directory, await readManifest(directory));
2000 + let manifest = await readManifest(directory);
2001 + if (manifest.terminal) throw new Error("selected run is terminal");
2002 + if (
2003 + manifest.runId !== selected.data.runId ||
2004 + manifest.sessionId !== ctx.sessionManager.getSessionId()
2006 + throw new Error("run is not linked to this active session branch");
2007 + manifest = await recoverRunLink({
2008 + directory,
2009 + manifest,
2010 + selected,
2011 + branch: ctx.sessionManager.getBranch(),
2012 + });
2013 + active.runId = manifest.runId;
2014 + active.interrupt = async () =>
2015 + await checkpointInterrupted(directory, manifest);
2016 + throwIfCancelled(active);
2017 + const originalLink = ctx.sessionManager.getBranch().find((entry) => {
2018 + if (typeof entry !== "object" || entry === null) return false;
2019 + const candidate = entry as {
2020 + id?: unknown;
2021 + type?: unknown;
2022 + customType?: unknown;
2023 + data?: unknown;
2024 + };
2025 + if (
2026 + candidate.id !== manifest.linkEntryId ||
2027 + candidate.type !== "custom" ||
2028 + candidate.customType !== "overkill-run"
2030 + return false;
2031 + try {
2032 + const data = validateRunEntry(candidate.data);
2033 + return data.kind === "suite" && data.runId === manifest.runId;
2034 + } catch {
2035 + return false;
2037 + });
2038 + if (!originalLink)
2039 + throw new Error("run is no longer linked to this active branch");
2040 + const sourcePath = await realpath(manifest.source.path);
2041 + throwIfCancelled(active);
2042 + const sourceBytes = await readFile(sourcePath);
2043 + throwIfCancelled(active);
2044 + if (fingerprintSource(sourceBytes) !== manifest.source.fingerprint)
2045 + throw new Error("source fingerprint changed; resume is unsafe");
2046 + if (!manifest.suite) throw new Error("suite run has no suite contract");
2047 + const suite = await validateTrustedSuite(
2048 + manifest.suite.path,
2049 + ctx.cwd,
2050 + ctx.isProjectTrusted(),
2051 + );
2052 + throwIfCancelled(active);
2053 + if ((await fingerprintSuite(suite.suitePath)) !== manifest.suite.fingerprint)
2054 + throw new Error("suite fingerprint changed; resume is unsafe");
2055 + throwIfCancelled(active);
2056 + const model = ctx.modelRegistry.find(
2057 + manifest.model.provider,
2058 + manifest.model.id,
2059 + );
2060 + if (
2061 + !model ||
2062 + !(await modelHasAuth(ctx, manifest.model.provider, manifest.model.id))
2064 + throw new Error("recorded model authentication is unavailable");
2065 + const loadedSkill = await resolveLoadedSkill(
2066 + sourcePath,
2067 + pi.getCommands(),
2068 + ctx.isProjectTrusted(),
2069 + ctx.cwd,
2070 + );
2071 + throwIfCancelled(active);
2072 + await executeBaseline({
2073 + pi,
2074 + ctx,
2075 + directory,
2076 + manifest,
2077 + source: parseSourceSkill(sourceBytes),
2078 + loadedSkill,
2079 + suite,
2080 + targetModel: model,
2081 + });
2084 +export async function shutdownOverkill(ctx: ExtensionContext): Promise<void> {
2085 + const sessionId = ctx.sessionManager.getSessionId();
2086 + if (!activeRun) {
2087 + if (startPending && pendingSessionId === sessionId) pendingCancelled = true;
2088 + return;
2090 + if (activeRun.sessionId !== sessionId) return;
2091 + activeRun.suppressOutput = true;
2092 + requestCancellation(activeRun);
2093 + try {
2094 + await activeRun.completed;
2095 + } catch {}
2098 +export default function overkill(pi: ExtensionAPI): void {
2099 + pi.on("session_shutdown", async (_event, ctx) => {
2100 + await shutdownOverkill(ctx);
2101 + });
2102 + registerOverkillEntryRenderer(pi);
2103 + pi.registerCommand("overkill", createOverkillCommand(pi));
2106 +export function createOverkillCommand(pi: ExtensionAPI) {
2107 + return {
2108 + description:
2109 + "Author or optimize a loaded Pi skill with trusted evaluators.",
2110 + getArgumentCompletions: (prefix: string) => {
2111 + const trimmed = prefix.trim();
2112 + if (trimmed === "aftermath" && !prefix.endsWith(" ")) return null;
2113 + if (trimmed === "aftermath" || trimmed.startsWith("aftermath ")) {
2114 + const nested = trimmed.slice("aftermath".length).trim();
2115 + const choices = ["status", "resume", "review", "discard", "cancel"]
2116 + .filter((command) => command.startsWith(nested))
2117 + .map((command) => ({
2118 + value: `aftermath ${command}`,
2119 + label: `aftermath ${command}`,
2120 + }));
2121 + return choices.length > 0 ? choices : null;
2123 + const choices = SUBCOMMANDS.filter((command) =>
2124 + command.startsWith(trimmed),
2125 + ).map((command) => ({ value: command, label: command }));
2126 + return choices.length > 0 ? choices : null;
2127 + },
2128 + handler: async (args: string, ctx: ExtensionCommandContext) => {
2129 + let parts: string[];
2130 + try {
2131 + parts = parseArguments(args);
2132 + } catch (error) {
2133 + ctx.ui.notify(
2134 + `Overkill command failed: ${error instanceof Error ? error.message : String(error)}`,
2135 + "error",
2136 + );
2137 + return;
2139 + const command = parts[0];
2140 + if (command === "suite") {
2141 + let [sourcePath, suitePath, ...extra] = parts.slice(1);
2142 + if (extra.length > 0) {
2143 + ctx.ui.notify(
2144 + "Usage: /overkill suite [skill-path] [suite-dir]",
2145 + "error",
2146 + );
2147 + return;
2149 + if (!ctx.isIdle()) {
2150 + ctx.ui.notify("Agent is busy.", "error");
2151 + return;
2153 + try {
2154 + if (!ctx.isProjectTrusted())
2155 + throw new Error("suite authoring requires a trusted project");
2156 + if (!sourcePath) {
2157 + if (!ctx.hasUI)
2158 + throw new Error(
2159 + "headless suite authoring requires an explicit skill path",
2160 + );
2161 + const skills = loadedSkills(pi);
2162 + if (skills.length === 0)
2163 + throw new Error("no loaded Pi skills are available");
2164 + const canonicalPaths = new Map<string, string>();
2165 + for (const skill of skills)
2166 + canonicalPaths.set(skill.path, await realpath(skill.path));
2167 + const selected = await chooseLoadedSkill(
2168 + ctx,
2169 + skills,
2170 + canonicalPaths,
2171 + "suite",
2172 + );
2173 + if (!selected) return;
2174 + sourcePath = selected.path;
2176 + const loaded = await resolveLoadedSkill(
2177 + sourcePath,
2178 + pi.getCommands(),
2179 + true,
2180 + ctx.cwd,
2181 + );
2182 + const projectRoot = await realpath(resolve(ctx.cwd));
2183 + if (suitePath) {
2184 + suitePath = await canonicalProspectivePath(
2185 + resolve(projectRoot, suitePath),
2186 + );
2187 + const projectRelative = relative(projectRoot, suitePath);
2188 + if (projectRelative.startsWith("..") || isAbsolute(projectRelative))
2189 + throw new Error("suite directory must be inside the project");
2191 + const parallelWarning = await unfinishedRunWarning(
2192 + undefined,
2193 + suitePath,
2194 + );
2195 + if (parallelWarning) ctx.ui.notify(parallelWarning, "warning");
2196 + pi.sendUserMessage(
2197 + buildSuiteAuthoringPrompt(loaded.path, suitePath, projectRoot),
2198 + );
2199 + } catch (error) {
2200 + ctx.ui.notify(
2201 + `Overkill suite authoring failed: ${error instanceof Error ? error.message : String(error)}`,
2202 + "error",
2203 + );
2205 + return;
2207 + if (command === "aftermath") {
2208 + const subcommand = parts[1];
2209 + if (
2210 + ["status", "resume", "review", "discard", "cancel"].includes(
2211 + subcommand ?? "",
2212 + ) &&
2213 + parts.length !== 2
2214 + ) {
2215 + ctx.ui.notify(`Usage: /overkill aftermath ${subcommand}`, "error");
2216 + return;
2218 + if (subcommand === "status") {
2219 + const run = selectedAftermathRun(ctx);
2220 + if (!run)
2221 + ctx.ui.notify(
2222 + "No Overkill Aftermath run is linked to this branch.",
2223 + "info",
2224 + );
2225 + else if (commandMode(ctx) === "json")
2226 + appendProgress(pi, ctx, run.data);
2227 + else if (commandMode(ctx) === "print")
2228 + writePrint(statusMessage(run.data));
2229 + else ctx.ui.notify(statusMessage(run.data), "info");
2230 + return;
2232 + if (subcommand === "cancel") {
2233 + if (
2234 + activeRun &&
2235 + activeRun.kind === "aftermath" &&
2236 + activeRun.sessionId === ctx.sessionManager.getSessionId()
2238 + requestCancellation(activeRun);
2239 + else ctx.ui.notify("No Overkill Aftermath run is active.", "info");
2240 + return;
2242 + if (["resume", "review", "discard"].includes(subcommand ?? "")) {
2243 + const run = selectedAftermathRun(ctx);
2244 + if (!run) {
2245 + ctx.ui.notify(
2246 + "No Overkill Aftermath run is linked to this branch.",
2247 + "info",
2248 + );
2249 + return;
2251 + const directory = join(
2252 + getAgentDir(),
2253 + "overkill-runs",
2254 + run.data.runId,
2255 + );
2256 + try {
2257 + const manifest = await readAftermathManifest(directory);
2258 + if (manifest.sessionId !== ctx.sessionManager.getSessionId())
2259 + throw new Error(
2260 + "run is not linked to this active session branch",
2261 + );
2262 + if (subcommand === "review") {
2263 + const artifacts = await readFrozenReviewArtifacts(directory);
2264 + const entry = {
2265 + ...run.data,
2266 + sourcePath: artifacts.sourcePath,
2267 + reportPath: artifacts.reportPath,
2268 + stagedPath: artifacts.stagedPath,
2269 + };
2270 + const paths = formatResultPaths(entry);
2271 + if (commandMode(ctx) === "json") appendProgress(pi, ctx, entry);
2272 + else if (commandMode(ctx) === "print") {
2273 + writePrint(paths);
2274 + if (artifacts.stagedPath && artifacts.staged)
2275 + writePrint(
2276 + buildReviewDiff(
2277 + artifacts.sourcePath,
2278 + artifacts.stagedPath,
2279 + artifacts.source,
2280 + artifacts.staged,
2281 + ),
2282 + );
2283 + else writePrint("No candidate staged.");
2284 + } else {
2285 + ctx.ui.notify(
2286 + artifacts.stagedPath
2287 + ? paths
2288 + : `${paths}\nNo candidate staged.`,
2289 + "info",
2290 + );
2291 + if (
2292 + commandMode(ctx) === "tui" &&
2293 + artifacts.stagedPath &&
2294 + artifacts.staged
2296 + ctx.ui.notify(
2297 + buildReviewDiff(
2298 + artifacts.sourcePath,
2299 + artifacts.stagedPath,
2300 + artifacts.source,
2301 + artifacts.staged,
2302 + ),
2303 + "info",
2304 + );
2306 + } else if (subcommand === "discard") {
2307 + if (manifest.terminal === "discarded")
2308 + throw new Error("discarded run artifacts are unavailable");
2309 + await discardLinkedRun({
2310 + runRoot: join(getAgentDir(), "overkill-runs"),
2311 + runDirectory: directory,
2312 + selected: run,
2313 + branch: ctx.sessionManager.getBranch(),
2314 + sessionId: ctx.sessionManager.getSessionId(),
2315 + allowNonterminal: true,
2316 + isRunActive: (runId) => activeRun?.runId === runId,
2317 + });
2318 + appendProgress(pi, ctx, {
2319 + ...run.data,
2320 + stage: "discarded",
2321 + terminal: "discarded",
2322 + });
2323 + clearProgress(ctx);
2324 + } else if (manifest.terminal) {
2325 + ctx.ui.notify(
2326 + "Aftermath mining is already complete; stored tasks are frozen.",
2327 + "info",
2328 + );
2329 + } else {
2330 + await withActiveRun(
2331 + ctx,
2332 + async () => await resumeAftermath(pi, ctx, run),
2333 + "aftermath",
2334 + );
2336 + } catch (error) {
2337 + if (!(error instanceof RolloutCancelledError))
2338 + ctx.ui.notify(
2339 + `Overkill Aftermath ${subcommand} failed: ${error instanceof Error ? error.message : String(error)}`,
2340 + "error",
2341 + );
2343 + return;
2345 + if (activeRun || startPending) {
2346 + ctx.ui.notify(
2347 + "This Pi process already has an Overkill rollout active; wait or cancel it before starting another.",
2348 + "error",
2349 + );
2350 + return;
2352 + startPending = true;
2353 + pendingSessionId = ctx.sessionManager.getSessionId();
2354 + pendingCancelled = false;
2355 + try {
2356 + await withActiveRun(
2357 + ctx,
2358 + async () => await runAftermath(pi, ctx, parts.slice(1)),
2359 + "aftermath",
2360 + );
2361 + } catch (error) {
2362 + if (!(error instanceof RolloutCancelledError))
2363 + ctx.ui.notify(
2364 + `Overkill Aftermath failed: ${error instanceof Error ? error.message : String(error)}`,
2365 + "error",
2366 + );
2367 + } finally {
2368 + startPending = false;
2369 + pendingSessionId = undefined;
2370 + pendingCancelled = false;
2372 + return;
2374 + if (
2375 + ["status", "discard", "cancel", "review", "resume"].includes(
2376 + command ?? "",
2377 + ) &&
2378 + parts.length !== 1
2379 + ) {
2380 + ctx.ui.notify(`Usage: /overkill ${command}`, "error");
2381 + return;
2383 + if (command === "status") {
2384 + const run = selectedRun(ctx);
2385 + if (!run) {
2386 + ctx.ui.notify(
2387 + "No Overkill suite run is linked to this branch.",
2388 + "info",
2389 + );
2390 + return;
2392 + const mode = commandMode(ctx);
2393 + if (mode === "json") appendProgress(pi, ctx, run.data);
2394 + else if (mode === "print") writePrint(statusMessage(run.data));
2395 + else ctx.ui.notify(statusMessage(run.data), "info");
2396 + return;
2398 + if (command === "discard") {
2399 + const run = selectedRun(ctx);
2400 + if (!run) {
2401 + ctx.ui.notify(
2402 + "No Overkill suite run is linked to this branch.",
2403 + "info",
2404 + );
2405 + return;
2407 + try {
2408 + await discardLinkedRun({
2409 + runRoot: join(getAgentDir(), "overkill-runs"),
2410 + runDirectory: join(getAgentDir(), "overkill-runs", run.data.runId),
2411 + selected: run,
2412 + branch: ctx.sessionManager.getBranch(),
2413 + sessionId: ctx.sessionManager.getSessionId(),
2414 + allowNonterminal: true,
2415 + isRunActive: (runId) => activeRun?.runId === runId,
2416 + });
2417 + appendProgress(pi, ctx, {
2418 + ...run.data,
2419 + stage: "discarded",
2420 + terminal: "discarded",
2421 + });
2422 + clearProgress(ctx);
2423 + ctx.ui.notify("Overkill run discarded.", "info");
2424 + } catch (error) {
2425 + ctx.ui.notify(
2426 + `Overkill discard failed: ${error instanceof Error ? error.message : String(error)}`,
2427 + "error",
2428 + );
2430 + return;
2432 + if (command === "cancel") {
2433 + if (
2434 + !activeRun ||
2435 + activeRun.sessionId !== ctx.sessionManager.getSessionId()
2436 + ) {
2437 + ctx.ui.notify("No Overkill rollout is active.", "info");
2438 + return;
2440 + requestCancellation(activeRun);
2441 + if (commandMode(ctx) !== "json")
2442 + ctx.ui.notify("Overkill cancellation requested.", "info");
2443 + return;
2445 + if (command === "review") {
2446 + const run = selectedRun(ctx);
2447 + if (!run) {
2448 + ctx.ui.notify(
2449 + "No Overkill suite run is linked to this branch.",
2450 + "info",
2451 + );
2452 + return;
2454 + if (!run.data.terminal) {
2455 + ctx.ui.notify("Overkill run is not terminal yet.", "info");
2456 + return;
2458 + if (run.data.terminal === "discarded") {
2459 + ctx.ui.notify(
2460 + "Overkill run was discarded; review artifacts are unavailable.",
2461 + "info",
2462 + );
2463 + return;
2465 + let artifacts: Awaited<ReturnType<typeof readFrozenReviewArtifacts>>;
2466 + try {
2467 + artifacts = await readFrozenReviewArtifacts(
2468 + join(getAgentDir(), "overkill-runs", run.data.runId),
2469 + );
2470 + } catch (error) {
2471 + ctx.ui.notify(
2472 + `Overkill review failed: ${error instanceof Error ? error.message : String(error)}`,
2473 + "error",
2474 + );
2475 + return;
2477 + const entry = {
2478 + ...run.data,
2479 + sourcePath: artifacts.sourcePath,
2480 + reportPath: artifacts.reportPath,
2481 + stagedPath: artifacts.stagedPath,
2482 + };
2483 + const paths = formatResultPaths(entry);
2484 + const mode = commandMode(ctx);
2485 + if (mode === "json") appendProgress(pi, ctx, entry);
2486 + else if (mode === "print") {
2487 + writePrint(paths);
2488 + if (artifacts.stagedPath && artifacts.staged)
2489 + writePrint(
2490 + buildReviewDiff(
2491 + artifacts.sourcePath,
2492 + artifacts.stagedPath,
2493 + artifacts.source,
2494 + artifacts.staged,
2495 + ),
2496 + );
2497 + else writePrint("No candidate staged.");
2498 + } else {
2499 + ctx.ui.notify(
2500 + artifacts.stagedPath ? paths : `${paths}\nNo candidate staged.`,
2501 + "info",
2502 + );
2503 + if (mode === "tui" && artifacts.stagedPath && artifacts.staged)
2504 + ctx.ui.notify(
2505 + buildReviewDiff(
2506 + artifacts.sourcePath,
2507 + artifacts.stagedPath,
2508 + artifacts.source,
2509 + artifacts.staged,
2510 + ),
2511 + "info",
2512 + );
2514 + return;
2516 + if (command === "resume") {
2517 + if (activeRun || startPending) {
2518 + ctx.ui.notify(
2519 + "This Pi process already has an Overkill rollout active; wait or cancel it before resuming another.",
2520 + "error",
2521 + );
2522 + return;
2524 + const run = selectedRun(ctx);
2525 + if (!run) {
2526 + ctx.ui.notify(
2527 + "No Overkill suite run is linked to this branch.",
2528 + "info",
2529 + );
2530 + return;
2532 + startPending = true;
2533 + pendingSessionId = ctx.sessionManager.getSessionId();
2534 + pendingCancelled = false;
2535 + try {
2536 + await withActiveRun(
2537 + ctx,
2538 + async () => await resumeBaseline(pi, ctx, run),
2539 + );
2540 + } catch (error) {
2541 + if (
2542 + error instanceof RolloutCancelledError ||
2543 + error instanceof OptimizerCancelledError
2545 + return;
2546 + ctx.ui.notify(
2547 + `Overkill resume failed: ${error instanceof Error ? error.message : String(error)}`,
2548 + "error",
2549 + );
2550 + } finally {
2551 + startPending = false;
2552 + pendingSessionId = undefined;
2553 + pendingCancelled = false;
2555 + return;
2557 + const [sourcePath, suitePath, ...extra] = parts;
2558 + if (!sourcePath || !suitePath || extra.length > 0) {
2559 + ctx.ui.notify("Usage: /overkill <skill-path> <suite-dir>", "error");
2560 + return;
2562 + if (activeRun || startPending) {
2563 + ctx.ui.notify(
2564 + "This Pi process already has an Overkill rollout active; wait or cancel it before starting another.",
2565 + "error",
2566 + );
2567 + return;
2569 + const invokingModel = ctx.model;
2570 + const targetModel =
2571 + invokingModel &&
2572 + ctx.modelRegistry.find(invokingModel.provider, invokingModel.id);
2573 + startPending = true;
2574 + pendingSessionId = ctx.sessionManager.getSessionId();
2575 + pendingCancelled = false;
2576 + try {
2577 + if (invokingModel && !targetModel)
2578 + throw new Error(
2579 + "selected model is unavailable from the invoking registry",
2580 + );
2581 + const preflightOptions = {
2582 + agentDir: getAgentDir(),
2583 + cwd: ctx.cwd,
2584 + sourcePath,
2585 + suitePath,
2586 + commands: pi.getCommands(),
2587 + projectTrusted: ctx.isProjectTrusted(),
2588 + sessionId: ctx.sessionManager.getSessionId(),
2589 + model: invokingModel
2590 + ? {
2591 + provider: invokingModel.provider,
2592 + id: invokingModel.id,
2593 + thinking: pi.getThinkingLevel(),
2595 + : undefined,
2596 + authAvailable: await modelHasAuth(
2597 + ctx,
2598 + invokingModel?.provider,
2599 + invokingModel?.id,
2600 + ),
2601 + };
2602 + const inspected = await inspectSuiteRun(preflightOptions);
2603 + const inspectedSuiteFingerprint = inspected.suite.fingerprint;
2604 + if (!inspectedSuiteFingerprint)
2605 + throw new Error("validated suite fingerprint is unavailable");
2606 + const confirmedFingerprints = {
2607 + sourceFingerprint: fingerprintSource(inspected.sourceBytes),
2608 + suiteFingerprint: inspectedSuiteFingerprint,
2609 + };
2610 + const parallelWarning = await unfinishedRunWarning(
2611 + inspected.sourcePath,
2612 + inspected.suite.suitePath,
2613 + );
2614 + if (commandMode(ctx) === "tui" || commandMode(ctx) === "rpc") {
2615 + const confirmed = await ctx.ui.confirm(
2616 + "Start Overkill run?",
2617 + buildConfirmation({
2618 + sourcePath: inspected.sourcePath,
2619 + suitePath: inspected.suite.suitePath,
2620 + splitCounts: inspected.suite.splitCounts,
2621 + warning: parallelWarning,
2622 + }),
2623 + );
2624 + if (!confirmed) {
2625 + ctx.ui.notify("Overkill start cancelled.", "info");
2626 + return;
2629 + if (pendingCancelled) throw new RolloutCancelledError();
2630 + await withActiveRun(ctx, async (active) => {
2631 + throwIfCancelled(active);
2632 + const run = await prepareSuiteRun(
2633 + preflightOptions,
2634 + confirmedFingerprints,
2635 + );
2636 + active.runId = run.manifest.runId;
2637 + active.interrupt = async () =>
2638 + await checkpointInterrupted(run.directory, run.manifest);
2639 + throwIfCancelled(active);
2640 + appendProgress(pi, ctx, runEntry(run));
2641 + const link = ctx.sessionManager.getBranch().at(-1);
2642 + if (!link || typeof link.id !== "string")
2643 + throw new Error("could not persist branch link");
2644 + run.manifest.linkEntryId = link.id;
2645 + await persistManifest(run.directory, run.manifest);
2646 + throwIfCancelled(active);
2647 + await executeBaseline({
2648 + pi,
2649 + ctx,
2650 + directory: run.directory,
2651 + manifest: run.manifest,
2652 + source: run.source,
2653 + loadedSkill: run.loadedSkill,
2654 + suite: run.suite,
2655 + targetModel,
2656 + });
2657 + });
2658 + } catch (error) {
2659 + if (
2660 + error instanceof RolloutCancelledError ||
2661 + error instanceof OptimizerCancelledError
2663 + return;
2664 + ctx.ui.notify(
2665 + `Overkill run failed: ${error instanceof Error ? error.message : String(error)}`,
2666 + "error",
2667 + );
2668 + } finally {
2669 + startPending = false;
2670 + pendingSessionId = undefined;
2671 + pendingCancelled = false;
2673 + },
2674 + };

extensions/overkill/index.ts

Modified
1 -import { createHash } from "node:crypto";
2 -import { writeSync } from "node:fs";
3 -import { readdir, readFile, realpath } from "node:fs/promises";
4 -import {
5 - basename,
6 - dirname,
7 - extname,
8 - isAbsolute,
9 - join,
10 - relative,
11 - resolve,
12 -} from "node:path";
13 -import {
14 - type ExtensionAPI,
15 - type ExtensionCommandContext,
16 - getAgentDir,
17 - keyHint,
18 - renderDiff,
19 - SessionManager,
20 -} from "@earendil-works/pi-coding-agent";
21 -import { Key, matchesKey, SelectList, Text } from "@earendil-works/pi-tui";
22 -import {
23 - buildDigest,
24 - type Digest,
25 - rankSkillEvidence,
26 -} from "./lib/aftermath-harvest.ts";
27 -import {
28 - assignSplits,
29 - createMiningSession,
30 - estimateMiningCalls,
31 - mineTaskProposals,
32 - type StructuredOutputSdkSeam,
33 - validateAndFreezeTasks,
34 -} from "./lib/aftermath-mining.ts";
35 -import {
36 - createGeneratedTaskScorer,
37 - createRubricJudgeSession,
38 - generatedTaskPrompt,
39 - rubricCacheKey,
40 - runRubricJudge,
41 -} from "./lib/aftermath-scoring.ts";
42 -import {
43 - type AftermathManifest,
44 - createAftermathDirectory,
45 - persistAftermathCost,
46 - persistAftermathManifest,
47 - persistFrozenTasks,
48 - persistHarvest,
49 - persistJudgeCache,
50 - persistMiningCheckpoint,
51 - persistMiningOutput,
52 - readAftermathCost,
53 - readAftermathManifest,
54 - readFrozenTasks,
55 - readHarvest,
56 - readJudgeCache,
57 - readMiningCheckpoint,
58 - readMiningOutput,
59 -} from "./lib/aftermath-state.ts";
60 -import {
61 - executeChecker,
62 - fingerprintSource,
63 - fingerprintSuite,
64 -} from "./lib/checker.ts";
65 -import {
66 - getValidatedSuiteItems,
67 - parseSourceSkill,
68 - resolveLoadedSkill,
69 - type ValidatedItem,
70 - validateTrustedSuite,
71 -} from "./lib/contracts.ts";
72 -import {
73 - buildConfirmation,
74 - buildReviewDiff,
75 - buildReviewDiffPreview,
76 - formatProgress,
77 - formatResultPaths,
78 - formatScore,
79 - sanitizeTerminalText,
80 - upperBoundCalls,
81 -} from "./lib/modes.ts";
82 -import {
83 - createOptimizerSession,
84 - defaultOptimizerSdk,
85 - OptimizerCancelledError,
86 - type OptimizerSdkSeam,
87 - runOptimizerStep,
88 -} from "./lib/optimizer.ts";
89 -import { parsePatch } from "./lib/patches.ts";
90 -import { inspectSuiteRun, prepareSuiteRun } from "./lib/preflight.ts";
91 -import {
92 - createIsolatedTargetSession,
93 - defaultTargetSdk,
94 - RolloutCancelledError,
95 - runSplitRollout,
96 - runValidationBaseline,
97 - type TargetSdkSeam,
98 -} from "./lib/rollouts.ts";
99 -import {
100 - discardLinkedRun,
101 - persistFinalReport,
102 - persistManifest,
103 - persistTrainingCheckpoint,
104 - type RunEntry,
105 - type RunManifest,
106 - readFrozenReviewArtifacts,
107 - readFrozenReviewArtifactsSync,
108 - readManifest,
109 - readScoreCache,
110 - readTrainingCheckpoint,
111 - recoverRunLink,
112 - selectNewestAftermathRun,
113 - selectNewestSuiteRun,
114 - validateRunEntry,
115 - writePrivateText,
116 -} from "./lib/run-state.ts";
117 -import {
118 - type AttemptedPatch,
119 - createTrainingState,
120 - type DurableTrainingState,
121 - runDurableTrainingSchedule,
122 -} from "./lib/training.ts";
1 +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2 +import { registerOverkillEntryRenderer } from "./entry-renderer.ts";
124 4 const SUBCOMMANDS = [
125 5 "aftermath",
131 11 "cancel",
132 12 ] as const;
134 -type ActiveRun = {
135 - kind: "suite" | "aftermath";
136 - runId: string;
137 - sessionId: string;
138 - controller: AbortController;
139 - suppressOutput: boolean;
140 - completed?: Promise<void>;
141 - cancelRequested?: boolean;
142 - interrupt?: () => Promise<void>;
143 - removeCancellationListeners?: () => void;
146 -let activeRun: ActiveRun | undefined;
147 -let startPending = false;
148 -let pendingSessionId: string | undefined;
149 -let pendingCancelled = false;
151 -export type ExplicitSuiteAcceptanceSeam = {
152 - targetSdk: TargetSdkSeam;
153 - optimizerSdk: OptimizerSdkSeam;
156 -export type AftermathSessionApi = {
157 - listAll: (
158 - sessionDir?: string,
159 - ) => Promise<Array<{ path: string; id: string }>>;
160 - open: (
161 - path: string,
162 - sessionDir?: string,
163 - ) => {
164 - getBranch: () => unknown[];
165 - getSessionId: () => string;
166 - getCwd: () => string;
167 - };
170 -export type AftermathAcceptanceSeam = ExplicitSuiteAcceptanceSeam & {
171 - sessionApi: AftermathSessionApi;
172 - structuredOutputSdk: StructuredOutputSdkSeam;
173 - /** Lets acceptance fakes request cancellation through the registered command. */
174 - bindCommand?: (handler: (args: string) => Promise<void>) => void;
177 -export type ModeAcceptanceSeam = {
178 - write: (fd: number, text: string) => void;
181 -const explicitSuiteAcceptanceSeam = Symbol.for(
182 - "@bugabinga/pi-ext-overkill/explicit-suite-acceptance-seam",
184 -const aftermathAcceptanceSeam = Symbol.for(
185 - "@bugabinga/pi-ext-overkill/aftermath-acceptance-seam",
187 -const modeAcceptanceSeam = Symbol.for(
188 - "@bugabinga/pi-ext-overkill/mode-acceptance-seam",
191 -function installedExplicitSuiteAcceptanceSeam():
192 - | ExplicitSuiteAcceptanceSeam
193 - | undefined {
194 - return (globalThis as Record<PropertyKey, unknown>)[
195 - explicitSuiteAcceptanceSeam
196 - ] as ExplicitSuiteAcceptanceSeam | undefined;
199 -function installedAftermathAcceptanceSeam():
200 - | AftermathAcceptanceSeam
201 - | undefined {
202 - return (globalThis as Record<PropertyKey, unknown>)[
203 - aftermathAcceptanceSeam
204 - ] as AftermathAcceptanceSeam | undefined;
207 -function installedModeAcceptanceSeam(): ModeAcceptanceSeam | undefined {
208 - return (globalThis as Record<PropertyKey, unknown>)[modeAcceptanceSeam] as
209 - | ModeAcceptanceSeam
210 - | undefined;
213 -/** Installs deterministic session boundaries while retaining command orchestration. */
214 -export function installExplicitSuiteAcceptanceSeam(
215 - seam: ExplicitSuiteAcceptanceSeam,
216 -): () => void {
217 - const globals = globalThis as Record<PropertyKey, unknown>;
218 - const previous = globals[explicitSuiteAcceptanceSeam];
219 - globals[explicitSuiteAcceptanceSeam] = seam;
220 - return () => {
221 - if (globals[explicitSuiteAcceptanceSeam] !== seam) return;
222 - if (previous === undefined) delete globals[explicitSuiteAcceptanceSeam];
223 - else globals[explicitSuiteAcceptanceSeam] = previous;
224 - };
227 -/** Captures direct mode output while preserving the fd selected by production code. */
228 -export function installModeAcceptanceSeam(
229 - seam: ModeAcceptanceSeam,
230 -): () => void {
231 - const globals = globalThis as Record<PropertyKey, unknown>;
232 - const previous = globals[modeAcceptanceSeam];
233 - globals[modeAcceptanceSeam] = seam;
234 - return () => {
235 - if (globals[modeAcceptanceSeam] !== seam) return;
236 - if (previous === undefined) delete globals[modeAcceptanceSeam];
237 - else globals[modeAcceptanceSeam] = previous;
238 - };
241 -/** Installs deterministic constructors at Aftermath's production boundaries. */
242 -export function installAftermathAcceptanceSeam(
243 - seam: AftermathAcceptanceSeam,
244 -): () => void {
245 - const globals = globalThis as Record<PropertyKey, unknown>;
246 - const previous = globals[aftermathAcceptanceSeam];
247 - globals[aftermathAcceptanceSeam] = seam;
248 - return () => {
249 - if (globals[aftermathAcceptanceSeam] !== seam) return;
250 - if (previous === undefined) delete globals[aftermathAcceptanceSeam];
251 - else globals[aftermathAcceptanceSeam] = previous;
252 - };
255 -export function buildSuiteAuthoringPrompt(
256 - sourcePath: string,
257 - suitePath: string | undefined,
258 - projectRoot: string,
259 -): string {
260 - const sourceName = basename(sourcePath);
261 - const skillName =
262 - sourceName === "SKILL.md"
263 - ? basename(dirname(sourcePath))
264 - : basename(sourceName, extname(sourceName));
265 - const destination = suitePath
266 - ? `Use ${JSON.stringify(suitePath)} as the suite directory.`
267 - : `Choose a suite directory inside ${JSON.stringify(projectRoot)}. Prefer an established project evaluator location; otherwise use the project-local scratch directory ${JSON.stringify(join(projectRoot, ".overkill", "suites", skillName))}.`;
268 - return `Create a trusted explicit evaluator suite for the loaded Pi skill at ${JSON.stringify(sourcePath)}.
269 -${destination}
271 -Read the complete skill and only its directly referenced files needed to understand behavior.
272 -The suite contract below is authoritative; do not inspect Overkill implementation or unrelated project documentation.
273 -Inspect the destination once immediately before writing; if evaluator files already exist or appear concurrently modified, stop and ask rather than overwriting them.
274 -Do not design locks, atomic reservations, or repeated concurrency probes.
275 -After that single inspection, write the four files immediately without narrating a plan.
276 -Create exactly this layout and nothing else inside the suite directory:
277 -- train/items.json
278 -- val/items.json
279 -- test/items.json
280 -- check.mjs
282 -Create a small first suite: exactly 3 train items, 2 validation items, and 2 test items, with globally unique nonempty string ids and nonempty prompts no larger than 16 KiB.
283 -Add checker-owned expected fields when useful.
284 -Use train cases for optimization, validation cases for candidate selection, and held-out test cases for final evaluation; keep scenarios distinct across splits.
285 -Prioritize the skill's highest-risk and most representative successful and failure workflows; report omitted coverage instead of adding more cases.
286 -Keep each item concise and use a small shared expectation vocabulary such as exact, required substrings, forbidden substrings, or simple patterns.
287 -Do not design a domain-specific testing framework.
288 -The checker must read one JSON object from stdin, inspect item, response, trajectory, and calledTools, then write exactly one JSON object with score in [0,1], Boolean passed, and optional feedback.
289 -Write all four files before refining them, and keep check.mjs under 120 lines.
290 -Use only Node.js standard library in check.mjs.
291 -Keep checks bounded, cross-platform, and independent of network, credentials, mutable external state, and the source skill's exact wording.
292 -Do not weaken trust-boundary, security, data-loss, or required human-confirmation behavior.
293 -Validate the JSON and checker locally with one passing and one failing input.
294 -Do not start Overkill automatically.
295 -Summarize coverage, limitations, and the exact /overkill <skill-path> <suite-dir> command the user should run after reviewing check.mjs.`;
298 -export function parseArguments(input: string): string[] {
299 - const parts: string[] = [];
300 - let index = 0;
301 - while (index < input.length) {
302 - while (/\s/u.test(input[index] ?? "")) index += 1;
303 - if (index >= input.length) break;
304 - let token = "";
305 - while (index < input.length && !/\s/u.test(input[index] ?? "")) {
306 - const quote = input[index];
307 - if (quote !== '"' && quote !== "'") {
308 - token += quote;
309 - index += 1;
310 - continue;
312 - index += 1;
313 - let closed = false;
314 - while (index < input.length) {
315 - const character = input[index];
316 - if (character === quote) {
317 - closed = true;
318 - index += 1;
319 - break;
321 - if (character === "\\" && input[index + 1] === quote) {
322 - token += quote;
323 - index += 2;
324 - } else {
325 - token += character;
326 - index += 1;
329 - if (!closed) throw new Error("unterminated quoted argument");
331 - parts.push(token);
333 - return parts;
336 -async function canonicalProspectivePath(path: string): Promise<string> {
337 - let existing = resolve(path);
338 - const missing: string[] = [];
339 - while (true) {
340 - try {
341 - return resolve(await realpath(existing), ...missing);
342 - } catch (error) {
343 - if (
344 - !(error instanceof Error) ||
345 - !("code" in error) ||
346 - error.code !== "ENOENT"
348 - throw error;
350 - const parent = dirname(existing);
351 - if (parent === existing) throw new Error("path has no existing ancestor");
352 - missing.unshift(basename(existing));
353 - existing = parent;
357 -function runEntry(
358 - run: {
359 - manifest: {
360 - runId: string;
361 - kind: "suite" | "aftermath";
362 - stage: string;
363 - terminal: RunEntry["terminal"];
364 - source: { path: string };
365 - };
366 - },
367 - progress: Pick<
368 - RunEntry,
369 - "completed" | "total" | "currentScore" | "bestScore"
370 - > = {},
371 - artifacts: Pick<RunEntry, "reportPath" | "stagedPath"> = {
372 - reportPath: null,
373 - stagedPath: null,
374 - },
375 -): RunEntry {
376 - return {
377 - version: 1,
378 - runId: run.manifest.runId,
379 - kind: run.manifest.kind,
380 - stage: run.manifest.stage,
381 - terminal: run.manifest.terminal,
382 - sourcePath: run.manifest.source.path,
383 - ...artifacts,
384 - ...progress,
385 - };
388 -async function modelHasAuth(
389 - ctx: ExtensionCommandContext,
390 - provider = ctx.model?.provider,
391 - id = ctx.model?.id,
392 -): Promise<boolean> {
393 - if (!provider || !id) return false;
394 - const available = await ctx.modelRegistry.getAvailable();
395 - return available.some(
396 - (model) => model.provider === provider && model.id === id,
397 - );
400 -function selectedRun(ctx: ExtensionCommandContext) {
401 - return selectNewestSuiteRun(ctx.sessionManager.getBranch());
404 -function selectedAftermathRun(ctx: ExtensionCommandContext) {
405 - return selectNewestAftermathRun(ctx.sessionManager.getBranch());
408 -export async function unfinishedRunWarning(
409 - sourcePath: string | undefined,
410 - suitePath?: string,
411 - agentDir = getAgentDir(),
412 -): Promise<string | undefined> {
413 - const root = join(agentDir, "overkill-runs");
414 - const entries = await readdir(root, { withFileTypes: true }).catch(() => []);
415 - const runIds: string[] = [];
416 - for (const entry of entries) {
417 - if (!entry.isDirectory()) continue;
418 - const manifest = await readManifest(join(root, entry.name)).catch(
419 - () => undefined,
420 - );
421 - if (
422 - manifest?.terminal === null &&
423 - ((sourcePath !== undefined && manifest.source.path === sourcePath) ||
424 - (suitePath !== undefined && manifest.suite?.path === suitePath))
426 - runIds.push(manifest.runId);
428 - if (runIds.length === 0) return undefined;
429 - return `${runIds.length} unfinished Overkill run${runIds.length === 1 ? "" : "s"} already target this skill or suite (${runIds.slice(0, 3).join(", ")}${runIds.length > 3 ? ", …" : ""}). They may be active in other Pi processes; parallel runs duplicate model cost, and suite edits can invalidate running evaluations.`;
432 -function statusMessage(run: RunEntry): string {
433 - if (
434 - run.kind === "suite" &&
435 - run.stage === "complete" &&
436 - run.terminal === "no_improvement" &&
437 - run.bestScore === 1
439 - return `Overkill: complete | completed: ${run.completed ?? 0}${run.total === undefined ? "" : `/${run.total}`} | outcome: baseline perfect; optimization skipped`;
440 - const fields = [`Overkill: ${run.stage}`];
441 - if (run.completed !== undefined)
442 - fields.push(
443 - `completed: ${run.completed}${run.total === undefined ? "" : `/${run.total}`}`,
444 - );
445 - if (run.currentScore !== undefined)
446 - fields.push(`current score: ${run.currentScore}`);
447 - if (run.bestScore !== undefined) fields.push(`best score: ${run.bestScore}`);
448 - fields.push(`terminal: ${run.terminal ?? "active"}`);
449 - if (run.stagedPath) fields.push(`staged: ${run.stagedPath}`);
450 - return fields.join(" | ");
453 -function commandMode(
454 - ctx: ExtensionCommandContext,
455 -): "tui" | "rpc" | "json" | "print" {
456 - return ctx.mode ?? "tui";
459 -function writePrint(line: string): void {
460 - const text = `${line}\n`;
461 - const seam = installedModeAcceptanceSeam();
462 - if (seam) seam.write(1, text);
463 - else writeSync(1, text);
466 -function appendProgressEntry(pi: ExtensionAPI, entry: RunEntry): void {
467 - pi.appendEntry("overkill-run", entry);
470 -function showProgress(ctx: ExtensionCommandContext, text: string): void {
471 - const mode = commandMode(ctx);
472 - if (mode === "rpc") ctx.ui.setStatus("overkill", text);
473 - if (mode === "tui")
474 - ctx.ui.setWidget("overkill-progress", [text], {
475 - placement: "belowEditor",
476 - });
479 -function appendProgress(
480 - pi: ExtensionAPI,
481 - ctx: ExtensionCommandContext,
482 - entry: RunEntry,
483 -): void {
484 - const mode = commandMode(ctx);
485 - const durable =
486 - entry.terminal !== null ||
487 - entry.stage === "preflight" ||
488 - entry.stage === "interrupted" ||
489 - (entry.kind === "aftermath" &&
490 - entry.stage === "mining" &&
491 - entry.completed === undefined);
492 - if (mode !== "tui" || durable) appendProgressEntry(pi, entry);
493 - const progress = formatProgress(entry);
494 - showProgress(ctx, progress);
495 - if (mode === "print") writePrint(progress);
498 -function clearProgress(ctx: ExtensionCommandContext): void {
499 - if (commandMode(ctx) === "rpc") ctx.ui.setStatus("overkill", undefined);
500 - if (commandMode(ctx) === "tui")
501 - ctx.ui.setWidget("overkill-progress", undefined);
504 -function requestCancellation(active: ActiveRun): void {
505 - if (active.cancelRequested) return;
506 - active.cancelRequested = true;
507 - active.controller.abort();
510 -function installCancellationListeners(
511 - active: ActiveRun,
512 - ctx: ExtensionCommandContext,
513 -): void {
514 - const mode = commandMode(ctx);
515 - const cleanup: Array<() => void> = [];
516 - if (mode === "tui")
517 - cleanup.push(
518 - ctx.ui.onTerminalInput((data) => {
519 - if (!matchesKey(data, Key.escape)) return undefined;
520 - requestCancellation(active);
521 - return { consume: true };
522 - }),
523 - );
524 - if (mode === "json" || mode === "print") {
525 - const onSignal = () => requestCancellation(active);
526 - process.once("SIGINT", onSignal);
527 - cleanup.push(() => process.removeListener("SIGINT", onSignal));
529 - active.removeCancellationListeners = () => {
530 - for (const remove of cleanup.splice(0)) remove();
531 - };
534 -async function withActiveRun(
535 - ctx: ExtensionCommandContext,
536 - operation: (active: ActiveRun) => Promise<void>,
537 - kind: ActiveRun["kind"] = "suite",
538 -): Promise<void> {
539 - if (activeRun) throw new Error("an Overkill rollout is already active");
540 - const active: ActiveRun = {
541 - kind,
542 - runId: "pending",
543 - sessionId: ctx.sessionManager.getSessionId(),
544 - controller: new AbortController(),
545 - suppressOutput: false,
546 - };
547 - activeRun = active;
548 - installCancellationListeners(active, ctx);
549 - const completed = (async () => {
550 - try {
551 - await operation(active);
552 - } catch (error) {
553 - if (active.controller.signal.aborted) {
554 - await active.interrupt?.();
555 - throw new RolloutCancelledError();
557 - try {
558 - await active.interrupt?.();
559 - } catch {
560 - // Preserve the original infrastructure failure.
562 - throw error;
563 - } finally {
564 - active.removeCancellationListeners?.();
565 - clearProgress(ctx);
566 - if (activeRun === active) activeRun = undefined;
568 - })();
569 - active.completed = completed;
570 - return await completed;
573 -async function checkpointInterrupted(
574 - directory: string,
575 - manifest: RunManifest,
576 -): Promise<void> {
577 - const training = await readTrainingCheckpoint(directory);
578 - if (training)
579 - await persistTrainingCheckpoint(directory, {
580 - ...training,
581 - stage: "interrupted",
582 - });
583 - manifest.stage = "interrupted";
584 - manifest.terminal = null;
585 - await persistManifest(directory, manifest);
588 -function throwIfCancelled(active: ActiveRun): void {
589 - if (active.controller.signal.aborted) throw new RolloutCancelledError();
592 -type CoreTaskAdapter = {
593 - items: (split: "train" | "validation" | "test") => readonly ValidatedItem[];
594 - scoreResponse: (options: {
595 - item: ValidatedItem;
596 - response: string;
597 - input: string;
598 - signal?: AbortSignal;
599 - }) => Promise<{ score: number; passed: boolean; feedback?: string }>;
600 - onTargetCall?: () => void | Promise<void>;
603 -async function executeBaseline(options: {
604 - pi: ExtensionAPI;
605 - ctx: ExtensionCommandContext;
606 - directory: string;
607 - manifest: RunManifest;
608 - source: ReturnType<typeof parseSourceSkill>;
609 - loadedSkill: Awaited<ReturnType<typeof resolveLoadedSkill>>;
610 - suite?: Awaited<ReturnType<typeof validateTrustedSuite>>;
611 - targetModel: unknown;
612 - taskAdapter?: CoreTaskAdapter;
613 - reportExtras?: () => Record<string, unknown>;
614 - deferCompletionNotice?: boolean;
615 -}): Promise<void> {
616 - const { ctx, manifest } = options;
617 - const itemsFor = (split: "train" | "validation" | "test") =>
618 - options.taskAdapter
619 - ? options.taskAdapter.items(split)
620 - : getValidatedSuiteItems(
621 - options.suite ??
622 - (() => {
623 - throw new Error("suite tasks are unavailable");
624 - })(),
625 - split === "validation" ? "val" : split,
626 - );
627 - const scorer = options.taskAdapter
628 - ? { scoreResponse: options.taskAdapter.scoreResponse }
629 - : { executeChecker };
630 - const active = activeRun;
631 - if (!active || active.sessionId !== ctx.sessionManager.getSessionId())
632 - throw new Error("Overkill run ownership is unavailable");
633 - active.runId = manifest.runId;
634 - active.interrupt = async () =>
635 - await checkpointInterrupted(options.directory, manifest);
636 - throwIfCancelled(active);
637 - const controller = active.controller;
638 - const acceptanceSeam =
639 - manifest.kind === "aftermath"
640 - ? installedAftermathAcceptanceSeam()
641 - : installedExplicitSuiteAcceptanceSeam();
642 - const recordTargetAttempt = async () => {
643 - if (options.taskAdapter?.onTargetCall) {
644 - await options.taskAdapter.onTargetCall();
645 - return;
647 - if (!options.suite || !manifest.suite)
648 - throw new Error("suite call contract is unavailable");
649 - const maximum = upperBoundCalls(options.suite.splitCounts).target;
650 - if (manifest.calls.target >= maximum)
651 - throw new Error("target call maximum exceeded");
652 - if (
653 - fingerprintSource(await readFile(manifest.source.path)) !==
654 - manifest.source.fingerprint
656 - throw new Error("source fingerprint changed");
657 - if (
658 - (await fingerprintSuite(options.suite.suitePath)) !==
659 - manifest.suite.fingerprint
661 - throw new Error("suite fingerprint changed");
662 - manifest.calls.target += 1;
663 - await persistManifest(options.directory, manifest);
664 - };
665 - let progress: Pick<
666 - RunEntry,
667 - "completed" | "total" | "currentScore" | "bestScore"
668 - > = {};
669 - const operation = (async () => {
670 - manifest.stage = "baseline";
671 - await persistManifest(options.directory, manifest);
672 - try {
673 - const result = await runValidationBaseline(
675 - directory: options.directory,
676 - cwd: ctx.cwd,
677 - agentDir: getAgentDir(),
678 - sourcePath: manifest.source.path,
679 - sourceFingerprint: manifest.source.fingerprint,
680 - source: options.source,
681 - loadedSkill: options.loadedSkill,
682 - model: manifest.model,
683 - suite: options.suite,
684 - items: itemsFor("validation"),
685 - signal: controller.signal,
686 - onTargetCall: recordTargetAttempt,
687 - onCheckerCall: async () => {
688 - if (options.taskAdapter) return;
689 - manifest.calls.checker += 1;
690 - await persistManifest(options.directory, manifest);
691 - },
692 - onProgress: async (next) => {
693 - progress = {
694 - completed: next.completed,
695 - total: next.total,
696 - ...(next.mean === undefined
697 - ? {}
698 - : { currentScore: next.mean, bestScore: next.mean }),
699 - };
700 - if (!active.suppressOutput)
701 - appendProgress(options.pi, ctx, runEntry({ manifest }, progress));
702 - },
703 - },
705 - createTarget: () =>
706 - createIsolatedTargetSession(
708 - cwd: ctx.cwd,
709 - agentDir: getAgentDir(),
710 - sourcePath: manifest.source.path,
711 - loadedSkill: options.loadedSkill,
712 - candidatePath: join(options.directory, "baseline-snapshot.md"),
713 - model: manifest.model,
714 - targetModel: options.targetModel,
715 - },
716 - acceptanceSeam?.targetSdk ?? defaultTargetSdk,
717 - ),
718 - ...scorer,
719 - },
720 - );
721 - manifest.stage = "baseline_complete";
722 - manifest.scores = {
723 - ...manifest.scores,
724 - baseline: result.mean,
725 - current: result.mean,
726 - best: result.mean,
727 - };
728 - await persistManifest(options.directory, manifest);
729 - if (controller.signal.aborted) throw new RolloutCancelledError();
730 - if (!active.suppressOutput)
731 - appendProgress(options.pi, ctx, runEntry({ manifest }, progress));
732 - if (result.mean === 1) {
733 - const reportPath = await persistFinalReport(options.directory, {
734 - version: 1,
735 - baselineScore: result.mean,
736 - bestScore: result.mean,
737 - accepted: 0,
738 - optimizerCalls: 0,
739 - targetCalls: manifest.calls.target,
740 - checkerCalls: manifest.calls.checker,
741 - terminal: "perfect",
742 - ...options.reportExtras?.(),
743 - });
744 - if (controller.signal.aborted) throw new RolloutCancelledError();
745 - manifest.stage = "complete";
746 - manifest.terminal = "no_improvement";
747 - manifest.artifacts.report = "report.json";
748 - await persistManifest(options.directory, manifest);
749 - if (!active.suppressOutput) {
750 - const entry = runEntry({ manifest }, progress, {
751 - reportPath,
752 - stagedPath: null,
753 - });
754 - appendProgress(options.pi, ctx, entry);
755 - const paths = formatResultPaths({
756 - sourcePath: manifest.source.path,
757 - reportPath,
758 - stagedPath: null,
759 - });
760 - if (commandMode(ctx) === "print")
761 - writePrint(
762 - `Baseline already scores 1; no strictly better candidate exists, so optimization was skipped.\n${paths}`,
763 - );
764 - else if (
765 - commandMode(ctx) !== "json" &&
766 - !options.deferCompletionNotice
768 - ctx.ui.notify(
769 - "Baseline already scores 1; no strictly better candidate exists, so optimization was skipped.",
770 - "info",
771 - );
773 - return;
775 - const baselineCache = await readScoreCache(options.directory);
776 - if (!baselineCache)
777 - throw new Error("validation baseline cache is unavailable");
778 - const sourceHash = createHash("sha256")
779 - .update(options.source.body, "utf8")
780 - .digest("hex");
781 - const saved = await readTrainingCheckpoint(options.directory);
782 - const readSnapshotBody = async (hash: string, filename: string) => {
783 - if (hash === sourceHash) return options.source.body;
784 - const snapshot = await readFile(
785 - join(options.directory, filename),
786 - "utf8",
787 - );
788 - if (!snapshot.startsWith(options.source.frontmatter))
789 - throw new Error("candidate snapshot frontmatter is unsafe");
790 - const body = snapshot.slice(options.source.frontmatter.length);
791 - if (createHash("sha256").update(body, "utf8").digest("hex") !== hash)
792 - throw new Error("candidate snapshot identity is unsafe");
793 - return body;
794 - };
795 - const readAttempt = async (filename: string): Promise<AttemptedPatch> => {
796 - const value: unknown = JSON.parse(
797 - await readFile(join(options.directory, filename), "utf8"),
798 - );
799 - if (typeof value !== "object" || value === null || Array.isArray(value))
800 - throw new Error("attempt artifact is malformed");
801 - const artifact = value as Record<string, unknown>;
802 - if (!Array.isArray(artifact.reports))
803 - throw new Error("attempt artifact reports are malformed");
804 - return {
805 - patch: parsePatch(artifact.patch),
806 - reports: artifact.reports as AttemptedPatch["reports"],
807 - ...(typeof artifact.validationScore === "number"
808 - ? { validationScore: artifact.validationScore }
809 - : {}),
810 - };
811 - };
812 - let latestTraining: DurableTrainingState;
813 - if (saved) {
814 - const attempts = await Promise.all(
815 - saved.attemptedPatchFiles.map(readAttempt),
816 - );
817 - const pendingArtifact = saved.pendingPatch
818 - ? await readAttempt(saved.pendingPatch.file)
819 - : undefined;
820 - latestTraining = {
821 - nextStep: saved.nextStep,
822 - body: await readSnapshotBody(
823 - saved.currentBodyHash,
824 - saved.currentSnapshot,
825 - ),
826 - bestBody: await readSnapshotBody(
827 - saved.bestBodyHash,
828 - saved.bestSnapshot,
829 - ),
830 - baselineScore: saved.baselineScore,
831 - currentScore: saved.currentScore,
832 - bestScore: saved.bestScore,
833 - accepted: saved.accepted,
834 - optimizerCalls: saved.optimizerCalls,
835 - consecutiveRejections: saved.consecutiveRejections,
836 - rejections: saved.rejections.map(({ edits, ...rejection }) => ({
837 - ...rejection,
838 - ...(edits === undefined
839 - ? {}
840 - : { edits: parsePatch({ edits }).edits }),
841 - })),
842 - attempts,
843 - ...(pendingArtifact && saved.pendingPatch
844 - ? {
845 - pendingPatch: {
846 - step: saved.pendingPatch.step,
847 - body: await readSnapshotBody(
848 - saved.pendingPatch.bodyHash,
849 - saved.currentSnapshot,
850 - ),
851 - patch: pendingArtifact.patch,
852 - },
854 - : {}),
855 - ...(saved.terminal === undefined ? {} : { terminal: saved.terminal }),
856 - ...(saved.finalTest === undefined
857 - ? {}
858 - : {
859 - finalTest: {
860 - body: await readSnapshotBody(
861 - saved.finalTest.bodyHash,
862 - saved.bestSnapshot,
863 - ),
864 - completedItemIds: saved.finalTest.completedItemIds,
865 - total: saved.finalTest.total,
866 - ...(saved.finalTest.score === undefined
867 - ? {}
868 - : { score: saved.finalTest.score }),
869 - },
870 - ...(saved.finalTest.score === undefined
871 - ? {}
872 - : { finalTestScore: saved.finalTest.score }),
873 - }),
874 - ...(saved.pendingOptimizer === undefined
875 - ? {}
876 - : { pendingOptimizer: saved.pendingOptimizer }),
877 - };
878 - } else
879 - latestTraining = createTrainingState({
880 - body: options.source.body,
881 - baseline: baselineCache.scores,
882 - });
883 - const checkpointTraining = async (state: DurableTrainingState) => {
884 - latestTraining = state;
885 - if (state.pendingPatch)
886 - await writePrivateText(
887 - options.directory,
888 - `attempt-${state.pendingPatch.step}.json`,
889 - `${JSON.stringify({ patch: state.pendingPatch.patch, reports: [] })}\n`,
890 - );
891 - for (const [index, attempt] of state.attempts.entries())
892 - await writePrivateText(
893 - options.directory,
894 - `attempt-${index + 1}.json`,
895 - `${JSON.stringify(attempt)}\n`,
896 - );
897 - const snapshotName = (body: string) => {
898 - const hash = createHash("sha256").update(body, "utf8").digest("hex");
899 - return hash === sourceHash
900 - ? "baseline-snapshot.md"
901 - : `candidate-${hash}.md`;
902 - };
903 - const stage =
904 - state.terminal &&
905 - state.accepted > 0 &&
906 - state.finalTestScore === undefined
907 - ? "final_test"
908 - : "training";
909 - await persistTrainingCheckpoint(options.directory, {
910 - version: 1,
911 - stage,
912 - nextStep: state.nextStep,
913 - currentBodyHash: createHash("sha256")
914 - .update(state.body, "utf8")
915 - .digest("hex"),
916 - bestBodyHash: createHash("sha256")
917 - .update(state.bestBody, "utf8")
918 - .digest("hex"),
919 - currentSnapshot: snapshotName(state.body),
920 - bestSnapshot: snapshotName(state.bestBody),
921 - ...(state.terminal === undefined ? {} : { terminal: state.terminal }),
922 - baselineScore: state.baselineScore,
923 - currentScore: state.currentScore,
924 - bestScore: state.bestScore,
925 - accepted: state.accepted,
926 - optimizerCalls: state.optimizerCalls,
927 - consecutiveRejections: state.consecutiveRejections,
928 - rejections: state.rejections.map(
929 - ({ reason, delta, reports, edits }) => ({
930 - reason,
931 - delta,
932 - ...(reports === undefined ? {} : { reports }),
933 - ...(edits === undefined ? {} : { edits }),
934 - }),
935 - ),
936 - attemptedPatchFiles: state.attempts.map(
937 - (_, index) => `attempt-${index + 1}.json`,
938 - ),
939 - ...(state.pendingPatch
940 - ? {
941 - pendingPatch: {
942 - step: state.pendingPatch.step,
943 - bodyHash: createHash("sha256")
944 - .update(state.pendingPatch.body, "utf8")
945 - .digest("hex"),
946 - file: `attempt-${state.pendingPatch.step}.json`,
947 - },
949 - : {}),
950 - ...(state.pendingOptimizer === undefined
951 - ? {}
952 - : { pendingOptimizer: state.pendingOptimizer }),
953 - ...(state.finalTest
954 - ? {
955 - finalTest: {
956 - bodyHash: createHash("sha256")
957 - .update(state.finalTest.body, "utf8")
958 - .digest("hex"),
959 - completedItemIds: state.finalTest.completedItemIds,
960 - total: state.finalTest.total,
961 - ...(state.finalTest.score === undefined
962 - ? {}
963 - : { score: state.finalTest.score }),
964 - },
966 - : {}),
967 - });
968 - manifest.stage = stage;
969 - manifest.scores = {
970 - ...manifest.scores,
971 - current: state.currentScore,
972 - best: state.bestScore,
973 - ...(state.finalTestScore === undefined
974 - ? {}
975 - : { test: state.finalTestScore }),
976 - };
977 - manifest.calls.optimizer = state.optimizerCalls;
978 - await persistManifest(options.directory, manifest);
979 - if (!active.suppressOutput)
980 - appendProgress(
981 - options.pi,
982 - ctx,
983 - runEntry(
984 - { manifest },
986 - ...(state.finalTest
987 - ? {
988 - completed: state.finalTest.completedItemIds.length,
989 - total: state.finalTest.total,
991 - : {}),
992 - currentScore: state.currentScore,
993 - bestScore: state.bestScore,
994 - },
995 - ),
996 - );
997 - };
998 - const training = await runDurableTrainingSchedule(latestTraining, {
999 - evaluate: async (body, split, onProgress) =>
1000 - await splitScores(split, body, onProgress),
1001 - optimize: async (input, onCall, onProgress) =>
1002 - await runOptimizerStep(
1003 - input,
1005 - createSession: () =>
1006 - createOptimizerSession(
1008 - cwd: ctx.cwd,
1009 - agentDir: getAgentDir(),
1010 - model: manifest.model,
1011 - targetModel: options.targetModel,
1012 - },
1013 - acceptanceSeam?.optimizerSdk ?? defaultOptimizerSdk,
1014 - ),
1015 - },
1016 - controller.signal,
1017 - latestTraining.pendingOptimizer,
1018 - onCall,
1019 - onProgress,
1020 - ),
1021 - checkpoint: checkpointTraining,
1022 - testItemIds: async () => itemsFor("test").map((item) => item.id),
1023 - signal: controller.signal,
1024 - });
1025 - if (controller.signal.aborted) throw new RolloutCancelledError();
1026 - const reportPath = await persistFinalReport(options.directory, {
1027 - version: 1,
1028 - baselineScore: training.baselineScore,
1029 - bestScore: training.bestScore,
1030 - accepted: training.accepted,
1031 - optimizerCalls: training.optimizerCalls,
1032 - targetCalls: manifest.calls.target,
1033 - checkerCalls: manifest.calls.checker,
1034 - terminal: training.terminal,
1035 - rejections: training.rejections,
1036 - attempts: training.attempts,
1037 - ...(training.testScore === undefined
1038 - ? {}
1039 - : { testScore: training.testScore }),
1040 - ...options.reportExtras?.(),
1041 - });
1042 - const stagedPath =
1043 - training.accepted > 0
1044 - ? await writePrivateText(
1045 - options.directory,
1046 - "staged-skill.md",
1047 - `${options.source.frontmatter}${training.bestBody}`,
1049 - : null;
1050 - await checkpointTraining(latestTraining);
1051 - if (controller.signal.aborted) throw new RolloutCancelledError();
1052 - const completeCheckpoint = await readTrainingCheckpoint(
1053 - options.directory,
1054 - );
1055 - if (!completeCheckpoint)
1056 - throw new Error("training checkpoint is unavailable");
1057 - await persistTrainingCheckpoint(options.directory, {
1058 - ...completeCheckpoint,
1059 - stage: "complete",
1060 - reportFile: "report.json",
1061 - ...(stagedPath === null ? {} : { stagedFile: "staged-skill.md" }),
1062 - });
1063 - manifest.stage = "complete";
1064 - manifest.terminal = training.accepted > 0 ? "staged" : "no_improvement";
1065 - manifest.scores = {
1066 - ...manifest.scores,
1067 - best: training.bestScore,
1068 - ...(training.testScore === undefined
1069 - ? {}
1070 - : { test: training.testScore }),
1071 - };
1072 - manifest.calls.optimizer = training.optimizerCalls;
1073 - manifest.artifacts.report = "report.json";
1074 - manifest.artifacts.staged =
1075 - stagedPath === null ? null : "staged-skill.md";
1076 - await persistManifest(options.directory, manifest);
1077 - if (controller.signal.aborted) throw new RolloutCancelledError();
1078 - if (!active.suppressOutput) {
1079 - const entry = runEntry(
1080 - { manifest },
1081 - { currentScore: training.bestScore, bestScore: training.bestScore },
1082 - { reportPath, stagedPath },
1083 - );
1084 - appendProgress(options.pi, ctx, entry);
1085 - const paths = formatResultPaths({
1086 - sourcePath: manifest.source.path,
1087 - reportPath,
1088 - stagedPath,
1089 - });
1090 - if (commandMode(ctx) === "print") writePrint(paths);
1091 - else if (commandMode(ctx) !== "json" && !options.deferCompletionNotice)
1092 - ctx.ui.notify(paths, "info");
1095 - async function splitScores(
1096 - split: "train" | "validation" | "test",
1097 - body: string,
1098 - onProgress?: (progress: {
1099 - completed: number;
1100 - total: number;
1101 - }) => Promise<void>,
1102 - ) {
1103 - const hash = createHash("sha256").update(body, "utf8").digest("hex");
1104 - const rollout = await runSplitRollout(
1106 - directory: options.directory,
1107 - cwd: ctx.cwd,
1108 - agentDir: getAgentDir(),
1109 - sourcePath: manifest.source.path,
1110 - sourceFingerprint: manifest.source.fingerprint,
1111 - source: options.source,
1112 - loadedSkill: options.loadedSkill,
1113 - model: manifest.model,
1114 - suite: options.suite,
1115 - items: itemsFor(split),
1116 - body,
1117 - split,
1118 - signal: controller.signal,
1119 - onProgress,
1120 - onTargetCall: recordTargetAttempt,
1121 - onCheckerCall: async () => {
1122 - if (options.taskAdapter) return;
1123 - manifest.calls.checker += 1;
1124 - await persistManifest(options.directory, manifest);
1125 - },
1126 - },
1128 - createTarget: () =>
1129 - createIsolatedTargetSession(
1131 - cwd: ctx.cwd,
1132 - agentDir: getAgentDir(),
1133 - sourcePath: manifest.source.path,
1134 - loadedSkill: options.loadedSkill,
1135 - candidatePath: join(
1136 - options.directory,
1137 - `candidate-${hash}.md`,
1138 - ),
1139 - model: manifest.model,
1140 - targetModel: options.targetModel,
1141 - },
1142 - acceptanceSeam?.targetSdk ?? defaultTargetSdk,
1143 - ),
1144 - ...scorer,
1145 - },
1146 - );
1147 - return rollout.scores;
1149 - } catch (error) {
1150 - if (
1151 - error instanceof RolloutCancelledError ||
1152 - error instanceof OptimizerCancelledError ||
1153 - controller.signal.aborted
1154 - ) {
1155 - await checkpointInterrupted(options.directory, manifest);
1156 - const entry = runEntry({ manifest }, progress);
1157 - if (active.suppressOutput) appendProgressEntry(options.pi, entry);
1158 - else appendProgress(options.pi, ctx, entry);
1159 - throw new RolloutCancelledError();
1161 - throw error;
1163 - })();
1164 - return await operation;
1167 -export async function harvestAftermath(options: {
1168 - ctx: ExtensionCommandContext;
1169 - knownSkillPaths: ReadonlyMap<string, string>;
1170 - sessionApi?: AftermathSessionApi;
1171 -}): Promise<{ digests: Digest[]; total: number; unreadable: number }> {
1172 - const api =
1173 - options.sessionApi ??
1174 - (SessionManager as unknown as NonNullable<typeof options.sessionApi>);
1175 - const sessionDir = options.ctx.sessionManager.getSessionDir();
1176 - const listed = await api.listAll(sessionDir);
1177 - const snapshots = new Map<string, Digest>();
1178 - const unreadableIds = new Set<string>();
1179 - for (const listedSession of listed) {
1180 - try {
1181 - const session = api.open(listedSession.path, sessionDir);
1182 - const id = session.getSessionId();
1183 - snapshots.set(
1184 - id,
1185 - buildDigest({
1186 - sessionId: id,
1187 - cwd: session.getCwd(),
1188 - branch: session.getBranch(),
1189 - knownSkillPaths: options.knownSkillPaths,
1190 - }),
1191 - );
1192 - } catch {
1193 - unreadableIds.add(listedSession.id);
1196 - const current = options.ctx.sessionManager;
1197 - const currentId = current.getSessionId();
1198 - snapshots.set(
1199 - currentId,
1200 - buildDigest({
1201 - sessionId: currentId,
1202 - cwd: options.ctx.cwd,
1203 - branch: current.getBranch(),
1204 - knownSkillPaths: options.knownSkillPaths,
1205 - }),
1206 - );
1207 - unreadableIds.delete(currentId);
1208 - const currentWasListed = listed.some((session) => session.id === currentId);
1209 - return {
1210 - digests: [...snapshots.values()].sort((left, right) =>
1211 - left.sessionId.localeCompare(right.sessionId),
1212 - ),
1213 - total: listed.length + (currentWasListed ? 0 : 1),
1214 - unreadable: unreadableIds.size,
1215 - };
1218 -type LoadedSkillTarget = { name: string; path: string };
1220 -function loadedSkills(pi: ExtensionAPI): LoadedSkillTarget[] {
1221 - return pi
1222 - .getCommands()
1223 - .flatMap((command) => {
1224 - if (
1225 - command.source !== "skill" ||
1226 - typeof command.sourceInfo?.path !== "string"
1228 - return [];
1229 - return [
1231 - name: command.name.replace(/^skill:/, ""),
1232 - path: command.sourceInfo.path,
1233 - },
1234 - ];
1235 - })
1236 - .sort(
1237 - (left, right) =>
1238 - left.name.localeCompare(right.name) ||
1239 - left.path.localeCompare(right.path),
1240 - );
1243 -export async function chooseLoadedSkill(
1244 - ctx: ExtensionCommandContext,
1245 - skills: readonly LoadedSkillTarget[],
1246 - canonicalPaths: ReadonlyMap<string, string>,
1247 - purpose: string,
1248 -): Promise<LoadedSkillTarget | undefined> {
1249 - if (commandMode(ctx) !== "tui" || skills.length <= 8) {
1250 - const labels = skills.map(
1251 - (skill) =>
1252 - `${skill.name} — ${canonicalPaths.get(skill.path) ?? skill.path}`,
1253 - );
1254 - const choice = await ctx.ui.select(
1255 - `Choose exactly one ${purpose} skill.`,
1256 - labels,
1257 - );
1258 - return skills[labels.indexOf(choice ?? "")];
1261 - const path = await ctx.ui.custom<string | null>(
1262 - (tui, theme, _keybindings, done) => {
1263 - const list = new SelectList(
1264 - skills.map((skill) => {
1265 - const canonical = canonicalPaths.get(skill.path) ?? skill.path;
1266 - const parent = dirname(canonical);
1267 - return {
1268 - value: skill.path,
1269 - label: skill.name,
1270 - description: join(
1271 - "…",
1272 - basename(dirname(parent)),
1273 - basename(parent),
1274 - basename(canonical),
1275 - ),
1276 - };
1277 - }),
1278 - Math.max(1, Math.min(8, tui.terminal.rows - 3)),
1280 - selectedPrefix: (text) => theme.fg("accent", text),
1281 - selectedText: (text) => theme.fg("accent", text),
1282 - description: (text) => theme.fg("muted", text),
1283 - scrollInfo: (text) => theme.fg("dim", text),
1284 - noMatch: (text) => theme.fg("warning", text),
1285 - },
1286 - );
1287 - list.onSelect = (item) => done(item.value);
1288 - list.onCancel = () => done(null);
1289 - return {
1290 - render: (width: number) => [
1291 - theme.fg("accent", `Choose one ${purpose} skill`),
1292 - ...list.render(width),
1293 - theme.fg("dim", "type filter · arrows move · enter · esc"),
1294 - ],
1295 - invalidate: () => list.invalidate(),
1296 - handleInput: (data: string) => {
1297 - list.handleInput(data);
1298 - tui.requestRender();
1299 - },
1300 - };
1301 - },
1302 - );
1303 - return skills.find((skill) => skill.path === path);
1306 -async function runAftermath(
1307 - pi: ExtensionAPI,
1308 - ctx: ExtensionCommandContext,
1309 - parts: string[],
1310 -): Promise<void> {
1311 - const active = activeRun;
1312 - if (active?.kind !== "aftermath")
1313 - throw new Error("Aftermath run ownership is unavailable");
1314 - throwIfCancelled(active);
1315 - const explicitPath = parts[0];
1316 - if (parts.length > 1)
1317 - throw new Error("Usage: /overkill aftermath [skill-path]");
1318 - const interrupted = selectedAftermathRun(ctx);
1319 - if (interrupted) {
1320 - const directory = join(
1321 - getAgentDir(),
1322 - "overkill-runs",
1323 - interrupted.data.runId,
1324 - );
1325 - const manifest = await readAftermathManifest(directory).catch(
1326 - () => undefined,
1327 - );
1328 - if (manifest && !manifest.terminal) {
1329 - const instructions =
1330 - "An incomplete Aftermath run is linked here; use /overkill aftermath resume, discard, or cancel.";
1331 - if (explicitPath) throw new Error(instructions);
1332 - if (ctx.hasUI) {
1333 - const choice = await ctx.ui.select("Resume incomplete Aftermath run?", [
1334 - "resume",
1335 - "discard",
1336 - "cancel",
1337 - ]);
1338 - if (choice === "resume") {
1339 - await resumeAftermath(pi, ctx, interrupted);
1340 - return;
1342 - if (choice === "discard") {
1343 - await discardLinkedRun({
1344 - runRoot: join(getAgentDir(), "overkill-runs"),
1345 - runDirectory: directory,
1346 - selected: interrupted,
1347 - branch: ctx.sessionManager.getBranch(),
1348 - sessionId: ctx.sessionManager.getSessionId(),
1349 - allowNonterminal: true,
1350 - isRunActive: (runId) => activeRun?.runId === runId,
1351 - });
1352 - appendProgress(pi, ctx, {
1353 - ...interrupted.data,
1354 - stage: "discarded",
1355 - terminal: "discarded",
1356 - });
1358 - return;
1360 - throw new Error(instructions);
1363 - if (
1364 - (commandMode(ctx) === "json" || commandMode(ctx) === "print") &&
1365 - !explicitPath
1367 - throw new Error(
1368 - "JSON and print Aftermath runs require an explicit skill path",
1369 - );
1370 - const skills = loadedSkills(pi);
1371 - if (skills.length === 0) {
1372 - ctx.ui.notify("No loaded Pi skills are available for Aftermath.", "info");
1373 - return;
1375 - const knownPaths = new Map<string, string>();
1376 - const canonicalSkillPaths = new Map<string, string>();
1377 - for (const skill of skills) {
1378 - try {
1379 - const canonical = await realpath(skill.path);
1380 - knownPaths.set(canonical, skill.name);
1381 - canonicalSkillPaths.set(skill.path, canonical);
1382 - } catch {}
1384 - const canonicalExplicit = explicitPath
1385 - ? await realpath(resolve(ctx.cwd, explicitPath)).catch(() => undefined)
1386 - : undefined;
1387 - let selected = explicitPath
1388 - ? skills.find(
1389 - (skill) =>
1390 - canonicalExplicit !== undefined &&
1391 - canonicalSkillPaths.get(skill.path) === canonicalExplicit,
1393 - : undefined;
1394 - if (explicitPath && !selected)
1395 - throw new Error("explicit Aftermath skill path is not a loaded skill");
1396 - if (!selected && ctx.hasUI)
1397 - selected = await chooseLoadedSkill(
1398 - ctx,
1399 - skills,
1400 - canonicalSkillPaths,
1401 - "Aftermath",
1402 - );
1403 - if (!selected) throw new Error("one confirmed Aftermath skill is required");
1404 - throwIfCancelled(active);
1405 - const sourcePath =
1406 - canonicalSkillPaths.get(selected.path) ?? (await realpath(selected.path));
1407 - const parallelWarning = await unfinishedRunWarning(sourcePath);
1408 - const harvested = await harvestAftermath({
1409 - ctx,
1410 - knownSkillPaths: knownPaths,
1411 - sessionApi: installedAftermathAcceptanceSeam()?.sessionApi,
1412 - });
1413 - throwIfCancelled(active);
1414 - const evidence = rankSkillEvidence(
1415 - harvested.digests,
1416 - new Set(skills.map((skill) => skill.name)),
1417 - );
1418 - const calls = estimateMiningCalls(harvested.digests.length);
1419 - const confirmation = [
1420 - `Readable sessions: ${harvested.digests.length}; unreadable: ${harvested.unreadable}.`,
1421 - "Each digest is capped at 16 KiB, persisted, and sent to the model without redaction.",
1422 - `Expected mining/aggregation calls: ${calls.mining}/${calls.aggregation}; malformed structured output retries once per call.`,
1423 - `Maximum mining/aggregation calls with retries: ${calls.mining * 2}/${calls.aggregation * 2}.`,
1424 - "Maximum replay calls after mining: 116 target, 116 rubric-judge, 12 optimizer.",
1425 - ...(parallelWarning ? [`Warning: ${parallelWarning}`] : []),
1426 - "Overkill will not write the source skill.",
1427 - ].join("\n");
1428 - if (
1429 - ctx.hasUI &&
1430 - !(await ctx.ui.confirm("Persist and mine Aftermath digests?", confirmation))
1432 - return;
1433 - throwIfCancelled(active);
1434 - const frozenModel = ctx.model;
1435 - const targetModel =
1436 - frozenModel && ctx.modelRegistry.find(frozenModel.provider, frozenModel.id);
1437 - if (!frozenModel || !targetModel || !(await modelHasAuth(ctx)))
1438 - throw new Error("selected model authentication is unavailable");
1439 - const sourceBytes = await readFile(sourcePath);
1440 - throwIfCancelled(active);
1441 - parseSourceSkill(sourceBytes);
1442 - await resolveLoadedSkill(
1443 - sourcePath,
1444 - pi.getCommands(),
1445 - ctx.isProjectTrusted(),
1446 - ctx.cwd,
1447 - );
1448 - throwIfCancelled(active);
1450 - const created = await createAftermathDirectory(getAgentDir());
1451 - active.runId = created.runId;
1452 - const manifest: AftermathManifest = {
1453 - version: 1 as const,
1454 - runId: created.runId,
1455 - kind: "aftermath" as const,
1456 - source: {
1457 - path: sourcePath,
1458 - fingerprint: fingerprintSource(sourceBytes),
1459 - },
1460 - model: {
1461 - provider: frozenModel.provider,
1462 - id: frozenModel.id,
1463 - thinking: pi.getThinkingLevel(),
1464 - },
1465 - sessionId: ctx.sessionManager.getSessionId(),
1466 - linkEntryId: null,
1467 - stage: "harvest",
1468 - miningPromptVersion: 2,
1469 - terminal: null,
1470 - scores: { baseline: null, current: null, best: null, test: null },
1471 - calls: { target: 0, checker: 0, optimizer: 0, judge: 0 },
1472 - artifacts: {
1473 - baselineSnapshot: "baseline-snapshot.md",
1474 - report: null,
1475 - staged: null,
1476 - },
1477 - target: { name: selected.name, path: sourcePath },
1478 - };
1479 - active.interrupt = async () =>
1480 - await persistAftermathManifest(created.directory, {
1481 - ...manifest,
1482 - stage: "interrupted",
1483 - });
1484 - await writePrivateText(
1485 - created.directory,
1486 - "baseline-snapshot.md",
1487 - sourceBytes,
1488 - );
1489 - const persisted = await persistHarvest(created.directory, {
1490 - digests: harvested.digests,
1491 - unreadable: harvested.unreadable,
1492 - total: harvested.total,
1493 - evidence: evidence.counts,
1494 - });
1495 - manifest.stage = "mining";
1496 - await persistAftermathManifest(created.directory, {
1497 - ...manifest,
1498 - harvestFingerprint: persisted.fingerprint,
1499 - });
1500 - appendProgress(pi, ctx, runEntry({ manifest }));
1501 - const link = ctx.sessionManager.getBranch().at(-1);
1502 - if (!link || typeof link.id !== "string")
1503 - throw new Error("could not persist Aftermath branch link");
1504 - manifest.linkEntryId = link.id;
1505 - await persistAftermathManifest(created.directory, {
1506 - ...manifest,
1507 - harvestFingerprint: persisted.fingerprint,
1508 - });
1509 - await mineAftermath({
1510 - pi,
1511 - ctx,
1512 - directory: created.directory,
1513 - manifest,
1514 - digests: harvested.digests,
1515 - targetModel,
1516 - harvestFingerprint: persisted.fingerprint,
1517 - });
1521 -async function offerAftermathCompletion(
1522 - pi: ExtensionAPI,
1523 - ctx: ExtensionCommandContext,
1524 - directory: string,
1525 -): Promise<void> {
1526 - if (!ctx.hasUI || (commandMode(ctx) !== "tui" && commandMode(ctx) !== "rpc"))
1527 - return;
1528 - const artifacts = await readFrozenReviewArtifacts(directory);
1529 - const manifest = await readAftermathManifest(directory);
1530 - const staged = artifacts.stagedPath !== null && artifacts.staged !== null;
1531 - const report = artifacts.reportPath
1532 - ? `Report: ${sanitizeTerminalText(artifacts.reportPath)}`
1533 - : "";
1534 - let summary: string;
1535 - if (staged) {
1536 - const scores = [
1537 - manifest.scores.baseline === null
1538 - ? null
1539 - : `baseline ${formatScore(manifest.scores.baseline)}`,
1540 - manifest.scores.best === null
1541 - ? null
1542 - : `validation ${formatScore(manifest.scores.best)}`,
1543 - manifest.scores.test === null
1544 - ? null
1545 - : `held-out ${formatScore(manifest.scores.test)}`,
1546 - ].filter((value): value is string => value !== null);
1547 - summary = `Aftermath complete — candidate staged.\n${scores.join(" → ")}\n${report}`;
1548 - } else if (manifest.scores.baseline === null) {
1549 - const frozen = await readFrozenTasks(directory);
1550 - summary = `Aftermath complete — no candidate.\nOnly ${frozen.tasks.length}/3 recurring tasks; replay was skipped.\n${report}`;
1551 - } else if (manifest.scores.baseline === 1) {
1552 - summary = `Aftermath complete — no candidate needed.\nBaseline already scores 1; no improvement is possible.\n${report}`;
1553 - } else {
1554 - summary = `Aftermath complete — no candidate.\nBest validation ${formatScore(manifest.scores.best ?? manifest.scores.baseline)} did not beat baseline ${formatScore(manifest.scores.baseline)}; no diff was staged.\n${report}`;
1556 - const choice = await ctx.ui.select(
1557 - summary,
1558 - staged ? ["review diff", "done", "discard"] : ["done", "discard report"],
1559 - );
1560 - if (choice === "review diff") {
1561 - const paths = [
1562 - artifacts.reportPath
1563 - ? `Report: ${sanitizeTerminalText(artifacts.reportPath)}`
1564 - : null,
1565 - artifacts.stagedPath
1566 - ? `Staged: ${sanitizeTerminalText(artifacts.stagedPath)}`
1567 - : null,
1569 - .filter((value): value is string => value !== null)
1570 - .join("\n");
1571 - ctx.ui.notify(
1572 - commandMode(ctx) === "tui" && artifacts.stagedPath && artifacts.staged
1573 - ? `${paths}\n${buildReviewDiff(
1574 - artifacts.sourcePath,
1575 - artifacts.stagedPath,
1576 - artifacts.source,
1577 - artifacts.staged,
1578 - )}`
1579 - : paths,
1580 - "info",
1581 - );
1582 - return;
1584 - if (choice === "done" || choice === undefined) return;
1585 - if (choice === "discard" || choice === "discard report") {
1586 - const selected = selectedAftermathRun(ctx);
1587 - if (!selected)
1588 - throw new Error("Aftermath run is no longer linked to this branch");
1589 - await discardLinkedRun({
1590 - runRoot: join(getAgentDir(), "overkill-runs"),
1591 - runDirectory: directory,
1592 - selected,
1593 - branch: ctx.sessionManager.getBranch(),
1594 - sessionId: ctx.sessionManager.getSessionId(),
1595 - });
1596 - appendProgress(pi, ctx, {
1597 - ...selected.data,
1598 - stage: "discarded",
1599 - terminal: "discarded",
1600 - });
1604 -async function mineAftermath(options: {
1605 - pi: ExtensionAPI;
1606 - ctx: ExtensionCommandContext;
1607 - directory: string;
1608 - manifest: AftermathManifest;
1609 - digests: Digest[];
1610 - targetModel: unknown;
1611 - harvestFingerprint: string;
1612 - completedChunks?: ReadonlyMap<number, unknown[]>;
1613 - completedLevels?: ReadonlyMap<number, unknown[][]>;
1614 - attemptedMiningCalls?: number;
1615 - frozen?: Awaited<ReturnType<typeof readFrozenTasks>>;
1616 -}): Promise<void> {
1617 - const active = activeRun;
1618 - if (active?.kind !== "aftermath")
1619 - throw new Error("Aftermath run ownership is unavailable");
1620 - const { ctx, manifest } = options;
1621 - if (manifest.miningPromptVersion !== 2)
1622 - throw new Error(
1623 - "Aftermath run predates target-relevant mining; discard it and start a new run",
1624 - );
1625 - const frozenSource = await readFile(
1626 - join(options.directory, "baseline-snapshot.md"),
1627 - );
1628 - if (fingerprintSource(frozenSource) !== manifest.source.fingerprint)
1629 - throw new Error("frozen source fingerprint changed; replay is unsafe");
1630 - const source = parseSourceSkill(frozenSource);
1631 - if (source.name !== manifest.target.name)
1632 - throw new Error("frozen source name changed; replay is unsafe");
1633 - manifest.stage = "mining";
1634 - active.interrupt = async () => {
1635 - const interrupted = {
1636 - ...manifest,
1637 - stage: "interrupted",
1638 - harvestFingerprint: options.harvestFingerprint,
1639 - };
1640 - await persistAftermathManifest(options.directory, interrupted);
1641 - const entry = runEntry({ manifest: interrupted });
1642 - if (active.suppressOutput) appendProgressEntry(options.pi, entry);
1643 - else appendProgress(options.pi, ctx, entry);
1644 - };
1645 - const chunks = new Set(options.completedChunks?.keys());
1646 - const levels = new Set<string>();
1647 - let attemptedMiningCalls = options.attemptedMiningCalls ?? 0;
1648 - for (const [level, outputs] of options.completedLevels ?? [])
1649 - for (const index of outputs.keys()) levels.add(`${level}:${index}`);
1650 - const miningTotal = estimateMiningCalls(options.digests.length).total;
1651 - const updateMiningProgress = (
1652 - phase?: "mining" | "aggregation",
1653 - attempt?: number,
1654 - maximumAttempts?: number,
1655 - ) => {
1656 - const completed = chunks.size + levels.size;
1657 - const progress = formatProgress({
1658 - stage: "mining",
1659 - completed,
1660 - total: miningTotal,
1661 - });
1662 - showProgress(
1663 - ctx,
1664 - phase
1665 - ? `${progress} · ${phase} call ${Math.min(completed + 1, miningTotal)} running${attempt && attempt > 1 ? ` · retry ${attempt}/${maximumAttempts}` : ""}`
1666 - : progress,
1667 - );
1668 - };
1669 - const persistCheckpoint = async () =>
1670 - await persistMiningCheckpoint(options.directory, {
1671 - attemptedCalls: attemptedMiningCalls,
1672 - completedChunks: [...chunks],
1673 - completedLevels: [...levels].map((item) => {
1674 - const [levelText, indexText] = item.split(":");
1675 - return { level: Number(levelText), index: Number(indexText) };
1676 - }),
1677 - });
1678 - const frozen =
1679 - options.frozen ??
1680 - (await (async () => {
1681 - updateMiningProgress();
1682 - const proposals = await mineTaskProposals({
1683 - digests: options.digests,
1684 - target: { name: source.name, description: source.description },
1685 - createSession: () =>
1686 - createMiningSession(
1688 - cwd: ctx.cwd,
1689 - agentDir: getAgentDir(),
1690 - model: manifest.model,
1691 - targetModel: options.targetModel,
1692 - },
1693 - installedAftermathAcceptanceSeam()?.structuredOutputSdk,
1694 - ),
1695 - signal: active.controller.signal,
1696 - completedChunks: options.completedChunks,
1697 - completedLevels: options.completedLevels,
1698 - onCall: async (phase, attempt, maximumAttempts) => {
1699 - attemptedMiningCalls += 1;
1700 - await persistCheckpoint();
1701 - updateMiningProgress(phase, attempt, maximumAttempts);
1702 - },
1703 - onChunk: async (index, output) => {
1704 - await persistMiningOutput(
1705 - options.directory,
1706 - `miner-chunk-${index}.json`,
1707 - output,
1708 - );
1709 - chunks.add(index);
1710 - await persistCheckpoint();
1711 - updateMiningProgress();
1712 - },
1713 - onLevel: async (level, index, output) => {
1714 - await persistMiningOutput(
1715 - options.directory,
1716 - `aggregate-${level}-${index}.json`,
1717 - output,
1718 - );
1719 - levels.add(`${level}:${index}`);
1720 - await persistCheckpoint();
1721 - updateMiningProgress();
1722 - },
1723 - });
1724 - throwIfCancelled(active);
1725 - const tasks = validateAndFreezeTasks({
1726 - proposals,
1727 - harvestedSessionIds: new Set(
1728 - options.digests.map((digest) => digest.sessionId),
1729 - ),
1730 - });
1731 - const splits = assignSplits(tasks);
1732 - const persisted = await persistFrozenTasks(
1733 - options.directory,
1734 - tasks,
1735 - splits,
1736 - );
1737 - return { ...persisted, tasks, splits };
1738 - })());
1739 - const { tasks, splits } = frozen;
1740 - if (
1741 - manifest.taskFingerprint !== undefined &&
1742 - manifest.taskFingerprint !== frozen.fingerprint
1744 - throw new Error("frozen task fingerprint changed; resume is unsafe");
1745 - manifest.harvestFingerprint = options.harvestFingerprint;
1746 - manifest.taskFingerprint = frozen.fingerprint;
1747 - const callBounds = upperBoundCalls({
1748 - train: splits.train.length,
1749 - val: splits.validation.length,
1750 - test: splits.test.length,
1751 - });
1752 - const taskById = new Map(tasks.map((task) => [task.id, task]));
1753 - const rubricCount = (split: "train" | "validation" | "test") =>
1754 - splits[split].filter((id) => taskById.get(id)?.rubric !== undefined).length;
1755 - const judgeMaximum =
1756 - rubricCount("validation") +
1757 - 3 * (rubricCount("train") + rubricCount("validation")) +
1758 - rubricCount("test");
1759 - manifest.scorers = tasks.map((task) => ({
1760 - id: task.id,
1761 - kind: task.checks ? ("deterministic" as const) : ("rubric" as const),
1762 - }));
1763 - manifest.splits = splits;
1764 - let costs = (await readAftermathCost(options.directory)) ?? {
1765 - targetMaximum: tasks.length < 3 ? 0 : callBounds.target,
1766 - judgeMaximum: tasks.length < 3 ? 0 : judgeMaximum,
1767 - targetConsumed: 0,
1768 - judgeConsumed: 0,
1769 - };
1770 - if (
1771 - costs.targetMaximum !== (tasks.length < 3 ? 0 : callBounds.target) ||
1772 - costs.judgeMaximum !== (tasks.length < 3 ? 0 : judgeMaximum)
1774 - throw new Error("Aftermath replay call maximum changed; resume is unsafe");
1775 - manifest.calls.target = costs.targetConsumed;
1776 - manifest.calls.judge = costs.judgeConsumed;
1777 - await persistAftermathCost(options.directory, costs);
1778 - const callNotice = `Replay budget: ≤${costs.targetMaximum} target + ${costs.judgeMaximum} judge calls; malformed judge retries consume this budget.`;
1779 - if (commandMode(ctx) === "print") writePrint(callNotice);
1780 - else if (commandMode(ctx) !== "json") ctx.ui.notify(callNotice, "info");
1781 - const allowCall = (kind: "target" | "judge") => {
1782 - const maximum =
1783 - kind === "target" ? costs.targetMaximum : costs.judgeMaximum;
1784 - const consumed =
1785 - kind === "target" ? costs.targetConsumed : costs.judgeConsumed;
1786 - if (consumed >= maximum)
1787 - throw new Error(`Aftermath ${kind} call maximum exceeded`);
1788 - };
1789 - const recordCall = async (kind: "target" | "judge") => {
1790 - allowCall(kind);
1791 - if (
1792 - kind === "target" &&
1793 - fingerprintSource(await readFile(manifest.source.path)) !==
1794 - manifest.source.fingerprint
1796 - throw new Error("source fingerprint changed; replay is unsafe");
1797 - costs = {
1798 - ...costs,
1799 - ...(kind === "target"
1800 - ? { targetConsumed: costs.targetConsumed + 1 }
1801 - : { judgeConsumed: costs.judgeConsumed + 1 }),
1802 - };
1803 - await persistAftermathCost(options.directory, costs);
1804 - manifest.calls[kind] =
1805 - kind === "target" ? costs.targetConsumed : costs.judgeConsumed;
1806 - await persistAftermathManifest(options.directory, manifest);
1807 - };
1808 - throwIfCancelled(active);
1809 - if (tasks.length < 3) {
1810 - const reportPath = await persistFinalReport(options.directory, {
1811 - version: 1,
1812 - kind: "aftermath",
1813 - taskFingerprint: frozen.fingerprint,
1814 - taskIds: tasks.map((task) => task.id),
1815 - splits,
1816 - targetCallMaximum: costs.targetMaximum,
1817 - judgeCallMaximum: costs.judgeMaximum,
1818 - targetCalls: costs.targetConsumed,
1819 - judgeCalls: costs.judgeConsumed,
1820 - terminal: "insufficient_tasks",
1821 - });
1822 - manifest.stage = "complete";
1823 - manifest.terminal = "no_improvement";
1824 - manifest.artifacts.report = "report.json";
1825 - await persistAftermathManifest(options.directory, manifest);
1826 - const entry = runEntry(
1827 - { manifest },
1828 - { completed: tasks.length, total: 3 },
1829 - { reportPath, stagedPath: null },
1830 - );
1831 - appendProgress(options.pi, ctx, entry);
1832 - if (commandMode(ctx) === "print")
1833 - writePrint(
1834 - `Aftermath complete — no candidate.\nOnly ${tasks.length}/3 recurring tasks; replay was skipped.\nReport: ${sanitizeTerminalText(reportPath)}`,
1835 - );
1836 - await offerAftermathCompletion(options.pi, ctx, options.directory);
1837 - return;
1839 - await persistAftermathManifest(options.directory, manifest);
1840 - const currentSource = await readFile(manifest.source.path);
1841 - if (fingerprintSource(currentSource) !== manifest.source.fingerprint)
1842 - throw new Error("source fingerprint changed; replay is unsafe");
1843 - const loadedSkill = await resolveLoadedSkill(
1844 - manifest.source.path,
1845 - options.pi.getCommands(),
1846 - ctx.isProjectTrusted(),
1847 - ctx.cwd,
1848 - );
1849 - const byId = new Map(tasks.map((task) => [task.id, task]));
1850 - const taskItems = (split: "train" | "validation" | "test") =>
1851 - splits[split].map((id) => {
1852 - const task = byId.get(id);
1853 - if (!task) throw new Error("frozen split task is unavailable");
1854 - return { id, prompt: generatedTaskPrompt(task) };
1855 - });
1856 - const judgeCache = await readJudgeCache(options.directory);
1857 - const scorer = createGeneratedTaskScorer(tasks, {
1858 - judge: async (task, response, signal) => {
1859 - const key = rubricCacheKey(task.id, response);
1860 - const cached = judgeCache[key];
1861 - if (cached) return cached;
1862 - const result = await runRubricJudge(
1863 - task,
1864 - response,
1866 - createSession: () =>
1867 - createRubricJudgeSession(
1869 - cwd: ctx.cwd,
1870 - agentDir: getAgentDir(),
1871 - model: manifest.model,
1872 - targetModel: options.targetModel,
1873 - },
1874 - installedAftermathAcceptanceSeam()?.structuredOutputSdk,
1875 - ),
1876 - },
1877 - signal,
1878 - async () => await recordCall("judge"),
1879 - );
1880 - judgeCache[result.cacheKey] = {
1881 - score: result.score,
1882 - passed: result.passed,
1883 - rationale: result.rationale,
1884 - };
1885 - await persistJudgeCache(options.directory, judgeCache);
1886 - return result;
1887 - },
1888 - });
1889 - await executeBaseline({
1890 - pi: options.pi,
1891 - ctx,
1892 - directory: options.directory,
1893 - manifest,
1894 - source,
1895 - loadedSkill,
1896 - targetModel: options.targetModel,
1897 - taskAdapter: {
1898 - items: taskItems,
1899 - scoreResponse: async ({ item, response, signal }) =>
1900 - await scorer({ id: item.id, response, signal }),
1901 - onTargetCall: async () => await recordCall("target"),
1902 - },
1903 - reportExtras: () => ({
1904 - taskFingerprint: frozen.fingerprint,
1905 - targetCallMaximum: costs.targetMaximum,
1906 - judgeCallMaximum: costs.judgeMaximum,
1907 - targetCalls: costs.targetConsumed,
1908 - judgeCalls: costs.judgeConsumed,
1909 - }),
1910 - deferCompletionNotice: true,
1911 - });
1912 - await offerAftermathCompletion(options.pi, ctx, options.directory);
1915 -async function resumeAftermath(
1916 - pi: ExtensionAPI,
1917 - ctx: ExtensionCommandContext,
1918 - selected: NonNullable<ReturnType<typeof selectedAftermathRun>>,
1919 -): Promise<void> {
1920 - const active = activeRun;
1921 - if (active?.kind !== "aftermath")
1922 - throw new Error("Aftermath run ownership is unavailable");
1923 - const directory = join(getAgentDir(), "overkill-runs", selected.data.runId);
1924 - let manifest = await readAftermathManifest(directory);
1925 - if (manifest.terminal)
1926 - throw new Error("only incomplete Aftermath runs can resume");
1927 - if (
1928 - manifest.runId !== selected.data.runId ||
1929 - manifest.sessionId !== ctx.sessionManager.getSessionId()
1931 - throw new Error("run is not linked to this active session branch");
1932 - manifest = (await recoverRunLink({
1933 - directory,
1934 - manifest,
1935 - selected,
1936 - branch: ctx.sessionManager.getBranch(),
1937 - })) as AftermathManifest;
1938 - const sourcePath = await realpath(manifest.source.path);
1939 - const source = await readFile(sourcePath);
1940 - if (fingerprintSource(source) !== manifest.source.fingerprint)
1941 - throw new Error("source fingerprint changed; resume is unsafe");
1942 - const targetModel = ctx.modelRegistry.find(
1943 - manifest.model.provider,
1944 - manifest.model.id,
1945 - );
1946 - if (
1947 - !targetModel ||
1948 - !(await modelHasAuth(ctx, manifest.model.provider, manifest.model.id))
1950 - throw new Error("recorded model authentication is unavailable");
1951 - const harvest = await readHarvest(directory);
1952 - if (harvest.fingerprint !== manifest.harvestFingerprint)
1953 - throw new Error("harvest fingerprint changed; resume is unsafe");
1954 - const frozen = await readFrozenTasks(directory).catch((error) => {
1955 - if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
1956 - throw error;
1957 - });
1958 - const checkpoint = await readMiningCheckpoint(directory);
1959 - const chunks = new Map<number, unknown[]>();
1960 - const levels = new Map<number, unknown[][]>();
1961 - for (const index of checkpoint?.completedChunks ?? [])
1962 - chunks.set(
1963 - index,
1964 - await readMiningOutput(directory, `miner-chunk-${index}.json`),
1965 - );
1966 - for (const { level, index } of checkpoint?.completedLevels ?? []) {
1967 - const outputs = levels.get(level) ?? [];
1968 - outputs[index] = await readMiningOutput(
1969 - directory,
1970 - `aggregate-${level}-${index}.json`,
1971 - );
1972 - levels.set(level, outputs);
1974 - active.runId = manifest.runId;
1975 - await mineAftermath({
1976 - pi,
1977 - ctx,
1978 - directory,
1979 - manifest,
1980 - digests: harvest.digests,
1981 - targetModel,
1982 - harvestFingerprint: harvest.fingerprint,
1983 - completedChunks: chunks,
1984 - completedLevels: levels,
1985 - attemptedMiningCalls: checkpoint?.attemptedCalls ?? 0,
1986 - frozen,
1987 - });
1990 -async function resumeBaseline(
1991 - pi: ExtensionAPI,
1992 - ctx: ExtensionCommandContext,
1993 - selected: NonNullable<ReturnType<typeof selectedRun>>,
1994 -): Promise<void> {
1995 - const directory = join(getAgentDir(), "overkill-runs", selected.data.runId);
1996 - const active = activeRun;
1997 - if (!active || active.sessionId !== ctx.sessionManager.getSessionId())
1998 - throw new Error("Overkill run ownership is unavailable");
1999 - active.runId = selected.data.runId;
2000 - active.interrupt = async () =>
2001 - await checkpointInterrupted(directory, await readManifest(directory));
2002 - let manifest = await readManifest(directory);
2003 - if (manifest.terminal) throw new Error("selected run is terminal");
2004 - if (
2005 - manifest.runId !== selected.data.runId ||
2006 - manifest.sessionId !== ctx.sessionManager.getSessionId()
2008 - throw new Error("run is not linked to this active session branch");
2009 - manifest = await recoverRunLink({
2010 - directory,
2011 - manifest,
2012 - selected,
2013 - branch: ctx.sessionManager.getBranch(),
2014 - });
2015 - active.runId = manifest.runId;
2016 - active.interrupt = async () =>
2017 - await checkpointInterrupted(directory, manifest);
2018 - throwIfCancelled(active);
2019 - const originalLink = ctx.sessionManager.getBranch().find((entry) => {
2020 - if (typeof entry !== "object" || entry === null) return false;
2021 - const candidate = entry as {
2022 - id?: unknown;
2023 - type?: unknown;
2024 - customType?: unknown;
2025 - data?: unknown;
2026 - };
2027 - if (
2028 - candidate.id !== manifest.linkEntryId ||
2029 - candidate.type !== "custom" ||
2030 - candidate.customType !== "overkill-run"
2032 - return false;
2033 - try {
2034 - const data = validateRunEntry(candidate.data);
2035 - return data.kind === "suite" && data.runId === manifest.runId;
2036 - } catch {
2037 - return false;
2039 - });
2040 - if (!originalLink)
2041 - throw new Error("run is no longer linked to this active branch");
2042 - const sourcePath = await realpath(manifest.source.path);
2043 - throwIfCancelled(active);
2044 - const sourceBytes = await readFile(sourcePath);
2045 - throwIfCancelled(active);
2046 - if (fingerprintSource(sourceBytes) !== manifest.source.fingerprint)
2047 - throw new Error("source fingerprint changed; resume is unsafe");
2048 - if (!manifest.suite) throw new Error("suite run has no suite contract");
2049 - const suite = await validateTrustedSuite(
2050 - manifest.suite.path,
2051 - ctx.cwd,
2052 - ctx.isProjectTrusted(),
2053 - );
2054 - throwIfCancelled(active);
2055 - if ((await fingerprintSuite(suite.suitePath)) !== manifest.suite.fingerprint)
2056 - throw new Error("suite fingerprint changed; resume is unsafe");
2057 - throwIfCancelled(active);
2058 - const model = ctx.modelRegistry.find(
2059 - manifest.model.provider,
2060 - manifest.model.id,
2061 - );
2062 - if (
2063 - !model ||
2064 - !(await modelHasAuth(ctx, manifest.model.provider, manifest.model.id))
2066 - throw new Error("recorded model authentication is unavailable");
2067 - const loadedSkill = await resolveLoadedSkill(
2068 - sourcePath,
2069 - pi.getCommands(),
2070 - ctx.isProjectTrusted(),
2071 - ctx.cwd,
2072 - );
2073 - throwIfCancelled(active);
2074 - await executeBaseline({
2075 - pi,
2076 - ctx,
2077 - directory,
2078 - manifest,
2079 - source: parseSourceSkill(sourceBytes),
2080 - loadedSkill,
2081 - suite,
2082 - targetModel: model,
2083 - });
14 +type Implementation = typeof import("./implementation.ts");
2086 16 export default function overkill(pi: ExtensionAPI): void {
17 + let implementation: Promise<Implementation> | undefined;
18 + const load = () => (implementation ??= import("./implementation.ts"));
2087 20 pi.on("session_shutdown", async (_event, ctx) => {
2088 - const sessionId = ctx.sessionManager.getSessionId();
2089 - if (!activeRun) {
2090 - if (startPending && pendingSessionId === sessionId)
2091 - pendingCancelled = true;
2092 - return;
2094 - if (activeRun.sessionId !== sessionId) return;
2095 - activeRun.suppressOutput = true;
2096 - requestCancellation(activeRun);
2097 - try {
2098 - await activeRun.completed;
2099 - } catch {}
21 + if (implementation) await (await implementation).shutdownOverkill(ctx);
2101 - pi.registerEntryRenderer<RunEntry>(
2102 - "overkill-run",
2103 - (entry, options, theme) => {
2104 - let data: RunEntry;
2105 - try {
2106 - data = validateRunEntry(entry.data);
2107 - } catch {
2108 - return undefined;
2110 - const summary = theme.fg("accent", formatProgress(data));
2111 - if (!options.expanded) {
2112 - const details = [
2113 - `${summary} ${theme.fg("dim", keyHint("app.tools.expand", "to expand"))}`,
2114 - ];
2115 - if (data.terminal === "staged")
2116 - try {
2117 - const artifacts = readFrozenReviewArtifactsSync(
2118 - join(getAgentDir(), "overkill-runs", data.runId),
2119 - );
2120 - if (artifacts.stagedPath && artifacts.staged)
2121 - details.push(
2122 - renderDiff(
2123 - buildReviewDiffPreview(
2124 - buildReviewDiff(
2125 - artifacts.sourcePath,
2126 - artifacts.stagedPath,
2127 - artifacts.source,
2128 - artifacts.staged,
2129 - ),
2130 - ),
2131 - ),
2132 - );
2133 - } catch {
2134 - // The durable run summary remains useful without review artifacts.
2136 - return new Text(details.join("\n"), 0, 0);
2138 - const details = [summary];
2139 - if (data.stage === "preflight") return new Text(summary, 0, 0);
2140 - try {
2141 - const artifacts = readFrozenReviewArtifactsSync(
2142 - join(getAgentDir(), "overkill-runs", data.runId),
2143 - );
2144 - details.push(
2145 - theme.fg(
2146 - "dim",
2147 - `Source: ${sanitizeTerminalText(artifacts.sourcePath)}`,
2148 - ),
2149 - );
2150 - if (artifacts.reportPath)
2151 - details.push(
2152 - theme.fg(
2153 - "dim",
2154 - `Report: ${sanitizeTerminalText(artifacts.reportPath)}`,
2155 - ),
2156 - );
2157 - if (artifacts.stagedPath && artifacts.staged) {
2158 - details.push(
2159 - theme.fg(
2160 - "dim",
2161 - `Staged: ${sanitizeTerminalText(artifacts.stagedPath)}`,
2162 - ),
2163 - );
2164 - details.push(
2165 - renderDiff(
2166 - buildReviewDiff(
2167 - artifacts.sourcePath,
2168 - artifacts.stagedPath,
2169 - artifacts.source,
2170 - artifacts.staged,
2171 - ),
2172 - ),
2173 - );
2175 - } catch {
2176 - details.push(theme.fg("warning", "Review artifacts unavailable."));
2178 - return new Text(details.join("\n"), 0, 0);
2179 - },
2180 - );
23 + registerOverkillEntryRenderer(pi);
2181 24 pi.registerCommand("overkill", {
2182 25 description:
2183 26 "Author or optimize a loaded Pi skill with trusted evaluators.",
2200 43 return choices.length > 0 ? choices : null;
2202 45 handler: async (args, ctx) => {
2203 - let parts: string[];
2204 - try {
2205 - parts = parseArguments(args);
2206 - } catch (error) {
2207 - ctx.ui.notify(
2208 - `Overkill command failed: ${error instanceof Error ? error.message : String(error)}`,
2209 - "error",
2210 - );
2211 - return;
2213 - const command = parts[0];
2214 - if (command === "suite") {
2215 - let [sourcePath, suitePath, ...extra] = parts.slice(1);
2216 - if (extra.length > 0) {
2217 - ctx.ui.notify(
2218 - "Usage: /overkill suite [skill-path] [suite-dir]",
2219 - "error",
2220 - );
2221 - return;
2223 - if (!ctx.isIdle()) {
2224 - ctx.ui.notify("Agent is busy.", "error");
2225 - return;
2227 - try {
2228 - if (!ctx.isProjectTrusted())
2229 - throw new Error("suite authoring requires a trusted project");
2230 - if (!sourcePath) {
2231 - if (!ctx.hasUI)
2232 - throw new Error(
2233 - "headless suite authoring requires an explicit skill path",
2234 - );
2235 - const skills = loadedSkills(pi);
2236 - if (skills.length === 0)
2237 - throw new Error("no loaded Pi skills are available");
2238 - const canonicalPaths = new Map<string, string>();
2239 - for (const skill of skills)
2240 - canonicalPaths.set(skill.path, await realpath(skill.path));
2241 - const selected = await chooseLoadedSkill(
2242 - ctx,
2243 - skills,
2244 - canonicalPaths,
2245 - "suite",
2246 - );
2247 - if (!selected) return;
2248 - sourcePath = selected.path;
2250 - const loaded = await resolveLoadedSkill(
2251 - sourcePath,
2252 - pi.getCommands(),
2253 - true,
2254 - ctx.cwd,
2255 - );
2256 - const projectRoot = await realpath(resolve(ctx.cwd));
2257 - if (suitePath) {
2258 - suitePath = await canonicalProspectivePath(
2259 - resolve(projectRoot, suitePath),
2260 - );
2261 - const projectRelative = relative(projectRoot, suitePath);
2262 - if (projectRelative.startsWith("..") || isAbsolute(projectRelative))
2263 - throw new Error("suite directory must be inside the project");
2265 - const parallelWarning = await unfinishedRunWarning(
2266 - undefined,
2267 - suitePath,
2268 - );
2269 - if (parallelWarning) ctx.ui.notify(parallelWarning, "warning");
2270 - pi.sendUserMessage(
2271 - buildSuiteAuthoringPrompt(loaded.path, suitePath, projectRoot),
2272 - );
2273 - } catch (error) {
2274 - ctx.ui.notify(
2275 - `Overkill suite authoring failed: ${error instanceof Error ? error.message : String(error)}`,
2276 - "error",
2277 - );
2279 - return;
2281 - if (command === "aftermath") {
2282 - const subcommand = parts[1];
2283 - if (
2284 - ["status", "resume", "review", "discard", "cancel"].includes(
2285 - subcommand ?? "",
2286 - ) &&
2287 - parts.length !== 2
2288 - ) {
2289 - ctx.ui.notify(`Usage: /overkill aftermath ${subcommand}`, "error");
2290 - return;
2292 - if (subcommand === "status") {
2293 - const run = selectedAftermathRun(ctx);
2294 - if (!run)
2295 - ctx.ui.notify(
2296 - "No Overkill Aftermath run is linked to this branch.",
2297 - "info",
2298 - );
2299 - else if (commandMode(ctx) === "json")
2300 - appendProgress(pi, ctx, run.data);
2301 - else if (commandMode(ctx) === "print")
2302 - writePrint(statusMessage(run.data));
2303 - else ctx.ui.notify(statusMessage(run.data), "info");
2304 - return;
2306 - if (subcommand === "cancel") {
2307 - if (
2308 - activeRun &&
2309 - activeRun.kind === "aftermath" &&
2310 - activeRun.sessionId === ctx.sessionManager.getSessionId()
2312 - requestCancellation(activeRun);
2313 - else ctx.ui.notify("No Overkill Aftermath run is active.", "info");
2314 - return;
2316 - if (["resume", "review", "discard"].includes(subcommand ?? "")) {
2317 - const run = selectedAftermathRun(ctx);
2318 - if (!run) {
2319 - ctx.ui.notify(
2320 - "No Overkill Aftermath run is linked to this branch.",
2321 - "info",
2322 - );
2323 - return;
2325 - const directory = join(
2326 - getAgentDir(),
2327 - "overkill-runs",
2328 - run.data.runId,
2329 - );
2330 - try {
2331 - const manifest = await readAftermathManifest(directory);
2332 - if (manifest.sessionId !== ctx.sessionManager.getSessionId())
2333 - throw new Error(
2334 - "run is not linked to this active session branch",
2335 - );
2336 - if (subcommand === "review") {
2337 - const artifacts = await readFrozenReviewArtifacts(directory);
2338 - const entry = {
2339 - ...run.data,
2340 - sourcePath: artifacts.sourcePath,
2341 - reportPath: artifacts.reportPath,
2342 - stagedPath: artifacts.stagedPath,
2343 - };
2344 - const paths = formatResultPaths(entry);
2345 - if (commandMode(ctx) === "json") appendProgress(pi, ctx, entry);
2346 - else if (commandMode(ctx) === "print") {
2347 - writePrint(paths);
2348 - if (artifacts.stagedPath && artifacts.staged)
2349 - writePrint(
2350 - buildReviewDiff(
2351 - artifacts.sourcePath,
2352 - artifacts.stagedPath,
2353 - artifacts.source,
2354 - artifacts.staged,
2355 - ),
2356 - );
2357 - else writePrint("No candidate staged.");
2358 - } else {
2359 - ctx.ui.notify(
2360 - artifacts.stagedPath
2361 - ? paths
2362 - : `${paths}\nNo candidate staged.`,
2363 - "info",
2364 - );
2365 - if (
2366 - commandMode(ctx) === "tui" &&
2367 - artifacts.stagedPath &&
2368 - artifacts.staged
2370 - ctx.ui.notify(
2371 - buildReviewDiff(
2372 - artifacts.sourcePath,
2373 - artifacts.stagedPath,
2374 - artifacts.source,
2375 - artifacts.staged,
2376 - ),
2377 - "info",
2378 - );
2380 - } else if (subcommand === "discard") {
2381 - if (manifest.terminal === "discarded")
2382 - throw new Error("discarded run artifacts are unavailable");
2383 - await discardLinkedRun({
2384 - runRoot: join(getAgentDir(), "overkill-runs"),
2385 - runDirectory: directory,
2386 - selected: run,
2387 - branch: ctx.sessionManager.getBranch(),
2388 - sessionId: ctx.sessionManager.getSessionId(),
2389 - allowNonterminal: true,
2390 - isRunActive: (runId) => activeRun?.runId === runId,
2391 - });
2392 - appendProgress(pi, ctx, {
2393 - ...run.data,
2394 - stage: "discarded",
2395 - terminal: "discarded",
2396 - });
2397 - clearProgress(ctx);
2398 - } else if (manifest.terminal) {
2399 - ctx.ui.notify(
2400 - "Aftermath mining is already complete; stored tasks are frozen.",
2401 - "info",
2402 - );
2403 - } else {
2404 - await withActiveRun(
2405 - ctx,
2406 - async () => await resumeAftermath(pi, ctx, run),
2407 - "aftermath",
2408 - );
2410 - } catch (error) {
2411 - if (!(error instanceof RolloutCancelledError))
2412 - ctx.ui.notify(
2413 - `Overkill Aftermath ${subcommand} failed: ${error instanceof Error ? error.message : String(error)}`,
2414 - "error",
2415 - );
2417 - return;
2419 - if (activeRun || startPending) {
2420 - ctx.ui.notify(
2421 - "This Pi process already has an Overkill rollout active; wait or cancel it before starting another.",
2422 - "error",
2423 - );
2424 - return;
2426 - startPending = true;
2427 - pendingSessionId = ctx.sessionManager.getSessionId();
2428 - pendingCancelled = false;
2429 - try {
2430 - await withActiveRun(
2431 - ctx,
2432 - async () => await runAftermath(pi, ctx, parts.slice(1)),
2433 - "aftermath",
2434 - );
2435 - } catch (error) {
2436 - if (!(error instanceof RolloutCancelledError))
2437 - ctx.ui.notify(
2438 - `Overkill Aftermath failed: ${error instanceof Error ? error.message : String(error)}`,
2439 - "error",
2440 - );
2441 - } finally {
2442 - startPending = false;
2443 - pendingSessionId = undefined;
2444 - pendingCancelled = false;
2446 - return;
2448 - if (
2449 - ["status", "discard", "cancel", "review", "resume"].includes(
2450 - command ?? "",
2451 - ) &&
2452 - parts.length !== 1
2453 - ) {
2454 - ctx.ui.notify(`Usage: /overkill ${command}`, "error");
2455 - return;
2457 - if (command === "status") {
2458 - const run = selectedRun(ctx);
2459 - if (!run) {
2460 - ctx.ui.notify(
2461 - "No Overkill suite run is linked to this branch.",
2462 - "info",
2463 - );
2464 - return;
2466 - const mode = commandMode(ctx);
2467 - if (mode === "json") appendProgress(pi, ctx, run.data);
2468 - else if (mode === "print") writePrint(statusMessage(run.data));
2469 - else ctx.ui.notify(statusMessage(run.data), "info");
2470 - return;
2472 - if (command === "discard") {
2473 - const run = selectedRun(ctx);
2474 - if (!run) {
2475 - ctx.ui.notify(
2476 - "No Overkill suite run is linked to this branch.",
2477 - "info",
2478 - );
2479 - return;
2481 - try {
2482 - await discardLinkedRun({
2483 - runRoot: join(getAgentDir(), "overkill-runs"),
2484 - runDirectory: join(getAgentDir(), "overkill-runs", run.data.runId),
2485 - selected: run,
2486 - branch: ctx.sessionManager.getBranch(),
2487 - sessionId: ctx.sessionManager.getSessionId(),
2488 - allowNonterminal: true,
2489 - isRunActive: (runId) => activeRun?.runId === runId,
2490 - });
2491 - appendProgress(pi, ctx, {
2492 - ...run.data,
2493 - stage: "discarded",
2494 - terminal: "discarded",
2495 - });
2496 - clearProgress(ctx);
2497 - ctx.ui.notify("Overkill run discarded.", "info");
2498 - } catch (error) {
2499 - ctx.ui.notify(
2500 - `Overkill discard failed: ${error instanceof Error ? error.message : String(error)}`,
2501 - "error",
2502 - );
2504 - return;
2506 - if (command === "cancel") {
2507 - if (
2508 - !activeRun ||
2509 - activeRun.sessionId !== ctx.sessionManager.getSessionId()
2510 - ) {
2511 - ctx.ui.notify("No Overkill rollout is active.", "info");
2512 - return;
2514 - requestCancellation(activeRun);
2515 - if (commandMode(ctx) !== "json")
2516 - ctx.ui.notify("Overkill cancellation requested.", "info");
2517 - return;
2519 - if (command === "review") {
2520 - const run = selectedRun(ctx);
2521 - if (!run) {
2522 - ctx.ui.notify(
2523 - "No Overkill suite run is linked to this branch.",
2524 - "info",
2525 - );
2526 - return;
2528 - if (!run.data.terminal) {
2529 - ctx.ui.notify("Overkill run is not terminal yet.", "info");
2530 - return;
2532 - if (run.data.terminal === "discarded") {
2533 - ctx.ui.notify(
2534 - "Overkill run was discarded; review artifacts are unavailable.",
2535 - "info",
2536 - );
2537 - return;
2539 - let artifacts: Awaited<ReturnType<typeof readFrozenReviewArtifacts>>;
2540 - try {
2541 - artifacts = await readFrozenReviewArtifacts(
2542 - join(getAgentDir(), "overkill-runs", run.data.runId),
2543 - );
2544 - } catch (error) {
2545 - ctx.ui.notify(
2546 - `Overkill review failed: ${error instanceof Error ? error.message : String(error)}`,
2547 - "error",
2548 - );
2549 - return;
2551 - const entry = {
2552 - ...run.data,
2553 - sourcePath: artifacts.sourcePath,
2554 - reportPath: artifacts.reportPath,
2555 - stagedPath: artifacts.stagedPath,
2556 - };
2557 - const paths = formatResultPaths(entry);
2558 - const mode = commandMode(ctx);
2559 - if (mode === "json") appendProgress(pi, ctx, entry);
2560 - else if (mode === "print") {
2561 - writePrint(paths);
2562 - if (artifacts.stagedPath && artifacts.staged)
2563 - writePrint(
2564 - buildReviewDiff(
2565 - artifacts.sourcePath,
2566 - artifacts.stagedPath,
2567 - artifacts.source,
2568 - artifacts.staged,
2569 - ),
2570 - );
2571 - else writePrint("No candidate staged.");
2572 - } else {
2573 - ctx.ui.notify(
2574 - artifacts.stagedPath ? paths : `${paths}\nNo candidate staged.`,
2575 - "info",
2576 - );
2577 - if (mode === "tui" && artifacts.stagedPath && artifacts.staged)
2578 - ctx.ui.notify(
2579 - buildReviewDiff(
2580 - artifacts.sourcePath,
2581 - artifacts.stagedPath,
2582 - artifacts.source,
2583 - artifacts.staged,
2584 - ),
2585 - "info",
2586 - );
2588 - return;
2590 - if (command === "resume") {
2591 - if (activeRun || startPending) {
2592 - ctx.ui.notify(
2593 - "This Pi process already has an Overkill rollout active; wait or cancel it before resuming another.",
2594 - "error",
2595 - );
2596 - return;
2598 - const run = selectedRun(ctx);
2599 - if (!run) {
2600 - ctx.ui.notify(
2601 - "No Overkill suite run is linked to this branch.",
2602 - "info",
2603 - );
2604 - return;
2606 - startPending = true;
2607 - pendingSessionId = ctx.sessionManager.getSessionId();
2608 - pendingCancelled = false;
2609 - try {
2610 - await withActiveRun(
2611 - ctx,
2612 - async () => await resumeBaseline(pi, ctx, run),
2613 - );
2614 - } catch (error) {
2615 - if (
2616 - error instanceof RolloutCancelledError ||
2617 - error instanceof OptimizerCancelledError
2619 - return;
2620 - ctx.ui.notify(
2621 - `Overkill resume failed: ${error instanceof Error ? error.message : String(error)}`,
2622 - "error",
2623 - );
2624 - } finally {
2625 - startPending = false;
2626 - pendingSessionId = undefined;
2627 - pendingCancelled = false;
2629 - return;
2631 - const [sourcePath, suitePath, ...extra] = parts;
2632 - if (!sourcePath || !suitePath || extra.length > 0) {
2633 - ctx.ui.notify("Usage: /overkill <skill-path> <suite-dir>", "error");
2634 - return;
2636 - if (activeRun || startPending) {
2637 - ctx.ui.notify(
2638 - "This Pi process already has an Overkill rollout active; wait or cancel it before starting another.",
2639 - "error",
2640 - );
2641 - return;
2643 - const invokingModel = ctx.model;
2644 - const targetModel =
2645 - invokingModel &&
2646 - ctx.modelRegistry.find(invokingModel.provider, invokingModel.id);
2647 - startPending = true;
2648 - pendingSessionId = ctx.sessionManager.getSessionId();
2649 - pendingCancelled = false;
2650 - try {
2651 - if (invokingModel && !targetModel)
2652 - throw new Error(
2653 - "selected model is unavailable from the invoking registry",
2654 - );
2655 - const preflightOptions = {
2656 - agentDir: getAgentDir(),
2657 - cwd: ctx.cwd,
2658 - sourcePath,
2659 - suitePath,
2660 - commands: pi.getCommands(),
2661 - projectTrusted: ctx.isProjectTrusted(),
2662 - sessionId: ctx.sessionManager.getSessionId(),
2663 - model: invokingModel
2664 - ? {
2665 - provider: invokingModel.provider,
2666 - id: invokingModel.id,
2667 - thinking: pi.getThinkingLevel(),
2669 - : undefined,
2670 - authAvailable: await modelHasAuth(
2671 - ctx,
2672 - invokingModel?.provider,
2673 - invokingModel?.id,
2674 - ),
2675 - };
2676 - const inspected = await inspectSuiteRun(preflightOptions);
2677 - const inspectedSuiteFingerprint = inspected.suite.fingerprint;
2678 - if (!inspectedSuiteFingerprint)
2679 - throw new Error("validated suite fingerprint is unavailable");
2680 - const confirmedFingerprints = {
2681 - sourceFingerprint: fingerprintSource(inspected.sourceBytes),
2682 - suiteFingerprint: inspectedSuiteFingerprint,
2683 - };
2684 - const parallelWarning = await unfinishedRunWarning(
2685 - inspected.sourcePath,
2686 - inspected.suite.suitePath,
2687 - );
2688 - if (commandMode(ctx) === "tui" || commandMode(ctx) === "rpc") {
2689 - const confirmed = await ctx.ui.confirm(
2690 - "Start Overkill run?",
2691 - buildConfirmation({
2692 - sourcePath: inspected.sourcePath,
2693 - suitePath: inspected.suite.suitePath,
2694 - splitCounts: inspected.suite.splitCounts,
2695 - warning: parallelWarning,
2696 - }),
2697 - );
2698 - if (!confirmed) {
2699 - ctx.ui.notify("Overkill start cancelled.", "info");
2700 - return;
2703 - if (pendingCancelled) throw new RolloutCancelledError();
2704 - await withActiveRun(ctx, async (active) => {
2705 - throwIfCancelled(active);
2706 - const run = await prepareSuiteRun(
2707 - preflightOptions,
2708 - confirmedFingerprints,
2709 - );
2710 - active.runId = run.manifest.runId;
2711 - active.interrupt = async () =>
2712 - await checkpointInterrupted(run.directory, run.manifest);
2713 - throwIfCancelled(active);
2714 - appendProgress(pi, ctx, runEntry(run));
2715 - const link = ctx.sessionManager.getBranch().at(-1);
2716 - if (!link || typeof link.id !== "string")
2717 - throw new Error("could not persist branch link");
2718 - run.manifest.linkEntryId = link.id;
2719 - await persistManifest(run.directory, run.manifest);
2720 - throwIfCancelled(active);
2721 - await executeBaseline({
2722 - pi,
2723 - ctx,
2724 - directory: run.directory,
2725 - manifest: run.manifest,
2726 - source: run.source,
2727 - loadedSkill: run.loadedSkill,
2728 - suite: run.suite,
2729 - targetModel,
2730 - });
2731 - });
2732 - } catch (error) {
2733 - if (
2734 - error instanceof RolloutCancelledError ||
2735 - error instanceof OptimizerCancelledError
2737 - return;
2738 - ctx.ui.notify(
2739 - `Overkill run failed: ${error instanceof Error ? error.message : String(error)}`,
2740 - "error",
2741 - );
2742 - } finally {
2743 - startPending = false;
2744 - pendingSessionId = undefined;
2745 - pendingCancelled = false;
46 + const command = (await load()).createOverkillCommand(pi);
47 + await command.handler(args, ctx);