1 +import { createHash }
from "node:crypto";
2 +import { writeSync }
from "node:fs";
3 +import { readdir, readFile, realpath }
from "node:fs/promises";
15 + type ExtensionCommandContext,
16 + type ExtensionContext,
19 +}
from "@earendil-works/pi-coding-agent";
20 +import { Key, matchesKey, SelectList }
from "@earendil-works/pi-tui";
21 +import { registerOverkillEntryRenderer }
from "./entry-renderer.ts";
26 +}
from "./lib/aftermath-harvest.ts";
29 + createMiningSession,
30 + estimateMiningCalls,
32 + type StructuredOutputSdkSeam,
33 + validateAndFreezeTasks,
34 +}
from "./lib/aftermath-mining.ts";
36 + createGeneratedTaskScorer,
37 + createRubricJudgeSession,
38 + generatedTaskPrompt,
41 +}
from "./lib/aftermath-scoring.ts";
43 + type AftermathManifest,
44 + createAftermathDirectory,
45 + persistAftermathCost,
46 + persistAftermathManifest,
50 + persistMiningCheckpoint,
51 + persistMiningOutput,
53 + readAftermathManifest,
57 + readMiningCheckpoint,
59 +}
from "./lib/aftermath-state.ts";
64 +}
from "./lib/checker.ts";
66 + getValidatedSuiteItems,
70 + validateTrustedSuite,
71 +}
from "./lib/contracts.ts";
78 + sanitizeTerminalText,
80 +}
from "./lib/modes.ts";
82 + createOptimizerSession,
83 + defaultOptimizerSdk,
84 + OptimizerCancelledError,
85 + type OptimizerSdkSeam,
87 +}
from "./lib/optimizer.ts";
88 +import { parsePatch }
from "./lib/patches.ts";
89 +import { inspectSuiteRun, prepareSuiteRun }
from "./lib/preflight.ts";
91 + createIsolatedTargetSession,
93 + RolloutCancelledError,
95 + runValidationBaseline,
97 +}
from "./lib/rollouts.ts";
100 + persistFinalReport,
102 + persistTrainingCheckpoint,
105 + readFrozenReviewArtifacts,
108 + readTrainingCheckpoint,
110 + selectNewestAftermathRun,
111 + selectNewestSuiteRun,
114 +}
from "./lib/run-state.ts";
116 + type AttemptedPatch,
117 + createTrainingState,
118 + type DurableTrainingState,
119 + runDurableTrainingSchedule,
120 +}
from "./lib/training.ts";
122 +const SUBCOMMANDS
= [
133 + kind
: "suite" | "aftermath";
136 + controller
: AbortController;
137 + suppressOutput
: boolean;
138 + completed
?: Promise<
void>;
139 + cancelRequested
?: boolean;
140 + interrupt?: ()
=> Promise<
void>;
141 + removeCancellationListeners?: ()
=> void;
144 +let activeRun
: ActiveRun | undefined;
145 +let startPending
= false;
146 +let pendingSessionId
: string | undefined;
147 +let pendingCancelled
= false;
149 +export type ExplicitSuiteAcceptanceSeam = {
150 + targetSdk
: TargetSdkSeam;
151 + optimizerSdk
: OptimizerSdkSeam;
154 +export type AftermathSessionApi = {
156 + sessionDir
?: string,
157 + )
=> Promise<
Array<{ path
: string; id
: string }>>;
160 + sessionDir
?: string,
162 + getBranch: ()
=> unknown[];
163 + getSessionId: ()
=> string;
164 + getCwd: ()
=> string;
168 +export type AftermathAcceptanceSeam = ExplicitSuiteAcceptanceSeam & {
169 + sessionApi
: AftermathSessionApi;
170 + structuredOutputSdk
: StructuredOutputSdkSeam;
172 + bindCommand?: (
handler: (args
: string)
=> Promise<
void>)
=> void;
175 +export type ModeAcceptanceSeam = {
176 + write: (fd
: number, text
: string)
=> void;
179 +const explicitSuiteAcceptanceSeam
= Symbol.
for(
180 + "@bugabinga/pi-ext-overkill/explicit-suite-acceptance-seam",
182 +const aftermathAcceptanceSeam
= Symbol.
for(
183 + "@bugabinga/pi-ext-overkill/aftermath-acceptance-seam",
185 +const modeAcceptanceSeam
= Symbol.
for(
186 + "@bugabinga/pi-ext-overkill/mode-acceptance-seam",
189 +function installedExplicitSuiteAcceptanceSeam()
: 190 + | ExplicitSuiteAcceptanceSeam 192 + return (globalThis
as Record<
PropertyKey,
unknown>)[
193 + explicitSuiteAcceptanceSeam
194 + ]
as ExplicitSuiteAcceptanceSeam | undefined;
197 +function installedAftermathAcceptanceSeam()
: 198 + | AftermathAcceptanceSeam 200 + return (globalThis
as Record<
PropertyKey,
unknown>)[
201 + aftermathAcceptanceSeam
202 + ]
as AftermathAcceptanceSeam | undefined;
205 +function installedModeAcceptanceSeam()
: ModeAcceptanceSeam | undefined {
206 + return (globalThis
as Record<
PropertyKey,
unknown>)[modeAcceptanceSeam] as
207 + | ModeAcceptanceSeam
212 +export function installExplicitSuiteAcceptanceSeam(
213 + seam
: ExplicitSuiteAcceptanceSeam,
215 + const globals
= globalThis
as Record<
PropertyKey,
unknown>;
216 + const previous
= globals[explicitSuiteAcceptanceSeam];
217 + globals[explicitSuiteAcceptanceSeam]
= seam;
219 + if (globals[explicitSuiteAcceptanceSeam]
!== seam)
return;
220 + if (previous
=== undefined)
delete globals[explicitSuiteAcceptanceSeam];
221 + else globals[explicitSuiteAcceptanceSeam]
= previous;
226 +export function installModeAcceptanceSeam(
227 + seam
: ModeAcceptanceSeam,
229 + const globals
= globalThis
as Record<
PropertyKey,
unknown>;
230 + const previous
= globals[modeAcceptanceSeam];
231 + globals[modeAcceptanceSeam]
= seam;
233 + if (globals[modeAcceptanceSeam]
!== seam)
return;
234 + if (previous
=== undefined)
delete globals[modeAcceptanceSeam];
235 + else globals[modeAcceptanceSeam]
= previous;
240 +export function installAftermathAcceptanceSeam(
241 + seam
: AftermathAcceptanceSeam,
243 + const globals
= globalThis
as Record<
PropertyKey,
unknown>;
244 + const previous
= globals[aftermathAcceptanceSeam];
245 + globals[aftermathAcceptanceSeam]
= seam;
247 + if (globals[aftermathAcceptanceSeam]
!== seam)
return;
248 + if (previous
=== undefined)
delete globals[aftermathAcceptanceSeam];
249 + else globals[aftermathAcceptanceSeam]
= previous;
253 +export function buildSuiteAuthoringPrompt(
254 + sourcePath
: string,
255 + suitePath
: string | undefined,
256 + projectRoot
: string,
258 + const sourceName
= basename(sourcePath);
260 + sourceName
=== "SKILL.md" 261 + ? basename(
dirname(sourcePath))
262 + : basename(sourceName,
extname(sourceName));
263 + const destination
= suitePath
264 + ? `Use ${JSON.stringify(suitePath)} as the suite directory.` 265 + : `Choose a suite directory inside ${JSON.stringify(projectRoot)}. Prefer an established project evaluator location; otherwise use the project-local scratch directory ${JSON.stringify(join(projectRoot, ".overkill", "suites", skillName))}.`;
266 + return `Create a trusted explicit evaluator suite for the loaded Pi skill at ${JSON.stringify(sourcePath)}. 269 +Read the complete skill and only its directly referenced files needed to understand behavior. 270 +The suite contract below is authoritative; do not inspect Overkill implementation or unrelated project documentation. 271 +Inspect the destination once immediately before writing; if evaluator files already exist or appear concurrently modified, stop and ask rather than overwriting them. 272 +Do not design locks, atomic reservations, or repeated concurrency probes. 273 +After that single inspection, write the four files immediately without narrating a plan. 274 +Create exactly this layout and nothing else inside the suite directory: 280 +Create a small first suite: exactly 3 train items, 2 validation items, and 2 test items, with globally unique nonempty string ids and nonempty prompts no larger than 16 KiB. 281 +Add checker-owned expected fields when useful. 282 +Use train cases for optimization, validation cases for candidate selection, and held-out test cases for final evaluation; keep scenarios distinct across splits. 283 +Prioritize the skill's highest-risk and most representative successful and failure workflows; report omitted coverage instead of adding more cases. 284 +Keep each item concise and use a small shared expectation vocabulary such as exact, required substrings, forbidden substrings, or simple patterns. 285 +Do not design a domain-specific testing framework. 286 +The checker must read one JSON object from stdin, inspect item, response, trajectory, and calledTools, then write exactly one JSON object with score in [0,1], Boolean passed, and optional feedback. 287 +Write all four files before refining them, and keep check.mjs under 120 lines. 288 +Use only Node.js standard library in check.mjs. 289 +Keep checks bounded, cross-platform, and independent of network, credentials, mutable external state, and the source skill's exact wording. 290 +Do not weaken trust-boundary, security, data-loss, or required human-confirmation behavior. 291 +Validate the JSON and checker locally with one passing and one failing input. 292 +Do not start Overkill automatically. 293 +Summarize coverage, limitations, and the exact /overkill <skill-path> <suite-dir> command the user should run after reviewing check.mjs.`;
296 +export function parseArguments(input
: string)
: string[] {
297 + const parts
: string[]
= [];
299 + while (index
< input.
length) {
300 + while (
/\s/u.
test(input[index]
?? "")) index
+= 1;
301 + if (index
>= input.
length)
break;
303 + while (index
< input.
length && !/\s/u.
test(input[index]
?? "")) {
304 + const quote
= input[index];
305 + if (quote
!== '"' && quote
!== "'") {
311 + let closed
= false;
312 + while (index
< input.
length) {
313 + const character
= input[index];
314 + if (character
=== quote) {
319 + if (character
=== "\\" && input[index
+ 1]
=== quote) {
323 + token
+= character;
327 + if (
!closed)
throw new Error(
"unterminated quoted argument");
334 +async function canonicalProspectivePath(path
: string)
: Promise<
string> {
335 + let existing
= resolve(path);
336 + const missing
: string[]
= [];
339 + return resolve(
await realpath(existing),
...missing);
342 + !(error
instanceof Error)
|| 343 + !(
"code" in error)
|| 344 + error.
code !== "ENOENT" 348 + const parent
= dirname(existing);
349 + if (parent
=== existing)
throw new Error(
"path has no existing ancestor");
350 + missing.
unshift(
basename(existing));
359 + kind
: "suite" | "aftermath";
361 + terminal
: RunEntry[
"terminal"];
362 + source
: { path
: string };
367 + "completed" | "total" | "currentScore" | "bestScore" 369 + artifacts
: Pick<
RunEntry,
"reportPath" | "stagedPath">
= {
376 + runId: run.manifest.runId,
377 + kind: run.manifest.kind,
378 + stage: run.manifest.stage,
379 + terminal: run.manifest.terminal,
380 + sourcePath: run.manifest.
source.path,
386 +async function modelHasAuth(
387 + ctx
: ExtensionCommandContext,
388 + provider
= ctx.model?.provider,
389 + id
= ctx.model?.
id,
390 +)
: Promise<
boolean> {
391 + if (
!provider
|| !id)
return false;
392 + const available
= await ctx.modelRegistry.
getAvailable();
393 + return available.
some(
394 + (model)
=> model.provider
=== provider
&& model.
id === id,
398 +function selectedRun(ctx
: ExtensionCommandContext) {
399 + return selectNewestSuiteRun(ctx.sessionManager.
getBranch());
402 +function selectedAftermathRun(ctx
: ExtensionCommandContext) {
403 + return selectNewestAftermathRun(ctx.sessionManager.
getBranch());
406 +export async function unfinishedRunWarning(
407 + sourcePath
: string | undefined,
408 + suitePath
?: string,
409 + agentDir
= getAgentDir(),
410 +)
: Promise<
string | undefined> {
411 + const root
= join(agentDir,
"overkill-runs");
412 + const entries
= await readdir(
root, { withFileTypes:
true }).
catch(()
=> []);
413 + const runIds
: string[]
= [];
414 + for (
const entry
of entries) {
415 + if (
!entry.
isDirectory())
continue;
416 + const manifest
= await readManifest(
join(
root, entry.
name)).
catch(
420 + manifest?.terminal
=== null && 421 + ((sourcePath
!== undefined && manifest.
source.path
=== sourcePath)
|| 422 + (suitePath
!== undefined && manifest.suite?.path
=== suitePath))
424 + runIds.
push(manifest.runId);
426 + if (runIds.
length === 0)
return undefined;
427 + return `${runIds.length} unfinished Overkill run${runIds.length === 1 ? "" : "s"} already target this skill or suite (${runIds.slice(0, 3).join(", ")}${runIds.length > 3 ? ", …" : ""}). They may be active in other Pi processes; parallel runs duplicate model cost, and suite edits can invalidate running evaluations.`;
430 +function statusMessage(run
: RunEntry)
: string {
432 + run.kind
=== "suite" && 433 + run.stage
=== "complete" && 434 + run.terminal
=== "no_improvement" && 435 + run.bestScore
=== 1 437 + return `Overkill: complete | completed: ${run.completed ?? 0}${run.total === undefined ? "" : `/${run.total}`} | outcome: baseline perfect; optimization skipped`;
438 + const fields
= [
`Overkill: ${run.stage}`];
439 + if (run.completed
!== undefined)
441 + `completed: ${run.completed}${run.total === undefined ? "" : `/${run.total}`}`,
443 + if (run.currentScore
!== undefined)
444 + fields.
push(
`current score: ${run.currentScore}`);
445 + if (run.bestScore
!== undefined) fields.
push(
`best score: ${run.bestScore}`);
446 + fields.
push(
`terminal: ${run.terminal ?? "active"}`);
447 + if (run.stagedPath) fields.
push(
`staged: ${run.stagedPath}`);
448 + return fields.
join(
" | ");
451 +function commandMode(
452 + ctx
: ExtensionCommandContext,
453 +)
: "tui" | "rpc" | "json" | "print" {
454 + return ctx.mode
?? "tui";
457 +function writePrint(line
: string)
: void {
458 + const text
= `${line}\n`;
459 + const seam
= installedModeAcceptanceSeam();
460 + if (seam) seam.
write(
1, text);
461 + else writeSync(
1, text);
464 +function appendProgressEntry(pi
: ExtensionAPI, entry
: RunEntry)
: void {
465 + pi.
appendEntry(
"overkill-run", entry);
468 +function showProgress(ctx
: ExtensionCommandContext, text
: string)
: void {
469 + const mode
= commandMode(ctx);
470 + if (mode
=== "rpc") ctx.ui.
setStatus(
"overkill", text);
471 + if (mode
=== "tui")
472 + ctx.ui.
setWidget(
"overkill-progress", [text], {
473 + placement:
"belowEditor",
477 +function appendProgress(
479 + ctx
: ExtensionCommandContext,
482 + const mode
= commandMode(ctx);
484 + entry.terminal
!== null || 485 + entry.stage
=== "preflight" || 486 + entry.stage
=== "interrupted" || 487 + (entry.kind
=== "aftermath" && 488 + entry.stage
=== "mining" && 489 + entry.completed
=== undefined);
490 + if (mode
!== "tui" || durable)
appendProgressEntry(pi, entry);
491 + const progress
= formatProgress(entry);
492 + showProgress(ctx, progress);
493 + if (mode
=== "print")
writePrint(progress);
496 +function clearProgress(ctx
: ExtensionCommandContext)
: void {
497 + if (
commandMode(ctx)
=== "rpc") ctx.ui.
setStatus(
"overkill",
undefined);
498 + if (
commandMode(ctx)
=== "tui")
499 + ctx.ui.
setWidget(
"overkill-progress",
undefined);
502 +function requestCancellation(active
: ActiveRun)
: void {
503 + if (active.cancelRequested)
return;
504 + active.cancelRequested
= true;
505 + active.controller.
abort();
508 +function installCancellationListeners(
510 + ctx
: ExtensionCommandContext,
512 + const mode
= commandMode(ctx);
513 + const cleanup
: Array<()
=> void>
= [];
514 + if (mode
=== "tui")
516 + ctx.ui.
onTerminalInput((data)
=> {
517 + if (
!matchesKey(data, Key.escape))
return undefined;
518 + requestCancellation(active);
519 + return { consume:
true };
522 + if (mode
=== "json" || mode
=== "print") {
523 + const onSignal = ()
=> requestCancellation(active);
524 + process.
once(
"SIGINT", onSignal);
525 + cleanup.
push(()
=> process.
removeListener(
"SIGINT", onSignal));
527 + active.
removeCancellationListeners = ()
=> {
528 + for (
const remove
of cleanup.
splice(
0))
remove();
532 +async function withActiveRun(
533 + ctx
: ExtensionCommandContext,
534 + operation: (active
: ActiveRun)
=> Promise<
void>,
535 + kind
: ActiveRun[
"kind"]
= "suite",
537 + if (activeRun)
throw new Error(
"an Overkill rollout is already active");
538 + const active
: ActiveRun = {
541 + sessionId: ctx.sessionManager.
getSessionId(),
542 + controller:
new AbortController(),
543 + suppressOutput:
false,
545 + activeRun
= active;
546 + installCancellationListeners(active, ctx);
547 + const completed
= (
async ()
=> {
549 + await operation(active);
551 + if (active.controller.signal.aborted) {
552 + await active.
interrupt?.();
553 + throw new RolloutCancelledError();
556 + await active.
interrupt?.();
562 + active.
removeCancellationListeners?.();
563 + clearProgress(ctx);
564 + if (activeRun
=== active) activeRun
= undefined;
567 + active.completed
= completed;
568 + return await completed;
571 +async function checkpointInterrupted(
573 + manifest
: RunManifest,
575 + const training
= await readTrainingCheckpoint(directory);
577 + await persistTrainingCheckpoint(directory, {
579 + stage:
"interrupted",
581 + manifest.stage
= "interrupted";
582 + manifest.terminal
= null;
583 + await persistManifest(directory, manifest);
586 +function throwIfCancelled(active
: ActiveRun)
: void {
587 + if (active.controller.signal.aborted)
throw new RolloutCancelledError();
590 +type CoreTaskAdapter = {
591 + items: (split
: "train" | "validation" | "test")
=> readonly ValidatedItem[];
592 + scoreResponse: (options
: {
593 + item
: ValidatedItem;
596 + signal
?: AbortSignal;
597 + })
=> Promise<{ score
: number; passed
: boolean; feedback
?: string }>;
598 + onTargetCall?: ()
=> void | Promise<
void>;
601 +async function executeBaseline(options
: {
603 + ctx
: ExtensionCommandContext;
605 + manifest
: RunManifest;
606 + source
: ReturnType<
typeof parseSourceSkill>;
607 + loadedSkill
: Awaited<
ReturnType<
typeof resolveLoadedSkill>>;
608 + suite
?: Awaited<
ReturnType<
typeof validateTrustedSuite>>;
609 + targetModel
: unknown;
610 + taskAdapter
?: CoreTaskAdapter;
611 + reportExtras?: ()
=> Record<
string,
unknown>;
612 + deferCompletionNotice
?: boolean;
614 + const { ctx, manifest }
= options;
615 + const itemsFor = (split
: "train" | "validation" | "test")
=> 616 + options.taskAdapter
617 + ? options.taskAdapter.
items(split)
618 + : getValidatedSuiteItems(
621 + throw new Error(
"suite tasks are unavailable");
623 + split
=== "validation" ? "val" : split,
625 + const scorer
= options.taskAdapter
626 + ? { scoreResponse: options.taskAdapter.scoreResponse }
627 + : { executeChecker };
628 + const active
= activeRun;
629 + if (
!active
|| active.sessionId
!== ctx.sessionManager.
getSessionId())
630 + throw new Error(
"Overkill run ownership is unavailable");
631 + active.runId
= manifest.runId;
632 + active.
interrupt = async ()
=> 633 + await checkpointInterrupted(options.directory, manifest);
634 + throwIfCancelled(active);
635 + const controller
= active.controller;
636 + const acceptanceSeam
= 637 + manifest.kind
=== "aftermath" 638 + ? installedAftermathAcceptanceSeam()
639 + : installedExplicitSuiteAcceptanceSeam();
640 + const recordTargetAttempt = async ()
=> {
641 + if (options.taskAdapter?.onTargetCall) {
642 + await options.taskAdapter.
onTargetCall();
645 + if (
!options.suite
|| !manifest.suite)
646 + throw new Error(
"suite call contract is unavailable");
647 + const maximum
= upperBoundCalls(options.suite.splitCounts).
target;
648 + if (manifest.calls.
target >= maximum)
649 + throw new Error(
"target call maximum exceeded");
651 + fingerprintSource(
await readFile(manifest.
source.path))
!== 652 + manifest.
source.fingerprint
654 + throw new Error(
"source fingerprint changed");
656 + (
await fingerprintSuite(options.suite.suitePath))
!== 657 + manifest.suite.fingerprint
659 + throw new Error(
"suite fingerprint changed");
660 + manifest.calls.
target += 1;
661 + await persistManifest(options.directory, manifest);
663 + let progress
: Pick<
665 + "completed" | "total" | "currentScore" | "bestScore" 667 + const operation
= (
async ()
=> {
668 + manifest.stage
= "baseline";
669 + await persistManifest(options.directory, manifest);
671 + const result
= await runValidationBaseline(
673 + directory: options.directory,
675 + agentDir:
getAgentDir(),
676 + sourcePath: manifest.
source.path,
677 + sourceFingerprint: manifest.
source.fingerprint,
678 + source: options.
source,
679 + loadedSkill: options.loadedSkill,
680 + model: manifest.model,
681 + suite: options.suite,
682 + items:
itemsFor(
"validation"),
683 + signal: controller.signal,
684 + onTargetCall: recordTargetAttempt,
685 + onCheckerCall:
async ()
=> {
686 + if (options.taskAdapter)
return;
687 + manifest.calls.checker
+= 1;
688 + await persistManifest(options.directory, manifest);
690 + onProgress:
async (next)
=> {
692 + completed: next.completed,
694 + ...(next.mean
=== undefined 696 + : { currentScore: next.mean, bestScore: next.mean }),
698 + if (
!active.suppressOutput)
699 + appendProgress(options.pi, ctx,
runEntry({ manifest }, progress));
703 + createTarget: ()
=> 704 + createIsolatedTargetSession(
707 + agentDir:
getAgentDir(),
708 + sourcePath: manifest.
source.path,
709 + loadedSkill: options.loadedSkill,
710 + candidatePath:
join(options.directory,
"baseline-snapshot.md"),
711 + model: manifest.model,
712 + targetModel: options.targetModel,
714 + acceptanceSeam?.targetSdk
?? defaultTargetSdk,
719 + manifest.stage
= "baseline_complete";
720 + manifest.scores
= {
721 + ...manifest.scores,
722 + baseline: result.mean,
723 + current: result.mean,
726 + await persistManifest(options.directory, manifest);
727 + if (controller.signal.aborted)
throw new RolloutCancelledError();
728 + if (
!active.suppressOutput)
729 + appendProgress(options.pi, ctx,
runEntry({ manifest }, progress));
730 + if (result.mean
=== 1) {
731 + const reportPath
= await persistFinalReport(options.directory, {
733 + baselineScore: result.mean,
734 + bestScore: result.mean,
737 + targetCalls: manifest.calls.
target,
738 + checkerCalls: manifest.calls.checker,
739 + terminal:
"perfect",
740 + ...options.
reportExtras?.(),
742 + if (controller.signal.aborted)
throw new RolloutCancelledError();
743 + manifest.stage
= "complete";
744 + manifest.terminal
= "no_improvement";
745 + manifest.artifacts.report
= "report.json";
746 + await persistManifest(options.directory, manifest);
747 + if (
!active.suppressOutput) {
748 + const entry
= runEntry({ manifest }, progress, {
752 + appendProgress(options.pi, ctx, entry);
753 + const paths
= formatResultPaths({
754 + sourcePath: manifest.
source.path,
758 + if (
commandMode(ctx)
=== "print")
760 + `Baseline already scores 1; no strictly better candidate exists, so optimization was skipped.\n${paths}`,
763 + commandMode(ctx)
!== "json" && 764 + !options.deferCompletionNotice
767 + "Baseline already scores 1; no strictly better candidate exists, so optimization was skipped.",
773 + const baselineCache
= await readScoreCache(options.directory);
774 + if (
!baselineCache)
775 + throw new Error(
"validation baseline cache is unavailable");
776 + const sourceHash
= createHash(
"sha256")
777 + .
update(options.
source.
body,
"utf8")
779 + const saved
= await readTrainingCheckpoint(options.directory);
780 + const readSnapshotBody = async (hash
: string, filename
: string)
=> {
781 + if (hash
=== sourceHash)
return options.
source.
body;
782 + const snapshot
= await readFile(
783 + join(options.directory, filename),
786 + if (
!snapshot.
startsWith(options.
source.frontmatter))
787 + throw new Error(
"candidate snapshot frontmatter is unsafe");
788 + const body
= snapshot.
slice(options.
source.frontmatter.
length);
789 + if (
createHash(
"sha256").
update(body,
"utf8").
digest(
"hex")
!== hash)
790 + throw new Error(
"candidate snapshot identity is unsafe");
793 + const readAttempt = async (filename
: string)
: Promise<
AttemptedPatch>
=> {
794 + const value
: unknown = JSON.
parse(
795 + await readFile(
join(options.directory, filename),
"utf8"),
797 + if (
typeof value
!== "object" || value
=== null || Array.
isArray(value))
798 + throw new Error(
"attempt artifact is malformed");
799 + const artifact
= value
as Record<
string,
unknown>;
800 + if (
!Array.
isArray(artifact.reports))
801 + throw new Error(
"attempt artifact reports are malformed");
803 + patch:
parsePatch(artifact.patch),
804 + reports: artifact.reports
as AttemptedPatch[
"reports"],
805 + ...(
typeof artifact.validationScore
=== "number" 806 + ? { validationScore: artifact.validationScore }
810 + let latestTraining
: DurableTrainingState;
812 + const attempts
= await Promise.
all(
813 + saved.attemptedPatchFiles.
map(readAttempt),
815 + const pendingArtifact
= saved.pendingPatch
816 + ? await readAttempt(saved.pendingPatch.file)
819 + nextStep: saved.nextStep,
820 + body:
await readSnapshotBody(
821 + saved.currentBodyHash,
822 + saved.currentSnapshot,
824 + bestBody:
await readSnapshotBody(
825 + saved.bestBodyHash,
826 + saved.bestSnapshot,
828 + baselineScore: saved.baselineScore,
829 + currentScore: saved.currentScore,
830 + bestScore: saved.bestScore,
831 + accepted: saved.accepted,
832 + optimizerCalls: saved.optimizerCalls,
833 + consecutiveRejections: saved.consecutiveRejections,
834 + rejections: saved.rejections.
map(({ edits,
...rejection })
=> ({
836 + ...(edits
=== undefined 838 + : { edits:
parsePatch({ edits }).edits }),
841 + ...(pendingArtifact
&& saved.pendingPatch
844 + step: saved.pendingPatch.step,
845 + body:
await readSnapshotBody(
846 + saved.pendingPatch.bodyHash,
847 + saved.currentSnapshot,
849 + patch: pendingArtifact.patch,
853 + ...(saved.terminal
=== undefined ? {}
: { terminal: saved.terminal }),
854 + ...(saved.finalTest
=== undefined 858 + body:
await readSnapshotBody(
859 + saved.finalTest.bodyHash,
860 + saved.bestSnapshot,
862 + completedItemIds: saved.finalTest.completedItemIds,
863 + total: saved.finalTest.total,
864 + ...(saved.finalTest.score
=== undefined 866 + : { score: saved.finalTest.score }),
868 + ...(saved.finalTest.score
=== undefined 870 + : { finalTestScore: saved.finalTest.score }),
872 + ...(saved.pendingOptimizer
=== undefined 874 + : { pendingOptimizer: saved.pendingOptimizer }),
877 + latestTraining
= createTrainingState({
878 + body: options.
source.
body,
879 + baseline: baselineCache.scores,
881 + const checkpointTraining = async (state
: DurableTrainingState)
=> {
882 + latestTraining
= state;
883 + if (state.pendingPatch)
884 + await writePrivateText(
886 + `attempt-${state.pendingPatch.step}.json`,
887 + `${JSON.stringify({ patch: state.pendingPatch.patch, reports: [] })}\n`,
889 + for (
const [index, attempt]
of state.attempts.
entries())
890 + await writePrivateText(
892 + `attempt-${index + 1}.json`,
893 + `${JSON.stringify(attempt)}\n`,
895 + const snapshotName = (body
: string)
=> {
896 + const hash
= createHash(
"sha256").
update(body,
"utf8").
digest(
"hex");
897 + return hash
=== sourceHash
898 + ? "baseline-snapshot.md" 899 + : `candidate-${hash}.md`;
903 + state.accepted
> 0 && 904 + state.finalTestScore
=== undefined 907 + await persistTrainingCheckpoint(options.directory, {
910 + nextStep: state.nextStep,
911 + currentBodyHash:
createHash(
"sha256")
912 + .
update(state.
body,
"utf8")
914 + bestBodyHash:
createHash(
"sha256")
915 + .
update(state.bestBody,
"utf8")
917 + currentSnapshot:
snapshotName(state.
body),
918 + bestSnapshot:
snapshotName(state.bestBody),
919 + ...(state.terminal
=== undefined ? {}
: { terminal: state.terminal }),
920 + baselineScore: state.baselineScore,
921 + currentScore: state.currentScore,
922 + bestScore: state.bestScore,
923 + accepted: state.accepted,
924 + optimizerCalls: state.optimizerCalls,
925 + consecutiveRejections: state.consecutiveRejections,
926 + rejections: state.rejections.
map(
927 + ({ reason, delta, reports, edits })
=> ({
930 + ...(reports
=== undefined ? {}
: { reports }),
931 + ...(edits
=== undefined ? {}
: { edits }),
934 + attemptedPatchFiles: state.attempts.
map(
935 + (_, index)
=> `attempt-${index + 1}.json`,
937 + ...(state.pendingPatch
940 + step: state.pendingPatch.step,
941 + bodyHash:
createHash(
"sha256")
942 + .
update(state.pendingPatch.
body,
"utf8")
944 + file:
`attempt-${state.pendingPatch.step}.json`,
948 + ...(state.pendingOptimizer
=== undefined 950 + : { pendingOptimizer: state.pendingOptimizer }),
951 + ...(state.finalTest
954 + bodyHash:
createHash(
"sha256")
955 + .
update(state.finalTest.
body,
"utf8")
957 + completedItemIds: state.finalTest.completedItemIds,
958 + total: state.finalTest.total,
959 + ...(state.finalTest.score
=== undefined 961 + : { score: state.finalTest.score }),
966 + manifest.stage
= stage;
967 + manifest.scores
= {
968 + ...manifest.scores,
969 + current: state.currentScore,
970 + best: state.bestScore,
971 + ...(state.finalTestScore
=== undefined 973 + : { test: state.finalTestScore }),
975 + manifest.calls.optimizer
= state.optimizerCalls;
976 + await persistManifest(options.directory, manifest);
977 + if (
!active.suppressOutput)
984 + ...(state.finalTest
986 + completed: state.finalTest.completedItemIds.
length,
987 + total: state.finalTest.total,
990 + currentScore: state.currentScore,
991 + bestScore: state.bestScore,
996 + const training
= await runDurableTrainingSchedule(latestTraining, {
997 + evaluate:
async (body, split, onProgress)
=> 998 + await splitScores(split, body, onProgress),
999 + optimize:
async (input, onCall, onProgress)
=> 1000 + await runOptimizerStep(
1003 + createSession: ()
=> 1004 + createOptimizerSession(
1007 + agentDir:
getAgentDir(),
1008 + model: manifest.model,
1009 + targetModel: options.targetModel,
1011 + acceptanceSeam?.optimizerSdk
?? defaultOptimizerSdk,
1014 + controller.signal,
1015 + latestTraining.pendingOptimizer,
1019 + checkpoint: checkpointTraining,
1020 + testItemIds:
async ()
=> itemsFor(
"test").
map((item)
=> item.
id),
1021 + signal: controller.signal,
1023 + if (controller.signal.aborted)
throw new RolloutCancelledError();
1024 + const reportPath
= await persistFinalReport(options.directory, {
1026 + baselineScore: training.baselineScore,
1027 + bestScore: training.bestScore,
1028 + accepted: training.accepted,
1029 + optimizerCalls: training.optimizerCalls,
1030 + targetCalls: manifest.calls.
target,
1031 + checkerCalls: manifest.calls.checker,
1032 + terminal: training.terminal,
1033 + rejections: training.rejections,
1034 + attempts: training.attempts,
1035 + ...(training.testScore
=== undefined 1037 + : { testScore: training.testScore }),
1038 + ...options.
reportExtras?.(),
1040 + const stagedPath
= 1041 + training.accepted
> 0 1042 + ? await writePrivateText(
1043 + options.directory,
1044 + "staged-skill.md",
1045 + `${options.source.frontmatter}${training.bestBody}`,
1048 + await checkpointTraining(latestTraining);
1049 + if (controller.signal.aborted)
throw new RolloutCancelledError();
1050 + const completeCheckpoint
= await readTrainingCheckpoint(
1051 + options.directory,
1053 + if (
!completeCheckpoint)
1054 + throw new Error(
"training checkpoint is unavailable");
1055 + await persistTrainingCheckpoint(options.directory, {
1056 + ...completeCheckpoint,
1057 + stage:
"complete",
1058 + reportFile:
"report.json",
1059 + ...(stagedPath
=== null ? {}
: { stagedFile:
"staged-skill.md" }),
1061 + manifest.stage
= "complete";
1062 + manifest.terminal
= training.accepted
> 0 ? "staged" : "no_improvement";
1063 + manifest.scores
= {
1064 + ...manifest.scores,
1065 + best: training.bestScore,
1066 + ...(training.testScore
=== undefined 1068 + : { test: training.testScore }),
1070 + manifest.calls.optimizer
= training.optimizerCalls;
1071 + manifest.artifacts.report
= "report.json";
1072 + manifest.artifacts.staged
= 1073 + stagedPath
=== null ? null : "staged-skill.md";
1074 + await persistManifest(options.directory, manifest);
1075 + if (controller.signal.aborted)
throw new RolloutCancelledError();
1076 + if (
!active.suppressOutput) {
1077 + const entry
= runEntry(
1079 + { currentScore: training.bestScore, bestScore: training.bestScore },
1080 + { reportPath, stagedPath },
1082 + appendProgress(options.pi, ctx, entry);
1083 + const paths
= formatResultPaths({
1084 + sourcePath: manifest.
source.path,
1088 + if (
commandMode(ctx)
=== "print")
writePrint(paths);
1089 + else if (
commandMode(ctx)
!== "json" && !options.deferCompletionNotice)
1090 + ctx.ui.
notify(paths,
"info");
1093 + async function splitScores(
1094 + split
: "train" | "validation" | "test",
1096 + onProgress?: (progress
: {
1097 + completed
: number;
1099 + })
=> Promise<
void>,
1101 + const hash
= createHash(
"sha256").
update(body,
"utf8").
digest(
"hex");
1102 + const rollout
= await runSplitRollout(
1104 + directory: options.directory,
1106 + agentDir:
getAgentDir(),
1107 + sourcePath: manifest.
source.path,
1108 + sourceFingerprint: manifest.
source.fingerprint,
1109 + source: options.
source,
1110 + loadedSkill: options.loadedSkill,
1111 + model: manifest.model,
1112 + suite: options.suite,
1113 + items:
itemsFor(split),
1116 + signal: controller.signal,
1118 + onTargetCall: recordTargetAttempt,
1119 + onCheckerCall:
async ()
=> {
1120 + if (options.taskAdapter)
return;
1121 + manifest.calls.checker
+= 1;
1122 + await persistManifest(options.directory, manifest);
1126 + createTarget: ()
=> 1127 + createIsolatedTargetSession(
1130 + agentDir:
getAgentDir(),
1131 + sourcePath: manifest.
source.path,
1132 + loadedSkill: options.loadedSkill,
1133 + candidatePath:
join(
1134 + options.directory,
1135 + `candidate-${hash}.md`,
1137 + model: manifest.model,
1138 + targetModel: options.targetModel,
1140 + acceptanceSeam?.targetSdk
?? defaultTargetSdk,
1145 + return rollout.scores;
1149 + error
instanceof RolloutCancelledError || 1150 + error
instanceof OptimizerCancelledError || 1151 + controller.signal.aborted
1153 + await checkpointInterrupted(options.directory, manifest);
1154 + const entry
= runEntry({ manifest }, progress);
1155 + if (active.suppressOutput)
appendProgressEntry(options.pi, entry);
1156 + else appendProgress(options.pi, ctx, entry);
1157 + throw new RolloutCancelledError();
1162 + return await operation;
1165 +export async function harvestAftermath(options
: {
1166 + ctx
: ExtensionCommandContext;
1167 + knownSkillPaths
: ReadonlyMap<
string,
string>;
1168 + sessionApi
?: AftermathSessionApi;
1169 +})
: Promise<{ digests
: Digest[]; total
: number; unreadable
: number }> {
1171 + options.sessionApi
?? 1172 + (SessionManager
as unknown as NonNullable<
typeof options.sessionApi>);
1173 + const sessionDir
= options.ctx.sessionManager.
getSessionDir();
1174 + const listed
= await api.
listAll(sessionDir);
1175 + const snapshots
= new Map<
string,
Digest>();
1176 + const unreadableIds
= new Set<
string>();
1177 + for (
const listedSession
of listed) {
1179 + const session
= api.
open(listedSession.path, sessionDir);
1180 + const id
= session.
getSessionId();
1185 + cwd: session.
getCwd(),
1186 + branch: session.
getBranch(),
1187 + knownSkillPaths: options.knownSkillPaths,
1191 + unreadableIds.
add(listedSession.
id);
1194 + const current
= options.ctx.sessionManager;
1195 + const currentId
= current.
getSessionId();
1199 + sessionId: currentId,
1200 + cwd: options.ctx.cwd,
1201 + branch: current.
getBranch(),
1202 + knownSkillPaths: options.knownSkillPaths,
1205 + unreadableIds.
delete(currentId);
1206 + const currentWasListed
= listed.
some((session)
=> session.
id === currentId);
1208 + digests: [
...snapshots.
values()].
sort((left, right)
=> 1209 + left.sessionId.
localeCompare(right.sessionId),
1211 + total: listed.
length + (currentWasListed
? 0 : 1),
1212 + unreadable: unreadableIds.
size,
1216 +type LoadedSkillTarget = { name
: string; path
: string };
1218 +function loadedSkills(pi
: ExtensionAPI)
: LoadedSkillTarget[] {
1221 + .
flatMap((command)
=> {
1223 + command.
source !== "skill" || 1224 + typeof command.sourceInfo?.path
!== "string" 1229 + name: command.
name.
replace(
/^skill:/,
""),
1230 + path: command.sourceInfo.path,
1236 + left.
name.
localeCompare(right.
name)
|| 1237 + left.path.
localeCompare(right.path),
1241 +export async function chooseLoadedSkill(
1242 + ctx
: ExtensionCommandContext,
1243 + skills
: readonly LoadedSkillTarget[],
1244 + canonicalPaths
: ReadonlyMap<
string,
string>,
1246 +)
: Promise<
LoadedSkillTarget | undefined> {
1247 + if (
commandMode(ctx)
!== "tui" || skills.
length <= 8) {
1248 + const labels
= skills.
map(
1250 + `${skill.name} — ${canonicalPaths.get(skill.path) ?? skill.path}`,
1252 + const choice
= await ctx.ui.
select(
1253 + `Choose exactly one ${purpose} skill.`,
1256 + return skills[labels.
indexOf(choice
?? "")];
1259 + const path
= await ctx.ui.
custom<
string | null>(
1260 + (tui, theme, _keybindings, done)
=> {
1261 + const list
= new SelectList(
1262 + skills.
map((skill)
=> {
1263 + const canonical
= canonicalPaths.
get(skill.path)
?? skill.path;
1264 + const parent
= dirname(canonical);
1266 + value: skill.path,
1267 + label: skill.
name,
1268 + description:
join(
1270 + basename(
dirname(parent)),
1272 + basename(canonical),
1276 + Math.
max(
1,
Math.
min(
8, tui.terminal.
rows - 3)),
1278 + selectedPrefix: (text)
=> theme.
fg(
"accent", text),
1279 + selectedText: (text)
=> theme.
fg(
"accent", text),
1280 + description: (text)
=> theme.
fg(
"muted", text),
1281 + scrollInfo: (text)
=> theme.
fg(
"dim", text),
1282 + noMatch: (text)
=> theme.
fg(
"warning", text),
1285 + list.
onSelect = (item)
=> done(item.
value);
1286 + list.
onCancel = ()
=> done(
null);
1288 + render: (width
: number)
=> [
1289 + theme.
fg(
"accent",
`Choose one ${purpose} skill`),
1290 + ...list.
render(width),
1291 + theme.
fg(
"dim",
"type filter · arrows move · enter · esc"),
1293 + invalidate: ()
=> list.
invalidate(),
1294 + handleInput: (data
: string)
=> {
1295 + list.
handleInput(data);
1296 + tui.
requestRender();
1301 + return skills.
find((skill)
=> skill.path
=== path);
1304 +async function runAftermath(
1306 + ctx
: ExtensionCommandContext,
1309 + const active
= activeRun;
1310 + if (active?.kind
!== "aftermath")
1311 + throw new Error(
"Aftermath run ownership is unavailable");
1312 + throwIfCancelled(active);
1313 + const explicitPath
= parts[
0];
1314 + if (parts.
length > 1)
1315 + throw new Error(
"Usage: /overkill aftermath [skill-path]");
1316 + const interrupted
= selectedAftermathRun(ctx);
1317 + if (interrupted) {
1318 + const directory
= join(
1321 + interrupted.
data.runId,
1323 + const manifest
= await readAftermathManifest(directory).
catch(
1326 + if (manifest
&& !manifest.terminal) {
1327 + const instructions
= 1328 + "An incomplete Aftermath run is linked here; use /overkill aftermath resume, discard, or cancel.";
1329 + if (explicitPath)
throw new Error(instructions);
1331 + const choice
= await ctx.ui.
select(
"Resume incomplete Aftermath run?", [
1336 + if (choice
=== "resume") {
1337 + await resumeAftermath(pi, ctx, interrupted);
1340 + if (choice
=== "discard") {
1341 + await discardLinkedRun({
1342 + runRoot:
join(
getAgentDir(),
"overkill-runs"),
1343 + runDirectory: directory,
1344 + selected: interrupted,
1345 + branch: ctx.sessionManager.
getBranch(),
1346 + sessionId: ctx.sessionManager.
getSessionId(),
1347 + allowNonterminal:
true,
1348 + isRunActive: (runId)
=> activeRun?.runId
=== runId,
1350 + appendProgress(pi, ctx, {
1351 + ...interrupted.
data,
1352 + stage:
"discarded",
1353 + terminal:
"discarded",
1358 + throw new Error(instructions);
1362 + (
commandMode(ctx)
=== "json" || commandMode(ctx)
=== "print")
&& 1366 + "JSON and print Aftermath runs require an explicit skill path",
1368 + const skills
= loadedSkills(pi);
1369 + if (skills.
length === 0) {
1370 + ctx.ui.
notify(
"No loaded Pi skills are available for Aftermath.",
"info");
1373 + const knownPaths
= new Map<
string,
string>();
1374 + const canonicalSkillPaths
= new Map<
string,
string>();
1375 + for (
const skill
of skills) {
1377 + const canonical
= await realpath(skill.path);
1378 + knownPaths.
set(canonical, skill.
name);
1379 + canonicalSkillPaths.
set(skill.path, canonical);
1382 + const canonicalExplicit
= explicitPath
1383 + ? await realpath(
resolve(ctx.cwd, explicitPath)).
catch(()
=> undefined)
1385 + let selected
= explicitPath
1388 + canonicalExplicit
!== undefined && 1389 + canonicalSkillPaths.
get(skill.path)
=== canonicalExplicit,
1392 + if (explicitPath
&& !selected)
1393 + throw new Error(
"explicit Aftermath skill path is not a loaded skill");
1394 + if (
!selected
&& ctx.hasUI)
1395 + selected
= await chooseLoadedSkill(
1398 + canonicalSkillPaths,
1401 + if (
!selected)
throw new Error(
"one confirmed Aftermath skill is required");
1402 + throwIfCancelled(active);
1403 + const sourcePath
= 1404 + canonicalSkillPaths.
get(selected.path)
?? (
await realpath(selected.path));
1405 + const parallelWarning
= await unfinishedRunWarning(sourcePath);
1406 + const harvested
= await harvestAftermath({
1408 + knownSkillPaths: knownPaths,
1409 + sessionApi:
installedAftermathAcceptanceSeam()?.sessionApi,
1411 + throwIfCancelled(active);
1412 + const evidence
= rankSkillEvidence(
1413 + harvested.digests,
1414 + new Set(skills.
map((skill)
=> skill.
name)),
1416 + const calls
= estimateMiningCalls(harvested.digests.
length);
1417 + const confirmation
= [
1418 + `Readable sessions: ${harvested.digests.length}; unreadable: ${harvested.unreadable}.`,
1419 + "Each digest is capped at 16 KiB, persisted, and sent to the model without redaction.",
1420 + `Expected mining/aggregation calls: ${calls.mining}/${calls.aggregation}; malformed structured output retries once per call.`,
1421 + `Maximum mining/aggregation calls with retries: ${calls.mining * 2}/${calls.aggregation * 2}.`,
1422 + "Maximum replay calls after mining: 116 target, 116 rubric-judge, 12 optimizer.",
1423 + ...(parallelWarning
? [
`Warning: ${parallelWarning}`]
: []),
1424 + "Overkill will not write the source skill.",
1428 + !(
await ctx.ui.
confirm(
"Persist and mine Aftermath digests?", confirmation))
1431 + throwIfCancelled(active);
1432 + const frozenModel
= ctx.model;
1433 + const targetModel
= 1434 + frozenModel
&& ctx.modelRegistry.
find(frozenModel.provider, frozenModel.
id);
1435 + if (
!frozenModel
|| !targetModel
|| !(
await modelHasAuth(ctx)))
1436 + throw new Error(
"selected model authentication is unavailable");
1437 + const sourceBytes
= await readFile(sourcePath);
1438 + throwIfCancelled(active);
1439 + parseSourceSkill(sourceBytes);
1440 + await resolveLoadedSkill(
1443 + ctx.
isProjectTrusted(),
1446 + throwIfCancelled(active);
1448 + const created
= await createAftermathDirectory(
getAgentDir());
1449 + active.runId
= created.runId;
1450 + const manifest
: AftermathManifest = {
1451 + version:
1 as const,
1452 + runId: created.runId,
1453 + kind:
"aftermath" as const,
1456 + fingerprint:
fingerprintSource(sourceBytes),
1459 + provider: frozenModel.provider,
1460 + id: frozenModel.
id,
1461 + thinking: pi.
getThinkingLevel(),
1463 + sessionId: ctx.sessionManager.
getSessionId(),
1464 + linkEntryId:
null,
1466 + miningPromptVersion:
2,
1468 + scores: { baseline:
null, current:
null, best:
null, test:
null },
1469 + calls: { target:
0, checker:
0, optimizer:
0, judge:
0 },
1471 + baselineSnapshot:
"baseline-snapshot.md",
1475 + target: { name: selected.
name, path: sourcePath },
1477 + active.
interrupt = async ()
=> 1478 + await persistAftermathManifest(created.directory, {
1480 + stage:
"interrupted",
1482 + await writePrivateText(
1483 + created.directory,
1484 + "baseline-snapshot.md",
1487 + const persisted
= await persistHarvest(created.directory, {
1488 + digests: harvested.digests,
1489 + unreadable: harvested.unreadable,
1490 + total: harvested.total,
1491 + evidence: evidence.counts,
1493 + manifest.stage
= "mining";
1494 + await persistAftermathManifest(created.directory, {
1496 + harvestFingerprint: persisted.fingerprint,
1498 + appendProgress(pi, ctx,
runEntry({ manifest }));
1499 + const link
= ctx.sessionManager.
getBranch().
at(
-1);
1500 + if (
!link
|| typeof link.
id !== "string")
1501 + throw new Error(
"could not persist Aftermath branch link");
1502 + manifest.linkEntryId
= link.
id;
1503 + await persistAftermathManifest(created.directory, {
1505 + harvestFingerprint: persisted.fingerprint,
1507 + await mineAftermath({
1510 + directory: created.directory,
1512 + digests: harvested.digests,
1514 + harvestFingerprint: persisted.fingerprint,
1519 +async function offerAftermathCompletion(
1521 + ctx
: ExtensionCommandContext,
1522 + directory
: string,
1524 + if (
!ctx.hasUI
|| (
commandMode(ctx)
!== "tui" && commandMode(ctx)
!== "rpc"))
1526 + const artifacts
= await readFrozenReviewArtifacts(directory);
1527 + const manifest
= await readAftermathManifest(directory);
1528 + const staged
= artifacts.stagedPath
!== null && artifacts.staged
!== null;
1529 + const report
= artifacts.reportPath
1530 + ? `Report: ${sanitizeTerminalText(artifacts.reportPath)}` 1532 + let summary
: string;
1535 + manifest.scores.baseline
=== null 1537 + : `baseline ${formatScore(manifest.scores.baseline)}`,
1538 + manifest.scores.best
=== null 1540 + : `validation ${formatScore(manifest.scores.best)}`,
1541 + manifest.scores.test
=== null 1543 + : `held-out ${formatScore(manifest.scores.test)}`,
1544 + ].
filter((value)
: value
is string => value
!== null);
1545 + summary
= `Aftermath complete — candidate staged.\n${scores.join(" → ")}\n${report}`;
1546 + }
else if (manifest.scores.baseline
=== null) {
1547 + const frozen
= await readFrozenTasks(directory);
1548 + summary
= `Aftermath complete — no candidate.\nOnly ${frozen.tasks.length}/3 recurring tasks; replay was skipped.\n${report}`;
1549 + }
else if (manifest.scores.baseline
=== 1) {
1550 + summary
= `Aftermath complete — no candidate needed.\nBaseline already scores 1; no improvement is possible.\n${report}`;
1552 + summary
= `Aftermath complete — no candidate.\nBest validation ${formatScore(manifest.scores.best ?? manifest.scores.baseline)} did not beat baseline ${formatScore(manifest.scores.baseline)}; no diff was staged.\n${report}`;
1554 + const choice
= await ctx.ui.
select(
1556 + staged
? [
"review diff",
"done",
"discard"]
: [
"done",
"discard report"],
1558 + if (choice
=== "review diff") {
1560 + artifacts.reportPath
1561 + ? `Report: ${sanitizeTerminalText(artifacts.reportPath)}` 1563 + artifacts.stagedPath
1564 + ? `Staged: ${sanitizeTerminalText(artifacts.stagedPath)}` 1567 + .
filter((value)
: value
is string => value
!== null)
1570 + commandMode(ctx)
=== "tui" && artifacts.stagedPath
&& artifacts.staged
1571 + ? `${paths}\n${buildReviewDiff( 1572 + artifacts.sourcePath, 1573 + artifacts.stagedPath, 1582 + if (choice
=== "done" || choice
=== undefined)
return;
1583 + if (choice
=== "discard" || choice
=== "discard report") {
1584 + const selected
= selectedAftermathRun(ctx);
1586 + throw new Error(
"Aftermath run is no longer linked to this branch");
1587 + await discardLinkedRun({
1588 + runRoot:
join(
getAgentDir(),
"overkill-runs"),
1589 + runDirectory: directory,
1591 + branch: ctx.sessionManager.
getBranch(),
1592 + sessionId: ctx.sessionManager.
getSessionId(),
1594 + appendProgress(pi, ctx, {
1596 + stage:
"discarded",
1597 + terminal:
"discarded",
1602 +async function mineAftermath(options
: {
1604 + ctx
: ExtensionCommandContext;
1605 + directory
: string;
1606 + manifest
: AftermathManifest;
1607 + digests
: Digest[];
1608 + targetModel
: unknown;
1609 + harvestFingerprint
: string;
1610 + completedChunks
?: ReadonlyMap<
number,
unknown[]>;
1611 + completedLevels
?: ReadonlyMap<
number,
unknown[][]>;
1612 + attemptedMiningCalls
?: number;
1613 + frozen
?: Awaited<
ReturnType<
typeof readFrozenTasks>>;
1614 +})
: Promise<
void> {
1615 + const active
= activeRun;
1616 + if (active?.kind
!== "aftermath")
1617 + throw new Error(
"Aftermath run ownership is unavailable");
1618 + const { ctx, manifest }
= options;
1619 + if (manifest.miningPromptVersion
!== 2)
1621 + "Aftermath run predates target-relevant mining; discard it and start a new run",
1623 + const frozenSource
= await readFile(
1624 + join(options.directory,
"baseline-snapshot.md"),
1626 + if (
fingerprintSource(frozenSource)
!== manifest.
source.fingerprint)
1627 + throw new Error(
"frozen source fingerprint changed; replay is unsafe");
1628 + const source
= parseSourceSkill(frozenSource);
1629 + if (source.
name !== manifest.
target.
name)
1630 + throw new Error(
"frozen source name changed; replay is unsafe");
1631 + manifest.stage
= "mining";
1632 + active.
interrupt = async ()
=> {
1633 + const interrupted
= {
1635 + stage:
"interrupted",
1636 + harvestFingerprint: options.harvestFingerprint,
1638 + await persistAftermathManifest(options.directory, interrupted);
1639 + const entry
= runEntry({ manifest: interrupted });
1640 + if (active.suppressOutput)
appendProgressEntry(options.pi, entry);
1641 + else appendProgress(options.pi, ctx, entry);
1643 + const chunks
= new Set(options.completedChunks?.
keys());
1644 + const levels
= new Set<
string>();
1645 + let attemptedMiningCalls
= options.attemptedMiningCalls
?? 0;
1646 + for (
const [level, outputs]
of options.completedLevels
?? [])
1647 + for (
const index
of outputs.
keys()) levels.
add(
`${level}:${index}`);
1648 + const miningTotal
= estimateMiningCalls(options.digests.
length).total;
1649 + const updateMiningProgress = (
1650 + phase
?: "mining" | "aggregation",
1652 + maximumAttempts
?: number,
1654 + const completed
= chunks.
size + levels.
size;
1655 + const progress
= formatProgress({
1658 + total: miningTotal,
1663 + ? `${progress} · ${phase} call ${Math.min(completed + 1, miningTotal)} running${attempt && attempt > 1 ? ` · retry ${attempt}/${maximumAttempts}` : ""}` 1667 + const persistCheckpoint = async ()
=> 1668 + await persistMiningCheckpoint(options.directory, {
1669 + attemptedCalls: attemptedMiningCalls,
1670 + completedChunks: [
...chunks],
1671 + completedLevels: [
...levels].
map((item)
=> {
1672 + const [levelText, indexText]
= item.
split(
":");
1673 + return { level:
Number(levelText), index:
Number(indexText) };
1678 + (
await (
async ()
=> {
1679 + updateMiningProgress();
1680 + const proposals
= await mineTaskProposals({
1681 + digests: options.digests,
1682 + target: { name: source.
name, description: source.
description },
1683 + createSession: ()
=> 1684 + createMiningSession(
1687 + agentDir:
getAgentDir(),
1688 + model: manifest.model,
1689 + targetModel: options.targetModel,
1691 + installedAftermathAcceptanceSeam()?.structuredOutputSdk,
1693 + signal: active.controller.signal,
1694 + completedChunks: options.completedChunks,
1695 + completedLevels: options.completedLevels,
1696 + onCall:
async (phase, attempt, maximumAttempts)
=> {
1697 + attemptedMiningCalls
+= 1;
1698 + await persistCheckpoint();
1699 + updateMiningProgress(phase, attempt, maximumAttempts);
1701 + onChunk:
async (index, output)
=> {
1702 + await persistMiningOutput(
1703 + options.directory,
1704 + `miner-chunk-${index}.json`,
1707 + chunks.
add(index);
1708 + await persistCheckpoint();
1709 + updateMiningProgress();
1711 + onLevel:
async (level, index, output)
=> {
1712 + await persistMiningOutput(
1713 + options.directory,
1714 + `aggregate-${level}-${index}.json`,
1717 + levels.
add(
`${level}:${index}`);
1718 + await persistCheckpoint();
1719 + updateMiningProgress();
1722 + throwIfCancelled(active);
1723 + const tasks
= validateAndFreezeTasks({
1725 + harvestedSessionIds:
new Set(
1726 + options.digests.
map((digest)
=> digest.sessionId),
1729 + const splits
= assignSplits(tasks);
1730 + const persisted
= await persistFrozenTasks(
1731 + options.directory,
1735 + return {
...persisted, tasks, splits };
1737 + const { tasks, splits }
= frozen;
1739 + manifest.taskFingerprint
!== undefined && 1740 + manifest.taskFingerprint
!== frozen.fingerprint
1742 + throw new Error(
"frozen task fingerprint changed; resume is unsafe");
1743 + manifest.harvestFingerprint
= options.harvestFingerprint;
1744 + manifest.taskFingerprint
= frozen.fingerprint;
1745 + const callBounds
= upperBoundCalls({
1746 + train: splits.train.
length,
1747 + val: splits.validation.
length,
1748 + test: splits.test.
length,
1750 + const taskById
= new Map(tasks.
map((task)
=> [task.
id, task]));
1751 + const rubricCount = (split
: "train" | "validation" | "test")
=> 1752 + splits[split].
filter((id)
=> taskById.
get(id)?.rubric
!== undefined).
length;
1753 + const judgeMaximum
= 1754 + rubricCount(
"validation")
+ 1755 + 3 * (
rubricCount(
"train")
+ rubricCount(
"validation"))
+ 1756 + rubricCount(
"test");
1757 + manifest.scorers
= tasks.
map((task)
=> ({
1759 + kind: task.checks
? (
"deterministic" as const)
: (
"rubric" as const),
1761 + manifest.splits
= splits;
1762 + let costs
= (
await readAftermathCost(options.directory))
?? {
1763 + targetMaximum: tasks.
length < 3 ? 0 : callBounds.
target,
1764 + judgeMaximum: tasks.
length < 3 ? 0 : judgeMaximum,
1765 + targetConsumed:
0,
1769 + costs.targetMaximum
!== (tasks.
length < 3 ? 0 : callBounds.
target)
|| 1770 + costs.judgeMaximum
!== (tasks.
length < 3 ? 0 : judgeMaximum)
1772 + throw new Error(
"Aftermath replay call maximum changed; resume is unsafe");
1773 + manifest.calls.
target = costs.targetConsumed;
1774 + manifest.calls.judge
= costs.judgeConsumed;
1775 + await persistAftermathCost(options.directory, costs);
1776 + const callNotice
= `Replay budget: ≤${costs.targetMaximum} target + ${costs.judgeMaximum} judge calls; malformed judge retries consume this budget.`;
1777 + if (
commandMode(ctx)
=== "print")
writePrint(callNotice);
1778 + else if (
commandMode(ctx)
!== "json") ctx.ui.
notify(callNotice,
"info");
1779 + const allowCall = (kind
: "target" | "judge")
=> {
1781 + kind
=== "target" ? costs.targetMaximum
: costs.judgeMaximum;
1783 + kind
=== "target" ? costs.targetConsumed
: costs.judgeConsumed;
1784 + if (consumed
>= maximum)
1785 + throw new Error(
`Aftermath ${kind} call maximum exceeded`);
1787 + const recordCall = async (kind
: "target" | "judge")
=> {
1790 + kind
=== "target" && 1791 + fingerprintSource(
await readFile(manifest.
source.path))
!== 1792 + manifest.
source.fingerprint
1794 + throw new Error(
"source fingerprint changed; replay is unsafe");
1797 + ...(kind
=== "target" 1798 + ? { targetConsumed: costs.targetConsumed
+ 1 }
1799 + : { judgeConsumed: costs.judgeConsumed
+ 1 }),
1801 + await persistAftermathCost(options.directory, costs);
1802 + manifest.calls[kind]
= 1803 + kind
=== "target" ? costs.targetConsumed
: costs.judgeConsumed;
1804 + await persistAftermathManifest(options.directory, manifest);
1806 + throwIfCancelled(active);
1807 + if (tasks.
length < 3) {
1808 + const reportPath
= await persistFinalReport(options.directory, {
1810 + kind:
"aftermath",
1811 + taskFingerprint: frozen.fingerprint,
1812 + taskIds: tasks.
map((task)
=> task.
id),
1814 + targetCallMaximum: costs.targetMaximum,
1815 + judgeCallMaximum: costs.judgeMaximum,
1816 + targetCalls: costs.targetConsumed,
1817 + judgeCalls: costs.judgeConsumed,
1818 + terminal:
"insufficient_tasks",
1820 + manifest.stage
= "complete";
1821 + manifest.terminal
= "no_improvement";
1822 + manifest.artifacts.report
= "report.json";
1823 + await persistAftermathManifest(options.directory, manifest);
1824 + const entry
= runEntry(
1826 + { completed: tasks.
length, total:
3 },
1827 + { reportPath, stagedPath:
null },
1829 + appendProgress(options.pi, ctx, entry);
1830 + if (
commandMode(ctx)
=== "print")
1832 + `Aftermath complete — no candidate.\nOnly ${tasks.length}/3 recurring tasks; replay was skipped.\nReport: ${sanitizeTerminalText(reportPath)}`,
1834 + await offerAftermathCompletion(options.pi, ctx, options.directory);
1837 + await persistAftermathManifest(options.directory, manifest);
1838 + const currentSource
= await readFile(manifest.
source.path);
1839 + if (
fingerprintSource(currentSource)
!== manifest.
source.fingerprint)
1840 + throw new Error(
"source fingerprint changed; replay is unsafe");
1841 + const loadedSkill
= await resolveLoadedSkill(
1842 + manifest.
source.path,
1843 + options.pi.
getCommands(),
1844 + ctx.
isProjectTrusted(),
1847 + const byId
= new Map(tasks.
map((task)
=> [task.
id, task]));
1848 + const taskItems = (split
: "train" | "validation" | "test")
=> 1849 + splits[split].
map((id)
=> {
1850 + const task
= byId.
get(id);
1851 + if (
!task)
throw new Error(
"frozen split task is unavailable");
1852 + return { id, prompt:
generatedTaskPrompt(task) };
1854 + const judgeCache
= await readJudgeCache(options.directory);
1855 + const scorer
= createGeneratedTaskScorer(tasks, {
1856 + judge:
async (task, response, signal)
=> {
1857 + const key
= rubricCacheKey(task.
id, response);
1858 + const cached
= judgeCache[key];
1859 + if (cached)
return cached;
1860 + const result
= await runRubricJudge(
1864 + createSession: ()
=> 1865 + createRubricJudgeSession(
1868 + agentDir:
getAgentDir(),
1869 + model: manifest.model,
1870 + targetModel: options.targetModel,
1872 + installedAftermathAcceptanceSeam()?.structuredOutputSdk,
1876 + async ()
=> await recordCall(
"judge"),
1878 + judgeCache[result.cacheKey]
= {
1879 + score: result.score,
1880 + passed: result.passed,
1881 + rationale: result.rationale,
1883 + await persistJudgeCache(options.directory, judgeCache);
1887 + await executeBaseline({
1890 + directory: options.directory,
1894 + targetModel: options.targetModel,
1897 + scoreResponse:
async ({ item, response, signal })
=> 1898 + await scorer({ id: item.
id, response, signal }),
1899 + onTargetCall:
async ()
=> await recordCall(
"target"),
1901 + reportExtras: ()
=> ({
1902 + taskFingerprint: frozen.fingerprint,
1903 + targetCallMaximum: costs.targetMaximum,
1904 + judgeCallMaximum: costs.judgeMaximum,
1905 + targetCalls: costs.targetConsumed,
1906 + judgeCalls: costs.judgeConsumed,
1908 + deferCompletionNotice:
true,
1910 + await offerAftermathCompletion(options.pi, ctx, options.directory);
1913 +async function resumeAftermath(
1915 + ctx
: ExtensionCommandContext,
1916 + selected
: NonNullable<
ReturnType<
typeof selectedAftermathRun>>,
1918 + const active
= activeRun;
1919 + if (active?.kind
!== "aftermath")
1920 + throw new Error(
"Aftermath run ownership is unavailable");
1921 + const directory
= join(
getAgentDir(),
"overkill-runs", selected.
data.runId);
1922 + let manifest
= await readAftermathManifest(directory);
1923 + if (manifest.terminal)
1924 + throw new Error(
"only incomplete Aftermath runs can resume");
1926 + manifest.runId
!== selected.
data.runId
|| 1927 + manifest.sessionId
!== ctx.sessionManager.
getSessionId()
1929 + throw new Error(
"run is not linked to this active session branch");
1930 + manifest
= (
await recoverRunLink({
1934 + branch: ctx.sessionManager.
getBranch(),
1935 + }))
as AftermathManifest;
1936 + const sourcePath
= await realpath(manifest.
source.path);
1937 + const source
= await readFile(sourcePath);
1938 + if (
fingerprintSource(source)
!== manifest.
source.fingerprint)
1939 + throw new Error(
"source fingerprint changed; resume is unsafe");
1940 + const targetModel
= ctx.modelRegistry.
find(
1941 + manifest.model.provider,
1942 + manifest.model.
id,
1946 + !(
await modelHasAuth(ctx, manifest.model.provider, manifest.model.
id))
1948 + throw new Error(
"recorded model authentication is unavailable");
1949 + const harvest
= await readHarvest(directory);
1950 + if (harvest.fingerprint
!== manifest.harvestFingerprint)
1951 + throw new Error(
"harvest fingerprint changed; resume is unsafe");
1952 + const frozen
= await readFrozenTasks(directory).
catch((error)
=> {
1953 + if ((error
as NodeJS.
ErrnoException).
code === "ENOENT")
return undefined;
1956 + const checkpoint
= await readMiningCheckpoint(directory);
1957 + const chunks
= new Map<
number,
unknown[]>();
1958 + const levels
= new Map<
number,
unknown[][]>();
1959 + for (
const index
of checkpoint?.completedChunks
?? [])
1962 + await readMiningOutput(directory,
`miner-chunk-${index}.json`),
1964 + for (
const { level, index }
of checkpoint?.completedLevels
?? []) {
1965 + const outputs
= levels.
get(level)
?? [];
1966 + outputs[index]
= await readMiningOutput(
1968 + `aggregate-${level}-${index}.json`,
1970 + levels.
set(level, outputs);
1972 + active.runId
= manifest.runId;
1973 + await mineAftermath({
1978 + digests: harvest.digests,
1980 + harvestFingerprint: harvest.fingerprint,
1981 + completedChunks: chunks,
1982 + completedLevels: levels,
1983 + attemptedMiningCalls: checkpoint?.attemptedCalls
?? 0,
1988 +async function resumeBaseline(
1990 + ctx
: ExtensionCommandContext,
1991 + selected
: NonNullable<
ReturnType<
typeof selectedRun>>,
1993 + const directory
= join(
getAgentDir(),
"overkill-runs", selected.
data.runId);
1994 + const active
= activeRun;
1995 + if (
!active
|| active.sessionId
!== ctx.sessionManager.
getSessionId())
1996 + throw new Error(
"Overkill run ownership is unavailable");
1997 + active.runId
= selected.
data.runId;
1998 + active.
interrupt = async ()
=> 1999 + await checkpointInterrupted(directory,
await readManifest(directory));
2000 + let manifest
= await readManifest(directory);
2001 + if (manifest.terminal)
throw new Error(
"selected run is terminal");
2003 + manifest.runId
!== selected.
data.runId
|| 2004 + manifest.sessionId
!== ctx.sessionManager.
getSessionId()
2006 + throw new Error(
"run is not linked to this active session branch");
2007 + manifest
= await recoverRunLink({
2011 + branch: ctx.sessionManager.
getBranch(),
2013 + active.runId
= manifest.runId;
2014 + active.
interrupt = async ()
=> 2015 + await checkpointInterrupted(directory, manifest);
2016 + throwIfCancelled(active);
2017 + const originalLink
= ctx.sessionManager.
getBranch().
find((entry)
=> {
2018 + if (
typeof entry
!== "object" || entry
=== null)
return false;
2019 + const candidate
= entry
as {
2022 + customType
?: unknown;
2026 + candidate.
id !== manifest.linkEntryId
|| 2027 + candidate.
type !== "custom" || 2028 + candidate.customType
!== "overkill-run" 2032 + const data
= validateRunEntry(candidate.
data);
2033 + return data.kind
=== "suite" && data.runId
=== manifest.runId;
2038 + if (
!originalLink)
2039 + throw new Error(
"run is no longer linked to this active branch");
2040 + const sourcePath
= await realpath(manifest.
source.path);
2041 + throwIfCancelled(active);
2042 + const sourceBytes
= await readFile(sourcePath);
2043 + throwIfCancelled(active);
2044 + if (
fingerprintSource(sourceBytes)
!== manifest.
source.fingerprint)
2045 + throw new Error(
"source fingerprint changed; resume is unsafe");
2046 + if (
!manifest.suite)
throw new Error(
"suite run has no suite contract");
2047 + const suite
= await validateTrustedSuite(
2048 + manifest.suite.path,
2050 + ctx.
isProjectTrusted(),
2052 + throwIfCancelled(active);
2053 + if ((
await fingerprintSuite(suite.suitePath))
!== manifest.suite.fingerprint)
2054 + throw new Error(
"suite fingerprint changed; resume is unsafe");
2055 + throwIfCancelled(active);
2056 + const model
= ctx.modelRegistry.
find(
2057 + manifest.model.provider,
2058 + manifest.model.
id,
2062 + !(
await modelHasAuth(ctx, manifest.model.provider, manifest.model.
id))
2064 + throw new Error(
"recorded model authentication is unavailable");
2065 + const loadedSkill
= await resolveLoadedSkill(
2068 + ctx.
isProjectTrusted(),
2071 + throwIfCancelled(active);
2072 + await executeBaseline({
2077 + source:
parseSourceSkill(sourceBytes),
2080 + targetModel: model,
2084 +export async function shutdownOverkill(ctx
: ExtensionContext)
: Promise<
void> {
2085 + const sessionId
= ctx.sessionManager.
getSessionId();
2087 + if (startPending
&& pendingSessionId
=== sessionId) pendingCancelled
= true;
2090 + if (activeRun.sessionId
!== sessionId)
return;
2091 + activeRun.suppressOutput
= true;
2092 + requestCancellation(activeRun);
2094 + await activeRun.completed;
2098 +export default function overkill(pi
: ExtensionAPI)
: void {
2099 + pi.
on(
"session_shutdown",
async (_event, ctx)
=> {
2100 + await shutdownOverkill(ctx);
2102 + registerOverkillEntryRenderer(pi);
2103 + pi.
registerCommand(
"overkill",
createOverkillCommand(pi));
2106 +export function createOverkillCommand(pi
: ExtensionAPI) {
2109 + "Author or optimize a loaded Pi skill with trusted evaluators.",
2110 + getArgumentCompletions: (prefix
: string)
=> {
2111 + const trimmed
= prefix.
trim();
2112 + if (trimmed
=== "aftermath" && !prefix.
endsWith(
" "))
return null;
2113 + if (trimmed
=== "aftermath" || trimmed.
startsWith(
"aftermath ")) {
2114 + const nested
= trimmed.
slice(
"aftermath".
length).
trim();
2115 + const choices
= [
"status",
"resume",
"review",
"discard",
"cancel"]
2116 + .
filter((command)
=> command.
startsWith(nested))
2117 + .
map((command)
=> ({
2118 + value:
`aftermath ${command}`,
2119 + label:
`aftermath ${command}`,
2121 + return choices.
length > 0 ? choices
: null;
2123 + const choices
= SUBCOMMANDS.
filter((command)
=> 2124 + command.
startsWith(trimmed),
2125 + ).
map((command)
=> ({ value: command, label: command }));
2126 + return choices.
length > 0 ? choices
: null;
2128 + handler:
async (args
: string, ctx
: ExtensionCommandContext)
=> {
2129 + let parts
: string[];
2131 + parts
= parseArguments(args);
2134 + `Overkill command failed: ${error instanceof Error ? error.message : String(error)}`,
2139 + const command
= parts[
0];
2140 + if (command
=== "suite") {
2141 + let [sourcePath, suitePath,
...extra]
= parts.
slice(
1);
2142 + if (extra.
length > 0) {
2144 + "Usage: /overkill suite [skill-path] [suite-dir]",
2149 + if (
!ctx.
isIdle()) {
2150 + ctx.ui.
notify(
"Agent is busy.",
"error");
2154 + if (
!ctx.
isProjectTrusted())
2155 + throw new Error(
"suite authoring requires a trusted project");
2156 + if (
!sourcePath) {
2159 + "headless suite authoring requires an explicit skill path",
2161 + const skills
= loadedSkills(pi);
2162 + if (skills.
length === 0)
2163 + throw new Error(
"no loaded Pi skills are available");
2164 + const canonicalPaths
= new Map<
string,
string>();
2165 + for (
const skill
of skills)
2166 + canonicalPaths.
set(skill.path,
await realpath(skill.path));
2167 + const selected
= await chooseLoadedSkill(
2173 + if (
!selected)
return;
2174 + sourcePath
= selected.path;
2176 + const loaded
= await resolveLoadedSkill(
2182 + const projectRoot
= await realpath(
resolve(ctx.cwd));
2184 + suitePath
= await canonicalProspectivePath(
2185 + resolve(projectRoot, suitePath),
2187 + const projectRelative
= relative(projectRoot, suitePath);
2188 + if (projectRelative.
startsWith(
"..")
|| isAbsolute(projectRelative))
2189 + throw new Error(
"suite directory must be inside the project");
2191 + const parallelWarning
= await unfinishedRunWarning(
2195 + if (parallelWarning) ctx.ui.
notify(parallelWarning,
"warning");
2196 + pi.
sendUserMessage(
2197 + buildSuiteAuthoringPrompt(loaded.path, suitePath, projectRoot),
2201 + `Overkill suite authoring failed: ${error instanceof Error ? error.message : String(error)}`,
2207 + if (command
=== "aftermath") {
2208 + const subcommand
= parts[
1];
2210 + [
"status",
"resume",
"review",
"discard",
"cancel"].
includes(
2213 + parts.
length !== 2 2215 + ctx.ui.
notify(
`Usage: /overkill aftermath ${subcommand}`,
"error");
2218 + if (subcommand
=== "status") {
2219 + const run
= selectedAftermathRun(ctx);
2222 + "No Overkill Aftermath run is linked to this branch.",
2225 + else if (
commandMode(ctx)
=== "json")
2226 + appendProgress(pi, ctx, run.
data);
2227 + else if (
commandMode(ctx)
=== "print")
2228 + writePrint(
statusMessage(run.
data));
2229 + else ctx.ui.
notify(
statusMessage(run.
data),
"info");
2232 + if (subcommand
=== "cancel") {
2235 + activeRun.kind
=== "aftermath" && 2236 + activeRun.sessionId
=== ctx.sessionManager.
getSessionId()
2238 + requestCancellation(activeRun);
2239 + else ctx.ui.
notify(
"No Overkill Aftermath run is active.",
"info");
2242 + if ([
"resume",
"review",
"discard"].
includes(subcommand
?? "")) {
2243 + const run
= selectedAftermathRun(ctx);
2246 + "No Overkill Aftermath run is linked to this branch.",
2251 + const directory
= join(
2257 + const manifest
= await readAftermathManifest(directory);
2258 + if (manifest.sessionId
!== ctx.sessionManager.
getSessionId())
2260 + "run is not linked to this active session branch",
2262 + if (subcommand
=== "review") {
2263 + const artifacts
= await readFrozenReviewArtifacts(directory);
2266 + sourcePath: artifacts.sourcePath,
2267 + reportPath: artifacts.reportPath,
2268 + stagedPath: artifacts.stagedPath,
2270 + const paths
= formatResultPaths(entry);
2271 + if (
commandMode(ctx)
=== "json")
appendProgress(pi, ctx, entry);
2272 + else if (
commandMode(ctx)
=== "print") {
2273 + writePrint(paths);
2274 + if (artifacts.stagedPath
&& artifacts.staged)
2277 + artifacts.sourcePath,
2278 + artifacts.stagedPath,
2283 + else writePrint(
"No candidate staged.");
2286 + artifacts.stagedPath
2288 + : `${paths}\nNo candidate staged.`,
2292 + commandMode(ctx)
=== "tui" && 2293 + artifacts.stagedPath
&& 2298 + artifacts.sourcePath,
2299 + artifacts.stagedPath,
2306 + }
else if (subcommand
=== "discard") {
2307 + if (manifest.terminal
=== "discarded")
2308 + throw new Error(
"discarded run artifacts are unavailable");
2309 + await discardLinkedRun({
2310 + runRoot:
join(
getAgentDir(),
"overkill-runs"),
2311 + runDirectory: directory,
2313 + branch: ctx.sessionManager.
getBranch(),
2314 + sessionId: ctx.sessionManager.
getSessionId(),
2315 + allowNonterminal:
true,
2316 + isRunActive: (runId)
=> activeRun?.runId
=== runId,
2318 + appendProgress(pi, ctx, {
2320 + stage:
"discarded",
2321 + terminal:
"discarded",
2323 + clearProgress(ctx);
2324 + }
else if (manifest.terminal) {
2326 + "Aftermath mining is already complete; stored tasks are frozen.",
2330 + await withActiveRun(
2332 + async ()
=> await resumeAftermath(pi, ctx, run),
2337 + if (
!(error
instanceof RolloutCancelledError))
2339 + `Overkill Aftermath ${subcommand} failed: ${error instanceof Error ? error.message : String(error)}`,
2345 + if (activeRun
|| startPending) {
2347 + "This Pi process already has an Overkill rollout active; wait or cancel it before starting another.",
2352 + startPending
= true;
2353 + pendingSessionId
= ctx.sessionManager.
getSessionId();
2354 + pendingCancelled
= false;
2356 + await withActiveRun(
2358 + async ()
=> await runAftermath(pi, ctx, parts.
slice(
1)),
2362 + if (
!(error
instanceof RolloutCancelledError))
2364 + `Overkill Aftermath failed: ${error instanceof Error ? error.message : String(error)}`,
2368 + startPending
= false;
2369 + pendingSessionId
= undefined;
2370 + pendingCancelled
= false;
2375 + [
"status",
"discard",
"cancel",
"review",
"resume"].
includes(
2378 + parts.
length !== 1 2380 + ctx.ui.
notify(
`Usage: /overkill ${command}`,
"error");
2383 + if (command
=== "status") {
2384 + const run
= selectedRun(ctx);
2387 + "No Overkill suite run is linked to this branch.",
2392 + const mode
= commandMode(ctx);
2393 + if (mode
=== "json")
appendProgress(pi, ctx, run.
data);
2394 + else if (mode
=== "print")
writePrint(
statusMessage(run.
data));
2395 + else ctx.ui.
notify(
statusMessage(run.
data),
"info");
2398 + if (command
=== "discard") {
2399 + const run
= selectedRun(ctx);
2402 + "No Overkill suite run is linked to this branch.",
2408 + await discardLinkedRun({
2409 + runRoot:
join(
getAgentDir(),
"overkill-runs"),
2410 + runDirectory:
join(
getAgentDir(),
"overkill-runs", run.
data.runId),
2412 + branch: ctx.sessionManager.
getBranch(),
2413 + sessionId: ctx.sessionManager.
getSessionId(),
2414 + allowNonterminal:
true,
2415 + isRunActive: (runId)
=> activeRun?.runId
=== runId,
2417 + appendProgress(pi, ctx, {
2419 + stage:
"discarded",
2420 + terminal:
"discarded",
2422 + clearProgress(ctx);
2423 + ctx.ui.
notify(
"Overkill run discarded.",
"info");
2426 + `Overkill discard failed: ${error instanceof Error ? error.message : String(error)}`,
2432 + if (command
=== "cancel") {
2435 + activeRun.sessionId
!== ctx.sessionManager.
getSessionId()
2437 + ctx.ui.
notify(
"No Overkill rollout is active.",
"info");
2440 + requestCancellation(activeRun);
2441 + if (
commandMode(ctx)
!== "json")
2442 + ctx.ui.
notify(
"Overkill cancellation requested.",
"info");
2445 + if (command
=== "review") {
2446 + const run
= selectedRun(ctx);
2449 + "No Overkill suite run is linked to this branch.",
2454 + if (
!run.
data.terminal) {
2455 + ctx.ui.
notify(
"Overkill run is not terminal yet.",
"info");
2458 + if (run.
data.terminal
=== "discarded") {
2460 + "Overkill run was discarded; review artifacts are unavailable.",
2465 + let artifacts
: Awaited<
ReturnType<
typeof readFrozenReviewArtifacts>>;
2467 + artifacts
= await readFrozenReviewArtifacts(
2468 + join(
getAgentDir(),
"overkill-runs", run.
data.runId),
2472 + `Overkill review failed: ${error instanceof Error ? error.message : String(error)}`,
2479 + sourcePath: artifacts.sourcePath,
2480 + reportPath: artifacts.reportPath,
2481 + stagedPath: artifacts.stagedPath,
2483 + const paths
= formatResultPaths(entry);
2484 + const mode
= commandMode(ctx);
2485 + if (mode
=== "json")
appendProgress(pi, ctx, entry);
2486 + else if (mode
=== "print") {
2487 + writePrint(paths);
2488 + if (artifacts.stagedPath
&& artifacts.staged)
2491 + artifacts.sourcePath,
2492 + artifacts.stagedPath,
2497 + else writePrint(
"No candidate staged.");
2500 + artifacts.stagedPath
? paths
: `${paths}\nNo candidate staged.`,
2503 + if (mode
=== "tui" && artifacts.stagedPath
&& artifacts.staged)
2506 + artifacts.sourcePath,
2507 + artifacts.stagedPath,
2516 + if (command
=== "resume") {
2517 + if (activeRun
|| startPending) {
2519 + "This Pi process already has an Overkill rollout active; wait or cancel it before resuming another.",
2524 + const run
= selectedRun(ctx);
2527 + "No Overkill suite run is linked to this branch.",
2532 + startPending
= true;
2533 + pendingSessionId
= ctx.sessionManager.
getSessionId();
2534 + pendingCancelled
= false;
2536 + await withActiveRun(
2538 + async ()
=> await resumeBaseline(pi, ctx, run),
2542 + error
instanceof RolloutCancelledError || 2543 + error
instanceof OptimizerCancelledError 2547 + `Overkill resume failed: ${error instanceof Error ? error.message : String(error)}`,
2551 + startPending
= false;
2552 + pendingSessionId
= undefined;
2553 + pendingCancelled
= false;
2557 + const [sourcePath, suitePath,
...extra]
= parts;
2558 + if (
!sourcePath
|| !suitePath
|| extra.
length > 0) {
2559 + ctx.ui.
notify(
"Usage: /overkill <skill-path> <suite-dir>",
"error");
2562 + if (activeRun
|| startPending) {
2564 + "This Pi process already has an Overkill rollout active; wait or cancel it before starting another.",
2569 + const invokingModel
= ctx.model;
2570 + const targetModel
= 2572 + ctx.modelRegistry.
find(invokingModel.provider, invokingModel.
id);
2573 + startPending
= true;
2574 + pendingSessionId
= ctx.sessionManager.
getSessionId();
2575 + pendingCancelled
= false;
2577 + if (invokingModel
&& !targetModel)
2579 + "selected model is unavailable from the invoking registry",
2581 + const preflightOptions
= {
2582 + agentDir:
getAgentDir(),
2586 + commands: pi.
getCommands(),
2587 + projectTrusted: ctx.
isProjectTrusted(),
2588 + sessionId: ctx.sessionManager.
getSessionId(),
2589 + model: invokingModel
2591 + provider: invokingModel.provider,
2592 + id: invokingModel.
id,
2593 + thinking: pi.
getThinkingLevel(),
2596 + authAvailable:
await modelHasAuth(
2598 + invokingModel?.provider,
2599 + invokingModel?.
id,
2602 + const inspected
= await inspectSuiteRun(preflightOptions);
2603 + const inspectedSuiteFingerprint
= inspected.suite.fingerprint;
2604 + if (
!inspectedSuiteFingerprint)
2605 + throw new Error(
"validated suite fingerprint is unavailable");
2606 + const confirmedFingerprints
= {
2607 + sourceFingerprint:
fingerprintSource(inspected.sourceBytes),
2608 + suiteFingerprint: inspectedSuiteFingerprint,
2610 + const parallelWarning
= await unfinishedRunWarning(
2611 + inspected.sourcePath,
2612 + inspected.suite.suitePath,
2614 + if (
commandMode(ctx)
=== "tui" || commandMode(ctx)
=== "rpc") {
2615 + const confirmed
= await ctx.ui.
confirm(
2616 + "Start Overkill run?",
2617 + buildConfirmation({
2618 + sourcePath: inspected.sourcePath,
2619 + suitePath: inspected.suite.suitePath,
2620 + splitCounts: inspected.suite.splitCounts,
2621 + warning: parallelWarning,
2625 + ctx.ui.
notify(
"Overkill start cancelled.",
"info");
2629 + if (pendingCancelled)
throw new RolloutCancelledError();
2630 + await withActiveRun(ctx,
async (active)
=> {
2631 + throwIfCancelled(active);
2632 + const run
= await prepareSuiteRun(
2634 + confirmedFingerprints,
2636 + active.runId
= run.manifest.runId;
2637 + active.
interrupt = async ()
=> 2638 + await checkpointInterrupted(run.directory, run.manifest);
2639 + throwIfCancelled(active);
2640 + appendProgress(pi, ctx,
runEntry(run));
2641 + const link
= ctx.sessionManager.
getBranch().
at(
-1);
2642 + if (
!link
|| typeof link.
id !== "string")
2643 + throw new Error(
"could not persist branch link");
2644 + run.manifest.linkEntryId
= link.
id;
2645 + await persistManifest(run.directory, run.manifest);
2646 + throwIfCancelled(active);
2647 + await executeBaseline({
2650 + directory: run.directory,
2651 + manifest: run.manifest,
2652 + source: run.
source,
2653 + loadedSkill: run.loadedSkill,
2660 + error
instanceof RolloutCancelledError || 2661 + error
instanceof OptimizerCancelledError 2665 + `Overkill run failed: ${error instanceof Error ? error.message : String(error)}`,
2669 + startPending
= false;
2670 + pendingSessionId
= undefined;
2671 + pendingCancelled
= false;