pi-ext
bugabingas pi extensions
owned by bugabinga
improve ultra result inspection
7f4ca029f9aed581541f2754fe6650127b38ca8d
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (9)
extensions/ultra/__tests__/engine.test.ts
Modified436 436 describe(
"runWorkflow — result shape", ()
=> {
437 - it(
"returns exactly { workflow, phases:[{id,ran,ok,dropped}], result, steered }",
async ()
=> {
437 + it(
"persists every phase's positional results for later inspection",
async ()
=> {
438 + const fr
= fakeRunner((args)
=> {
439 + if (args.phase
=== "review") {
440 + return args.item
=== "drop" 441 + ? { ok:
false, value:
null, reason:
"bad schema" }
442 + : { ok:
true, value: { findings: [{ title:
String(args.item), file:
"a.ts" }] } };
444 + return { ok:
true, value: { title:
"refuted", real:
false, file:
"a.ts" } };
448 + return:
"{verify.results | where real}",
453 + over: [
"kept",
"drop"],
454 + step: { prompt:
"review {item}" },
459 + over:
"{review.results[].findings[]}",
460 + step: { prompt:
"verify {item}" },
465 + const res
= await runWorkflow(
468 + baseDeps({ stepRunner: fr.runner, concurrency:
2 }),
471 + expect(res.result).
toEqual([]);
472 + expect(res.phaseResults.review).
toEqual([
473 + { ok:
true, value: { findings: [{ title:
"kept", file:
"a.ts" }] } },
474 + { ok:
false, value:
null, reason:
"bad schema" },
476 + expect(res.phaseResults.verify).
toEqual([
477 + { ok:
true, value: { title:
"refuted", real:
false, file:
"a.ts" } },
481 + it(
"returns { workflow, phases:[{id,ran,ok,dropped}], phaseResults, result, steered }",
async ()
=> {
438 482 const fr
= fakeRunner((args)
=> ({ ok:
true, value: args.item
?? "v" }));
453 497 expect(
Object.
keys(res).
sort()).
toEqual([
459 504 expect(res).
toEqual({
460 505 workflow:
"demo",
461 506 phases: [{ id:
"p", ran:
1, ok:
1, dropped:
0 }],
507 + phaseResults: { p: [{ ok:
true, value:
"a" }] },
extensions/ultra/__tests__/progress.test.ts
Modified171 171 expect(s.
rows[
0].
action).not.
toContain(
"RAW RESULT");
174 + it(
"summarizes structured_output rows instead of dumping JSON", ()
=> {
180 + toolName:
"structured_output",
183 + { title:
"A very long finding title", file:
"a.ts" },
184 + { title:
"Another finding", file:
"b.ts" },
187 + text:
JSON.
stringify({ findings: [{ title:
"A very long finding title" }] }),
191 + expect(s.
rows[
0].
action).
toBe(
"structured output · 2 findings");
192 + expect(s.
rows[
0].
action).not.
toContain(
"{\"findings\"");
174 195 it(
"folds a scripted stream into one upserted row + transcript", ()
=> {
176 197 pe({ agentId:
"a", phase:
"p1", kind:
"start" }),
extensions/ultra/__tests__/ui.test.ts
Modified297 297 expect(out).
toContain(
"BBB-DELTA");
300 + it(
"auto-focuses the first agent in a new phase so verifier transcripts are visible during the run", ()
=> {
301 + const { overlay }
= makeOverlay({ rows:
12 });
302 + for (
let i
= 0; i
< 3; i
++) {
303 + overlay.
handleProgress(
304 + pe({ agentId:
`review#${i}`, phase:
"review", kind:
"start" }),
306 + overlay.
handleProgress(
307 + pe({ agentId:
`review#${i}`, phase:
"review", kind:
"end", status:
"done" }),
311 + overlay.
handleProgress(
312 + pe({ agentId:
"verify#0", phase:
"verify", kind:
"start" }),
314 + overlay.
handleProgress(
315 + pe({ agentId:
"verify#0", phase:
"verify", kind:
"delta", text:
"VERIFY-DELTA" }),
318 + const out
= overlay.
render(
60).
join(
"\n");
319 + expect(out).
toContain(
"transcript: verify#0");
320 + expect(out).
toContain(
"VERIFY-DELTA");
300 323 it(
"keeps rendered overlay within terminal height minus a 3-row safety margin", ()
=> {
301 324 const { overlay }
= makeOverlay({ rows:
12 });
302 325 for (
let i
= 0; i
< 8; i
++) {
extensions/ultra/__tests__/wire.test.ts
Modified 3 3 import { describe, expect, it, vi }
from "vitest";
4 4 import type { UltraProgressEvent }
from "../progress.ts";
5 5 import type { DiscoveredWorkflow, WorkflowSpec }
from "../spec.ts";
7 7 completeWorkflowNames,
8 + formatUltraResultMarkdown,
8 9 formatWorkflowResultMessage,
9 10 makeToolProgressSink,
39 +describe(
"formatUltraResultMarkdown", ()
=> {
40 + it(
"renders empty final result with refuted findings and drops", ()
=> {
41 + const markdown
= formatUltraResultMarkdown({
44 + { id:
"review", ran:
3, ok:
2, dropped:
1 },
45 + { id:
"verify", ran:
2, ok:
2, dropped:
0 },
49 + { ok:
false, value:
null, reason:
"timeout" },
54 + { title:
"possible issue", file:
"a.ts" },
63 + title:
"possible issue",
66 + why:
"not reproducible",
75 + expect(markdown).
toContain(
"# ultra review");
76 + expect(markdown).
toContain(
"No final results");
77 + expect(markdown).
toContain(
"review: 2/3 ok, 1 dropped");
78 + expect(markdown).
toContain(
"review#0: timeout");
79 + expect(markdown).
toContain(
"Refuted findings");
80 + expect(markdown).
toContain(
"possible issue");
81 + expect(markdown).
toContain(
"not reproducible");
84 + it(
"renders final result titles before raw JSON", ()
=> {
85 + const markdown
= formatUltraResultMarkdown({
87 + phases: [{ id:
"verify", ran:
1, ok:
1, dropped:
0 }],
89 + result: [{ title:
"real bug", file:
"src/a.ts", why:
"confirmed" }],
93 + expect(markdown).
toContain(
"Final results");
94 + expect(markdown).
toContain(
"real bug");
95 + expect(markdown).
toContain(
"src/a.ts");
96 + expect(markdown).not.
toContain(
'```json');
99 + it(
"includes raw JSON only when expanded", ()
=> {
101 + workflow:
"review",
108 + expect(
formatUltraResultMarkdown(result)).not.
toContain(
'```json');
109 + expect(
formatUltraResultMarkdown(result,
true)).
toContain(
'```json');
38 117 describe(
"formatWorkflowResultMessage", ()
=> {
39 118 it(
"formats the aggregate JSON so /ultra command results enter the session", ()
=> {
40 119 const content
= formatWorkflowResultMessage({
43 122 {
id:
"review",
ran:
3,
ok:
3,
dropped:
0 },
44 123 {
id:
"verify",
ran:
2,
ok:
1,
dropped:
1 },
129 + value: { findings: [{ title:
"candidate", file:
"a.ts" }] },
133 + { ok:
true, value: { title:
"real bug", real:
true } },
46 136 result: [{ title:
"real bug", real:
true }],
50 140 expect(content).
toContain(
'ultra workflow "review" result');
51 141 expect(content).
toContain(
'"title": "real bug"');
142 + expect(content).
toContain(
'"phaseResults"');
143 + expect(content).
toContain(
'"candidate"');
52 144 expect(content).
toContain(
'"id": "verify"');
56 148 const content
= formatWorkflowResultMessage({
57 149 workflow:
"review",
58 150 phases: [{ id:
"review", ran:
3, ok:
3, dropped:
0 }],
151 + phaseResults: { review: [{ ok:
true, value: { findings: [] } }] },
extensions/ultra/engine.ts
Modified 68 +export interface PhaseSlotResult {
69 75 export interface WorkflowResult {
71 77 phases
: PhaseResult[];
79 + phaseResults
: Record<
string,
PhaseSlotResult[]>;
91 99 ):
Promise<WorkflowResult
> {
92 100 const results: Record
<string, unknown[]> = {};
101 + const phaseResults: Record
<string, PhaseSlotResult[]> = {};
93 102 const phases: PhaseResult[]
= [];
94 103 const steeredBox
= { value:
false };
95 104 const forward
= makeForwarder(deps, steeredBox);
144 - const values
= slots.
map((slot)
=> {
153 + const values
: unknown[]
= [];
154 + const outcomeSlots
: PhaseSlotResult[]
= [];
155 + for (
const slot
of slots) {
145 156 if (
isAborted(slot)) {
159 + outcomeSlots.
push({ ok:
false, value:
null, reason:
"aborted" });
165 + values.
push(slot.
value);
166 + outcomeSlots.
push({ ok:
true, value: slot.
value });
171 + outcomeSlots.
push({
174 + reason: slot.reason
?? "dropped",
158 178 results[phase.
id]
= values;
179 + phaseResults[phase.
id]
= outcomeSlots;
159 180 phases.
push({ id: phase.
id, ran, ok, dropped });
167 188 ? resolveSelector(spec.return, { args, results })
168 189 : results[lastPhase.
id];
170 - return { workflow: spec.
name, phases, result, steered: steeredBox.
value };
192 + workflow: spec.
name,
196 + steered: steeredBox.
value,
extensions/ultra/index.ts
Modified 17 17 type ExtensionContext,
20 21 } from
"@earendil-works/pi-coding-agent";
22 +import { Markdown }
from "@earendil-works/pi-tui";
21 23 import { Type }
from "typebox";
46 48 import { renderUltraResult, UltraOverlayComponent }
from "./ui.ts";
48 50 completeWorkflowNames,
51 + formatUltraResultMarkdown,
49 52 formatWorkflowResultMessage,
50 53 makeToolProgressSink,
53 56 } from
"./wire.ts";
55 58 export default function ultraExtension(pi
: ExtensionAPI) {
59 + pi.
registerMessageRenderer<
WorkflowResult>(
61 + (message, options, theme)
=> 63 + formatUltraResultMarkdown(message.details, options.expanded),
67 + {
color: (text)
=> theme.
fg(
"customMessageText", text) },
58 73 const settings = ()
: UltraSettings => extensions/ultra/progress.ts
Modified113 113 if (pattern
&& path)
return `${pattern} in ${path}`;
114 114 return pattern
?? path
?? previewArgs(args);
116 + case "structured_output":
117 + return structuredOutputSummary(args);
117 119 return previewArgs(args);
123 +function plural(n
: number, word
: string)
: string {
124 + return `${n} ${word}${n === 1 ? "" : "s"}`;
127 +function structuredOutputSummary(value
: Record<
string,
unknown>)
: string {
128 + if (
Array.
isArray(value.findings))
return plural(value.findings.
length,
"finding");
129 + if (
typeof value.real
=== "boolean") {
130 + const title
= firstLine(value.
title);
131 + return title
? `real=${value.real} · ${cap(title)}` : `real=${value.real}`;
133 + const keys
= Object.
keys(value);
134 + return keys.
length > 0 ? `returned ${keys.join(", ")}` : "returned output";
284 300 function actionLabel(event
: UltraProgressEvent)
: string {
285 301 const text
= event.args
!== undefined ? conciseArgs(
event.toolName,
event.args)
: event.
text ?? "";
302 + if (
event.toolName
=== "structured_output")
return text
? `structured output · ${text}` : "structured output";
286 303 if (
event.toolName)
287 304 return text
? `${event.toolName} ${text}` : event.toolName;
extensions/ultra/ui.ts
Modified327 327 handleProgress(
event: UltraProgressEvent):
void {
328 + const previousPhase
= this.state.phase;
328 329 this.state
= reduceProgress(this.state,
event);
330 + if (
event.kind
=== "start" && event.phase
!== previousPhase) {
331 + const nextFocus
= this.state.
rows.
findIndex(
332 + (row)
=> row.agentId
=== event.agentId,
334 + if (nextFocus
>= 0) {
335 + this.focusIndex
= nextFocus;
336 + this.followMode
= true;
337 + this.scrollOffset
= 0;
329 340 if (this.focusIndex
>= this.state.
rows.
length) {
330 341 this.focusIndex
= Math.
max(
0, this.state.
rows.
length - 1);
extensions/ultra/wire.ts
Modified 9 -import type { WorkflowResult }
from "./engine.ts";
9 +import type { PhaseSlotResult, WorkflowResult }
from "./engine.ts";
11 11 initialReducerState,
27 +function record(value
: unknown)
: Record<
string,
unknown>
| undefined {
28 + return typeof value
=== "object" && value
!== null && !Array.
isArray(value)
29 + ? (value
as Record<
string,
unknown>)
33 +function text(value
: unknown)
: string | undefined {
34 + return typeof value
=== "string" && value.
length > 0 ? value
: undefined;
37 +function itemLine(value
: unknown)
: string {
38 + const r
= record(value);
39 + if (
!r)
return `- ${JSON.stringify(value)}`;
40 + const title
= text(r.
title)
?? text(r.file)
?? JSON.
stringify(value);
41 + const file
= text(r.file);
42 + const why
= text(r.why)
?? text(r.reason)
?? text(r.detail);
43 + return `- **${title}**${file && file !== title ? ` — \`${file}\`` : ""}${why ? `\n ${why}` : ""}`;
46 +function resultItems(value
: unknown)
: unknown[] {
47 + if (
Array.
isArray(value))
return value;
48 + return value
=== undefined || value
=== null ? []
: [value];
51 +function refutedSlots(result
: WorkflowResult)
: PhaseSlotResult[] {
52 + return Object.
values(result.phaseResults)
55 + const v
= record(slot.
value);
56 + return slot.ok
&& v?.real
=== false;
60 +function dropLines(result
: WorkflowResult)
: string[] {
61 + const lines
: string[]
= [];
62 + for (
const [phase, slots]
of Object.
entries(result.phaseResults)) {
63 + slots.
forEach((slot, i)
=> {
64 + if (
!slot.ok) lines.
push(
`- ${phase}#${i}: ${slot.reason ?? "dropped"}`);
70 +export function formatUltraResultMarkdown(
71 + result
: WorkflowResult,
74 + const finalItems
= resultItems(result.result);
75 + const lines
= [
`# ultra ${result.workflow}${result.steered ? " (steered)" : ""}`];
79 + .
map((p)
=> `${p.id}: ${p.ok}/${p.ran} ok, ${p.dropped} dropped`)
80 + .
join(
" · ")
|| "No phases.",
84 + if (finalItems.
length > 0) {
85 + lines.
push(
"## Final results",
...finalItems.
map(itemLine),
"");
87 + lines.
push(
"## No final results",
"No items survived the workflow return selector.",
"");
90 + const refuted
= refutedSlots(result);
91 + if (refuted.
length > 0) {
92 + lines.
push(
"## Refuted findings",
...refuted.
map((s)
=> itemLine(s.
value)),
"");
95 + const drops
= dropLines(result);
96 + if (drops.
length > 0) lines.
push(
"## Drops",
...drops,
"");
98 + if (expanded) lines.
push(
"## Raw JSON",
"```json",
JSON.
stringify(result,
null,
2),
"```");
99 + return lines.
join(
"\n").
trimEnd();
27 102 export function formatWorkflowResultMessage(result
: WorkflowResult)
: string {
28 103 const empty
= Array.
isArray(result.result)
&& result.result.
length === 0;