repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

improve ultra result inspection

7f4ca029f9aed581541f2754fe6650127b38ca8d

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (9)

extensions/ultra/__tests__/engine.test.ts

Modified
434 434 // ---------------------------------------------------------------------------
436 436 describe("runWorkflow — result shape", () => {
437 - it("returns exactly { workflow, phases:[{id,ran,ok,dropped}], result, steered }", async () => {
437 + it("persists every phase's positional results for later inspection", async () => {
438 + const fr = fakeRunner((args) => {
439 + if (args.phase === "review") {
440 + return args.item === "drop"
441 + ? { ok: false, value: null, reason: "bad schema" }
442 + : { ok: true, value: { findings: [{ title: String(args.item), file: "a.ts" }] } };
444 + return { ok: true, value: { title: "refuted", real: false, file: "a.ts" } };
445 + });
446 + const s = spec({
447 + name: "review",
448 + return: "{verify.results | where real}",
449 + phases: [
451 + id: "review",
452 + kind: "fanout",
453 + over: ["kept", "drop"],
454 + step: { prompt: "review {item}" },
455 + },
457 + id: "verify",
458 + kind: "fanout",
459 + over: "{review.results[].findings[]}",
460 + step: { prompt: "verify {item}" },
461 + },
462 + ],
463 + });
465 + const res = await runWorkflow(
466 + s,
467 + undefined,
468 + baseDeps({ stepRunner: fr.runner, concurrency: 2 }),
469 + );
471 + expect(res.result).toEqual([]);
472 + expect(res.phaseResults.review).toEqual([
473 + { ok: true, value: { findings: [{ title: "kept", file: "a.ts" }] } },
474 + { ok: false, value: null, reason: "bad schema" },
475 + ]);
476 + expect(res.phaseResults.verify).toEqual([
477 + { ok: true, value: { title: "refuted", real: false, file: "a.ts" } },
478 + ]);
479 + });
481 + it("returns { workflow, phases:[{id,ran,ok,dropped}], phaseResults, result, steered }", async () => {
438 482 const fr = fakeRunner((args) => ({ ok: true, value: args.item ?? "v" }));
439 483 const s = spec({
440 484 name: "demo",
453 497 expect(Object.keys(res).sort()).toEqual([
498 + "phaseResults",
454 499 "phases",
455 500 "result",
456 501 "steered",
459 504 expect(res).toEqual({
460 505 workflow: "demo",
461 506 phases: [{ id: "p", ran: 1, ok: 1, dropped: 0 }],
507 + phaseResults: { p: [{ ok: true, value: "a" }] },
462 508 result: ["a"],
463 509 steered: false,

extensions/ultra/__tests__/progress.test.ts

Modified
171 171 expect(s.rows[0].action).not.toContain("RAW RESULT");
174 + it("summarizes structured_output rows instead of dumping JSON", () => {
175 + const s = fold([
176 + pe({
177 + agentId: "a",
178 + phase: "review",
179 + kind: "action",
180 + toolName: "structured_output",
181 + args: {
182 + findings: [
183 + { title: "A very long finding title", file: "a.ts" },
184 + { title: "Another finding", file: "b.ts" },
185 + ],
186 + },
187 + text: JSON.stringify({ findings: [{ title: "A very long finding title" }] }),
188 + }),
189 + ]);
191 + expect(s.rows[0].action).toBe("structured output · 2 findings");
192 + expect(s.rows[0].action).not.toContain("{\"findings\"");
193 + });
174 195 it("folds a scripted stream into one upserted row + transcript", () => {
175 196 const s = fold([
176 197 pe({ agentId: "a", phase: "p1", kind: "start" }),

extensions/ultra/__tests__/ui.test.ts

Modified
297 297 expect(out).toContain("BBB-DELTA");
300 + it("auto-focuses the first agent in a new phase so verifier transcripts are visible during the run", () => {
301 + const { overlay } = makeOverlay({ rows: 12 });
302 + for (let i = 0; i < 3; i++) {
303 + overlay.handleProgress(
304 + pe({ agentId: `review#${i}`, phase: "review", kind: "start" }),
305 + );
306 + overlay.handleProgress(
307 + pe({ agentId: `review#${i}`, phase: "review", kind: "end", status: "done" }),
308 + );
311 + overlay.handleProgress(
312 + pe({ agentId: "verify#0", phase: "verify", kind: "start" }),
313 + );
314 + overlay.handleProgress(
315 + pe({ agentId: "verify#0", phase: "verify", kind: "delta", text: "VERIFY-DELTA" }),
316 + );
318 + const out = overlay.render(60).join("\n");
319 + expect(out).toContain("transcript: verify#0");
320 + expect(out).toContain("VERIFY-DELTA");
321 + });
300 323 it("keeps rendered overlay within terminal height minus a 3-row safety margin", () => {
301 324 const { overlay } = makeOverlay({ rows: 12 });
302 325 for (let i = 0; i < 8; i++) {

extensions/ultra/__tests__/wire.test.ts

Modified
3 3 import { describe, expect, it, vi } from "vitest";
4 4 import type { UltraProgressEvent } from "../progress.ts";
5 5 import type { DiscoveredWorkflow, WorkflowSpec } from "../spec.ts";
6 6 import {
7 7 completeWorkflowNames,
8 + formatUltraResultMarkdown,
8 9 formatWorkflowResultMessage,
9 10 makeToolProgressSink,
10 11 mapPositionalArgs,
35 36 // makeToolProgressSink — folds engine events, pushes through onUpdate
36 37 // ---------------------------------------------------------------------------
39 +describe("formatUltraResultMarkdown", () => {
40 + it("renders empty final result with refuted findings and drops", () => {
41 + const markdown = formatUltraResultMarkdown({
42 + workflow: "review",
43 + phases: [
44 + { id: "review", ran: 3, ok: 2, dropped: 1 },
45 + { id: "verify", ran: 2, ok: 2, dropped: 0 },
46 + ],
47 + phaseResults: {
48 + review: [
49 + { ok: false, value: null, reason: "timeout" },
50 + {
51 + ok: true,
52 + value: {
53 + findings: [
54 + { title: "possible issue", file: "a.ts" },
55 + ],
56 + },
57 + },
58 + ],
59 + verify: [
60 + {
61 + ok: true,
62 + value: {
63 + title: "possible issue",
64 + real: false,
65 + file: "a.ts",
66 + why: "not reproducible",
67 + },
68 + },
69 + ],
70 + },
71 + result: [],
72 + steered: false,
73 + });
75 + expect(markdown).toContain("# ultra review");
76 + expect(markdown).toContain("No final results");
77 + expect(markdown).toContain("review: 2/3 ok, 1 dropped");
78 + expect(markdown).toContain("review#0: timeout");
79 + expect(markdown).toContain("Refuted findings");
80 + expect(markdown).toContain("possible issue");
81 + expect(markdown).toContain("not reproducible");
82 + });
84 + it("renders final result titles before raw JSON", () => {
85 + const markdown = formatUltraResultMarkdown({
86 + workflow: "review",
87 + phases: [{ id: "verify", ran: 1, ok: 1, dropped: 0 }],
88 + phaseResults: {},
89 + result: [{ title: "real bug", file: "src/a.ts", why: "confirmed" }],
90 + steered: false,
91 + });
93 + expect(markdown).toContain("Final results");
94 + expect(markdown).toContain("real bug");
95 + expect(markdown).toContain("src/a.ts");
96 + expect(markdown).not.toContain('```json');
97 + });
99 + it("includes raw JSON only when expanded", () => {
100 + const result = {
101 + workflow: "review",
102 + phases: [],
103 + phaseResults: {},
104 + result: [],
105 + steered: false,
106 + };
108 + expect(formatUltraResultMarkdown(result)).not.toContain('```json');
109 + expect(formatUltraResultMarkdown(result, true)).toContain('```json');
110 + });
111 +});
113 +// ---------------------------------------------------------------------------
114 +// formatWorkflowResultMessage — session/LLM payload
115 +// ---------------------------------------------------------------------------
38 117 describe("formatWorkflowResultMessage", () => {
39 118 it("formats the aggregate JSON so /ultra command results enter the session", () => {
40 119 const content = formatWorkflowResultMessage({
43 122 { id: "review", ran: 3, ok: 3, dropped: 0 },
44 123 { id: "verify", ran: 2, ok: 1, dropped: 1 },
125 + phaseResults: {
126 + review: [
128 + ok: true,
129 + value: { findings: [{ title: "candidate", file: "a.ts" }] },
130 + },
131 + ],
132 + verify: [
133 + { ok: true, value: { title: "real bug", real: true } },
134 + ],
135 + },
46 136 result: [{ title: "real bug", real: true }],
47 137 steered: false,
50 140 expect(content).toContain('ultra workflow "review" result');
51 141 expect(content).toContain('"title": "real bug"');
142 + expect(content).toContain('"phaseResults"');
143 + expect(content).toContain('"candidate"');
52 144 expect(content).toContain('"id": "verify"');
56 148 const content = formatWorkflowResultMessage({
57 149 workflow: "review",
58 150 phases: [{ id: "review", ran: 3, ok: 3, dropped: 0 }],
151 + phaseResults: { review: [{ ok: true, value: { findings: [] } }] },
59 152 result: [],
60 153 steered: false,

extensions/ultra/engine.ts

Modified
3 3 // ultra — phase engine (Task 6).
5 5 // Sequences a workflow's phases, owns the interpolation mini-syntax (it alone
6 6 // holds `args` + prior-phase `results`), fans each phase out through the
7 7 // concurrency pool, normalises drops/aborts, resolves the `return` selector, and
8 -// produces the documented aggregate `{ workflow, phases, result, steered }`.
8 +// produces the documented aggregate `{ workflow, phases, phaseResults, result, steered }`.
10 10 // Everything model/network-touching is behind the injected `stepRunner` (Task
11 11 // 5's `StepRunner` shape), so this module is fully offline-testable with a fake
65 65 dropped: number;
68 +export interface PhaseSlotResult {
69 + ok: boolean;
70 + value: unknown;
71 + reason?: string;
68 74 /** The documented aggregate the `workflow` tool / `/ultra` command returns. */
69 75 export interface WorkflowResult {
70 76 workflow: string;
71 77 phases: PhaseResult[];
78 + /** Full positional step outcomes for every phase, including drop reasons. */
79 + phaseResults: Record<string, PhaseSlotResult[]>;
72 80 result: unknown;
73 81 steered: boolean;
90 98 deps: EngineDeps,
91 99 ): Promise<WorkflowResult> {
92 100 const results: Record<string, unknown[]> = {};
101 + const phaseResults: Record<string, PhaseSlotResult[]> = {};
93 102 const phases: PhaseResult[] = [];
94 103 const steeredBox = { value: false };
95 104 const forward = makeForwarder(deps, steeredBox);
141 150 let ran = 0;
142 151 let ok = 0;
143 152 let dropped = 0;
144 - const values = slots.map((slot) => {
153 + const values: unknown[] = [];
154 + const outcomeSlots: PhaseSlotResult[] = [];
155 + for (const slot of slots) {
145 156 if (isAborted(slot)) {
146 157 dropped++;
147 - return null;
158 + values.push(null);
159 + outcomeSlots.push({ ok: false, value: null, reason: "aborted" });
160 + continue;
149 162 ran++;
150 163 if (slot.ok) {
152 - return slot.value;
165 + values.push(slot.value);
166 + outcomeSlots.push({ ok: true, value: slot.value });
167 + continue;
154 169 dropped++;
155 - return null;
156 - });
170 + values.push(null);
171 + outcomeSlots.push({
172 + ok: false,
173 + value: null,
174 + reason: slot.reason ?? "dropped",
175 + });
158 178 results[phase.id] = values;
179 + phaseResults[phase.id] = outcomeSlots;
159 180 phases.push({ id: phase.id, ran, ok, dropped });
160 181 // An all-dropped phase (`ok === 0`) is left visible in these counts and the
161 182 // workflow continues — a later phase may legitimately act on an empty set.
167 188 ? resolveSelector(spec.return, { args, results })
168 189 : results[lastPhase.id];
170 - return { workflow: spec.name, phases, result, steered: steeredBox.value };
191 + return {
192 + workflow: spec.name,
193 + phases,
194 + phaseResults,
195 + result,
196 + steered: steeredBox.value,
197 + };
173 200 // ---------------------------------------------------------------------------

extensions/ultra/index.ts

Modified
16 16 type ExtensionAPI,
17 17 type ExtensionContext,
18 18 getAgentDir,
19 + getMarkdownTheme,
19 20 SessionManager,
20 21 } from "@earendil-works/pi-coding-agent";
22 +import { Markdown } from "@earendil-works/pi-tui";
21 23 import { Type } from "typebox";
22 24 import {
23 25 type CompleteApi,
46 48 import { renderUltraResult, UltraOverlayComponent } from "./ui.ts";
47 49 import {
48 50 completeWorkflowNames,
51 + formatUltraResultMarkdown,
49 52 formatWorkflowResultMessage,
50 53 makeToolProgressSink,
51 54 parseCommandLine,
53 56 } from "./wire.ts";
55 58 export default function ultraExtension(pi: ExtensionAPI) {
59 + pi.registerMessageRenderer<WorkflowResult>(
60 + "ultra-result",
61 + (message, options, theme) =>
62 + new Markdown(
63 + formatUltraResultMarkdown(message.details, options.expanded),
64 + 0,
65 + 0,
66 + getMarkdownTheme(),
67 + { color: (text) => theme.fg("customMessageText", text) },
68 + ),
69 + );
56 71 // Settings: resolved per-invocation so the runtime `settingsManager` (which
57 72 // can change mid-session) always wins. The angel precedence pattern.
58 73 const settings = (): UltraSettings =>

extensions/ultra/progress.ts

Modified
113 113 if (pattern && path) return `${pattern} in ${path}`;
114 114 return pattern ?? path ?? previewArgs(args);
116 + case "structured_output":
117 + return structuredOutputSummary(args);
116 118 default:
117 119 return previewArgs(args);
123 +function plural(n: number, word: string): string {
124 + return `${n} ${word}${n === 1 ? "" : "s"}`;
127 +function structuredOutputSummary(value: Record<string, unknown>): string {
128 + if (Array.isArray(value.findings)) return plural(value.findings.length, "finding");
129 + if (typeof value.real === "boolean") {
130 + const title = firstLine(value.title);
131 + return title ? `real=${value.real} · ${cap(title)}` : `real=${value.real}`;
133 + const keys = Object.keys(value);
134 + return keys.length > 0 ? `returned ${keys.join(", ")}` : "returned output";
122 138 * Short preview of a tool result. Prefers `result.details` — the validated
123 139 * structured-output payload path (P1) — falling back to `result.content`, then
283 299 /** The coarse board label for an `action` event. */
284 300 function actionLabel(event: UltraProgressEvent): string {
285 301 const text = event.args !== undefined ? conciseArgs(event.toolName, event.args) : event.text ?? "";
302 + if (event.toolName === "structured_output") return text ? `structured output · ${text}` : "structured output";
286 303 if (event.toolName)
287 304 return text ? `${event.toolName} ${text}` : event.toolName;
288 305 return text;

extensions/ultra/ui.ts

Modified
325 325 // ---- progress sink: fold + throttled re-render -----------------------
327 327 handleProgress(event: UltraProgressEvent): void {
328 + const previousPhase = this.state.phase;
328 329 this.state = reduceProgress(this.state, event);
330 + if (event.kind === "start" && event.phase !== previousPhase) {
331 + const nextFocus = this.state.rows.findIndex(
332 + (row) => row.agentId === event.agentId,
333 + );
334 + if (nextFocus >= 0) {
335 + this.focusIndex = nextFocus;
336 + this.followMode = true;
337 + this.scrollOffset = 0;
329 340 if (this.focusIndex >= this.state.rows.length) {
330 341 this.focusIndex = Math.max(0, this.state.rows.length - 1);

extensions/ultra/wire.ts

Modified
6 6 // filter, and name-vs-inline spec resolution. `index.ts` is the thin SDK glue
7 7 // that wires these to the real `pi.registerTool` / `pi.registerCommand` / `ctx`.
9 -import type { WorkflowResult } from "./engine.ts";
9 +import type { PhaseSlotResult, WorkflowResult } from "./engine.ts";
10 10 import {
11 11 initialReducerState,
12 12 type ProgressMeta,
24 24 // Result message formatting
25 25 // ---------------------------------------------------------------------------
27 +function record(value: unknown): Record<string, unknown> | undefined {
28 + return typeof value === "object" && value !== null && !Array.isArray(value)
29 + ? (value as Record<string, unknown>)
30 + : undefined;
33 +function text(value: unknown): string | undefined {
34 + return typeof value === "string" && value.length > 0 ? value : undefined;
37 +function itemLine(value: unknown): string {
38 + const r = record(value);
39 + if (!r) return `- ${JSON.stringify(value)}`;
40 + const title = text(r.title) ?? text(r.file) ?? JSON.stringify(value);
41 + const file = text(r.file);
42 + const why = text(r.why) ?? text(r.reason) ?? text(r.detail);
43 + return `- **${title}**${file && file !== title ? ` — \`${file}\`` : ""}${why ? `\n ${why}` : ""}`;
46 +function resultItems(value: unknown): unknown[] {
47 + if (Array.isArray(value)) return value;
48 + return value === undefined || value === null ? [] : [value];
51 +function refutedSlots(result: WorkflowResult): PhaseSlotResult[] {
52 + return Object.values(result.phaseResults)
53 + .flat()
54 + .filter((slot) => {
55 + const v = record(slot.value);
56 + return slot.ok && v?.real === false;
57 + });
60 +function dropLines(result: WorkflowResult): string[] {
61 + const lines: string[] = [];
62 + for (const [phase, slots] of Object.entries(result.phaseResults)) {
63 + slots.forEach((slot, i) => {
64 + if (!slot.ok) lines.push(`- ${phase}#${i}: ${slot.reason ?? "dropped"}`);
65 + });
66 + }
67 + return lines;
70 +export function formatUltraResultMarkdown(
71 + result: WorkflowResult,
72 + expanded = false,
73 +): string {
74 + const finalItems = resultItems(result.result);
75 + const lines = [`# ultra ${result.workflow}${result.steered ? " (steered)" : ""}`];
76 + lines.push("");
77 + lines.push(
78 + result.phases
79 + .map((p) => `${p.id}: ${p.ok}/${p.ran} ok, ${p.dropped} dropped`)
80 + .join(" · ") || "No phases.",
81 + );
82 + lines.push("");
84 + if (finalItems.length > 0) {
85 + lines.push("## Final results", ...finalItems.map(itemLine), "");
86 + } else {
87 + lines.push("## No final results", "No items survived the workflow return selector.", "");
88 + }
90 + const refuted = refutedSlots(result);
91 + if (refuted.length > 0) {
92 + lines.push("## Refuted findings", ...refuted.map((s) => itemLine(s.value)), "");
93 + }
95 + const drops = dropLines(result);
96 + if (drops.length > 0) lines.push("## Drops", ...drops, "");
98 + if (expanded) lines.push("## Raw JSON", "```json", JSON.stringify(result, null, 2), "```");
99 + return lines.join("\n").trimEnd();
27 102 export function formatWorkflowResultMessage(result: WorkflowResult): string {
28 103 const empty = Array.isArray(result.result) && result.result.length === 0;
29 104 return [