repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

docs(ultra): document linear failure recovery

fcd115916696eac3a9196c414c235738b67077cc

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (7)

extensions/ultra/README.md

Modified
80 80 Workflow results include aggregate token and cost usage from completed sub-agent turns when the provider reports it.
81 81 The final JSON contains `tokenUsage` with `input`, `output`, `total`, `cacheRead`, `cacheWrite`, and `cost`.
83 +Every dropped step produces a typed diagnostic in `phaseFailures` with its code, message, retryability, attempt count, model, thinking level, original item, and a bounded final transcript tail when available.
84 +Collapsed output shows only short failure reasons; expanded output shows the diagnostic fields.
85 +A later phase may consume `{PHASE.failures}` through `when`, `over`, prompts, or `return` to perform one explicit linear recovery pass.
86 +Ultra never drops work because of elapsed time and never creates automatic recovery loops.
83 88 ## Settings
85 90 Read from the `ultra` block of agent settings, `<cwd>/.pi/settings.json`, and

extensions/ultra/__tests__/wire.test.ts

Modified
39 39 // ---------------------------------------------------------------------------
41 41 describe("formatUltraResultMarkdown", () => {
42 - it("renders empty final result with refuted findings and drops", () => {
43 - const markdown = formatUltraResultMarkdown({
42 + it("renders compact drops and expanded typed diagnostics", () => {
43 + const failure = {
44 + code: "transport" as const,
45 + message: "provider error",
46 + retryable: true,
47 + attempts: 3,
48 + model: "openai/gpt",
49 + thinkingLevel: "high" as const,
50 + transcriptTail: "last bad response",
51 + };
52 + const result = {
44 53 workflow: "review",
45 54 phases: [
46 55 { id: "review", ran: 3, ok: 2, dropped: 1 },
47 56 { id: "verify", ran: 2, ok: 2, dropped: 0 },
49 58 phaseResults: {
50 - review: [
51 - { ok: false, value: null, reason: "provider error" },
52 - {
53 - ok: true,
54 - value: {
55 - findings: [{ title: "possible issue", file: "a.ts" }],
56 - },
57 - },
58 - ],
59 + review: [{ ok: false as const, value: null, failure }],
59 60 verify: [
61 - ok: true,
62 + ok: true as const,
62 63 value: {
63 64 title: "possible issue",
64 65 real: false,
72 + phaseFailures: {
73 + review: [{ agentId: "review#0", index: 0, item: "scope", ...failure }],
74 + verify: [],
75 + },
71 76 result: [],
72 77 steered: false,
73 - });
78 + };
75 - expect(markdown).toContain("# ultra review");
76 - expect(markdown).toContain("No final results");
77 - expect(markdown).toContain("review: 2/3 ok, 1 dropped");
78 - expect(markdown).toContain("review#0: provider error");
79 - expect(markdown).toContain("Refuted findings");
80 - expect(markdown).toContain("possible issue");
81 - expect(markdown).toContain("not reproducible");
80 + const collapsed = formatUltraResultMarkdown(result);
81 + expect(collapsed).toContain("# ultra review");
82 + expect(collapsed).toContain("No final results");
83 + expect(collapsed).toContain("review: 2/3 ok, 1 dropped");
84 + expect(collapsed).toContain("review#0 [transport]: provider error");
85 + expect(collapsed).not.toContain("last bad response");
86 + expect(collapsed).toContain("Refuted findings");
88 + const expanded = formatUltraResultMarkdown(result, true);
89 + expect(expanded).toContain("retryable: yes");
90 + expect(expanded).toContain("attempts: 3");
91 + expect(expanded).toContain("model: openai/gpt");
92 + expect(expanded).toContain('item: "scope"');
93 + expect(expanded).toContain("last bad response");
84 96 it("renders final result titles before raw JSON", () => {
86 98 workflow: "review",
87 99 phases: [{ id: "verify", ran: 1, ok: 1, dropped: 0 }],
88 100 phaseResults: {},
101 + phaseFailures: {},
89 102 result: [{ title: "real bug", file: "src/a.ts", why: "confirmed" }],
90 103 steered: false,
101 114 workflow: "review",
102 115 phases: [],
103 116 phaseResults: {},
117 + phaseFailures: {},
104 118 result: [],
105 119 steered: false,
106 120 tokenUsage: {
121 135 workflow: "review",
122 136 phases: [],
123 137 phaseResults: {},
138 + phaseFailures: {},
124 139 result: [],
125 140 steered: false,
140 155 workflow: "review",
141 156 phases: [{ id: "p", ran: 1, ok: 1, dropped: 0 }],
142 157 phaseResults: {},
158 + phaseFailures: {},
143 159 result: [{ title: "bug" }],
144 160 steered: false,
145 161 tokenUsage: {
182 198 verify: [{ ok: true, value: { title: "real bug", real: true } }],
200 + phaseFailures: { review: [], verify: [] },
184 201 result: [{ title: "real bug", real: true }],
185 202 steered: false,
188 205 expect(content).toContain('ultra workflow "review" result');
189 206 expect(content).toContain('"title": "real bug"');
190 207 expect(content).toContain('"phaseResults"');
208 + expect(content).toContain('"phaseFailures"');
191 209 expect(content).toContain('"candidate"');
192 210 expect(content).toContain('"id": "verify"');
197 215 workflow: "review",
198 216 phases: [{ id: "review", ran: 3, ok: 3, dropped: 0 }],
199 217 phaseResults: { review: [{ ok: true, value: { findings: [] } }] },
218 + phaseFailures: { review: [] },
200 219 result: [],
201 220 steered: false,

extensions/ultra/engine.ts

Modified
3 3 // ultra — phase engine (Task 6).
5 5 // Sequences a workflow's phases, owns the interpolation mini-syntax (it alone
6 6 // holds `args` + prior-phase `results`), fans each phase out through the
7 7 // concurrency pool, normalises drops/aborts, resolves the `return` selector, and
8 -// produces the documented aggregate `{ workflow, phases, phaseResults, result, steered, tokenUsage }`.
8 +// produces the documented aggregate `{ workflow, phases, phaseResults, phaseFailures, result, steered, tokenUsage }`.
10 10 // Everything model/network-touching is behind the injected `stepRunner` (Task
11 11 // 5's `StepRunner` shape), so this module is fully offline-testable with a fake
14 14 // RESOLVED; the runner uses it verbatim (runner.ts top-of-file contract).
15 15 // - The engine assigns every planned step an `agentId` (`<phase>#<index>`).
16 16 // Failed and un-launched aborted slots receive synthetic `end` events with
17 -// reducer-safe `"dropped"` status; the free-form reason rides in `text`.
17 +// reducer-safe `"dropped"` status; the failure message rides in `text`.
18 18 // - The authoritative per-phase `dropped` count is derived from the step
19 19 // results (`isAborted` ∥ `!ok`), independent of the progress stream — the
20 20 // synthetic `end` is board-only.
110 110 * Run a validated workflow spec to completion. Phases run in order; each phase's
111 - * `results` (nulls retained positionally) feed the next phase's interpolation.
111 + * `results` and typed `failures` feed the next phase's interpolation.
112 112 * Returns the aggregate result; never throws on a step drop or an abort (those
113 113 * are normalised into the counts). A malformed interpolation token IS a spec
114 114 * authoring error and propagates (fail-fast), not a runtime drop.

extensions/ultra/index.ts

Modified
224 224 workflow: spec.name,
225 225 phases: [],
226 226 phaseResults: {},
227 + phaseFailures: {},
227 228 result: { error: message },
228 229 steered: false,
229 230 tokenUsage: {
368 369 workflow: run.workflow,
369 370 phases: [],
370 371 phaseResults: {},
372 + phaseFailures: {},
371 373 result: { status: run.status, error: run.error },
372 374 steered: false,
373 375 tokenUsage: {

extensions/ultra/skills/authoring-workflows/SKILL.md

Modified
47 47 - `fanout` needs an `over` — a literal array, or an interpolation token over
48 48 a prior phase's results (e.g. `{review.results[].findings[]}`).
49 49 - Optional `when` uses one existing selector; empty/false skips the phase and produces empty results.
50 + - A recovery fan-out may use `{PHASE.failures}` for both `when` and `over`; recovery remains one explicit later phase, never a loop.
50 51 - The step `prompt`, its `tools`, and how tightly its structured JSON output must be constrained by a schema.
52 53 3. **Apply the read-only fan-out rule.** A `fanout` step with tools must stay
60 61 worked example in the reference.
62 63 5. **Wire interpolation.** Use only the closed grammar (`{args.NAME}`, `{item}`,
63 - `{PHASE.results}`, `{PHASE.results[].FIELD[]}`, `{… | where FIELD}`). Keep
64 + `{PHASE.results}`, `{PHASE.failures}`, `{PHASE.results[].FIELD[]}`, `{… | where FIELD}`). Keep
64 65 literal `{`/`}` out of prompts except these tokens. Set `return` to the
65 66 selector for the workflow's final value.

extensions/ultra/skills/authoring-workflows/references/spec-format.md

Modified
33 33 `when` uses the existing single-selector grammar.
34 34 If it resolves to `false`, `null`, `undefined`, an empty string, or an empty array, the phase is skipped and records `{ ran: 0, ok: 0, dropped: 0 }` with empty results.
35 35 Any other value runs the phase.
36 -A skipped phase may still be referenced later through its empty `{PHASE.results}`.
36 +A skipped phase may still be referenced later through empty `{PHASE.results}` and `{PHASE.failures}` arrays.
37 37 There are no loops or arbitrary expressions.
39 39 `over` is either a literal array or a single interpolation token string (e.g.
102 102 {args.NAME} workflow input value
103 103 {item} current fanout item (string verbatim; else compact JSON)
104 104 {PHASE.results} a prior phase's results array (nulls retained positionally)
105 +{PHASE.failures} a prior phase's typed failure array
105 106 {PHASE.results[].FIELD[]} flatten: concat each result's FIELD array (nulls/missing skipped)
106 107 {EXPR | where FIELD} keep elements whose boolean FIELD is truthy
111 112 raw `{`/`}` (e.g. an inline JSON example) in a prompt; describe the shape in
112 113 words and use a `schema` instead.
115 +### Explicit linear recovery
117 +A dropped step appears in `{PHASE.failures}` with `agentId`, `index`, optional original `item`, `code`, `message`, `retryable`, `attempts`, optional `model`, optional `thinkingLevel`, and an optional 2,000-character `transcriptTail`.
118 +Failure codes are `auth`, `transport`, `invalid-json`, `schema`, `missing-output`, `aborted`, or `session`.
120 +Use the non-empty failure array to condition and fan out one later recovery phase:
122 +```json
124 + "id": "recover",
125 + "kind": "fanout",
126 + "when": "{work.failures}",
127 + "over": "{work.failures}",
128 + "step": {
129 + "prompt": "Recover this failed task and return the required result. Failure: {item}",
130 + "model": "big",
131 + "thinkingLevel": "high"
134 +```
136 +Ultra has no timeout, automatic rerun, loop, or back-edge.
137 +`ultra.maxRetries` only retries invalid tool-less completion output inside one step.
138 +Workflow authors explicitly place any recovery phase after the failed phase.
114 140 ### Schema join-keys
116 142 When one phase fans out over another's findings, give the producer schema and

extensions/ultra/wire.ts

Modified
60 -function dropLines(result: WorkflowResult): string[] {
60 +function dropLines(result: WorkflowResult, expanded: boolean): string[] {
61 61 const lines: string[] = [];
62 - for (const [phase, slots] of Object.entries(result.phaseResults)) {
63 - slots.forEach((slot, i) => {
64 - if (!slot.ok) lines.push(`- ${phase}#${i}: ${slot.reason ?? "dropped"}`);
65 - });
62 + for (const failures of Object.values(result.phaseFailures)) {
63 + for (const failure of failures) {
64 + lines.push(`- ${failure.agentId} [${failure.code}]: ${failure.message}`);
65 + if (!expanded) continue;
66 + lines.push(
67 + ` retryable: ${failure.retryable ? "yes" : "no"}`,
68 + ` attempts: ${failure.attempts}`,
69 + );
70 + if (failure.model) lines.push(` model: ${failure.model}`);
71 + if (failure.thinkingLevel)
72 + lines.push(` thinking: ${failure.thinkingLevel}`);
73 + if (failure.item !== undefined)
74 + lines.push(` item: ${JSON.stringify(failure.item)}`);
75 + if (failure.transcriptTail)
76 + lines.push(
77 + ` transcript tail: ${JSON.stringify(failure.transcriptTail)}`,
78 + );
79 + }
67 81 return lines;
114 - const drops = dropLines(result);
128 + const drops = dropLines(result, expanded);
115 129 if (drops.length > 0) lines.push("## Drops", ...drops, "");
117 131 if (expanded)