repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

feat(ultra): improve workflow planning and review results

18cd51ed5a996c01bb38afb74fa9bc35ced0fcb7

Oliver Krylow <o.krylow@isp-insoft.de> ·

SSH signature · not checked

Browse filesCompare with HEAD

Changed files (5)

extensions/ultra/README.md

Modified
53 53 (no base ref reviews the working tree; a base ref reviews everything since it).
55 55 The shipped `feature` workflow drives a small feature end to end —
56 -research → plan → adversarial critique → conditional implement → conditional security/quality review —
56 +research → plan → adversarial critique → strengthened plan → implement → conditional security/quality review —
57 57 run it with `/ultra feature <describe the task>` (the whole trailing text is the task).
58 -Implementation runs only when critique approves the plan.
58 +A dedicated read-only phase rewrites the plan to resolve critique blockers before implementation.
59 59 Only its `implement` phase writes; every other phase is read-only.
61 61 The shipped `bug` workflow drives investigate → regression test → conditional fix → adversarial verification.
65 65 ## Results
67 +Completed review command and background runs show a human-readable summary, verified and refuted findings, drops, and recommended next steps. Expand the result to inspect raw JSON.
67 69 The `workflow` tool accepts `background: true` to start a run and return immediately.
68 70 When the detached run finishes, ultra sends its result back into the conversation.
69 71 `/ultra runs` opens a saved-run picker for inspecting, resuming, stopping, or deleting runs.

extensions/ultra/__tests__/feature.test.ts

Modified
6 6 import { resolveSelector } from "../interp.ts";
7 7 import { discoverWorkflows, parseWorkflow } from "../spec.ts";
9 -// The shipped `feature` workflow: research → plan → critique → conditional
9 +// The shipped `feature` workflow: research → plan → critique → refine →
10 10 // implement → conditional review. It lives beside the bundled-workflows lookup.
11 11 const here = path.dirname(fileURLToPath(import.meta.url));
12 12 const FEATURE_PATH = path.join(here, "..", "workflows", "feature.json");
27 27 "research",
28 28 "plan",
29 29 "critique",
30 + "refine",
30 31 "implement",
31 32 "review",
45 46 research: ["small", "medium"],
46 47 plan: ["big", "high"],
47 48 critique: ["big", "xhigh"],
49 + refine: ["big", "high"],
48 50 implement: ["medium", "high"],
49 51 review: ["big", "high"],
53 55 it("takes a free-text task as a positional arg consumed by every phase", () => {
54 56 const spec = parseWorkflow(loadRaw());
55 57 expect(spec.args?.params).toEqual(["task"]);
56 - // research/critique/implement all scope to the task.
57 - for (const id of ["research", "critique", "implement"]) {
58 + // Every reasoning/writing phase remains scoped to the task.
59 + for (const id of ["research", "critique", "refine", "implement"]) {
58 60 const p = spec.phases.find((ph) => ph.id === id);
59 61 expect(p?.step.prompt).toContain("{args.task}");
66 68 spec.phases.find((p) => p.id === id)?.step.prompt ?? "";
67 69 expect(get("plan")).toContain("{research.results}");
68 70 expect(get("critique")).toContain("{plan.results}");
69 - expect(get("implement")).toContain("{plan.results}");
70 - expect(get("implement")).toContain("{critique.results}");
71 + expect(get("refine")).toContain("{plan.results}");
72 + expect(get("refine")).toContain("{critique.results}");
73 + expect(get("implement")).toContain("{refine.results}");
71 74 expect(get("review")).toContain("{implement.results}");
90 - it("implements only approved critiques and reviews only completed implementations", () => {
93 + it("strengthens the plan with critique feedback and reviews only completed implementations", () => {
91 94 const spec = parseWorkflow(loadRaw());
95 + const refine = spec.phases.find((p) => p.id === "refine");
92 96 const implement = spec.phases.find((p) => p.id === "implement");
93 97 const review = spec.phases.find((p) => p.id === "review");
95 - expect(implement?.when).toBe("{critique.results | where approved}");
99 + expect(refine?.step.prompt).toContain("resolves EVERY blocking item");
100 + expect(implement?.when).toBeUndefined();
101 + expect(implement?.step.prompt).toContain("{refine.results}");
96 102 expect(review?.when).toBe("{implement.results | where implemented}");
97 103 const implementation = spec.schemas?.Implementation as {
98 104 required?: string[];

extensions/ultra/__tests__/wire.test.ts

Modified
80 80 const collapsed = formatUltraResultMarkdown(result);
81 81 expect(collapsed).toContain("# ultra review");
82 - expect(collapsed).toContain("No final results");
82 + expect(collapsed).toContain("No verified findings.");
83 + expect(collapsed).toContain("1 candidate finding refuted.");
84 + expect(collapsed).toContain("Review incomplete: 1 step dropped.");
85 + expect(collapsed).toContain(
86 + "Inspect dropped steps and rerun review before relying on the result.",
87 + );
83 88 expect(collapsed).toContain("review: 2/3 ok, 1 dropped");
84 89 expect(collapsed).toContain("review#0 [transport]: provider error");
85 90 expect(collapsed).not.toContain("last bad response");
134 139 steered: false,
137 - expect(markdown).toContain("Final results");
142 + expect(markdown).toContain("Verified findings");
138 143 expect(markdown).toContain("real bug");
139 144 expect(markdown).toContain("src/a.ts");
140 145 expect(markdown).not.toContain("```json");
206 211 'ultra background run "review" complete: run-1',
208 213 expect(formatBackgroundResultMessage("run-1", result)).toContain(
209 - '"title": "bug"',
214 + "## Verified findings\n- **bug**",
214 219 describe("formatWorkflowResultMessage", () => {
215 - it("formats the aggregate JSON so /ultra command results enter the session", () => {
220 + it("formats a human summary so /ultra command results enter the session", () => {
216 221 const content = formatWorkflowResultMessage({
217 222 workflow: "review",
218 223 phases: [
219 224 { id: "review", ran: 3, ok: 3, dropped: 0 },
220 - { id: "verify", ran: 2, ok: 1, dropped: 1 },
225 + { id: "verify", ran: 1, ok: 1, dropped: 0 },
222 227 phaseResults: {
223 228 review: [
233 238 steered: false,
236 - expect(content).toContain('ultra workflow "review" result');
237 - expect(content).toContain('"title": "real bug"');
238 - expect(content).toContain('"phaseResults"');
239 - expect(content).toContain('"phaseFailures"');
240 - expect(content).toContain('"candidate"');
241 - expect(content).toContain('"id": "verify"');
241 + expect(content).toContain("# ultra review");
242 + expect(content).toContain("1 verified finding.");
243 + expect(content).toContain("Verified findings");
244 + expect(content).toContain("real bug");
245 + expect(content).toContain("Address verified findings, then rerun review.");
246 + expect(content).not.toContain("```json");
247 + expect(content).not.toContain("phaseResults");
244 250 it("makes an empty result explicit instead of looking like no run happened", () => {
245 251 const content = formatWorkflowResultMessage({
246 252 workflow: "review",
247 253 phases: [{ id: "review", ran: 3, ok: 3, dropped: 0 }],
248 - phaseResults: { review: [{ ok: true, value: { findings: [] } }] },
254 + phaseResults: {
255 + review: [{ ok: true, value: { findings: [] } }],
256 + verify: [
258 + ok: true,
259 + value: {
260 + title: "speculative issue",
261 + file: "a.ts",
262 + real: false,
263 + why: "not reproducible",
264 + },
265 + },
266 + ],
267 + },
249 268 phaseFailures: { review: [] },
250 269 result: [],
251 270 steered: false,
254 - expect(content).toContain("Result is empty");
255 - expect(content).toContain('"result": []');
273 + expect(content).toContain("No verified findings.");
274 + expect(content).toContain("1 candidate finding refuted.");
275 + expect(content).toContain("No code changes recommended from this review.");
276 + expect(content).toContain("Continue normal project validation.");
277 + expect(content).not.toContain("Result is empty");
278 + expect(content).not.toContain('"result": []');
279 + });
281 + it("preserves structured details for non-review workflows", () => {
282 + const content = formatWorkflowResultMessage({
283 + workflow: "feature",
284 + phases: [],
285 + phaseResults: {},
286 + phaseFailures: {},
287 + result: { plan: "ship it" },
288 + steered: false,
289 + });
291 + expect(content).toContain('ultra workflow "feature" result');
292 + expect(content).toContain('"plan": "ship it"');

extensions/ultra/wire.ts

Modified
95 95 return `Usage: ${usage.total} tokens (${usage.input} in, ${usage.output} out)${cost}`;
98 +function plural(count: number, singular: string): string {
99 + return `${count} ${singular}${count === 1 ? "" : "s"}`;
102 +function summaryLine(
103 + result: WorkflowResult,
104 + finalCount: number,
105 + refutedCount: number,
106 + droppedCount: number,
107 +): string {
108 + const incomplete =
109 + droppedCount > 0
110 + ? ` Review incomplete: ${plural(droppedCount, "step")} dropped.`
111 + : "";
112 + if (result.workflow === "review") {
113 + const findings =
114 + finalCount === 0
115 + ? "No verified findings."
116 + : `${plural(finalCount, "verified finding")}.`;
117 + const refuted =
118 + refutedCount > 0
119 + ? ` ${plural(refutedCount, "candidate finding")} refuted.`
120 + : "";
121 + return `${findings}${refuted}${incomplete}`;
123 + return `${plural(finalCount, "final result")}.${incomplete}`;
126 +function recommendationLines(
127 + result: WorkflowResult,
128 + finalCount: number,
129 + droppedCount: number,
130 +): string[] {
131 + if (result.workflow === "review") {
132 + if (droppedCount > 0)
133 + return [
134 + "- Inspect dropped steps and rerun review before relying on the result.",
135 + ];
136 + if (finalCount > 0)
137 + return ["- Address verified findings, then rerun review."];
138 + return [
139 + "- No code changes recommended from this review.",
140 + "- Continue normal project validation.",
141 + ];
143 + if (droppedCount > 0)
144 + return ["- Inspect dropped steps and rerun the workflow."];
145 + if (finalCount > 0) return ["- Continue with the final results above."];
146 + return ["- No follow-up action indicated."];
98 149 export function formatUltraResultMarkdown(
99 150 result: WorkflowResult,
100 151 expanded = false,
113 164 if (usage) lines.push(usage);
114 165 lines.push("");
167 + const refuted = refutedSlots(result);
168 + const droppedCount = result.phases.reduce(
169 + (total, phase) => total + phase.dropped,
170 + 0,
171 + );
172 + lines.push(
173 + "## Summary",
174 + summaryLine(result, finalItems.length, refuted.length, droppedCount),
175 + "",
176 + );
116 178 if (finalItems.length > 0) {
117 - lines.push("## Final results", ...finalItems.map(itemLine), "");
118 - } else {
119 179 lines.push(
120 - "## No final results",
121 - "No items survived the workflow return selector.",
180 + result.workflow === "review"
181 + ? "## Verified findings"
182 + : "## Final results",
183 + ...finalItems.map(itemLine),
126 - const refuted = refutedSlots(result);
127 188 if (refuted.length > 0) {
128 189 lines.push(
129 190 "## Refuted findings",
135 196 const drops = dropLines(result, expanded);
136 197 if (drops.length > 0) lines.push("## Drops", ...drops, "");
198 + lines.push(
199 + "## Recommendations",
200 + ...recommendationLines(result, finalItems.length, droppedCount),
201 + "",
202 + );
138 204 if (expanded)
139 205 lines.push(
148 214 export function formatWorkflowResultMessage(result: WorkflowResult): string {
149 - const empty = Array.isArray(result.result) && result.result.length === 0;
215 + if (result.workflow === "review") return formatUltraResultMarkdown(result);
150 216 return [
151 217 `ultra workflow "${result.workflow}" result${result.steered ? " (steered)" : ""}:`,
152 - empty
153 - ? "\nResult is empty: no items survived the workflow return selector."
154 - : "",
155 218 "```json",
156 219 JSON.stringify(result, null, 2),
157 220 "```",
158 - ]
159 - .filter(Boolean)
160 - .join("\n");
221 + ].join("\n");
163 224 export function formatBackgroundAcceptedMessage(

extensions/ultra/workflows/feature.json

Modified
2 2 "name": "feature",
3 - "description": "Research the codebase, plan a feature, adversarially critique the plan, implement it, then review for security and quality.",
3 + "description": "Research the codebase, plan a feature, critique and strengthen the plan, implement it, then review for security and quality.",
4 4 "args": { "hint": "<task>", "params": ["task"] },
5 5 "schemas": {
6 6 "Research": {
126 126 "schema": "Critique"
130 + "id": "refine",
131 + "kind": "single",
132 + "step": {
133 + "prompt": "Strengthen the implementation plan for: {args.task}. Rewrite it as a complete replacement that resolves EVERY blocking item from this critique: {critique.results}. Original plan: {plan.results}. Research: {research.results}. Read further if needed; do not modify anything. Return JSON matching the Plan schema.",
134 + "tools": ["read", "grep", "find", "ls", "bash"],
135 + "model": "big",
136 + "thinkingLevel": "high",
137 + "schema": "Plan"
139 + },
130 141 "id": "implement",
131 142 "kind": "single",
132 - "when": "{critique.results | where approved}",
133 143 "step": {
134 - "prompt": "Implement the feature: {args.task}. Follow the plan and resolve EVERY blocking item from the critique. Plan: {plan.results}. Critique: {critique.results}. Make the actual edits with the edit/write tools, keep the change focused, and follow the conventions surfaced in research. Run focused checks and set implemented=true only when the changes and checks complete. Then return JSON matching the Implementation schema (list the files you changed and a concise summary).",
144 + "prompt": "Implement the feature: {args.task}. Follow the strengthened plan: {refine.results}. Make the actual edits with the edit/write tools, keep the change focused, and follow the conventions surfaced in research. Run focused checks and set implemented=true only when the changes and checks complete. Then return JSON matching the Implementation schema (list the files you changed and a concise summary).",
135 145 "tools": ["read", "grep", "find", "ls", "bash", "edit", "write"],
136 146 "model": "medium",
137 147 "thinkingLevel": "high",