repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

goal: gate updates behind judge review

f984f326c4548c7868d7b968df157887fe6c4775

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (6)

drivers/goal_suite.py

Added
new file mode 100644
1 +title = "Goal loop"
3 +extensions = [
4 + "goal",
5 +]
7 +targets = [
8 + "goal",
9 +]
11 +cols = 100
12 +rows = 28
13 +cwd = "."
15 +steps = [
16 + ("wait", 5000),
17 + ("command", "/goal status", 3500),
18 + ("command", "/goal In this demo, answer exactly: goal loop ready.", 18000),
19 + ("command", "/goal status", 6500),
20 + ("wait", 7000),

extensions/goal/README.md

Modified
2 2 # goal
4 4 Persistent `/goal` loop for Pi.
6 +## Install / load
8 +Loaded through the root pi-ext package. See [../../README.md](../../README.md).
6 10 ## Commands
8 12 ```text
17 21 /goal clear-criteria
24 +Debug command, registered only when `PI_GOAL_PROBE=1`:
26 +```text
27 +/goal-probe direct|microtask|timeout
28 +```
20 30 ## Tool
22 32 ```text
23 33 update_goal({ status: "complete" | "blocked", reason: string })
36 +Marks the active goal complete or blocked. Active only after at least one judge review. Disabled again when the goal is complete, blocked, paused, cleared, or resumed into a fresh 0-turn state. Errors when no active goal exists or when the goal belongs to another session.
26 38 ## Settings
28 40 ```json
30 42 "goal": {
31 43 "maxTurns": 20,
32 - "judgeModel": "openai-codex/gpt-5.4-mini",
44 + "judgeModel": "provider/model",
33 45 "tailBytes": 4096
38 -`judgeModel` uses Pi model refs: `provider/model`.
50 +- `maxTurns`: turn budget for a new goal. Default `20`, clamped to `1..10000`.
51 +- `judgeModel`: optional Pi model ref, `provider/model`. When unset, the current session model is used.
52 +- `tailBytes`: bytes read from the latest assistant response for judgment. Default `4096`, clamped to `512..65536`.
54 +## Hooks
56 +```text
57 +session_start
58 +before_agent_start
59 +agent_start
60 +agent_end
61 +session_shutdown
62 +```
64 +## Behavior
66 +- `/goal <objective>` creates an active goal with empty criteria, current session ownership, and the configured turn budget.
67 +- Goal state is stored as append-only custom `goal-state` entries. Replay scans backward; newest valid state or clear tombstone wins.
68 +- `before_agent_start` injects the active goal and criteria into the system prompt.
69 +- After each agent turn, the extension writes a `goal-judge` start entry, sets status to `goal judging N/M`, notifies `Goal judge running: turn N/M · <objective>`, then asks a judge model for strict JSON: done, blocked, reason.
70 +- Judge completion writes a `goal-judge` verdict entry and notifies `Goal judge verdict: complete|blocked|continue · <reason>`.
71 +- Done verdict marks the goal complete. Blocked verdict marks it blocked.
72 +- Incomplete verdict increments `turnsUsed`; exhausted budget pauses the goal.
73 +- `update_goal` is hidden until the first judge review, then hidden again for terminal or paused goals.
74 +- If still active, continuation is sent when the session becomes idle.
75 +- `/goal resume` rebinds a goal to the current session and resets `turnsUsed`.
76 +- `/goal clear` writes a clear tombstone and removes status.
77 +- UI status key: `goal` for active, paused, and blocked goals. Complete/cleared goals clear status and notify instead.
79 +## Demo
81 +<!-- demo:goal_suite:start -->
82 +![Goal loop](assets/goal_suite.gif)
83 +<!-- demo:goal_suite:end -->

extensions/goal/__tests__/harness.test.ts

Modified
101 101 describe("goal integration", () => {
102 - it("update_goal tool marks active goal complete", async () => {
103 - t = await createTestSession({ extensions: [goalPath()] });
102 + it("update_goal is unavailable before judge review", async () => {
103 + t = await createTestSession({ extensions: [goalPath()], propagateErrors: false });
104 104 await t.run({
105 105 prompt: "/goal ship it",
106 106 actions: [
112 112 const result = t.events.toolResultsFor("update_goal")[0];
113 - expect(result.text).toBe("Goal marked complete: verified");
114 - expect(latestGoal()).toMatchObject({ objective: "ship it", status: "complete", lastReason: "verified" });
113 + expect(result.isError).toBe(true);
114 + expect(result.text).toContain("Tool update_goal not found");
115 + expect(latestGoal()).toMatchObject({ objective: "ship it", status: "active" });
117 - it("update_goal errors without active goal", async () => {
118 + it("update_goal is unavailable without active judged goal", async () => {
118 119 t = await createTestSession({ extensions: [goalPath()], propagateErrors: false });
119 120 await t.run({
120 121 prompt: "try tool",
124 125 const result = t.events.toolResultsFor("update_goal")[0];
125 126 expect(result.isError).toBe(true);
126 - expect(result.text).toContain("No active goal.");
127 + expect(result.text).toContain("Tool update_goal not found");
129 130 it("/goal complete is a user override and updates passive status", async () => {
141 142 .filter((call) => call.method === "setStatus" && call.args[0] === "goal")
142 143 .map((call) => call.args[1]);
143 144 expect(statusTexts).toContain("goal active 0/20");
144 - expect(statusTexts).toContain("goal complete 0/20");
145 + expect(statusTexts).toContain(undefined);
145 146 } finally {
146 147 restore();
157 158 await waitFor(() => latestGoal()?.status === "complete");
159 160 expect(latestGoal()).toMatchObject({ objective: "finish feature", status: "complete", lastReason: "all criteria verified" });
161 + expect(customEvents("goal-judge")).toMatchObject([
162 + { event: "start", objective: "finish feature", turn: 1, maxTurns: 20 },
163 + { event: "verdict", objective: "finish feature", turn: 1, maxTurns: 20, verdict: { done: true, blocked: false, reason: "all criteria verified" } },
164 + ]);
165 + expect(t.events.ui.filter((call) => call.method === "setStatus" && call.args[0] === "goal").map((call) => call.args[1])).toContain("goal judging 1/20");
160 166 } finally {
161 167 restore();

extensions/goal/__tests__/unit.test.ts

Modified
77 77 expect(__goalTest.statusLine(goal({ sessionId: "current" }) as never, "current")).not.toContain("owned by another session");
80 + it("hides turn budget for terminal goal statuses", () => {
81 + expect(__goalTest.statusText(goal({ status: "active" }) as never)).toBe("goal active 2/5 · 2c");
82 + expect(__goalTest.statusText(goal({ status: "paused" }) as never)).toBe("goal paused 2/5 · 2c");
83 + expect(__goalTest.statusText(goal({ status: "complete", turnsUsed: 0 }) as never)).toBeUndefined();
84 + expect(__goalTest.statusText(goal({ status: "blocked", turnsUsed: 0 }) as never)).toBe("goal blocked");
85 + expect(__goalTest.statusLine(goal({ status: "complete", turnsUsed: 0 }) as never)).not.toContain("0/5");
86 + });
88 + it("shows useful judge progress context", () => {
89 + expect(__goalTest.judgeRunningText(goal({ turnsUsed: 2, maxTurns: 5 }) as never)).toBe("Goal judge running: turn 3/5 · 2 criteria · Ship the feature");
90 + expect(__goalTest.judgeRunningText(goal({ objective: "x".repeat(100), criteria: [] }) as never)).toBe(`Goal judge running: turn 3/5 · ${"x".repeat(71)}…`);
91 + });
80 93 it("snapshots user-facing prompt/status output", () => {
81 94 const state = goal({ lastReason: "still checking" }) as never;
82 95 expect(JSON.stringify({
90 103 "status": "active · 2/5 turns · 2 criteria · still checking\\nShip the feature",
91 104 "criteria": "1. tests pass\\n2. docs updated",
92 - "active": "\\n\\nActive /goal:\\n<objective>\\nShip the feature\\n</objective>\\n\\nCriteria:\\n1. tests pass\\n2. docs updated\\n\\nKeep working toward this standing goal. Do not redefine success smaller. Before claiming completion, verify current files, commands, tests, or deliverables. Use update_goal only when complete or genuinely blocked.",
93 - "continuation": "Continue working toward this standing goal.\\n\\n<objective>\\nShip the feature\\n</objective>\\n\\nCriteria:\\n1. tests pass\\n2. docs updated\\n\\nDo not redefine success smaller. Inspect current state before claiming completion. If incomplete, make concrete progress. If complete or blocked, call update_goal.",
105 + "active": "\\n\\nActive /goal:\\n<objective>\\nShip the feature\\n</objective>\\n\\nCriteria:\\n1. tests pass\\n2. docs updated\\n\\nKeep working toward this standing goal. Do not redefine success smaller. Before claiming completion, verify current files, commands, tests, or deliverables. Use update_goal only after judge review and only when complete or genuinely blocked.",
106 + "continuation": "Continue working toward this standing goal.\\n\\n<objective>\\nShip the feature\\n</objective>\\n\\nCriteria:\\n1. tests pass\\n2. docs updated\\n\\nDo not redefine success smaller. Inspect current state before claiming completion. If incomplete, make concrete progress. If complete or blocked after judge review, call update_goal.",
94 107 "kickoff": "Start working toward this standing goal.\\n\\n<objective>\\nShip the feature\\n</objective>\\n\\nDo not stop just because one turn ends. Keep making concrete progress until the goal is complete, blocked, paused, cleared, or the turn budget is exhausted."

extensions/goal/assets/goal_suite.gif

Added
new file mode 100644

Current version contains binary or non-UTF-8 data.

extensions/goal/index.ts

Modified
40 40 const GOAL_STATE_ENTRY = "goal-state";
41 +const GOAL_JUDGE_ENTRY = "goal-judge";
41 42 const PROBE_ENTRY = "goal-probe";
42 43 const PROBE_MESSAGE = "GOAL_PROBE_CONTINUATION";
43 44 const DEFAULT_MAX_TURNS = 20;
128 129 return next;
132 +function showsTurnBudget(state: GoalState): boolean {
133 + return state.status === "active" || state.status === "paused";
136 +function showsPersistentStatus(state: GoalState): boolean {
137 + return state.status !== "complete";
140 +function shortText(text: string, max = 72): string {
141 + const oneLine = text.replace(/\s+/g, " ").trim();
142 + return oneLine.length <= max ? oneLine : `${oneLine.slice(0, max - 1)}…`;
145 +function judgeRunningText(state: GoalState): string {
146 + const criteria = state.criteria.length === 0 ? "" : ` · ${state.criteria.length} criteria`;
147 + return `Goal judge running: turn ${state.turnsUsed + 1}/${state.maxTurns}${criteria} · ${shortText(state.objective)}`;
131 150 function statusLine(state: GoalState, currentSessionId?: string): string {
151 + const turns = showsTurnBudget(state) ? ` · ${state.turnsUsed}/${state.maxTurns} turns` : "";
132 152 const criteria = state.criteria.length === 0 ? "" : ` · ${state.criteria.length} criteria`;
133 153 const reason = state.lastReason ? ` · ${state.lastReason}` : "";
134 154 const owner = state.sessionId && currentSessionId && state.sessionId !== currentSessionId ? " · owned by another session" : "";
135 - return `${state.status} · ${state.turnsUsed}/${state.maxTurns} turns${criteria}${owner}${reason}\n${state.objective}`;
155 + return `${state.status}${turns}${criteria}${owner}${reason}\n${state.objective}`;
138 158 function statusText(state: GoalState | undefined): string | undefined {
139 - if (!state) return undefined;
159 + if (!state || !showsPersistentStatus(state)) return undefined;
160 + if (!showsTurnBudget(state)) return `goal ${state.status}`;
140 161 const criteria = state.criteria.length === 0 ? "" : ` · ${state.criteria.length}c`;
141 162 return `goal ${state.status} ${state.turnsUsed}/${state.maxTurns}${criteria}`;
165 +function judgeStatusText(state: GoalState): string {
166 + const criteria = state.criteria.length === 0 ? "" : ` · ${state.criteria.length}c`;
167 + return `goal judging ${state.turnsUsed + 1}/${state.maxTurns}${criteria}`;
144 170 function updateStatus(ctx: ExtensionContext, state: GoalState | undefined): void {
145 171 if (!ctx.hasUI) return;
146 172 ctx.ui.setStatus(STATUS_KEY, statusText(state));
175 +function canUseUpdateGoal(state: GoalState | undefined): state is GoalState {
176 + return !!state && state.status === "active" && state.turnsUsed > 0;
179 +function setToolActive(pi: ExtensionAPI, name: string, active: boolean): void {
180 + const names = pi.getActiveTools();
181 + const has = names.includes(name);
182 + if (active === has) return;
183 + pi.setActiveTools(active ? [...names, name] : names.filter((tool) => tool !== name));
149 186 function currentSessionId(ctx: ExtensionContext): string | undefined {
150 187 return ctx.sessionManager.getSessionId?.();
173 210 const criteria = state.criteria.length === 0
174 211 ? "none"
175 212 : state.criteria.map((criterion, index) => `${index + 1}. ${criterion}`).join("\n");
176 - return `\n\nActive /goal:\n<objective>\n${state.objective}\n</objective>\n\nCriteria:\n${criteria}\n\nKeep working toward this standing goal. Do not redefine success smaller. Before claiming completion, verify current files, commands, tests, or deliverables. Use update_goal only when complete or genuinely blocked.`;
213 + return `\n\nActive /goal:\n<objective>\n${state.objective}\n</objective>\n\nCriteria:\n${criteria}\n\nKeep working toward this standing goal. Do not redefine success smaller. Before claiming completion, verify current files, commands, tests, or deliverables. Use update_goal only after judge review and only when complete or genuinely blocked.`;
179 216 function continuationPrompt(state: GoalState): string {
180 217 const criteria = state.criteria.length === 0
181 218 ? "none"
182 219 : state.criteria.map((criterion, index) => `${index + 1}. ${criterion}`).join("\n");
183 - return `Continue working toward this standing goal.\n\n<objective>\n${state.objective}\n</objective>\n\nCriteria:\n${criteria}\n\nDo not redefine success smaller. Inspect current state before claiming completion. If incomplete, make concrete progress. If complete or blocked, call update_goal.`;
220 + return `Continue working toward this standing goal.\n\n<objective>\n${state.objective}\n</objective>\n\nCriteria:\n${criteria}\n\nDo not redefine success smaller. Inspect current state before claiming completion. If incomplete, make concrete progress. If complete or blocked after judge review, call update_goal.`;
186 223 function kickoffPrompt(state: GoalState): string {
265 +function appendJudge(pi: ExtensionAPI, event: "start" | "verdict", state: GoalState, verdict?: JudgeVerdict): void {
266 + pi.appendEntry(GOAL_JUDGE_ENTRY, {
267 + event,
268 + goalId: state.id,
269 + objective: state.objective,
270 + turn: state.turnsUsed + 1,
271 + maxTurns: state.maxTurns,
272 + verdict,
273 + timestamp: Date.now(),
274 + });
228 277 async function judgeGoal(ctx: ExtensionContext, state: GoalState, settings: GoalSettings): Promise<JudgeVerdict> {
229 278 const model = resolveJudgeModel(ctx, settings);
230 279 if (!model) return { done: false, blocked: false, reason: "no judge model" };
240 289 timestamp: Date.now(),
243 - { apiKey: auth.apiKey, headers: auth.headers, signal: ctx.signal, reasoning: "off" },
292 + { apiKey: auth.apiKey, headers: auth.headers, signal: ctx.signal, reasoning: "off", cacheRetention: "none" },
245 294 const raw = response.content
246 295 .filter((part): part is { type: "text"; text: string } => part.type === "text")
249 298 return parseJudge(raw);
301 +function isStaleContextError(error: unknown): boolean {
302 + return error instanceof Error && error.message.includes("extension ctx is stale");
252 305 function sendWhenIdle(pi: ExtensionAPI, ctx: ExtensionContext, state: GoalState, scheduled: { value: boolean }, attempt = 0) {
253 306 setTimeout(() => {
254 - const latest = latestGoalState(ctx);
307 + let latest: GoalState | undefined;
308 + try { latest = latestGoalState(ctx); }
309 + catch (error) { if (isStaleContextError(error)) { scheduled.value = false; return; } throw error; }
255 310 if (!latest || latest.id !== state.id || latest.status !== "active" || !ownsGoal(ctx, latest)) {
256 311 scheduled.value = false;
257 312 return;
306 361 let state: GoalState | undefined;
307 362 let continuationScheduled = { value: false };
308 363 let probe: ProbeState | undefined;
364 + const syncUpdateGoalTool = () => setToolActive(pi, "update_goal", canUseUpdateGoal(state));
310 366 pi.on("session_start", (_event, ctx) => {
311 367 state = latestGoalState(ctx);
312 368 updateStatus(ctx, state);
369 + syncUpdateGoalTool();
315 372 pi.on("before_agent_start", (event, ctx) => {
321 378 if (probe?.active) appendProbe(pi, "agent-start", { mode: probe.mode });
324 - pi.on("session_shutdown", (_event, ctx) => updateStatus(ctx, undefined));
381 + pi.on("session_shutdown", (_event, ctx) => {
382 + updateStatus(ctx, undefined);
383 + setToolActive(pi, "update_goal", false);
384 + });
326 386 pi.on("agent_end", async (_event, ctx) => {
327 387 if (probe?.active) {
337 397 updateStatus(ctx, state);
398 + syncUpdateGoalTool();
338 399 if (!state || state.status !== "active" || !ownsGoal(ctx, state) || continuationScheduled.value) return;
339 400 const judged = state;
401 + appendJudge(pi, "start", judged);
402 + if (ctx.hasUI) {
403 + ctx.ui.setStatus(STATUS_KEY, judgeStatusText(judged));
404 + ctx.ui.notify(judgeRunningText(judged), "info");
340 406 let verdict: JudgeVerdict;
342 408 verdict = await judgeGoal(ctx, judged, loadSettings(pi));
344 410 verdict = { done: false, blocked: false, reason: `judge failed: ${(err as Error).message}` };
346 412 if (state !== judged || state.status !== "active") return;
413 + appendJudge(pi, "verdict", judged, verdict);
414 + if (ctx.hasUI) ctx.ui.notify(`Goal judge verdict: ${verdict.done ? "complete" : verdict.blocked ? "blocked" : "continue"} · ${verdict.reason}`, verdict.done ? "info" : verdict.blocked ? "warning" : "info");
347 415 if (verdict.done) {
348 416 state = updateGoal(pi, judged, { status: "complete", lastReason: verdict.reason });
349 417 updateStatus(ctx, state);
418 + syncUpdateGoalTool();
350 419 ctx.ui.notify(`✓ Goal complete: ${verdict.reason}`, "info");
351 420 return;
353 422 if (verdict.blocked) {
354 423 state = updateGoal(pi, judged, { status: "blocked", lastReason: verdict.reason });
355 424 updateStatus(ctx, state);
425 + syncUpdateGoalTool();
356 426 ctx.ui.notify(`Goal blocked: ${verdict.reason}`, "warning");
357 427 return;
360 430 if (turnsUsed >= judged.maxTurns) {
361 431 state = updateGoal(pi, judged, { status: "paused", turnsUsed, lastReason: `turn budget exhausted: ${verdict.reason}` });
362 432 updateStatus(ctx, state);
433 + syncUpdateGoalTool();
363 434 ctx.ui.notify(`Goal paused — ${turnsUsed}/${judged.maxTurns} turns used`, "warning");
364 435 return;
366 437 state = updateGoal(pi, judged, { turnsUsed, lastReason: verdict.reason });
367 438 updateStatus(ctx, state);
439 + syncUpdateGoalTool();
368 440 continuationScheduled.value = true;
369 441 sendWhenIdle(pi, ctx, state, continuationScheduled);
373 445 name: "update_goal",
374 446 label: "Update Goal",
375 447 description: "Mark the active /goal complete or blocked.",
376 - promptSnippet: "Mark the active /goal complete or blocked.",
448 + promptSnippet: "Mark the active /goal complete or blocked after judge review.",
377 449 promptGuidelines: [
378 - "Use update_goal only when the active /goal is actually complete or genuinely blocked.",
450 + "Use update_goal only after a judge review has run and the active /goal is actually complete or genuinely blocked.",
379 451 "Before update_goal complete, verify the objective and all criteria against current files, tests, commands, or deliverables.",
381 453 parameters: Type.Object({
385 457 async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
386 458 if (!state || state.status !== "active") throw new Error("No active goal.");
387 459 if (!ownsGoal(ctx, state)) throw new Error("Goal belongs to another session.");
460 + if (!canUseUpdateGoal(state)) throw new Error("Goal has not been judged yet.");
388 461 state = updateGoal(pi, state, { status: params.status, lastReason: params.reason });
389 462 updateStatus(ctx, state);
463 + syncUpdateGoalTool();
390 464 return { content: [{ type: "text", text: `Goal marked ${params.status}: ${params.reason}` }], details: { goal: state } };
408 482 if (state.status !== "active") return ctx.ui.notify("Goal is not active.", "warning");
409 483 state = updateGoal(pi, state, { status: "paused" });
410 484 updateStatus(ctx, state);
485 + syncUpdateGoalTool();
411 486 ctx.ui.notify("Goal paused", "info");
412 487 return;
416 491 const rebinding = state.sessionId && state.sessionId !== currentSessionId(ctx);
417 492 state = updateGoal(pi, state, { status: "active", turnsUsed: 0, sessionId: currentSessionId(ctx) });
418 493 updateStatus(ctx, state);
494 + syncUpdateGoalTool();
419 495 ctx.ui.notify(rebinding ? "Goal resumed in this session" : "Goal resumed", rebinding ? "warning" : "info");
420 496 continuationScheduled.value = true;
421 497 sendWhenIdle(pi, ctx, state, continuationScheduled);
426 502 const reason = rest.trim() || "marked complete by user";
427 503 state = updateGoal(pi, state, { status: "complete", lastReason: reason });
428 504 updateStatus(ctx, state);
505 + syncUpdateGoalTool();
429 506 ctx.ui.notify("Goal complete", "info");
430 507 return;
435 512 state = undefined;
436 513 pi.appendEntry(GOAL_STATE_ENTRY, { cleared: true, updatedAt: Date.now() });
437 514 updateStatus(ctx, state);
515 + syncUpdateGoalTool();
438 516 ctx.ui.notify("Goal cleared", "info");
439 517 return;
443 521 if (!rest.trim()) return ctx.ui.notify("Usage: /goal add <criterion>", "error");
444 522 state = updateGoal(pi, state, { criteria: [...state.criteria, rest.trim()] });
445 523 updateStatus(ctx, state);
524 + syncUpdateGoalTool();
446 525 ctx.ui.notify("Goal criterion added", "info");
447 526 return;
452 531 if (!Number.isInteger(index) || index < 0 || index >= state.criteria.length) return ctx.ui.notify("Usage: /goal remove <n>", "error");
453 532 state = updateGoal(pi, state, { criteria: state.criteria.filter((_, i) => i !== index) });
454 533 updateStatus(ctx, state);
534 + syncUpdateGoalTool();
455 535 ctx.ui.notify("Goal criterion removed", "info");
456 536 return;
463 543 if (!ownedGoalOrNotify(ctx, state)) return;
464 544 state = updateGoal(pi, state, { criteria: [] });
465 545 updateStatus(ctx, state);
546 + syncUpdateGoalTool();
466 547 ctx.ui.notify("Goal criteria cleared", "info");
467 548 return;
482 563 appendGoal(pi, state);
483 564 updateStatus(ctx, state);
565 + syncUpdateGoalTool();
484 566 ctx.ui.notify(`Goal set (${state.maxTurns}-turn budget)`, "info");
485 567 pi.sendUserMessage(kickoffPrompt(state));
510 592 latestAssistantTail,
511 593 latestGoalState,
512 594 loadSettings,
595 + judgeRunningText,
513 596 parseJudge,
514 597 parseModelRef,
515 598 statusLine,
599 + statusText,
516 600 textFromContent,