repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

feat(ultra): track token usage

154439b9990e4cef343c001978421ee46d9b00bd

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (9)

CHANGELOG.md

Modified
25 25 `web_search`), while the step `tools` allowlist stays the only gate on what is
26 26 active and ultra's own `workflow` tool is stripped from every sub-agent
27 27 allowlist (recursion defense); set `"none"` to keep sub-agents on core tools
28 +- Add ultra `modelTiers` setting so workflow steps can use `small`, `medium`, or
29 + `big` aliases, with unset tiers falling back to the session model and optional
30 + thinking suffixes such as `openai/gpt-5-mini:low`
31 +- Add ultra aggregate token/cost accounting in workflow results from reported
32 + sub-agent usage
28 33 - Ship the ultra `feature` workflow (research → plan → adversarial critique →
29 34 implement → security/quality review), and make `/ultra` positional args soak
30 35 the trailing free text into the last declared param so `/ultra feature <task>`

extensions/ultra/README.md

Modified
51 51 run it with `/ultra feature <describe the task>` (the whole trailing text is the
52 52 task). Only its `implement` phase writes; every other phase is read-only.
54 +## Results
56 +Workflow results include aggregate token and cost usage from completed sub-agent
57 +turns when the provider reports it.
58 +The final JSON contains `tokenUsage` with `input`, `output`, `total`,
59 +`cacheRead`, `cacheWrite`, and `cost`.
54 61 ## Settings
56 63 Read from the `ultra` block of agent settings, `<cwd>/.pi/settings.json`, and

extensions/ultra/__tests__/engine.test.ts

Modified
487 - it("returns { workflow, phases:[{id,ran,ok,dropped}], phaseResults, result, steered }", async () => {
488 - const fr = fakeRunner((args) => ({ ok: true, value: args.item ?? "v" }));
487 + it("returns { workflow, phases:[{id,ran,ok,dropped}], phaseResults, result, steered, tokenUsage }", async () => {
488 + const fr = fakeRunner((args) => ({
489 + ok: true,
490 + value: args.item ?? "v",
491 + usage: {
492 + input: 7,
493 + output: 3,
494 + total: 10,
495 + cacheRead: 2,
496 + cacheWrite: 1,
497 + cost: 0.4,
498 + },
499 + }));
489 500 const s = spec({
490 501 name: "demo",
491 502 return: "{p.results}",
505 516 "phases",
506 517 "result",
507 518 "steered",
519 + "tokenUsage",
508 520 "workflow",
510 522 expect(res).toEqual({
513 525 phaseResults: { p: [{ ok: true, value: "a" }] },
514 526 result: ["a"],
515 527 steered: false,
528 + tokenUsage: {
529 + input: 7,
530 + output: 3,
531 + total: 10,
532 + cacheRead: 2,
533 + cacheWrite: 1,
534 + cost: 0.4,
535 + },
517 537 expect(Object.keys(res.phases[0]).sort()).toEqual([
518 538 "dropped",

extensions/ultra/__tests__/runner.test.ts

Modified
156 156 /** A normal drive: tool_start -> text_delta -> structured(valid) -> tool_end -> agent_end. */
157 -function happyDrive(params: unknown = { verdict: "ok" }) {
157 +function happyDrive(params: unknown = { verdict: "ok" }, usage?: unknown) {
158 158 return async ({ emit, callStructured }: DriveApi) => {
159 159 emit({ type: "agent_start" });
160 160 emit({
181 181 result,
182 182 isError: false,
184 - emit({ type: "agent_end", messages: [], willRetry: false });
184 + emit({
185 + type: "agent_end",
186 + messages: usage ? [{ role: "assistant", usage }] : [],
187 + willRetry: false,
188 + });
194 198 const completeFn = vi.fn(
195 199 async (_req: CompleteRequest): Promise<CompleteResponse> => ({
196 200 text: '```json\n{ "verdict": "ok" }\n```',
201 + usage: {
202 + input: 12,
203 + output: 4,
204 + cacheRead: 3,
205 + cacheWrite: 2,
206 + cost: { total: 0.25 },
207 + },
199 210 const drivers = makeDrivers({ completeFn });
202 213 const result = await runStep(toollessStep, ctx, drivers, baseOpts());
204 - expect(result).toEqual({ ok: true, value: { verdict: "ok" } });
215 + expect(result).toEqual({
216 + ok: true,
217 + value: { verdict: "ok" },
218 + usage: {
219 + input: 12,
220 + output: 4,
221 + total: 16,
222 + cacheRead: 3,
223 + cacheWrite: 2,
224 + cost: 0.25,
225 + },
226 + });
206 228 // resolveAuth invoked with the resolved model (correction #4)
207 229 const model = (drivers.resolveModel as ReturnType<typeof vi.fn>).mock
515 537 expect(harness.captured.args?.tools).not.toContain("workflow");
540 + it("captures token and cost usage from the final assistant message", async () => {
541 + const harness = makeSessionHarness({
542 + drive: happyDrive(
543 + { verdict: "metered" },
545 + input: 10,
546 + output: 5,
547 + cacheRead: 1,
548 + cacheWrite: 2,
549 + cost: { total: 0.5 },
550 + },
551 + ),
552 + });
553 + const drivers = makeDrivers({ sessionFactory: harness.factory });
555 + const result = await runStep(toolStep, makeCtx(), drivers, baseOpts());
557 + expect(result).toEqual({
558 + ok: true,
559 + value: { verdict: "metered" },
560 + usage: {
561 + input: 10,
562 + output: 5,
563 + total: 15,
564 + cacheRead: 1,
565 + cacheWrite: 2,
566 + cost: 0.5,
567 + },
568 + });
569 + });
518 571 it("the structured-output tool returns terminate:true and details = the validated params", async () => {
519 572 let returned: { details: unknown; terminate?: boolean } | undefined;
520 573 const harness = makeSessionHarness({

extensions/ultra/__tests__/wire.test.ts

Modified
94 94 expect(markdown).not.toContain("```json");
97 + it("renders aggregate token and cost usage when present", () => {
98 + const markdown = formatUltraResultMarkdown({
99 + workflow: "review",
100 + phases: [],
101 + phaseResults: {},
102 + result: [],
103 + steered: false,
104 + tokenUsage: {
105 + input: 100,
106 + output: 40,
107 + total: 140,
108 + cacheRead: 20,
109 + cacheWrite: 5,
110 + cost: 0.1234,
111 + },
112 + });
114 + expect(markdown).toContain("Usage: 140 tokens (100 in, 40 out) · $0.1234");
115 + });
97 117 it("includes raw JSON only when expanded", () => {
98 118 const result = {
99 119 workflow: "review",

extensions/ultra/drivers.ts

Modified
111 111 stopReason?: string;
112 112 errorMessage?: string;
113 113 content?: { type: string; text?: string }[];
114 + usage?: unknown;
116 117 export type CompleteApi = (
162 163 .map((c) => c.text)
163 164 .join("");
164 - return { text, stopReason: res.stopReason, errorMessage: res.errorMessage };
165 + return {
166 + text,
167 + stopReason: res.stopReason,
168 + errorMessage: res.errorMessage,
169 + usage: res.usage,
170 + };

extensions/ultra/engine.ts

Modified
3 3 // ultra — phase engine (Task 6).
5 5 // Sequences a workflow's phases, owns the interpolation mini-syntax (it alone
6 6 // holds `args` + prior-phase `results`), fans each phase out through the
7 7 // concurrency pool, normalises drops/aborts, resolves the `return` selector, and
8 -// produces the documented aggregate `{ workflow, phases, phaseResults, result, steered }`.
8 +// produces the documented aggregate `{ workflow, phases, phaseResults, result, steered, tokenUsage }`.
10 10 // Everything model/network-touching is behind the injected `stepRunner` (Task
11 11 // 5's `StepRunner` shape), so this module is fully offline-testable with a fake
32 32 } from "./interp.ts";
33 33 import { isAborted, mapWithConcurrency } from "./pool.ts";
34 34 import type { AgentControls, UltraProgressEvent } from "./progress.ts";
35 -import type { StepRunner } from "./runner.ts";
35 +import type { StepRunner, TokenUsage } from "./runner.ts";
36 36 import type { Phase, Step, WorkflowSpec } from "./spec.ts";
38 38 // ---------------------------------------------------------------------------
79 79 phaseResults: Record<string, PhaseSlotResult[]>;
80 80 result: unknown;
81 81 steered: boolean;
82 + tokenUsage: TokenUsage;
84 85 // ---------------------------------------------------------------------------
100 101 const results: Record<string, unknown[]> = {};
101 102 const phaseResults: Record<string, PhaseSlotResult[]> = {};
102 103 const phases: PhaseResult[] = [];
104 + const tokenUsage = zeroUsage();
103 105 const steeredBox = { value: false };
104 106 const forward = makeForwarder(deps, steeredBox);
159 161 outcomeSlots.push({ ok: false, value: null, reason: "aborted" });
160 162 continue;
164 + addUsage(tokenUsage, slot.usage);
162 165 ran++;
163 166 if (slot.ok) {
194 197 phaseResults,
195 198 result,
196 199 steered: steeredBox.value,
200 + tokenUsage,
201 205 // Internals
202 206 // ---------------------------------------------------------------------------
208 +function zeroUsage(): TokenUsage {
209 + return {
210 + input: 0,
211 + output: 0,
212 + total: 0,
213 + cacheRead: 0,
214 + cacheWrite: 0,
215 + cost: 0,
216 + };
219 +function addUsage(into: TokenUsage, usage: TokenUsage | undefined): void {
220 + if (!usage) return;
221 + into.input += usage.input;
222 + into.output += usage.output;
223 + into.total += usage.total;
224 + into.cacheRead += usage.cacheRead;
225 + into.cacheWrite += usage.cacheWrite;
226 + into.cost += usage.cost;
204 229 /** The fanout/single item list. `single` runs its step exactly once (any `over`
205 230 * is ignored in v1); `fanout` resolves `over` to its element array. */
206 231 function phaseItems(phase: Phase, ctx: InterpContext): unknown[] {

extensions/ultra/runner.ts

Modified
47 47 ok: boolean;
48 48 value: unknown;
49 49 reason?: string;
50 + usage?: TokenUsage;
52 53 /** Per-model auth, mirroring `ModelRegistry.getApiKeyAndHeaders`. */
76 77 /** Normalised completion: assistant text + the provider stop reason. */
78 +export interface TokenUsage {
79 + input: number;
80 + output: number;
81 + total: number;
82 + cacheRead: number;
83 + cacheWrite: number;
84 + cost: number;
77 87 export interface CompleteResponse {
78 88 text: string;
79 89 stopReason?: string;
80 90 errorMessage?: string;
91 + usage?: unknown;
83 94 export type CompleteFn = (req: CompleteRequest) => Promise<CompleteResponse>;
269 +function number(value: unknown): number {
270 + const n = Number(value ?? 0);
271 + return Number.isFinite(n) ? n : 0;
274 +function usageCost(usage: Record<string, unknown>): number {
275 + const cost = usage.cost;
276 + return typeof cost === "object" && cost !== null
277 + ? number((cost as Record<string, unknown>).total)
278 + : number(cost);
281 +function normalizeUsage(usage: unknown): TokenUsage | undefined {
282 + if (!usage || typeof usage !== "object" || Array.isArray(usage))
283 + return undefined;
284 + const u = usage as Record<string, unknown>;
285 + const input = number(u.input);
286 + const output = number(u.output);
287 + const total = number(u.total ?? u.totalTokens ?? input + output);
288 + return {
289 + input,
290 + output,
291 + total,
292 + cacheRead: number(u.cacheRead),
293 + cacheWrite: number(u.cacheWrite),
294 + cost: usageCost(u),
295 + };
298 +function lastAssistantUsage(messages: unknown): TokenUsage | undefined {
299 + if (!Array.isArray(messages)) return undefined;
300 + for (let i = messages.length - 1; i >= 0; i--) {
301 + const message = messages[i];
302 + if (!message || typeof message !== "object") continue;
303 + const m = message as Record<string, unknown>;
304 + if (m.role === "assistant") return normalizeUsage(m.usage);
306 + return undefined;
258 309 /** Strip a leading ```json (or ```) fence and a trailing ``` fence (angel's behavior). */
259 310 function stripFences(raw: string): string {
260 311 const FENCE_OPEN = /^\s*`{3}(?:json)?\s*\n?/i;
515 566 const text = response?.text ?? "";
516 567 const transportFailure = completionFailureReason(response, guarded.error);
517 568 if (transportFailure) return { done: false, reason: transportFailure, text };
518 - return evaluateJson(text, validator);
569 + return evaluateJson(text, validator, normalizeUsage(response.usage));
521 572 function completionFailureReason(
535 586 function evaluateJson(
536 587 text: string,
537 588 validator: CompiledValidator | undefined,
589 + usage: TokenUsage | undefined,
538 590 ): CompleteAttempt {
539 591 const parsed = parseJsonObject(text);
540 592 if (!parsed.ok) return { done: false, reason: parsed.reason, text };
548 - return { done: true, result: { ok: true, value: parsed.value } };
600 + return { done: true, result: { ok: true, value: parsed.value, usage } };
551 603 function appendCorrection(
602 654 const sessionManager = ctxLike.newSessionManager();
603 655 let session: SessionLike | undefined;
604 656 let unsubscribe: () => void = () => {};
657 + let usage: TokenUsage | undefined;
607 660 session = await drivers.sessionFactory({
628 681 unsubscribe = session.subscribe((event) => {
682 + if (event.type === "agent_end") {
683 + usage = lastAssistantUsage((event as { messages?: unknown }).messages);
629 685 const mapped = mapSessionEvent(event, agentId, phase);
630 686 if (mapped) onProgress?.({ ...mapped, controls });
643 699 return { ok: false, value: null, reason: guarded.outcome };
646 - if (captured) return { ok: true, value: captured.value };
702 + if (captured) return { ok: true, value: captured.value, usage };
647 703 return { ok: false, value: null, reason: "no structured output produced" };
648 704 } finally {
649 705 unsubscribe();

extensions/ultra/wire.ts

Modified
67 67 return lines;
70 +function usageLine(result: WorkflowResult): string | undefined {
71 + const usage = result.tokenUsage;
72 + if (!usage || usage.total === 0) return undefined;
73 + const cost = usage.cost > 0 ? ` · $${usage.cost.toFixed(4)}` : "";
74 + return `Usage: ${usage.total} tokens (${usage.input} in, ${usage.output} out)${cost}`;
70 77 export function formatUltraResultMarkdown(
71 78 result: WorkflowResult,
72 79 expanded = false,
81 88 .map((p) => `${p.id}: ${p.ok}/${p.ran} ok, ${p.dropped} dropped`)
82 89 .join(" · ") || "No phases.",
91 + const usage = usageLine(result);
92 + if (usage) lines.push(usage);
84 93 lines.push("");
86 95 if (finalItems.length > 0) {