pi-ext
bugabingas pi extensions
owned by bugabinga
feat(ultra): track token usage
154439b9990e4cef343c001978421ee46d9b00bd
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (9)
25 25 `web_search`), while the step `tools` allowlist stays the only gate on what is
26 26 active and ultra's own `workflow` tool is stripped from every sub-agent
27 27 allowlist (recursion defense); set `"none"` to keep sub-agents on core tools
28 +- Add ultra `modelTiers` setting so workflow steps can use `small`, `medium`, or
29 + `big` aliases, with unset tiers falling back to the session model and optional
30 + thinking suffixes such as `openai/gpt-5-mini:low`
31 +- Add ultra aggregate token/cost accounting in workflow results from reported
28 33 - Ship the ultra `feature` workflow (research → plan → adversarial critique →
29 34 implement → security/quality review), and make `/ultra` positional args soak
30 35 the trailing free text into the last declared param so `/ultra feature <task>`
extensions/ultra/README.md
Modified51 51 run it with `/ultra feature <describe the task>` (the whole trailing text is the
52 52 task). Only its `implement` phase writes; every other phase is read-only.
56 +Workflow results include aggregate token and cost usage from completed sub-agent
57 +turns when the provider reports it.
58 +The final JSON contains `tokenUsage` with `input`, `output`, `total`,
59 +`cacheRead`, `cacheWrite`, and `cost`.
56 63 Read from the `ultra` block of agent settings, `<cwd>/.pi/settings.json`, and
extensions/ultra/__tests__/engine.test.ts
Modified487 - it(
"returns { workflow, phases:[{id,ran,ok,dropped}], phaseResults, result, steered }",
async ()
=> {
488 - const fr
= fakeRunner((args)
=> ({ ok:
true, value: args.item
?? "v" }));
487 + it(
"returns { workflow, phases:[{id,ran,ok,dropped}], phaseResults, result, steered, tokenUsage }",
async ()
=> {
488 + const fr
= fakeRunner((args)
=> ({
490 + value: args.item
?? "v",
491 502 return:
"{p.results}",
510 522 expect(res).
toEqual({
513 525 phaseResults: {
p: [{ ok:
true, value:
"a" }] },
517 537 expect(
Object.
keys(res.phases[
0]).
sort()).
toEqual([
extensions/ultra/__tests__/runner.test.ts
Modified157 -function happyDrive(params
: unknown = { verdict:
"ok" }) {
157 +function happyDrive(params
: unknown = { verdict:
"ok" }, usage
?: unknown) {
158 158 return async ({ emit, callStructured }
: DriveApi)
=> {
159 159 emit({ type:
"agent_start" });
184 - emit({ type:
"agent_end", messages: [], willRetry:
false });
186 + messages: usage
? [{ role:
"assistant", usage }]
: [],
194 198 const completeFn
= vi.
fn(
195 199 async (_req
: CompleteRequest)
: Promise<
CompleteResponse>
=> ({
196 200 text:
'```json\n{ "verdict": "ok" }\n```',
206 + cost: { total:
0.25 },
199 210 const drivers
= makeDrivers({ completeFn });
202 213 const result
= await runStep(toollessStep, ctx, drivers,
baseOpts());
204 - expect(result).
toEqual({ ok:
true, value: { verdict:
"ok" } });
215 + expect(result).
toEqual({
217 + value: { verdict:
"ok" },
207 229 const model
= (drivers.resolveModel
as ReturnType<
typeof vi.fn>).mock
515 537 expect(harness.captured.args?.tools).not.
toContain(
"workflow");
540 + it(
"captures token and cost usage from the final assistant message",
async ()
=> {
541 + const harness
= makeSessionHarness({
543 + { verdict:
"metered" },
549 + cost: { total:
0.5 },
553 + const drivers
= makeDrivers({ sessionFactory: harness.factory });
555 + const result
= await runStep(toolStep,
makeCtx(), drivers,
baseOpts());
557 + expect(result).
toEqual({
559 + value: { verdict:
"metered" },
518 571 it(
"the structured-output tool returns terminate:true and details = the validated params",
async ()
=> {
519 572 let returned
: { details
: unknown; terminate
?: boolean }
| undefined;
520 573 const harness
= makeSessionHarness({
extensions/ultra/__tests__/wire.test.ts
Modified 94 94 expect(markdown).not.
toContain(
"```json");
97 + it(
"renders aggregate token and cost usage when present", ()
=> {
98 + const markdown
= formatUltraResultMarkdown({
114 + expect(markdown).
toContain(
"Usage: 140 tokens (100 in, 40 out) · $0.1234");
97 117 it(
"includes raw JSON only when expanded", ()
=> {
99 119 workflow:
"review",
extensions/ultra/drivers.ts
Modified111 111 stopReason
?: string;
112 112 errorMessage
?: string;
113 113 content
?: { type: string; text
?: string }[];
116 117 export type CompleteApi = (
162 163 .
map((c)
=> c.
text)
164 - return { text, stopReason: res.stopReason, errorMessage: res.errorMessage };
167 + stopReason: res.stopReason,
168 + errorMessage: res.errorMessage,
extensions/ultra/engine.ts
Modified 32 32 } from
"./interp.ts";
33 33 import { isAborted, mapWithConcurrency }
from "./pool.ts";
34 34 import type { AgentControls, UltraProgressEvent }
from "./progress.ts";
35 -import type { StepRunner }
from "./runner.ts";
35 +import type { StepRunner, TokenUsage }
from "./runner.ts";
36 36 import type { Phase, Step, WorkflowSpec }
from "./spec.ts";
79 79 phaseResults: Record
<string, PhaseSlotResult[]
>;
82 + tokenUsage: TokenUsage;
100 101 const results
: Record<
string,
unknown[]>
= {};
101 102 const phaseResults
: Record<
string,
PhaseSlotResult[]>
= {};
102 103 const phases
: PhaseResult[]
= [];
104 + const tokenUsage
= zeroUsage();
103 105 const steeredBox
= { value:
false };
104 106 const forward
= makeForwarder(deps, steeredBox);
159 161 outcomeSlots.
push({ ok:
false, value:
null, reason:
"aborted" });
164 + addUsage(tokenUsage, slot.usage);
196 199 steered: steeredBox.
value,
208 +function zeroUsage()
: TokenUsage {
219 +function addUsage(into
: TokenUsage, usage
: TokenUsage | undefined)
: void {
220 + if (
!usage)
return;
221 + into.
input += usage.
input;
222 + into.output
+= usage.output;
223 + into.total
+= usage.total;
224 + into.cacheRead
+= usage.cacheRead;
225 + into.cacheWrite
+= usage.cacheWrite;
226 + into.cost
+= usage.cost;
206 231 function phaseItems(phase
: Phase, ctx
: InterpContext)
: unknown[] {
extensions/ultra/runner.ts
Modified 78 +export interface TokenUsage {
77 87 export interface CompleteResponse {
79 89 stopReason
?: string;
80 90 errorMessage
?: string;
83 94 export type CompleteFn = (req
: CompleteRequest)
=> Promise<
CompleteResponse>;
269 +function number(value
: unknown)
: number {
270 + const n
= Number(value
?? 0);
271 + return Number.
isFinite(n)
? n
: 0;
274 +function usageCost(usage
: Record<
string,
unknown>)
: number {
275 + const cost
= usage.cost;
276 + return typeof cost
=== "object" && cost
!== null 277 + ? number((cost
as Record<
string,
unknown>).total)
281 +function normalizeUsage(usage
: unknown)
: TokenUsage | undefined {
282 + if (
!usage
|| typeof usage
!== "object" || Array.
isArray(usage))
284 + const u
= usage
as Record<
string,
unknown>;
285 + const input
= number(u.
input);
286 + const output
= number(u.output);
287 + const total
= number(u.total
?? u.totalTokens
?? input
+ output);
292 + cacheRead:
number(u.cacheRead),
293 + cacheWrite:
number(u.cacheWrite),
294 + cost:
usageCost(u),
298 +function lastAssistantUsage(messages
: unknown)
: TokenUsage | undefined {
299 + if (
!Array.
isArray(messages))
return undefined;
300 + for (
let i
= messages.
length - 1; i
>= 0; i
--) {
301 + const message
= messages[i];
302 + if (
!message
|| typeof message
!== "object")
continue;
303 + const m
= message
as Record<
string,
unknown>;
304 + if (m.role
=== "assistant")
return normalizeUsage(m.usage);
259 310 function stripFences(raw
: string)
: string {
260 311 const FENCE_OPEN
= /^\s*`{3}(?:json)?\s*\n?/i;
515 566 const text
= response?.
text ?? "";
516 567 const transportFailure
= completionFailureReason(response, guarded.error);
517 568 if (transportFailure)
return { done:
false, reason: transportFailure, text };
518 - return evaluateJson(text, validator);
569 + return evaluateJson(text, validator,
normalizeUsage(response.usage));
521 572 function completionFailureReason(
535 586 function evaluateJson(
537 588 validator
: CompiledValidator | undefined,
589 + usage
: TokenUsage | undefined,
538 590 )
: CompleteAttempt {
539 591 const parsed
= parseJsonObject(text);
540 592 if (
!parsed.ok)
return { done:
false, reason: parsed.reason, text };
548 - return { done:
true, result: { ok:
true, value: parsed.
value } };
600 + return { done:
true, result: { ok:
true, value: parsed.
value, usage } };
551 603 function appendCorrection(
602 654 const sessionManager
= ctxLike.
newSessionManager();
603 655 let session
: SessionLike | undefined;
604 656 let unsubscribe: ()
=> void = ()
=> {};
657 + let usage
: TokenUsage | undefined;
607 660 session
= await drivers.
sessionFactory({
628 681 unsubscribe
= session.
subscribe((event)
=> {
682 + if (
event.
type === "agent_end") {
683 + usage
= lastAssistantUsage((
event as { messages
?: unknown }).messages);
629 685 const mapped
= mapSessionEvent(
event, agentId, phase);
630 686 if (mapped)
onProgress?.({
...mapped, controls });
643 699 return { ok:
false, value:
null, reason: guarded.outcome };
646 - if (captured)
return { ok:
true, value: captured.
value };
702 + if (captured)
return { ok:
true, value: captured.
value, usage };
647 703 return { ok:
false, value:
null, reason:
"no structured output produced" };
extensions/ultra/wire.ts
Modified 70 +function usageLine(result
: WorkflowResult)
: string | undefined {
71 + const usage
= result.tokenUsage;
72 + if (
!usage
|| usage.total
=== 0)
return undefined;
73 + const cost
= usage.cost
> 0 ? ` · $${usage.cost.toFixed(4)}` : "";
74 + return `Usage: ${usage.total} tokens (${usage.input} in, ${usage.output} out)${cost}`;
70 77 export function formatUltraResultMarkdown(
71 78 result
: WorkflowResult,
81 88 .
map((p)
=> `${p.id}: ${p.ok}/${p.ran} ok, ${p.dropped} dropped`)
82 89 .
join(
" · ")
|| "No phases.",
91 + const usage
= usageLine(result);
92 + if (usage) lines.
push(usage);
86 95 if (finalItems.
length > 0) {