Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/usage/__tests__/accounting.test.ts

Raw
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { stripVTControlCharacters } from "node:util";
import {
	type ExtensionUIContext,
	initTheme,
	type Theme,
} from "@earendil-works/pi-coding-agent";
import { visibleWidth } from "@earendil-works/pi-tui";
import { afterEach, expect, it, vi } from "vitest";
import { createPiSession } from "../../../test/fixture/create-pi-session.ts";
import { collectUsageData, showUsage } from "../implementation.ts";

let root: string | undefined;
afterEach(() => {
	if (root) rmSync(root, { recursive: true, force: true });
	vi.useRealTimers();
	vi.unstubAllEnvs();
});
function fixture() {
	root = mkdtempSync(join(tmpdir(), "pi-usage-accounting-"));
	const dir = join(root, "project");
	mkdirSync(dir);
	return dir;
}
function writeSession(dir: string, name: string, entries: unknown[]) {
	writeFileSync(
		join(dir, `${name}.jsonl`),
		entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n",
	);
}
function header(id: string) {
	return {
		type: "session",
		version: 3,
		id,
		timestamp: new Date().toISOString(),
	};
}
function response(
	id: string | undefined,
	cost: number,
	provider = "provider-a",
	responseId?: string,
) {
	return {
		type: "message",
		...(id === undefined ? {} : { id }),
		parentId: null,
		timestamp: new Date().toISOString(),
		message: {
			role: "assistant",
			provider,
			model: "model",
			timestamp: Date.now(),
			...(responseId ? { responseId } : {}),
			usage: {
				input: 10,
				output: 2,
				cacheRead: 3,
				cacheWrite: 1,
				cost: { total: cost },
			},
		},
	};
}

it.each([
	{ flags: [true, true], billed: true, total: 3 },
	{ flags: [true, false], billed: false, total: 3 },
	{ flags: [true, undefined], billed: false, total: 3 },
	{ flags: [true, true], billed: true, total: 0 },
])(
	"preserves recorded cost provenance $flags with total $total through dashboard",
	async ({ flags, billed, total }) => {
		const dir = fixture();
		const entries = flags.map((flag, i) => {
			const entry = response(String(i), total === 0 ? 0 : i + 1);
			return {
				...entry,
				message: {
					...entry.message,
					usage: {
						...entry.message.usage,
						cost: {
							total: total === 0 ? 0 : i + 1,
							input: 999,
							...(flag === undefined ? {} : { totalFromProvider: flag }),
						},
					},
				},
			};
		});
		writeSession(dir, "provenance", [header("provenance"), ...entries]);
		const data = await collectUsageData(dir);
		for (const period of [data?.today, data?.thisWeek, data?.allTime]) {
			expect(period?.totals).toMatchObject({
				cost: total,
				totalFromProvider: billed,
				messages: 2,
			});
			const provider = period?.providers.get("provider-a");
			expect(provider).toMatchObject({
				cost: total,
				totalFromProvider: billed,
			});
			expect(provider?.models.get("model")).toMatchObject({
				cost: total,
				totalFromProvider: billed,
			});
		}
		const customOptions: unknown[] = [];
		let dismiss: (() => void) | undefined;
		let dashboard:
			| { render(width: number): string[]; handleInput(input: string): void }
			| undefined;
		const theme = {
			fg: (_color: string, text: string) => text,
			bold: (text: string) => text,
		} as Theme;
		let finished = false;
		const showing = showUsage("", {
			hasUI: true,
			sessionManager: { getSessionDir: () => dir },
			ui: {
				custom: (
					factory: (
						tui: unknown,
						theme: Theme,
						kb: unknown,
						done: (value: unknown) => void,
					) => unknown,
					options?: unknown,
				) =>
					new Promise((resolve) => {
						customOptions.push(options);
						const component = factory(
							{ requestRender() {} },
							theme,
							{},
							resolve,
						);
						if (
							dashboard === undefined &&
							(component as { handleInput?: unknown }).handleInput &&
							!(component as { signal?: unknown }).signal
						) {
							dashboard = component as typeof dashboard;
							dismiss = () => dashboard!.handleInput("q");
						}
					}),
			},
		} as never).then(() => {
			finished = true;
		});
		await vi.waitFor(() => expect(dashboard).toBeDefined());
		expect(finished).toBe(false);
		expect(customOptions).toEqual([
			undefined,
			{ blocked: { kind: "question", message: expect.any(String) } },
		]);
		const rows = dashboard!.render(120).join("\n");
		expect(rows.includes("≈")).toBe(!billed);
		dashboard!.handleInput("\r");
		expect(dashboard!.render(120).join("\n").includes("≈")).toBe(!billed);
		expect(finished).toBe(false);
		dismiss!();
		await showing;
		expect(finished).toBe(true);
	},
);

it("distinguishes routed responders while preserving requested identity and recorded billing", async () => {
	const dir = fixture();
	const routed = response("routed", 0);
	writeSession(dir, "routed", [
		header("routed"),
		{
			...routed,
			message: {
				...routed.message,
				responseModel: "concrete",
				usage: {
					...routed.message.usage,
					cost: { total: 0, totalFromProvider: true },
				},
			},
		},
		{
			...response("other", 2),
			message: { ...response("other", 2).message, responseModel: "unknown" },
		},
		{
			...response("same", 3),
			message: { ...response("same", 3).message, responseModel: "model" },
		},
		response("legacy", 4),
	]);
	const stats = (await collectUsageData(dir))?.allTime;
	expect(stats?.totals).toMatchObject({
		messages: 4,
		cost: 9,
		totalFromProvider: false,
	});
	const models = stats?.providers.get("provider-a")?.models;
	expect(models?.get("concrete ← model")).toMatchObject({
		messages: 1,
		cost: 0,
		totalFromProvider: true,
	});
	expect(models?.get("unknown ← model")?.cost).toBe(2);
	expect(models?.get("model")).toMatchObject({ messages: 2, cost: 7 });
});

it.each(["dark", "light"])(
	"keeps the concrete responder visible in expanded narrow and wide rows with native %s theme",
	async (themeName) => {
		const dir = fixture();
		const requested = "requested-router-model-with-a-very-long-identifier";
		const entry = response("routed", 2);
		writeSession(dir, "routed", [
			header("routed"),
			{
				...entry,
				message: {
					...entry.message,
					model: requested,
					responseModel: "served",
				},
			},
		]);
		initTheme(themeName, false);
		const pi = await createPiSession({ extensionFactories: [] });
		try {
			const ctx = pi.session.extensionRunner.createCommandContext();
			const nativeTheme = ctx.ui.theme;
			let screens = 0;
			const custom: ExtensionUIContext["custom"] = async <T>(
				factory: Parameters<ExtensionUIContext["custom"]>[0],
				options?: Parameters<ExtensionUIContext["custom"]>[1],
			): Promise<T> => {
				const screen = ++screens;
				let finish!: (result: T) => void;
				const pending = new Promise<T>((resolve) => {
					finish = resolve;
				});
				const component = await factory(
					{ requestRender() {} } as Parameters<typeof factory>[0],
					nativeTheme,
					{} as Parameters<typeof factory>[2],
					(value) => finish(value as T),
				);
				if (screen === 1) {
					expect(options?.blocked).toBeUndefined();
				} else {
					expect(options?.blocked?.kind).toBe("question");
					for (const width of [68, 120]) {
						expect(
							component
								.render(width)
								.map(stripVTControlCharacters)
								.some((row) => row.includes("served")),
						).toBe(false);
					}
					component.handleInput!("\r");
					for (const width of [68, 120]) {
						const rows = component.render(width);
						expect(rows.every((row) => visibleWidth(row) <= width)).toBe(true);
						const served = rows
							.map(stripVTControlCharacters)
							.filter((row) => row.trimStart().startsWith("served ← "));
						expect(served).toHaveLength(1);
						expect(served[0]).toContain("≈$2.00");
					}
					component.handleInput!("q");
				}
				return pending;
			};
			await showUsage("", {
				...ctx,
				hasUI: true,
				sessionManager: { getSessionDir: () => dir },
				ui: { ...ctx.ui, custom },
			} as never);
			expect(screens).toBe(2);
			const models = (await collectUsageData(dir))?.allTime.providers.get(
				"provider-a",
			)?.models;
			expect([...models!.keys()]).toEqual([`served ← ${requested}`]);
		} finally {
			await pi.dispose();
		}
	},
);

it("reads the configured session store for an in-memory session", async () => {
	fixture();
	vi.stubEnv("PI_CODING_AGENT_DIR", root!);
	const dir = join(root!, "sessions", "project");
	mkdirSync(dir, { recursive: true });
	writeSession(dir, "stored", [
		header("stored-session"),
		response("stored-response", 2),
	]);
	const data = await collectUsageData("");
	expect(data?.allTime.totals).toMatchObject({
		sessions: 1,
		messages: 1,
		cost: 2,
	});
});

it("retains independent equal-sized responses and legacy rows while suppressing copied fork records", async () => {
	vi.useFakeTimers({ toFake: ["Date"] });
	vi.setSystemTime(new Date(2026, 5, 10, 12));
	const dir = fixture();
	const original = response("original", 1, "provider-a", "response-1");
	writeSession(dir, "a", [
		header("session-a"),
		original,
		response("different", 2, "provider-b"),
	]);
	writeSession(dir, "b", [
		header("session-b"),
		{ ...original, parentId: "rewritten-by-fork" },
		response("original", 1, "provider-a", "response-2"),
	]);
	writeSession(dir, "legacy", [
		header("legacy"),
		response(undefined, 4),
		response(undefined, 4),
	]);
	const data = await collectUsageData(dir);
	for (const stats of [data?.allTime, data?.today, data?.thisWeek]) {
		expect(stats?.totals).toMatchObject({
			messages: 5,
			cost: 12,
			sessions: 3,
			tokens: {
				total: 80,
				input: 50,
				output: 10,
				cacheRead: 15,
				cacheWrite: 5,
			},
		});
		expect(stats?.providers.get("provider-a")?.cost).toBe(10);
		expect(stats?.providers.get("provider-b")?.models.get("model")?.cost).toBe(
			2,
		);
	}
});

it("does not let a headerless file suppress a valid record in another file", async () => {
	const dir = fixture();
	const entry = response("shared", 2);
	writeSession(dir, "00-invalid", [entry]);
	writeSession(dir, "10-valid", [header("valid"), entry]);
	expect((await collectUsageData(dir))?.allTime.totals).toMatchObject({
		messages: 1,
		cost: 2,
		sessions: 1,
	});
});

it("accepts numeric and ISO response times with entry-time fallback across reporting periods", async () => {
	vi.useFakeTimers({ toFake: ["Date"] });
	vi.setSystemTime(new Date(2026, 5, 10, 12));
	const dir = fixture();
	const today = response("today", 1);
	const iso = response("iso", 2);
	const fallback = response("fallback", 3);
	const lastWeek = new Date(2026, 5, 3, 12);
	writeSession(dir, "dates", [
		header("dates"),
		today,
		{
			...iso,
			message: { ...iso.message, timestamp: new Date().toISOString() },
		},
		{
			...fallback,
			timestamp: lastWeek.toISOString(),
			message: { ...fallback.message, timestamp: undefined },
		},
	]);
	const data = await collectUsageData(dir);
	expect(data?.today.totals.cost).toBe(3);
	expect(data?.thisWeek.totals.cost).toBe(3);
	expect(data?.lastWeek.totals.cost).toBe(3);
	expect(data?.allTime.totals.cost).toBe(6);
});

it("reads every project of the default store but only the configured flat store", async () => {
	fixture();
	vi.stubEnv("PI_CODING_AGENT_DIR", root!);
	const defaultProject = join(root!, "sessions", "project-a");
	mkdirSync(defaultProject, { recursive: true });
	mkdirSync(join(root!, "sessions", "project-b"));
	writeSession(defaultProject, "a", [header("a"), response("a", 1)]);
	writeSession(join(root!, "sessions", "project-b"), "b", [
		header("b"),
		response("b", 2),
	]);
	const custom = join(root!, "state", "sessions");
	mkdirSync(join(custom, "nested"), { recursive: true });
	writeSession(custom, "c", [header("c"), response("c", 4)]);
	writeSession(join(custom, "nested"), "d", [header("d"), response("d", 8)]);
	writeSession(join(root!, "state"), "foreign", [
		header("foreign"),
		response("foreign", 16),
	]);
	expect((await collectUsageData(defaultProject))?.allTime.totals.cost).toBe(3);
	expect((await collectUsageData(custom))?.allTime.totals.cost).toBe(4);
});

it("reads persisted native usage without assigning aggregate tool usage to a provider", async () => {
	const dir = fixture();
	const timestamp = new Date().toISOString();
	const media = {
		type: "message",
		id: "media",
		timestamp,
		message: {
			role: "toolResult",
			toolName: "media_generate",
			usage: { input: 2, cost: { total: 2, totalFromProvider: false } },
		},
	};
	writeSession(dir, "parent", [
		header("parent"),
		response("assistant", 1),
		media,
		{
			type: "usage",
			id: "standalone",
			timestamp,
			kind: "speech",
			provider: "minimax",
			model: "speech-2.6-hd",
			usage: { input: 3, cost: { total: 3, totalFromProvider: false } },
		},
		{
			type: "compaction",
			id: "compaction",
			timestamp,
			usage: { input: 4, cost: { total: 4, totalFromProvider: false } },
		},
		{
			type: "branch_summary",
			id: "summary",
			timestamp,
			usage: { input: 5, cost: { total: 5, totalFromProvider: false } },
		},
	]);
	writeSession(dir, "child", [
		{ ...header("child"), parentSession: "parent.jsonl" },
		response("child", 20),
	]);
	writeSession(dir, "fork", [
		{ ...header("fork"), parentSession: "parent.jsonl" },
		{ ...media, parentId: "forked-parent" },
	]);

	const allTime = (await collectUsageData(dir))?.allTime;
	expect(allTime?.totals).toMatchObject({ messages: 6, cost: 35, sessions: 3 });
	expect(allTime?.providers.get("provider-a")?.cost).toBe(21);
	expect(
		allTime?.providers.get("minimax")?.models.get("speech-2.6-hd")?.cost,
	).toBe(3);
	expect(allTime?.providers.get("unattributed native")).toMatchObject({
		cost: 11,
		messages: 3,
	});
});