repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

extensions/iq/implementation.ts

Raw
import type { Dirent } from "node:fs";
import { readdir, stat } from "node:fs/promises";
import { basename, join, resolve } from "node:path";
import type { Api, Model } from "@earendil-works/pi-ai";
import {
	type ExtensionCommandContext,
	type ExtensionContext,
	SettingsManager,
} from "@earendil-works/pi-coding-agent";
import {
	aggregateModels,
	defaultModelReference,
	estimateScan,
	fileSha256,
	parseModelReference,
	prepareSession,
	readIqSettings,
	type SessionPlan,
	type SourceSignature,
	sessionPlan,
} from "./core.js";
import { judgeSession } from "./judge.js";
import {
	acquireScanLock,
	iqStateDirectory,
	loadRecords,
	recordMatches,
	recordsForJudge,
	saveRecord,
} from "./storage.js";
import {
	type DashboardAction,
	type DashboardData,
	IqDashboardComponent,
	IqScanComponent,
	type ScanDisplayState,
} from "./ui.js";

type AnyModel = Model<Api>;

interface DiscoverySnapshot {
	sessionDir: string;
	candidates: SourceSignature[];
	discoveredAt: number;
}

interface ProgressOutcome<T> {
	value?: T;
	error?: Error;
	cancelled: boolean;
}

interface PreflightResult {
	plans: SessionPlan[];
	unranked: number;
	failures: number;
}

let discoveryCache: DiscoverySnapshot | undefined;
const eligibilityCache = new Map<
	string,
	{ size: number; mtimeMs: number; ranked: boolean }
>();

function asError(value: unknown): Error {
	return value instanceof Error ? value : new Error(String(value));
}

function boundedError(value: unknown): string {
	return asError(value).message.replaceAll(/\s+/g, " ").slice(0, 300);
}

async function safeReadDir(path: string): Promise<Dirent[]> {
	try {
		return await readdir(path, { withFileTypes: true });
	} catch (error) {
		if (isNodeError(error) && error.code === "ENOENT") return [];
		throw error;
	}
}

function isNodeError(error: unknown): error is NodeJS.ErrnoException {
	return error instanceof Error && "code" in error;
}

async function collectSessionFiles(
	directory: string,
	files: string[],
	signal: AbortSignal,
): Promise<void> {
	signal.throwIfAborted();
	for (const entry of await safeReadDir(directory)) {
		signal.throwIfAborted();
		const path = join(directory, entry.name);
		if (entry.isDirectory()) await collectSessionFiles(path, files, signal);
		else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(path);
	}
}

export async function discoverSessions(
	sessionDir: string,
	currentSessionFile: string | undefined,
	signal: AbortSignal,
): Promise<DiscoverySnapshot> {
	const files: string[] = [];
	await collectSessionFiles(sessionDir, files, signal);
	const current = currentSessionFile ? resolve(currentSessionFile) : undefined;
	const candidates: SourceSignature[] = [];
	for (const path of files.sort()) {
		signal.throwIfAborted();
		if (current && resolve(path) === current) continue;
		try {
			const info = await stat(path);
			candidates.push({ path, size: info.size, mtimeMs: info.mtimeMs });
		} catch {
			// A session can disappear between directory enumeration and stat.
		}
	}
	return { sessionDir, candidates, discoveredAt: Date.now() };
}

export async function discoverInBackground(
	ctx: ExtensionContext,
	signal: AbortSignal,
	isCurrent: () => boolean,
): Promise<void> {
	const sessionDir = ctx.sessionManager.getSessionDir();
	const current = ctx.sessionManager.getSessionFile();
	const snapshot = await discoverSessions(sessionDir, current, signal);
	if (!signal.aborted && isCurrent()) discoveryCache = snapshot;
}

function settings(ctx: ExtensionCommandContext) {
	const manager = SettingsManager.create(ctx.cwd);
	const global = manager.getGlobalSettings();
	const project = manager.getProjectSettings();
	return {
		iq: readIqSettings(global, project),
		defaultModel: defaultModelReference(global, project),
	};
}

function resolveJudge(ctx: ExtensionCommandContext): {
	model: AnyModel;
	reference: string;
} {
	const configured = settings(ctx);
	const reference = configured.iq.judgeModel ?? configured.defaultModel;
	if (!reference) {
		if (!ctx.model)
			throw new Error("No configured default model for IQ judging");
		return {
			model: ctx.model,
			reference: `${ctx.model.provider}/${ctx.model.id}`,
		};
	}
	const parsed = parseModelReference(reference);
	if (!parsed)
		throw new Error(`Invalid iq.judgeModel: ${JSON.stringify(reference)}`);
	const model = ctx.modelRegistry.find(parsed.provider, parsed.modelId);
	if (!model) throw new Error(`IQ judge model not found: ${reference}`);
	return { model, reference };
}

async function currentDiscovery(
	ctx: ExtensionCommandContext,
	signal: AbortSignal,
	force = false,
): Promise<DiscoverySnapshot> {
	const sessionDir = ctx.sessionManager.getSessionDir();
	if (!force && discoveryCache?.sessionDir === sessionDir)
		return discoveryCache;
	const snapshot = await discoverSessions(
		sessionDir,
		ctx.sessionManager.getSessionFile(),
		signal,
	);
	discoveryCache = snapshot;
	return snapshot;
}

async function dashboardData(
	ctx: ExtensionCommandContext,
	judge: string,
): Promise<DashboardData> {
	const discovery = await currentDiscovery(ctx, new AbortController().signal);
	const stateDirectory = iqStateDirectory();
	const loaded = await loadRecords(stateDirectory);
	const active = recordsForJudge(loaded.records, judge).sort((left, right) =>
		right.scoredAt.localeCompare(left.scoredAt),
	);
	const unscored = discovery.candidates.filter(
		(candidate) =>
			!active.some((record) => recordMatches(record, candidate, judge)),
	);
	const unranked = unscored.filter((candidate) => {
		const cached = eligibilityCache.get(candidate.path);
		return (
			cached?.size === candidate.size &&
			cached.mtimeMs === candidate.mtimeMs &&
			!cached.ranked
		);
	}).length;
	return {
		judge,
		stateDirectory,
		leaderboard: aggregateModels(active),
		records: active,
		pending: unscored.length - unranked,
		unranked,
		storageErrors: loaded.errors.length,
	};
}

async function showDashboard(
	ctx: ExtensionCommandContext,
	judge: string,
): Promise<DashboardAction> {
	const data = await dashboardData(ctx, judge);
	return ctx.ui.custom<DashboardAction>((tui, theme, _keybindings, done) => {
		const component = new IqDashboardComponent(
			theme,
			data,
			() => tui.requestRender(),
			done,
		);
		return component;
	});
}

async function runProgress<T>(
	ctx: ExtensionCommandContext,
	state: ScanDisplayState,
	work: (
		signal: AbortSignal,
		state: ScanDisplayState,
		requestRender: () => void,
	) => Promise<T>,
): Promise<ProgressOutcome<T>> {
	return ctx.ui.custom<ProgressOutcome<T>>((tui, theme, _keybindings, done) => {
		const component = new IqScanComponent(tui, theme, state);
		let finished = false;
		const finish = (outcome: ProgressOutcome<T>) => {
			if (finished) return;
			finished = true;
			component.dispose();
			done(outcome);
		};
		void work(component.signal, state, () => tui.requestRender())
			.then((value) => finish({ value, cancelled: false }))
			.catch((error: unknown) =>
				finish({
					error: asError(error),
					cancelled: component.signal.aborted,
				}),
			);
		return component;
	});
}

async function preflight(
	ctx: ExtensionCommandContext,
	model: AnyModel,
	judge: string,
	stateDirectory: string,
): Promise<ProgressOutcome<PreflightResult>> {
	const discovery = await currentDiscovery(
		ctx,
		new AbortController().signal,
		true,
	);
	const loaded = await loadRecords(stateDirectory);
	const active = recordsForJudge(loaded.records, judge);
	const candidates = discovery.candidates.filter(
		(candidate) =>
			!active.some((record) => recordMatches(record, candidate, judge)),
	);
	const state: ScanDisplayState = {
		phase: "prepare",
		model: `${model.provider}/${model.id}`,
		currentName: "",
		session: 0,
		sessions: candidates.length,
		chunk: 0,
		chunks: 0,
		evidence: 0,
		cost: 0,
		failures: 0,
		startedAt: Date.now(),
	};
	return runProgress(ctx, state, async (signal, display, render) => {
		const plans: SessionPlan[] = [];
		let unranked = 0;
		let failures = 0;
		for (let index = 0; index < candidates.length; index++) {
			signal.throwIfAborted();
			const candidate = candidates[index];
			if (!candidate) continue;
			display.session = index + 1;
			display.currentName = basename(candidate.path);
			render();
			try {
				const plan = sessionPlan(
					await prepareSession(candidate.path, Date.now(), signal),
				);
				eligibilityCache.set(candidate.path, {
					size: plan.source.size,
					mtimeMs: plan.source.mtimeMs,
					ranked: plan.eligibility.ranked,
				});
				if (plan.eligibility.ranked) plans.push(plan);
				else unranked++;
			} catch {
				failures++;
				display.failures = failures;
			}
		}
		return { plans, unranked, failures };
	});
}

function formatTokens(tokens: number): string {
	if (tokens < 1_000) return String(tokens);
	if (tokens < 1_000_000) return `${(tokens / 1_000).toFixed(1)}k`;
	return `${(tokens / 1_000_000).toFixed(2)}M`;
}

async function scorePlans(
	ctx: ExtensionCommandContext,
	model: AnyModel,
	plans: readonly SessionPlan[],
	stateDirectory: string,
): Promise<ProgressOutcome<{ scored: number; failures: number }>> {
	const state: ScanDisplayState = {
		phase: "judge",
		model: `${model.provider}/${model.id}`,
		currentName: "",
		session: 0,
		sessions: plans.length,
		chunk: 0,
		chunks: 0,
		evidence: 0,
		cost: 0,
		failures: 0,
		startedAt: Date.now(),
	};
	return runProgress(ctx, state, async (signal, display, render) => {
		let scored = 0;
		let failures = 0;
		let completedCost = 0;
		for (let index = 0; index < plans.length; index++) {
			signal.throwIfAborted();
			const plan = plans[index];
			if (!plan) continue;
			display.session = index + 1;
			display.currentName = plan.source.name || basename(plan.source.path);
			display.chunk = 0;
			display.chunks = plan.chunkCount;
			display.evidence = 0;
			render();
			try {
				const prepared = await prepareSession(
					plan.source.path,
					Date.now(),
					signal,
				);
				if (
					prepared.source.size !== plan.source.size ||
					prepared.source.mtimeMs !== plan.source.mtimeMs ||
					prepared.source.sha256 !== plan.source.sha256
				)
					throw new Error("session changed after cost confirmation");
				const record = await judgeSession(
					ctx,
					model,
					prepared,
					signal,
					(progress) => {
						display.chunk = progress.chunk;
						display.chunks = progress.chunks;
						display.evidence = progress.evidence;
						display.cost = completedCost + progress.usage.cost;
						render();
					},
				);
				const beforeHash = await stat(plan.source.path);
				const hash = await fileSha256(plan.source.path, signal);
				const afterHash = await stat(plan.source.path);
				if (
					beforeHash.size !== record.source.size ||
					beforeHash.mtimeMs !== record.source.mtimeMs ||
					afterHash.size !== beforeHash.size ||
					afterHash.mtimeMs !== beforeHash.mtimeMs ||
					hash !== record.source.sha256
				)
					throw new Error("session changed while being judged");
				await saveRecord(record, stateDirectory);
				completedCost += record.usage.cost;
				display.cost = completedCost;
				scored++;
			} catch (error) {
				if (signal.aborted) throw error;
				failures++;
				display.failures = failures;
				render();
			}
		}
		return { scored, failures };
	});
}

async function scan(
	ctx: ExtensionCommandContext,
	stateDirectory = iqStateDirectory(),
): Promise<void> {
	let selected: { model: AnyModel; reference: string };
	try {
		selected = resolveJudge(ctx);
	} catch (error) {
		ctx.ui.notify(boundedError(error), "error");
		return;
	}
	const auth = await ctx.modelRegistry.getApiKeyAndHeaders(selected.model);
	if (!auth.ok) {
		ctx.ui.notify(auth.error || "IQ judge authentication failed", "error");
		return;
	}
	let prepared: Awaited<ReturnType<typeof preflight>>;
	try {
		prepared = await preflight(
			ctx,
			selected.model,
			selected.reference,
			stateDirectory,
		);
	} catch (error) {
		ctx.ui.notify(`IQ discovery failed: ${boundedError(error)}`, "error");
		return;
	}
	if (prepared.cancelled) {
		ctx.ui.notify("IQ scan cancelled", "info");
		return;
	}
	if (prepared.error || !prepared.value) {
		ctx.ui.notify(
			`IQ preparation failed: ${boundedError(prepared.error)}`,
			"error",
		);
		return;
	}
	if (prepared.value.plans.length === 0) {
		ctx.ui.notify(
			`No rankable unscored sessions · ${prepared.value.unranked} unranked · ${prepared.value.failures} unreadable`,
			"info",
		);
		return;
	}
	const estimate = estimateScan(prepared.value.plans, selected.model);
	const providers = new Set(
		prepared.value.plans
			.map((plan) => plan.source.model?.split("/")[0])
			.filter((provider): provider is string => Boolean(provider)),
	);
	const trustBoundary =
		providers.size === 1 && providers.has(selected.model.provider)
			? "same provider as source sessions"
			: `session content is sent to ${selected.model.provider}`;
	const confirmed = await ctx.ui.confirm(
		`Score ${prepared.value.plans.length} session${prepared.value.plans.length === 1 ? "" : "s"}?`,
		[
			`Judge: ${selected.reference}`,
			`Estimated: ${estimate.calls} calls · ${formatTokens(estimate.inputTokens + estimate.outputTokens)} tokens · $${estimate.cost.toFixed(2)}`,
			`Boundary: ${trustBoundary}`,
			`${prepared.value.unranked} mixed, trivial, or unsettled sessions stay unranked.`,
		].join("\n"),
	);
	if (!confirmed) {
		ctx.ui.notify("IQ scan cancelled", "info");
		return;
	}
	let release: (() => Promise<void>) | undefined;
	try {
		release = await acquireScanLock(stateDirectory);
	} catch (error) {
		ctx.ui.notify(boundedError(error), "warning");
		return;
	}
	let result: Awaited<ReturnType<typeof scorePlans>>;
	try {
		const latest = recordsForJudge(
			(await loadRecords(stateDirectory)).records,
			selected.reference,
		);
		const remaining = prepared.value.plans.filter(
			(plan) =>
				!latest.some((record) =>
					recordMatches(record, plan.source, selected.reference),
				),
		);
		if (remaining.length === 0) {
			ctx.ui.notify("IQ sessions were scored by another process", "info");
			return;
		}
		result = await scorePlans(ctx, selected.model, remaining, stateDirectory);
	} finally {
		await release();
	}
	if (result.cancelled) {
		ctx.ui.notify("IQ scan cancelled; completed scores were kept", "info");
		return;
	}
	if (result.error || !result.value) {
		ctx.ui.notify(`IQ scan failed: ${boundedError(result.error)}`, "error");
		return;
	}
	discoveryCache = undefined;
	ctx.ui.notify(
		`IQ scored ${result.value.scored} session${result.value.scored === 1 ? "" : "s"}${result.value.failures ? ` · ${result.value.failures} failed` : ""}`,
		result.value.failures ? "warning" : "info",
	);
}

export async function runIqCommand(
	args: string,
	ctx: ExtensionCommandContext,
	stateDirectory?: string,
): Promise<void> {
	if (!ctx.hasUI) return;
	if (ctx.mode !== "tui") {
		ctx.ui.notify("IQ requires interactive TUI mode", "error");
		return;
	}
	const command = args.trim().toLowerCase();
	if (command && command !== "scan") {
		ctx.ui.notify("Usage: /iq [scan]", "error");
		return;
	}
	if (command === "scan") {
		await scan(ctx, stateDirectory);
		return;
	}
	let selected: { model: AnyModel; reference: string };
	try {
		selected = resolveJudge(ctx);
	} catch (error) {
		ctx.ui.notify(boundedError(error), "error");
		return;
	}
	for (;;) {
		let action: DashboardAction;
		try {
			action = await showDashboard(ctx, selected.reference);
		} catch (error) {
			ctx.ui.notify(`IQ dashboard failed: ${boundedError(error)}`, "error");
			return;
		}
		if (action !== "scan") return;
		await scan(ctx, stateDirectory);
	}
}

export const __test = {
	dashboardData,
	resolveJudge,
};