import type { Dirent } from "node:fs"; import { readdir, stat } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; import type { Api, Model } from "@earendil-works/pi-ai"; import { type ExtensionCommandContext, type ExtensionContext, SettingsManager, } from "@earendil-works/pi-coding-agent"; import { aggregateModels, defaultModelReference, estimateScan, fileSha256, parseModelReference, prepareSession, readIqSettings, type SessionPlan, type SourceSignature, sessionPlan, } from "./core.js"; import { judgeSession } from "./judge.js"; import { acquireScanLock, iqStateDirectory, loadRecords, recordMatches, recordsForJudge, saveRecord, } from "./storage.js"; import { type DashboardAction, type DashboardData, IqDashboardComponent, IqScanComponent, type ScanDisplayState, } from "./ui.js"; type AnyModel = Model; interface DiscoverySnapshot { sessionDir: string; candidates: SourceSignature[]; discoveredAt: number; } interface ProgressOutcome { value?: T; error?: Error; cancelled: boolean; } interface PreflightResult { plans: SessionPlan[]; unranked: number; failures: number; } let discoveryCache: DiscoverySnapshot | undefined; const eligibilityCache = new Map< string, { size: number; mtimeMs: number; ranked: boolean } >(); function asError(value: unknown): Error { return value instanceof Error ? value : new Error(String(value)); } function boundedError(value: unknown): string { return asError(value).message.replaceAll(/\s+/g, " ").slice(0, 300); } async function safeReadDir(path: string): Promise { try { return await readdir(path, { withFileTypes: true }); } catch (error) { if (isNodeError(error) && error.code === "ENOENT") return []; throw error; } } function isNodeError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && "code" in error; } async function collectSessionFiles( directory: string, files: string[], signal: AbortSignal, ): Promise { signal.throwIfAborted(); for (const entry of await safeReadDir(directory)) { signal.throwIfAborted(); const path = join(directory, entry.name); if (entry.isDirectory()) await collectSessionFiles(path, files, signal); else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(path); } } export async function discoverSessions( sessionDir: string, currentSessionFile: string | undefined, signal: AbortSignal, ): Promise { const files: string[] = []; await collectSessionFiles(sessionDir, files, signal); const current = currentSessionFile ? resolve(currentSessionFile) : undefined; const candidates: SourceSignature[] = []; for (const path of files.sort()) { signal.throwIfAborted(); if (current && resolve(path) === current) continue; try { const info = await stat(path); candidates.push({ path, size: info.size, mtimeMs: info.mtimeMs }); } catch { // A session can disappear between directory enumeration and stat. } } return { sessionDir, candidates, discoveredAt: Date.now() }; } export async function discoverInBackground( ctx: ExtensionContext, signal: AbortSignal, isCurrent: () => boolean, ): Promise { const sessionDir = ctx.sessionManager.getSessionDir(); const current = ctx.sessionManager.getSessionFile(); const snapshot = await discoverSessions(sessionDir, current, signal); if (!signal.aborted && isCurrent()) discoveryCache = snapshot; } function settings(ctx: ExtensionCommandContext) { const manager = SettingsManager.create(ctx.cwd); const global = manager.getGlobalSettings(); const project = manager.getProjectSettings(); return { iq: readIqSettings(global, project), defaultModel: defaultModelReference(global, project), }; } function resolveJudge(ctx: ExtensionCommandContext): { model: AnyModel; reference: string; } { const configured = settings(ctx); const reference = configured.iq.judgeModel ?? configured.defaultModel; if (!reference) { if (!ctx.model) throw new Error("No configured default model for IQ judging"); return { model: ctx.model, reference: `${ctx.model.provider}/${ctx.model.id}`, }; } const parsed = parseModelReference(reference); if (!parsed) throw new Error(`Invalid iq.judgeModel: ${JSON.stringify(reference)}`); const model = ctx.modelRegistry.find(parsed.provider, parsed.modelId); if (!model) throw new Error(`IQ judge model not found: ${reference}`); return { model, reference }; } async function currentDiscovery( ctx: ExtensionCommandContext, signal: AbortSignal, force = false, ): Promise { const sessionDir = ctx.sessionManager.getSessionDir(); if (!force && discoveryCache?.sessionDir === sessionDir) return discoveryCache; const snapshot = await discoverSessions( sessionDir, ctx.sessionManager.getSessionFile(), signal, ); discoveryCache = snapshot; return snapshot; } async function dashboardData( ctx: ExtensionCommandContext, judge: string, ): Promise { const discovery = await currentDiscovery(ctx, new AbortController().signal); const stateDirectory = iqStateDirectory(); const loaded = await loadRecords(stateDirectory); const active = recordsForJudge(loaded.records, judge).sort((left, right) => right.scoredAt.localeCompare(left.scoredAt), ); const unscored = discovery.candidates.filter( (candidate) => !active.some((record) => recordMatches(record, candidate, judge)), ); const unranked = unscored.filter((candidate) => { const cached = eligibilityCache.get(candidate.path); return ( cached?.size === candidate.size && cached.mtimeMs === candidate.mtimeMs && !cached.ranked ); }).length; return { judge, stateDirectory, leaderboard: aggregateModels(active), records: active, pending: unscored.length - unranked, unranked, storageErrors: loaded.errors.length, }; } async function showDashboard( ctx: ExtensionCommandContext, judge: string, ): Promise { const data = await dashboardData(ctx, judge); return ctx.ui.custom((tui, theme, _keybindings, done) => { const component = new IqDashboardComponent( theme, data, () => tui.requestRender(), done, ); return component; }); } async function runProgress( ctx: ExtensionCommandContext, state: ScanDisplayState, work: ( signal: AbortSignal, state: ScanDisplayState, requestRender: () => void, ) => Promise, ): Promise> { return ctx.ui.custom>((tui, theme, _keybindings, done) => { const component = new IqScanComponent(tui, theme, state); let finished = false; const finish = (outcome: ProgressOutcome) => { if (finished) return; finished = true; component.dispose(); done(outcome); }; void work(component.signal, state, () => tui.requestRender()) .then((value) => finish({ value, cancelled: false })) .catch((error: unknown) => finish({ error: asError(error), cancelled: component.signal.aborted, }), ); return component; }); } async function preflight( ctx: ExtensionCommandContext, model: AnyModel, judge: string, stateDirectory: string, ): Promise> { const discovery = await currentDiscovery( ctx, new AbortController().signal, true, ); const loaded = await loadRecords(stateDirectory); const active = recordsForJudge(loaded.records, judge); const candidates = discovery.candidates.filter( (candidate) => !active.some((record) => recordMatches(record, candidate, judge)), ); const state: ScanDisplayState = { phase: "prepare", model: `${model.provider}/${model.id}`, currentName: "", session: 0, sessions: candidates.length, chunk: 0, chunks: 0, evidence: 0, cost: 0, failures: 0, startedAt: Date.now(), }; return runProgress(ctx, state, async (signal, display, render) => { const plans: SessionPlan[] = []; let unranked = 0; let failures = 0; for (let index = 0; index < candidates.length; index++) { signal.throwIfAborted(); const candidate = candidates[index]; if (!candidate) continue; display.session = index + 1; display.currentName = basename(candidate.path); render(); try { const plan = sessionPlan( await prepareSession(candidate.path, Date.now(), signal), ); eligibilityCache.set(candidate.path, { size: plan.source.size, mtimeMs: plan.source.mtimeMs, ranked: plan.eligibility.ranked, }); if (plan.eligibility.ranked) plans.push(plan); else unranked++; } catch { failures++; display.failures = failures; } } return { plans, unranked, failures }; }); } function formatTokens(tokens: number): string { if (tokens < 1_000) return String(tokens); if (tokens < 1_000_000) return `${(tokens / 1_000).toFixed(1)}k`; return `${(tokens / 1_000_000).toFixed(2)}M`; } async function scorePlans( ctx: ExtensionCommandContext, model: AnyModel, plans: readonly SessionPlan[], stateDirectory: string, ): Promise> { const state: ScanDisplayState = { phase: "judge", model: `${model.provider}/${model.id}`, currentName: "", session: 0, sessions: plans.length, chunk: 0, chunks: 0, evidence: 0, cost: 0, failures: 0, startedAt: Date.now(), }; return runProgress(ctx, state, async (signal, display, render) => { let scored = 0; let failures = 0; let completedCost = 0; for (let index = 0; index < plans.length; index++) { signal.throwIfAborted(); const plan = plans[index]; if (!plan) continue; display.session = index + 1; display.currentName = plan.source.name || basename(plan.source.path); display.chunk = 0; display.chunks = plan.chunkCount; display.evidence = 0; render(); try { const prepared = await prepareSession( plan.source.path, Date.now(), signal, ); if ( prepared.source.size !== plan.source.size || prepared.source.mtimeMs !== plan.source.mtimeMs || prepared.source.sha256 !== plan.source.sha256 ) throw new Error("session changed after cost confirmation"); const record = await judgeSession( ctx, model, prepared, signal, (progress) => { display.chunk = progress.chunk; display.chunks = progress.chunks; display.evidence = progress.evidence; display.cost = completedCost + progress.usage.cost; render(); }, ); const beforeHash = await stat(plan.source.path); const hash = await fileSha256(plan.source.path, signal); const afterHash = await stat(plan.source.path); if ( beforeHash.size !== record.source.size || beforeHash.mtimeMs !== record.source.mtimeMs || afterHash.size !== beforeHash.size || afterHash.mtimeMs !== beforeHash.mtimeMs || hash !== record.source.sha256 ) throw new Error("session changed while being judged"); await saveRecord(record, stateDirectory); completedCost += record.usage.cost; display.cost = completedCost; scored++; } catch (error) { if (signal.aborted) throw error; failures++; display.failures = failures; render(); } } return { scored, failures }; }); } async function scan( ctx: ExtensionCommandContext, stateDirectory = iqStateDirectory(), ): Promise { let selected: { model: AnyModel; reference: string }; try { selected = resolveJudge(ctx); } catch (error) { ctx.ui.notify(boundedError(error), "error"); return; } const auth = await ctx.modelRegistry.getApiKeyAndHeaders(selected.model); if (!auth.ok) { ctx.ui.notify(auth.error || "IQ judge authentication failed", "error"); return; } let prepared: Awaited>; try { prepared = await preflight( ctx, selected.model, selected.reference, stateDirectory, ); } catch (error) { ctx.ui.notify(`IQ discovery failed: ${boundedError(error)}`, "error"); return; } if (prepared.cancelled) { ctx.ui.notify("IQ scan cancelled", "info"); return; } if (prepared.error || !prepared.value) { ctx.ui.notify( `IQ preparation failed: ${boundedError(prepared.error)}`, "error", ); return; } if (prepared.value.plans.length === 0) { ctx.ui.notify( `No rankable unscored sessions · ${prepared.value.unranked} unranked · ${prepared.value.failures} unreadable`, "info", ); return; } const estimate = estimateScan(prepared.value.plans, selected.model); const providers = new Set( prepared.value.plans .map((plan) => plan.source.model?.split("/")[0]) .filter((provider): provider is string => Boolean(provider)), ); const trustBoundary = providers.size === 1 && providers.has(selected.model.provider) ? "same provider as source sessions" : `session content is sent to ${selected.model.provider}`; const confirmed = await ctx.ui.confirm( `Score ${prepared.value.plans.length} session${prepared.value.plans.length === 1 ? "" : "s"}?`, [ `Judge: ${selected.reference}`, `Estimated: ${estimate.calls} calls · ${formatTokens(estimate.inputTokens + estimate.outputTokens)} tokens · $${estimate.cost.toFixed(2)}`, `Boundary: ${trustBoundary}`, `${prepared.value.unranked} mixed, trivial, or unsettled sessions stay unranked.`, ].join("\n"), ); if (!confirmed) { ctx.ui.notify("IQ scan cancelled", "info"); return; } let release: (() => Promise) | undefined; try { release = await acquireScanLock(stateDirectory); } catch (error) { ctx.ui.notify(boundedError(error), "warning"); return; } let result: Awaited>; try { const latest = recordsForJudge( (await loadRecords(stateDirectory)).records, selected.reference, ); const remaining = prepared.value.plans.filter( (plan) => !latest.some((record) => recordMatches(record, plan.source, selected.reference), ), ); if (remaining.length === 0) { ctx.ui.notify("IQ sessions were scored by another process", "info"); return; } result = await scorePlans(ctx, selected.model, remaining, stateDirectory); } finally { await release(); } if (result.cancelled) { ctx.ui.notify("IQ scan cancelled; completed scores were kept", "info"); return; } if (result.error || !result.value) { ctx.ui.notify(`IQ scan failed: ${boundedError(result.error)}`, "error"); return; } discoveryCache = undefined; ctx.ui.notify( `IQ scored ${result.value.scored} session${result.value.scored === 1 ? "" : "s"}${result.value.failures ? ` · ${result.value.failures} failed` : ""}`, result.value.failures ? "warning" : "info", ); } export async function runIqCommand( args: string, ctx: ExtensionCommandContext, stateDirectory?: string, ): Promise { if (!ctx.hasUI) return; if (ctx.mode !== "tui") { ctx.ui.notify("IQ requires interactive TUI mode", "error"); return; } const command = args.trim().toLowerCase(); if (command && command !== "scan") { ctx.ui.notify("Usage: /iq [scan]", "error"); return; } if (command === "scan") { await scan(ctx, stateDirectory); return; } let selected: { model: AnyModel; reference: string }; try { selected = resolveJudge(ctx); } catch (error) { ctx.ui.notify(boundedError(error), "error"); return; } for (;;) { let action: DashboardAction; try { action = await showDashboard(ctx, selected.reference); } catch (error) { ctx.ui.notify(`IQ dashboard failed: ${boundedError(error)}`, "error"); return; } if (action !== "scan") return; await scan(ctx, stateDirectory); } } export const __test = { dashboardData, resolveJudge, };