repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by bugabinga
extensions/iq/implementation.ts
Rawimport type { Dirent } from "node:fs";
import { readdir, stat } from "node:fs/promises";
import { basename, join, resolve } from "node:path";
import type { Api, Model } from "@earendil-works/pi-ai";
import {
type ExtensionCommandContext,
type ExtensionContext,
SettingsManager,
} from "@earendil-works/pi-coding-agent";
import {
aggregateModels,
defaultModelReference,
estimateScan,
fileSha256,
parseModelReference,
prepareSession,
readIqSettings,
type SessionPlan,
type SourceSignature,
sessionPlan,
} from "./core.js";
import { judgeSession } from "./judge.js";
import {
acquireScanLock,
iqStateDirectory,
loadRecords,
recordMatches,
recordsForJudge,
saveRecord,
} from "./storage.js";
import {
type DashboardAction,
type DashboardData,
IqDashboardComponent,
IqScanComponent,
type ScanDisplayState,
} from "./ui.js";
type AnyModel = Model<Api>;
interface DiscoverySnapshot {
sessionDir: string;
candidates: SourceSignature[];
discoveredAt: number;
}
interface ProgressOutcome<T> {
value?: T;
error?: Error;
cancelled: boolean;
}
interface PreflightResult {
plans: SessionPlan[];
unranked: number;
failures: number;
}
let discoveryCache: DiscoverySnapshot | undefined;
const eligibilityCache = new Map<
string,
{ size: number; mtimeMs: number; ranked: boolean }
>();
function asError(value: unknown): Error {
return value instanceof Error ? value : new Error(String(value));
}
function boundedError(value: unknown): string {
return asError(value).message.replaceAll(/\s+/g, " ").slice(0, 300);
}
async function safeReadDir(path: string): Promise<Dirent[]> {
try {
return await readdir(path, { withFileTypes: true });
} catch (error) {
if (isNodeError(error) && error.code === "ENOENT") return [];
throw error;
}
}
function isNodeError(error: unknown): error is NodeJS.ErrnoException {
return error instanceof Error && "code" in error;
}
async function collectSessionFiles(
directory: string,
files: string[],
signal: AbortSignal,
): Promise<void> {
signal.throwIfAborted();
for (const entry of await safeReadDir(directory)) {
signal.throwIfAborted();
const path = join(directory, entry.name);
if (entry.isDirectory()) await collectSessionFiles(path, files, signal);
else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(path);
}
}
export async function discoverSessions(
sessionDir: string,
currentSessionFile: string | undefined,
signal: AbortSignal,
): Promise<DiscoverySnapshot> {
const files: string[] = [];
await collectSessionFiles(sessionDir, files, signal);
const current = currentSessionFile ? resolve(currentSessionFile) : undefined;
const candidates: SourceSignature[] = [];
for (const path of files.sort()) {
signal.throwIfAborted();
if (current && resolve(path) === current) continue;
try {
const info = await stat(path);
candidates.push({ path, size: info.size, mtimeMs: info.mtimeMs });
} catch {
// A session can disappear between directory enumeration and stat.
}
}
return { sessionDir, candidates, discoveredAt: Date.now() };
}
export async function discoverInBackground(
ctx: ExtensionContext,
signal: AbortSignal,
isCurrent: () => boolean,
): Promise<void> {
const sessionDir = ctx.sessionManager.getSessionDir();
const current = ctx.sessionManager.getSessionFile();
const snapshot = await discoverSessions(sessionDir, current, signal);
if (!signal.aborted && isCurrent()) discoveryCache = snapshot;
}
function settings(ctx: ExtensionCommandContext) {
const manager = SettingsManager.create(ctx.cwd);
const global = manager.getGlobalSettings();
const project = manager.getProjectSettings();
return {
iq: readIqSettings(global, project),
defaultModel: defaultModelReference(global, project),
};
}
function resolveJudge(ctx: ExtensionCommandContext): {
model: AnyModel;
reference: string;
} {
const configured = settings(ctx);
const reference = configured.iq.judgeModel ?? configured.defaultModel;
if (!reference) {
if (!ctx.model)
throw new Error("No configured default model for IQ judging");
return {
model: ctx.model,
reference: `${ctx.model.provider}/${ctx.model.id}`,
};
}
const parsed = parseModelReference(reference);
if (!parsed)
throw new Error(`Invalid iq.judgeModel: ${JSON.stringify(reference)}`);
const model = ctx.modelRegistry.find(parsed.provider, parsed.modelId);
if (!model) throw new Error(`IQ judge model not found: ${reference}`);
return { model, reference };
}
async function currentDiscovery(
ctx: ExtensionCommandContext,
signal: AbortSignal,
force = false,
): Promise<DiscoverySnapshot> {
const sessionDir = ctx.sessionManager.getSessionDir();
if (!force && discoveryCache?.sessionDir === sessionDir)
return discoveryCache;
const snapshot = await discoverSessions(
sessionDir,
ctx.sessionManager.getSessionFile(),
signal,
);
discoveryCache = snapshot;
return snapshot;
}
async function dashboardData(
ctx: ExtensionCommandContext,
judge: string,
): Promise<DashboardData> {
const discovery = await currentDiscovery(ctx, new AbortController().signal);
const stateDirectory = iqStateDirectory();
const loaded = await loadRecords(stateDirectory);
const active = recordsForJudge(loaded.records, judge).sort((left, right) =>
right.scoredAt.localeCompare(left.scoredAt),
);
const unscored = discovery.candidates.filter(
(candidate) =>
!active.some((record) => recordMatches(record, candidate, judge)),
);
const unranked = unscored.filter((candidate) => {
const cached = eligibilityCache.get(candidate.path);
return (
cached?.size === candidate.size &&
cached.mtimeMs === candidate.mtimeMs &&
!cached.ranked
);
}).length;
return {
judge,
stateDirectory,
leaderboard: aggregateModels(active),
records: active,
pending: unscored.length - unranked,
unranked,
storageErrors: loaded.errors.length,
};
}
async function showDashboard(
ctx: ExtensionCommandContext,
judge: string,
): Promise<DashboardAction> {
const data = await dashboardData(ctx, judge);
return ctx.ui.custom<DashboardAction>((tui, theme, _keybindings, done) => {
const component = new IqDashboardComponent(
theme,
data,
() => tui.requestRender(),
done,
);
return component;
});
}
async function runProgress<T>(
ctx: ExtensionCommandContext,
state: ScanDisplayState,
work: (
signal: AbortSignal,
state: ScanDisplayState,
requestRender: () => void,
) => Promise<T>,
): Promise<ProgressOutcome<T>> {
return ctx.ui.custom<ProgressOutcome<T>>((tui, theme, _keybindings, done) => {
const component = new IqScanComponent(tui, theme, state);
let finished = false;
const finish = (outcome: ProgressOutcome<T>) => {
if (finished) return;
finished = true;
component.dispose();
done(outcome);
};
void work(component.signal, state, () => tui.requestRender())
.then((value) => finish({ value, cancelled: false }))
.catch((error: unknown) =>
finish({
error: asError(error),
cancelled: component.signal.aborted,
}),
);
return component;
});
}
async function preflight(
ctx: ExtensionCommandContext,
model: AnyModel,
judge: string,
stateDirectory: string,
): Promise<ProgressOutcome<PreflightResult>> {
const discovery = await currentDiscovery(
ctx,
new AbortController().signal,
true,
);
const loaded = await loadRecords(stateDirectory);
const active = recordsForJudge(loaded.records, judge);
const candidates = discovery.candidates.filter(
(candidate) =>
!active.some((record) => recordMatches(record, candidate, judge)),
);
const state: ScanDisplayState = {
phase: "prepare",
model: `${model.provider}/${model.id}`,
currentName: "",
session: 0,
sessions: candidates.length,
chunk: 0,
chunks: 0,
evidence: 0,
cost: 0,
failures: 0,
startedAt: Date.now(),
};
return runProgress(ctx, state, async (signal, display, render) => {
const plans: SessionPlan[] = [];
let unranked = 0;
let failures = 0;
for (let index = 0; index < candidates.length; index++) {
signal.throwIfAborted();
const candidate = candidates[index];
if (!candidate) continue;
display.session = index + 1;
display.currentName = basename(candidate.path);
render();
try {
const plan = sessionPlan(
await prepareSession(candidate.path, Date.now(), signal),
);
eligibilityCache.set(candidate.path, {
size: plan.source.size,
mtimeMs: plan.source.mtimeMs,
ranked: plan.eligibility.ranked,
});
if (plan.eligibility.ranked) plans.push(plan);
else unranked++;
} catch {
failures++;
display.failures = failures;
}
}
return { plans, unranked, failures };
});
}
function formatTokens(tokens: number): string {
if (tokens < 1_000) return String(tokens);
if (tokens < 1_000_000) return `${(tokens / 1_000).toFixed(1)}k`;
return `${(tokens / 1_000_000).toFixed(2)}M`;
}
async function scorePlans(
ctx: ExtensionCommandContext,
model: AnyModel,
plans: readonly SessionPlan[],
stateDirectory: string,
): Promise<ProgressOutcome<{ scored: number; failures: number }>> {
const state: ScanDisplayState = {
phase: "judge",
model: `${model.provider}/${model.id}`,
currentName: "",
session: 0,
sessions: plans.length,
chunk: 0,
chunks: 0,
evidence: 0,
cost: 0,
failures: 0,
startedAt: Date.now(),
};
return runProgress(ctx, state, async (signal, display, render) => {
let scored = 0;
let failures = 0;
let completedCost = 0;
for (let index = 0; index < plans.length; index++) {
signal.throwIfAborted();
const plan = plans[index];
if (!plan) continue;
display.session = index + 1;
display.currentName = plan.source.name || basename(plan.source.path);
display.chunk = 0;
display.chunks = plan.chunkCount;
display.evidence = 0;
render();
try {
const prepared = await prepareSession(
plan.source.path,
Date.now(),
signal,
);
if (
prepared.source.size !== plan.source.size ||
prepared.source.mtimeMs !== plan.source.mtimeMs ||
prepared.source.sha256 !== plan.source.sha256
)
throw new Error("session changed after cost confirmation");
const record = await judgeSession(
ctx,
model,
prepared,
signal,
(progress) => {
display.chunk = progress.chunk;
display.chunks = progress.chunks;
display.evidence = progress.evidence;
display.cost = completedCost + progress.usage.cost;
render();
},
);
const beforeHash = await stat(plan.source.path);
const hash = await fileSha256(plan.source.path, signal);
const afterHash = await stat(plan.source.path);
if (
beforeHash.size !== record.source.size ||
beforeHash.mtimeMs !== record.source.mtimeMs ||
afterHash.size !== beforeHash.size ||
afterHash.mtimeMs !== beforeHash.mtimeMs ||
hash !== record.source.sha256
)
throw new Error("session changed while being judged");
await saveRecord(record, stateDirectory);
completedCost += record.usage.cost;
display.cost = completedCost;
scored++;
} catch (error) {
if (signal.aborted) throw error;
failures++;
display.failures = failures;
render();
}
}
return { scored, failures };
});
}
async function scan(
ctx: ExtensionCommandContext,
stateDirectory = iqStateDirectory(),
): Promise<void> {
let selected: { model: AnyModel; reference: string };
try {
selected = resolveJudge(ctx);
} catch (error) {
ctx.ui.notify(boundedError(error), "error");
return;
}
const auth = await ctx.modelRegistry.getApiKeyAndHeaders(selected.model);
if (!auth.ok) {
ctx.ui.notify(auth.error || "IQ judge authentication failed", "error");
return;
}
let prepared: Awaited<ReturnType<typeof preflight>>;
try {
prepared = await preflight(
ctx,
selected.model,
selected.reference,
stateDirectory,
);
} catch (error) {
ctx.ui.notify(`IQ discovery failed: ${boundedError(error)}`, "error");
return;
}
if (prepared.cancelled) {
ctx.ui.notify("IQ scan cancelled", "info");
return;
}
if (prepared.error || !prepared.value) {
ctx.ui.notify(
`IQ preparation failed: ${boundedError(prepared.error)}`,
"error",
);
return;
}
if (prepared.value.plans.length === 0) {
ctx.ui.notify(
`No rankable unscored sessions · ${prepared.value.unranked} unranked · ${prepared.value.failures} unreadable`,
"info",
);
return;
}
const estimate = estimateScan(prepared.value.plans, selected.model);
const providers = new Set(
prepared.value.plans
.map((plan) => plan.source.model?.split("/")[0])
.filter((provider): provider is string => Boolean(provider)),
);
const trustBoundary =
providers.size === 1 && providers.has(selected.model.provider)
? "same provider as source sessions"
: `session content is sent to ${selected.model.provider}`;
const confirmed = await ctx.ui.confirm(
`Score ${prepared.value.plans.length} session${prepared.value.plans.length === 1 ? "" : "s"}?`,
[
`Judge: ${selected.reference}`,
`Estimated: ${estimate.calls} calls · ${formatTokens(estimate.inputTokens + estimate.outputTokens)} tokens · $${estimate.cost.toFixed(2)}`,
`Boundary: ${trustBoundary}`,
`${prepared.value.unranked} mixed, trivial, or unsettled sessions stay unranked.`,
].join("\n"),
);
if (!confirmed) {
ctx.ui.notify("IQ scan cancelled", "info");
return;
}
let release: (() => Promise<void>) | undefined;
try {
release = await acquireScanLock(stateDirectory);
} catch (error) {
ctx.ui.notify(boundedError(error), "warning");
return;
}
let result: Awaited<ReturnType<typeof scorePlans>>;
try {
const latest = recordsForJudge(
(await loadRecords(stateDirectory)).records,
selected.reference,
);
const remaining = prepared.value.plans.filter(
(plan) =>
!latest.some((record) =>
recordMatches(record, plan.source, selected.reference),
),
);
if (remaining.length === 0) {
ctx.ui.notify("IQ sessions were scored by another process", "info");
return;
}
result = await scorePlans(ctx, selected.model, remaining, stateDirectory);
} finally {
await release();
}
if (result.cancelled) {
ctx.ui.notify("IQ scan cancelled; completed scores were kept", "info");
return;
}
if (result.error || !result.value) {
ctx.ui.notify(`IQ scan failed: ${boundedError(result.error)}`, "error");
return;
}
discoveryCache = undefined;
ctx.ui.notify(
`IQ scored ${result.value.scored} session${result.value.scored === 1 ? "" : "s"}${result.value.failures ? ` · ${result.value.failures} failed` : ""}`,
result.value.failures ? "warning" : "info",
);
}
export async function runIqCommand(
args: string,
ctx: ExtensionCommandContext,
stateDirectory?: string,
): Promise<void> {
if (!ctx.hasUI) return;
if (ctx.mode !== "tui") {
ctx.ui.notify("IQ requires interactive TUI mode", "error");
return;
}
const command = args.trim().toLowerCase();
if (command && command !== "scan") {
ctx.ui.notify("Usage: /iq [scan]", "error");
return;
}
if (command === "scan") {
await scan(ctx, stateDirectory);
return;
}
let selected: { model: AnyModel; reference: string };
try {
selected = resolveJudge(ctx);
} catch (error) {
ctx.ui.notify(boundedError(error), "error");
return;
}
for (;;) {
let action: DashboardAction;
try {
action = await showDashboard(ctx, selected.reference);
} catch (error) {
ctx.ui.notify(`IQ dashboard failed: ${boundedError(error)}`, "error");
return;
}
if (action !== "scan") return;
await scan(ctx, stateDirectory);
}
}
export const __test = {
dashboardData,
resolveJudge,
};