pi-ext

import { setTimeout as delay } from "node:timers/promises";
import {
	type ExtensionContext,
	ModelRegistry,
	ModelRuntime,
} from "@earendil-works/pi-coding-agent";
import { buildProviders, type WebSearchOptions } from "../search.js";
import { saveJsonResult } from "../store.js";

// Repeatable live comparison, not an overall search-quality benchmark.
// Resolve existing credentials through Pi; never log credentials or response errors.
const cases = [
	{
		query: "FastAPI response model return type documentation",
		target: "fastapi.tiangolo.com/tutorial/response-model",
	},
	{
		query: "Node.js readFile filesystem official documentation",
		target: "nodejs.org/api/fs.html",
	},
	{
		query: "Rust async fn traits stabilized 1.75 limitations",
		target: "blog.rust-lang.org/2023/12/21/async-fn-rpit-in-traits",
	},
	{
		query: "Nushell get optional column ignore missing documentation",
		target: "nushell.sh/commands/docs/get.html",
	},
	{
		query:
			"Bundesnetzagentur Internet zu langsam Minderungsrecht Nachweis Breitbandmessung",
		target: "bundesnetzagentur.de",
	},
];
const allSources = ["google", "parallel", "codex", "minimax", "ddg", "zai"];
const runtime = await ModelRuntime.create({
	allowModelNetwork: false,
	signal: AbortSignal.timeout(30_000),
});
const ctx = { modelRegistry: new ModelRegistry(runtime) } as ExtensionContext;
const runs: object[] = [];
const blocked = new Set<string>();
const report = {
	at: new Date().toISOString(),
	node: process.version,
	platform: process.platform,
	count: 5,
	timeoutMs: 45_000,
	cases,
	runs,
};

for (const [index, sample] of cases.entries()) {
	const options: WebSearchOptions = {
		query: sample.query,
		count: report.count,
		timeout: report.timeoutMs,
		ctx,
	};
	const providers = await buildProviders(options);
	// Rotate order to reduce a consistent warmup/order advantage.
	const order = [
		...providers.slice(index % providers.length),
		...providers.slice(0, index % providers.length),
	];
	for (const source of allSources.filter(
		(source) => !providers.some((provider) => provider.source === source),
	))
		runs.push({ query: index, source, status: "not configured" });
	for (const provider of order) {
		if (blocked.has(provider.source)) {
			runs.push({
				query: index,
				source: provider.source,
				status: "skipped after auth/quota/challenge failure",
			});
			continue;
		}
		options.signal = AbortSignal.timeout(report.timeoutMs);
		const start = performance.now();
		try {
			const results = (await provider.search()).slice(0, report.count);
			const hit = results.findIndex((result) => {
				try {
					const url = new URL(result.url);
					return `${url.hostname}${url.pathname}`.includes(sample.target);
				} catch {
					return false;
				}
			});
			const record = {
				query: index,
				source: provider.source,
				status: results.length ? "ok" : "empty",
				ms: Math.round(performance.now() - start),
				count: results.length,
				targetRank: hit < 0 ? null : hit + 1,
				snippetChars: results.reduce(
					(sum, result) => sum + result.snippet.length,
					0,
				),
				results,
			};
			runs.push(record);
			console.log(
				JSON.stringify({
					...record,
					results: results.map(({ title, url }) => ({ title, url })),
				}),
			);
		} catch (error) {
			const err = error as Error & { status?: number; code?: string };
			const http =
				err.status ??
				(Number(err.message?.match(/(?:HTTP|Codex search)\s+(\d{3})/)?.[1]) ||
					undefined);
			const category = /captcha|traffic block|unusual traffic/i.test(
				err.message,
			)
				? "challenge"
				: /timeout|abort/i.test(err.name + err.message)
					? "timeout/abort"
					: "error";
			const record = {
				query: index,
				source: provider.source,
				status: category,
				ms: Math.round(performance.now() - start),
				http,
				code: err.code,
			};
			runs.push(record);
			console.log(JSON.stringify(record));
			if (category === "challenge" || [401, 403, 429].includes(http ?? 0))
				blocked.add(provider.source);
		}
		await delay(2100);
	}
}
const saved = await saveJsonResult(
	"web-search-evaluation",
	process.cwd(),
	report,
);
console.log(`Full evaluation: ${saved.fullOutputPath}`);