pi-ext

// End-to-end benchmark against a real headless Firefox: measures the batch
// loop where spool writes can overlap real WebSocket round-trips.
// Baseline = inline spool awaits (__bench__/baseline.ts), candidate = deferred
// spool drain (lib.ts). One sessionless /session connection carries both
// variants; browsing contexts are created up front. Interleaved pairs,
// Wilcoxon signed-rank.
// Usage: node --experimental-strip-types __bench__/e2e.mts [pairs]

import { spawn } from "node:child_process";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { performance } from "node:perf_hooks";
import { runBatchBaseline } from "./baseline.ts";
import { runBatch, type BidiResponse, type BatchCtx, type Frame } from "../lib.ts";

const ITER = Number(process.argv[2] ?? 12);

function stats(s: number[]) {
	const sorted = [...s].sort((a, b) => a - b);
	const mean = s.reduce((a, b) => a + b, 0) / s.length;
	return { n: s.length, median: sorted[Math.floor(sorted.length / 2)], mean, sd: Math.sqrt(s.reduce((a, b) => a + (b - mean) ** 2, 0) / s.length) };
}
function erf(x: number): number {
	const t = 1 / (1 + 0.3275911 * Math.abs(x));
	const y = 1 - ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.exp(-x * x);
	return x >= 0 ? y : -y;
}
function wilcoxon(d: number[]): number {
	const nz = d.filter((x) => x !== 0).map((x) => ({ abs: Math.abs(x), sign: Math.sign(x), rank: 0 }));
	nz.sort((a, b) => a.abs - b.abs);
	nz.forEach((o, i) => (o.rank = i + 1));
	const w = nz.filter((o) => o.sign > 0).reduce((a, o) => a + o.rank, 0);
	const n = nz.length;
	const z = (w - (n * (n + 1)) / 4) / Math.sqrt((n * (n + 1) * (2 * n + 1)) / 24);
	return 2 * (1 - 0.5 * (1 + erf(Math.abs(z) / Math.SQRT2)));
}

// Launch Firefox
const profileDir = await mkdtemp(join(tmpdir(), "ffx-e2e-"));
const child = spawn(process.env.FIREFOX_BIN ?? "firefox", ["--headless", "--no-remote", "--profile", profileDir, "--remote-debugging-port", "0"], { stdio: ["ignore", "pipe", "pipe"] });
const baseUrl = await new Promise<string>((resolve, reject) => {
	const t = setTimeout(() => reject(new Error("no listener")), 30_000);
	const grab = (buf: Buffer) => {
		const m = String(buf).match(/WebDriver BiDi listening on (ws:\/\/\S+)/);
		if (m) {
			clearTimeout(t);
			resolve(m[1]);
		}
	};
	child.stdout?.on("data", grab);
	child.stderr?.on("data", grab);
});

// One sessionless connection; session.new attaches it. All commands and both
// variants run through this socket.
const ws = await connect(`${baseUrl}/session`);
const pending = new Map<number, (r: BidiResponse) => void>();
ws.onmessage = (m) => {
	const f = JSON.parse(String(m.data));
	if (f.type === "event") return;
	pending.get(f.id)?.(f);
};
function connect(url: string): Promise<WebSocket> {
	return new Promise((resolve, reject) => {
		const w = new WebSocket(url);
		const t = setTimeout(() => reject(new Error("connect timeout")), 10_000);
		w.onopen = () => {
			clearTimeout(t);
			resolve(w);
		};
		w.onerror = () => {
			clearTimeout(t);
			reject(new Error("connect failed"));
		};
	});
}
let nextId = 1;
function call(method: string, params: Record<string, unknown>): Promise<BidiResponse> {
	const id = nextId++;
	return new Promise((resolve) => {
		pending.set(id, resolve);
		ws.send(JSON.stringify({ id, method, params }));
	});
}

const sn = await call("session.new", { capabilities: { alwaysMatch: { acceptInsecureCerts: true } } });
if (sn.type !== "success") throw new Error(`session.new failed: ${JSON.stringify(sn)}`);

async function makeSlot(spoolParent: string): Promise<{ ctx: BatchCtx; ctxId: string; nextId: () => number; n: { v: number } }> {
	const cr = await call("browsingContext.create", { type: "tab" });
	const ctxId = (cr.result as { context: string }).context;
	const spoolDir = await mkdtemp(join(tmpdir(), spoolParent));
	const n = { v: 500 };
	return {
		ctx: { pending, ledger: { subscriptions: [], intercepts: [], preloadScripts: [] }, spoolDir },
		ctxId,
		nextId: () => n.v++,
		n,
	};
}

const framesFor = (ctxId: string, ids: () => number): Frame[] => [
	{ id: ids(), method: "browsingContext.navigate", params: { context: ctxId, url: "data:text/html,<b>x</b>", wait: "complete" } },
	{ id: ids(), method: "script.evaluate", params: { expression: `"A".repeat(${300 * 1024})`, awaitPromise: false, target: { context: ctxId } } },
	{ id: ids(), method: "script.evaluate", params: { expression: `"A".repeat(${300 * 1024})`, awaitPromise: false, target: { context: ctxId } } },
	{ id: ids(), method: "script.evaluate", params: { expression: `"A".repeat(${300 * 1024})`, awaitPromise: false, target: { context: ctxId } } },
	{ id: ids(), method: "browsingContext.navigate", params: { context: ctxId, url: "data:text/html,<i>y</i>", wait: "complete" } },
];

const slotA = await makeSlot("ffx-e2e-a-");
const slotB = await makeSlot("ffx-e2e-b-");

// Warmup
await runBatchBaseline(ws, slotA.ctx, framesFor(slotA.ctxId, slotA.nextId), new Map(), 30000);
await runBatch(ws, slotB.ctx, framesFor(slotB.ctxId, slotB.nextId), new Map(), 30000);

const pairs: { base: number; cand: number }[] = [];
for (let i = 0; i < ITER; i++) {
	{
		const t0 = performance.now();
		await runBatchBaseline(ws, slotA.ctx, framesFor(slotA.ctxId, slotA.nextId), new Map(), 30000);
		pairs.push({ base: performance.now() - t0, cand: 0 });
	}
	{
		const t1 = performance.now();
		await runBatch(ws, slotB.ctx, framesFor(slotB.ctxId, slotB.nextId), new Map(), 30000);
		pairs[i].cand = performance.now() - t1;
	}
}

const baseSamples = pairs.map((p) => p.base);
const candSamples = pairs.map((p) => p.cand);
const diffs = pairs.map((p) => p.base - p.cand);
console.log(JSON.stringify({
	pairs: ITER,
	samples_ms: { baseline_inlineSpool: baseSamples, candidate_deferredSpool: candSamples },
	stats: { baseline: stats(baseSamples), candidate: stats(candSamples) },
	paired_diff_ms: { mean: diffs.reduce((a, b) => a + b, 0) / diffs.length },
	wilcoxon_p: wilcoxon(diffs),
}));

ws.close();
child.kill("SIGTERM");
await rm(profileDir, { recursive: true, force: true }).catch(() => {});