pi-ext

import { mkdir, mkdtemp, rm } from "node:fs/promises";
import { join, resolve } from "node:path";
import { performance } from "node:perf_hooks";
import { Browser, executeOperations } from "../browser.ts";

const root = resolve(import.meta.dirname, "../../../.pi/tmp/chrome-cdp-bench");
const samples = Number(process.argv[2] ?? 30);
const workload = process.argv[3] ?? "batch";
const frameCount = 16;
if (workload !== "batch" && workload !== "spool" && workload !== "compare")
	throw new Error("workload must be batch, spool, or compare");
if (!Number.isInteger(samples) || samples < 3 || samples > 200) {
	throw new Error("samples must be an integer between 3 and 200");
}
await mkdir(root, { recursive: true });
await mkdir(join(root, "profiles"), { recursive: true });
const frame = { id: 7, result: { data: "x".repeat(512 * 1024) } };

async function measure(extraSerialization = false): Promise<number> {
	const b = new Browser();
	b.spoolDir = await mkdtemp(join(root, "spool-"));
	try {
		const start = performance.now();
		if (workload !== "spool") {
			// Reproduce the former label size probe without changing production code.
			b.send = async () => {
				if (extraSerialization) Buffer.byteLength(JSON.stringify(frame));
				return frame;
			};
			const result = await executeOperations(
				b,
				Array.from({ length: frameCount }, (_, i) => ({
					kind: "cdp",
					method: "Browser.getVersion",
					label: `reply${i}`,
				})),
			);
			if (
				result.failure ||
				result.output.length !== frameCount ||
				result.output.some((item) => !("path" in (item as { raw: object }).raw))
			)
				throw new Error("batch response mismatch");
		} else {
			for (let i = 0; i < frameCount; i++) {
				const result = await b.spoolFrame(frame);
				if (!("path" in result) || result.bytes < 512 * 1024)
					throw new Error("oversized frame not spooled");
			}
		}
		return (performance.now() - start) / frameCount;
	} finally {
		await rm(b.spoolDir, { recursive: true, force: true });
	}
}

const stats = (times: number[]) => {
	const sorted = [...times].sort((a, b) => a - b);
	const mean = times.reduce((a, b) => a + b, 0) / times.length;
	return {
		meanMs: mean,
		medianMs: sorted[Math.floor(sorted.length / 2)],
		sdMs: Math.sqrt(
			times.reduce((a, b) => a + (b - mean) ** 2, 0) / times.length,
		),
		minMs: sorted[0],
		maxMs: sorted.at(-1),
	};
};
if (workload === "compare") {
	for (let i = 0; i < 5; i++) {
		await measure(true);
		await measure(false);
	}
	const old: number[] = [],
		current: number[] = [];
	for (let i = 0; i < samples; i++) {
		if (i % 2 === 0) {
			old.push(await measure(true));
			current.push(await measure(false));
		} else {
			current.push(await measure(false));
			old.push(await measure(true));
		}
	}
	const wins = old.filter((value, i) => value > current[i]).length;
	const tail = Math.min(wins, samples - wins);
	let term = 2 ** -samples,
		p = term;
	for (let k = 1; k <= tail; k++) {
		term *= (samples - k + 1) / k;
		p += term;
	}
	console.log(
		JSON.stringify({
			workload: "batch/512KiB",
			node: process.version,
			samples,
			framesPerSample: frameCount,
			old: stats(old),
			current: stats(current),
			pairedWins: wins,
			signTestP: Math.min(1, 2 * p),
			oldMs: old,
			currentMs: current,
		}),
	);
} else {
	for (let i = 0; i < 5; i++) await measure();
	const times: number[] = [];
	for (let i = 0; i < samples; i++) times.push(await measure());
	console.log(
		JSON.stringify({
			workload: `${workload}/512KiB`,
			node: process.version,
			samples,
			framesPerSample: frameCount,
			...stats(times),
			samplesMs: times,
		}),
	);
}