Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/activity/__bench__/view.ts

Raw
import { deepStrictEqual } from "node:assert";
import { pathToFileURL } from "node:url";
import headless from "@xterm/headless";
import { terminalView } from "../view.ts";

// Copy view.ts before editing; point this env var at that file for paired runs.
const baseline = process.env.PI_ACTIVITY_BENCH_BASELINE;
const reference: typeof terminalView | undefined = baseline
	? (await import(pathToFileURL(baseline).href)).terminalView
	: undefined;
const { Terminal } = headless;
const iterations = 100;
const blocks = 25;
const mean = (values: number[]) =>
	values.reduce((sum, value) => sum + value, 0) / values.length;
const median = (values: number[]) =>
	[...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
const spread = (values: number[]) => {
	const center = median(values);
	const mad = median(values.map((value) => Math.abs(value - center)));
	return (value: number) =>
		mad === 0 || Math.abs(value - center) <= 3 * 1.4826 * mad;
};

for (const scenario of [
	"plain",
	"styled",
	"alternating",
	"unicode",
	"palette",
]) {
	const terminal = new Terminal({
		allowProposedApi: true,
		cols: 200,
		rows: 60,
		scrollback: 0,
	});
	try {
		const style = "\x1b[1;3;4;7;38;2;90;120;150;48;5;17m";
		const row =
			scenario === "palette"
				? Array.from(
						{ length: 200 },
						(_, index) => `\x1b[0;38;5;${index}mx`,
					).join("")
				: scenario === "alternating"
					? `${style}x\x1b[0my`.repeat(100)
					: scenario === "unicode"
						? `${style}界e\u0301😀`.repeat(40)
						: `${scenario === "styled" ? style : ""}${"x".repeat(terminal.cols)}`;
		await new Promise<void>((resolve) =>
			terminal.write(Array(terminal.rows).fill(row).join("\r\n"), resolve),
		);
		if (reference)
			deepStrictEqual(terminalView(terminal, true), reference(terminal, true));
		const variants = reference ? [reference, terminalView] : [terminalView];
		for (const project of variants)
			for (let warmup = 0; warmup < 100; warmup++) project(terminal, true);
		const samples = variants.map(() => [] as number[]);
		for (let block = 0; block < blocks; block++) {
			// Alternate order to reduce thermal/load drift between paired samples.
			const order =
				block % 2 ? [...variants.keys()].reverse() : [...variants.keys()];
			for (const index of order) {
				const start = performance.now();
				for (let iteration = 0; iteration < iterations; iteration++)
					variants[index](terminal, true);
				samples[index].push((performance.now() - start) / iterations);
			}
		}
		const filters = samples.map(spread);
		const kept = Array.from({ length: blocks }, (_, index) => index).filter(
			(index) =>
				samples.every((values, variant) => filters[variant](values[index])),
		);
		const means = samples.map((values) =>
			mean(kept.map((index) => values[index])),
		);
		let comparison: object | undefined;
		if (reference) {
			// Deterministic paired bootstrap: resample blocks, not inner iterations.
			let seed = 42;
			const effects: number[] = [];
			for (let trial = 0; trial < 10000; trial++) {
				let before = 0;
				let after = 0;
				for (let draw = 0; draw < kept.length; draw++) {
					seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
					const index = kept[Math.floor((seed / 2 ** 32) * kept.length)];
					before += samples[0][index];
					after += samples[1][index];
				}
				effects.push(100 * (1 - after / before));
			}
			effects.sort((a, b) => a - b);
			comparison = {
				reductionPercent: 100 * (1 - means[1] / means[0]),
				ci95Percent: [effects[250], effects[9750]],
			};
		}
		console.log(
			JSON.stringify({
				case: scenario,
				cols: terminal.cols,
				rows: terminal.rows,
				iterations,
				blocks,
				removedPairs: blocks - kept.length,
				meanMs: means,
				stddevMs: samples.map((values, variant) =>
					Math.sqrt(
						kept.reduce(
							(sum, index) => sum + (values[index] - means[variant]) ** 2,
							0,
						) /
							(kept.length - 1),
					),
				),
				unfilteredMeanMs: samples.map(mean),
				comparison,
				samplesMs: samples,
			}),
		);
	} finally {
		terminal.dispose();
	}
}