pi-ext

#!/usr/bin/env node

import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { performance } from "node:perf_hooks";
import { fileURLToPath, pathToFileURL } from "node:url";
import { executableOnPath, resolvePiInstallation } from "./audit.mjs";

const RUNS = 20;
const WARMUPS = 3;
const script = fileURLToPath(import.meta.url);

export function checkedSpawn(command, args, options = {}) {
	const result = spawnSync(command, args, {
		encoding: "utf8",
		windowsHide: true,
		...options,
	});
	if (result.error) throw result.error;
	if (result.status !== 0) {
		const outcome = result.signal
			? `signal ${result.signal}`
			: `code ${result.status}`;
		const detail = (result.stderr || result.stdout || "").trim();
		throw new Error(
			`${command} exited with ${outcome}${detail ? `: ${detail}` : ""}`,
		);
	}
	return result;
}

function installation() {
	return resolvePiInstallation(executableOnPath("pi"));
}

function spawnedVersion() {
	const { cli } = installation();
	return checkedSpawn(process.execPath, [cli, "--version"]).stdout.trim();
}

function manifestVersion() {
	return installation().version;
}

function command(mode) {
	return [process.execPath, script, mode]
		.map((argument) => JSON.stringify(argument.replaceAll("\\", "/")))
		.join(" ");
}

function writeBenchstat(pathname, result) {
	const lines = [
		`goos: ${process.platform}`,
		`goarch: ${process.arch}`,
		"pkg: pi-ext/modernize",
		...result.times.map(
			(seconds) =>
				`BenchmarkLocalPiVersion 1 ${Math.round(seconds * 1e9)} ns/op`,
		),
	];
	writeFileSync(pathname, `${lines.join("\n")}\n`);
}

function trace() {
	const samples = [];
	for (let index = 0; index < 10; index++) {
		const started = performance.now();
		const executable = executableOnPath("pi");
		const foundExecutable = performance.now();
		const { cli } = resolvePiInstallation(executable);
		const resolvedPackage = performance.now();
		checkedSpawn(process.execPath, [cli, "--version"]);
		const readVersion = performance.now();
		samples.push({
			executable: foundExecutable - started,
			package: resolvedPackage - foundExecutable,
			spawn: readVersion - resolvedPackage,
		});
	}
	for (const key of ["executable", "package", "spawn"]) {
		const mean =
			samples.reduce((total, sample) => total + sample[key], 0) /
			samples.length;
		console.log(`${key}: ${mean.toFixed(2)} ms mean`);
	}
}

function compare() {
	const baselineVersion = spawnedVersion();
	const candidateVersion = manifestVersion();
	if (baselineVersion !== candidateVersion)
		throw new Error(
			`version mismatch: subprocess ${baselineVersion}, manifest ${candidateVersion}`,
		);

	const directory = mkdtempSync(path.join(tmpdir(), "pi-modernize-bench-"));
	try {
		const json = path.join(directory, "hyperfine.json");
		checkedSpawn(
			"hyperfine",
			[
				"--shell=none",
				"--style",
				"basic",
				"--warmup",
				String(WARMUPS),
				"--runs",
				String(RUNS),
				"--export-json",
				json,
				"--command-name",
				"subprocess",
				command("spawn"),
				"--command-name",
				"manifest",
				command("manifest"),
			],
			{ stdio: "inherit" },
		);
		const results = JSON.parse(readFileSync(json, "utf8")).results;
		const baseline = path.join(directory, "subprocess.txt");
		const candidate = path.join(directory, "manifest.txt");
		writeBenchstat(baseline, results[0]);
		writeBenchstat(candidate, results[1]);
		checkedSpawn("benchstat", [baseline, candidate], { stdio: "inherit" });
	} finally {
		rmSync(directory, { recursive: true, force: true });
	}
}

function main() {
	switch (process.argv[2]) {
		case "spawn":
			spawnedVersion();
			break;
		case "manifest":
			manifestVersion();
			break;
		case "trace":
			trace();
			break;
		case undefined:
			compare();
			break;
		default:
			throw new Error(`unknown benchmark mode: ${process.argv[2]}`);
	}
}

const invoked = process.argv[1]
	? import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
	: false;
if (invoked) main();