Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/devdocs/__bench__/read.ts

Raw
import assert from "node:assert/strict";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { Session } from "node:inspector/promises";
import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { performance } from "node:perf_hooks";
import { pathToFileURL } from "node:url";
import type { Devdocs as Engine } from "../src/engine.ts";
import type { Adapter, Options, ReadResult } from "../src/types.ts";

// Fixed real-cache workloads: 10 warmups, 30 measured reads by default.
// DEVDOCS_BENCH_MODULE selects an engine snapshot for separate baseline runs.
// DEVDOCS_BENCH_PROFILE writes a native CPU profile; do not mix profiled timings
// with unprofiled comparisons. No network, personal settings, or retained databases.
const imported: { Devdocs: typeof Engine } = await import(
	process.env["DEVDOCS_BENCH_MODULE"]
		? pathToFileURL(process.env["DEVDOCS_BENCH_MODULE"]).href
		: "../src/engine.ts"
);
const rounds = Number(process.env["DEVDOCS_BENCH_ROUNDS"] ?? 30);
if (!Number.isInteger(rounds) || rounds < 5 || rounds > 200)
	throw new Error("benchmark rounds must be 5..200");
const profile = process.env["DEVDOCS_BENCH_PROFILE"];
if (profile && !isAbsolute(profile))
	throw new Error("profile path must be absolute");
const profiler = profile ? new Session() : undefined;
const workloads = [
	{
		name: "single",
		sections: 1,
		pattern: "ordinary reference ",
		size: 256,
		budget: 4000,
	},
	{
		name: "multi-ascii",
		sections: 20,
		pattern: "ordinary reference ",
		size: 2000,
		budget: 64000,
	},
	{
		name: "multi-unicode",
		sections: 20,
		pattern: 'α😀e\u0301漢\n"\\\t',
		size: 2000,
		budget: 64000,
	},
	{
		name: "truncated",
		sections: 20,
		pattern: 'α😀e\u0301漢\n"\\\t',
		size: 10000,
		budget: 4000,
	},
];
const quantile = (values: number[], p: number) =>
	values[Math.min(values.length - 1, Math.floor(values.length * p))] ?? 0;
const root = await mkdtemp(join(tmpdir(), "devdocs-read-bench-"));
try {
	if (profiler) {
		profiler.connect();
		await profiler.post("Profiler.enable");
		await profiler.post("Profiler.start");
	}
	for (const workload of workloads) {
		const body = workload.pattern
			.repeat(Math.ceil(workload.size / workload.pattern.length))
			.slice(0, workload.size);
		const adapter: Adapter = {
			id: "benchmark",
			rank: 1,
			formatVersion: 1,
			coverage: "corpus",
			discover: async () => [
				{
					id: "fixture",
					source: "benchmark",
					target: "fixture",
					version: "1.0",
					origin: "https://docs.example/fixture",
					title: "Fixture",
					scope: "public",
					evidence: "benchmark",
				},
			],
			retrieve: async (candidate) => ({
				candidate,
				sections: Array.from({ length: workload.sections }, (_, i) => ({
					locator: String(i),
					heading: "Reference " + i,
					body,
				})),
				attribution: "Benchmark fixture",
				validator: "unchanged",
			}),
		};
		const options: Options = {
			databasePath: join(root, workload.name + ".db"),
			sources: [],
			enabledSources: [adapter.id],
			offline: true,
			cacheBytes: 16 * 1024 * 1024,
			ttlMs: 60000,
		};
		const docs = new imported.Devdocs(options, [adapter]);
		try {
			const found = await docs.search({
				target: "fixture",
				query: "reference",
				cwd: root,
			});
			assert.equal(found.status, "ok");
			if (found.status !== "ok")
				throw new Error("benchmark fixture did not resolve");
			const revision = found.results[0]?.ref.split(":")[0];
			assert.ok(revision);
			const refs = Array.from(
				{ length: workload.sections },
				(_, i) => revision + ":" + i,
			);
			let last: ReadResult | undefined;
			for (let i = 0; i < 10; i++)
				last = await docs.read(refs, workload.budget, root);
			const elapsed: number[] = [],
				cpu: number[] = [];
			for (let i = 0; i < rounds; i++) {
				const before = process.cpuUsage(),
					start = performance.now();
				last = await docs.read(refs, workload.budget, root);
				elapsed.push(performance.now() - start);
				const used = process.cpuUsage(before);
				cpu.push((used.user + used.system) / 1000);
			}
			assert.ok(last);
			assert.ok([...JSON.stringify(last)].length <= workload.budget);
			assert.ok(last.sections[0]?.content);
			const sorted = [...elapsed].sort((a, b) => a - b);
			console.log(
				JSON.stringify({
					name: workload.name,
					node: process.version,
					rounds,
					warmup: 10,
					medianMs: quantile(sorted, 0.5),
					p95Ms: quantile(sorted, 0.95),
					meanCpuMs: cpu.reduce((a, b) => a + b, 0) / rounds,
					maxRssKiB: process.resourceUsage().maxRSS,
					elapsedMs: elapsed,
					cpuMs: cpu,
				}),
			);
		} finally {
			await docs.close();
		}
	}
} finally {
	try {
		if (profiler && profile) {
			const data = await profiler.post("Profiler.stop");
			await writeFile(profile, JSON.stringify(data.profile));
		}
	} finally {
		profiler?.disconnect();
		await rm(root, { recursive: true, force: true });
	}
}