repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/devdocs/__bench__/read.ts
Rawimport assert from "node:assert/strict";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { Session } from "node:inspector/promises";
import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { performance } from "node:perf_hooks";
import { pathToFileURL } from "node:url";
import type { Devdocs as Engine } from "../src/engine.ts";
import type { Adapter, Options, ReadResult } from "../src/types.ts";
// Fixed real-cache workloads: 10 warmups, 30 measured reads by default.
// DEVDOCS_BENCH_MODULE selects an engine snapshot for separate baseline runs.
// DEVDOCS_BENCH_PROFILE writes a native CPU profile; do not mix profiled timings
// with unprofiled comparisons. No network, personal settings, or retained databases.
const imported: { Devdocs: typeof Engine } = await import(
process.env["DEVDOCS_BENCH_MODULE"]
? pathToFileURL(process.env["DEVDOCS_BENCH_MODULE"]).href
: "../src/engine.ts"
);
const rounds = Number(process.env["DEVDOCS_BENCH_ROUNDS"] ?? 30);
if (!Number.isInteger(rounds) || rounds < 5 || rounds > 200)
throw new Error("benchmark rounds must be 5..200");
const profile = process.env["DEVDOCS_BENCH_PROFILE"];
if (profile && !isAbsolute(profile))
throw new Error("profile path must be absolute");
const profiler = profile ? new Session() : undefined;
const workloads = [
{
name: "single",
sections: 1,
pattern: "ordinary reference ",
size: 256,
budget: 4000,
},
{
name: "multi-ascii",
sections: 20,
pattern: "ordinary reference ",
size: 2000,
budget: 64000,
},
{
name: "multi-unicode",
sections: 20,
pattern: 'α😀e\u0301漢\n"\\\t',
size: 2000,
budget: 64000,
},
{
name: "truncated",
sections: 20,
pattern: 'α😀e\u0301漢\n"\\\t',
size: 10000,
budget: 4000,
},
];
const quantile = (values: number[], p: number) =>
values[Math.min(values.length - 1, Math.floor(values.length * p))] ?? 0;
const root = await mkdtemp(join(tmpdir(), "devdocs-read-bench-"));
try {
if (profiler) {
profiler.connect();
await profiler.post("Profiler.enable");
await profiler.post("Profiler.start");
}
for (const workload of workloads) {
const body = workload.pattern
.repeat(Math.ceil(workload.size / workload.pattern.length))
.slice(0, workload.size);
const adapter: Adapter = {
id: "benchmark",
rank: 1,
formatVersion: 1,
coverage: "corpus",
discover: async () => [
{
id: "fixture",
source: "benchmark",
target: "fixture",
version: "1.0",
origin: "https://docs.example/fixture",
title: "Fixture",
scope: "public",
evidence: "benchmark",
},
],
retrieve: async (candidate) => ({
candidate,
sections: Array.from({ length: workload.sections }, (_, i) => ({
locator: String(i),
heading: "Reference " + i,
body,
})),
attribution: "Benchmark fixture",
validator: "unchanged",
}),
};
const options: Options = {
databasePath: join(root, workload.name + ".db"),
sources: [],
enabledSources: [adapter.id],
offline: true,
cacheBytes: 16 * 1024 * 1024,
ttlMs: 60000,
};
const docs = new imported.Devdocs(options, [adapter]);
try {
const found = await docs.search({
target: "fixture",
query: "reference",
cwd: root,
});
assert.equal(found.status, "ok");
if (found.status !== "ok")
throw new Error("benchmark fixture did not resolve");
const revision = found.results[0]?.ref.split(":")[0];
assert.ok(revision);
const refs = Array.from(
{ length: workload.sections },
(_, i) => revision + ":" + i,
);
let last: ReadResult | undefined;
for (let i = 0; i < 10; i++)
last = await docs.read(refs, workload.budget, root);
const elapsed: number[] = [],
cpu: number[] = [];
for (let i = 0; i < rounds; i++) {
const before = process.cpuUsage(),
start = performance.now();
last = await docs.read(refs, workload.budget, root);
elapsed.push(performance.now() - start);
const used = process.cpuUsage(before);
cpu.push((used.user + used.system) / 1000);
}
assert.ok(last);
assert.ok([...JSON.stringify(last)].length <= workload.budget);
assert.ok(last.sections[0]?.content);
const sorted = [...elapsed].sort((a, b) => a - b);
console.log(
JSON.stringify({
name: workload.name,
node: process.version,
rounds,
warmup: 10,
medianMs: quantile(sorted, 0.5),
p95Ms: quantile(sorted, 0.95),
meanCpuMs: cpu.reduce((a, b) => a + b, 0) / rounds,
maxRssKiB: process.resourceUsage().maxRSS,
elapsedMs: elapsed,
cpuMs: cpu,
}),
);
} finally {
await docs.close();
}
}
} finally {
try {
if (profiler && profile) {
const data = await profiler.post("Profiler.stop");
await writeFile(profile, JSON.stringify(data.profile));
}
} finally {
profiler?.disconnect();
await rm(root, { recursive: true, force: true });
}
}