import assert from "node:assert/strict"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { Session } from "node:inspector/promises"; import { tmpdir } from "node:os"; import { isAbsolute, join } from "node:path"; import { performance } from "node:perf_hooks"; import { pathToFileURL } from "node:url"; import type { Devdocs as Engine } from "../src/engine.ts"; import type { Adapter, Options, ReadResult } from "../src/types.ts"; // Fixed real-cache workloads: 10 warmups, 30 measured reads by default. // DEVDOCS_BENCH_MODULE selects an engine snapshot for separate baseline runs. // DEVDOCS_BENCH_PROFILE writes a native CPU profile; do not mix profiled timings // with unprofiled comparisons. No network, personal settings, or retained databases. const imported: { Devdocs: typeof Engine } = await import( process.env["DEVDOCS_BENCH_MODULE"] ? pathToFileURL(process.env["DEVDOCS_BENCH_MODULE"]).href : "../src/engine.ts" ); const rounds = Number(process.env["DEVDOCS_BENCH_ROUNDS"] ?? 30); if (!Number.isInteger(rounds) || rounds < 5 || rounds > 200) throw new Error("benchmark rounds must be 5..200"); const profile = process.env["DEVDOCS_BENCH_PROFILE"]; if (profile && !isAbsolute(profile)) throw new Error("profile path must be absolute"); const profiler = profile ? new Session() : undefined; const workloads = [ { name: "single", sections: 1, pattern: "ordinary reference ", size: 256, budget: 4000, }, { name: "multi-ascii", sections: 20, pattern: "ordinary reference ", size: 2000, budget: 64000, }, { name: "multi-unicode", sections: 20, pattern: 'α😀e\u0301漢\n"\\\t', size: 2000, budget: 64000, }, { name: "truncated", sections: 20, pattern: 'α😀e\u0301漢\n"\\\t', size: 10000, budget: 4000, }, ]; const quantile = (values: number[], p: number) => values[Math.min(values.length - 1, Math.floor(values.length * p))] ?? 0; const root = await mkdtemp(join(tmpdir(), "devdocs-read-bench-")); try { if (profiler) { profiler.connect(); await profiler.post("Profiler.enable"); await profiler.post("Profiler.start"); } for (const workload of workloads) { const body = workload.pattern .repeat(Math.ceil(workload.size / workload.pattern.length)) .slice(0, workload.size); const adapter: Adapter = { id: "benchmark", rank: 1, formatVersion: 1, coverage: "corpus", discover: async () => [ { id: "fixture", source: "benchmark", target: "fixture", version: "1.0", origin: "https://docs.example/fixture", title: "Fixture", scope: "public", evidence: "benchmark", }, ], retrieve: async (candidate) => ({ candidate, sections: Array.from({ length: workload.sections }, (_, i) => ({ locator: String(i), heading: "Reference " + i, body, })), attribution: "Benchmark fixture", validator: "unchanged", }), }; const options: Options = { databasePath: join(root, workload.name + ".db"), sources: [], enabledSources: [adapter.id], offline: true, cacheBytes: 16 * 1024 * 1024, ttlMs: 60000, }; const docs = new imported.Devdocs(options, [adapter]); try { const found = await docs.search({ target: "fixture", query: "reference", cwd: root, }); assert.equal(found.status, "ok"); if (found.status !== "ok") throw new Error("benchmark fixture did not resolve"); const revision = found.results[0]?.ref.split(":")[0]; assert.ok(revision); const refs = Array.from( { length: workload.sections }, (_, i) => revision + ":" + i, ); let last: ReadResult | undefined; for (let i = 0; i < 10; i++) last = await docs.read(refs, workload.budget, root); const elapsed: number[] = [], cpu: number[] = []; for (let i = 0; i < rounds; i++) { const before = process.cpuUsage(), start = performance.now(); last = await docs.read(refs, workload.budget, root); elapsed.push(performance.now() - start); const used = process.cpuUsage(before); cpu.push((used.user + used.system) / 1000); } assert.ok(last); assert.ok([...JSON.stringify(last)].length <= workload.budget); assert.ok(last.sections[0]?.content); const sorted = [...elapsed].sort((a, b) => a - b); console.log( JSON.stringify({ name: workload.name, node: process.version, rounds, warmup: 10, medianMs: quantile(sorted, 0.5), p95Ms: quantile(sorted, 0.95), meanCpuMs: cpu.reduce((a, b) => a + b, 0) / rounds, maxRssKiB: process.resourceUsage().maxRSS, elapsedMs: elapsed, cpuMs: cpu, }), ); } finally { await docs.close(); } } } finally { try { if (profiler && profile) { const data = await profiler.post("Profiler.stop"); await writeFile(profile, JSON.stringify(data.profile)); } } finally { profiler?.disconnect(); await rm(root, { recursive: true, force: true }); } }