import { execFile } from "node:child_process"; import { readdir, readFile, realpath, stat } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, sep } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { readContainer } from "../containers.ts"; import { docsetPaths } from "../docset.ts"; import { readFormat } from "../formats.ts"; import { cleanText, digest, MAX_DOCUMENT_BYTES } from "../normalize.ts"; import { symbolIndex } from "../symbols.ts"; import { type Adapter, matchesVersion, type RegisteredSource, } from "../types.ts"; import { upstreamAdapter } from "./upstream.ts"; export async function command( file: string, args: string[], cwd: string, signal?: AbortSignal, ): Promise { return new Promise((resolve, reject) => { execFile( file, args, { cwd, signal, timeout: 5000, maxBuffer: MAX_DOCUMENT_BYTES, encoding: "utf8", env: { ...process.env, MANPAGER: "cat", PAGER: "cat", GIT_PAGER: "cat", NO_COLOR: "1", }, }, (err, stdout) => err ? reject(new Error("local documentation command unavailable")) : resolve(cleanText(stdout)), ); }); } async function executable(name: string): Promise { for (const root of (process.env["PATH"] ?? "") .split(process.platform === "win32" ? ";" : ":") .slice(0, 100)) { if (!isAbsolute(root)) continue; for (const suffix of process.platform === "win32" ? [".exe", ".cmd", ""] : [""]) { try { const p = await realpath(join(root, name + suffix)); if ((await stat(p)).isFile()) return p; } catch {} } } return null; } export function installedVersion(output: string): string | null { return ( /(?:^|[^a-zA-Z0-9])v?(\d+\.\d+(?:\.\d+)?(?:[-+][\w.-]+)?)/.exec( output, )?.[1] ?? null ); } export function localAdapter(): Adapter { return { id: "local", rank: 1, formatVersion: 1, coverage: "document", async discover(r) { const manual = /^man:(?:([1-9][a-z]*):)?([a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99})$/.exec( r.target, ); const infoTarget = /^info:([a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99})$/.exec( r.target, ); if (manual || infoTarget) { if (r.version) return []; const name = manual?.[2] ?? infoTarget?.[1] ?? ""; const found = manual ? await command( "man", ["-w", ...(manual[1] ? [manual[1]] : []), name], r.cwd, r.signal, ) : await command("info", ["--where", name], r.cwd, r.signal); const path = await realpath(found.trim().split("\n")[0] ?? ""); return [ { id: digest(path), target: r.target, title: name, version: null, source: "local", origin: pathToFileURL(path).href, scope: manual ? "man" : "info", evidence: null, }, ]; } if ( !/^(?:@[a-zA-Z0-9_.-]+\/)?[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99}$/.test( r.target, ) ) return []; const path = r.target.includes("/") ? null : await executable(r.target); if (!path) { let current = r.cwd; for (let depth = 0; depth < 20; depth++) { try { const root = join(current, "node_modules", r.target); const metadata = join(root, "package.json"); if ((await stat(metadata)).size > 131072) throw new Error("package metadata exceeds bounds"); const data: unknown = JSON.parse(await readFile(metadata, "utf8")); const version = data && typeof data === "object" && "version" in data && typeof data.version === "string" ? data.version : null; if (!matchesVersion(r.version, version)) return []; for (const name of [ "README.md", "readme.md", "README", "Readme.md", ]) { try { const doc = await realpath(join(root, name)); if (!(await stat(doc)).isFile()) continue; return [ { id: digest(doc), target: r.target, title: r.target, version, source: "local", origin: pathToFileURL(doc).href, scope: "package", evidence: version ? "installed package metadata" : null, }, ]; } catch {} } } catch {} const parent = dirname(current); if (parent === current) break; current = parent; } return []; } let version: string | null = null; try { version = installedVersion( await command(path, ["--version"], r.cwd, r.signal), ); } catch {} if (!matchesVersion(r.version, version)) return []; return [ { id: digest(path), target: r.target, title: r.target, version, source: "local", origin: pathToFileURL(path).href, scope: "installed", evidence: version ? "installed --version" : null, }, ]; }, async retrieve(c, r) { const path = fileURLToPath(c.origin); if (c.scope === "package") { if ((await stat(path)).size > MAX_DOCUMENT_BYTES) throw new Error("package documentation exceeds limit"); const text = await readFile(path, "utf8"); return { candidate: c, sections: readFormat(text, "markdown"), attribution: "Installed package documentation", validator: digest(text), }; } const text = c.scope === "man" ? await command("man", ["-P", "cat", "-l", path], r.cwd, r.signal) : c.scope === "info" ? await command( "info", ["--output=-", "--file", path], r.cwd, r.signal, ) : await command(path, ["--help"], r.cwd, r.signal); if (!text.trim()) throw new Error("no local documentation"); return { candidate: c, sections: readFormat("# " + c.title + "\n\n" + text, "markdown"), attribution: "Installed documentation", validator: null, }; }, }; } export function fileAdapter(source: RegisteredSource): Adapter { if (source.url) { const remote = upstreamAdapter({ sources: [ { id: source.id, aliases: [], domain: "registered documentation", home: source.url, index: /\/llms\.txt$/.test(new URL(source.url).pathname) ? source.url : null, format: source.format, authority: "publisher", role: "reference", adapter: "upstream", versionSupport: "Configured documentation version; unknown when omitted", description: source.target, ...(source.version ? { version: source.version, evidence: "configured documentation version", } : {}), }, ], }); return { id: source.id, rank: 3, formatVersion: 1, coverage: "document", async discover(request) { if ( request.target !== source.target || !matchesVersion(request.version, source.version) ) return []; return (await remote.discover({ ...request, target: source.id })).map( (candidate) => ({ ...candidate, target: source.target, source: source.id, scope: (source.private ? digest(request.cwd) + ":" : "") + candidate.scope, }), ); }, async retrieve(candidate, request, validator) { const result = await remote.retrieve( { ...candidate, target: source.id, source: "upstream" }, request, validator, ); return result === "unchanged" ? result : { ...result, candidate }; }, }; } return { id: source.id, rank: 1, formatVersion: 1, coverage: "document", async discover(r) { if ( r.target !== source.target || !matchesVersion(r.version, source.version) ) return []; const path = await realpath(source.path); const info = await stat(path); if ( (!info.isFile() && !info.isDirectory()) || info.size > MAX_DOCUMENT_BYTES ) throw new Error("local documentation file exceeds bounds"); return [ { id: digest(path), target: source.target, title: source.target, version: source.version, source: source.id, origin: pathToFileURL(path).href, scope: (source.private ? digest(r.cwd) : "public") + (info.isDirectory() || ["javadoc", "sphinx", "docset", "doxygen"].includes(source.format) ? ":query:" + digest(r.query) : ""), evidence: source.version ? "configured documentation version" : null, }, ]; }, async retrieve(c, request, validator) { const root = fileURLToPath(c.origin); const info = await stat(root); const paths: string[] = []; if (info.isDirectory() && source.format === "docset") { const documents = await realpath( join(root, "Contents", "Resources", "Documents"), ); const index = await realpath( join(root, "Contents", "Resources", "docSet.dsidx"), ); for (const path of await docsetPaths( index, request.query, request.signal, )) { const located = path.split("#")[0] ?? ""; if ( !located || isAbsolute(located) || located.includes("\\") || located.split("/").some((part) => part === "..") ) throw new Error("unsafe docset index path"); const candidate = await realpath(join(documents, located)); const inside = relative(documents, candidate); if ( inside === ".." || inside.startsWith(".." + sep) || isAbsolute(inside) ) throw new Error("docset page escapes document root"); if (!paths.includes(candidate)) paths.push(candidate); } } else if ( info.isDirectory() && ["javadoc", "doxygen", "sphinx"].includes(source.format) ) { const indexes = source.format === "javadoc" ? [ "member-search-index.js", "type-search-index.js", "package-search-index.js", "module-search-index.js", ] : source.format === "sphinx" ? ["objects.inv"] : ["index.xml", "index.tag", "doxygen.tag"]; for (const name of indexes) { try { const path = join(root, name); if ((await stat(path)).isFile()) paths.push(path); } catch {} } if (!paths.length) { const fallback = join(root, "index.html"); if ((await stat(fallback)).isFile()) paths.push(fallback); } } else if (info.isDirectory()) { const pending = [{ path: root, depth: 0 }]; let visited = 0; while (pending.length) { const directory = pending.shift(); if (!directory) break; for (const entry of await readdir(directory.path, { withFileTypes: true, })) { if (++visited > 1000) throw new Error("documentation tree exceeds 1000 entries"); if (entry.isSymbolicLink()) continue; const path = join(directory.path, entry.name); if ( entry.isDirectory() && directory.depth < 4 && !["node_modules", ".git"].includes(entry.name) ) pending.push({ path, depth: directory.depth + 1 }); else if ( entry.isFile() && /\.(?:md|mdx|rst|txt|html?|json|pdf|jar|zip)$/.test(entry.name) ) paths.push(path); } } const terms = request.query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []; const score = (path: string) => terms.reduce( (sum, term) => sum + Number(path.toLowerCase().includes(term)), 0, ); paths.sort((a, b) => score(b) - score(a) || a.localeCompare(b)); paths.splice(10); } else paths.push(root); const output: import("../types.ts").Section[] = []; const fingerprints: string[] = []; let bytes = 0; for (const path of paths) { request.signal?.throwIfAborted(); const canonical = await realpath(path); const inside = relative(root, canonical); if ( info.isDirectory() && (inside === ".." || inside.startsWith(".." + sep) || isAbsolute(inside)) ) throw new Error("documentation file escapes root"); const size = (await stat(canonical)).size; bytes += size; if (size > MAX_DOCUMENT_BYTES || bytes > MAX_DOCUMENT_BYTES) throw new Error("local documentation exceeds 8 MiB"); const data = await readFile(canonical); if (data.byteLength > MAX_DOCUMENT_BYTES) throw new Error("local documentation exceeds 8 MiB"); fingerprints.push( digest(JSON.stringify([canonical, data.toString("base64")])), ); const indexSections = source.format === "sphinx" && /objects\.inv$/.test(path) ? await readContainer(data, "sphinx", request.query, request.signal) : source.format === "javadoc" && /-search-index\.js$/.test(path) ? symbolIndex(data.toString("utf8"), "javadoc", request.query) : source.format === "doxygen" && /\.(?:xml|tag)$/.test(path) ? symbolIndex(data.toString("utf8"), "doxygen", request.query) : null; if (indexSections) { const allowed = info.isDirectory() ? root : dirname(root); for (const symbol of indexSections.slice(0, 3)) { const located = decodeURIComponent( symbol.locator.split("#")[0] ?? "", ); if ( !located || isAbsolute(located) || located.includes("\\") || /^[a-z][a-z\d+.-]*:/i.test(located) ) throw new Error("unsafe local index link"); const linked = await realpath(join(dirname(canonical), located)); const inside = relative(allowed, linked); if ( inside === ".." || inside.startsWith(".." + sep) || isAbsolute(inside) ) throw new Error("local index link escapes documentation root"); bytes += (await stat(linked)).size; if (bytes > MAX_DOCUMENT_BYTES) throw new Error("indexed documentation exceeds bounds"); const body = await readFile(linked, "utf8"); if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES) throw new Error("indexed documentation exceeds bounds"); fingerprints.push(digest(JSON.stringify([linked, body]))); const anchor = decodeURIComponent( symbol.locator.split("#")[1] ?? "", ); const sections = readFormat(body, "html"); const exact = sections.filter((s) => s.locator === anchor); output.push( ...(exact.length ? exact : sections).map((s) => ({ ...s, locator: symbol.locator + ":" + s.locator, body: s.body + "\n\nSource: " + pathToFileURL(linked).href + (anchor ? "#" + encodeURIComponent(anchor) : ""), })), ); } continue; } const binary = source.format === "pdf" || (source.format === "sphinx" && /objects\.inv$/.test(path)) || (source.format === "javadoc" && /\.(?:jar|zip)$/.test(path)); const parsed = binary ? await readContainer( data, source.format === "pdf" ? "pdf" : source.format === "sphinx" ? "sphinx" : "javadoc", request.query, request.signal, ) : readFormat( data.toString("utf8"), info.isDirectory() ? /\.html?$/.test(path) ? "html" : /\.json$/.test(path) ? "json" : /\.rst$/.test(path) ? "rst" : /\.mdx$/.test(path) ? "mdx" : "markdown" : source.format, ); output.push( ...parsed.map((section) => ({ ...section, locator: (info.isDirectory() ? inside + "#" : "") + section.locator, body: section.body + (info.isDirectory() ? "\n\nSource: " + pathToFileURL(canonical).href : ""), })), ); } const fingerprint = digest(JSON.stringify([request.query, fingerprints])); if (fingerprint === validator) return "unchanged"; return { candidate: c, sections: output, attribution: "User-registered documentation", validator: fingerprint, }; }, }; }