Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/devdocs/src/adapters/local.ts

Raw
import { execFile } from "node:child_process";
import { readdir, readFile, realpath, stat } from "node:fs/promises";
import { dirname, isAbsolute, join, relative, sep } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { readContainer } from "../containers.ts";
import { docsetPaths } from "../docset.ts";
import { readFormat } from "../formats.ts";
import { cleanText, digest, MAX_DOCUMENT_BYTES } from "../normalize.ts";
import { symbolIndex } from "../symbols.ts";
import {
	type Adapter,
	matchesVersion,
	type RegisteredSource,
} from "../types.ts";
import { upstreamAdapter } from "./upstream.ts";

export async function command(
	file: string,
	args: string[],
	cwd: string,
	signal?: AbortSignal,
): Promise<string> {
	return new Promise((resolve, reject) => {
		execFile(
			file,
			args,
			{
				cwd,
				signal,
				timeout: 5000,
				maxBuffer: MAX_DOCUMENT_BYTES,
				encoding: "utf8",
				env: {
					...process.env,
					MANPAGER: "cat",
					PAGER: "cat",
					GIT_PAGER: "cat",
					NO_COLOR: "1",
				},
			},
			(err, stdout) =>
				err
					? reject(new Error("local documentation command unavailable"))
					: resolve(cleanText(stdout)),
		);
	});
}
async function executable(name: string): Promise<string | null> {
	for (const root of (process.env["PATH"] ?? "")
		.split(process.platform === "win32" ? ";" : ":")
		.slice(0, 100)) {
		if (!isAbsolute(root)) continue;
		for (const suffix of process.platform === "win32"
			? [".exe", ".cmd", ""]
			: [""]) {
			try {
				const p = await realpath(join(root, name + suffix));
				if ((await stat(p)).isFile()) return p;
			} catch {}
		}
	}
	return null;
}
export function installedVersion(output: string): string | null {
	return (
		/(?:^|[^a-zA-Z0-9])v?(\d+\.\d+(?:\.\d+)?(?:[-+][\w.-]+)?)/.exec(
			output,
		)?.[1] ?? null
	);
}
export function localAdapter(): Adapter {
	return {
		id: "local",
		rank: 1,
		formatVersion: 1,
		coverage: "document",
		async discover(r) {
			const manual =
				/^man:(?:([1-9][a-z]*):)?([a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99})$/.exec(
					r.target,
				);
			const infoTarget = /^info:([a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99})$/.exec(
				r.target,
			);
			if (manual || infoTarget) {
				if (r.version) return [];
				const name = manual?.[2] ?? infoTarget?.[1] ?? "";
				const found = manual
					? await command(
							"man",
							["-w", ...(manual[1] ? [manual[1]] : []), name],
							r.cwd,
							r.signal,
						)
					: await command("info", ["--where", name], r.cwd, r.signal);
				const path = await realpath(found.trim().split("\n")[0] ?? "");
				return [
					{
						id: digest(path),
						target: r.target,
						title: name,
						version: null,
						source: "local",
						origin: pathToFileURL(path).href,
						scope: manual ? "man" : "info",
						evidence: null,
					},
				];
			}
			if (
				!/^(?:@[a-zA-Z0-9_.-]+\/)?[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99}$/.test(
					r.target,
				)
			)
				return [];
			const path = r.target.includes("/") ? null : await executable(r.target);
			if (!path) {
				let current = r.cwd;
				for (let depth = 0; depth < 20; depth++) {
					try {
						const root = join(current, "node_modules", r.target);
						const metadata = join(root, "package.json");
						if ((await stat(metadata)).size > 131072)
							throw new Error("package metadata exceeds bounds");
						const data: unknown = JSON.parse(await readFile(metadata, "utf8"));
						const version =
							data &&
							typeof data === "object" &&
							"version" in data &&
							typeof data.version === "string"
								? data.version
								: null;
						if (!matchesVersion(r.version, version)) return [];
						for (const name of [
							"README.md",
							"readme.md",
							"README",
							"Readme.md",
						]) {
							try {
								const doc = await realpath(join(root, name));
								if (!(await stat(doc)).isFile()) continue;
								return [
									{
										id: digest(doc),
										target: r.target,
										title: r.target,
										version,
										source: "local",
										origin: pathToFileURL(doc).href,
										scope: "package",
										evidence: version ? "installed package metadata" : null,
									},
								];
							} catch {}
						}
					} catch {}
					const parent = dirname(current);
					if (parent === current) break;
					current = parent;
				}
				return [];
			}
			let version: string | null = null;
			try {
				version = installedVersion(
					await command(path, ["--version"], r.cwd, r.signal),
				);
			} catch {}
			if (!matchesVersion(r.version, version)) return [];
			return [
				{
					id: digest(path),
					target: r.target,
					title: r.target,
					version,
					source: "local",
					origin: pathToFileURL(path).href,
					scope: "installed",
					evidence: version ? "installed --version" : null,
				},
			];
		},
		async retrieve(c, r) {
			const path = fileURLToPath(c.origin);
			if (c.scope === "package") {
				if ((await stat(path)).size > MAX_DOCUMENT_BYTES)
					throw new Error("package documentation exceeds limit");
				const text = await readFile(path, "utf8");
				return {
					candidate: c,
					sections: readFormat(text, "markdown"),
					attribution: "Installed package documentation",
					validator: digest(text),
				};
			}
			const text =
				c.scope === "man"
					? await command("man", ["-P", "cat", "-l", path], r.cwd, r.signal)
					: c.scope === "info"
						? await command(
								"info",
								["--output=-", "--file", path],
								r.cwd,
								r.signal,
							)
						: await command(path, ["--help"], r.cwd, r.signal);
			if (!text.trim()) throw new Error("no local documentation");
			return {
				candidate: c,
				sections: readFormat("# " + c.title + "\n\n" + text, "markdown"),
				attribution: "Installed documentation",
				validator: null,
			};
		},
	};
}
export function fileAdapter(source: RegisteredSource): Adapter {
	if (source.url) {
		const remote = upstreamAdapter({
			sources: [
				{
					id: source.id,
					aliases: [],
					domain: "registered documentation",
					home: source.url,
					index: /\/llms\.txt$/.test(new URL(source.url).pathname)
						? source.url
						: null,
					format: source.format,
					authority: "publisher",
					role: "reference",
					adapter: "upstream",
					versionSupport:
						"Configured documentation version; unknown when omitted",
					description: source.target,
					...(source.version
						? {
								version: source.version,
								evidence: "configured documentation version",
							}
						: {}),
				},
			],
		});
		return {
			id: source.id,
			rank: 3,
			formatVersion: 1,
			coverage: "document",
			async discover(request) {
				if (
					request.target !== source.target ||
					!matchesVersion(request.version, source.version)
				)
					return [];
				return (await remote.discover({ ...request, target: source.id })).map(
					(candidate) => ({
						...candidate,
						target: source.target,
						source: source.id,
						scope:
							(source.private ? digest(request.cwd) + ":" : "") +
							candidate.scope,
					}),
				);
			},
			async retrieve(candidate, request, validator) {
				const result = await remote.retrieve(
					{ ...candidate, target: source.id, source: "upstream" },
					request,
					validator,
				);
				return result === "unchanged" ? result : { ...result, candidate };
			},
		};
	}
	return {
		id: source.id,
		rank: 1,
		formatVersion: 1,
		coverage: "document",
		async discover(r) {
			if (
				r.target !== source.target ||
				!matchesVersion(r.version, source.version)
			)
				return [];
			const path = await realpath(source.path);
			const info = await stat(path);
			if (
				(!info.isFile() && !info.isDirectory()) ||
				info.size > MAX_DOCUMENT_BYTES
			)
				throw new Error("local documentation file exceeds bounds");
			return [
				{
					id: digest(path),
					target: source.target,
					title: source.target,
					version: source.version,
					source: source.id,
					origin: pathToFileURL(path).href,
					scope:
						(source.private ? digest(r.cwd) : "public") +
						(info.isDirectory() ||
						["javadoc", "sphinx", "docset", "doxygen"].includes(source.format)
							? ":query:" + digest(r.query)
							: ""),
					evidence: source.version ? "configured documentation version" : null,
				},
			];
		},
		async retrieve(c, request, validator) {
			const root = fileURLToPath(c.origin);
			const info = await stat(root);
			const paths: string[] = [];
			if (info.isDirectory() && source.format === "docset") {
				const documents = await realpath(
					join(root, "Contents", "Resources", "Documents"),
				);
				const index = await realpath(
					join(root, "Contents", "Resources", "docSet.dsidx"),
				);
				for (const path of await docsetPaths(
					index,
					request.query,
					request.signal,
				)) {
					const located = path.split("#")[0] ?? "";
					if (
						!located ||
						isAbsolute(located) ||
						located.includes("\\") ||
						located.split("/").some((part) => part === "..")
					)
						throw new Error("unsafe docset index path");
					const candidate = await realpath(join(documents, located));
					const inside = relative(documents, candidate);
					if (
						inside === ".." ||
						inside.startsWith(".." + sep) ||
						isAbsolute(inside)
					)
						throw new Error("docset page escapes document root");
					if (!paths.includes(candidate)) paths.push(candidate);
				}
			} else if (
				info.isDirectory() &&
				["javadoc", "doxygen", "sphinx"].includes(source.format)
			) {
				const indexes =
					source.format === "javadoc"
						? [
								"member-search-index.js",
								"type-search-index.js",
								"package-search-index.js",
								"module-search-index.js",
							]
						: source.format === "sphinx"
							? ["objects.inv"]
							: ["index.xml", "index.tag", "doxygen.tag"];
				for (const name of indexes) {
					try {
						const path = join(root, name);
						if ((await stat(path)).isFile()) paths.push(path);
					} catch {}
				}
				if (!paths.length) {
					const fallback = join(root, "index.html");
					if ((await stat(fallback)).isFile()) paths.push(fallback);
				}
			} else if (info.isDirectory()) {
				const pending = [{ path: root, depth: 0 }];
				let visited = 0;
				while (pending.length) {
					const directory = pending.shift();
					if (!directory) break;
					for (const entry of await readdir(directory.path, {
						withFileTypes: true,
					})) {
						if (++visited > 1000)
							throw new Error("documentation tree exceeds 1000 entries");
						if (entry.isSymbolicLink()) continue;
						const path = join(directory.path, entry.name);
						if (
							entry.isDirectory() &&
							directory.depth < 4 &&
							!["node_modules", ".git"].includes(entry.name)
						)
							pending.push({ path, depth: directory.depth + 1 });
						else if (
							entry.isFile() &&
							/\.(?:md|mdx|rst|txt|html?|json|pdf|jar|zip)$/.test(entry.name)
						)
							paths.push(path);
					}
				}
				const terms =
					request.query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
				const score = (path: string) =>
					terms.reduce(
						(sum, term) => sum + Number(path.toLowerCase().includes(term)),
						0,
					);
				paths.sort((a, b) => score(b) - score(a) || a.localeCompare(b));
				paths.splice(10);
			} else paths.push(root);
			const output: import("../types.ts").Section[] = [];
			const fingerprints: string[] = [];
			let bytes = 0;
			for (const path of paths) {
				request.signal?.throwIfAborted();
				const canonical = await realpath(path);
				const inside = relative(root, canonical);
				if (
					info.isDirectory() &&
					(inside === ".." ||
						inside.startsWith(".." + sep) ||
						isAbsolute(inside))
				)
					throw new Error("documentation file escapes root");
				const size = (await stat(canonical)).size;
				bytes += size;
				if (size > MAX_DOCUMENT_BYTES || bytes > MAX_DOCUMENT_BYTES)
					throw new Error("local documentation exceeds 8 MiB");
				const data = await readFile(canonical);
				if (data.byteLength > MAX_DOCUMENT_BYTES)
					throw new Error("local documentation exceeds 8 MiB");
				fingerprints.push(
					digest(JSON.stringify([canonical, data.toString("base64")])),
				);
				const indexSections =
					source.format === "sphinx" && /objects\.inv$/.test(path)
						? await readContainer(data, "sphinx", request.query, request.signal)
						: source.format === "javadoc" && /-search-index\.js$/.test(path)
							? symbolIndex(data.toString("utf8"), "javadoc", request.query)
							: source.format === "doxygen" && /\.(?:xml|tag)$/.test(path)
								? symbolIndex(data.toString("utf8"), "doxygen", request.query)
								: null;
				if (indexSections) {
					const allowed = info.isDirectory() ? root : dirname(root);
					for (const symbol of indexSections.slice(0, 3)) {
						const located = decodeURIComponent(
							symbol.locator.split("#")[0] ?? "",
						);
						if (
							!located ||
							isAbsolute(located) ||
							located.includes("\\") ||
							/^[a-z][a-z\d+.-]*:/i.test(located)
						)
							throw new Error("unsafe local index link");
						const linked = await realpath(join(dirname(canonical), located));
						const inside = relative(allowed, linked);
						if (
							inside === ".." ||
							inside.startsWith(".." + sep) ||
							isAbsolute(inside)
						)
							throw new Error("local index link escapes documentation root");
						bytes += (await stat(linked)).size;
						if (bytes > MAX_DOCUMENT_BYTES)
							throw new Error("indexed documentation exceeds bounds");
						const body = await readFile(linked, "utf8");
						if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES)
							throw new Error("indexed documentation exceeds bounds");
						fingerprints.push(digest(JSON.stringify([linked, body])));
						const anchor = decodeURIComponent(
							symbol.locator.split("#")[1] ?? "",
						);
						const sections = readFormat(body, "html");
						const exact = sections.filter((s) => s.locator === anchor);
						output.push(
							...(exact.length ? exact : sections).map((s) => ({
								...s,
								locator: symbol.locator + ":" + s.locator,
								body:
									s.body +
									"\n\nSource: " +
									pathToFileURL(linked).href +
									(anchor ? "#" + encodeURIComponent(anchor) : ""),
							})),
						);
					}
					continue;
				}
				const binary =
					source.format === "pdf" ||
					(source.format === "sphinx" && /objects\.inv$/.test(path)) ||
					(source.format === "javadoc" && /\.(?:jar|zip)$/.test(path));
				const parsed = binary
					? await readContainer(
							data,
							source.format === "pdf"
								? "pdf"
								: source.format === "sphinx"
									? "sphinx"
									: "javadoc",
							request.query,
							request.signal,
						)
					: readFormat(
							data.toString("utf8"),
							info.isDirectory()
								? /\.html?$/.test(path)
									? "html"
									: /\.json$/.test(path)
										? "json"
										: /\.rst$/.test(path)
											? "rst"
											: /\.mdx$/.test(path)
												? "mdx"
												: "markdown"
								: source.format,
						);
				output.push(
					...parsed.map((section) => ({
						...section,
						locator: (info.isDirectory() ? inside + "#" : "") + section.locator,
						body:
							section.body +
							(info.isDirectory()
								? "\n\nSource: " + pathToFileURL(canonical).href
								: ""),
					})),
				);
			}
			const fingerprint = digest(JSON.stringify([request.query, fingerprints]));
			if (fingerprint === validator) return "unchanged";
			return {
				candidate: c,
				sections: output,
				attribution: "User-registered documentation",
				validator: fingerprint,
			};
		},
	};
}