Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/devdocs/src/adapters/upstream.ts

Raw
import {
	type CatalogSource,
	type FormatId,
	sourceCatalog,
} from "../catalog.ts";
import { readContainer } from "../containers.ts";
import { readFormat } from "../formats.ts";
import { digest, MAX_DOCUMENT_BYTES } from "../normalize.ts";
import { symbolIndex } from "../symbols.ts";
import type { Adapter, Candidate, Document, Request } from "../types.ts";
import { matchesVersion } from "../types.ts";
import { httpError, ProviderError } from "./network.ts";

export type UpstreamFetch = (
	url: string,
	init: RequestInit,
) => Promise<Response>;
export type UpstreamOptions = {
	fetch?: UpstreamFetch;
	/** Additional publisher roots; duplicate IDs replace built-in catalog entries. */
	sources?: readonly CatalogSource[];
	maxBytes?: number;
	timeoutMs?: number;
};
type Page = {
	url: string;
	text: string;
	data: Uint8Array;
	type: string;
	validator: string | null;
	unchanged: boolean;
};

/** Documentation may be self-hosted. Query strings and userinfo can carry credentials. */
function publicUrl(value: string, base?: string): URL {
	let url: URL;
	try {
		url = new URL(value, base);
	} catch {
		throw new Error("invalid documentation URL");
	}
	if (
		url.protocol !== "https:" ||
		url.username ||
		url.password ||
		url.search ||
		url.href.length > 4096
	)
		throw new Error(
			"documentation URL must be HTTPS without credentials or query parameters",
		);
	return url;
}
function finiteLimit(value: number, maximum: number, name: string): number {
	if (!Number.isInteger(value) || value < 1 || value > maximum)
		throw new Error(`invalid ${name} limit`);
	return value;
}
function inferredFormat(
	url: string,
	contentType: string,
	fallback: FormatId,
): FormatId {
	const path = new URL(url).pathname.toLowerCase();
	if (/-search-index\.js$/.test(path)) return "javadoc";
	if (
		path.endsWith(".tag") ||
		(path.endsWith(".xml") && fallback === "doxygen")
	)
		return "doxygen";
	if (path.endsWith(".mdx")) return "mdx";
	if (path.endsWith(".rst")) return "rst";
	if (path.endsWith(".pdf") || contentType.includes("application/pdf"))
		return "pdf";
	if (path.endsWith(".json") || contentType.includes("application/json"))
		return "json";
	if (/\.(?:jar|zip)$/.test(path)) return "javadoc";
	if (path.endsWith("objects.inv")) return "sphinx";
	if (/\.(?:tgz|gz)$/.test(path)) throw new Error("archive reader unavailable");
	if (/\.(?:md|markdown)$/.test(path) || contentType.includes("text/markdown"))
		return "markdown";
	if (
		contentType.includes("text/html") ||
		contentType.includes("application/xhtml")
	)
		return "html";
	if (path.endsWith(".txt") || contentType.includes("text/plain"))
		return "text";
	if (contentType && !contentType.startsWith("text/"))
		throw new Error("unsupported documentation media type");
	return fallback;
}
function queryScoped(url: string): boolean {
	return /\.(?:jar|zip|tag|xml)$|objects\.inv$|-search-index\.js$/.test(
		new URL(url).pathname,
	);
}
function words(value: string): string[] {
	return [
		...new Set(value.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? []),
	].slice(0, 64);
}

export function upstreamAdapter(options: UpstreamOptions = {}): Adapter {
	const fetchPage = options.fetch ?? ((url, init) => fetch(url, init));
	const maxBytes = finiteLimit(
		options.maxBytes ?? 2 * 1024 * 1024,
		MAX_DOCUMENT_BYTES,
		"response bytes",
	);
	const timeoutMs = finiteLimit(options.timeoutMs ?? 10000, 60000, "deadline");
	const catalog = new Map(sourceCatalog.map((entry) => [entry.id, entry]));
	for (const entry of options.sources ?? []) catalog.set(entry.id, entry);

	async function load(
		origin: string,
		request: Request,
		validator?: string | null,
	): Promise<Page> {
		const initial = publicUrl(origin);
		initial.hash = "";
		const controller = new AbortController();
		const signal = request.signal
			? AbortSignal.any([request.signal, controller.signal])
			: controller.signal;
		const timer = setTimeout(() => controller.abort(), timeoutMs);
		let rejectAbort: (() => void) | undefined;
		const aborted = new Promise<never>((_, reject) => {
			rejectAbort = () =>
				reject(
					new Error("documentation request cancelled or deadline exceeded"),
				);
			signal.addEventListener("abort", rejectAbort, { once: true });
		});
		const work = async (): Promise<Page> => {
			let url = initial;
			for (let redirects = 0; redirects <= 3; redirects++) {
				signal.throwIfAborted();
				const headers: Record<string, string> = {
					accept: "text/markdown, text/html, text/plain;q=0.9",
				};
				if (validator)
					headers[
						validator.startsWith('"') || validator.startsWith("W/")
							? "if-none-match"
							: "if-modified-since"
					] = validator;
				const response = await fetchPage(url.href, {
					signal,
					redirect: "manual",
					credentials: "omit",
					referrerPolicy: "no-referrer",
					headers,
				});
				if (signal.aborted) {
					void response.body?.cancel().catch(() => {});
					signal.throwIfAborted();
				}
				if ([301, 302, 303, 307, 308].includes(response.status)) {
					void response.body?.cancel().catch(() => {});
					const location = response.headers.get("location");
					if (!location || redirects === 3)
						throw new Error("documentation redirect limit or missing location");
					const next = publicUrl(location, url.href);
					if (next.origin !== initial.origin)
						throw new Error("cross-origin documentation redirect refused");
					next.hash = "";
					url = next;
					continue;
				}
				if (response.status === 304) {
					void response.body?.cancel().catch(() => {});
					if (!validator)
						throw new Error("unexpected documentation validation response");
					return {
						url: url.href,
						text: "",
						data: new Uint8Array(),
						type: "",
						validator,
						unchanged: true,
					};
				}
				if (!response.ok) {
					void response.body?.cancel().catch(() => {});
					throw httpError(response);
				}
				const length = Number(response.headers.get("content-length"));
				if (Number.isFinite(length) && length > maxBytes) {
					void response.body?.cancel().catch(() => {});
					throw new Error("documentation response exceeds byte limit");
				}
				if (!response.body) throw new Error("empty documentation response");
				const reader = response.body.getReader();
				const cancel = () => {
					void reader.cancel().catch(() => {});
				};
				signal.addEventListener("abort", cancel, { once: true });
				const chunks: Uint8Array[] = [];
				let size = 0;
				try {
					while (true) {
						const { done, value } = await reader.read();
						signal.throwIfAborted();
						if (done) break;
						size += value.byteLength;
						if (size > maxBytes)
							throw new Error("documentation response exceeds byte limit");
						chunks.push(value);
					}
				} finally {
					signal.removeEventListener("abort", cancel);
					void reader.cancel().catch(() => {});
					reader.releaseLock();
				}
				return {
					url: url.href,
					text: Buffer.concat(chunks, size).toString("utf8"),
					data: Buffer.concat(chunks, size),
					type: (response.headers.get("content-type") ?? "").toLowerCase(),
					validator:
						response.headers.get("etag") ??
						response.headers.get("last-modified"),
					unchanged: false,
				};
			}
			throw new Error("documentation redirect limit");
		};
		try {
			return await Promise.race([work(), aborted]);
		} catch (error) {
			if (error instanceof ProviderError) throw error;
			throw new Error(
				"upstream documentation unavailable (HTTP, safety, size or deadline)",
			);
		} finally {
			clearTimeout(timer);
			if (rejectAbort) signal.removeEventListener("abort", rejectAbort);
		}
	}

	function candidate(
		source: CatalogSource | undefined,
		target: string,
		url: string,
		scope = "public",
	): Candidate {
		const origin = publicUrl(url).href;
		return {
			id: digest(JSON.stringify([source?.id ?? "url", origin])),
			target,
			version: source?.version && source.evidence ? source.version : null,
			origin,
			title: source?.description ?? target,
			source: "upstream",
			scope,
			evidence: source?.version && source.evidence ? source.evidence : null,
		};
	}
	async function machineIndex(
		source: CatalogSource | undefined,
		target: string,
		index: string,
		request: Request,
	): Promise<Candidate[]> {
		const page = await load(index, request);
		const terms = words(request.query);
		const links: { url: string; score: number; ordinal: number }[] = [];
		const seen = new Set<string>();
		let count = 0;
		for (const match of page.text.matchAll(
			/\[([^\]\n]{1,512})\]\(<?([^\s)>]{1,4096})>?(?:\s+"[^"\n]*")?\)/g,
		)) {
			if (++count > 256) break;
			let url: URL;
			try {
				url = publicUrl(match[2] ?? "", page.url);
			} catch {
				continue;
			}
			// A machine index does not authorize traversal to another publisher.
			if (
				url.origin !== new URL(page.url).origin ||
				/\/llms(?:-full)?\.txt$/.test(url.pathname) ||
				seen.has(url.href)
			)
				continue;
			seen.add(url.href);
			const label = (match[1] ?? "").toLowerCase();
			let path: string;
			try {
				path = decodeURI(url.pathname).toLowerCase();
			} catch {
				continue;
			}
			const score = terms.reduce(
				(sum, term) =>
					sum + (label.includes(term) ? 4 : 0) + (path.includes(term) ? 2 : 0),
				0,
			);
			if (score) links.push({ url: url.href, score, ordinal: count });
		}
		links.sort((a, b) => b.score - a.score || a.ordinal - b.ordinal);
		const selected = links[0];
		return selected ? [candidate(source, target, selected.url)] : [];
	}
	return {
		id: "upstream",
		rank: 3,
		formatVersion: 1,
		coverage: "document",
		async discover(request) {
			request.signal?.throwIfAborted();
			const maven =
				/^(?:maven|javadoc):([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.+-]+)$/.exec(
					request.target,
				);
			if (maven) {
				const [, group = "", artifact = "", version = ""] = maven;
				if (!matchesVersion(request.version, version)) return [];
				const url = `https://repo.maven.apache.org/maven2/${group.replaceAll(".", "/")}/${artifact}/${version}/${artifact}-${version}-javadoc.jar`;
				return [
					{
						...candidate(
							undefined,
							request.target,
							url,
							"query:" + digest(request.query),
						),
						version,
						evidence: "Maven repository artifact coordinates",
					},
				];
			}
			if (/^https?:\/\//i.test(request.target)) {
				const url = publicUrl(request.target);
				return /\/llms\.txt$/.test(url.pathname)
					? machineIndex(undefined, url.href, url.href, request)
					: [
							candidate(
								undefined,
								url.href,
								url.href,
								queryScoped(url.href)
									? "query:" + digest(request.query)
									: "public",
							),
						];
			}
			const target = request.target.toLowerCase().trim();
			const matches = [...catalog.values()].filter(
				(entry) =>
					entry.adapter === "upstream" &&
					[entry.id, ...entry.aliases].some(
						(alias) => target === alias.toLowerCase(),
					),
			);
			if (matches.length !== 1)
				return matches.map((entry) =>
					candidate(entry, entry.id, entry.index ?? entry.home),
				);
			const source = matches[0];
			if (!source) return [];
			if (source.index)
				return machineIndex(source, source.id, source.index, request);
			return [
				candidate(
					source,
					source.id,
					source.home,
					queryScoped(source.home)
						? "query:" + digest(request.query)
						: "public",
				),
			];
		},
		async retrieve(
			selected,
			request,
			validator,
		): Promise<Document | "unchanged"> {
			const inventory = new URL(selected.origin).pathname.endsWith(
				"objects.inv",
			);
			const symbolFormat = /-search-index\.js$/.test(selected.origin)
				? "javadoc"
				: /\.(?:tag|xml)$/.test(selected.origin)
					? "doxygen"
					: null;
			const indexed = inventory || symbolFormat !== null;
			const page = await load(
				selected.origin,
				request,
				indexed ? null : validator,
			);
			if (page.unchanged) return "unchanged";
			const source = catalog.get(selected.target);
			const format = inferredFormat(
				page.url,
				page.type,
				source?.format ?? "html",
			);
			let content = symbolFormat
				? symbolIndex(page.text, symbolFormat, request.query)
				: format === "pdf" ||
						(format === "javadoc" &&
							/\.(?:jar|zip)$/.test(new URL(page.url).pathname)) ||
						inventory
					? await readContainer(
							page.data,
							inventory ? "sphinx" : format === "pdf" ? "pdf" : "javadoc",
							request.query,
							request.signal,
						)
					: readFormat(page.text, format);
			if (indexed && content[0]) {
				const linked = publicUrl(content[0].locator, page.url);
				if (linked.origin !== new URL(page.url).origin)
					throw new Error("cross-origin inventory link refused");
				const body = await load(linked.href, request);
				content = readFormat(
					body.text,
					inferredFormat(body.url, body.type, "html"),
				).map((section) => ({
					...section,
					body: section.body + "\n\nSource: " + linked.href,
				}));
			}
			if (!content.length)
				throw new Error("upstream document has no extractable sections");
			return {
				candidate: selected,
				sections: content,
				attribution: `${source?.authority ?? "publisher authority unverified"}: ${source?.id ?? new URL(selected.origin).hostname}; ${selected.origin}`,
				validator: indexed ? null : page.validator,
			};
		},
	};
}