import { type CatalogSource, type FormatId, sourceCatalog, } from "../catalog.ts"; import { readContainer } from "../containers.ts"; import { readFormat } from "../formats.ts"; import { digest, MAX_DOCUMENT_BYTES } from "../normalize.ts"; import { symbolIndex } from "../symbols.ts"; import type { Adapter, Candidate, Document, Request } from "../types.ts"; import { matchesVersion } from "../types.ts"; import { httpError, ProviderError } from "./network.ts"; export type UpstreamFetch = ( url: string, init: RequestInit, ) => Promise; export type UpstreamOptions = { fetch?: UpstreamFetch; /** Additional publisher roots; duplicate IDs replace built-in catalog entries. */ sources?: readonly CatalogSource[]; maxBytes?: number; timeoutMs?: number; }; type Page = { url: string; text: string; data: Uint8Array; type: string; validator: string | null; unchanged: boolean; }; /** Documentation may be self-hosted. Query strings and userinfo can carry credentials. */ function publicUrl(value: string, base?: string): URL { let url: URL; try { url = new URL(value, base); } catch { throw new Error("invalid documentation URL"); } if ( url.protocol !== "https:" || url.username || url.password || url.search || url.href.length > 4096 ) throw new Error( "documentation URL must be HTTPS without credentials or query parameters", ); return url; } function finiteLimit(value: number, maximum: number, name: string): number { if (!Number.isInteger(value) || value < 1 || value > maximum) throw new Error(`invalid ${name} limit`); return value; } function inferredFormat( url: string, contentType: string, fallback: FormatId, ): FormatId { const path = new URL(url).pathname.toLowerCase(); if (/-search-index\.js$/.test(path)) return "javadoc"; if ( path.endsWith(".tag") || (path.endsWith(".xml") && fallback === "doxygen") ) return "doxygen"; if (path.endsWith(".mdx")) return "mdx"; if (path.endsWith(".rst")) return "rst"; if (path.endsWith(".pdf") || contentType.includes("application/pdf")) return "pdf"; if (path.endsWith(".json") || contentType.includes("application/json")) return "json"; if (/\.(?:jar|zip)$/.test(path)) return "javadoc"; if (path.endsWith("objects.inv")) return "sphinx"; if (/\.(?:tgz|gz)$/.test(path)) throw new Error("archive reader unavailable"); if (/\.(?:md|markdown)$/.test(path) || contentType.includes("text/markdown")) return "markdown"; if ( contentType.includes("text/html") || contentType.includes("application/xhtml") ) return "html"; if (path.endsWith(".txt") || contentType.includes("text/plain")) return "text"; if (contentType && !contentType.startsWith("text/")) throw new Error("unsupported documentation media type"); return fallback; } function queryScoped(url: string): boolean { return /\.(?:jar|zip|tag|xml)$|objects\.inv$|-search-index\.js$/.test( new URL(url).pathname, ); } function words(value: string): string[] { return [ ...new Set(value.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? []), ].slice(0, 64); } export function upstreamAdapter(options: UpstreamOptions = {}): Adapter { const fetchPage = options.fetch ?? ((url, init) => fetch(url, init)); const maxBytes = finiteLimit( options.maxBytes ?? 2 * 1024 * 1024, MAX_DOCUMENT_BYTES, "response bytes", ); const timeoutMs = finiteLimit(options.timeoutMs ?? 10000, 60000, "deadline"); const catalog = new Map(sourceCatalog.map((entry) => [entry.id, entry])); for (const entry of options.sources ?? []) catalog.set(entry.id, entry); async function load( origin: string, request: Request, validator?: string | null, ): Promise { const initial = publicUrl(origin); initial.hash = ""; const controller = new AbortController(); const signal = request.signal ? AbortSignal.any([request.signal, controller.signal]) : controller.signal; const timer = setTimeout(() => controller.abort(), timeoutMs); let rejectAbort: (() => void) | undefined; const aborted = new Promise((_, reject) => { rejectAbort = () => reject( new Error("documentation request cancelled or deadline exceeded"), ); signal.addEventListener("abort", rejectAbort, { once: true }); }); const work = async (): Promise => { let url = initial; for (let redirects = 0; redirects <= 3; redirects++) { signal.throwIfAborted(); const headers: Record = { accept: "text/markdown, text/html, text/plain;q=0.9", }; if (validator) headers[ validator.startsWith('"') || validator.startsWith("W/") ? "if-none-match" : "if-modified-since" ] = validator; const response = await fetchPage(url.href, { signal, redirect: "manual", credentials: "omit", referrerPolicy: "no-referrer", headers, }); if (signal.aborted) { void response.body?.cancel().catch(() => {}); signal.throwIfAborted(); } if ([301, 302, 303, 307, 308].includes(response.status)) { void response.body?.cancel().catch(() => {}); const location = response.headers.get("location"); if (!location || redirects === 3) throw new Error("documentation redirect limit or missing location"); const next = publicUrl(location, url.href); if (next.origin !== initial.origin) throw new Error("cross-origin documentation redirect refused"); next.hash = ""; url = next; continue; } if (response.status === 304) { void response.body?.cancel().catch(() => {}); if (!validator) throw new Error("unexpected documentation validation response"); return { url: url.href, text: "", data: new Uint8Array(), type: "", validator, unchanged: true, }; } if (!response.ok) { void response.body?.cancel().catch(() => {}); throw httpError(response); } const length = Number(response.headers.get("content-length")); if (Number.isFinite(length) && length > maxBytes) { void response.body?.cancel().catch(() => {}); throw new Error("documentation response exceeds byte limit"); } if (!response.body) throw new Error("empty documentation response"); const reader = response.body.getReader(); const cancel = () => { void reader.cancel().catch(() => {}); }; signal.addEventListener("abort", cancel, { once: true }); const chunks: Uint8Array[] = []; let size = 0; try { while (true) { const { done, value } = await reader.read(); signal.throwIfAborted(); if (done) break; size += value.byteLength; if (size > maxBytes) throw new Error("documentation response exceeds byte limit"); chunks.push(value); } } finally { signal.removeEventListener("abort", cancel); void reader.cancel().catch(() => {}); reader.releaseLock(); } return { url: url.href, text: Buffer.concat(chunks, size).toString("utf8"), data: Buffer.concat(chunks, size), type: (response.headers.get("content-type") ?? "").toLowerCase(), validator: response.headers.get("etag") ?? response.headers.get("last-modified"), unchanged: false, }; } throw new Error("documentation redirect limit"); }; try { return await Promise.race([work(), aborted]); } catch (error) { if (error instanceof ProviderError) throw error; throw new Error( "upstream documentation unavailable (HTTP, safety, size or deadline)", ); } finally { clearTimeout(timer); if (rejectAbort) signal.removeEventListener("abort", rejectAbort); } } function candidate( source: CatalogSource | undefined, target: string, url: string, scope = "public", ): Candidate { const origin = publicUrl(url).href; return { id: digest(JSON.stringify([source?.id ?? "url", origin])), target, version: source?.version && source.evidence ? source.version : null, origin, title: source?.description ?? target, source: "upstream", scope, evidence: source?.version && source.evidence ? source.evidence : null, }; } async function machineIndex( source: CatalogSource | undefined, target: string, index: string, request: Request, ): Promise { const page = await load(index, request); const terms = words(request.query); const links: { url: string; score: number; ordinal: number }[] = []; const seen = new Set(); let count = 0; for (const match of page.text.matchAll( /\[([^\]\n]{1,512})\]\(]{1,4096})>?(?:\s+"[^"\n]*")?\)/g, )) { if (++count > 256) break; let url: URL; try { url = publicUrl(match[2] ?? "", page.url); } catch { continue; } // A machine index does not authorize traversal to another publisher. if ( url.origin !== new URL(page.url).origin || /\/llms(?:-full)?\.txt$/.test(url.pathname) || seen.has(url.href) ) continue; seen.add(url.href); const label = (match[1] ?? "").toLowerCase(); let path: string; try { path = decodeURI(url.pathname).toLowerCase(); } catch { continue; } const score = terms.reduce( (sum, term) => sum + (label.includes(term) ? 4 : 0) + (path.includes(term) ? 2 : 0), 0, ); if (score) links.push({ url: url.href, score, ordinal: count }); } links.sort((a, b) => b.score - a.score || a.ordinal - b.ordinal); const selected = links[0]; return selected ? [candidate(source, target, selected.url)] : []; } return { id: "upstream", rank: 3, formatVersion: 1, coverage: "document", async discover(request) { request.signal?.throwIfAborted(); const maven = /^(?:maven|javadoc):([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.+-]+)$/.exec( request.target, ); if (maven) { const [, group = "", artifact = "", version = ""] = maven; if (!matchesVersion(request.version, version)) return []; const url = `https://repo.maven.apache.org/maven2/${group.replaceAll(".", "/")}/${artifact}/${version}/${artifact}-${version}-javadoc.jar`; return [ { ...candidate( undefined, request.target, url, "query:" + digest(request.query), ), version, evidence: "Maven repository artifact coordinates", }, ]; } if (/^https?:\/\//i.test(request.target)) { const url = publicUrl(request.target); return /\/llms\.txt$/.test(url.pathname) ? machineIndex(undefined, url.href, url.href, request) : [ candidate( undefined, url.href, url.href, queryScoped(url.href) ? "query:" + digest(request.query) : "public", ), ]; } const target = request.target.toLowerCase().trim(); const matches = [...catalog.values()].filter( (entry) => entry.adapter === "upstream" && [entry.id, ...entry.aliases].some( (alias) => target === alias.toLowerCase(), ), ); if (matches.length !== 1) return matches.map((entry) => candidate(entry, entry.id, entry.index ?? entry.home), ); const source = matches[0]; if (!source) return []; if (source.index) return machineIndex(source, source.id, source.index, request); return [ candidate( source, source.id, source.home, queryScoped(source.home) ? "query:" + digest(request.query) : "public", ), ]; }, async retrieve( selected, request, validator, ): Promise { const inventory = new URL(selected.origin).pathname.endsWith( "objects.inv", ); const symbolFormat = /-search-index\.js$/.test(selected.origin) ? "javadoc" : /\.(?:tag|xml)$/.test(selected.origin) ? "doxygen" : null; const indexed = inventory || symbolFormat !== null; const page = await load( selected.origin, request, indexed ? null : validator, ); if (page.unchanged) return "unchanged"; const source = catalog.get(selected.target); const format = inferredFormat( page.url, page.type, source?.format ?? "html", ); let content = symbolFormat ? symbolIndex(page.text, symbolFormat, request.query) : format === "pdf" || (format === "javadoc" && /\.(?:jar|zip)$/.test(new URL(page.url).pathname)) || inventory ? await readContainer( page.data, inventory ? "sphinx" : format === "pdf" ? "pdf" : "javadoc", request.query, request.signal, ) : readFormat(page.text, format); if (indexed && content[0]) { const linked = publicUrl(content[0].locator, page.url); if (linked.origin !== new URL(page.url).origin) throw new Error("cross-origin inventory link refused"); const body = await load(linked.href, request); content = readFormat( body.text, inferredFormat(body.url, body.type, "html"), ).map((section) => ({ ...section, body: section.body + "\n\nSource: " + linked.href, })); } if (!content.length) throw new Error("upstream document has no extractable sections"); return { candidate: selected, sections: content, attribution: `${source?.authority ?? "publisher authority unverified"}: ${source?.id ?? new URL(selected.origin).hostname}; ${selected.origin}`, validator: indexed ? null : page.validator, }; }, }; }