repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/devdocs/src/adapters/upstream.ts
Rawimport {
type CatalogSource,
type FormatId,
sourceCatalog,
} from "../catalog.ts";
import { readContainer } from "../containers.ts";
import { readFormat } from "../formats.ts";
import { digest, MAX_DOCUMENT_BYTES } from "../normalize.ts";
import { symbolIndex } from "../symbols.ts";
import type { Adapter, Candidate, Document, Request } from "../types.ts";
import { matchesVersion } from "../types.ts";
import { httpError, ProviderError } from "./network.ts";
export type UpstreamFetch = (
url: string,
init: RequestInit,
) => Promise<Response>;
export type UpstreamOptions = {
fetch?: UpstreamFetch;
/** Additional publisher roots; duplicate IDs replace built-in catalog entries. */
sources?: readonly CatalogSource[];
maxBytes?: number;
timeoutMs?: number;
};
type Page = {
url: string;
text: string;
data: Uint8Array;
type: string;
validator: string | null;
unchanged: boolean;
};
/** Documentation may be self-hosted. Query strings and userinfo can carry credentials. */
function publicUrl(value: string, base?: string): URL {
let url: URL;
try {
url = new URL(value, base);
} catch {
throw new Error("invalid documentation URL");
}
if (
url.protocol !== "https:" ||
url.username ||
url.password ||
url.search ||
url.href.length > 4096
)
throw new Error(
"documentation URL must be HTTPS without credentials or query parameters",
);
return url;
}
function finiteLimit(value: number, maximum: number, name: string): number {
if (!Number.isInteger(value) || value < 1 || value > maximum)
throw new Error(`invalid ${name} limit`);
return value;
}
function inferredFormat(
url: string,
contentType: string,
fallback: FormatId,
): FormatId {
const path = new URL(url).pathname.toLowerCase();
if (/-search-index\.js$/.test(path)) return "javadoc";
if (
path.endsWith(".tag") ||
(path.endsWith(".xml") && fallback === "doxygen")
)
return "doxygen";
if (path.endsWith(".mdx")) return "mdx";
if (path.endsWith(".rst")) return "rst";
if (path.endsWith(".pdf") || contentType.includes("application/pdf"))
return "pdf";
if (path.endsWith(".json") || contentType.includes("application/json"))
return "json";
if (/\.(?:jar|zip)$/.test(path)) return "javadoc";
if (path.endsWith("objects.inv")) return "sphinx";
if (/\.(?:tgz|gz)$/.test(path)) throw new Error("archive reader unavailable");
if (/\.(?:md|markdown)$/.test(path) || contentType.includes("text/markdown"))
return "markdown";
if (
contentType.includes("text/html") ||
contentType.includes("application/xhtml")
)
return "html";
if (path.endsWith(".txt") || contentType.includes("text/plain"))
return "text";
if (contentType && !contentType.startsWith("text/"))
throw new Error("unsupported documentation media type");
return fallback;
}
function queryScoped(url: string): boolean {
return /\.(?:jar|zip|tag|xml)$|objects\.inv$|-search-index\.js$/.test(
new URL(url).pathname,
);
}
function words(value: string): string[] {
return [
...new Set(value.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? []),
].slice(0, 64);
}
export function upstreamAdapter(options: UpstreamOptions = {}): Adapter {
const fetchPage = options.fetch ?? ((url, init) => fetch(url, init));
const maxBytes = finiteLimit(
options.maxBytes ?? 2 * 1024 * 1024,
MAX_DOCUMENT_BYTES,
"response bytes",
);
const timeoutMs = finiteLimit(options.timeoutMs ?? 10000, 60000, "deadline");
const catalog = new Map(sourceCatalog.map((entry) => [entry.id, entry]));
for (const entry of options.sources ?? []) catalog.set(entry.id, entry);
async function load(
origin: string,
request: Request,
validator?: string | null,
): Promise<Page> {
const initial = publicUrl(origin);
initial.hash = "";
const controller = new AbortController();
const signal = request.signal
? AbortSignal.any([request.signal, controller.signal])
: controller.signal;
const timer = setTimeout(() => controller.abort(), timeoutMs);
let rejectAbort: (() => void) | undefined;
const aborted = new Promise<never>((_, reject) => {
rejectAbort = () =>
reject(
new Error("documentation request cancelled or deadline exceeded"),
);
signal.addEventListener("abort", rejectAbort, { once: true });
});
const work = async (): Promise<Page> => {
let url = initial;
for (let redirects = 0; redirects <= 3; redirects++) {
signal.throwIfAborted();
const headers: Record<string, string> = {
accept: "text/markdown, text/html, text/plain;q=0.9",
};
if (validator)
headers[
validator.startsWith('"') || validator.startsWith("W/")
? "if-none-match"
: "if-modified-since"
] = validator;
const response = await fetchPage(url.href, {
signal,
redirect: "manual",
credentials: "omit",
referrerPolicy: "no-referrer",
headers,
});
if (signal.aborted) {
void response.body?.cancel().catch(() => {});
signal.throwIfAborted();
}
if ([301, 302, 303, 307, 308].includes(response.status)) {
void response.body?.cancel().catch(() => {});
const location = response.headers.get("location");
if (!location || redirects === 3)
throw new Error("documentation redirect limit or missing location");
const next = publicUrl(location, url.href);
if (next.origin !== initial.origin)
throw new Error("cross-origin documentation redirect refused");
next.hash = "";
url = next;
continue;
}
if (response.status === 304) {
void response.body?.cancel().catch(() => {});
if (!validator)
throw new Error("unexpected documentation validation response");
return {
url: url.href,
text: "",
data: new Uint8Array(),
type: "",
validator,
unchanged: true,
};
}
if (!response.ok) {
void response.body?.cancel().catch(() => {});
throw httpError(response);
}
const length = Number(response.headers.get("content-length"));
if (Number.isFinite(length) && length > maxBytes) {
void response.body?.cancel().catch(() => {});
throw new Error("documentation response exceeds byte limit");
}
if (!response.body) throw new Error("empty documentation response");
const reader = response.body.getReader();
const cancel = () => {
void reader.cancel().catch(() => {});
};
signal.addEventListener("abort", cancel, { once: true });
const chunks: Uint8Array[] = [];
let size = 0;
try {
while (true) {
const { done, value } = await reader.read();
signal.throwIfAborted();
if (done) break;
size += value.byteLength;
if (size > maxBytes)
throw new Error("documentation response exceeds byte limit");
chunks.push(value);
}
} finally {
signal.removeEventListener("abort", cancel);
void reader.cancel().catch(() => {});
reader.releaseLock();
}
return {
url: url.href,
text: Buffer.concat(chunks, size).toString("utf8"),
data: Buffer.concat(chunks, size),
type: (response.headers.get("content-type") ?? "").toLowerCase(),
validator:
response.headers.get("etag") ??
response.headers.get("last-modified"),
unchanged: false,
};
}
throw new Error("documentation redirect limit");
};
try {
return await Promise.race([work(), aborted]);
} catch (error) {
if (error instanceof ProviderError) throw error;
throw new Error(
"upstream documentation unavailable (HTTP, safety, size or deadline)",
);
} finally {
clearTimeout(timer);
if (rejectAbort) signal.removeEventListener("abort", rejectAbort);
}
}
function candidate(
source: CatalogSource | undefined,
target: string,
url: string,
scope = "public",
): Candidate {
const origin = publicUrl(url).href;
return {
id: digest(JSON.stringify([source?.id ?? "url", origin])),
target,
version: source?.version && source.evidence ? source.version : null,
origin,
title: source?.description ?? target,
source: "upstream",
scope,
evidence: source?.version && source.evidence ? source.evidence : null,
};
}
async function machineIndex(
source: CatalogSource | undefined,
target: string,
index: string,
request: Request,
): Promise<Candidate[]> {
const page = await load(index, request);
const terms = words(request.query);
const links: { url: string; score: number; ordinal: number }[] = [];
const seen = new Set<string>();
let count = 0;
for (const match of page.text.matchAll(
/\[([^\]\n]{1,512})\]\(<?([^\s)>]{1,4096})>?(?:\s+"[^"\n]*")?\)/g,
)) {
if (++count > 256) break;
let url: URL;
try {
url = publicUrl(match[2] ?? "", page.url);
} catch {
continue;
}
// A machine index does not authorize traversal to another publisher.
if (
url.origin !== new URL(page.url).origin ||
/\/llms(?:-full)?\.txt$/.test(url.pathname) ||
seen.has(url.href)
)
continue;
seen.add(url.href);
const label = (match[1] ?? "").toLowerCase();
let path: string;
try {
path = decodeURI(url.pathname).toLowerCase();
} catch {
continue;
}
const score = terms.reduce(
(sum, term) =>
sum + (label.includes(term) ? 4 : 0) + (path.includes(term) ? 2 : 0),
0,
);
if (score) links.push({ url: url.href, score, ordinal: count });
}
links.sort((a, b) => b.score - a.score || a.ordinal - b.ordinal);
const selected = links[0];
return selected ? [candidate(source, target, selected.url)] : [];
}
return {
id: "upstream",
rank: 3,
formatVersion: 1,
coverage: "document",
async discover(request) {
request.signal?.throwIfAborted();
const maven =
/^(?:maven|javadoc):([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.-]+):([a-zA-Z0-9_.+-]+)$/.exec(
request.target,
);
if (maven) {
const [, group = "", artifact = "", version = ""] = maven;
if (!matchesVersion(request.version, version)) return [];
const url = `https://repo.maven.apache.org/maven2/${group.replaceAll(".", "/")}/${artifact}/${version}/${artifact}-${version}-javadoc.jar`;
return [
{
...candidate(
undefined,
request.target,
url,
"query:" + digest(request.query),
),
version,
evidence: "Maven repository artifact coordinates",
},
];
}
if (/^https?:\/\//i.test(request.target)) {
const url = publicUrl(request.target);
return /\/llms\.txt$/.test(url.pathname)
? machineIndex(undefined, url.href, url.href, request)
: [
candidate(
undefined,
url.href,
url.href,
queryScoped(url.href)
? "query:" + digest(request.query)
: "public",
),
];
}
const target = request.target.toLowerCase().trim();
const matches = [...catalog.values()].filter(
(entry) =>
entry.adapter === "upstream" &&
[entry.id, ...entry.aliases].some(
(alias) => target === alias.toLowerCase(),
),
);
if (matches.length !== 1)
return matches.map((entry) =>
candidate(entry, entry.id, entry.index ?? entry.home),
);
const source = matches[0];
if (!source) return [];
if (source.index)
return machineIndex(source, source.id, source.index, request);
return [
candidate(
source,
source.id,
source.home,
queryScoped(source.home)
? "query:" + digest(request.query)
: "public",
),
];
},
async retrieve(
selected,
request,
validator,
): Promise<Document | "unchanged"> {
const inventory = new URL(selected.origin).pathname.endsWith(
"objects.inv",
);
const symbolFormat = /-search-index\.js$/.test(selected.origin)
? "javadoc"
: /\.(?:tag|xml)$/.test(selected.origin)
? "doxygen"
: null;
const indexed = inventory || symbolFormat !== null;
const page = await load(
selected.origin,
request,
indexed ? null : validator,
);
if (page.unchanged) return "unchanged";
const source = catalog.get(selected.target);
const format = inferredFormat(
page.url,
page.type,
source?.format ?? "html",
);
let content = symbolFormat
? symbolIndex(page.text, symbolFormat, request.query)
: format === "pdf" ||
(format === "javadoc" &&
/\.(?:jar|zip)$/.test(new URL(page.url).pathname)) ||
inventory
? await readContainer(
page.data,
inventory ? "sphinx" : format === "pdf" ? "pdf" : "javadoc",
request.query,
request.signal,
)
: readFormat(page.text, format);
if (indexed && content[0]) {
const linked = publicUrl(content[0].locator, page.url);
if (linked.origin !== new URL(page.url).origin)
throw new Error("cross-origin inventory link refused");
const body = await load(linked.href, request);
content = readFormat(
body.text,
inferredFormat(body.url, body.type, "html"),
).map((section) => ({
...section,
body: section.body + "\n\nSource: " + linked.href,
}));
}
if (!content.length)
throw new Error("upstream document has no extractable sections");
return {
candidate: selected,
sections: content,
attribution: `${source?.authority ?? "publisher authority unverified"}: ${source?.id ?? new URL(selected.origin).hostname}; ${selected.origin}`,
validator: indexed ? null : page.validator,
};
},
};
}