import { readFormat } from "../formats.ts"; import { digest } from "../normalize.ts"; import { type Adapter, matchesVersion } from "../types.ts"; import { createNetwork, type NetworkOptions, ProviderError, parseJson, record, string, } from "./network.ts"; const LIMIT = 2 * 1024 * 1024; const key = (value: string) => value.toLowerCase().replace(/[^a-z0-9]+/g, ""); export function devdocsAdapter(options: NetworkOptions = {}): Adapter { const catalog = createNetwork("https://devdocs.io", options); const documents = createNetwork("https://documents.devdocs.io", options); return { id: "devdocs", rank: 2, formatVersion: 1, coverage: "query", async discover(r) { if ( !r.target || r.target.length > 500 || !r.query || r.query.length > 500 || (r.version?.length ?? 0) > 100 ) throw new ProviderError("invalid"); const response = await catalog("https://devdocs.io/docs.json", { maxBytes: LIMIT, signal: r.signal, }); const docs = parseJson(response.text); if (!Array.isArray(docs)) throw new ProviderError("invalid"); let matches = docs .map(record) .filter((doc) => { const slug = string(doc["slug"]); return ( /^[a-z0-9_.~-]+$/i.test(slug) && [doc["name"], slug, slug.split("~")[0]].some( (name) => key(string(name)) === key(r.target), ) ); }) .filter((doc) => matchesVersion( r.version, string(doc["release"]) || string(doc["version"]) || null, ), ); // An exact slug identifies a docset; otherwise retain genuine catalog ambiguity. const exact = matches.filter( (doc) => string(doc["slug"]).toLowerCase() === r.target.toLowerCase(), ); if (exact.length) matches = exact; if ( matches.length > 1 && new Set(matches.map((doc) => string(doc["slug"]).split("~")[0])) .size === 1 ) { const version = (doc: Record) => (string(doc["release"]) || string(doc["version"])) .replace(/^v/, "") .split(/[.-]/) .map((part) => Number(part) || 0); matches.sort((a, b) => { const left = version(a), right = version(b); for (let i = 0; i < Math.max(left.length, right.length); i++) { const difference = (right[i] ?? 0) - (left[i] ?? 0); if (difference) return difference; } return 0; }); matches = matches.slice(0, 1); } return matches.slice(0, 20).map((doc) => ({ id: string(doc["slug"]), target: r.target, version: string(doc["release"]) || string(doc["version"]) || null, origin: `https://devdocs.io/${string(doc["slug"])}/`, title: string(doc["name"]), source: "devdocs", scope: "page-query:" + digest(r.query), evidence: "DevDocs catalog release/version", })); }, async retrieve(c, r, validator) { if ( !/^[a-z0-9_.~-]+$/i.test(c.id) || c.scope !== "page-query:" + digest(r.query) ) throw new ProviderError("invalid"); const base = `https://documents.devdocs.io/${c.id}/`; const index = record( parseJson( ( await documents(base + "index.json", { maxBytes: LIMIT, signal: r.signal, }) ).text, ), ); if (!Array.isArray(index["entries"])) throw new ProviderError("invalid"); const terms = r.query .toLowerCase() .match(/[\p{L}\p{N}_]+/gu) ?.slice(0, 32) ?? []; const entries = index["entries"] .map(record) .map((entry, order) => { const name = string(entry["name"]).toLowerCase(); const score = terms.reduce( (sum, term) => sum + (name.includes(term) ? 1 : 0), 0, ); return { path: string(entry["path"]).split("#")[0] ?? "", score, order, }; }) .filter( (entry) => entry["score"] > 0 && entry["path"] && entry["path"] .split("/") .every( (part) => part !== "." && part !== ".." && /^[a-z0-9_.+-]+$/i.test(part), ), ) .sort((a, b) => b.score - a.score || a.order - b.order); const path = entries[0]?.path; if (!path) throw new ProviderError("unavailable"); const old = validator ? record(parseJson(validator)) : {}; const response = await documents(base + path + ".html", { maxBytes: LIMIT, signal: r.signal, validator: old["path"] === path ? string(old["validator"]) || null : null, }); if (response.unchanged) return "unchanged"; const converted = readFormat(response.text, "html"); if (!converted.length) throw new ProviderError("invalid"); const origin = `${c.origin}${path}`; return { candidate: c, sections: converted.map((section) => ({ ...section, body: `${section.body}\n\nSource: ${origin}`, })), attribution: "DevDocs; underlying documentation retains its original license. " + origin, validator: JSON.stringify({ path, validator: response.validator }), }; }, }; }