import { randomUUID } from "node:crypto"; import { context7Adapter } from "./adapters/context7.ts"; import { devdocsAdapter } from "./adapters/devdocs.ts"; import { fileAdapter, localAdapter } from "./adapters/local.ts"; import { ProviderError } from "./adapters/network.ts"; import { upstreamAdapter } from "./adapters/upstream.ts"; import { formatReaders, sourceCatalog } from "./catalog.ts"; import { Cache } from "./database.ts"; import { digest, MAX_DOCUMENT_BYTES } from "./normalize.ts"; import { type Adapter, type Candidate, matchesVersion, type Options, type ReadResult, type ReadSection, type Request, type SearchHit, type SearchResult, type StatusResult, type StoredSection, } from "./types.ts"; export const MIN_BUDGET = 512; export function countTokens(value: string): number { let count = 0; for (const _ of value) count++; return count; } type Head = { revision: string; validator: string | null; checked: number }; type RawHit = { revision: string; ordinal: number; heading: string; snippet: string; target: string; version: string | null; source: string; origin: string; checked: number; }; type Discovery = { candidates: Candidate[]; checked: number }; async function waitForCaller( pending: Promise, signal?: AbortSignal, ): Promise { if (!signal) return pending; signal.throwIfAborted(); let abort: (() => void) | undefined; const stopped = new Promise((_, reject) => { abort = () => reject(new Error("documentation request aborted")); signal.addEventListener("abort", abort, { once: true }); }); try { return await Promise.race([pending, stopped]); } finally { if (abort) signal.removeEventListener("abort", abort); } } export class Devdocs { readonly cache: Cache; readonly adapters: Adapter[]; private closed = false; private configured: Promise | undefined; private async configure(): Promise { this.configured ??= this.cache.call("limit", { capacity: this.options.cacheBytes, }); await this.configured; } private controller = new AbortController(); private jobs = new Map>(); private failures = new Map(); readonly options: Options; private context7Key: string | undefined; constructor(options: Options, adapters?: Adapter[], context7Key?: string) { this.options = options; this.context7Key = context7Key; this.cache = new Cache(options.databasePath); this.adapters = ( adapters ?? [ localAdapter(), ...options.sources.map(fileAdapter), devdocsAdapter(), upstreamAdapter(), context7Adapter(context7Key ? { apiKey: context7Key } : {}), ] ) .filter((a) => options.enabledSources.includes(a.id)) .sort((a, b) => a.rank - b.rank); } private namespace(a: Adapter, cwd: string): string { const source = this.options.sources.find((s) => s.id === a.id); return digest( JSON.stringify([ a.id, a.formatVersion, a.identity ?? null, source ?? null, source?.private ? cwd : null, a.id === "context7" ? digest(this.context7Key ?? "anonymous") : null, ]), ); } private key(a: Adapter, c: Candidate): string { return digest( JSON.stringify([ a.id, a.formatVersion, c.id, c.origin, c.version, c.scope, ]), ); } private async update( a: Adapter, c: Candidate, r: Request, key: string, head: Head | null, force: boolean, ): Promise { if (this.closed || (this.options.offline && a.rank > 1)) return; const existing = this.jobs.get(key); if (existing) return existing; const prior = this.failures.get(key); if (!force && prior && Date.now() < prior.retryAt) return; if (this.jobs.size >= 2) throw new ProviderError("busy"); const job = (async () => { const owner = randomUUID(); if (!(await this.cache.call("lease", { key, owner }))) return; const signal = AbortSignal.any([ this.controller.signal, AbortSignal.timeout(45000), ]); try { const doc = await a.retrieve( { ...c, scope: c.scope.slice(c.scope.indexOf(":") + 1) }, { ...r, signal }, head?.validator, ); signal.throwIfAborted(); if (doc === "unchanged") { if (!head) throw new Error("invalid unchanged response"); await this.cache.call("validate", { key, owner, revision: head.revision, validator: head.validator, }); } else { if ( doc.candidate.source !== a.id || doc.candidate.id !== c.id || doc.candidate.origin !== c.origin || doc.candidate.version !== c.version || !matchesVersion(r.version, doc.candidate.version) ) throw new Error("source identity/version changed"); if ( doc.sections.length > 10000 || doc.sections.some((section) => section.body.includes("\0")) || Buffer.byteLength(JSON.stringify(doc)) > MAX_DOCUMENT_BYTES ) throw new Error("document exceeds bounds"); doc.candidate = { ...doc.candidate, scope: c.scope, requestedVersion: r.requestedVersion !== undefined ? r.requestedVersion : (r.version ?? null), }; await this.cache.call("put", { key, owner, document: doc, capacity: this.options.cacheBytes, }); } this.failures.delete(key); } catch (error) { const attempts = Math.min(10, (prior?.attempts ?? 0) + 1); const delay = error instanceof ProviderError && error.retryAfterMs !== null ? Math.max(error.retryAfterMs, 60000) : Math.min(3600000, 60000 * 2 ** (attempts - 1)); if (this.failures.size >= 1000) this.failures.delete(this.failures.keys().next().value ?? ""); this.failures.set(key, { retryAt: Date.now() + delay, attempts }); throw error; } finally { await this.cache.call("release", { key, owner }); } })(); this.jobs.set(key, job); try { await job; } finally { this.jobs.delete(key); } } async search(r: Request, refresh = false): Promise { if (this.closed) throw new Error("devdocs session closed"); await this.configure(); const requestedVersion = r.version ?? null; r = { ...r, requestedVersion }; const reasons: string[] = []; for (const a of this.adapters) { r.signal?.throwIfAborted(); const lookup = digest( JSON.stringify([ this.namespace(a, r.cwd), r.target, r.version ?? null, r.query, ]), ); const broadLookup = digest( JSON.stringify([ this.namespace(a, r.cwd), r.target, r.version ?? null, "retained-documents", ]), ); let fromBroad = false; const remembered = await this.cache.call("discovery", { key: lookup, }); let candidates = remembered?.candidates ?? []; if ( !remembered && this.options.offline && a.rank > 1 && a.coverage !== "query" ) { candidates = ( await this.cache.call("discovery", { key: broadLookup, }) )?.candidates ?? []; fromBroad = true; } if ( a.rank === 1 || !remembered || refresh || Date.now() - remembered.checked >= this.options.ttlMs ) { if (a.rank === 1 || !this.options.offline) { try { const discovered = await a.discover({ ...r, signal: r.signal ? AbortSignal.any([r.signal, this.controller.signal]) : this.controller.signal, }); candidates = discovered .filter((c) => matchesVersion(r.version, c.version)) .slice(0, 20) .map((c) => ({ ...c, scope: this.namespace(a, r.cwd) + ":" + c.scope, requestedVersion, })); if (candidates.length) { await this.cache.call("remember", { key: lookup, candidates, capacity: this.options.cacheBytes, }); if ( candidates.length === 1 && a.coverage !== "query" && !candidates[0]?.scope.includes(":query:") ) await this.cache.call("remember", { key: broadLookup, candidates, merge: true, capacity: this.options.cacheBytes, }); } } catch { r.signal?.throwIfAborted(); reasons.push( a.id + ": discovery failed; using eligible cached discovery", ); if (!candidates.length && a.coverage !== "query") { candidates = ( await this.cache.call("discovery", { key: broadLookup, }) )?.candidates ?? []; fromBroad = true; } } } else reasons.push(a.id + ": offline"); } if (fromBroad) { let selected: Candidate | undefined; for (const candidate of candidates) { if (!matchesVersion(r.version, candidate.version)) continue; const hits = await this.cache.call("search", { key: this.key(a, candidate), query: r.query, limit: 1, }); if (hits.length) { selected = candidate; break; } } candidates = selected ? [selected] : []; } if (candidates.length > 1) return { status: "ambiguous", candidates }; const c = candidates[0]; if (!c) { reasons.push(a.id + ": target/version unavailable"); continue; } if (!r.version && a.id === "local" && c.version) r = { ...r, version: c.version }; const key = this.key(a, c); let head = await this.cache.call("head", { key }); let failed = false; if (!head || refresh || a.rank === 1) { try { await waitForCaller( this.update(a, c, r, key, head, refresh), r.signal, ); head = await this.cache.call("head", { key }); } catch (error) { failed = true; reasons.push( a.id + ": " + (error instanceof ProviderError ? error.code : /capacity/.test(String(error)) ? "capacity" : "refresh failed") + "; prior eligible content retained", ); } } else if ( Date.now() - head.checked >= this.options.ttlMs && !this.options.offline ) { void this.update(a, c, r, key, head, false).catch(() => {}); reasons.push(a.id + ": stale content; background validation scheduled"); } r.signal?.throwIfAborted(); if (!head) continue; let hits = await this.cache.call("search", { key, query: r.query, limit: 10, }); if ( !hits.length && !(await this.cache.call("head", { key })) && (!this.options.offline || a.rank === 1) ) { try { await waitForCaller(this.update(a, c, r, key, null, true), r.signal); hits = await this.cache.call("search", { key, query: r.query, limit: 10, }); await this.cache.call("remember", { key: lookup, candidates, capacity: this.options.cacheBytes, }); reasons.push(a.id + ": cache rebuilt after invalidation"); } catch { reasons.push(a.id + ": invalidated cache could not be rebuilt"); } } if (!hits.length) { reasons.push(a.id + ": no matching sections"); continue; } return { status: "ok", results: hits.map( (h): SearchHit => ({ ref: h.revision + ":" + h.ordinal, heading: h.heading, snippet: h.snippet.replace(/\s+/g, " "), target: h.target, version: h.version, source: h.source, origin: h.origin, freshness: failed || Date.now() - h.checked >= this.options.ttlMs ? "stale" : "fresh", checkedAt: new Date(h.checked).toISOString(), requestedVersion, }), ), reasons, }; } return { status: "unavailable", reasons }; } async read(refs: string[], budget = 4000, cwd = ""): Promise { if (!Number.isInteger(budget) || budget < MIN_BUDGET || budget > 64000) throw new Error("budget must be 512..64000 Unicode code-point tokens"); if (refs.length < 1 || refs.length > 20) throw new Error("expected 1..20 refs"); await this.configure(); const parsed = refs.map((ref) => { const m = /^([a-f0-9]{64}):(\d{1,5})(?::(\d{1,8}))?$/.exec(ref); if (!m) throw new Error("invalid section ref"); return { revision: m[1], ordinal: Number(m[2]), offset: Number(m[3] ?? 0), }; }); const stored = await this.cache.call<(StoredSection | null)[]>("readMany", { refs: parsed, budget, ttlMs: this.options.ttlMs, }); const items: ReadSection[] = []; for (let i = 0; i < refs.length; i++) { const ref = refs[i] ?? ""; const section = stored[i]; if (!section) throw new Error("expired-ref: section no longer retained"); const adapter = this.adapters.find((a) => a.id === section.source); if (!adapter) throw new Error("source-disabled: section not eligible"); if (!section.scope.startsWith(this.namespace(adapter, cwd) + ":")) throw new Error( "source-access: reference belongs to another source configuration or authorization context", ); const offset = parsed[i]?.offset ?? 0; const chars = [...section.body]; if (offset >= section.totalLength) throw new Error("invalid continuation position"); const item: ReadSection = { ref: ref.split(":").slice(0, 2).join(":"), heading: section.heading, target: section.target, version: section.version, source: section.source, origin: section.origin, checkedAt: section.checkedAt, attribution: section.attribution, content: "", requestedVersion: section.requestedVersion, freshness: section.freshness ?? "stale", }; const base = section.revision + ":" + section.ordinal; const build = (end: number): ReadResult => ({ status: "ok", sections: [ ...items, { ...item, content: chars.slice(0, end).join("") }, ], truncated: offset + end < section.totalLength || i + 1 < refs.length, continuation: offset + end < section.totalLength ? [base + ":" + (offset + end), ...refs.slice(i + 1)] : refs.slice(i + 1), }); let low = 0, high = chars.length; // A fitting penultimate prefix is a safe lower bound; completion can // shrink JSON by removing its continuation, so a full-fit probe is not. // Skip budget-sized bodies: their prefixes leave no room for metadata. if ( high > 1 && high < budget && countTokens(JSON.stringify(build(high - 1))) <= budget ) low = high - 1; while (low < high) { const mid = Math.ceil((low + high) / 2); if (countTokens(JSON.stringify(build(mid))) <= budget) low = mid; else high = mid - 1; } if (low === 0) { if (items.length) return { status: "ok", sections: items, truncated: true, continuation: refs.slice(i), }; throw new Error( "budget insufficient for provenance and content progress", ); } if (offset + low < section.totalLength) return build(low); item.content = chars.slice(0, low).join(""); items.push(item); } return { status: "ok", sections: items, truncated: false, continuation: [], }; } async status(): Promise< StatusResult & { catalog: { id: string; home: string; format: string; adapter: string }[]; formats: { id: string; available: boolean }[]; } > { await this.configure(); return { status: "ok", tokenizer: "unicode-codepoint-v1", minimumBudget: MIN_BUDGET, sources: this.adapters.map((a) => ({ id: a.id, rank: a.rank, coverage: a.coverage, })), cache: await this.cache.status(), offline: this.options.offline, context7Mode: this.adapters.some((a) => a.id === "context7") ? this.context7Key ? "keyed" : "anonymous" : "disabled", catalog: sourceCatalog.map((s) => ({ id: s.id, home: s.home, format: s.format, adapter: s.adapter, })), formats: formatReaders.map((f) => ({ id: f.id, available: f.available })), }; } async close(): Promise { this.closed = true; this.controller.abort(); await Promise.allSettled([...this.jobs.values()]); await this.cache.close(); } }