Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/devdocs/src/engine.ts

Raw
import { randomUUID } from "node:crypto";
import { context7Adapter } from "./adapters/context7.ts";
import { devdocsAdapter } from "./adapters/devdocs.ts";
import { fileAdapter, localAdapter } from "./adapters/local.ts";
import { ProviderError } from "./adapters/network.ts";
import { upstreamAdapter } from "./adapters/upstream.ts";
import { formatReaders, sourceCatalog } from "./catalog.ts";
import { Cache } from "./database.ts";
import { digest, MAX_DOCUMENT_BYTES } from "./normalize.ts";
import {
	type Adapter,
	type Candidate,
	matchesVersion,
	type Options,
	type ReadResult,
	type ReadSection,
	type Request,
	type SearchHit,
	type SearchResult,
	type StatusResult,
	type StoredSection,
} from "./types.ts";

export const MIN_BUDGET = 512;
export function countTokens(value: string): number {
	let count = 0;
	for (const _ of value) count++;
	return count;
}
type Head = { revision: string; validator: string | null; checked: number };
type RawHit = {
	revision: string;
	ordinal: number;
	heading: string;
	snippet: string;
	target: string;
	version: string | null;
	source: string;
	origin: string;
	checked: number;
};
type Discovery = { candidates: Candidate[]; checked: number };
async function waitForCaller<T>(
	pending: Promise<T>,
	signal?: AbortSignal,
): Promise<T> {
	if (!signal) return pending;
	signal.throwIfAborted();
	let abort: (() => void) | undefined;
	const stopped = new Promise<never>((_, reject) => {
		abort = () => reject(new Error("documentation request aborted"));
		signal.addEventListener("abort", abort, { once: true });
	});
	try {
		return await Promise.race([pending, stopped]);
	} finally {
		if (abort) signal.removeEventListener("abort", abort);
	}
}
export class Devdocs {
	readonly cache: Cache;
	readonly adapters: Adapter[];
	private closed = false;
	private configured: Promise<unknown> | undefined;
	private async configure(): Promise<void> {
		this.configured ??= this.cache.call("limit", {
			capacity: this.options.cacheBytes,
		});
		await this.configured;
	}
	private controller = new AbortController();
	private jobs = new Map<string, Promise<void>>();
	private failures = new Map<string, { retryAt: number; attempts: number }>();
	readonly options: Options;
	private context7Key: string | undefined;
	constructor(options: Options, adapters?: Adapter[], context7Key?: string) {
		this.options = options;
		this.context7Key = context7Key;
		this.cache = new Cache(options.databasePath);
		this.adapters = (
			adapters ?? [
				localAdapter(),
				...options.sources.map(fileAdapter),
				devdocsAdapter(),
				upstreamAdapter(),
				context7Adapter(context7Key ? { apiKey: context7Key } : {}),
			]
		)
			.filter((a) => options.enabledSources.includes(a.id))
			.sort((a, b) => a.rank - b.rank);
	}
	private namespace(a: Adapter, cwd: string): string {
		const source = this.options.sources.find((s) => s.id === a.id);
		return digest(
			JSON.stringify([
				a.id,
				a.formatVersion,
				a.identity ?? null,
				source ?? null,
				source?.private ? cwd : null,
				a.id === "context7" ? digest(this.context7Key ?? "anonymous") : null,
			]),
		);
	}
	private key(a: Adapter, c: Candidate): string {
		return digest(
			JSON.stringify([
				a.id,
				a.formatVersion,
				c.id,
				c.origin,
				c.version,
				c.scope,
			]),
		);
	}
	private async update(
		a: Adapter,
		c: Candidate,
		r: Request,
		key: string,
		head: Head | null,
		force: boolean,
	): Promise<void> {
		if (this.closed || (this.options.offline && a.rank > 1)) return;
		const existing = this.jobs.get(key);
		if (existing) return existing;
		const prior = this.failures.get(key);
		if (!force && prior && Date.now() < prior.retryAt) return;
		if (this.jobs.size >= 2) throw new ProviderError("busy");
		const job = (async () => {
			const owner = randomUUID();
			if (!(await this.cache.call<boolean>("lease", { key, owner }))) return;
			const signal = AbortSignal.any([
				this.controller.signal,
				AbortSignal.timeout(45000),
			]);
			try {
				const doc = await a.retrieve(
					{ ...c, scope: c.scope.slice(c.scope.indexOf(":") + 1) },
					{ ...r, signal },
					head?.validator,
				);
				signal.throwIfAborted();
				if (doc === "unchanged") {
					if (!head) throw new Error("invalid unchanged response");
					await this.cache.call("validate", {
						key,
						owner,
						revision: head.revision,
						validator: head.validator,
					});
				} else {
					if (
						doc.candidate.source !== a.id ||
						doc.candidate.id !== c.id ||
						doc.candidate.origin !== c.origin ||
						doc.candidate.version !== c.version ||
						!matchesVersion(r.version, doc.candidate.version)
					)
						throw new Error("source identity/version changed");
					if (
						doc.sections.length > 10000 ||
						doc.sections.some((section) => section.body.includes("\0")) ||
						Buffer.byteLength(JSON.stringify(doc)) > MAX_DOCUMENT_BYTES
					)
						throw new Error("document exceeds bounds");
					doc.candidate = {
						...doc.candidate,
						scope: c.scope,
						requestedVersion:
							r.requestedVersion !== undefined
								? r.requestedVersion
								: (r.version ?? null),
					};
					await this.cache.call("put", {
						key,
						owner,
						document: doc,
						capacity: this.options.cacheBytes,
					});
				}
				this.failures.delete(key);
			} catch (error) {
				const attempts = Math.min(10, (prior?.attempts ?? 0) + 1);
				const delay =
					error instanceof ProviderError && error.retryAfterMs !== null
						? Math.max(error.retryAfterMs, 60000)
						: Math.min(3600000, 60000 * 2 ** (attempts - 1));
				if (this.failures.size >= 1000)
					this.failures.delete(this.failures.keys().next().value ?? "");
				this.failures.set(key, { retryAt: Date.now() + delay, attempts });
				throw error;
			} finally {
				await this.cache.call("release", { key, owner });
			}
		})();
		this.jobs.set(key, job);
		try {
			await job;
		} finally {
			this.jobs.delete(key);
		}
	}
	async search(r: Request, refresh = false): Promise<SearchResult> {
		if (this.closed) throw new Error("devdocs session closed");
		await this.configure();
		const requestedVersion = r.version ?? null;
		r = { ...r, requestedVersion };
		const reasons: string[] = [];
		for (const a of this.adapters) {
			r.signal?.throwIfAborted();
			const lookup = digest(
				JSON.stringify([
					this.namespace(a, r.cwd),
					r.target,
					r.version ?? null,
					r.query,
				]),
			);
			const broadLookup = digest(
				JSON.stringify([
					this.namespace(a, r.cwd),
					r.target,
					r.version ?? null,
					"retained-documents",
				]),
			);
			let fromBroad = false;
			const remembered = await this.cache.call<Discovery | null>("discovery", {
				key: lookup,
			});
			let candidates = remembered?.candidates ?? [];
			if (
				!remembered &&
				this.options.offline &&
				a.rank > 1 &&
				a.coverage !== "query"
			) {
				candidates =
					(
						await this.cache.call<Discovery | null>("discovery", {
							key: broadLookup,
						})
					)?.candidates ?? [];
				fromBroad = true;
			}
			if (
				a.rank === 1 ||
				!remembered ||
				refresh ||
				Date.now() - remembered.checked >= this.options.ttlMs
			) {
				if (a.rank === 1 || !this.options.offline) {
					try {
						const discovered = await a.discover({
							...r,
							signal: r.signal
								? AbortSignal.any([r.signal, this.controller.signal])
								: this.controller.signal,
						});
						candidates = discovered
							.filter((c) => matchesVersion(r.version, c.version))
							.slice(0, 20)
							.map((c) => ({
								...c,
								scope: this.namespace(a, r.cwd) + ":" + c.scope,
								requestedVersion,
							}));
						if (candidates.length) {
							await this.cache.call("remember", {
								key: lookup,
								candidates,
								capacity: this.options.cacheBytes,
							});
							if (
								candidates.length === 1 &&
								a.coverage !== "query" &&
								!candidates[0]?.scope.includes(":query:")
							)
								await this.cache.call("remember", {
									key: broadLookup,
									candidates,
									merge: true,
									capacity: this.options.cacheBytes,
								});
						}
					} catch {
						r.signal?.throwIfAborted();
						reasons.push(
							a.id + ": discovery failed; using eligible cached discovery",
						);
						if (!candidates.length && a.coverage !== "query") {
							candidates =
								(
									await this.cache.call<Discovery | null>("discovery", {
										key: broadLookup,
									})
								)?.candidates ?? [];
							fromBroad = true;
						}
					}
				} else reasons.push(a.id + ": offline");
			}
			if (fromBroad) {
				let selected: Candidate | undefined;
				for (const candidate of candidates) {
					if (!matchesVersion(r.version, candidate.version)) continue;
					const hits = await this.cache.call<RawHit[]>("search", {
						key: this.key(a, candidate),
						query: r.query,
						limit: 1,
					});
					if (hits.length) {
						selected = candidate;
						break;
					}
				}
				candidates = selected ? [selected] : [];
			}
			if (candidates.length > 1) return { status: "ambiguous", candidates };
			const c = candidates[0];
			if (!c) {
				reasons.push(a.id + ": target/version unavailable");
				continue;
			}
			if (!r.version && a.id === "local" && c.version)
				r = { ...r, version: c.version };
			const key = this.key(a, c);
			let head = await this.cache.call<Head | null>("head", { key });
			let failed = false;
			if (!head || refresh || a.rank === 1) {
				try {
					await waitForCaller(
						this.update(a, c, r, key, head, refresh),
						r.signal,
					);
					head = await this.cache.call<Head | null>("head", { key });
				} catch (error) {
					failed = true;
					reasons.push(
						a.id +
							": " +
							(error instanceof ProviderError
								? error.code
								: /capacity/.test(String(error))
									? "capacity"
									: "refresh failed") +
							"; prior eligible content retained",
					);
				}
			} else if (
				Date.now() - head.checked >= this.options.ttlMs &&
				!this.options.offline
			) {
				void this.update(a, c, r, key, head, false).catch(() => {});
				reasons.push(a.id + ": stale content; background validation scheduled");
			}
			r.signal?.throwIfAborted();
			if (!head) continue;
			let hits = await this.cache.call<RawHit[]>("search", {
				key,
				query: r.query,
				limit: 10,
			});
			if (
				!hits.length &&
				!(await this.cache.call<Head | null>("head", { key })) &&
				(!this.options.offline || a.rank === 1)
			) {
				try {
					await waitForCaller(this.update(a, c, r, key, null, true), r.signal);
					hits = await this.cache.call<RawHit[]>("search", {
						key,
						query: r.query,
						limit: 10,
					});
					await this.cache.call("remember", {
						key: lookup,
						candidates,
						capacity: this.options.cacheBytes,
					});
					reasons.push(a.id + ": cache rebuilt after invalidation");
				} catch {
					reasons.push(a.id + ": invalidated cache could not be rebuilt");
				}
			}
			if (!hits.length) {
				reasons.push(a.id + ": no matching sections");
				continue;
			}
			return {
				status: "ok",
				results: hits.map(
					(h): SearchHit => ({
						ref: h.revision + ":" + h.ordinal,
						heading: h.heading,
						snippet: h.snippet.replace(/\s+/g, " "),
						target: h.target,
						version: h.version,
						source: h.source,
						origin: h.origin,
						freshness:
							failed || Date.now() - h.checked >= this.options.ttlMs
								? "stale"
								: "fresh",
						checkedAt: new Date(h.checked).toISOString(),
						requestedVersion,
					}),
				),
				reasons,
			};
		}
		return { status: "unavailable", reasons };
	}
	async read(refs: string[], budget = 4000, cwd = ""): Promise<ReadResult> {
		if (!Number.isInteger(budget) || budget < MIN_BUDGET || budget > 64000)
			throw new Error("budget must be 512..64000 Unicode code-point tokens");
		if (refs.length < 1 || refs.length > 20)
			throw new Error("expected 1..20 refs");
		await this.configure();
		const parsed = refs.map((ref) => {
			const m = /^([a-f0-9]{64}):(\d{1,5})(?::(\d{1,8}))?$/.exec(ref);
			if (!m) throw new Error("invalid section ref");
			return {
				revision: m[1],
				ordinal: Number(m[2]),
				offset: Number(m[3] ?? 0),
			};
		});
		const stored = await this.cache.call<(StoredSection | null)[]>("readMany", {
			refs: parsed,
			budget,
			ttlMs: this.options.ttlMs,
		});
		const items: ReadSection[] = [];
		for (let i = 0; i < refs.length; i++) {
			const ref = refs[i] ?? "";
			const section = stored[i];
			if (!section) throw new Error("expired-ref: section no longer retained");
			const adapter = this.adapters.find((a) => a.id === section.source);
			if (!adapter) throw new Error("source-disabled: section not eligible");
			if (!section.scope.startsWith(this.namespace(adapter, cwd) + ":"))
				throw new Error(
					"source-access: reference belongs to another source configuration or authorization context",
				);
			const offset = parsed[i]?.offset ?? 0;
			const chars = [...section.body];
			if (offset >= section.totalLength)
				throw new Error("invalid continuation position");
			const item: ReadSection = {
				ref: ref.split(":").slice(0, 2).join(":"),
				heading: section.heading,
				target: section.target,
				version: section.version,
				source: section.source,
				origin: section.origin,
				checkedAt: section.checkedAt,
				attribution: section.attribution,
				content: "",
				requestedVersion: section.requestedVersion,
				freshness: section.freshness ?? "stale",
			};
			const base = section.revision + ":" + section.ordinal;
			const build = (end: number): ReadResult => ({
				status: "ok",
				sections: [
					...items,
					{ ...item, content: chars.slice(0, end).join("") },
				],
				truncated: offset + end < section.totalLength || i + 1 < refs.length,
				continuation:
					offset + end < section.totalLength
						? [base + ":" + (offset + end), ...refs.slice(i + 1)]
						: refs.slice(i + 1),
			});
			let low = 0,
				high = chars.length;
			// A fitting penultimate prefix is a safe lower bound; completion can
			// shrink JSON by removing its continuation, so a full-fit probe is not.
			// Skip budget-sized bodies: their prefixes leave no room for metadata.
			if (
				high > 1 &&
				high < budget &&
				countTokens(JSON.stringify(build(high - 1))) <= budget
			)
				low = high - 1;
			while (low < high) {
				const mid = Math.ceil((low + high) / 2);
				if (countTokens(JSON.stringify(build(mid))) <= budget) low = mid;
				else high = mid - 1;
			}
			if (low === 0) {
				if (items.length)
					return {
						status: "ok",
						sections: items,
						truncated: true,
						continuation: refs.slice(i),
					};
				throw new Error(
					"budget insufficient for provenance and content progress",
				);
			}
			if (offset + low < section.totalLength) return build(low);
			item.content = chars.slice(0, low).join("");
			items.push(item);
		}
		return {
			status: "ok",
			sections: items,
			truncated: false,
			continuation: [],
		};
	}
	async status(): Promise<
		StatusResult & {
			catalog: { id: string; home: string; format: string; adapter: string }[];
			formats: { id: string; available: boolean }[];
		}
	> {
		await this.configure();
		return {
			status: "ok",
			tokenizer: "unicode-codepoint-v1",
			minimumBudget: MIN_BUDGET,
			sources: this.adapters.map((a) => ({
				id: a.id,
				rank: a.rank,
				coverage: a.coverage,
			})),
			cache: await this.cache.status(),
			offline: this.options.offline,
			context7Mode: this.adapters.some((a) => a.id === "context7")
				? this.context7Key
					? "keyed"
					: "anonymous"
				: "disabled",
			catalog: sourceCatalog.map((s) => ({
				id: s.id,
				home: s.home,
				format: s.format,
				adapter: s.adapter,
			})),
			formats: formatReaders.map((f) => ({ id: f.id, available: f.available })),
		};
	}
	async close(): Promise<void> {
		this.closed = true;
		this.controller.abort();
		await Promise.allSettled([...this.jobs.values()]);
		await this.cache.close();
	}
}