repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/devdocs/src/engine.ts
Rawimport { randomUUID } from "node:crypto";
import { context7Adapter } from "./adapters/context7.ts";
import { devdocsAdapter } from "./adapters/devdocs.ts";
import { fileAdapter, localAdapter } from "./adapters/local.ts";
import { ProviderError } from "./adapters/network.ts";
import { upstreamAdapter } from "./adapters/upstream.ts";
import { formatReaders, sourceCatalog } from "./catalog.ts";
import { Cache } from "./database.ts";
import { digest, MAX_DOCUMENT_BYTES } from "./normalize.ts";
import {
type Adapter,
type Candidate,
matchesVersion,
type Options,
type ReadResult,
type ReadSection,
type Request,
type SearchHit,
type SearchResult,
type StatusResult,
type StoredSection,
} from "./types.ts";
export const MIN_BUDGET = 512;
export function countTokens(value: string): number {
let count = 0;
for (const _ of value) count++;
return count;
}
type Head = { revision: string; validator: string | null; checked: number };
type RawHit = {
revision: string;
ordinal: number;
heading: string;
snippet: string;
target: string;
version: string | null;
source: string;
origin: string;
checked: number;
};
type Discovery = { candidates: Candidate[]; checked: number };
async function waitForCaller<T>(
pending: Promise<T>,
signal?: AbortSignal,
): Promise<T> {
if (!signal) return pending;
signal.throwIfAborted();
let abort: (() => void) | undefined;
const stopped = new Promise<never>((_, reject) => {
abort = () => reject(new Error("documentation request aborted"));
signal.addEventListener("abort", abort, { once: true });
});
try {
return await Promise.race([pending, stopped]);
} finally {
if (abort) signal.removeEventListener("abort", abort);
}
}
export class Devdocs {
readonly cache: Cache;
readonly adapters: Adapter[];
private closed = false;
private configured: Promise<unknown> | undefined;
private async configure(): Promise<void> {
this.configured ??= this.cache.call("limit", {
capacity: this.options.cacheBytes,
});
await this.configured;
}
private controller = new AbortController();
private jobs = new Map<string, Promise<void>>();
private failures = new Map<string, { retryAt: number; attempts: number }>();
readonly options: Options;
private context7Key: string | undefined;
constructor(options: Options, adapters?: Adapter[], context7Key?: string) {
this.options = options;
this.context7Key = context7Key;
this.cache = new Cache(options.databasePath);
this.adapters = (
adapters ?? [
localAdapter(),
...options.sources.map(fileAdapter),
devdocsAdapter(),
upstreamAdapter(),
context7Adapter(context7Key ? { apiKey: context7Key } : {}),
]
)
.filter((a) => options.enabledSources.includes(a.id))
.sort((a, b) => a.rank - b.rank);
}
private namespace(a: Adapter, cwd: string): string {
const source = this.options.sources.find((s) => s.id === a.id);
return digest(
JSON.stringify([
a.id,
a.formatVersion,
a.identity ?? null,
source ?? null,
source?.private ? cwd : null,
a.id === "context7" ? digest(this.context7Key ?? "anonymous") : null,
]),
);
}
private key(a: Adapter, c: Candidate): string {
return digest(
JSON.stringify([
a.id,
a.formatVersion,
c.id,
c.origin,
c.version,
c.scope,
]),
);
}
private async update(
a: Adapter,
c: Candidate,
r: Request,
key: string,
head: Head | null,
force: boolean,
): Promise<void> {
if (this.closed || (this.options.offline && a.rank > 1)) return;
const existing = this.jobs.get(key);
if (existing) return existing;
const prior = this.failures.get(key);
if (!force && prior && Date.now() < prior.retryAt) return;
if (this.jobs.size >= 2) throw new ProviderError("busy");
const job = (async () => {
const owner = randomUUID();
if (!(await this.cache.call<boolean>("lease", { key, owner }))) return;
const signal = AbortSignal.any([
this.controller.signal,
AbortSignal.timeout(45000),
]);
try {
const doc = await a.retrieve(
{ ...c, scope: c.scope.slice(c.scope.indexOf(":") + 1) },
{ ...r, signal },
head?.validator,
);
signal.throwIfAborted();
if (doc === "unchanged") {
if (!head) throw new Error("invalid unchanged response");
await this.cache.call("validate", {
key,
owner,
revision: head.revision,
validator: head.validator,
});
} else {
if (
doc.candidate.source !== a.id ||
doc.candidate.id !== c.id ||
doc.candidate.origin !== c.origin ||
doc.candidate.version !== c.version ||
!matchesVersion(r.version, doc.candidate.version)
)
throw new Error("source identity/version changed");
if (
doc.sections.length > 10000 ||
doc.sections.some((section) => section.body.includes("\0")) ||
Buffer.byteLength(JSON.stringify(doc)) > MAX_DOCUMENT_BYTES
)
throw new Error("document exceeds bounds");
doc.candidate = {
...doc.candidate,
scope: c.scope,
requestedVersion:
r.requestedVersion !== undefined
? r.requestedVersion
: (r.version ?? null),
};
await this.cache.call("put", {
key,
owner,
document: doc,
capacity: this.options.cacheBytes,
});
}
this.failures.delete(key);
} catch (error) {
const attempts = Math.min(10, (prior?.attempts ?? 0) + 1);
const delay =
error instanceof ProviderError && error.retryAfterMs !== null
? Math.max(error.retryAfterMs, 60000)
: Math.min(3600000, 60000 * 2 ** (attempts - 1));
if (this.failures.size >= 1000)
this.failures.delete(this.failures.keys().next().value ?? "");
this.failures.set(key, { retryAt: Date.now() + delay, attempts });
throw error;
} finally {
await this.cache.call("release", { key, owner });
}
})();
this.jobs.set(key, job);
try {
await job;
} finally {
this.jobs.delete(key);
}
}
async search(r: Request, refresh = false): Promise<SearchResult> {
if (this.closed) throw new Error("devdocs session closed");
await this.configure();
const requestedVersion = r.version ?? null;
r = { ...r, requestedVersion };
const reasons: string[] = [];
for (const a of this.adapters) {
r.signal?.throwIfAborted();
const lookup = digest(
JSON.stringify([
this.namespace(a, r.cwd),
r.target,
r.version ?? null,
r.query,
]),
);
const broadLookup = digest(
JSON.stringify([
this.namespace(a, r.cwd),
r.target,
r.version ?? null,
"retained-documents",
]),
);
let fromBroad = false;
const remembered = await this.cache.call<Discovery | null>("discovery", {
key: lookup,
});
let candidates = remembered?.candidates ?? [];
if (
!remembered &&
this.options.offline &&
a.rank > 1 &&
a.coverage !== "query"
) {
candidates =
(
await this.cache.call<Discovery | null>("discovery", {
key: broadLookup,
})
)?.candidates ?? [];
fromBroad = true;
}
if (
a.rank === 1 ||
!remembered ||
refresh ||
Date.now() - remembered.checked >= this.options.ttlMs
) {
if (a.rank === 1 || !this.options.offline) {
try {
const discovered = await a.discover({
...r,
signal: r.signal
? AbortSignal.any([r.signal, this.controller.signal])
: this.controller.signal,
});
candidates = discovered
.filter((c) => matchesVersion(r.version, c.version))
.slice(0, 20)
.map((c) => ({
...c,
scope: this.namespace(a, r.cwd) + ":" + c.scope,
requestedVersion,
}));
if (candidates.length) {
await this.cache.call("remember", {
key: lookup,
candidates,
capacity: this.options.cacheBytes,
});
if (
candidates.length === 1 &&
a.coverage !== "query" &&
!candidates[0]?.scope.includes(":query:")
)
await this.cache.call("remember", {
key: broadLookup,
candidates,
merge: true,
capacity: this.options.cacheBytes,
});
}
} catch {
r.signal?.throwIfAborted();
reasons.push(
a.id + ": discovery failed; using eligible cached discovery",
);
if (!candidates.length && a.coverage !== "query") {
candidates =
(
await this.cache.call<Discovery | null>("discovery", {
key: broadLookup,
})
)?.candidates ?? [];
fromBroad = true;
}
}
} else reasons.push(a.id + ": offline");
}
if (fromBroad) {
let selected: Candidate | undefined;
for (const candidate of candidates) {
if (!matchesVersion(r.version, candidate.version)) continue;
const hits = await this.cache.call<RawHit[]>("search", {
key: this.key(a, candidate),
query: r.query,
limit: 1,
});
if (hits.length) {
selected = candidate;
break;
}
}
candidates = selected ? [selected] : [];
}
if (candidates.length > 1) return { status: "ambiguous", candidates };
const c = candidates[0];
if (!c) {
reasons.push(a.id + ": target/version unavailable");
continue;
}
if (!r.version && a.id === "local" && c.version)
r = { ...r, version: c.version };
const key = this.key(a, c);
let head = await this.cache.call<Head | null>("head", { key });
let failed = false;
if (!head || refresh || a.rank === 1) {
try {
await waitForCaller(
this.update(a, c, r, key, head, refresh),
r.signal,
);
head = await this.cache.call<Head | null>("head", { key });
} catch (error) {
failed = true;
reasons.push(
a.id +
": " +
(error instanceof ProviderError
? error.code
: /capacity/.test(String(error))
? "capacity"
: "refresh failed") +
"; prior eligible content retained",
);
}
} else if (
Date.now() - head.checked >= this.options.ttlMs &&
!this.options.offline
) {
void this.update(a, c, r, key, head, false).catch(() => {});
reasons.push(a.id + ": stale content; background validation scheduled");
}
r.signal?.throwIfAborted();
if (!head) continue;
let hits = await this.cache.call<RawHit[]>("search", {
key,
query: r.query,
limit: 10,
});
if (
!hits.length &&
!(await this.cache.call<Head | null>("head", { key })) &&
(!this.options.offline || a.rank === 1)
) {
try {
await waitForCaller(this.update(a, c, r, key, null, true), r.signal);
hits = await this.cache.call<RawHit[]>("search", {
key,
query: r.query,
limit: 10,
});
await this.cache.call("remember", {
key: lookup,
candidates,
capacity: this.options.cacheBytes,
});
reasons.push(a.id + ": cache rebuilt after invalidation");
} catch {
reasons.push(a.id + ": invalidated cache could not be rebuilt");
}
}
if (!hits.length) {
reasons.push(a.id + ": no matching sections");
continue;
}
return {
status: "ok",
results: hits.map(
(h): SearchHit => ({
ref: h.revision + ":" + h.ordinal,
heading: h.heading,
snippet: h.snippet.replace(/\s+/g, " "),
target: h.target,
version: h.version,
source: h.source,
origin: h.origin,
freshness:
failed || Date.now() - h.checked >= this.options.ttlMs
? "stale"
: "fresh",
checkedAt: new Date(h.checked).toISOString(),
requestedVersion,
}),
),
reasons,
};
}
return { status: "unavailable", reasons };
}
async read(refs: string[], budget = 4000, cwd = ""): Promise<ReadResult> {
if (!Number.isInteger(budget) || budget < MIN_BUDGET || budget > 64000)
throw new Error("budget must be 512..64000 Unicode code-point tokens");
if (refs.length < 1 || refs.length > 20)
throw new Error("expected 1..20 refs");
await this.configure();
const parsed = refs.map((ref) => {
const m = /^([a-f0-9]{64}):(\d{1,5})(?::(\d{1,8}))?$/.exec(ref);
if (!m) throw new Error("invalid section ref");
return {
revision: m[1],
ordinal: Number(m[2]),
offset: Number(m[3] ?? 0),
};
});
const stored = await this.cache.call<(StoredSection | null)[]>("readMany", {
refs: parsed,
budget,
ttlMs: this.options.ttlMs,
});
const items: ReadSection[] = [];
for (let i = 0; i < refs.length; i++) {
const ref = refs[i] ?? "";
const section = stored[i];
if (!section) throw new Error("expired-ref: section no longer retained");
const adapter = this.adapters.find((a) => a.id === section.source);
if (!adapter) throw new Error("source-disabled: section not eligible");
if (!section.scope.startsWith(this.namespace(adapter, cwd) + ":"))
throw new Error(
"source-access: reference belongs to another source configuration or authorization context",
);
const offset = parsed[i]?.offset ?? 0;
const chars = [...section.body];
if (offset >= section.totalLength)
throw new Error("invalid continuation position");
const item: ReadSection = {
ref: ref.split(":").slice(0, 2).join(":"),
heading: section.heading,
target: section.target,
version: section.version,
source: section.source,
origin: section.origin,
checkedAt: section.checkedAt,
attribution: section.attribution,
content: "",
requestedVersion: section.requestedVersion,
freshness: section.freshness ?? "stale",
};
const base = section.revision + ":" + section.ordinal;
const build = (end: number): ReadResult => ({
status: "ok",
sections: [
...items,
{ ...item, content: chars.slice(0, end).join("") },
],
truncated: offset + end < section.totalLength || i + 1 < refs.length,
continuation:
offset + end < section.totalLength
? [base + ":" + (offset + end), ...refs.slice(i + 1)]
: refs.slice(i + 1),
});
let low = 0,
high = chars.length;
// A fitting penultimate prefix is a safe lower bound; completion can
// shrink JSON by removing its continuation, so a full-fit probe is not.
// Skip budget-sized bodies: their prefixes leave no room for metadata.
if (
high > 1 &&
high < budget &&
countTokens(JSON.stringify(build(high - 1))) <= budget
)
low = high - 1;
while (low < high) {
const mid = Math.ceil((low + high) / 2);
if (countTokens(JSON.stringify(build(mid))) <= budget) low = mid;
else high = mid - 1;
}
if (low === 0) {
if (items.length)
return {
status: "ok",
sections: items,
truncated: true,
continuation: refs.slice(i),
};
throw new Error(
"budget insufficient for provenance and content progress",
);
}
if (offset + low < section.totalLength) return build(low);
item.content = chars.slice(0, low).join("");
items.push(item);
}
return {
status: "ok",
sections: items,
truncated: false,
continuation: [],
};
}
async status(): Promise<
StatusResult & {
catalog: { id: string; home: string; format: string; adapter: string }[];
formats: { id: string; available: boolean }[];
}
> {
await this.configure();
return {
status: "ok",
tokenizer: "unicode-codepoint-v1",
minimumBudget: MIN_BUDGET,
sources: this.adapters.map((a) => ({
id: a.id,
rank: a.rank,
coverage: a.coverage,
})),
cache: await this.cache.status(),
offline: this.options.offline,
context7Mode: this.adapters.some((a) => a.id === "context7")
? this.context7Key
? "keyed"
: "anonymous"
: "disabled",
catalog: sourceCatalog.map((s) => ({
id: s.id,
home: s.home,
format: s.format,
adapter: s.adapter,
})),
formats: formatReaders.map((f) => ({ id: f.id, available: f.available })),
};
}
async close(): Promise<void> {
this.closed = true;
this.controller.abort();
await Promise.allSettled([...this.jobs.values()]);
await this.cache.close();
}
}