repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/devdocs/src/adapters/local.ts
Rawimport { execFile } from "node:child_process";
import { readdir, readFile, realpath, stat } from "node:fs/promises";
import { dirname, isAbsolute, join, relative, sep } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { readContainer } from "../containers.ts";
import { docsetPaths } from "../docset.ts";
import { readFormat } from "../formats.ts";
import { cleanText, digest, MAX_DOCUMENT_BYTES } from "../normalize.ts";
import { symbolIndex } from "../symbols.ts";
import {
type Adapter,
matchesVersion,
type RegisteredSource,
} from "../types.ts";
import { upstreamAdapter } from "./upstream.ts";
export async function command(
file: string,
args: string[],
cwd: string,
signal?: AbortSignal,
): Promise<string> {
return new Promise((resolve, reject) => {
execFile(
file,
args,
{
cwd,
signal,
timeout: 5000,
maxBuffer: MAX_DOCUMENT_BYTES,
encoding: "utf8",
env: {
...process.env,
MANPAGER: "cat",
PAGER: "cat",
GIT_PAGER: "cat",
NO_COLOR: "1",
},
},
(err, stdout) =>
err
? reject(new Error("local documentation command unavailable"))
: resolve(cleanText(stdout)),
);
});
}
async function executable(name: string): Promise<string | null> {
for (const root of (process.env["PATH"] ?? "")
.split(process.platform === "win32" ? ";" : ":")
.slice(0, 100)) {
if (!isAbsolute(root)) continue;
for (const suffix of process.platform === "win32"
? [".exe", ".cmd", ""]
: [""]) {
try {
const p = await realpath(join(root, name + suffix));
if ((await stat(p)).isFile()) return p;
} catch {}
}
}
return null;
}
export function installedVersion(output: string): string | null {
return (
/(?:^|[^a-zA-Z0-9])v?(\d+\.\d+(?:\.\d+)?(?:[-+][\w.-]+)?)/.exec(
output,
)?.[1] ?? null
);
}
export function localAdapter(): Adapter {
return {
id: "local",
rank: 1,
formatVersion: 1,
coverage: "document",
async discover(r) {
const manual =
/^man:(?:([1-9][a-z]*):)?([a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99})$/.exec(
r.target,
);
const infoTarget = /^info:([a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99})$/.exec(
r.target,
);
if (manual || infoTarget) {
if (r.version) return [];
const name = manual?.[2] ?? infoTarget?.[1] ?? "";
const found = manual
? await command(
"man",
["-w", ...(manual[1] ? [manual[1]] : []), name],
r.cwd,
r.signal,
)
: await command("info", ["--where", name], r.cwd, r.signal);
const path = await realpath(found.trim().split("\n")[0] ?? "");
return [
{
id: digest(path),
target: r.target,
title: name,
version: null,
source: "local",
origin: pathToFileURL(path).href,
scope: manual ? "man" : "info",
evidence: null,
},
];
}
if (
!/^(?:@[a-zA-Z0-9_.-]+\/)?[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,99}$/.test(
r.target,
)
)
return [];
const path = r.target.includes("/") ? null : await executable(r.target);
if (!path) {
let current = r.cwd;
for (let depth = 0; depth < 20; depth++) {
try {
const root = join(current, "node_modules", r.target);
const metadata = join(root, "package.json");
if ((await stat(metadata)).size > 131072)
throw new Error("package metadata exceeds bounds");
const data: unknown = JSON.parse(await readFile(metadata, "utf8"));
const version =
data &&
typeof data === "object" &&
"version" in data &&
typeof data.version === "string"
? data.version
: null;
if (!matchesVersion(r.version, version)) return [];
for (const name of [
"README.md",
"readme.md",
"README",
"Readme.md",
]) {
try {
const doc = await realpath(join(root, name));
if (!(await stat(doc)).isFile()) continue;
return [
{
id: digest(doc),
target: r.target,
title: r.target,
version,
source: "local",
origin: pathToFileURL(doc).href,
scope: "package",
evidence: version ? "installed package metadata" : null,
},
];
} catch {}
}
} catch {}
const parent = dirname(current);
if (parent === current) break;
current = parent;
}
return [];
}
let version: string | null = null;
try {
version = installedVersion(
await command(path, ["--version"], r.cwd, r.signal),
);
} catch {}
if (!matchesVersion(r.version, version)) return [];
return [
{
id: digest(path),
target: r.target,
title: r.target,
version,
source: "local",
origin: pathToFileURL(path).href,
scope: "installed",
evidence: version ? "installed --version" : null,
},
];
},
async retrieve(c, r) {
const path = fileURLToPath(c.origin);
if (c.scope === "package") {
if ((await stat(path)).size > MAX_DOCUMENT_BYTES)
throw new Error("package documentation exceeds limit");
const text = await readFile(path, "utf8");
return {
candidate: c,
sections: readFormat(text, "markdown"),
attribution: "Installed package documentation",
validator: digest(text),
};
}
const text =
c.scope === "man"
? await command("man", ["-P", "cat", "-l", path], r.cwd, r.signal)
: c.scope === "info"
? await command(
"info",
["--output=-", "--file", path],
r.cwd,
r.signal,
)
: await command(path, ["--help"], r.cwd, r.signal);
if (!text.trim()) throw new Error("no local documentation");
return {
candidate: c,
sections: readFormat("# " + c.title + "\n\n" + text, "markdown"),
attribution: "Installed documentation",
validator: null,
};
},
};
}
export function fileAdapter(source: RegisteredSource): Adapter {
if (source.url) {
const remote = upstreamAdapter({
sources: [
{
id: source.id,
aliases: [],
domain: "registered documentation",
home: source.url,
index: /\/llms\.txt$/.test(new URL(source.url).pathname)
? source.url
: null,
format: source.format,
authority: "publisher",
role: "reference",
adapter: "upstream",
versionSupport:
"Configured documentation version; unknown when omitted",
description: source.target,
...(source.version
? {
version: source.version,
evidence: "configured documentation version",
}
: {}),
},
],
});
return {
id: source.id,
rank: 3,
formatVersion: 1,
coverage: "document",
async discover(request) {
if (
request.target !== source.target ||
!matchesVersion(request.version, source.version)
)
return [];
return (await remote.discover({ ...request, target: source.id })).map(
(candidate) => ({
...candidate,
target: source.target,
source: source.id,
scope:
(source.private ? digest(request.cwd) + ":" : "") +
candidate.scope,
}),
);
},
async retrieve(candidate, request, validator) {
const result = await remote.retrieve(
{ ...candidate, target: source.id, source: "upstream" },
request,
validator,
);
return result === "unchanged" ? result : { ...result, candidate };
},
};
}
return {
id: source.id,
rank: 1,
formatVersion: 1,
coverage: "document",
async discover(r) {
if (
r.target !== source.target ||
!matchesVersion(r.version, source.version)
)
return [];
const path = await realpath(source.path);
const info = await stat(path);
if (
(!info.isFile() && !info.isDirectory()) ||
info.size > MAX_DOCUMENT_BYTES
)
throw new Error("local documentation file exceeds bounds");
return [
{
id: digest(path),
target: source.target,
title: source.target,
version: source.version,
source: source.id,
origin: pathToFileURL(path).href,
scope:
(source.private ? digest(r.cwd) : "public") +
(info.isDirectory() ||
["javadoc", "sphinx", "docset", "doxygen"].includes(source.format)
? ":query:" + digest(r.query)
: ""),
evidence: source.version ? "configured documentation version" : null,
},
];
},
async retrieve(c, request, validator) {
const root = fileURLToPath(c.origin);
const info = await stat(root);
const paths: string[] = [];
if (info.isDirectory() && source.format === "docset") {
const documents = await realpath(
join(root, "Contents", "Resources", "Documents"),
);
const index = await realpath(
join(root, "Contents", "Resources", "docSet.dsidx"),
);
for (const path of await docsetPaths(
index,
request.query,
request.signal,
)) {
const located = path.split("#")[0] ?? "";
if (
!located ||
isAbsolute(located) ||
located.includes("\\") ||
located.split("/").some((part) => part === "..")
)
throw new Error("unsafe docset index path");
const candidate = await realpath(join(documents, located));
const inside = relative(documents, candidate);
if (
inside === ".." ||
inside.startsWith(".." + sep) ||
isAbsolute(inside)
)
throw new Error("docset page escapes document root");
if (!paths.includes(candidate)) paths.push(candidate);
}
} else if (
info.isDirectory() &&
["javadoc", "doxygen", "sphinx"].includes(source.format)
) {
const indexes =
source.format === "javadoc"
? [
"member-search-index.js",
"type-search-index.js",
"package-search-index.js",
"module-search-index.js",
]
: source.format === "sphinx"
? ["objects.inv"]
: ["index.xml", "index.tag", "doxygen.tag"];
for (const name of indexes) {
try {
const path = join(root, name);
if ((await stat(path)).isFile()) paths.push(path);
} catch {}
}
if (!paths.length) {
const fallback = join(root, "index.html");
if ((await stat(fallback)).isFile()) paths.push(fallback);
}
} else if (info.isDirectory()) {
const pending = [{ path: root, depth: 0 }];
let visited = 0;
while (pending.length) {
const directory = pending.shift();
if (!directory) break;
for (const entry of await readdir(directory.path, {
withFileTypes: true,
})) {
if (++visited > 1000)
throw new Error("documentation tree exceeds 1000 entries");
if (entry.isSymbolicLink()) continue;
const path = join(directory.path, entry.name);
if (
entry.isDirectory() &&
directory.depth < 4 &&
!["node_modules", ".git"].includes(entry.name)
)
pending.push({ path, depth: directory.depth + 1 });
else if (
entry.isFile() &&
/\.(?:md|mdx|rst|txt|html?|json|pdf|jar|zip)$/.test(entry.name)
)
paths.push(path);
}
}
const terms =
request.query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
const score = (path: string) =>
terms.reduce(
(sum, term) => sum + Number(path.toLowerCase().includes(term)),
0,
);
paths.sort((a, b) => score(b) - score(a) || a.localeCompare(b));
paths.splice(10);
} else paths.push(root);
const output: import("../types.ts").Section[] = [];
const fingerprints: string[] = [];
let bytes = 0;
for (const path of paths) {
request.signal?.throwIfAborted();
const canonical = await realpath(path);
const inside = relative(root, canonical);
if (
info.isDirectory() &&
(inside === ".." ||
inside.startsWith(".." + sep) ||
isAbsolute(inside))
)
throw new Error("documentation file escapes root");
const size = (await stat(canonical)).size;
bytes += size;
if (size > MAX_DOCUMENT_BYTES || bytes > MAX_DOCUMENT_BYTES)
throw new Error("local documentation exceeds 8 MiB");
const data = await readFile(canonical);
if (data.byteLength > MAX_DOCUMENT_BYTES)
throw new Error("local documentation exceeds 8 MiB");
fingerprints.push(
digest(JSON.stringify([canonical, data.toString("base64")])),
);
const indexSections =
source.format === "sphinx" && /objects\.inv$/.test(path)
? await readContainer(data, "sphinx", request.query, request.signal)
: source.format === "javadoc" && /-search-index\.js$/.test(path)
? symbolIndex(data.toString("utf8"), "javadoc", request.query)
: source.format === "doxygen" && /\.(?:xml|tag)$/.test(path)
? symbolIndex(data.toString("utf8"), "doxygen", request.query)
: null;
if (indexSections) {
const allowed = info.isDirectory() ? root : dirname(root);
for (const symbol of indexSections.slice(0, 3)) {
const located = decodeURIComponent(
symbol.locator.split("#")[0] ?? "",
);
if (
!located ||
isAbsolute(located) ||
located.includes("\\") ||
/^[a-z][a-z\d+.-]*:/i.test(located)
)
throw new Error("unsafe local index link");
const linked = await realpath(join(dirname(canonical), located));
const inside = relative(allowed, linked);
if (
inside === ".." ||
inside.startsWith(".." + sep) ||
isAbsolute(inside)
)
throw new Error("local index link escapes documentation root");
bytes += (await stat(linked)).size;
if (bytes > MAX_DOCUMENT_BYTES)
throw new Error("indexed documentation exceeds bounds");
const body = await readFile(linked, "utf8");
if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES)
throw new Error("indexed documentation exceeds bounds");
fingerprints.push(digest(JSON.stringify([linked, body])));
const anchor = decodeURIComponent(
symbol.locator.split("#")[1] ?? "",
);
const sections = readFormat(body, "html");
const exact = sections.filter((s) => s.locator === anchor);
output.push(
...(exact.length ? exact : sections).map((s) => ({
...s,
locator: symbol.locator + ":" + s.locator,
body:
s.body +
"\n\nSource: " +
pathToFileURL(linked).href +
(anchor ? "#" + encodeURIComponent(anchor) : ""),
})),
);
}
continue;
}
const binary =
source.format === "pdf" ||
(source.format === "sphinx" && /objects\.inv$/.test(path)) ||
(source.format === "javadoc" && /\.(?:jar|zip)$/.test(path));
const parsed = binary
? await readContainer(
data,
source.format === "pdf"
? "pdf"
: source.format === "sphinx"
? "sphinx"
: "javadoc",
request.query,
request.signal,
)
: readFormat(
data.toString("utf8"),
info.isDirectory()
? /\.html?$/.test(path)
? "html"
: /\.json$/.test(path)
? "json"
: /\.rst$/.test(path)
? "rst"
: /\.mdx$/.test(path)
? "mdx"
: "markdown"
: source.format,
);
output.push(
...parsed.map((section) => ({
...section,
locator: (info.isDirectory() ? inside + "#" : "") + section.locator,
body:
section.body +
(info.isDirectory()
? "\n\nSource: " + pathToFileURL(canonical).href
: ""),
})),
);
}
const fingerprint = digest(JSON.stringify([request.query, fingerprints]));
if (fingerprint === validator) return "unchanged";
return {
candidate: c,
sections: output,
attribution: "User-registered documentation",
validator: fingerprint,
};
},
};
}