repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/devdocs/src/formats.ts
Rawimport type { FormatId } from "./catalog.ts";
import { cleanText, MAX_DOCUMENT_BYTES } from "./normalize.ts";
import type { Section } from "./types.ts";
const entities: Record<string, string> = {
amp: "&",
lt: "<",
gt: ">",
quot: '"',
apos: "'",
nbsp: " ",
copy: "©",
reg: "®",
trade: "™",
ndash: "–",
mdash: "—",
hellip: "…",
laquo: "«",
raquo: "»",
lsquo: "‘",
rsquo: "’",
ldquo: "“",
rdquo: "”",
bull: "•",
middot: "·",
times: "×",
divide: "÷",
minus: "−",
le: "≤",
ge: "≥",
ne: "≠",
};
function decode(value: string): string {
return value.replace(
/&(#x[\da-f]+|#\d+|[a-z][\da-z]+);/gi,
(whole, name: string) => {
if (!name.startsWith("#")) return entities[name] ?? whole;
const point =
name[1]?.toLowerCase() === "x"
? Number.parseInt(name.slice(2), 16)
: Number(name.slice(1));
return point > 0 &&
point <= 0x10ffff &&
!(point >= 0xd800 && point <= 0xdfff)
? String.fromCodePoint(point)
: "�";
},
);
}
function bounded(value: string): void {
if (Buffer.byteLength(value) > MAX_DOCUMENT_BYTES)
throw new Error("format input exceeds 8 MiB limit");
}
function attribute(tag: string, name: string): string | undefined {
const match = new RegExp(
`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`,
"i",
).exec(tag);
return match ? decode(match[1] ?? match[2] ?? match[3] ?? "") : undefined;
}
function safeAnchor(value: string | undefined): string | undefined {
return value && /^[\w.():,$/[\]<>;%-]{1,512}$/.test(value)
? value
: undefined;
}
/** Bounded static extraction, not an HTML5 DOM or executable renderer. */
export function htmlMarkdown(html: string): string {
bounded(html);
const output: string[] = [];
const hidden = new Set([
"script",
"style",
"nav",
"template",
"noscript",
"aside",
"footer",
]);
let skip: string | undefined;
let skipDepth = 0;
let pre: string[] | undefined;
let inline: string[] | undefined;
let heading:
| { depth: number; id: string | undefined; text: string[] }
| undefined;
let pendingAnchor: string | undefined;
let tokens = 0;
const emit = (text: string) => {
if (pre) pre.push(text);
else if (inline) inline.push(text);
else if (heading) heading.text.push(text.replace(/\s+/g, " "));
else output.push(text);
};
const finishInline = () => {
if (!inline) return;
const value = inline.join("");
inline = undefined;
let width = 1;
for (const run of value.matchAll(/`+/g))
width = Math.max(width, run[0].length + 1);
const fence = "`".repeat(width);
const pad = value.startsWith("`") || value.endsWith("`") ? " " : "";
emit(`${fence}${pad}${value}${pad}${fence}`);
};
const finishHeading = () => {
finishInline();
if (!heading) return;
const title = heading.text.join("").trim();
output.push(
`\n\n${"#".repeat(heading.depth)} ${title}${heading.id ? ` {#${heading.id}}` : ""}\n\n`,
);
heading = undefined;
};
const finishPre = () => {
if (!pre) return;
const body = pre.join("").replace(/^\n|\n$/g, "");
let width = 3;
for (const run of body.matchAll(/`+/g))
width = Math.max(width, run[0].length + 1);
const fence = "`".repeat(width);
output.push(`\n\n${fence}\n${body}\n${fence}\n\n`);
pre = undefined;
};
// A quoted '>' stays inside its tag; malformed markup remains inert text.
for (const match of html.matchAll(
/<!--[\s\S]*?(?:-->|$)|<\/?[a-zA-Z][^<>"']*(?:(?:"[^"<]*"|'[^'<]*')[^<>"']*)*>|<![^<>]*>|[^<]+|</g,
)) {
if (++tokens > 200000) throw new Error("HTML token limit exceeded");
const token = match[0];
if (token.startsWith("<!")) continue;
const tag = /^<(\/?)\s*([a-z\d]+)/i.exec(token);
if (!tag) {
if (!skip) emit(pre ? decode(token) : decode(token).replace(/\s+/g, " "));
continue;
}
const closing = tag[1] === "/";
const name = tag[2]?.toLowerCase() ?? "";
if (skip) {
if (name === skip) {
if (closing) {
if (--skipDepth === 0) skip = undefined;
} else if (skip !== "script" && skip !== "style") skipDepth++;
}
continue;
}
if (!closing && hidden.has(name)) {
skip = name;
skipDepth = 1;
continue;
}
if (name === "pre") {
if (closing) finishPre();
else {
finishHeading();
pre = [];
}
continue;
}
if (pre) {
if (name === "br") pre.push("\n");
continue;
}
if (/^h[1-6]$/.test(name)) {
finishHeading();
if (!closing) {
heading = {
depth: Number(name[1]),
id: safeAnchor(attribute(token, "id")) ?? pendingAnchor,
text: [],
};
pendingAnchor = undefined;
}
continue;
}
if (!closing && ["a", "section", "div"].includes(name)) {
const id = safeAnchor(
attribute(token, "id") ??
(name === "a" ? attribute(token, "name") : undefined),
);
if (id) {
if (heading) heading.id ??= id;
else pendingAnchor = id;
}
}
if (name === "code" || name === "tt") {
if (closing) finishInline();
else if (!inline) inline = [];
continue;
}
if (name === "br") emit("\n");
else if (name === "li" && !closing) emit("\n- ");
else if (name === "td" || name === "th") emit(" | ");
else if (
[
"p",
"div",
"section",
"article",
"main",
"ul",
"ol",
"dl",
"dt",
"dd",
"tr",
"blockquote",
].includes(name)
)
emit("\n\n");
}
finishHeading();
finishPre();
finishInline();
// Do not collapse whitespace globally: preformatted code must survive byte-for-byte.
return cleanText(output.join("")).trim();
}
function staticMdx(input: string): string {
let fence: { char: string; width: number } | undefined;
let importing = false;
const lines = input.split(/\r?\n/).map((line) => {
const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
if (marker) {
const run = marker[1] ?? "";
if (!fence) fence = { char: run[0] ?? "`", width: run.length };
else if (
run[0] === fence.char &&
run.length >= fence.width &&
!marker[2]?.trim()
)
fence = undefined;
return line;
}
if (fence) return line;
if (/^import\s/.test(line)) importing = true;
if (importing) {
if (/;\s*$/.test(line) || /(?:from\s*)?["'][^"']+["']\s*$/.test(line))
importing = false;
return "";
}
// Inline code remains verbatim; ambiguous JSX/macros remain inert text.
return line
.split(/(`+[^`]*`+)/g)
.map((part, index) =>
index % 2
? part
: part.replace(
/<\/?[A-Za-z][\w.:-]*(?=[\s>]|\/>)(?:[^<>"'{}]|"[^"]*"|'[^']*'|\{[^{}]*\})*\/?>|<>|<\/>/g,
"",
),
)
.join("");
});
return `${lines.join("\n")}\n\n[MDX static reader: imports and JSX wrappers omitted; expressions and macros are unsupported inert text, never executed.]`;
}
function rstMarkdown(input: string): string {
const lines = input.replace(/\r\n?/g, "\n").split("\n");
const output: string[] = [];
const styles: string[] = [];
const adornment = (line: string) =>
/^([=~`^"+#*:_-])\1{2,}$/.test(line.trim());
for (let i = 0; i < lines.length; i++) {
const line = lines[i] ?? "";
const next = lines[i + 1] ?? "";
if (
adornment(line) &&
next.trim() &&
lines[i + 2]?.trim() === line.trim()
) {
const style = `over:${line.trim()[0]}`;
if (!styles.includes(style)) styles.push(style);
output.push(
`${"#".repeat(Math.min(6, styles.indexOf(style) + 1))} ${next.trim()}`,
);
i += 2;
} else if (
line.trim() &&
adornment(next) &&
next.trim().length >= line.trim().length
) {
const style = next.trim()[0] ?? "=";
if (!styles.includes(style)) styles.push(style);
output.push(
`${"#".repeat(Math.min(6, styles.indexOf(style) + 1))} ${line.trim()}`,
);
i++;
} else if (/^\s*\.\.\s+\S+::/.test(line) || /::$/.test(line)) {
const directive = /^\s*\.\.\s+\S+::/.test(line);
if (!directive) output.push(line.replace(/::$/, ":"));
const block = directive ? [line] : [];
while (
i + 1 < lines.length &&
(!lines[i + 1]?.trim() || /^\s+/.test(lines[i + 1] ?? ""))
)
block.push(lines[++i] ?? "");
let width = 3;
for (const run of block.join("\n").matchAll(/`+/g))
width = Math.max(width, run[0].length + 1);
const fence = "`".repeat(width);
output.push(`${fence}\n${block.join("\n")}\n${fence}`);
if (directive)
output.push(
"[reStructuredText directive unsupported; retained as literal text, never executed.]",
);
} else output.push(line);
}
return output.join("\n");
}
function jsonSections(input: string): Section[] {
const value: unknown = JSON.parse(input);
const result: Section[] = [];
const pending = [{ value, path: "" }];
let visited = 0;
while (pending.length) {
const node = pending.pop();
if (!node) break;
if (++visited > 20000 || node.path.length > 4096)
throw new Error("JSON registry structure exceeds limit");
const entries =
node.value && typeof node.value === "object" && !Array.isArray(node.value)
? Object.entries(node.value)
: [];
if (
entries.length &&
entries.every(
([, child]) =>
child !== null && typeof child === "object" && !Array.isArray(child),
)
) {
for (let i = entries.length - 1; i >= 0; i--) {
const entry = entries[i];
if (entry)
pending.push({
value: entry[1],
path: `${node.path}/${entry[0].replace(/~/g, "~0").replace(/\//g, "~1")}`,
});
}
} else
result.push({
locator: node.path || "/",
heading: node.path || "JSON registry",
body: JSON.stringify(node.value),
});
if (pending.length + result.length > 10000)
throw new Error("JSON registry exceeds section limit");
}
return result;
}
export function readFormat(input: string, format: FormatId): Section[] {
bounded(input);
if (format === "man") {
input = input
.replace(/^\.\\".*$/gm, "")
.replace(/^\.TH\s+(\S+).*$/gm, "# $1")
.replace(/^\.SH\s+"?([^\n]+?)"?\s*$/gm, "## $1")
.replace(/^\.SS\s+"?([^\n]+?)"?\s*$/gm, "### $1")
.replace(/^\.(?:B|I|BR|IR|RB|RI)\s+(.+)$/gm, "$1")
.replace(/^\.[A-Za-z][A-Za-z0-9]*.*$/gm, "")
.replace(/\\f[BRIP]|\\&/g, "")
.replace(/\\-/g, "-");
format = "markdown";
}
if (format === "info") {
input = input.replace(/^File:[^\n]*?Node:\s*([^,\n]+)[^\n]*$/gm, "# $1");
format = "markdown";
}
if (format === "help") format = "text";
if (format === "json") return jsonSections(input);
const html = ["html", "javadoc", "doxygen", "sphinx"].includes(format);
if (!html && !["markdown", "mdx", "rst", "text"].includes(format))
throw new Error(`format unavailable: ${format}`);
const normalized = cleanText(
html
? htmlMarkdown(input)
: format === "mdx"
? staticMdx(input)
: format === "rst"
? rstMarkdown(input)
: input,
);
if (format === "text")
return normalized.trim()
? [
{
locator: "document",
heading: "Documentation",
body: normalized.trim(),
},
]
: [];
// Track fence character and width: longer fences may contain shorter fences.
const result: Section[] = [];
const path: string[] = [];
let body: string[] = [];
let heading = "Documentation";
let locator: string | undefined;
let fence: { char: string; width: number } | undefined;
const flush = () => {
const text = body.join("\n").trim();
if (text.split("\n").some((line) => line.trim() && !/^#{1,6}\s/.test(line)))
result.push({
locator: locator ?? String(result.length),
heading,
body: text,
});
if (result.length > 10000)
throw new Error("document exceeds section limit");
body = [];
};
for (const line of normalized.replace(/\r\n?/g, "\n").split("\n")) {
const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
if (marker) {
const run = marker[1] ?? "";
if (!fence) fence = { char: run[0] ?? "`", width: run.length };
else if (
run[0] === fence.char &&
run.length >= fence.width &&
!marker[2]?.trim()
)
fence = undefined;
body.push(line);
continue;
}
const match = !fence && /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line);
if (!match) {
body.push(line);
continue;
}
flush();
const depth = match[1]?.length ?? 1;
const title = match[2] ?? "";
const anchor = / \{#([^{}]+)\}$/.exec(title);
const cleanTitle = anchor ? title.slice(0, anchor.index) : title;
path.length = depth - 1;
path[depth - 1] = cleanTitle;
heading = path.filter(Boolean).join(" > ");
locator = anchor?.[1];
body.push(`${match[1]} ${cleanTitle}`);
}
flush();
return result;
}