Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/devdocs/src/formats.ts

Raw
import type { FormatId } from "./catalog.ts";
import { cleanText, MAX_DOCUMENT_BYTES } from "./normalize.ts";
import type { Section } from "./types.ts";

const entities: Record<string, string> = {
	amp: "&",
	lt: "<",
	gt: ">",
	quot: '"',
	apos: "'",
	nbsp: " ",
	copy: "©",
	reg: "®",
	trade: "™",
	ndash: "–",
	mdash: "—",
	hellip: "…",
	laquo: "«",
	raquo: "»",
	lsquo: "‘",
	rsquo: "’",
	ldquo: "“",
	rdquo: "”",
	bull: "•",
	middot: "·",
	times: "×",
	divide: "÷",
	minus: "−",
	le: "≤",
	ge: "≥",
	ne: "≠",
};
function decode(value: string): string {
	return value.replace(
		/&(#x[\da-f]+|#\d+|[a-z][\da-z]+);/gi,
		(whole, name: string) => {
			if (!name.startsWith("#")) return entities[name] ?? whole;
			const point =
				name[1]?.toLowerCase() === "x"
					? Number.parseInt(name.slice(2), 16)
					: Number(name.slice(1));
			return point > 0 &&
				point <= 0x10ffff &&
				!(point >= 0xd800 && point <= 0xdfff)
				? String.fromCodePoint(point)
				: "�";
		},
	);
}
function bounded(value: string): void {
	if (Buffer.byteLength(value) > MAX_DOCUMENT_BYTES)
		throw new Error("format input exceeds 8 MiB limit");
}
function attribute(tag: string, name: string): string | undefined {
	const match = new RegExp(
		`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`,
		"i",
	).exec(tag);
	return match ? decode(match[1] ?? match[2] ?? match[3] ?? "") : undefined;
}
function safeAnchor(value: string | undefined): string | undefined {
	return value && /^[\w.():,$/[\]<>;%-]{1,512}$/.test(value)
		? value
		: undefined;
}

/** Bounded static extraction, not an HTML5 DOM or executable renderer. */
export function htmlMarkdown(html: string): string {
	bounded(html);
	const output: string[] = [];
	const hidden = new Set([
		"script",
		"style",
		"nav",
		"template",
		"noscript",
		"aside",
		"footer",
	]);
	let skip: string | undefined;
	let skipDepth = 0;
	let pre: string[] | undefined;
	let inline: string[] | undefined;
	let heading:
		| { depth: number; id: string | undefined; text: string[] }
		| undefined;
	let pendingAnchor: string | undefined;
	let tokens = 0;
	const emit = (text: string) => {
		if (pre) pre.push(text);
		else if (inline) inline.push(text);
		else if (heading) heading.text.push(text.replace(/\s+/g, " "));
		else output.push(text);
	};
	const finishInline = () => {
		if (!inline) return;
		const value = inline.join("");
		inline = undefined;
		let width = 1;
		for (const run of value.matchAll(/`+/g))
			width = Math.max(width, run[0].length + 1);
		const fence = "`".repeat(width);
		const pad = value.startsWith("`") || value.endsWith("`") ? " " : "";
		emit(`${fence}${pad}${value}${pad}${fence}`);
	};
	const finishHeading = () => {
		finishInline();
		if (!heading) return;
		const title = heading.text.join("").trim();
		output.push(
			`\n\n${"#".repeat(heading.depth)} ${title}${heading.id ? ` {#${heading.id}}` : ""}\n\n`,
		);
		heading = undefined;
	};
	const finishPre = () => {
		if (!pre) return;
		const body = pre.join("").replace(/^\n|\n$/g, "");
		let width = 3;
		for (const run of body.matchAll(/`+/g))
			width = Math.max(width, run[0].length + 1);
		const fence = "`".repeat(width);
		output.push(`\n\n${fence}\n${body}\n${fence}\n\n`);
		pre = undefined;
	};
	// A quoted '>' stays inside its tag; malformed markup remains inert text.
	for (const match of html.matchAll(
		/<!--[\s\S]*?(?:-->|$)|<\/?[a-zA-Z][^<>"']*(?:(?:"[^"<]*"|'[^'<]*')[^<>"']*)*>|<![^<>]*>|[^<]+|</g,
	)) {
		if (++tokens > 200000) throw new Error("HTML token limit exceeded");
		const token = match[0];
		if (token.startsWith("<!")) continue;
		const tag = /^<(\/?)\s*([a-z\d]+)/i.exec(token);
		if (!tag) {
			if (!skip) emit(pre ? decode(token) : decode(token).replace(/\s+/g, " "));
			continue;
		}
		const closing = tag[1] === "/";
		const name = tag[2]?.toLowerCase() ?? "";
		if (skip) {
			if (name === skip) {
				if (closing) {
					if (--skipDepth === 0) skip = undefined;
				} else if (skip !== "script" && skip !== "style") skipDepth++;
			}
			continue;
		}
		if (!closing && hidden.has(name)) {
			skip = name;
			skipDepth = 1;
			continue;
		}
		if (name === "pre") {
			if (closing) finishPre();
			else {
				finishHeading();
				pre = [];
			}
			continue;
		}
		if (pre) {
			if (name === "br") pre.push("\n");
			continue;
		}
		if (/^h[1-6]$/.test(name)) {
			finishHeading();
			if (!closing) {
				heading = {
					depth: Number(name[1]),
					id: safeAnchor(attribute(token, "id")) ?? pendingAnchor,
					text: [],
				};
				pendingAnchor = undefined;
			}
			continue;
		}
		if (!closing && ["a", "section", "div"].includes(name)) {
			const id = safeAnchor(
				attribute(token, "id") ??
					(name === "a" ? attribute(token, "name") : undefined),
			);
			if (id) {
				if (heading) heading.id ??= id;
				else pendingAnchor = id;
			}
		}
		if (name === "code" || name === "tt") {
			if (closing) finishInline();
			else if (!inline) inline = [];
			continue;
		}
		if (name === "br") emit("\n");
		else if (name === "li" && !closing) emit("\n- ");
		else if (name === "td" || name === "th") emit(" | ");
		else if (
			[
				"p",
				"div",
				"section",
				"article",
				"main",
				"ul",
				"ol",
				"dl",
				"dt",
				"dd",
				"tr",
				"blockquote",
			].includes(name)
		)
			emit("\n\n");
	}
	finishHeading();
	finishPre();
	finishInline();
	// Do not collapse whitespace globally: preformatted code must survive byte-for-byte.
	return cleanText(output.join("")).trim();
}

function staticMdx(input: string): string {
	let fence: { char: string; width: number } | undefined;
	let importing = false;
	const lines = input.split(/\r?\n/).map((line) => {
		const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
		if (marker) {
			const run = marker[1] ?? "";
			if (!fence) fence = { char: run[0] ?? "`", width: run.length };
			else if (
				run[0] === fence.char &&
				run.length >= fence.width &&
				!marker[2]?.trim()
			)
				fence = undefined;
			return line;
		}
		if (fence) return line;
		if (/^import\s/.test(line)) importing = true;
		if (importing) {
			if (/;\s*$/.test(line) || /(?:from\s*)?["'][^"']+["']\s*$/.test(line))
				importing = false;
			return "";
		}
		// Inline code remains verbatim; ambiguous JSX/macros remain inert text.
		return line
			.split(/(`+[^`]*`+)/g)
			.map((part, index) =>
				index % 2
					? part
					: part.replace(
							/<\/?[A-Za-z][\w.:-]*(?=[\s>]|\/>)(?:[^<>"'{}]|"[^"]*"|'[^']*'|\{[^{}]*\})*\/?>|<>|<\/>/g,
							"",
						),
			)
			.join("");
	});
	return `${lines.join("\n")}\n\n[MDX static reader: imports and JSX wrappers omitted; expressions and macros are unsupported inert text, never executed.]`;
}

function rstMarkdown(input: string): string {
	const lines = input.replace(/\r\n?/g, "\n").split("\n");
	const output: string[] = [];
	const styles: string[] = [];
	const adornment = (line: string) =>
		/^([=~`^"+#*:_-])\1{2,}$/.test(line.trim());
	for (let i = 0; i < lines.length; i++) {
		const line = lines[i] ?? "";
		const next = lines[i + 1] ?? "";
		if (
			adornment(line) &&
			next.trim() &&
			lines[i + 2]?.trim() === line.trim()
		) {
			const style = `over:${line.trim()[0]}`;
			if (!styles.includes(style)) styles.push(style);
			output.push(
				`${"#".repeat(Math.min(6, styles.indexOf(style) + 1))} ${next.trim()}`,
			);
			i += 2;
		} else if (
			line.trim() &&
			adornment(next) &&
			next.trim().length >= line.trim().length
		) {
			const style = next.trim()[0] ?? "=";
			if (!styles.includes(style)) styles.push(style);
			output.push(
				`${"#".repeat(Math.min(6, styles.indexOf(style) + 1))} ${line.trim()}`,
			);
			i++;
		} else if (/^\s*\.\.\s+\S+::/.test(line) || /::$/.test(line)) {
			const directive = /^\s*\.\.\s+\S+::/.test(line);
			if (!directive) output.push(line.replace(/::$/, ":"));
			const block = directive ? [line] : [];
			while (
				i + 1 < lines.length &&
				(!lines[i + 1]?.trim() || /^\s+/.test(lines[i + 1] ?? ""))
			)
				block.push(lines[++i] ?? "");
			let width = 3;
			for (const run of block.join("\n").matchAll(/`+/g))
				width = Math.max(width, run[0].length + 1);
			const fence = "`".repeat(width);
			output.push(`${fence}\n${block.join("\n")}\n${fence}`);
			if (directive)
				output.push(
					"[reStructuredText directive unsupported; retained as literal text, never executed.]",
				);
		} else output.push(line);
	}
	return output.join("\n");
}

function jsonSections(input: string): Section[] {
	const value: unknown = JSON.parse(input);
	const result: Section[] = [];
	const pending = [{ value, path: "" }];
	let visited = 0;
	while (pending.length) {
		const node = pending.pop();
		if (!node) break;
		if (++visited > 20000 || node.path.length > 4096)
			throw new Error("JSON registry structure exceeds limit");
		const entries =
			node.value && typeof node.value === "object" && !Array.isArray(node.value)
				? Object.entries(node.value)
				: [];
		if (
			entries.length &&
			entries.every(
				([, child]) =>
					child !== null && typeof child === "object" && !Array.isArray(child),
			)
		) {
			for (let i = entries.length - 1; i >= 0; i--) {
				const entry = entries[i];
				if (entry)
					pending.push({
						value: entry[1],
						path: `${node.path}/${entry[0].replace(/~/g, "~0").replace(/\//g, "~1")}`,
					});
			}
		} else
			result.push({
				locator: node.path || "/",
				heading: node.path || "JSON registry",
				body: JSON.stringify(node.value),
			});
		if (pending.length + result.length > 10000)
			throw new Error("JSON registry exceeds section limit");
	}
	return result;
}

export function readFormat(input: string, format: FormatId): Section[] {
	bounded(input);
	if (format === "man") {
		input = input
			.replace(/^\.\\".*$/gm, "")
			.replace(/^\.TH\s+(\S+).*$/gm, "# $1")
			.replace(/^\.SH\s+"?([^\n]+?)"?\s*$/gm, "## $1")
			.replace(/^\.SS\s+"?([^\n]+?)"?\s*$/gm, "### $1")
			.replace(/^\.(?:B|I|BR|IR|RB|RI)\s+(.+)$/gm, "$1")
			.replace(/^\.[A-Za-z][A-Za-z0-9]*.*$/gm, "")
			.replace(/\\f[BRIP]|\\&/g, "")
			.replace(/\\-/g, "-");
		format = "markdown";
	}
	if (format === "info") {
		input = input.replace(/^File:[^\n]*?Node:\s*([^,\n]+)[^\n]*$/gm, "# $1");
		format = "markdown";
	}
	if (format === "help") format = "text";
	if (format === "json") return jsonSections(input);
	const html = ["html", "javadoc", "doxygen", "sphinx"].includes(format);
	if (!html && !["markdown", "mdx", "rst", "text"].includes(format))
		throw new Error(`format unavailable: ${format}`);
	const normalized = cleanText(
		html
			? htmlMarkdown(input)
			: format === "mdx"
				? staticMdx(input)
				: format === "rst"
					? rstMarkdown(input)
					: input,
	);
	if (format === "text")
		return normalized.trim()
			? [
					{
						locator: "document",
						heading: "Documentation",
						body: normalized.trim(),
					},
				]
			: [];
	// Track fence character and width: longer fences may contain shorter fences.
	const result: Section[] = [];
	const path: string[] = [];
	let body: string[] = [];
	let heading = "Documentation";
	let locator: string | undefined;
	let fence: { char: string; width: number } | undefined;
	const flush = () => {
		const text = body.join("\n").trim();
		if (text.split("\n").some((line) => line.trim() && !/^#{1,6}\s/.test(line)))
			result.push({
				locator: locator ?? String(result.length),
				heading,
				body: text,
			});
		if (result.length > 10000)
			throw new Error("document exceeds section limit");
		body = [];
	};
	for (const line of normalized.replace(/\r\n?/g, "\n").split("\n")) {
		const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
		if (marker) {
			const run = marker[1] ?? "";
			if (!fence) fence = { char: run[0] ?? "`", width: run.length };
			else if (
				run[0] === fence.char &&
				run.length >= fence.width &&
				!marker[2]?.trim()
			)
				fence = undefined;
			body.push(line);
			continue;
		}
		const match = !fence && /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line);
		if (!match) {
			body.push(line);
			continue;
		}
		flush();
		const depth = match[1]?.length ?? 1;
		const title = match[2] ?? "";
		const anchor = / \{#([^{}]+)\}$/.exec(title);
		const cleanTitle = anchor ? title.slice(0, anchor.index) : title;
		path.length = depth - 1;
		path[depth - 1] = cleanTitle;
		heading = path.filter(Boolean).join(" > ");
		locator = anchor?.[1];
		body.push(`${match[1]} ${cleanTitle}`);
	}
	flush();
	return result;
}