import type { FormatId } from "./catalog.ts"; import { cleanText, MAX_DOCUMENT_BYTES } from "./normalize.ts"; import type { Section } from "./types.ts"; const entities: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", copy: "©", reg: "®", trade: "™", ndash: "–", mdash: "—", hellip: "…", laquo: "«", raquo: "»", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bull: "•", middot: "·", times: "×", divide: "÷", minus: "−", le: "≤", ge: "≥", ne: "≠", }; function decode(value: string): string { return value.replace( /&(#x[\da-f]+|#\d+|[a-z][\da-z]+);/gi, (whole, name: string) => { if (!name.startsWith("#")) return entities[name] ?? whole; const point = name[1]?.toLowerCase() === "x" ? Number.parseInt(name.slice(2), 16) : Number(name.slice(1)); return point > 0 && point <= 0x10ffff && !(point >= 0xd800 && point <= 0xdfff) ? String.fromCodePoint(point) : "�"; }, ); } function bounded(value: string): void { if (Buffer.byteLength(value) > MAX_DOCUMENT_BYTES) throw new Error("format input exceeds 8 MiB limit"); } function attribute(tag: string, name: string): string | undefined { const match = new RegExp( `\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i", ).exec(tag); return match ? decode(match[1] ?? match[2] ?? match[3] ?? "") : undefined; } function safeAnchor(value: string | undefined): string | undefined { return value && /^[\w.():,$/[\]<>;%-]{1,512}$/.test(value) ? value : undefined; } /** Bounded static extraction, not an HTML5 DOM or executable renderer. */ export function htmlMarkdown(html: string): string { bounded(html); const output: string[] = []; const hidden = new Set([ "script", "style", "nav", "template", "noscript", "aside", "footer", ]); let skip: string | undefined; let skipDepth = 0; let pre: string[] | undefined; let inline: string[] | undefined; let heading: | { depth: number; id: string | undefined; text: string[] } | undefined; let pendingAnchor: string | undefined; let tokens = 0; const emit = (text: string) => { if (pre) pre.push(text); else if (inline) inline.push(text); else if (heading) heading.text.push(text.replace(/\s+/g, " ")); else output.push(text); }; const finishInline = () => { if (!inline) return; const value = inline.join(""); inline = undefined; let width = 1; for (const run of value.matchAll(/`+/g)) width = Math.max(width, run[0].length + 1); const fence = "`".repeat(width); const pad = value.startsWith("`") || value.endsWith("`") ? " " : ""; emit(`${fence}${pad}${value}${pad}${fence}`); }; const finishHeading = () => { finishInline(); if (!heading) return; const title = heading.text.join("").trim(); output.push( `\n\n${"#".repeat(heading.depth)} ${title}${heading.id ? ` {#${heading.id}}` : ""}\n\n`, ); heading = undefined; }; const finishPre = () => { if (!pre) return; const body = pre.join("").replace(/^\n|\n$/g, ""); let width = 3; for (const run of body.matchAll(/`+/g)) width = Math.max(width, run[0].length + 1); const fence = "`".repeat(width); output.push(`\n\n${fence}\n${body}\n${fence}\n\n`); pre = undefined; }; // A quoted '>' stays inside its tag; malformed markup remains inert text. for (const match of html.matchAll( /|$)|<\/?[a-zA-Z][^<>"']*(?:(?:"[^"<]*"|'[^'<]*')[^<>"']*)*>|]*>|[^<]+| 200000) throw new Error("HTML token limit exceeded"); const token = match[0]; if (token.startsWith(" { const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line); if (marker) { const run = marker[1] ?? ""; if (!fence) fence = { char: run[0] ?? "`", width: run.length }; else if ( run[0] === fence.char && run.length >= fence.width && !marker[2]?.trim() ) fence = undefined; return line; } if (fence) return line; if (/^import\s/.test(line)) importing = true; if (importing) { if (/;\s*$/.test(line) || /(?:from\s*)?["'][^"']+["']\s*$/.test(line)) importing = false; return ""; } // Inline code remains verbatim; ambiguous JSX/macros remain inert text. return line .split(/(`+[^`]*`+)/g) .map((part, index) => index % 2 ? part : part.replace( /<\/?[A-Za-z][\w.:-]*(?=[\s>]|\/>)(?:[^<>"'{}]|"[^"]*"|'[^']*'|\{[^{}]*\})*\/?>|<>|<\/>/g, "", ), ) .join(""); }); return `${lines.join("\n")}\n\n[MDX static reader: imports and JSX wrappers omitted; expressions and macros are unsupported inert text, never executed.]`; } function rstMarkdown(input: string): string { const lines = input.replace(/\r\n?/g, "\n").split("\n"); const output: string[] = []; const styles: string[] = []; const adornment = (line: string) => /^([=~`^"+#*:_-])\1{2,}$/.test(line.trim()); for (let i = 0; i < lines.length; i++) { const line = lines[i] ?? ""; const next = lines[i + 1] ?? ""; if ( adornment(line) && next.trim() && lines[i + 2]?.trim() === line.trim() ) { const style = `over:${line.trim()[0]}`; if (!styles.includes(style)) styles.push(style); output.push( `${"#".repeat(Math.min(6, styles.indexOf(style) + 1))} ${next.trim()}`, ); i += 2; } else if ( line.trim() && adornment(next) && next.trim().length >= line.trim().length ) { const style = next.trim()[0] ?? "="; if (!styles.includes(style)) styles.push(style); output.push( `${"#".repeat(Math.min(6, styles.indexOf(style) + 1))} ${line.trim()}`, ); i++; } else if (/^\s*\.\.\s+\S+::/.test(line) || /::$/.test(line)) { const directive = /^\s*\.\.\s+\S+::/.test(line); if (!directive) output.push(line.replace(/::$/, ":")); const block = directive ? [line] : []; while ( i + 1 < lines.length && (!lines[i + 1]?.trim() || /^\s+/.test(lines[i + 1] ?? "")) ) block.push(lines[++i] ?? ""); let width = 3; for (const run of block.join("\n").matchAll(/`+/g)) width = Math.max(width, run[0].length + 1); const fence = "`".repeat(width); output.push(`${fence}\n${block.join("\n")}\n${fence}`); if (directive) output.push( "[reStructuredText directive unsupported; retained as literal text, never executed.]", ); } else output.push(line); } return output.join("\n"); } function jsonSections(input: string): Section[] { const value: unknown = JSON.parse(input); const result: Section[] = []; const pending = [{ value, path: "" }]; let visited = 0; while (pending.length) { const node = pending.pop(); if (!node) break; if (++visited > 20000 || node.path.length > 4096) throw new Error("JSON registry structure exceeds limit"); const entries = node.value && typeof node.value === "object" && !Array.isArray(node.value) ? Object.entries(node.value) : []; if ( entries.length && entries.every( ([, child]) => child !== null && typeof child === "object" && !Array.isArray(child), ) ) { for (let i = entries.length - 1; i >= 0; i--) { const entry = entries[i]; if (entry) pending.push({ value: entry[1], path: `${node.path}/${entry[0].replace(/~/g, "~0").replace(/\//g, "~1")}`, }); } } else result.push({ locator: node.path || "/", heading: node.path || "JSON registry", body: JSON.stringify(node.value), }); if (pending.length + result.length > 10000) throw new Error("JSON registry exceeds section limit"); } return result; } export function readFormat(input: string, format: FormatId): Section[] { bounded(input); if (format === "man") { input = input .replace(/^\.\\".*$/gm, "") .replace(/^\.TH\s+(\S+).*$/gm, "# $1") .replace(/^\.SH\s+"?([^\n]+?)"?\s*$/gm, "## $1") .replace(/^\.SS\s+"?([^\n]+?)"?\s*$/gm, "### $1") .replace(/^\.(?:B|I|BR|IR|RB|RI)\s+(.+)$/gm, "$1") .replace(/^\.[A-Za-z][A-Za-z0-9]*.*$/gm, "") .replace(/\\f[BRIP]|\\&/g, "") .replace(/\\-/g, "-"); format = "markdown"; } if (format === "info") { input = input.replace(/^File:[^\n]*?Node:\s*([^,\n]+)[^\n]*$/gm, "# $1"); format = "markdown"; } if (format === "help") format = "text"; if (format === "json") return jsonSections(input); const html = ["html", "javadoc", "doxygen", "sphinx"].includes(format); if (!html && !["markdown", "mdx", "rst", "text"].includes(format)) throw new Error(`format unavailable: ${format}`); const normalized = cleanText( html ? htmlMarkdown(input) : format === "mdx" ? staticMdx(input) : format === "rst" ? rstMarkdown(input) : input, ); if (format === "text") return normalized.trim() ? [ { locator: "document", heading: "Documentation", body: normalized.trim(), }, ] : []; // Track fence character and width: longer fences may contain shorter fences. const result: Section[] = []; const path: string[] = []; let body: string[] = []; let heading = "Documentation"; let locator: string | undefined; let fence: { char: string; width: number } | undefined; const flush = () => { const text = body.join("\n").trim(); if (text.split("\n").some((line) => line.trim() && !/^#{1,6}\s/.test(line))) result.push({ locator: locator ?? String(result.length), heading, body: text, }); if (result.length > 10000) throw new Error("document exceeds section limit"); body = []; }; for (const line of normalized.replace(/\r\n?/g, "\n").split("\n")) { const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line); if (marker) { const run = marker[1] ?? ""; if (!fence) fence = { char: run[0] ?? "`", width: run.length }; else if ( run[0] === fence.char && run.length >= fence.width && !marker[2]?.trim() ) fence = undefined; body.push(line); continue; } const match = !fence && /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line); if (!match) { body.push(line); continue; } flush(); const depth = match[1]?.length ?? 1; const title = match[2] ?? ""; const anchor = / \{#([^{}]+)\}$/.exec(title); const cleanTitle = anchor ? title.slice(0, anchor.index) : title; path.length = depth - 1; path[depth - 1] = cleanTitle; heading = path.filter(Boolean).join(" > "); locator = anchor?.[1]; body.push(`${match[1]} ${cleanTitle}`); } flush(); return result; }