Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/devdocs/src/containers.ts

Raw
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { inflate, inflateRaw } from "node:zlib";
import { type UnzipFileInfo, unzip } from "fflate";
import { readFormat } from "./formats.ts";
import { MAX_DOCUMENT_BYTES } from "./normalize.ts";
import type { Section } from "./types.ts";

const inflateAsync = promisify(inflate);
const inflateRawAsync = promisify(inflateRaw);

function safePath(name: string): boolean {
	// Reject encoded traversal too: locators may later become URLs.
	let decoded: string;
	try {
		decoded = decodeURIComponent(name);
	} catch {
		return false;
	}
	return (
		!!decoded &&
		decoded.length <= 2048 &&
		!/[\\\x00-\x1f:]/.test(decoded) &&
		!decoded.startsWith("/") &&
		!decoded.split("/").some((part) => part === ".." || part === ".")
	);
}

async function zipSections(
	input: Uint8Array,
	query: string,
	signal?: AbortSignal,
): Promise<Section[]> {
	const entries: UnzipFileInfo[] = [];
	const names = new Set<string>();
	let total = 0;
	// First pass inspects all central-directory headers without inflating anything.
	await new Promise<void>((resolve, reject) => {
		try {
			unzip(
				input,
				{
					filter: (entry) => {
						if (entries.length >= 10000)
							throw new Error("ZIP exceeds 10000 entry limit");
						if (
							!safePath(entry.name) ||
							names.has(decodeURIComponent(entry.name))
						)
							throw new Error("ZIP unsafe or duplicate path");
						names.add(decodeURIComponent(entry.name));
						total += entry.originalSize;
						if (
							!Number.isSafeInteger(entry.originalSize) ||
							entry.originalSize < 0 ||
							total > MAX_DOCUMENT_BYTES
						)
							throw new Error("ZIP expanded size exceeds 8 MiB limit");
						entries.push(entry);
						return false;
					},
				},
				(error) => (error ? reject(error) : resolve()),
			);
		} catch (error) {
			reject(error);
		}
	});
	// fflate exposes sizes but not offsets. Validate the bounded classic ZIP layout
	// before decoding with zlib: fflate's fixed output buffers silently truncate
	// forged-small originalSize values and still perform the excess inflate work.
	const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
	const u16 = (at: number) => view.getUint16(at, true);
	const u32 = (at: number) => view.getUint32(at, true);
	let end = input.length - 22;
	for (; end >= Math.max(0, input.length - 65557); end--) {
		if (u32(end) === 0x06054b50 && end + 22 + u16(end + 20) === input.length)
			break;
	}
	if (end < 0 || u32(end) !== 0x06054b50)
		throw new Error("Invalid ZIP end record");
	if (
		u16(end + 4) ||
		u16(end + 6) ||
		u16(end + 8) !== entries.length ||
		u16(end + 10) !== entries.length
	)
		throw new Error("Unsupported multidisk/ZIP64 archive");
	const directory = u32(end + 16);
	if (directory + u32(end + 12) !== end)
		throw new Error("Invalid ZIP directory bounds");
	let cursor = directory;
	const payloads = new Map<string, Uint8Array>();
	const ranges: { start: number; end: number }[] = [];
	for (const entry of entries) {
		if (cursor + 46 > end || u32(cursor) !== 0x02014b50)
			throw new Error("Invalid ZIP directory entry");
		const flags = u16(cursor + 8);
		const method = u16(cursor + 10);
		const nameSize = u16(cursor + 28);
		const next = cursor + 46 + nameSize + u16(cursor + 30) + u16(cursor + 32);
		const local = u32(cursor + 42);
		if (
			next > end ||
			local + 30 > directory ||
			u32(local) !== 0x04034b50 ||
			flags & 1 ||
			u16(cursor + 34) ||
			![0, 8].includes(method)
		)
			throw new Error("Unsupported or invalid ZIP entry");
		if (
			u16(local + 6) !== flags ||
			u16(local + 8) !== method ||
			u16(local + 26) !== nameSize ||
			u32(cursor + 20) !== entry.size ||
			u32(cursor + 24) !== entry.originalSize
		)
			throw new Error("ZIP header mismatch");
		if (
			!(flags & 8) &&
			(u32(local + 18) !== entry.size || u32(local + 22) !== entry.originalSize)
		)
			throw new Error("ZIP local size mismatch");
		for (let i = 0; i < nameSize; i++)
			if (input[local + 30 + i] !== input[cursor + 46 + i])
				throw new Error("ZIP filename mismatch");
		const start = local + 30 + nameSize + u16(local + 28);
		const finish = start + entry.size;
		if (
			finish > directory ||
			(method === 0 && entry.size !== entry.originalSize)
		)
			throw new Error("ZIP payload bounds or size mismatch");
		ranges.push({ start: local, end: finish });
		payloads.set(entry.name, input.subarray(start, finish));
		cursor = next;
	}
	if (cursor !== end) throw new Error("ZIP directory size mismatch");
	ranges.sort((a, b) => a.start - b.start);
	for (let i = 1; i < ranges.length; i++)
		if ((ranges[i - 1]?.end ?? 0) > (ranges[i]?.start ?? 0))
			throw new Error("Overlapping ZIP entries");
	signal?.throwIfAborted();
	const terms = query
		.toLowerCase()
		.split(/[^\p{L}\p{N}_]+/u)
		.filter(Boolean)
		.slice(0, 32);
	const score = (entry: UnzipFileInfo) =>
		terms.reduce(
			(sum, term) => sum + Number(entry.name.toLowerCase().includes(term)),
			0,
		);
	const selected = entries
		.filter((entry) => /\.html?$/i.test(entry.name))
		.sort((a, b) => score(b) - score(a) || a.name.localeCompare(b.name))
		.slice(0, 10);
	const result: Section[] = [];
	let emitted = 0;
	for (const entry of selected) {
		signal?.throwIfAborted();
		const payload = payloads.get(entry.name);
		if (!payload) throw new Error("ZIP payload missing");
		const data =
			entry.compression === 0
				? payload
				: await inflateRawAsync(payload, {
						maxOutputLength: Math.max(1, entry.originalSize),
					});
		signal?.throwIfAborted();
		if (data.byteLength !== entry.originalSize)
			throw new Error("ZIP entry size mismatch");
		for (const section of readFormat(
			new TextDecoder().decode(data),
			"javadoc",
		)) {
			const located = {
				...section,
				locator: `${entry.name.split("/").map(encodeURIComponent).join("/")}#${section.locator}`,
				heading: `${entry.name} > ${section.heading}`,
			};
			emitted +=
				Buffer.byteLength(located.body) +
				Buffer.byteLength(located.heading) +
				Buffer.byteLength(located.locator);
			result.push(located);
			if (emitted > MAX_DOCUMENT_BYTES)
				throw new Error("ZIP sections exceed 8 MiB limit");
			if (result.length > 10000) throw new Error("ZIP exceeds section limit");
		}
	}
	return result;
}

async function inventorySections(
	input: Uint8Array,
	query: string,
	signal?: AbortSignal,
): Promise<Section[]> {
	let offset = 0;
	const headers: string[] = [];
	for (let i = 0; i < 4; i++) {
		const end = input.indexOf(10, offset);
		if (end < 0 || end - offset > 4096)
			throw new Error("Invalid Sphinx inventory header");
		headers.push(
			new TextDecoder().decode(input.subarray(offset, end)).replace(/\r$/, ""),
		);
		offset = end + 1;
	}
	if (
		headers[0] !== "# Sphinx inventory version 2" ||
		!headers[1]?.startsWith("# Project: ") ||
		!headers[2]?.startsWith("# Version: ") ||
		!headers[3]?.includes("compressed using zlib")
	)
		throw new Error(
			"Unsupported Sphinx inventory; expected objects.inv v2 zlib",
		);
	const expanded = await inflateAsync(input.subarray(offset), {
		maxOutputLength: MAX_DOCUMENT_BYTES,
	});
	signal?.throwIfAborted();
	const text = new TextDecoder("utf-8", { fatal: true }).decode(expanded);
	const terms = query.toLowerCase().split(/\s+/).filter(Boolean).slice(0, 32);
	const result: Section[] = [];
	let emitted = 0;
	let count = 0;
	for (const line of text.split("\n")) {
		if (!line.trim()) continue;
		if (++count > 10000)
			throw new Error("Sphinx inventory exceeds 10000 entry limit");
		const match = /^(\S+)\s+(\S+:\S+)\s+(-?\d+)\s+(\S+)\s+(.+)$/.exec(line);
		if (!match) throw new Error("Invalid Sphinx inventory entry");
		const [, name = "", type = "", priority = "", rawUri = "", display = ""] =
			match;
		const uri = rawUri.endsWith("$") ? rawUri.slice(0, -1) + name : rawUri;
		if (
			(/^[a-z][a-z\d+.-]*:/i.test(uri) && !/^https?:\/\//i.test(uri)) ||
			uri.startsWith("//") ||
			/[\\\x00-\x1f]/.test(uri)
		)
			throw new Error("Unsafe Sphinx inventory URI");
		if (
			terms.length &&
			!terms.some((term) =>
				`${name} ${type} ${display} ${uri}`.toLowerCase().includes(term),
			)
		)
			continue;
		const section = {
			locator: uri,
			heading: display === "-" ? name : display,
			body: `Inventory only; discovery metadata, not documentation body.\n${headers[1]}\n${headers[2]}\nSymbol: ${name}\nType: ${type}\nPriority: ${priority}\nURI: ${uri}\nDisplay: ${display}`,
		};
		emitted +=
			Buffer.byteLength(section.body) +
			Buffer.byteLength(section.heading) +
			Buffer.byteLength(section.locator);
		if (emitted > MAX_DOCUMENT_BYTES)
			throw new Error("Sphinx sections exceed 8 MiB limit");
		result.push(section);
	}
	return result;
}

function pdfSections(
	input: Uint8Array,
	signal?: AbortSignal,
): Promise<Section[]> {
	return new Promise((resolve, reject) => {
		// No shell, paths, installation, remote scripts or caller-controlled arguments.
		const child = execFile(
			"pdftotext",
			["-layout", "-enc", "UTF-8", "-", "-"],
			{
				timeout: 10000,
				killSignal: "SIGKILL",
				maxBuffer: MAX_DOCUMENT_BYTES,
				encoding: "utf8",
				windowsHide: true,
				...(signal ? { signal } : {}),
			},
			(error, stdout) => {
				if (error) {
					reject(
						new Error(
							(error as NodeJS.ErrnoException).code === "ENOENT"
								? "PDF unavailable: pdftotext is not installed"
								: "PDF extraction failed, aborted, or exceeded deadline/output limit",
							{ cause: error },
						),
					);
				} else {
					try {
						resolve(readFormat(stdout, "text"));
					} catch (failure) {
						reject(failure);
					}
				}
			},
		);
		// EPIPE accompanies parser failures; execFile callback owns the diagnostic.
		child.stdin?.on("error", () => {});
		child.stdin?.end(input);
	});
}

/** Binary readers only; callers retain source attribution around returned sections. */
export async function readContainer(
	input: Uint8Array,
	format: "javadoc" | "sphinx" | "pdf",
	query: string,
	signal?: AbortSignal,
): Promise<Section[]> {
	signal?.throwIfAborted();
	if (input.byteLength > MAX_DOCUMENT_BYTES)
		throw new Error("container input exceeds 8 MiB limit");
	if (query.length > 4096) throw new Error("container query exceeds limit");
	if (format === "javadoc") return zipSections(input, query, signal);
	if (format === "sphinx") return inventorySections(input, query, signal);
	if (format === "pdf") return pdfSections(input, signal);
	throw new Error(`container format unavailable: ${String(format)}`);
}