repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/devdocs/src/containers.ts
Rawimport { execFile } from "node:child_process";
import { promisify } from "node:util";
import { inflate, inflateRaw } from "node:zlib";
import { type UnzipFileInfo, unzip } from "fflate";
import { readFormat } from "./formats.ts";
import { MAX_DOCUMENT_BYTES } from "./normalize.ts";
import type { Section } from "./types.ts";
const inflateAsync = promisify(inflate);
const inflateRawAsync = promisify(inflateRaw);
function safePath(name: string): boolean {
// Reject encoded traversal too: locators may later become URLs.
let decoded: string;
try {
decoded = decodeURIComponent(name);
} catch {
return false;
}
return (
!!decoded &&
decoded.length <= 2048 &&
!/[\\\x00-\x1f:]/.test(decoded) &&
!decoded.startsWith("/") &&
!decoded.split("/").some((part) => part === ".." || part === ".")
);
}
async function zipSections(
input: Uint8Array,
query: string,
signal?: AbortSignal,
): Promise<Section[]> {
const entries: UnzipFileInfo[] = [];
const names = new Set<string>();
let total = 0;
// First pass inspects all central-directory headers without inflating anything.
await new Promise<void>((resolve, reject) => {
try {
unzip(
input,
{
filter: (entry) => {
if (entries.length >= 10000)
throw new Error("ZIP exceeds 10000 entry limit");
if (
!safePath(entry.name) ||
names.has(decodeURIComponent(entry.name))
)
throw new Error("ZIP unsafe or duplicate path");
names.add(decodeURIComponent(entry.name));
total += entry.originalSize;
if (
!Number.isSafeInteger(entry.originalSize) ||
entry.originalSize < 0 ||
total > MAX_DOCUMENT_BYTES
)
throw new Error("ZIP expanded size exceeds 8 MiB limit");
entries.push(entry);
return false;
},
},
(error) => (error ? reject(error) : resolve()),
);
} catch (error) {
reject(error);
}
});
// fflate exposes sizes but not offsets. Validate the bounded classic ZIP layout
// before decoding with zlib: fflate's fixed output buffers silently truncate
// forged-small originalSize values and still perform the excess inflate work.
const view = new DataView(input.buffer, input.byteOffset, input.byteLength);
const u16 = (at: number) => view.getUint16(at, true);
const u32 = (at: number) => view.getUint32(at, true);
let end = input.length - 22;
for (; end >= Math.max(0, input.length - 65557); end--) {
if (u32(end) === 0x06054b50 && end + 22 + u16(end + 20) === input.length)
break;
}
if (end < 0 || u32(end) !== 0x06054b50)
throw new Error("Invalid ZIP end record");
if (
u16(end + 4) ||
u16(end + 6) ||
u16(end + 8) !== entries.length ||
u16(end + 10) !== entries.length
)
throw new Error("Unsupported multidisk/ZIP64 archive");
const directory = u32(end + 16);
if (directory + u32(end + 12) !== end)
throw new Error("Invalid ZIP directory bounds");
let cursor = directory;
const payloads = new Map<string, Uint8Array>();
const ranges: { start: number; end: number }[] = [];
for (const entry of entries) {
if (cursor + 46 > end || u32(cursor) !== 0x02014b50)
throw new Error("Invalid ZIP directory entry");
const flags = u16(cursor + 8);
const method = u16(cursor + 10);
const nameSize = u16(cursor + 28);
const next = cursor + 46 + nameSize + u16(cursor + 30) + u16(cursor + 32);
const local = u32(cursor + 42);
if (
next > end ||
local + 30 > directory ||
u32(local) !== 0x04034b50 ||
flags & 1 ||
u16(cursor + 34) ||
![0, 8].includes(method)
)
throw new Error("Unsupported or invalid ZIP entry");
if (
u16(local + 6) !== flags ||
u16(local + 8) !== method ||
u16(local + 26) !== nameSize ||
u32(cursor + 20) !== entry.size ||
u32(cursor + 24) !== entry.originalSize
)
throw new Error("ZIP header mismatch");
if (
!(flags & 8) &&
(u32(local + 18) !== entry.size || u32(local + 22) !== entry.originalSize)
)
throw new Error("ZIP local size mismatch");
for (let i = 0; i < nameSize; i++)
if (input[local + 30 + i] !== input[cursor + 46 + i])
throw new Error("ZIP filename mismatch");
const start = local + 30 + nameSize + u16(local + 28);
const finish = start + entry.size;
if (
finish > directory ||
(method === 0 && entry.size !== entry.originalSize)
)
throw new Error("ZIP payload bounds or size mismatch");
ranges.push({ start: local, end: finish });
payloads.set(entry.name, input.subarray(start, finish));
cursor = next;
}
if (cursor !== end) throw new Error("ZIP directory size mismatch");
ranges.sort((a, b) => a.start - b.start);
for (let i = 1; i < ranges.length; i++)
if ((ranges[i - 1]?.end ?? 0) > (ranges[i]?.start ?? 0))
throw new Error("Overlapping ZIP entries");
signal?.throwIfAborted();
const terms = query
.toLowerCase()
.split(/[^\p{L}\p{N}_]+/u)
.filter(Boolean)
.slice(0, 32);
const score = (entry: UnzipFileInfo) =>
terms.reduce(
(sum, term) => sum + Number(entry.name.toLowerCase().includes(term)),
0,
);
const selected = entries
.filter((entry) => /\.html?$/i.test(entry.name))
.sort((a, b) => score(b) - score(a) || a.name.localeCompare(b.name))
.slice(0, 10);
const result: Section[] = [];
let emitted = 0;
for (const entry of selected) {
signal?.throwIfAborted();
const payload = payloads.get(entry.name);
if (!payload) throw new Error("ZIP payload missing");
const data =
entry.compression === 0
? payload
: await inflateRawAsync(payload, {
maxOutputLength: Math.max(1, entry.originalSize),
});
signal?.throwIfAborted();
if (data.byteLength !== entry.originalSize)
throw new Error("ZIP entry size mismatch");
for (const section of readFormat(
new TextDecoder().decode(data),
"javadoc",
)) {
const located = {
...section,
locator: `${entry.name.split("/").map(encodeURIComponent).join("/")}#${section.locator}`,
heading: `${entry.name} > ${section.heading}`,
};
emitted +=
Buffer.byteLength(located.body) +
Buffer.byteLength(located.heading) +
Buffer.byteLength(located.locator);
result.push(located);
if (emitted > MAX_DOCUMENT_BYTES)
throw new Error("ZIP sections exceed 8 MiB limit");
if (result.length > 10000) throw new Error("ZIP exceeds section limit");
}
}
return result;
}
async function inventorySections(
input: Uint8Array,
query: string,
signal?: AbortSignal,
): Promise<Section[]> {
let offset = 0;
const headers: string[] = [];
for (let i = 0; i < 4; i++) {
const end = input.indexOf(10, offset);
if (end < 0 || end - offset > 4096)
throw new Error("Invalid Sphinx inventory header");
headers.push(
new TextDecoder().decode(input.subarray(offset, end)).replace(/\r$/, ""),
);
offset = end + 1;
}
if (
headers[0] !== "# Sphinx inventory version 2" ||
!headers[1]?.startsWith("# Project: ") ||
!headers[2]?.startsWith("# Version: ") ||
!headers[3]?.includes("compressed using zlib")
)
throw new Error(
"Unsupported Sphinx inventory; expected objects.inv v2 zlib",
);
const expanded = await inflateAsync(input.subarray(offset), {
maxOutputLength: MAX_DOCUMENT_BYTES,
});
signal?.throwIfAborted();
const text = new TextDecoder("utf-8", { fatal: true }).decode(expanded);
const terms = query.toLowerCase().split(/\s+/).filter(Boolean).slice(0, 32);
const result: Section[] = [];
let emitted = 0;
let count = 0;
for (const line of text.split("\n")) {
if (!line.trim()) continue;
if (++count > 10000)
throw new Error("Sphinx inventory exceeds 10000 entry limit");
const match = /^(\S+)\s+(\S+:\S+)\s+(-?\d+)\s+(\S+)\s+(.+)$/.exec(line);
if (!match) throw new Error("Invalid Sphinx inventory entry");
const [, name = "", type = "", priority = "", rawUri = "", display = ""] =
match;
const uri = rawUri.endsWith("$") ? rawUri.slice(0, -1) + name : rawUri;
if (
(/^[a-z][a-z\d+.-]*:/i.test(uri) && !/^https?:\/\//i.test(uri)) ||
uri.startsWith("//") ||
/[\\\x00-\x1f]/.test(uri)
)
throw new Error("Unsafe Sphinx inventory URI");
if (
terms.length &&
!terms.some((term) =>
`${name} ${type} ${display} ${uri}`.toLowerCase().includes(term),
)
)
continue;
const section = {
locator: uri,
heading: display === "-" ? name : display,
body: `Inventory only; discovery metadata, not documentation body.\n${headers[1]}\n${headers[2]}\nSymbol: ${name}\nType: ${type}\nPriority: ${priority}\nURI: ${uri}\nDisplay: ${display}`,
};
emitted +=
Buffer.byteLength(section.body) +
Buffer.byteLength(section.heading) +
Buffer.byteLength(section.locator);
if (emitted > MAX_DOCUMENT_BYTES)
throw new Error("Sphinx sections exceed 8 MiB limit");
result.push(section);
}
return result;
}
function pdfSections(
input: Uint8Array,
signal?: AbortSignal,
): Promise<Section[]> {
return new Promise((resolve, reject) => {
// No shell, paths, installation, remote scripts or caller-controlled arguments.
const child = execFile(
"pdftotext",
["-layout", "-enc", "UTF-8", "-", "-"],
{
timeout: 10000,
killSignal: "SIGKILL",
maxBuffer: MAX_DOCUMENT_BYTES,
encoding: "utf8",
windowsHide: true,
...(signal ? { signal } : {}),
},
(error, stdout) => {
if (error) {
reject(
new Error(
(error as NodeJS.ErrnoException).code === "ENOENT"
? "PDF unavailable: pdftotext is not installed"
: "PDF extraction failed, aborted, or exceeded deadline/output limit",
{ cause: error },
),
);
} else {
try {
resolve(readFormat(stdout, "text"));
} catch (failure) {
reject(failure);
}
}
},
);
// EPIPE accompanies parser failures; execFile callback owns the diagnostic.
child.stdin?.on("error", () => {});
child.stdin?.end(input);
});
}
/** Binary readers only; callers retain source attribution around returned sections. */
export async function readContainer(
input: Uint8Array,
format: "javadoc" | "sphinx" | "pdf",
query: string,
signal?: AbortSignal,
): Promise<Section[]> {
signal?.throwIfAborted();
if (input.byteLength > MAX_DOCUMENT_BYTES)
throw new Error("container input exceeds 8 MiB limit");
if (query.length > 4096) throw new Error("container query exceeds limit");
if (format === "javadoc") return zipSections(input, query, signal);
if (format === "sphinx") return inventorySections(input, query, signal);
if (format === "pdf") return pdfSections(input, signal);
throw new Error(`container format unavailable: ${String(format)}`);
}