Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/web/fetch.ts

Raw
// HTTP fetch helper — returns body as Buffer + content-type for routing

import { abortable } from "./abort.js";
import type { FetchResult } from "./constants.js";
import {
	BROWSER_HEADERS,
	DEFAULT_MAX_BYTES,
	DEFAULT_TIMEOUT_MS,
} from "./constants.js";
import { readBoundedText } from "./http-body.js";
import {
	assertSafeHttpUrl,
	type DnsResolver,
	resolveRedirectUrl,
} from "./safety.js";

/** A response the server refused; distinct from URL-safety, size, and transport failures. */
export class HttpStatusError extends Error {
	readonly status: number;

	constructor(status: number, statusText: string) {
		super(`HTTP ${status}: ${statusText}`);
		this.name = "HttpStatusError";
		this.status = status;
	}
}

export interface FetchUrlOptions {
	maxBytes?: number;
	maxRedirects?: number;
	timeoutMs?: number;
	resolver?: DnsResolver;
}

export async function fetchUrlStream(
	url: string,
	onProgress?: (loaded: number, total: number) => void,
	signal?: AbortSignal,
	options: FetchUrlOptions = {},
): Promise<FetchResult> {
	const deadline = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
	const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
	requestSignal.throwIfAborted();
	let safe = await abortable(
		assertSafeHttpUrl(url, {
			resolver: options.resolver,
			signal: requestSignal,
		}),
		requestSignal,
	);
	const maxRedirects = options.maxRedirects ?? 5;

	for (let redirects = 0; redirects <= maxRedirects; redirects++) {
		requestSignal.throwIfAborted();
		const response = await fetchOnce(safe.normalizedUrl, requestSignal);
		if (!isRedirect(response.status)) {
			return readResponse(
				response,
				safe.normalizedUrl,
				onProgress,
				options.maxBytes ?? DEFAULT_MAX_BYTES,
				requestSignal,
			);
		}
		const location = response.headers.get("location");
		await response.body?.cancel().catch(() => {});
		if (!location)
			throw new Error(`HTTP ${response.status}: redirect missing location`);
		if (redirects === maxRedirects)
			throw new Error(`Too many redirects (${maxRedirects})`);
		safe = await abortable(
			resolveRedirectUrl(safe.normalizedUrl, location, {
				resolver: options.resolver,
				signal: requestSignal,
			}),
			requestSignal,
		);
	}
	throw new Error(`Too many redirects (${maxRedirects})`);
}

async function fetchOnce(url: string, signal: AbortSignal): Promise<Response> {
	const parsed = new URL(url);
	return await fetch(url, {
		redirect: "manual",
		headers: {
			...BROWSER_HEADERS,
			Referer: `${parsed.protocol}//${parsed.host}/`,
		},
		signal,
	});
}

async function readResponse(
	response: Response,
	finalUrl: string,
	onProgress: ((loaded: number, total: number) => void) | undefined,
	maxBytes: number,
	signal: AbortSignal,
): Promise<FetchResult> {
	if (!response.ok) {
		await response.body?.cancel().catch(() => {});
		throw new HttpStatusError(response.status, response.statusText);
	}
	if (!response.body) throw new Error("No response body");
	const contentType = response.headers.get("content-type") ?? "";
	const contentLength =
		Number.parseInt(response.headers.get("content-length") ?? "0", 10) || 0;
	if (contentLength > maxBytes) {
		await response.body.cancel().catch(() => {});
		throw new Error(
			`Response exceeds maxBytes (${contentLength} > ${maxBytes})`,
		);
	}

	const reader = response.body.getReader();
	let loaded = 0;
	const chunks: Uint8Array[] = [];
	try {
		while (true) {
			signal.throwIfAborted();
			const { done, value } = await abortable(reader.read(), signal);
			signal.throwIfAborted();
			if (done) break;
			loaded += value.length;
			if (loaded > maxBytes)
				throw new Error(`Response exceeds maxBytes (${loaded} > ${maxBytes})`);
			chunks.push(value);
			onProgress?.(loaded, contentLength);
		}
	} catch (error) {
		await reader.cancel().catch(() => {});
		throw error;
	} finally {
		reader.releaseLock();
	}

	return {
		body: Buffer.concat(chunks),
		contentType,
		contentLength,
		status: response.status,
		finalUrl,
	};
}

function isRedirect(status: number): boolean {
	return status >= 300 && status < 400;
}

// Search pages use manual redirects so challenges cannot redirect into private networks.
export async function fetchSearchHtml(
	url: string,
	signal?: AbortSignal,
	timeout?: number,
	request: Pick<RequestInit, "method" | "headers" | "body"> = {},
): Promise<string> {
	const deadline = AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT_MS);
	const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
	requestSignal.throwIfAborted();
	await abortable(
		assertSafeHttpUrl(url, { signal: requestSignal }),
		requestSignal,
	);
	requestSignal.throwIfAborted();
	const response = await fetch(url, {
		...request,
		headers: request.headers ?? BROWSER_HEADERS,
		redirect: "manual",
		signal: requestSignal,
	});
	requestSignal.throwIfAborted();

	if (!response.ok) {
		await response.body?.cancel().catch(() => {});
		throw new Error(`HTTP ${response.status}: ${response.statusText}`);
	}
	return await readBoundedText(response, undefined, requestSignal);
}