// HTTP fetch helper — returns body as Buffer + content-type for routing import type { FetchResult } from "./constants.js"; import { BROWSER_HEADERS, DEFAULT_MAX_BYTES, DEFAULT_TIMEOUT_MS, } from "./constants.js"; import { assertSafeHttpUrl, type DnsResolver, resolveRedirectUrl, } from "./safety.js"; export interface FetchUrlOptions { maxBytes?: number; maxRedirects?: number; timeoutMs?: number; resolver?: DnsResolver; } export async function fetchUrlStream( url: string, onProgress?: (loaded: number, total: number) => void, signal?: AbortSignal, options: FetchUrlOptions = {}, ): Promise { let safe = await assertSafeHttpUrl(url, { resolver: options.resolver }); const maxRedirects = options.maxRedirects ?? 5; const timeoutSignal = signal ?? AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS); for (let redirects = 0; redirects <= maxRedirects; redirects++) { const response = await fetchOnce(safe.normalizedUrl, timeoutSignal); if (!isRedirect(response.status)) { return readResponse( response, safe.normalizedUrl, onProgress, options.maxBytes ?? DEFAULT_MAX_BYTES, ); } const location = response.headers.get("location"); await response.body?.cancel().catch(() => {}); if (!location) throw new Error(`HTTP ${response.status}: redirect missing location`); if (redirects === maxRedirects) throw new Error(`Too many redirects (${maxRedirects})`); safe = await resolveRedirectUrl(safe.normalizedUrl, location, { resolver: options.resolver, }); } throw new Error(`Too many redirects (${maxRedirects})`); } async function fetchOnce(url: string, signal: AbortSignal): Promise { const parsed = new URL(url); return await fetch(url, { redirect: "manual", headers: { ...BROWSER_HEADERS, Referer: `${parsed.protocol}//${parsed.host}/`, }, signal, }); } async function readResponse( response: Response, finalUrl: string, onProgress: ((loaded: number, total: number) => void) | undefined, maxBytes: number, ): Promise { if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`); if (!response.body) throw new Error("No response body"); const contentType = response.headers.get("content-type") ?? ""; const contentLength = Number.parseInt(response.headers.get("content-length") ?? "0", 10) || 0; if (contentLength > maxBytes) { throw new Error( `Response exceeds maxBytes (${contentLength} > ${maxBytes})`, ); } const reader = response.body.getReader(); let loaded = 0; const chunks: Uint8Array[] = []; while (true) { const { done, value } = await reader.read(); if (done) break; loaded += value.length; if (loaded > maxBytes) { await reader.cancel().catch(() => {}); throw new Error(`Response exceeds maxBytes (${loaded} > ${maxBytes})`); } chunks.push(value); onProgress?.(loaded, contentLength); } return { body: Buffer.concat(chunks), contentType, contentLength, status: response.status, finalUrl, }; } function isRedirect(status: number): boolean { return status >= 300 && status < 400; } // Search pages use manual redirects so challenges cannot redirect into private networks. export async function fetchSearchHtml( url: string, signal?: AbortSignal, timeout?: number, request: Pick = {}, ): Promise { signal?.throwIfAborted(); await assertSafeHttpUrl(url); const deadline = AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT_MS); const response = await fetch(url, { ...request, headers: request.headers ?? BROWSER_HEADERS, redirect: "manual", signal: signal ? AbortSignal.any([signal, deadline]) : deadline, }); if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`); return response.text(); }