// HTTP fetch helper — returns body as Buffer + content-type for routing import { abortable } from "./abort.js"; import type { FetchResult } from "./constants.js"; import { BROWSER_HEADERS, DEFAULT_MAX_BYTES, DEFAULT_TIMEOUT_MS, } from "./constants.js"; import { readBoundedText } from "./http-body.js"; import { assertSafeHttpUrl, type DnsResolver, resolveRedirectUrl, } from "./safety.js"; /** A response the server refused; distinct from URL-safety, size, and transport failures. */ export class HttpStatusError extends Error { readonly status: number; constructor(status: number, statusText: string) { super(`HTTP ${status}: ${statusText}`); this.name = "HttpStatusError"; this.status = status; } } export interface FetchUrlOptions { maxBytes?: number; maxRedirects?: number; timeoutMs?: number; resolver?: DnsResolver; } export async function fetchUrlStream( url: string, onProgress?: (loaded: number, total: number) => void, signal?: AbortSignal, options: FetchUrlOptions = {}, ): Promise { const deadline = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS); const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; requestSignal.throwIfAborted(); let safe = await abortable( assertSafeHttpUrl(url, { resolver: options.resolver, signal: requestSignal, }), requestSignal, ); const maxRedirects = options.maxRedirects ?? 5; for (let redirects = 0; redirects <= maxRedirects; redirects++) { requestSignal.throwIfAborted(); const response = await fetchOnce(safe.normalizedUrl, requestSignal); if (!isRedirect(response.status)) { return readResponse( response, safe.normalizedUrl, onProgress, options.maxBytes ?? DEFAULT_MAX_BYTES, requestSignal, ); } const location = response.headers.get("location"); await response.body?.cancel().catch(() => {}); if (!location) throw new Error(`HTTP ${response.status}: redirect missing location`); if (redirects === maxRedirects) throw new Error(`Too many redirects (${maxRedirects})`); safe = await abortable( resolveRedirectUrl(safe.normalizedUrl, location, { resolver: options.resolver, signal: requestSignal, }), requestSignal, ); } throw new Error(`Too many redirects (${maxRedirects})`); } async function fetchOnce(url: string, signal: AbortSignal): Promise { const parsed = new URL(url); return await fetch(url, { redirect: "manual", headers: { ...BROWSER_HEADERS, Referer: `${parsed.protocol}//${parsed.host}/`, }, signal, }); } async function readResponse( response: Response, finalUrl: string, onProgress: ((loaded: number, total: number) => void) | undefined, maxBytes: number, signal: AbortSignal, ): Promise { if (!response.ok) { await response.body?.cancel().catch(() => {}); throw new HttpStatusError(response.status, response.statusText); } if (!response.body) throw new Error("No response body"); const contentType = response.headers.get("content-type") ?? ""; const contentLength = Number.parseInt(response.headers.get("content-length") ?? "0", 10) || 0; if (contentLength > maxBytes) { await response.body.cancel().catch(() => {}); throw new Error( `Response exceeds maxBytes (${contentLength} > ${maxBytes})`, ); } const reader = response.body.getReader(); let loaded = 0; const chunks: Uint8Array[] = []; try { while (true) { signal.throwIfAborted(); const { done, value } = await abortable(reader.read(), signal); signal.throwIfAborted(); if (done) break; loaded += value.length; if (loaded > maxBytes) throw new Error(`Response exceeds maxBytes (${loaded} > ${maxBytes})`); chunks.push(value); onProgress?.(loaded, contentLength); } } catch (error) { await reader.cancel().catch(() => {}); throw error; } finally { reader.releaseLock(); } return { body: Buffer.concat(chunks), contentType, contentLength, status: response.status, finalUrl, }; } function isRedirect(status: number): boolean { return status >= 300 && status < 400; } // Search pages use manual redirects so challenges cannot redirect into private networks. export async function fetchSearchHtml( url: string, signal?: AbortSignal, timeout?: number, request: Pick = {}, ): Promise { const deadline = AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT_MS); const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; requestSignal.throwIfAborted(); await abortable( assertSafeHttpUrl(url, { signal: requestSignal }), requestSignal, ); requestSignal.throwIfAborted(); const response = await fetch(url, { ...request, headers: request.headers ?? BROWSER_HEADERS, redirect: "manual", signal: requestSignal, }); requestSignal.throwIfAborted(); if (!response.ok) { await response.body?.cancel().catch(() => {}); throw new Error(`HTTP ${response.status}: ${response.statusText}`); } return await readBoundedText(response, undefined, requestSignal); }