repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/web/fetch.ts
Raw// HTTP fetch helper — returns body as Buffer + content-type for routing
import { abortable } from "./abort.js";
import type { FetchResult } from "./constants.js";
import {
BROWSER_HEADERS,
DEFAULT_MAX_BYTES,
DEFAULT_TIMEOUT_MS,
} from "./constants.js";
import { readBoundedText } from "./http-body.js";
import {
assertSafeHttpUrl,
type DnsResolver,
resolveRedirectUrl,
} from "./safety.js";
/** A response the server refused; distinct from URL-safety, size, and transport failures. */
export class HttpStatusError extends Error {
readonly status: number;
constructor(status: number, statusText: string) {
super(`HTTP ${status}: ${statusText}`);
this.name = "HttpStatusError";
this.status = status;
}
}
export interface FetchUrlOptions {
maxBytes?: number;
maxRedirects?: number;
timeoutMs?: number;
resolver?: DnsResolver;
}
export async function fetchUrlStream(
url: string,
onProgress?: (loaded: number, total: number) => void,
signal?: AbortSignal,
options: FetchUrlOptions = {},
): Promise<FetchResult> {
const deadline = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
requestSignal.throwIfAborted();
let safe = await abortable(
assertSafeHttpUrl(url, {
resolver: options.resolver,
signal: requestSignal,
}),
requestSignal,
);
const maxRedirects = options.maxRedirects ?? 5;
for (let redirects = 0; redirects <= maxRedirects; redirects++) {
requestSignal.throwIfAborted();
const response = await fetchOnce(safe.normalizedUrl, requestSignal);
if (!isRedirect(response.status)) {
return readResponse(
response,
safe.normalizedUrl,
onProgress,
options.maxBytes ?? DEFAULT_MAX_BYTES,
requestSignal,
);
}
const location = response.headers.get("location");
await response.body?.cancel().catch(() => {});
if (!location)
throw new Error(`HTTP ${response.status}: redirect missing location`);
if (redirects === maxRedirects)
throw new Error(`Too many redirects (${maxRedirects})`);
safe = await abortable(
resolveRedirectUrl(safe.normalizedUrl, location, {
resolver: options.resolver,
signal: requestSignal,
}),
requestSignal,
);
}
throw new Error(`Too many redirects (${maxRedirects})`);
}
async function fetchOnce(url: string, signal: AbortSignal): Promise<Response> {
const parsed = new URL(url);
return await fetch(url, {
redirect: "manual",
headers: {
...BROWSER_HEADERS,
Referer: `${parsed.protocol}//${parsed.host}/`,
},
signal,
});
}
async function readResponse(
response: Response,
finalUrl: string,
onProgress: ((loaded: number, total: number) => void) | undefined,
maxBytes: number,
signal: AbortSignal,
): Promise<FetchResult> {
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new HttpStatusError(response.status, response.statusText);
}
if (!response.body) throw new Error("No response body");
const contentType = response.headers.get("content-type") ?? "";
const contentLength =
Number.parseInt(response.headers.get("content-length") ?? "0", 10) || 0;
if (contentLength > maxBytes) {
await response.body.cancel().catch(() => {});
throw new Error(
`Response exceeds maxBytes (${contentLength} > ${maxBytes})`,
);
}
const reader = response.body.getReader();
let loaded = 0;
const chunks: Uint8Array[] = [];
try {
while (true) {
signal.throwIfAborted();
const { done, value } = await abortable(reader.read(), signal);
signal.throwIfAborted();
if (done) break;
loaded += value.length;
if (loaded > maxBytes)
throw new Error(`Response exceeds maxBytes (${loaded} > ${maxBytes})`);
chunks.push(value);
onProgress?.(loaded, contentLength);
}
} catch (error) {
await reader.cancel().catch(() => {});
throw error;
} finally {
reader.releaseLock();
}
return {
body: Buffer.concat(chunks),
contentType,
contentLength,
status: response.status,
finalUrl,
};
}
function isRedirect(status: number): boolean {
return status >= 300 && status < 400;
}
// Search pages use manual redirects so challenges cannot redirect into private networks.
export async function fetchSearchHtml(
url: string,
signal?: AbortSignal,
timeout?: number,
request: Pick<RequestInit, "method" | "headers" | "body"> = {},
): Promise<string> {
const deadline = AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT_MS);
const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
requestSignal.throwIfAborted();
await abortable(
assertSafeHttpUrl(url, { signal: requestSignal }),
requestSignal,
);
requestSignal.throwIfAborted();
const response = await fetch(url, {
...request,
headers: request.headers ?? BROWSER_HEADERS,
redirect: "manual",
signal: requestSignal,
});
requestSignal.throwIfAborted();
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
return await readBoundedText(response, undefined, requestSignal);
}