import { setTimeout as delay } from "node:timers/promises"; import { AwsClient } from "aws4fetch"; import type { S3Config } from "./config.js"; export { type S3Config, s3Config } from "./config.js"; const ERROR_TEXT_LIMIT = 500; const MAX_ATTEMPTS = 3; const REQUEST_TIMEOUT_MS = 30_000; export interface StoredObject { body: Uint8Array; etag?: string | undefined; } export interface StoredRange { body: Uint8Array; etag: string; start: number; totalBytes: number; } export class S3Error extends Error { constructor( readonly status: number, message: string, ) { super(message); this.name = "S3Error"; } } type SignedClient = Pick; function objectUrl(config: S3Config, key: string): string { const endpoint = config.endpoint.replace(/\/+$/, ""); const path = [config.bucket, ...key.split("/")] .map((part) => encodeURIComponent(part)) .join("/"); return `${endpoint}/${path}`; } function shouldRetry(status: number): boolean { return status === 408 || status === 429 || status >= 500; } function strongEtag(value: string | null): string | undefined { const etag = value?.trim().replace(/^W\//, ""); return etag || undefined; } export class S3SizeLimitError extends Error { constructor() { super("S3 response exceeds size limit"); this.name = "S3SizeLimitError"; } } async function readBody( response: Response, maxBytes?: number, signal?: AbortSignal, truncate = false, ): Promise { const contentLength = Number(response.headers.get("content-length")); if ( !truncate && maxBytes !== undefined && Number.isFinite(contentLength) && contentLength > maxBytes ) { await response.body?.cancel(); throw new S3SizeLimitError(); } if (!response.body) return new Uint8Array(); const reader = response.body.getReader(); const chunks: Uint8Array[] = []; let length = 0; const abort = () => { void reader.cancel().catch(() => undefined); }; signal?.addEventListener("abort", abort, { once: true }); try { while (true) { signal?.throwIfAborted(); const { done, value } = await reader.read(); signal?.throwIfAborted(); if (done) break; const remaining = maxBytes === undefined ? value.byteLength : maxBytes - length; if (!truncate && value.byteLength > remaining) throw new S3SizeLimitError(); const chunk = truncate ? value.subarray(0, Math.max(0, remaining)) : value; chunks.push(chunk); length += chunk.byteLength; if (truncate && maxBytes !== undefined && length >= maxBytes) { await reader.cancel(); break; } } const body = new Uint8Array(length); let offset = 0; for (const chunk of chunks) { body.set(chunk, offset); offset += chunk.byteLength; } return body; } catch (error) { await reader.cancel().catch(() => undefined); throw error; } finally { signal?.removeEventListener("abort", abort); reader.releaseLock(); } } async function boundedError( response: Response, signal?: AbortSignal, ): Promise { const body = await readBody(response, ERROR_TEXT_LIMIT + 1, signal, true); const text = new TextDecoder().decode(body).replace(/\s+/g, " ").trim(); return (text || response.statusText || "request failed").slice( 0, ERROR_TEXT_LIMIT, ); } export class S3Store { private readonly client: SignedClient; constructor( readonly config: S3Config, client?: SignedClient, ) { this.client = client ?? new AwsClient({ accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey, service: "s3", region: config.region, retries: 0, }); } private async request( key: string, init: RequestInit, signal?: AbortSignal, ): Promise { let lastError: unknown; for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { signal?.throwIfAborted(); try { const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS); const response = await this.client.fetch(objectUrl(this.config, key), { ...init, signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }); if (!shouldRetry(response.status) || attempt === MAX_ATTEMPTS - 1) return response; await response.body?.cancel(); } catch (error) { lastError = error; signal?.throwIfAborted(); if (attempt === MAX_ATTEMPTS - 1) throw error; } try { await delay(250 * 2 ** attempt, undefined, signal ? { signal } : {}); } catch (error) { signal?.throwIfAborted(); throw error; } } throw lastError instanceof Error ? lastError : new Error("S3 request failed"); } async get( key: string, options: { signal?: AbortSignal | undefined; maxBytes?: number | undefined; } = {}, ): Promise { if ( options.maxBytes !== undefined && (!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 1) ) throw new RangeError("Invalid response byte limit"); const response = await this.request( key, { method: "GET", headers: { "accept-encoding": "identity" } }, options.signal, ); if (response.status === 404) { await response.body?.cancel(); return undefined; } if (!response.ok) throw new S3Error( response.status, await boundedError(response, options.signal), ); return { body: await readBody(response, options.maxBytes, options.signal), etag: strongEtag(response.headers.get("etag")), }; } /** Bounded inspection reads must never download an entire object as fallback. */ async getRange( key: string, options: { start: number; endInclusive: number; etag?: string; signal?: AbortSignal; }, ): Promise { const { start, endInclusive, etag, signal } = options; const maximum = endInclusive - start + 1; if ( !Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(endInclusive) || maximum < 1 || maximum > 128 * 1024 ) throw new RangeError("Invalid inspection byte range"); if (etag !== undefined && !/^"[^"\r\n]+"$/.test(etag)) throw new Error("Inspection requires a strong object validator"); const response = await this.request( key, { method: "GET", headers: { "accept-encoding": "identity", range: `bytes=${start}-${endInclusive}`, ...(etag === undefined ? {} : { "if-match": etag }), }, }, signal, ); if (response.status !== 206) { await response.body?.cancel(); if (response.status === 412) throw new S3Error(412, "Archive changed; restart inspection"); if (response.status === 416) throw new S3Error(416, "Inspection offset is outside the archive"); throw new S3Error( response.status, "Archive storage did not return the requested byte range", ); } const match = /^bytes (\d+)-(\d+)\/(\d+)$/.exec( response.headers.get("content-range") ?? "", ); const validator = response.headers.get("etag")?.trim(); const encoding = response.headers.get("content-encoding"); const first = Number(match?.[1]); const last = Number(match?.[2]); const totalBytes = Number(match?.[3]); if ( !match || !Number.isSafeInteger(totalBytes) || totalBytes <= start || first !== start || last !== Math.min(endInclusive, totalBytes - 1) || !validator || !/^"[^"\r\n]+"$/.test(validator) || (etag !== undefined && validator !== etag) || (encoding !== null && encoding !== "identity") ) { await response.body?.cancel(); throw new Error( "Invalid archive byte-range response or object validator", ); } const body = await readBody(response, last - first + 1, signal); if (body.byteLength !== last - first + 1) throw new Error("Incomplete archive byte-range response"); return { body, etag: validator, start, totalBytes }; } async put( key: string, body: string | Uint8Array, options: { contentType?: string | undefined; ifMatch?: string | undefined; ifNoneMatch?: string | undefined; signal?: AbortSignal | undefined; } = {}, ): Promise { const headers = new Headers({ "content-type": options.contentType ?? "application/octet-stream", }); const ifMatch = strongEtag(options.ifMatch ?? null); if (ifMatch) headers.set("if-match", ifMatch); if (options.ifNoneMatch) headers.set("if-none-match", options.ifNoneMatch); const requestBody = typeof body === "string" ? body : Uint8Array.from(body).buffer; const response = await this.request( key, { method: "PUT", headers, body: requestBody, }, options.signal, ); if (!response.ok) throw new S3Error( response.status, await boundedError(response, options.signal), ); await response.body?.cancel(); return strongEtag(response.headers.get("etag")); } }