Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/bak/s3.ts

Raw
import { setTimeout as delay } from "node:timers/promises";
import { AwsClient } from "aws4fetch";
import type { S3Config } from "./config.js";

export { type S3Config, s3Config } from "./config.js";

const ERROR_TEXT_LIMIT = 500;
const MAX_ATTEMPTS = 3;
const REQUEST_TIMEOUT_MS = 30_000;

export interface StoredObject {
	body: Uint8Array;
	etag?: string | undefined;
}

export interface StoredRange {
	body: Uint8Array;
	etag: string;
	start: number;
	totalBytes: number;
}

export class S3Error extends Error {
	constructor(
		readonly status: number,
		message: string,
	) {
		super(message);
		this.name = "S3Error";
	}
}

type SignedClient = Pick<AwsClient, "fetch">;

function objectUrl(config: S3Config, key: string): string {
	const endpoint = config.endpoint.replace(/\/+$/, "");
	const path = [config.bucket, ...key.split("/")]
		.map((part) => encodeURIComponent(part))
		.join("/");
	return `${endpoint}/${path}`;
}

function shouldRetry(status: number): boolean {
	return status === 408 || status === 429 || status >= 500;
}

function strongEtag(value: string | null): string | undefined {
	const etag = value?.trim().replace(/^W\//, "");
	return etag || undefined;
}

export class S3SizeLimitError extends Error {
	constructor() {
		super("S3 response exceeds size limit");
		this.name = "S3SizeLimitError";
	}
}

async function readBody(
	response: Response,
	maxBytes?: number,
	signal?: AbortSignal,
	truncate = false,
): Promise<Uint8Array> {
	const contentLength = Number(response.headers.get("content-length"));
	if (
		!truncate &&
		maxBytes !== undefined &&
		Number.isFinite(contentLength) &&
		contentLength > maxBytes
	) {
		await response.body?.cancel();
		throw new S3SizeLimitError();
	}
	if (!response.body) return new Uint8Array();
	const reader = response.body.getReader();
	const chunks: Uint8Array[] = [];
	let length = 0;
	const abort = () => {
		void reader.cancel().catch(() => undefined);
	};
	signal?.addEventListener("abort", abort, { once: true });
	try {
		while (true) {
			signal?.throwIfAborted();
			const { done, value } = await reader.read();
			signal?.throwIfAborted();
			if (done) break;
			const remaining =
				maxBytes === undefined ? value.byteLength : maxBytes - length;
			if (!truncate && value.byteLength > remaining)
				throw new S3SizeLimitError();
			const chunk = truncate
				? value.subarray(0, Math.max(0, remaining))
				: value;
			chunks.push(chunk);
			length += chunk.byteLength;
			if (truncate && maxBytes !== undefined && length >= maxBytes) {
				await reader.cancel();
				break;
			}
		}
		const body = new Uint8Array(length);
		let offset = 0;
		for (const chunk of chunks) {
			body.set(chunk, offset);
			offset += chunk.byteLength;
		}
		return body;
	} catch (error) {
		await reader.cancel().catch(() => undefined);
		throw error;
	} finally {
		signal?.removeEventListener("abort", abort);
		reader.releaseLock();
	}
}

async function boundedError(
	response: Response,
	signal?: AbortSignal,
): Promise<string> {
	const body = await readBody(response, ERROR_TEXT_LIMIT + 1, signal, true);
	const text = new TextDecoder().decode(body).replace(/\s+/g, " ").trim();
	return (text || response.statusText || "request failed").slice(
		0,
		ERROR_TEXT_LIMIT,
	);
}

export class S3Store {
	private readonly client: SignedClient;

	constructor(
		readonly config: S3Config,
		client?: SignedClient,
	) {
		this.client =
			client ??
			new AwsClient({
				accessKeyId: config.accessKeyId,
				secretAccessKey: config.secretAccessKey,
				service: "s3",
				region: config.region,
				retries: 0,
			});
	}

	private async request(
		key: string,
		init: RequestInit,
		signal?: AbortSignal,
	): Promise<Response> {
		let lastError: unknown;
		for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
			signal?.throwIfAborted();
			try {
				const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
				const response = await this.client.fetch(objectUrl(this.config, key), {
					...init,
					signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
				});
				if (!shouldRetry(response.status) || attempt === MAX_ATTEMPTS - 1)
					return response;
				await response.body?.cancel();
			} catch (error) {
				lastError = error;
				signal?.throwIfAborted();
				if (attempt === MAX_ATTEMPTS - 1) throw error;
			}
			try {
				await delay(250 * 2 ** attempt, undefined, signal ? { signal } : {});
			} catch (error) {
				signal?.throwIfAborted();
				throw error;
			}
		}
		throw lastError instanceof Error
			? lastError
			: new Error("S3 request failed");
	}

	async get(
		key: string,
		options: {
			signal?: AbortSignal | undefined;
			maxBytes?: number | undefined;
		} = {},
	): Promise<StoredObject | undefined> {
		if (
			options.maxBytes !== undefined &&
			(!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 1)
		)
			throw new RangeError("Invalid response byte limit");
		const response = await this.request(
			key,
			{ method: "GET", headers: { "accept-encoding": "identity" } },
			options.signal,
		);
		if (response.status === 404) {
			await response.body?.cancel();
			return undefined;
		}
		if (!response.ok)
			throw new S3Error(
				response.status,
				await boundedError(response, options.signal),
			);
		return {
			body: await readBody(response, options.maxBytes, options.signal),
			etag: strongEtag(response.headers.get("etag")),
		};
	}

	/** Bounded inspection reads must never download an entire object as fallback. */
	async getRange(
		key: string,
		options: {
			start: number;
			endInclusive: number;
			etag?: string;
			signal?: AbortSignal;
		},
	): Promise<StoredRange> {
		const { start, endInclusive, etag, signal } = options;
		const maximum = endInclusive - start + 1;
		if (
			!Number.isSafeInteger(start) ||
			start < 0 ||
			!Number.isSafeInteger(endInclusive) ||
			maximum < 1 ||
			maximum > 128 * 1024
		)
			throw new RangeError("Invalid inspection byte range");
		if (etag !== undefined && !/^"[^"\r\n]+"$/.test(etag))
			throw new Error("Inspection requires a strong object validator");
		const response = await this.request(
			key,
			{
				method: "GET",
				headers: {
					"accept-encoding": "identity",
					range: `bytes=${start}-${endInclusive}`,
					...(etag === undefined ? {} : { "if-match": etag }),
				},
			},
			signal,
		);
		if (response.status !== 206) {
			await response.body?.cancel();
			if (response.status === 412)
				throw new S3Error(412, "Archive changed; restart inspection");
			if (response.status === 416)
				throw new S3Error(416, "Inspection offset is outside the archive");
			throw new S3Error(
				response.status,
				"Archive storage did not return the requested byte range",
			);
		}
		const match = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
			response.headers.get("content-range") ?? "",
		);
		const validator = response.headers.get("etag")?.trim();
		const encoding = response.headers.get("content-encoding");
		const first = Number(match?.[1]);
		const last = Number(match?.[2]);
		const totalBytes = Number(match?.[3]);
		if (
			!match ||
			!Number.isSafeInteger(totalBytes) ||
			totalBytes <= start ||
			first !== start ||
			last !== Math.min(endInclusive, totalBytes - 1) ||
			!validator ||
			!/^"[^"\r\n]+"$/.test(validator) ||
			(etag !== undefined && validator !== etag) ||
			(encoding !== null && encoding !== "identity")
		) {
			await response.body?.cancel();
			throw new Error(
				"Invalid archive byte-range response or object validator",
			);
		}
		const body = await readBody(response, last - first + 1, signal);
		if (body.byteLength !== last - first + 1)
			throw new Error("Incomplete archive byte-range response");
		return { body, etag: validator, start, totalBytes };
	}

	async put(
		key: string,
		body: string | Uint8Array,
		options: {
			contentType?: string | undefined;
			ifMatch?: string | undefined;
			ifNoneMatch?: string | undefined;
			signal?: AbortSignal | undefined;
		} = {},
	): Promise<string | undefined> {
		const headers = new Headers({
			"content-type": options.contentType ?? "application/octet-stream",
		});
		const ifMatch = strongEtag(options.ifMatch ?? null);
		if (ifMatch) headers.set("if-match", ifMatch);
		if (options.ifNoneMatch) headers.set("if-none-match", options.ifNoneMatch);
		const requestBody =
			typeof body === "string" ? body : Uint8Array.from(body).buffer;
		const response = await this.request(
			key,
			{
				method: "PUT",
				headers,
				body: requestBody,
			},
			options.signal,
		);
		if (!response.ok)
			throw new S3Error(
				response.status,
				await boundedError(response, options.signal),
			);
		await response.body?.cancel();
		return strongEtag(response.headers.get("etag"));
	}
}