repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/bak/s3.ts
Rawimport { setTimeout as delay } from "node:timers/promises";
import { AwsClient } from "aws4fetch";
import type { S3Config } from "./config.js";
export { type S3Config, s3Config } from "./config.js";
const ERROR_TEXT_LIMIT = 500;
const MAX_ATTEMPTS = 3;
const REQUEST_TIMEOUT_MS = 30_000;
export interface StoredObject {
body: Uint8Array;
etag?: string | undefined;
}
export interface StoredRange {
body: Uint8Array;
etag: string;
start: number;
totalBytes: number;
}
export class S3Error extends Error {
constructor(
readonly status: number,
message: string,
) {
super(message);
this.name = "S3Error";
}
}
type SignedClient = Pick<AwsClient, "fetch">;
function objectUrl(config: S3Config, key: string): string {
const endpoint = config.endpoint.replace(/\/+$/, "");
const path = [config.bucket, ...key.split("/")]
.map((part) => encodeURIComponent(part))
.join("/");
return `${endpoint}/${path}`;
}
function shouldRetry(status: number): boolean {
return status === 408 || status === 429 || status >= 500;
}
function strongEtag(value: string | null): string | undefined {
const etag = value?.trim().replace(/^W\//, "");
return etag || undefined;
}
export class S3SizeLimitError extends Error {
constructor() {
super("S3 response exceeds size limit");
this.name = "S3SizeLimitError";
}
}
async function readBody(
response: Response,
maxBytes?: number,
signal?: AbortSignal,
truncate = false,
): Promise<Uint8Array> {
const contentLength = Number(response.headers.get("content-length"));
if (
!truncate &&
maxBytes !== undefined &&
Number.isFinite(contentLength) &&
contentLength > maxBytes
) {
await response.body?.cancel();
throw new S3SizeLimitError();
}
if (!response.body) return new Uint8Array();
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let length = 0;
const abort = () => {
void reader.cancel().catch(() => undefined);
};
signal?.addEventListener("abort", abort, { once: true });
try {
while (true) {
signal?.throwIfAborted();
const { done, value } = await reader.read();
signal?.throwIfAborted();
if (done) break;
const remaining =
maxBytes === undefined ? value.byteLength : maxBytes - length;
if (!truncate && value.byteLength > remaining)
throw new S3SizeLimitError();
const chunk = truncate
? value.subarray(0, Math.max(0, remaining))
: value;
chunks.push(chunk);
length += chunk.byteLength;
if (truncate && maxBytes !== undefined && length >= maxBytes) {
await reader.cancel();
break;
}
}
const body = new Uint8Array(length);
let offset = 0;
for (const chunk of chunks) {
body.set(chunk, offset);
offset += chunk.byteLength;
}
return body;
} catch (error) {
await reader.cancel().catch(() => undefined);
throw error;
} finally {
signal?.removeEventListener("abort", abort);
reader.releaseLock();
}
}
async function boundedError(
response: Response,
signal?: AbortSignal,
): Promise<string> {
const body = await readBody(response, ERROR_TEXT_LIMIT + 1, signal, true);
const text = new TextDecoder().decode(body).replace(/\s+/g, " ").trim();
return (text || response.statusText || "request failed").slice(
0,
ERROR_TEXT_LIMIT,
);
}
export class S3Store {
private readonly client: SignedClient;
constructor(
readonly config: S3Config,
client?: SignedClient,
) {
this.client =
client ??
new AwsClient({
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
service: "s3",
region: config.region,
retries: 0,
});
}
private async request(
key: string,
init: RequestInit,
signal?: AbortSignal,
): Promise<Response> {
let lastError: unknown;
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
signal?.throwIfAborted();
try {
const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
const response = await this.client.fetch(objectUrl(this.config, key), {
...init,
signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
});
if (!shouldRetry(response.status) || attempt === MAX_ATTEMPTS - 1)
return response;
await response.body?.cancel();
} catch (error) {
lastError = error;
signal?.throwIfAborted();
if (attempt === MAX_ATTEMPTS - 1) throw error;
}
try {
await delay(250 * 2 ** attempt, undefined, signal ? { signal } : {});
} catch (error) {
signal?.throwIfAborted();
throw error;
}
}
throw lastError instanceof Error
? lastError
: new Error("S3 request failed");
}
async get(
key: string,
options: {
signal?: AbortSignal | undefined;
maxBytes?: number | undefined;
} = {},
): Promise<StoredObject | undefined> {
if (
options.maxBytes !== undefined &&
(!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 1)
)
throw new RangeError("Invalid response byte limit");
const response = await this.request(
key,
{ method: "GET", headers: { "accept-encoding": "identity" } },
options.signal,
);
if (response.status === 404) {
await response.body?.cancel();
return undefined;
}
if (!response.ok)
throw new S3Error(
response.status,
await boundedError(response, options.signal),
);
return {
body: await readBody(response, options.maxBytes, options.signal),
etag: strongEtag(response.headers.get("etag")),
};
}
/** Bounded inspection reads must never download an entire object as fallback. */
async getRange(
key: string,
options: {
start: number;
endInclusive: number;
etag?: string;
signal?: AbortSignal;
},
): Promise<StoredRange> {
const { start, endInclusive, etag, signal } = options;
const maximum = endInclusive - start + 1;
if (
!Number.isSafeInteger(start) ||
start < 0 ||
!Number.isSafeInteger(endInclusive) ||
maximum < 1 ||
maximum > 128 * 1024
)
throw new RangeError("Invalid inspection byte range");
if (etag !== undefined && !/^"[^"\r\n]+"$/.test(etag))
throw new Error("Inspection requires a strong object validator");
const response = await this.request(
key,
{
method: "GET",
headers: {
"accept-encoding": "identity",
range: `bytes=${start}-${endInclusive}`,
...(etag === undefined ? {} : { "if-match": etag }),
},
},
signal,
);
if (response.status !== 206) {
await response.body?.cancel();
if (response.status === 412)
throw new S3Error(412, "Archive changed; restart inspection");
if (response.status === 416)
throw new S3Error(416, "Inspection offset is outside the archive");
throw new S3Error(
response.status,
"Archive storage did not return the requested byte range",
);
}
const match = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
response.headers.get("content-range") ?? "",
);
const validator = response.headers.get("etag")?.trim();
const encoding = response.headers.get("content-encoding");
const first = Number(match?.[1]);
const last = Number(match?.[2]);
const totalBytes = Number(match?.[3]);
if (
!match ||
!Number.isSafeInteger(totalBytes) ||
totalBytes <= start ||
first !== start ||
last !== Math.min(endInclusive, totalBytes - 1) ||
!validator ||
!/^"[^"\r\n]+"$/.test(validator) ||
(etag !== undefined && validator !== etag) ||
(encoding !== null && encoding !== "identity")
) {
await response.body?.cancel();
throw new Error(
"Invalid archive byte-range response or object validator",
);
}
const body = await readBody(response, last - first + 1, signal);
if (body.byteLength !== last - first + 1)
throw new Error("Incomplete archive byte-range response");
return { body, etag: validator, start, totalBytes };
}
async put(
key: string,
body: string | Uint8Array,
options: {
contentType?: string | undefined;
ifMatch?: string | undefined;
ifNoneMatch?: string | undefined;
signal?: AbortSignal | undefined;
} = {},
): Promise<string | undefined> {
const headers = new Headers({
"content-type": options.contentType ?? "application/octet-stream",
});
const ifMatch = strongEtag(options.ifMatch ?? null);
if (ifMatch) headers.set("if-match", ifMatch);
if (options.ifNoneMatch) headers.set("if-none-match", options.ifNoneMatch);
const requestBody =
typeof body === "string" ? body : Uint8Array.from(body).buffer;
const response = await this.request(
key,
{
method: "PUT",
headers,
body: requestBody,
},
options.signal,
);
if (!response.ok)
throw new S3Error(
response.status,
await boundedError(response, options.signal),
);
await response.body?.cancel();
return strongEtag(response.headers.get("etag"));
}
}