Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/bak/__tests__/raw.test.ts

Raw
import { Value } from "typebox/value";
import { describe, expect, it } from "vitest";
import { BakArchive } from "../archive.js";
import { BakQuery } from "../query.js";
import { S3Store } from "../s3.js";
import { statePaths } from "../state.js";
import { BakOutput, type BakRawResult, type BakResult } from "../tool.js";

const HOST = "12345678-1234-1234-1234-123456789abc";
const SESSION = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const DATE = "2026-01-01T00:00:00.000Z";
const HEADER = JSON.stringify({
	type: "session",
	id: SESSION,
	version: 3,
	timestamp: DATE,
	cwd: "/project",
	parentSession: "/parent.jsonl",
});
const CONFIG = {
	endpoint: "https://example.invalid",
	bucket: "bucket",
	region: "auto",
	accessKeyId: "fake-id",
	secretAccessKey: "fake-secret",
};

type Source = {
	size: number;
	slice: (start: number, end: number) => Uint8Array;
};
function reader(
	body: Buffer | Source,
	secrets: string[] = [],
	now = () => 0,
	metadataCwd = "/project",
) {
	const source: Source = Buffer.isBuffer(body)
		? { size: body.length, slice: (start, end) => body.subarray(start, end) }
		: body;
	const requests: Array<{
		key: string;
		range: string | null;
		ifMatch: string | null;
	}> = [];
	const state: { etag: string; status?: number } = { etag: '"version-one"' };
	const client = {
		fetch: async (input: RequestInfo | URL, init?: RequestInit) => {
			const request = new Request(input, init);
			expect(request.method).toBe("GET");
			const key = decodeURIComponent(
				new URL(request.url).pathname.slice("/bucket/".length),
			);
			const range = request.headers.get("range");
			requests.push({ key, range, ifMatch: request.headers.get("if-match") });
			if (key === "catalog/hosts.json")
				return new Response(
					JSON.stringify({
						version: 1,
						hosts: [{ hostId: HOST, alias: "desktop" }],
					}),
				);
			if (key === `hosts/${HOST}.json`)
				return new Response(
					JSON.stringify({
						version: 1,
						hostId: HOST,
						alias: "desktop",
						sessions: [
							{
								id: SESSION,
								key: `sessions/${HOST}/${SESSION}.jsonl`,
								cwd: metadataCwd,
								created: DATE,
							},
						],
					}),
				);
			expect(key).toBe(`sessions/${HOST}/${SESSION}.jsonl`);
			if (state.status) return new Response(null, { status: state.status });
			if (!range)
				throw new Error("whole-session GET forbidden in this fixture");
			if (
				request.headers.has("if-match") &&
				request.headers.get("if-match") !== state.etag
			)
				return new Response(null, { status: 412 });
			const match = /^bytes=(\d+)-(\d+)$/.exec(range)!;
			const start = Number(match[1]);
			const end = Math.min(Number(match[2]), source.size - 1);
			if (start >= source.size) return new Response(null, { status: 416 });
			return new Response(
				Uint8Array.from(source.slice(start, end + 1)).buffer,
				{
					status: 206,
					headers: {
						etag: state.etag,
						"content-range": `bytes ${start}-${end}/${source.size}`,
					},
				},
			);
		},
	};
	const query = new BakQuery(
		new BakArchive(new S3Store(CONFIG, client), statePaths()),
		secrets,
		now,
	);
	return { query, requests, state };
}

function raw(result: BakResult): BakRawResult {
	expect(Value.Check(BakOutput, result)).toBe(true);
	expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThanOrEqual(
		48 * 1024,
	);
	if (result.action !== "read" || result.view !== "raw-jsonl")
		throw new Error("expected raw JSONL");
	return result;
}

async function collect(query: BakQuery): Promise<string> {
	let page = raw(
		await query.execute({
			action: "read",
			view: "raw-jsonl",
			hostId: HOST,
			sessionId: SESSION,
		}),
	);
	let text = page.text;
	while (page.nextCursor) {
		const previousEnd = page.endByteExclusive;
		page = raw(
			await query.execute({ action: "read", cursor: page.nextCursor }),
		);
		expect(page.startByte).toBe(previousEnd);
		expect(page.endByteExclusive).toBeGreaterThan(page.startByte);
		text += page.text;
	}
	expect(page.endByteExclusive).toBe(page.totalBytes);
	return text;
}

describe("complete archived JSONL inspection", () => {
	it("reconstructs every field, unknown types, media and header without whole-file parsing", async () => {
		const entries = [
			{
				type: "message",
				message: {
					role: "system",
					content: "system prompt",
					toolsAdded: [{ name: "tool", parameters: {} }],
				},
			},
			{
				type: "message",
				message: {
					role: "assistant",
					content: [
						{
							type: "thinking",
							thinking: "reasoning",
							thinkingSignature: "signature",
						},
						{ type: "toolCall", arguments: { path: "/exact/path" } },
					],
					usage: { cost: { total: 12.34 } },
				},
			},
			{
				type: "message",
				message: {
					role: "toolResult",
					content: [
						{ type: "image", data: "base64-image" },
						{ type: "audio", data: "base64-audio" },
					],
					details: { private: true },
					structuredContent: { result: "structured" },
				},
			},
			{
				type: "compaction",
				details: {
					decay: { atoms: [{ key: "rule", allocation: 0 }], elapsedMs: 12345 },
				},
				summary: "checkpoint",
			},
			{
				type: "custom",
				customType: "decay-diagnostic",
				data: { stage: "summarizer", reason: "capacity exceeded" },
			},
			{
				type: "custom_message",
				display: false,
				content: "hidden",
				details: { secret: "state" },
			},
			{ type: "context_edit", targetId: "old", replacement: null },
			{
				type: "future-extension-entry",
				unknown: { nested: ["😀", '\\"\n'.repeat(12000)] },
			},
		];
		const body = `${HEADER}\n${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n{"type":"future","hugeNumber":900719925474099312345,"duplicate":1,"duplicate":2}\n`;
		const { query, requests } = reader(Buffer.from(body));
		expect(await collect(query)).toBe(body);
		expect(
			requests
				.filter((request) => request.key.startsWith("sessions/"))
				.every((request) => request.range !== null),
		).toBe(true);
		expect(
			requests.filter((request) => request.key === "catalog/hosts.json"),
		).toHaveLength(1);
	});

	it("inspects a 40 MiB entry near its end with only bounded range reads", async () => {
		const prefix = Buffer.from(`${HEADER}\n{"type":"custom","data":"`);
		const payloadSize = 40 * 1024 * 1024;
		const suffix = Buffer.from(
			'"}\n{"type":"custom","customType":"decay-diagnostic","data":{"reason":"late failure"}}\n',
		);
		const total = prefix.length + payloadSize + suffix.length;
		let fetched = 0;
		const source: Source = {
			size: total,
			slice: (start, end) => {
				fetched += end - start;
				const buffer = Buffer.alloc(end - start, "x");
				for (const [segment, at] of [
					[prefix, 0],
					[suffix, prefix.length + payloadSize],
				] as const) {
					const first = Math.max(start, at),
						last = Math.min(end, at + segment.length);
					if (last > first)
						buffer.set(segment.subarray(first - at, last - at), first - start);
				}
				return buffer;
			},
		};
		const { query } = reader(source);
		const page = raw(
			await query.execute({
				action: "read",
				view: "raw-jsonl",
				hostId: HOST,
				sessionId: SESSION,
				byteOffset: prefix.length + payloadSize - 100,
			}),
		);
		expect(page.text).toContain('"reason":"late failure"');
		expect(page.totalBytes).toBe(total);
		expect(page.nextCursor).toBeNull();
		expect(fetched).toBeLessThan(32 * 1024);
	});

	it("preserves UTF-8 across pages and rejects offsets inside an unmasked code point", async () => {
		const prefix = `${HEADER}\n{"type":"future","data":"`;
		const body = `${prefix}${"a".repeat(16383 - Buffer.byteLength(prefix))}😀${"é".repeat(10000)}"}\n`;
		const { query } = reader(Buffer.from(body));
		expect(await collect(query)).toBe(body);
		await expect(
			query.execute({
				action: "read",
				view: "raw-jsonl",
				hostId: HOST,
				sessionId: SESSION,
				byteOffset: 16384,
			}),
		).rejects.toThrow("UTF-8 boundary");
	});

	it.each(['Ab/C\\"\n😀', "token-token"])(
		"masks literal and JSON-escaped credential spellings across page and arbitrary-offset boundaries: %s",
		async (secret) => {
			const canonical = JSON.stringify(secret).slice(1, -1);
			const escaped = secret
				.split("")
				.map(
					(unit, index) =>
						`\\u${unit.charCodeAt(0).toString(16).padStart(4, "0")[index % 2 ? "toUpperCase" : "toLowerCase"]()}`,
				)
				.join("");
			const mixed = secret
				.split("")
				.map((unit, index) =>
					index % 2
						? JSON.stringify(unit).slice(1, -1)
						: `\\u${unit.charCodeAt(0).toString(16).padStart(4, "0")}`,
				)
				.join("");
			for (const spelling of [
				canonical,
				canonical.replaceAll("/", "\\/"),
				escaped,
				mixed,
			]) {
				for (const split of [
					1,
					Math.floor(spelling.length / 2),
					spelling.length - 1,
				]) {
					const prefix = `${HEADER}\n{"type":"custom","data":"`;
					const padding = "x".repeat(16384 - Buffer.byteLength(prefix) - split);
					const body = Buffer.from(`${prefix}${padding}${spelling} tail"}\n`);
					const { query } = reader(body, [secret]);
					const reconstructed = await collect(query);
					expect(reconstructed).not.toContain(spelling);
					const value = JSON.parse(reconstructed.split("\n")[1]!)
						.data as string;
					expect(value).toBe(
						`${padding}${"*".repeat(Buffer.byteLength(spelling))} tail`,
					);
					const start = Buffer.byteLength(prefix + padding) + split;
					const page = raw(
						await query.execute({
							action: "read",
							view: "raw-jsonl",
							hostId: HOST,
							sessionId: SESSION,
							byteOffset: start,
						}),
					);
					expect(page.text).toBe(
						`${"*".repeat(Buffer.byteLength(spelling) - split)} tail"}\n`,
					);
				}
			}
		},
	);

	it("preserves BOMs and output-budget clipping without dropping source bytes", async () => {
		const bomBody = `\uFEFF${HEADER}\n${"a".repeat(16384)}\uFEFF${"😀".repeat(1000)}`;
		expect(await collect(reader(Buffer.from(bomBody)).query)).toBe(bomBody);
		const escapedBody = `${HEADER}\n${"\u0000".repeat(40000)}`;
		expect(await collect(reader(Buffer.from(escapedBody)).query)).toBe(
			escapedBody,
		);
	});

	it.each([401, 403])(
		"reports raw archive authorization failure accurately: %i",
		async (status) => {
			const { query, state } = reader(Buffer.from(HEADER));
			state.status = status;
			await expect(
				query.execute({
					action: "read",
					view: "raw-jsonl",
					hostId: HOST,
					sessionId: SESSION,
				}),
			).rejects.toThrow("access denied");
		},
	);

	it("rejects size changes even when a storage service reuses its ETag", async () => {
		const bytes = Buffer.from(`${HEADER}\n${"x".repeat(40000)}`);
		const source: Source = {
			size: bytes.length,
			slice: (start, end) => bytes.subarray(start, end),
		};
		const { query } = reader(source);
		const page = raw(
			await query.execute({
				action: "read",
				view: "raw-jsonl",
				hostId: HOST,
				sessionId: SESSION,
			}),
		);
		source.size++;
		await expect(
			query.execute({ action: "read", cursor: page.nextCursor! }),
		).rejects.toThrow("changed");
	});

	it("masks overlapping credentials in attached metadata as well as raw text", async () => {
		const { query } = reader(
			Buffer.from(`${HEADER}\n{"data":"ababaXYZ"}`),
			["aba", "baXYZ"],
			() => 0,
			"ababaXYZ",
		);
		const page = raw(
			await query.execute({
				action: "read",
				view: "raw-jsonl",
				hostId: HOST,
				sessionId: SESSION,
			}),
		);
		expect(page.session.cwd).toBe("[redacted]");
		expect(page.text).not.toContain("abaXYZ");
	});

	it("masks overlapping credential occurrences before slicing", async () => {
		const { query } = reader(
			Buffer.from(`${HEADER}\n{"type":"custom","data":"ababaXYZ"}`),
			["aba", "baXYZ"],
		);
		expect(await collect(query)).toBe(
			`${HEADER}\n{"type":"custom","data":"********"}`,
		);
	});

	it("fails explicitly if its cursor is cleared during a range read", async () => {
		const bytes = Buffer.from(`${HEADER}\n${"x".repeat(40000)}`);
		let clearOnRead = false;
		const source: Source = {
			size: bytes.length,
			slice: (start, end) => {
				if (clearOnRead) query.clear();
				return bytes.subarray(start, end);
			},
		};
		const { query } = reader(source);
		const page = raw(
			await query.execute({
				action: "read",
				view: "raw-jsonl",
				hostId: HOST,
				sessionId: SESSION,
			}),
		);
		clearOnRead = true;
		await expect(
			query.execute({ action: "read", cursor: page.nextCursor! }),
		).rejects.toThrow("evicted during inspection");
	});

	it("pins object versions, replays cursors, rejects changed selectors and expires without remote work", async () => {
		let now = 0;
		const { query, state, requests } = reader(
			Buffer.from(`${HEADER}\n${"x".repeat(40000)}`),
			[],
			() => now,
		);
		const page = raw(
			await query.execute({
				action: "read",
				view: "raw-jsonl",
				hostId: HOST,
				sessionId: SESSION,
			}),
		);
		const next = raw(
			await query.execute({ action: "read", cursor: page.nextCursor! }),
		);
		expect(
			await query.execute({ action: "read", cursor: page.nextCursor! }),
		).toEqual(next);
		await expect(
			query.execute({
				action: "read",
				cursor: page.nextCursor!,
				hostId: HOST,
			} as never),
		).rejects.toThrow("selectors");
		state.etag = '"new-version"';
		await expect(
			query.execute({ action: "read", cursor: page.nextCursor! }),
		).rejects.toThrow("changed");
		expect(
			requests
				.filter((request) => request.ifMatch !== null)
				.every((request) => request.ifMatch === '"version-one"'),
		).toBe(true);
		now = 10 * 60 * 1000;
		const count = requests.length;
		await expect(
			query.execute({ action: "read", cursor: page.nextCursor! }),
		).rejects.toThrow("expired");
		expect(requests).toHaveLength(count);
	});
});