repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/bak/__tests__/raw.test.ts
Rawimport { Value } from "typebox/value";
import { describe, expect, it } from "vitest";
import { BakArchive } from "../archive.js";
import { BakQuery } from "../query.js";
import { S3Store } from "../s3.js";
import { statePaths } from "../state.js";
import { BakOutput, type BakRawResult, type BakResult } from "../tool.js";
const HOST = "12345678-1234-1234-1234-123456789abc";
const SESSION = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const DATE = "2026-01-01T00:00:00.000Z";
const HEADER = JSON.stringify({
type: "session",
id: SESSION,
version: 3,
timestamp: DATE,
cwd: "/project",
parentSession: "/parent.jsonl",
});
const CONFIG = {
endpoint: "https://example.invalid",
bucket: "bucket",
region: "auto",
accessKeyId: "fake-id",
secretAccessKey: "fake-secret",
};
type Source = {
size: number;
slice: (start: number, end: number) => Uint8Array;
};
function reader(
body: Buffer | Source,
secrets: string[] = [],
now = () => 0,
metadataCwd = "/project",
) {
const source: Source = Buffer.isBuffer(body)
? { size: body.length, slice: (start, end) => body.subarray(start, end) }
: body;
const requests: Array<{
key: string;
range: string | null;
ifMatch: string | null;
}> = [];
const state: { etag: string; status?: number } = { etag: '"version-one"' };
const client = {
fetch: async (input: RequestInfo | URL, init?: RequestInit) => {
const request = new Request(input, init);
expect(request.method).toBe("GET");
const key = decodeURIComponent(
new URL(request.url).pathname.slice("/bucket/".length),
);
const range = request.headers.get("range");
requests.push({ key, range, ifMatch: request.headers.get("if-match") });
if (key === "catalog/hosts.json")
return new Response(
JSON.stringify({
version: 1,
hosts: [{ hostId: HOST, alias: "desktop" }],
}),
);
if (key === `hosts/${HOST}.json`)
return new Response(
JSON.stringify({
version: 1,
hostId: HOST,
alias: "desktop",
sessions: [
{
id: SESSION,
key: `sessions/${HOST}/${SESSION}.jsonl`,
cwd: metadataCwd,
created: DATE,
},
],
}),
);
expect(key).toBe(`sessions/${HOST}/${SESSION}.jsonl`);
if (state.status) return new Response(null, { status: state.status });
if (!range)
throw new Error("whole-session GET forbidden in this fixture");
if (
request.headers.has("if-match") &&
request.headers.get("if-match") !== state.etag
)
return new Response(null, { status: 412 });
const match = /^bytes=(\d+)-(\d+)$/.exec(range)!;
const start = Number(match[1]);
const end = Math.min(Number(match[2]), source.size - 1);
if (start >= source.size) return new Response(null, { status: 416 });
return new Response(
Uint8Array.from(source.slice(start, end + 1)).buffer,
{
status: 206,
headers: {
etag: state.etag,
"content-range": `bytes ${start}-${end}/${source.size}`,
},
},
);
},
};
const query = new BakQuery(
new BakArchive(new S3Store(CONFIG, client), statePaths()),
secrets,
now,
);
return { query, requests, state };
}
function raw(result: BakResult): BakRawResult {
expect(Value.Check(BakOutput, result)).toBe(true);
expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThanOrEqual(
48 * 1024,
);
if (result.action !== "read" || result.view !== "raw-jsonl")
throw new Error("expected raw JSONL");
return result;
}
async function collect(query: BakQuery): Promise<string> {
let page = raw(
await query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
}),
);
let text = page.text;
while (page.nextCursor) {
const previousEnd = page.endByteExclusive;
page = raw(
await query.execute({ action: "read", cursor: page.nextCursor }),
);
expect(page.startByte).toBe(previousEnd);
expect(page.endByteExclusive).toBeGreaterThan(page.startByte);
text += page.text;
}
expect(page.endByteExclusive).toBe(page.totalBytes);
return text;
}
describe("complete archived JSONL inspection", () => {
it("reconstructs every field, unknown types, media and header without whole-file parsing", async () => {
const entries = [
{
type: "message",
message: {
role: "system",
content: "system prompt",
toolsAdded: [{ name: "tool", parameters: {} }],
},
},
{
type: "message",
message: {
role: "assistant",
content: [
{
type: "thinking",
thinking: "reasoning",
thinkingSignature: "signature",
},
{ type: "toolCall", arguments: { path: "/exact/path" } },
],
usage: { cost: { total: 12.34 } },
},
},
{
type: "message",
message: {
role: "toolResult",
content: [
{ type: "image", data: "base64-image" },
{ type: "audio", data: "base64-audio" },
],
details: { private: true },
structuredContent: { result: "structured" },
},
},
{
type: "compaction",
details: {
decay: { atoms: [{ key: "rule", allocation: 0 }], elapsedMs: 12345 },
},
summary: "checkpoint",
},
{
type: "custom",
customType: "decay-diagnostic",
data: { stage: "summarizer", reason: "capacity exceeded" },
},
{
type: "custom_message",
display: false,
content: "hidden",
details: { secret: "state" },
},
{ type: "context_edit", targetId: "old", replacement: null },
{
type: "future-extension-entry",
unknown: { nested: ["😀", '\\"\n'.repeat(12000)] },
},
];
const body = `${HEADER}\n${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n{"type":"future","hugeNumber":900719925474099312345,"duplicate":1,"duplicate":2}\n`;
const { query, requests } = reader(Buffer.from(body));
expect(await collect(query)).toBe(body);
expect(
requests
.filter((request) => request.key.startsWith("sessions/"))
.every((request) => request.range !== null),
).toBe(true);
expect(
requests.filter((request) => request.key === "catalog/hosts.json"),
).toHaveLength(1);
});
it("inspects a 40 MiB entry near its end with only bounded range reads", async () => {
const prefix = Buffer.from(`${HEADER}\n{"type":"custom","data":"`);
const payloadSize = 40 * 1024 * 1024;
const suffix = Buffer.from(
'"}\n{"type":"custom","customType":"decay-diagnostic","data":{"reason":"late failure"}}\n',
);
const total = prefix.length + payloadSize + suffix.length;
let fetched = 0;
const source: Source = {
size: total,
slice: (start, end) => {
fetched += end - start;
const buffer = Buffer.alloc(end - start, "x");
for (const [segment, at] of [
[prefix, 0],
[suffix, prefix.length + payloadSize],
] as const) {
const first = Math.max(start, at),
last = Math.min(end, at + segment.length);
if (last > first)
buffer.set(segment.subarray(first - at, last - at), first - start);
}
return buffer;
},
};
const { query } = reader(source);
const page = raw(
await query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
byteOffset: prefix.length + payloadSize - 100,
}),
);
expect(page.text).toContain('"reason":"late failure"');
expect(page.totalBytes).toBe(total);
expect(page.nextCursor).toBeNull();
expect(fetched).toBeLessThan(32 * 1024);
});
it("preserves UTF-8 across pages and rejects offsets inside an unmasked code point", async () => {
const prefix = `${HEADER}\n{"type":"future","data":"`;
const body = `${prefix}${"a".repeat(16383 - Buffer.byteLength(prefix))}😀${"é".repeat(10000)}"}\n`;
const { query } = reader(Buffer.from(body));
expect(await collect(query)).toBe(body);
await expect(
query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
byteOffset: 16384,
}),
).rejects.toThrow("UTF-8 boundary");
});
it.each(['Ab/C\\"\n😀', "token-token"])(
"masks literal and JSON-escaped credential spellings across page and arbitrary-offset boundaries: %s",
async (secret) => {
const canonical = JSON.stringify(secret).slice(1, -1);
const escaped = secret
.split("")
.map(
(unit, index) =>
`\\u${unit.charCodeAt(0).toString(16).padStart(4, "0")[index % 2 ? "toUpperCase" : "toLowerCase"]()}`,
)
.join("");
const mixed = secret
.split("")
.map((unit, index) =>
index % 2
? JSON.stringify(unit).slice(1, -1)
: `\\u${unit.charCodeAt(0).toString(16).padStart(4, "0")}`,
)
.join("");
for (const spelling of [
canonical,
canonical.replaceAll("/", "\\/"),
escaped,
mixed,
]) {
for (const split of [
1,
Math.floor(spelling.length / 2),
spelling.length - 1,
]) {
const prefix = `${HEADER}\n{"type":"custom","data":"`;
const padding = "x".repeat(16384 - Buffer.byteLength(prefix) - split);
const body = Buffer.from(`${prefix}${padding}${spelling} tail"}\n`);
const { query } = reader(body, [secret]);
const reconstructed = await collect(query);
expect(reconstructed).not.toContain(spelling);
const value = JSON.parse(reconstructed.split("\n")[1]!)
.data as string;
expect(value).toBe(
`${padding}${"*".repeat(Buffer.byteLength(spelling))} tail`,
);
const start = Buffer.byteLength(prefix + padding) + split;
const page = raw(
await query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
byteOffset: start,
}),
);
expect(page.text).toBe(
`${"*".repeat(Buffer.byteLength(spelling) - split)} tail"}\n`,
);
}
}
},
);
it("preserves BOMs and output-budget clipping without dropping source bytes", async () => {
const bomBody = `\uFEFF${HEADER}\n${"a".repeat(16384)}\uFEFF${"😀".repeat(1000)}`;
expect(await collect(reader(Buffer.from(bomBody)).query)).toBe(bomBody);
const escapedBody = `${HEADER}\n${"\u0000".repeat(40000)}`;
expect(await collect(reader(Buffer.from(escapedBody)).query)).toBe(
escapedBody,
);
});
it.each([401, 403])(
"reports raw archive authorization failure accurately: %i",
async (status) => {
const { query, state } = reader(Buffer.from(HEADER));
state.status = status;
await expect(
query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
}),
).rejects.toThrow("access denied");
},
);
it("rejects size changes even when a storage service reuses its ETag", async () => {
const bytes = Buffer.from(`${HEADER}\n${"x".repeat(40000)}`);
const source: Source = {
size: bytes.length,
slice: (start, end) => bytes.subarray(start, end),
};
const { query } = reader(source);
const page = raw(
await query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
}),
);
source.size++;
await expect(
query.execute({ action: "read", cursor: page.nextCursor! }),
).rejects.toThrow("changed");
});
it("masks overlapping credentials in attached metadata as well as raw text", async () => {
const { query } = reader(
Buffer.from(`${HEADER}\n{"data":"ababaXYZ"}`),
["aba", "baXYZ"],
() => 0,
"ababaXYZ",
);
const page = raw(
await query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
}),
);
expect(page.session.cwd).toBe("[redacted]");
expect(page.text).not.toContain("abaXYZ");
});
it("masks overlapping credential occurrences before slicing", async () => {
const { query } = reader(
Buffer.from(`${HEADER}\n{"type":"custom","data":"ababaXYZ"}`),
["aba", "baXYZ"],
);
expect(await collect(query)).toBe(
`${HEADER}\n{"type":"custom","data":"********"}`,
);
});
it("fails explicitly if its cursor is cleared during a range read", async () => {
const bytes = Buffer.from(`${HEADER}\n${"x".repeat(40000)}`);
let clearOnRead = false;
const source: Source = {
size: bytes.length,
slice: (start, end) => {
if (clearOnRead) query.clear();
return bytes.subarray(start, end);
},
};
const { query } = reader(source);
const page = raw(
await query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
}),
);
clearOnRead = true;
await expect(
query.execute({ action: "read", cursor: page.nextCursor! }),
).rejects.toThrow("evicted during inspection");
});
it("pins object versions, replays cursors, rejects changed selectors and expires without remote work", async () => {
let now = 0;
const { query, state, requests } = reader(
Buffer.from(`${HEADER}\n${"x".repeat(40000)}`),
[],
() => now,
);
const page = raw(
await query.execute({
action: "read",
view: "raw-jsonl",
hostId: HOST,
sessionId: SESSION,
}),
);
const next = raw(
await query.execute({ action: "read", cursor: page.nextCursor! }),
);
expect(
await query.execute({ action: "read", cursor: page.nextCursor! }),
).toEqual(next);
await expect(
query.execute({
action: "read",
cursor: page.nextCursor!,
hostId: HOST,
} as never),
).rejects.toThrow("selectors");
state.etag = '"new-version"';
await expect(
query.execute({ action: "read", cursor: page.nextCursor! }),
).rejects.toThrow("changed");
expect(
requests
.filter((request) => request.ifMatch !== null)
.every((request) => request.ifMatch === '"version-one"'),
).toBe(true);
now = 10 * 60 * 1000;
const count = requests.length;
await expect(
query.execute({ action: "read", cursor: page.nextCursor! }),
).rejects.toThrow("expired");
expect(requests).toHaveLength(count);
});
});