repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/bak/query.ts
Rawimport { randomUUID } from "node:crypto";
import { setImmediate } from "node:timers/promises";
import { ArchiveLimitError, type BakArchive } from "./archive.js";
import { type RawWindow, readRawWindow } from "./raw.js";
import { S3Error, S3SizeLimitError } from "./s3.js";
import type { CatalogCache, SessionRecord } from "./state.js";
import type {
BakFragment,
BakOmitted,
BakParams,
BakRawResult,
BakResult,
BakSession,
} from "./tool.js";
const DEFAULT_LIMIT = 20;
const SNAPSHOT_TTL_MS = 10 * 60 * 1000;
const MAX_SNAPSHOTS = 4;
const MAX_SNAPSHOT_BYTES = 32 * 1024 * 1024;
const MAX_PAGE_BYTES = 48 * 1024;
const MAX_FRAGMENT_CHARS = 4096;
type CompactData =
| { action: "list" | "search"; items: BakSession[]; view?: undefined }
| {
action: "read";
view: "archive-history";
items: BakFragment[];
session: BakSession;
omitted: BakOmitted;
};
type RawData = {
action: "read";
view: "raw-jsonl";
session: BakSession;
etag: string;
totalBytes: number;
};
type SnapshotData = CompactData | RawData;
type SnapshotMetadata = { createdAt: number; fetchedAt: string; bytes: number };
type Snapshot = SnapshotData & SnapshotMetadata;
type CompactSnapshot = CompactData & SnapshotMetadata;
type RawSnapshot = RawData & SnapshotMetadata;
class QueryError extends Error {}
function fail(message: string): never {
throw new QueryError(`Bak query: ${message}`);
}
function isObject(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function safeEnd(value: string, start: number, maximum: number): number {
let end = Math.min(value.length, start + maximum);
const last = value.charCodeAt(end - 1);
if (end < value.length && last >= 0xd800 && last <= 0xdbff) end--;
return end;
}
function compare(left: string, right: string): number {
return left < right ? -1 : left > right ? 1 : 0;
}
/** Query-owned snapshots never write cache, identity or restored session files. */
export class BakQuery {
private readonly snapshots = new Map<string, Snapshot>();
private snapshotBytes = 0;
private active = 0;
constructor(
private readonly archive: Pick<
BakArchive,
"readCatalog" | "readSession" | "readSessionRange"
>,
private readonly secrets: readonly string[] = [],
private readonly now: () => number = Date.now,
) {}
clear(): void {
this.snapshots.clear();
this.snapshotBytes = 0;
}
async execute(params: BakParams, signal?: AbortSignal): Promise<BakResult> {
if (this.active >= 2)
throw new Error(
"Bak query: two reads already running; retry when one completes",
);
this.active++;
try {
signal?.throwIfAborted();
const limit =
("limit" in params ? params.limit : undefined) ?? DEFAULT_LIMIT;
if (!Number.isInteger(limit) || limit < 1 || limit > 100)
fail("invalid page limit");
if ("cursor" in params) {
if (
Object.keys(params).some(
(key) => !["action", "cursor", "limit"].includes(key),
)
)
fail(
"cursor continuation cannot change selectors; restart without cursor",
);
this.expire();
const cursor = /^([0-9a-f-]{36})\.(0|[1-9][0-9]{0,15})$/.exec(
params.cursor,
);
if (!cursor) fail("invalid cursor; restart without cursor");
const token = cursor[1];
if (!token) fail("invalid cursor; restart without cursor");
const snapshot = this.snapshots.get(token);
if (!snapshot)
fail("cursor expired or is unknown; restart without cursor");
if (snapshot.action !== params.action)
fail("cursor action mismatch; restart without cursor");
const offset = Number(cursor[2]);
if (!Number.isSafeInteger(offset))
fail("invalid cursor offset; restart without cursor");
if (snapshot.view === "raw-jsonl") {
if ("limit" in params)
fail("raw inspection uses byte pages, not item limits");
if (offset > snapshot.totalBytes)
fail("invalid cursor offset; restart without cursor");
const window = await this.rawWindow(
snapshot.session,
offset,
snapshot.etag,
signal,
);
if (window.totalBytes !== snapshot.totalBytes)
fail("archive changed; restart inspection");
if (this.snapshots.get(token) !== snapshot)
fail(
"cursor expired or was evicted during inspection; restart without cursor",
);
return this.rawResult(token, snapshot, offset, window);
}
if (offset > snapshot.items.length)
fail("invalid cursor offset; restart without cursor");
return this.page(token, offset, limit);
}
const query =
params.action === "search"
? params.query.trim().toLowerCase()
: undefined;
if (query !== undefined && (!query || query.length > 256))
fail("search needs a non-empty query of at most 256 characters");
const catalog = await this.archive.readCatalog(signal);
signal?.throwIfAborted();
const sessions = await this.sessions(
catalog,
params.hostId,
query,
signal,
);
let snapshot: SnapshotData;
if (params.action === "read") {
const session = sessions.find(
(item) => item.sessionId === params.sessionId,
);
if (!session)
fail(
"session is not indexed; list sessions again or wait for backup",
);
if ("view" in params && params.view === "raw-jsonl") {
const offset = params.byteOffset ?? 0;
if (
!Number.isSafeInteger(offset) ||
offset < 0 ||
offset > Number.MAX_SAFE_INTEGER - 128 * 1024
)
fail("invalid raw archive byte offset");
const window = await this.rawWindow(
session,
offset,
undefined,
signal,
);
const token = this.store({
action: "read",
view: "raw-jsonl",
session,
etag: window.etag,
totalBytes: window.totalBytes,
});
const stored = this.snapshots.get(token);
if (stored?.view !== "raw-jsonl")
fail("inspection cursor is unavailable");
return this.rawResult(token, stored, offset, window);
}
const body = await this.archive.readSession(
params.hostId,
params.sessionId,
signal,
);
signal?.throwIfAborted();
const { fragments, omitted } = await this.project(body, signal);
snapshot = {
action: "read",
view: "archive-history",
items: fragments,
session,
omitted,
};
} else snapshot = { action: params.action, items: sessions };
signal?.throwIfAborted();
return this.page(this.store(snapshot), 0, limit);
} catch (error) {
if (signal?.aborted)
throw new Error("Bak query: cancelled or deadline exceeded");
if (error instanceof QueryError) throw new Error(error.message);
if (
error instanceof S3SizeLimitError ||
error instanceof ArchiveLimitError
)
throw new Error(
params.action === "read"
? 'Bak query: compact archive read exceeds a safety limit; retry read with view: "raw-jsonl"'
: "Bak query: archive response exceeds a safety limit",
);
if (
error instanceof S3Error &&
(error.status === 401 || error.status === 403)
)
throw new Error(
"Bak query: archive access denied; check bucket read credentials",
);
throw new Error(
"Bak query: archive unavailable or invalid; retry or check /bak status",
);
} finally {
this.active--;
}
}
private async rawWindow(
session: BakSession,
offset: number,
etag: string | undefined,
signal?: AbortSignal,
): Promise<RawWindow> {
try {
const window = await readRawWindow(
this.archive,
session.hostId,
session.sessionId,
offset,
this.secrets,
etag,
signal,
);
signal?.throwIfAborted();
return window;
} catch (error) {
if (signal?.aborted) throw error;
if (
error instanceof S3Error &&
(error.status === 401 || error.status === 403)
)
throw error;
if (error instanceof S3Error && error.status === 412)
fail("archive changed; restart inspection");
if (error instanceof S3Error && error.status === 416)
fail("raw byte offset is outside the archive");
fail(
"raw inspection failed; use a UTF-8 boundary offset and storage supporting byte ranges with strong ETags",
);
}
}
private rawResult(
token: string,
snapshot: RawSnapshot,
offset: number,
window: RawWindow,
): BakRawResult {
const result = (text: string): BakRawResult => {
const endByteExclusive = offset + Buffer.byteLength(text);
return {
action: "read",
view: "raw-jsonl",
trust: "untrusted-history",
fetchedAt: snapshot.fetchedAt,
session: structuredClone(snapshot.session),
startByte: offset,
endByteExclusive,
totalBytes: snapshot.totalBytes,
redaction: "configured-credentials",
text,
nextCursor:
endByteExclusive < snapshot.totalBytes
? `${token}.${endByteExclusive}`
: null,
};
};
let low = 0;
let high = window.text.length;
while (low < high) {
const middle = Math.ceil((low + high) / 2);
const candidate = window.text.slice(0, safeEnd(window.text, 0, middle));
if (
Buffer.byteLength(JSON.stringify(result(candidate))) <= MAX_PAGE_BYTES
)
low = middle;
else high = middle - 1;
}
const page = result(window.text.slice(0, safeEnd(window.text, 0, low)));
if (page.endByteExclusive === offset && offset < snapshot.totalBytes)
fail("raw page did not advance");
return page;
}
private async sessions(
cache: CatalogCache,
hostId: string | undefined,
query: string | undefined,
signal?: AbortSignal,
): Promise<BakSession[]> {
const result: BakSession[] = [];
let seen = 0;
for (const index of cache.indexes) {
if (hostId !== undefined && index.hostId !== hostId) continue;
for (const record of index.sessions) {
if (++seen % 256 === 0) {
await setImmediate();
signal?.throwIfAborted();
}
if (
query !== undefined &&
![record.name, record.cwd, record.id, index.alias, index.hostId].some(
(value) =>
value !== undefined &&
this.redact(value).toLowerCase().includes(query),
)
)
continue;
result.push(this.session(index.hostId, index.alias, record));
}
}
return result.sort(
(left, right) =>
compare(right.created, left.created) ||
compare(left.hostId, right.hostId) ||
compare(left.sessionId, right.sessionId),
);
}
private session(
hostId: string,
alias: string,
record: SessionRecord,
): BakSession {
// Redacting an identifier must not silently create an unusable or invalid reference.
if (this.redact(hostId) !== hostId || this.redact(record.id) !== record.id)
fail(
"archive identifiers overlap configured credentials; cannot expose safe references",
);
const name = record.name === undefined ? null : this.text(record.name, 512);
const cwd = this.text(record.cwd, 512);
const hostAlias = this.text(alias, 64);
return {
hostId,
hostAlias: hostAlias.value,
sessionId: record.id,
name: name?.value ?? null,
cwd: cwd.value,
created: this.text(record.created, 64).value,
metadataTruncated:
cwd.truncated || hostAlias.truncated || (name?.truncated ?? false),
};
}
private async project(
body: Uint8Array,
signal?: AbortSignal,
): Promise<{ fragments: BakFragment[]; omitted: BakOmitted }> {
const lines = new TextDecoder("utf-8", { fatal: true })
.decode(body)
.split("\n");
const fragments: BakFragment[] = [];
const omitted: BakOmitted = {
thinking: 0,
images: 0,
system: 0,
customState: 0,
toolArguments: 0,
other: 0,
};
const header = this.json(lines[0] ?? "");
if (!isObject(header) || header["type"] !== "session")
fail("malformed archive session");
for (let index = 1; index < lines.length; index++) {
if (index % 128 === 0) {
await setImmediate();
signal?.throwIfAborted();
}
const line = lines[index] ?? "";
if (!line.trim()) continue;
const entry = this.json(line);
if (!isObject(entry) || typeof entry["type"] !== "string")
fail("malformed archive session");
this.entryFragments(entry, index + 1, fragments, omitted);
}
return { fragments, omitted };
}
private json(line: string): unknown {
try {
return JSON.parse(line) as unknown;
} catch {
fail("malformed archive session JSONL");
}
}
private entryFragments(
entry: Record<string, unknown>,
line: number,
fragments: BakFragment[],
omitted: BakOmitted,
): void {
const nullable = (value: unknown, maximum = 512) =>
typeof value === "string" ? this.text(value, maximum).value : null;
const base = {
entryId: nullable(entry["id"]),
parentId: nullable(entry["parentId"]),
timestamp: nullable(entry["timestamp"], 64),
line,
};
let kind: BakFragment["kind"];
let text: string;
let toolName: string | null = null;
let isError: boolean | null = null;
if (entry["type"] === "message" && isObject(entry["message"])) {
const message = entry["message"];
const role = message["role"];
if (role === "system") {
omitted.system++;
return;
}
if (
role === "bashExecution" &&
typeof message["command"] === "string" &&
typeof message["output"] === "string"
) {
kind = role;
text = `$ ${message["command"]}\n${message["output"]}`;
isError =
message["cancelled"] === true ||
(typeof message["exitCode"] === "number" &&
message["exitCode"] !== 0);
} else if (
role === "user" ||
role === "assistant" ||
role === "toolResult"
) {
kind = role;
text = this.content(message["content"], omitted, role === "assistant");
if (role === "toolResult") {
toolName = nullable(message["toolName"]);
isError =
typeof message["isError"] === "boolean" ? message["isError"] : null;
}
} else {
omitted.other++;
return;
}
} else if (entry["type"] === "custom_message") {
kind = "custom_message";
text = this.content(entry["content"], omitted);
} else if (
(entry["type"] === "compaction" || entry["type"] === "branch_summary") &&
typeof entry["summary"] === "string"
) {
kind = entry["type"];
text = entry["summary"];
} else {
if (entry["type"] === "custom") omitted.customState++;
else omitted.other++;
return;
}
const clean = this.redact(text);
const pieces: string[] = [];
for (let offset = 0; offset < clean.length; ) {
const end = safeEnd(clean, offset, MAX_FRAGMENT_CHARS);
pieces.push(clean.slice(offset, end));
offset = end;
}
for (const [part, piece] of pieces.entries())
fragments.push({
...base,
kind,
toolName,
isError,
part: part + 1,
parts: pieces.length,
text: piece,
});
}
private content(
value: unknown,
omitted: BakOmitted,
assistant = false,
): string {
if (typeof value === "string") return value;
if (!Array.isArray(value)) {
omitted.other++;
return "";
}
const text: string[] = [];
for (const part of value) {
if (!isObject(part)) {
omitted.other++;
continue;
}
if (part["type"] === "text" && typeof part["text"] === "string")
text.push(part["text"]);
else if (part["type"] === "thinking") omitted.thinking++;
else if (part["type"] === "image") omitted.images++;
else if (assistant && part["type"] === "toolCall") {
omitted.toolArguments++;
const name =
typeof part["name"] === "string"
? this.text(part["name"], 512).value
: "unknown";
text.push(`\n[tool call: ${name}; arguments omitted]\n`);
} else omitted.other++;
}
return text.join("");
}
private redact(value: string): string {
const secrets = this.secrets.filter(Boolean);
const positions = secrets.map((secret) => value.indexOf(secret));
const parts: string[] = [];
let copied = 0;
for (;;) {
const start = Math.min(...positions.filter((position) => position >= 0));
if (!Number.isFinite(start)) break;
let end = start;
let advanced: boolean;
do {
advanced = false;
for (const [index, secret] of secrets.entries()) {
const position = positions[index] ?? -1;
if (position < 0 || position > end) continue;
end = Math.max(end, position + secret.length);
positions[index] = value.indexOf(secret, position + 1);
advanced = true;
}
} while (advanced);
parts.push(value.slice(copied, start), "[redacted]");
copied = end;
}
parts.push(value.slice(copied));
return parts.join("");
}
private text(
value: string,
maximum: number,
): { value: string; truncated: boolean } {
const clean = this.redact(value);
return {
value: clean.slice(0, safeEnd(clean, 0, maximum)),
truncated: clean.length > maximum,
};
}
private store(data: SnapshotData): string {
this.expire();
const createdAt = this.now();
const snapshot: Snapshot = {
...data,
createdAt,
fetchedAt: new Date(createdAt).toISOString(),
bytes: 0,
};
snapshot.bytes = Buffer.byteLength(JSON.stringify(snapshot));
if (snapshot.bytes > MAX_SNAPSHOT_BYTES)
fail("snapshot exceeds memory limit; narrow by host or session");
while (
this.snapshots.size >= MAX_SNAPSHOTS ||
this.snapshotBytes + snapshot.bytes > MAX_SNAPSHOT_BYTES
) {
const first = this.snapshots.entries().next().value;
if (!first) break;
this.remove(first[0], first[1]);
}
const token = randomUUID();
this.snapshots.set(token, snapshot);
this.snapshotBytes += snapshot.bytes;
return token;
}
private page(token: string, offset: number, limit: number): BakResult {
const snapshot = this.snapshots.get(token);
if (!snapshot) fail("cursor expired or is unknown; restart without cursor");
if (snapshot.view === "raw-jsonl")
fail("raw inspection needs a byte-range read");
let end = offset;
// Reserve the longest possible continuation so JSON escaping cannot exceed budget.
let bytes = Buffer.byteLength(
JSON.stringify(
this.result(
snapshot,
offset,
offset,
`${token}.${snapshot.items.length}`,
),
),
);
while (end < snapshot.items.length && end < offset + limit) {
const addition =
Buffer.byteLength(JSON.stringify(snapshot.items[end])) +
(end > offset ? 1 : 0);
if (bytes + addition > MAX_PAGE_BYTES) break;
bytes += addition;
end++;
}
if (end === offset && offset < snapshot.items.length)
fail("one item exceeds page budget");
return structuredClone(
this.result(
snapshot,
offset,
end,
end < snapshot.items.length ? `${token}.${end}` : null,
),
);
}
private result(
snapshot: CompactSnapshot,
offset: number,
end: number,
nextCursor: string | null,
): BakResult {
const page = {
fetchedAt: snapshot.fetchedAt,
trust: "untrusted-history" as const,
totalItems: snapshot.items.length,
nextCursor,
};
if (snapshot.action === "read")
return {
...page,
action: "read",
view: "archive-history",
order: "append",
branches: "all",
session: snapshot.session,
omitted: snapshot.omitted,
fragments: snapshot.items.slice(offset, end),
};
return {
...page,
action: snapshot.action,
scope: "indexed-sessions",
sessions: snapshot.items.slice(offset, end),
};
}
private expire(): void {
for (const [token, snapshot] of this.snapshots)
if (this.now() - snapshot.createdAt >= SNAPSHOT_TTL_MS)
this.remove(token, snapshot);
}
private remove(token: string, snapshot: Snapshot): void {
this.snapshots.delete(token);
this.snapshotBytes -= snapshot.bytes;
}
}