Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/autopsy/__tests__/contract.test.ts

Raw
import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { initTheme } from "@earendil-works/pi-coding-agent";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
	createPiSession,
	type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import { groupSystemPrompt } from "../index.ts";
import type { InspectorData } from "../src/types.ts";

function extensionPath(): string {
	return fileURLToPath(new URL("../index.ts", import.meta.url));
}

async function openInspector(
	session: PiSession,
): Promise<{ url: string; data: InspectorData }> {
	initTheme(undefined, false);
	const ctx = session.session.extensionRunner.createCommandContext();
	const notify = vi.spyOn(ctx.ui, "notify");
	const command = session.extensions[0]?.commands.get("autopsy");
	if (!command) throw new Error("missing autopsy command");
	await command.handler("", ctx);
	const url = String(notify.mock.calls.at(-1)?.[0]).match(
		/http:\/\/127\.0\.0\.1:\d+\/\w+\//u,
	)?.[0];
	if (!url) throw new Error("missing inspector URL");
	const response = await fetch(`${url}data`);
	expect(response.status).toBe(200);
	return { url, data: (await response.json()) as InspectorData };
}

describe("autopsy contract", () => {
	let session: PiSession | undefined;

	afterEach(async () => {
		await session?.dispose();
		session = undefined;
		vi.restoreAllMocks();
	});

	it("groups Pi 0.86 top-level prompt sections without mistaking nested XML", () => {
		const grouped = groupSystemPrompt(`Custom identity

<tools>
- read: Read files
</tools>

<rules>
- Be concise
</rules>

<general_guidelines>
General guidance
</general_guidelines>

<tool_guidelines>
<tool name="read">
Read carefully
</tool>
</tool_guidelines>

<project_context>
Project-specific instructions and guidelines:

<project_instructions path="/repo/AGENTS.md">
PROJECT RULE
</project_instructions>
</project_context>

<skills>
<available_skills>
</available_skills>
</skills>

<cwd>
/repo
</cwd>

<custom_context>
CUSTOM
</custom_context>`);

		expect(grouped).toContain("[PREAMBLE]\nCustom identity");
		expect(grouped).toContain("[TOOLS]\n<tools>");
		expect(grouped).toContain("[RULES]\n<rules>");
		expect(grouped).toContain("[GENERAL GUIDELINES]\n<general_guidelines>");
		expect(grouped).toContain("[TOOL GUIDELINES]\n<tool_guidelines>");
		expect(grouped).toContain("[CTX]\n<project_context>");
		expect(grouped).toContain("[SKILLS]\n<skills>");
		expect(grouped).toContain("[CWD]\n<cwd>");
		expect(grouped).toContain("[SECTION custom_context]\n<custom_context>");
		expect(grouped).not.toContain("[SECTION project_instructions]");
	});

	it("keeps nested same-name sections intact", () => {
		const grouped = groupSystemPrompt(`<addendum>
outer
<addendum>
inner
</addendum>
outer tail
</addendum>

<cwd>
/repo
</cwd>`);

		expect(grouped).toBe(`[APPEND]
<addendum>
outer
<addendum>
inner
</addendum>
outer tail
</addendum>

[CWD]
<cwd>
/repo
</cwd>`);
	});

	it("delivers an unchanged final prompt from an actual Pi run through the browser endpoint", async () => {
		const expected =
			"  Exact identity\n\n<tools>\n- read\n</tools>\n<script>literal</script>\n  ";
		session = await createPiSession({
			extensionPaths: [extensionPath()],
			extensionFactories: [
				(pi) => {
					// Loaded after autopsy: capture must include later prompt transformations.
					pi.on("before_agent_start", () => ({ systemPrompt: expected }));
				},
			],
			env: { PATH: "/no-browser-openers", TERMUX_VERSION: "fixture" },
		});
		await session.session.prompt("hello inspector");
		expect(session.providerState.callCount).toBeGreaterThan(0);
		const { url, data } = await openInspector(session);
		expect(data.prompts).toEqual([]);
		const prompts = await (await fetch(`${url}prompts`)).json();
		expect(prompts.at(-1)?.prompt).toBe(expected);
		expect(expected).toContain("Exact identity");
		expect(prompts.at(-1)?.grouped).toBe(groupSystemPrompt(expected));
		const filtered = await (
			await fetch(`${url}data?role=user&limit=1&order=oldest&q=hello`)
		).json();
		expect(filtered.entries).toHaveLength(1);
		expect(filtered.entries[0].role).toBe("user");
		expect(filtered.page.total).toBe(1);
		const metrics = await fetch(`${url}metrics?role=assistant&limit=1`);
		expect(metrics.status).toBe(200);
		const events = (await metrics.text())
			.trim()
			.split("\n")
			.map((line) => JSON.parse(line));
		expect(events.at(-1)).toEqual({ type: "end" });
		expect(events.slice(0, -1).length).toBeGreaterThan(0);
		expect(
			events
				.slice(0, -1)
				.every(
					(event) =>
						event.entry.role === "assistant" &&
						event.entry.text === "" &&
						event.entry.raw === "",
				),
		).toBe(true);
		expect(
			data.entries.some(
				(entry) =>
					entry.role === "user" && entry.text.includes("hello inspector"),
			),
		).toBe(true);
		expect(data.entries.some((entry) => entry.role === "assistant")).toBe(true);
		await session.dispose();
		session = undefined;
		await expect(fetch(`${url}data`)).rejects.toThrow();
	});

	it("retains bounded captures and clears them when the runtime restarts", async () => {
		session = await createPiSession({
			extensionPaths: [extensionPath()],
			env: { PATH: "/no-browser-openers", TERMUX_VERSION: "fixture" },
		});
		const runner = session.session.extensionRunner;
		await session.session.prompt("capture");
		for (let i = 0; i < 34; i++) await runner.emit({ type: "agent_start" });
		const { url, data } = await openInspector(session);
		expect(data.prompts).toEqual([]);
		const captures = await (await fetch(`${url}prompts`)).json();
		expect(captures).toHaveLength(32);
		expect(captures.at(-1)?.id).toBe(35);
		expect(data.warnings.some((warning) => warning.includes("expired"))).toBe(
			true,
		);
		await runner.emit({ type: "session_shutdown", reason: "reload" });
		await expect(fetch(`${url}data`)).rejects.toThrow();
		await runner.emit({ type: "session_start", reason: "reload" });
		const fresh = await openInspector(session);
		expect(fresh.url).not.toBe(url);
		expect(fresh.data.prompts).toEqual([]);
		expect(fresh.data.entries.length).toBeGreaterThan(0);
	});

	it("does not leave an inspector running when open overlaps shutdown", async () => {
		session = await createPiSession({
			extensionPaths: [extensionPath()],
			env: { PATH: "/no-browser-openers", TERMUX_VERSION: "fixture" },
		});
		initTheme(undefined, false);
		const runner = session.session.extensionRunner;
		const ctx = runner.createCommandContext();
		const urls: string[] = [];
		ctx.ui.notify = (message) => {
			const url = message.match(/http:\/\/127\.0\.0\.1:\d+\/\w+\//u)?.[0];
			if (url) urls.push(url);
		};
		const command = session.extensions[0]!.commands.get("autopsy")!;
		await Promise.all([
			command.handler("", ctx),
			runner.emit({ type: "session_shutdown", reason: "reload" }),
		]);
		await command.handler("", ctx);
		for (const url of urls) await expect(fetch(`${url}data`)).rejects.toThrow();
		const count = urls.length;
		await command.handler("", ctx);
		expect(urls).toHaveLength(count);
		await runner.emit({ type: "session_start", reason: "reload" });
		const fresh = await openInspector(session);
		expect((await fetch(fresh.url)).status).toBe(200);
	});

	it("writes its opt-in lifecycle trace inside the fixture sandbox", async () => {
		const root = mkdtempSync(path.join(tmpdir(), "pi-ext-autopsy-debug-"));
		try {
			session = await createPiSession({
				cwd: root,
				extensionPaths: [extensionPath()],
				env: {
					PI_AUTOPSY_DEBUG: "1",
					PATH: "/no-browser-openers",
					TERMUX_VERSION: "fixture",
				},
			});
			const runner = session.session.extensionRunner;
			await runner.emit({ type: "session_start", reason: "startup" });
			initTheme(undefined, false);
			const extension = session.extensions[0];
			if (!extension) throw new Error("missing extension");
			await extension.commands
				.get("autopsy")
				?.handler("", runner.createCommandContext());
			await runner.emit({ type: "session_shutdown", reason: "quit" });
			const stateHome = session.env["XDG_STATE_HOME"];
			if (!stateHome) throw new Error("missing sandbox state home");
			const directory = path.join(stateHome, "pi-ext", "debug", "autopsy");
			const [file] = readdirSync(directory);
			if (!file) throw new Error("missing debug file");
			const events = readFileSync(path.join(directory, file), "utf8")
				.trim()
				.split("\n")
				.map((line) => JSON.parse(line).event);
			expect(events).toEqual(
				expect.arrayContaining([
					"session.start",
					"session.shutdown",
					"prompt.skip",
				]),
			);
		} finally {
			await session?.dispose();
			session = undefined;
			rmSync(root, { recursive: true, force: true });
		}
	});

	it("loads in real Pi extension runtime", async () => {
		const extPath = extensionPath();
		session = await createPiSession({
			extensionPaths: [extPath],
			env: { PATH: "/no-browser-openers", TERMUX_VERSION: "fixture" },
		});

		expect(session.extensions).toHaveLength(1);
		const extension = session.extensions[0];
		if (!extension) throw new Error("missing extension");
		expect(extension.path).toBe(extPath);
		expect(extension.commands.has("autopsy")).toBe(true);
		expect(extension.commands.has("prompt-autopsy")).toBe(false);

		initTheme(undefined, false);
		const ctx = session.session.extensionRunner.createCommandContext();
		const notify = vi.spyOn(ctx.ui, "notify");
		const command = extension.commands.get("autopsy");
		if (!command) throw new Error("missing autopsy command");
		await command.handler("", ctx);
		const notification = notify.mock.calls.at(-1);
		expect(notification?.[1]).toBe("warning");
		const url = String(notification?.[0]).match(
			/http:\/\/127\.0\.0\.1:\d+\/\w+\//u,
		)?.[0];
		expect(url).toBeDefined();
		const data = (await (await fetch(`${url}data`)).json()) as InspectorData;
		expect(data.prompts).toEqual([]);
		expect((await fetch(String(url))).status).toBe(200);
	});
});