Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/mockup/__tests__/contract.test.ts

Raw
import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
	createPiSession,
	type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import { createMockupRuntime } from "../mockup-tool.ts";

function extensionPath(): string {
	return fileURLToPath(new URL("../index.ts", import.meta.url));
}

async function submitReview(
	url: string,
	action: "accept" | "iterate",
	id: string,
) {
	const token = new URL(url).searchParams.get("k");
	if (!token) throw new Error("mockup URL is missing its review token");
	const response = await fetch(new URL("/api/submit", url), {
		method: "POST",
		headers: {
			Authorization: `Bearer ${token}`,
			"Content-Type": "application/json",
		},
		body: JSON.stringify({
			action,
			selected: [id],
			notes: "reviewed",
			annotations: [],
			client: {
				userAgent: "mockup-contract-test",
				viewport: { width: 800, height: 600, devicePixelRatio: 1 },
			},
		}),
	});
	if (!response.ok)
		throw new Error(
			`mockup submission failed: ${response.status} ${await response.text()}`,
		);
}

describe("mockup pi-test-harness", () => {
	let session: PiSession | undefined;

	afterEach(async () => {
		await session?.dispose();
		session = undefined;
	});

	it("contributes review context through a named prompt section", async () => {
		session = await createPiSession({ extensionPaths: [extensionPath()] });
		session.session.sessionManager.appendMessage({
			role: "toolResult",
			toolCallId: "mockup-call",
			toolName: "mockup",
			content: [{ type: "text", text: "accepted" }],
			details: {
				sessionId: session.session.sessionManager.getSessionId(),
				chainId: "chain",
				iteration: 1,
				url: "http://127.0.0.1",
				history: [],
				contextCapsule: "review context",
			},
			isError: false,
			timestamp: Date.now(),
		} as never);

		const result = await session.session.extensionRunner.emitBeforeAgentStart(
			{ messageType: "user", prompt: "prompt" },
			{ cwd: session.cwd },
		);
		expect(result.systemPromptOptions.sections["mockup_context"]).toBe(
			"review context",
		);
		expect(result.systemPromptOptions.forceSystemPrompt).toBeUndefined();
	});

	it("writes enabled lifecycle debug records in an isolated sandbox", async () => {
		const root = mkdtempSync(path.join(tmpdir(), "pi-ext-mockup-debug-"));
		try {
			session = await createPiSession({
				cwd: root,
				env: { PI_MOCKUP_DEBUG: "1" },
				extensionPaths: [extensionPath()],
			});
			await session.session.extensionRunner.emit({
				type: "session_start",
				reason: "startup",
			});
			const definition = session.extensions[0]?.tools.get("mockup")?.definition;
			if (!definition?.execute) throw new Error("missing mockup tool");
			expect(definition.exposure).toBe("model-only");
			expect(definition.annotations?.readOnlyHint).toBe(false);
			expect(
				createMockupRuntime({ openBrowser: () => false }).tool.annotations
					?.readOnlyHint,
			).toBe(false);
			expect(definition.renderShell).toBeUndefined();
			const base = session.session.extensionRunner.createToolContext(
				"mockup-debug",
				undefined,
			);
			const ctx = Object.create(base) as typeof base;
			Object.defineProperties(ctx, {
				mode: { value: "tui" },
				hasUI: { value: true },
				ui: {
					value: {
						notify: () => undefined,
						select: async () => "Cancel",
					},
				},
			});
			const cancelled = await definition.execute(
				"mockup-debug-cancel",
				{
					title: "Debug",
					question: "Continue?",
					options: [{ id: "one", label: "One", html: "<p>one</p>" }],
				},
				undefined,
				undefined,
				ctx,
			);
			expect(cancelled.details).toEqual({ kind: "cancelled" });
			await expect(
				definition.execute("mockup-debug-fail", {}, undefined, undefined, ctx),
			).rejects.toThrow();
			await session.session.extensionRunner.emit({
				type: "session_shutdown",
				reason: "quit",
			});
			const stateHome = session.env["XDG_STATE_HOME"];
			if (!stateHome) throw new Error("missing sandbox state home");
			const debugDirectory = path.join(stateHome, "pi-ext", "debug", "mockup");
			const events = readdirSync(debugDirectory).flatMap((file) =>
				readFileSync(path.join(debugDirectory, file), "utf8")
					.trim()
					.split("\n")
					.map((line) => JSON.parse(line)),
			);
			expect(events).toEqual(
				expect.arrayContaining([
					expect.objectContaining({
						extension: "mockup",
						event: "tool.outcome",
						outcome: "cancelled",
					}),
					expect.objectContaining({
						extension: "mockup",
						event: "tool.execute.finish",
						outcome: "cancelled",
					}),
					expect.objectContaining({
						extension: "mockup",
						event: "tool.outcome",
						outcome: "failed",
					}),
					expect.objectContaining({
						extension: "mockup",
						event: "tool.execute.error",
						outcome: "failed",
					}),
				]),
			);
		} finally {
			await session?.dispose();
			session = undefined;
			rmSync(root, { recursive: true, force: true });
		}
	});

	it("serializes model-issued batch calls through the real Pi runtime", async () => {
		session = await createPiSession({
			extensionPaths: [extensionPath()],
			env: { PI_MOCKUP_NO_OPEN: "1" },
			responses: [
				fauxAssistantMessage(
					[
						fauxToolCall(
							"mockup",
							{
								title: "First",
								question: "Review first",
								options: [
									{ id: "first", label: "First", html: "<p>first</p>" },
								],
							},
							{ id: "first-call" },
						),
						fauxToolCall(
							"mockup",
							{
								title: "Second",
								question: "Review second",
								options: [
									{ id: "second", label: "Second", html: "<p>second</p>" },
								],
							},
							{ id: "second-call" },
						),
					],
					{ stopReason: "toolUse" },
				),
				fauxAssistantMessage("done"),
			],
		});
		const starts: string[] = [];
		const urls = new Map<string, string>();
		const unsubscribe = session.session.subscribe((event) => {
			if (event.type === "tool_execution_start" && event.toolName === "mockup")
				starts.push(event.toolCallId);
			if (
				event.type === "tool_execution_update" &&
				event.toolName === "mockup"
			) {
				const details = event.partialResult?.details as
					| { url?: string }
					| undefined;
				if (details?.url) urls.set(event.toolCallId, details.url);
			}
		});
		const run = session.session.prompt("Review both mockups.");
		try {
			await vi.waitFor(() => expect(urls.has("first-call")).toBe(true));
			await new Promise((resolve) => setTimeout(resolve, 30));
			expect(starts).toContain("first-call");
			expect(urls.size).toBe(1);
			await submitReview(urls.get("first-call")!, "iterate", "first");
			await vi.waitFor(() => expect(urls.has("second-call")).toBe(true));
			expect(starts).toEqual(["first-call", "second-call"]);
			await submitReview(urls.get("second-call")!, "accept", "second");
			await run;
			expect(session.errors).toEqual([]);
		} finally {
			unsubscribe();
		}
	});

	it("closes the held review server when the real Pi session is disposed", async () => {
		session = await createPiSession({
			extensionPaths: [extensionPath()],
			env: { PI_MOCKUP_NO_OPEN: "1" },
			responses: [
				fauxAssistantMessage(
					[
						fauxToolCall(
							"mockup",
							{
								title: "Held",
								question: "Review held",
								options: [{ id: "held", label: "Held", html: "<p>held</p>" }],
							},
							{ id: "held-call" },
						),
						fauxToolCall(
							"mockup",
							{
								title: "Queued",
								question: "Review queued",
								options: [
									{ id: "queued", label: "Queued", html: "<p>queued</p>" },
								],
							},
							{ id: "queued-call" },
						),
					],
					{ stopReason: "toolUse" },
				),
			],
		});
		let url: string | undefined;
		const urls = new Map<string, string>();
		const unsubscribe = session.session.subscribe((event) => {
			if (event.type !== "tool_execution_update" || event.toolName !== "mockup")
				return;
			const details = event.partialResult?.details as
				| { url?: string }
				| undefined;
			if (details?.url) {
				urls.set(event.toolCallId, details.url);
				if (event.toolCallId === "held-call") url = details.url;
			}
		});
		const run = session.session
			.prompt("Open held review.")
			.catch(() => undefined);
		try {
			await vi.waitFor(() => expect(url).toBeDefined());
			await new Promise((resolve) => setTimeout(resolve, 30));
			expect(urls.size).toBe(1);
			await session.dispose();
			session = undefined;
			await run;
			expect(urls.has("queued-call")).toBe(false);
			await expect(fetch(url!)).rejects.toThrow();
		} finally {
			unsubscribe();
		}
	});

	it("loads the tool and its authoring skill in the real Pi runtime", async () => {
		const extPath = extensionPath();
		session = await createPiSession({ extensionPaths: [extPath] });

		expect(session.extensions).toHaveLength(1);
		expect(session.extensions[0]!.path).toBe(extPath);
		expect(session.session.getActiveToolNames()).toContain("mockup");
		expect(session.session.systemPrompt).toContain(
			"- mockup: Preview HTML mockups in Pi or review them interactively in a browser",
		);
		const guidancePath = path.resolve(
			import.meta.dirname,
			"../guidance/mockup-html/SKILL.md",
		);
		const skill = session.session.resourceLoader
			.getSkills()
			.skills.find((candidate) => candidate.name === "mockup-html");
		expect(skill?.filePath).toBe(guidancePath);
		expect(session.session.systemPrompt).toContain("<name>mockup-html</name>");
		expect(session.session.systemPrompt).toContain(guidancePath);
		expect(session.session.systemPrompt).not.toContain(
			"Rich interactions, scripts, and animations are allowed",
		);
	});
});