import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; import { createPiSession, type PiSession, } from "../../../test/fixture/create-pi-session.ts"; import { createMockupRuntime } from "../mockup-tool.ts"; function extensionPath(): string { return fileURLToPath(new URL("../index.ts", import.meta.url)); } async function submitReview( url: string, action: "accept" | "iterate", id: string, ) { const token = new URL(url).searchParams.get("k"); if (!token) throw new Error("mockup URL is missing its review token"); const response = await fetch(new URL("/api/submit", url), { method: "POST", headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", }, body: JSON.stringify({ action, selected: [id], notes: "reviewed", annotations: [], client: { userAgent: "mockup-contract-test", viewport: { width: 800, height: 600, devicePixelRatio: 1 }, }, }), }); if (!response.ok) throw new Error( `mockup submission failed: ${response.status} ${await response.text()}`, ); } describe("mockup pi-test-harness", () => { let session: PiSession | undefined; afterEach(async () => { await session?.dispose(); session = undefined; }); it("contributes review context through a named prompt section", async () => { session = await createPiSession({ extensionPaths: [extensionPath()] }); session.session.sessionManager.appendMessage({ role: "toolResult", toolCallId: "mockup-call", toolName: "mockup", content: [{ type: "text", text: "accepted" }], details: { sessionId: session.session.sessionManager.getSessionId(), chainId: "chain", iteration: 1, url: "http://127.0.0.1", history: [], contextCapsule: "review context", }, isError: false, timestamp: Date.now(), } as never); const result = await session.session.extensionRunner.emitBeforeAgentStart( { messageType: "user", prompt: "prompt" }, { cwd: session.cwd }, ); expect(result.systemPromptOptions.sections["mockup_context"]).toBe( "review context", ); expect(result.systemPromptOptions.forceSystemPrompt).toBeUndefined(); }); it("writes enabled lifecycle debug records in an isolated sandbox", async () => { const root = mkdtempSync(path.join(tmpdir(), "pi-ext-mockup-debug-")); try { session = await createPiSession({ cwd: root, env: { PI_MOCKUP_DEBUG: "1" }, extensionPaths: [extensionPath()], }); await session.session.extensionRunner.emit({ type: "session_start", reason: "startup", }); const definition = session.extensions[0]?.tools.get("mockup")?.definition; if (!definition?.execute) throw new Error("missing mockup tool"); expect(definition.exposure).toBe("model-only"); expect(definition.annotations?.readOnlyHint).toBe(false); expect( createMockupRuntime({ openBrowser: () => false }).tool.annotations ?.readOnlyHint, ).toBe(false); expect(definition.renderShell).toBeUndefined(); const base = session.session.extensionRunner.createToolContext( "mockup-debug", undefined, ); const ctx = Object.create(base) as typeof base; Object.defineProperties(ctx, { mode: { value: "tui" }, hasUI: { value: true }, ui: { value: { notify: () => undefined, select: async () => "Cancel", }, }, }); const cancelled = await definition.execute( "mockup-debug-cancel", { title: "Debug", question: "Continue?", options: [{ id: "one", label: "One", html: "

one

" }], }, undefined, undefined, ctx, ); expect(cancelled.details).toEqual({ kind: "cancelled" }); await expect( definition.execute("mockup-debug-fail", {}, undefined, undefined, ctx), ).rejects.toThrow(); await session.session.extensionRunner.emit({ type: "session_shutdown", reason: "quit", }); const stateHome = session.env["XDG_STATE_HOME"]; if (!stateHome) throw new Error("missing sandbox state home"); const debugDirectory = path.join(stateHome, "pi-ext", "debug", "mockup"); const events = readdirSync(debugDirectory).flatMap((file) => readFileSync(path.join(debugDirectory, file), "utf8") .trim() .split("\n") .map((line) => JSON.parse(line)), ); expect(events).toEqual( expect.arrayContaining([ expect.objectContaining({ extension: "mockup", event: "tool.outcome", outcome: "cancelled", }), expect.objectContaining({ extension: "mockup", event: "tool.execute.finish", outcome: "cancelled", }), expect.objectContaining({ extension: "mockup", event: "tool.outcome", outcome: "failed", }), expect.objectContaining({ extension: "mockup", event: "tool.execute.error", outcome: "failed", }), ]), ); } finally { await session?.dispose(); session = undefined; rmSync(root, { recursive: true, force: true }); } }); it("serializes model-issued batch calls through the real Pi runtime", async () => { session = await createPiSession({ extensionPaths: [extensionPath()], env: { PI_MOCKUP_NO_OPEN: "1" }, responses: [ fauxAssistantMessage( [ fauxToolCall( "mockup", { title: "First", question: "Review first", options: [ { id: "first", label: "First", html: "

first

" }, ], }, { id: "first-call" }, ), fauxToolCall( "mockup", { title: "Second", question: "Review second", options: [ { id: "second", label: "Second", html: "

second

" }, ], }, { id: "second-call" }, ), ], { stopReason: "toolUse" }, ), fauxAssistantMessage("done"), ], }); const starts: string[] = []; const urls = new Map(); const unsubscribe = session.session.subscribe((event) => { if (event.type === "tool_execution_start" && event.toolName === "mockup") starts.push(event.toolCallId); if ( event.type === "tool_execution_update" && event.toolName === "mockup" ) { const details = event.partialResult?.details as | { url?: string } | undefined; if (details?.url) urls.set(event.toolCallId, details.url); } }); const run = session.session.prompt("Review both mockups."); try { await vi.waitFor(() => expect(urls.has("first-call")).toBe(true)); await new Promise((resolve) => setTimeout(resolve, 30)); expect(starts).toContain("first-call"); expect(urls.size).toBe(1); await submitReview(urls.get("first-call")!, "iterate", "first"); await vi.waitFor(() => expect(urls.has("second-call")).toBe(true)); expect(starts).toEqual(["first-call", "second-call"]); await submitReview(urls.get("second-call")!, "accept", "second"); await run; expect(session.errors).toEqual([]); } finally { unsubscribe(); } }); it("closes the held review server when the real Pi session is disposed", async () => { session = await createPiSession({ extensionPaths: [extensionPath()], env: { PI_MOCKUP_NO_OPEN: "1" }, responses: [ fauxAssistantMessage( [ fauxToolCall( "mockup", { title: "Held", question: "Review held", options: [{ id: "held", label: "Held", html: "

held

" }], }, { id: "held-call" }, ), fauxToolCall( "mockup", { title: "Queued", question: "Review queued", options: [ { id: "queued", label: "Queued", html: "

queued

" }, ], }, { id: "queued-call" }, ), ], { stopReason: "toolUse" }, ), ], }); let url: string | undefined; const urls = new Map(); const unsubscribe = session.session.subscribe((event) => { if (event.type !== "tool_execution_update" || event.toolName !== "mockup") return; const details = event.partialResult?.details as | { url?: string } | undefined; if (details?.url) { urls.set(event.toolCallId, details.url); if (event.toolCallId === "held-call") url = details.url; } }); const run = session.session .prompt("Open held review.") .catch(() => undefined); try { await vi.waitFor(() => expect(url).toBeDefined()); await new Promise((resolve) => setTimeout(resolve, 30)); expect(urls.size).toBe(1); await session.dispose(); session = undefined; await run; expect(urls.has("queued-call")).toBe(false); await expect(fetch(url!)).rejects.toThrow(); } finally { unsubscribe(); } }); it("loads the tool and its authoring skill in the real Pi runtime", async () => { const extPath = extensionPath(); session = await createPiSession({ extensionPaths: [extPath] }); expect(session.extensions).toHaveLength(1); expect(session.extensions[0]!.path).toBe(extPath); expect(session.session.getActiveToolNames()).toContain("mockup"); expect(session.session.systemPrompt).toContain( "- mockup: Preview HTML mockups in Pi or review them interactively in a browser", ); const guidancePath = path.resolve( import.meta.dirname, "../guidance/mockup-html/SKILL.md", ); const skill = session.session.resourceLoader .getSkills() .skills.find((candidate) => candidate.name === "mockup-html"); expect(skill?.filePath).toBe(guidancePath); expect(session.session.systemPrompt).toContain("mockup-html"); expect(session.session.systemPrompt).toContain(guidancePath); expect(session.session.systemPrompt).not.toContain( "Rich interactions, scripts, and animations are allowed", ); }); });