import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync, } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; import { createCodemodeExtension, type ExtensionUIContext, withFileMutationQueue, } from "@earendil-works/pi-coding-agent"; import { Value } from "typebox/value"; import { afterEach, describe, expect, it, vi } from "vitest"; import { createPiSession, type PiSession, } from "../../../test/fixture/create-pi-session.ts"; import { createBoardRuntime } from "../board-runtime.ts"; import { applyInitialization } from "../init.ts"; const ENTRY = fileURLToPath(new URL("../index.ts", import.meta.url)); function codemodeResult(session: PiSession) { const result = session.session.messages .filter( (message) => message.role === "toolResult" && message.toolName === "codemode", ) .at(-1); if (result?.role !== "toolResult") throw new Error("Missing codemode result"); return result; } const recordingUi = ( select: (title: string, items: string[]) => string | undefined, ): ExtensionUIContext => ({ select: async (title: string, items: string[]) => select(title, items), confirm: async () => false, input: async () => undefined, notify: () => {}, onTerminalInput: () => () => {}, setStatus: () => {}, setWorkingMessage: () => {}, setWorkingVisible: () => {}, setWorkingIndicator: () => {}, setHiddenThinkingLabel: () => {}, setWidget: () => {}, setFooter: () => {}, setHeader: () => {}, setTitle: () => {}, custom: async () => undefined, pasteToEditor: () => {}, setEditorText: () => {}, getEditorText: () => "", editor: async () => undefined, addAutocompleteProvider: () => {}, setEditorComponent: () => {}, getEditorComponent: () => undefined, getAllThemes: () => [], getTheme: () => undefined, setTheme: () => ({ success: false, error: "UI not available" }), getToolsExpanded: () => false, setToolsExpanded: () => {}, }) as unknown as ExtensionUIContext; describe("the-system pi-test-harness", () => { let session: PiSession | undefined; afterEach(async () => { await session?.dispose(); session = undefined; }); it("loads in real Pi extension runtime", async () => { session = await createPiSession({ extensionPaths: [ENTRY] }); expect(session.extensions).toHaveLength(1); expect(session.extensions[0]!.path).toBe(ENTRY); expect(session.session.getActiveToolNames()).not.toContain("mockup"); }); it("writes debug lifecycle records in its sandbox when enabled", async () => { session = await createPiSession({ env: { PI_THE_SYSTEM_DEBUG: "1" }, extensionPaths: [ENTRY], }); const context = session.session.extensionRunner.createCommandContext(); const [lifecycleExtension] = session.extensions; if (!lifecycleExtension) throw new Error("the-system extension not loaded"); for (const handler of lifecycleExtension.handlers.get("session_start") ?? []) await handler({ type: "session_start" }, context); for (const handler of lifecycleExtension.handlers.get("session_shutdown") ?? []) await handler({ type: "session_shutdown" }, context); const directory = path.join( session.cwd, ".test-state", "pi-ext", "debug", "the-system", ); const records = readFileSync( path.join(directory, readdirSync(directory)[0]!), "utf8", ) .trim() .split("\n") .map((line) => JSON.parse(line)); expect(records.map((record) => record.event)).toEqual( expect.arrayContaining(["session.start", "session.shutdown"]), ); }); it("routes authenticated board exec into this Pi session and shuts down cleanly", async () => { session = await createPiSession({ extensionPaths: [ENTRY], env: { NODE_ENV: "test" }, }); applyInitialization(session.cwd, "PX"); const directory = path.join( session.cwd, ".system", "specs", "PX-SPEC-SPEC0001-contract", ); mkdirSync(directory, { recursive: true }); writeFileSync( path.join(directory, "index.md"), `---\nid: PX-SPEC-SPEC0001\ntype: spec\ntitle: Contract\n---\n\n# Contract\n`, ); const extension = session.extensions.find((item) => item.path === ENTRY); if (!extension) throw new Error("the-system extension not loaded"); const command = extension.commands.get("system"); if (!command) throw new Error("system command not registered"); const notices: string[] = []; const ui = recordingUi(() => undefined); session.session.extensionRunner.setUIContext( { ...ui, notify: (message: string) => notices.push(message) }, "tui", ); const send = vi .spyOn(session.session, "sendUserMessage") .mockResolvedValue(); const context = session.session.extensionRunner.createCommandContext(); let idle = true; await command.handler("board", { ...context, isIdle: () => idle }); const url = notices .join("\\n") .match(/http:\/\/127\.0\.0\.1:\d+\/board\.html\?k=[\w-]+/)?.[0]; expect(url).toBeDefined(); const authenticated = await fetch(url!, { redirect: "manual" }); expect(authenticated.status).toBe(303); const cookie = authenticated.headers.get("set-cookie")?.split(";")[0]; expect(cookie).toBeDefined(); const origin = new URL(url!).origin; const exec = await fetch(`${origin}/api/exec`, { method: "POST", headers: { cookie: cookie!, origin, "content-type": "application/json" }, body: JSON.stringify({ id: "PX-SPEC-SPEC0001" }), }); expect(exec.status).toBe(200); expect(send).toHaveBeenCalledWith( expect.stringContaining("# /system exec: PX-SPEC-SPEC0001"), undefined, ); send.mockClear(); idle = false; const busy = await fetch(`${origin}/api/exec`, { method: "POST", headers: { cookie: cookie!, origin, "content-type": "application/json" }, body: JSON.stringify({ id: "PX-SPEC-SPEC0001" }), }); expect(busy.status).toBe(400); expect(await busy.json()).toEqual({ error: expect.stringContaining("Wait for the current Pi run"), }); expect(send).not.toHaveBeenCalled(); for (const handler of extension.handlers.get("session_shutdown") ?? []) await handler({ type: "session_shutdown" }, context); }); it("serializes codemode board batches and returns bundle slots as structured data", async () => { session = await createPiSession({ extensionPaths: [ENTRY], extensionFactories: [createCodemodeExtension()], userSettings: { defaultTools: ["+codemode"] }, responses: [ fauxAssistantMessage( fauxToolCall("codemode", { code: ` await Promise.all([ tools.system_board_update({ items: [{ id: "PX-SPEC-SPEC0001", state: "todo", note: "Spec reviewed." }] }), tools.system_board_update({ items: [{ id: "PX-ISSUE-ISSUE001", state: "done", note: "Issue verified." }] }), ]); let rejected = false; try { await tools.system_board_update({ items: [{ id: "PX-SPEC-MISSING", state: "done", note: "Missing item." }] }); } catch { rejected = true; } await tools.system_board_update({ items: [{ id: "PX-ISSUE-ISSUE001", state: "done", note: "Queue recovered." }] }); const slot = await tools.system_bundle_slot({ type: "plan", title: "Concurrent board updates" }); text(JSON.stringify({ rejected, slot })); `, }), { stopReason: "toolUse" }, ), fauxAssistantMessage("DONE"), ], }); applyInitialization(session.cwd, "PX"); for (const [relative, source] of [ [ ".system/specs/PX-SPEC-SPEC0001-contract/index.md", "---\nid: PX-SPEC-SPEC0001\ntype: spec\ntitle: Contract\n---\n\n# Contract\n", ], [ ".system/plans/PX-PLAN-PLAN0001-work/index.md", "---\nid: PX-PLAN-PLAN0001\ntype: plan\ntitle: Work\nspec: PX-SPEC-SPEC0001\n---\n\n# Work\n", ], [ ".system/issues/PX-ISSUE-ISSUE001-bug/index.md", "---\nid: PX-ISSUE-ISSUE001\ntype: issue\ntitle: Bug\nspecs: []\n---\n\n# Bug\n", ], ] as const) { const target = path.join(session.cwd, relative); mkdirSync(path.dirname(target), { recursive: true }); writeFileSync(target, source); } await session.session.prompt( "Run concurrent board updates and reserve a plan bundle.", ); const result = codemodeResult(session); expect(result.isError).toBe(false); const output = result.content .filter((part) => part.type === "text") .map((part) => part.text) .join("\n"); const jsonStart = output.indexOf("{"); const jsonEnd = output.lastIndexOf("}") + 1; const { rejected, slot } = JSON.parse(output.slice(jsonStart, jsonEnd)) as { rejected: boolean; slot: Record; }; expect(rejected).toBe(true); const board = JSON.parse( readFileSync(path.join(session.cwd, ".system/board.json"), "utf8"), ); expect(board.items["PX-SPEC-SPEC0001"]).toMatchObject({ state: "todo", source: "agent", note: "Spec reviewed.", }); expect(board.items["PX-ISSUE-ISSUE001"]).toMatchObject({ state: "done", source: "agent", note: "Queue recovered.", }); const extension = session.extensions.find((item) => item.path === ENTRY); const definition = extension?.tools.get("system_bundle_slot")?.definition; if (!definition?.outputSchema) throw new Error("Missing bundle slot output schema"); expect(Value.Check(definition.outputSchema, slot)).toBe(true); expect(slot).toMatchObject({ type: "plan", title: "Concurrent board updates", id: expect.stringMatching(/^PX-PLAN-[A-Z0-9_]{8}$/), path: `.system/plans/${slot["id"]}-concurrent-board-updates/index.md`, }); expect(session.errors).toEqual([]); }); it.each(["system_board_update", "system_board_propose"])( "cancels queued %s before any mutation", async (name) => { session = await createPiSession({ extensionPaths: [ENTRY] }); applyInitialization(session.cwd, "PX"); const board = path.join(session.cwd, ".system/board.json"); const before = existsSync(board) ? readFileSync(board, "utf8") : undefined; let release!: () => void; let entered!: () => void; const held = new Promise((resolve) => { release = resolve; }); const started = new Promise((resolve) => { entered = resolve; }); const locked = withFileMutationQueue(board, async () => { entered(); await held; }); await started; const tool = session.extensions[0]!.tools.get(name)!.definition; const controller = new AbortController(); const reason = new Error("cancelled while queued"); const pending = tool.execute( "queued", { items: [] }, controller.signal, undefined, session.session.extensionRunner.createToolContext( "queued", controller.signal, ), ); controller.abort(reason); const rejected = expect(pending).rejects.toThrow(/cancel/); release(); await Promise.all([locked, rejected]); expect(existsSync(board) ? readFileSync(board, "utf8") : undefined).toBe( before, ); }, ); it("uses replacement proposals in an already-open board server", async () => { session = await createPiSession({}); applyInitialization(session.cwd, "PX"); const issue = path.join( session.cwd, ".system/issues/PX-ISSUE-ISSUE001-bug/index.md", ); mkdirSync(path.dirname(issue), { recursive: true }); writeFileSync( issue, "---\nid: PX-ISSUE-ISSUE001\ntype: issue\ntitle: Bug\nspecs: []\n---\n\n# Bug\n", ); const runtime = createBoardRuntime(); const ctx = session.session.extensionRunner.createToolContext( "proposal", undefined, ); const propose = (state: string) => runtime.tool.execute( "proposal", { items: [ { id: "PX-ISSUE-ISSUE001", state, rationale: "Reviewed evidence." }, ], }, undefined, undefined, ctx, ); try { await propose("open"); const opened = await runtime.open(session.cwd); const url = opened.match(/http:\/\/127\.0\.0\.1:[^\s]+/)?.[0]; if (!url) throw new Error("Missing local board URL"); const page = await fetch(url, { redirect: "manual" }); const cookie = page.headers.get("set-cookie")?.split(";")[0]; if (!cookie) throw new Error("Missing board cookie"); const origin = new URL(url).origin; await propose("done"); const accept = (state: string) => fetch(`${origin}/api/proposal`, { method: "POST", headers: { "Content-Type": "application/json", Cookie: cookie, Origin: origin, }, body: JSON.stringify({ id: "PX-ISSUE-ISSUE001", state }), }); expect((await accept("open")).status).toBe(400); expect((await accept("done")).status).toBe(200); } finally { runtime.shutdown(); } }); it("registers scoped tools deferred and callable without a System stage", async () => { session = await createPiSession({ extensionPaths: [ENTRY] }); applyInitialization(session.cwd, "PX"); const extension = session.extensions.find((item) => item.path === ENTRY); if (!extension) throw new Error("the-system extension not loaded"); for (const name of [ "system_bundle_slot", "system_board_propose", "system_board_update", ]) { const tool = extension.tools.get(name)?.definition; expect(tool?.exposure).toBe("deferred"); expect(tool?.annotations?.readOnlyHint).toBe(false); expect(session.session.getCallableToolNames()).toContain(name); expect(session.session.getActiveToolNames()).not.toContain(name); expect(session.session.systemPrompt).not.toContain(`- ${name}:`); } }); it.each([true, false])( "preserves nested codemode mutation approval: %s", async (approve) => { session = await createPiSession({ extensionPaths: [ENTRY], extensionFactories: [createCodemodeExtension()], userSettings: { defaultTools: ["+codemode"] }, responses: [ fauxAssistantMessage( fauxToolCall("codemode", { code: 'await tools.edit({ path: ".system/SYSTEM.md", edits: [{ oldText: "Approved.", newText: "Human-approved." }] });', }), ), fauxAssistantMessage("approval requested"), ], }); applyInitialization(session.cwd, "PX"); const extension = session.extensions.find((item) => item.path === ENTRY); if (!extension) throw new Error("the-system extension not loaded"); const systemCommand = extension.commands.get("system"); if (!systemCommand) throw new Error("system command not registered"); vi.spyOn(session.session, "sendUserMessage").mockResolvedValue(); await systemCommand.handler( "spec nested confirmation", session.session.extensionRunner.createCommandContext(), ); writeFileSync( path.join(session.cwd, ".system", "SYSTEM.md"), "Approved.\n", ); const nestedStarts: { toolName: string; parentToolCallId?: string | undefined; }[] = []; const unsubscribe = session.session.subscribe((event) => { if (event.type === "tool_execution_start" && event.toolName === "edit") nestedStarts.push({ toolName: event.toolName, parentToolCallId: event.parentToolCallId, }); }); const selectCalls: { args: [string, string[]] }[] = []; session.session.extensionRunner.setUIContext( recordingUi((title, items) => { selectCalls.push({ args: [title, items] }); return items[approve ? 0 : 1]; }), "tui", ); await session.session.prompt("run the scripted nested edit"); unsubscribe(); const [approval] = selectCalls; expect(approval?.args).toEqual([ "The System ownership approval", [ "Approve once: edit .system/SYSTEM.md", "Deny: keep .system/SYSTEM.md unchanged", ], ]); expect(nestedStarts).toEqual([ { toolName: "edit", parentToolCallId: expect.any(String) }, ]); expect( readFileSync(path.join(session.cwd, ".system", "SYSTEM.md"), "utf8"), ).toBe(approve ? "Human-approved.\n" : "Approved.\n"); }, ); });