Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/the-system/__tests__/contract.test.ts

Raw
import {
	existsSync,
	mkdirSync,
	readdirSync,
	readFileSync,
	writeFileSync,
} from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
import {
	createCodemodeExtension,
	type ExtensionUIContext,
	withFileMutationQueue,
} from "@earendil-works/pi-coding-agent";
import { Value } from "typebox/value";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
	createPiSession,
	type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import { createBoardRuntime } from "../board-runtime.ts";
import { applyInitialization } from "../init.ts";

const ENTRY = fileURLToPath(new URL("../index.ts", import.meta.url));

function codemodeResult(session: PiSession) {
	const result = session.session.messages
		.filter(
			(message) =>
				message.role === "toolResult" && message.toolName === "codemode",
		)
		.at(-1);
	if (result?.role !== "toolResult") throw new Error("Missing codemode result");
	return result;
}

const recordingUi = (
	select: (title: string, items: string[]) => string | undefined,
): ExtensionUIContext =>
	({
		select: async (title: string, items: string[]) => select(title, items),
		confirm: async () => false,
		input: async () => undefined,
		notify: () => {},
		onTerminalInput: () => () => {},
		setStatus: () => {},
		setWorkingMessage: () => {},
		setWorkingVisible: () => {},
		setWorkingIndicator: () => {},
		setHiddenThinkingLabel: () => {},
		setWidget: () => {},
		setFooter: () => {},
		setHeader: () => {},
		setTitle: () => {},
		custom: async () => undefined,
		pasteToEditor: () => {},
		setEditorText: () => {},
		getEditorText: () => "",
		editor: async () => undefined,
		addAutocompleteProvider: () => {},
		setEditorComponent: () => {},
		getEditorComponent: () => undefined,
		getAllThemes: () => [],
		getTheme: () => undefined,
		setTheme: () => ({ success: false, error: "UI not available" }),
		getToolsExpanded: () => false,
		setToolsExpanded: () => {},
	}) as unknown as ExtensionUIContext;

describe("the-system pi-test-harness", () => {
	let session: PiSession | undefined;

	afterEach(async () => {
		await session?.dispose();
		session = undefined;
	});

	it("loads in real Pi extension runtime", async () => {
		session = await createPiSession({ extensionPaths: [ENTRY] });

		expect(session.extensions).toHaveLength(1);
		expect(session.extensions[0]!.path).toBe(ENTRY);
		expect(session.session.getActiveToolNames()).not.toContain("mockup");
	});

	it("writes debug lifecycle records in its sandbox when enabled", async () => {
		session = await createPiSession({
			env: { PI_THE_SYSTEM_DEBUG: "1" },
			extensionPaths: [ENTRY],
		});
		const context = session.session.extensionRunner.createCommandContext();
		const [lifecycleExtension] = session.extensions;
		if (!lifecycleExtension) throw new Error("the-system extension not loaded");
		for (const handler of lifecycleExtension.handlers.get("session_start") ??
			[])
			await handler({ type: "session_start" }, context);
		for (const handler of lifecycleExtension.handlers.get("session_shutdown") ??
			[])
			await handler({ type: "session_shutdown" }, context);
		const directory = path.join(
			session.cwd,
			".test-state",
			"pi-ext",
			"debug",
			"the-system",
		);
		const records = readFileSync(
			path.join(directory, readdirSync(directory)[0]!),
			"utf8",
		)
			.trim()
			.split("\n")
			.map((line) => JSON.parse(line));
		expect(records.map((record) => record.event)).toEqual(
			expect.arrayContaining(["session.start", "session.shutdown"]),
		);
	});

	it("routes authenticated board exec into this Pi session and shuts down cleanly", async () => {
		session = await createPiSession({
			extensionPaths: [ENTRY],
			env: { NODE_ENV: "test" },
		});
		applyInitialization(session.cwd, "PX");
		const directory = path.join(
			session.cwd,
			".system",
			"specs",
			"PX-SPEC-SPEC0001-contract",
		);
		mkdirSync(directory, { recursive: true });
		writeFileSync(
			path.join(directory, "index.md"),
			`---\nid: PX-SPEC-SPEC0001\ntype: spec\ntitle: Contract\n---\n\n# Contract\n`,
		);
		const extension = session.extensions.find((item) => item.path === ENTRY);
		if (!extension) throw new Error("the-system extension not loaded");
		const command = extension.commands.get("system");
		if (!command) throw new Error("system command not registered");
		const notices: string[] = [];
		const ui = recordingUi(() => undefined);
		session.session.extensionRunner.setUIContext(
			{ ...ui, notify: (message: string) => notices.push(message) },
			"tui",
		);
		const send = vi
			.spyOn(session.session, "sendUserMessage")
			.mockResolvedValue();
		const context = session.session.extensionRunner.createCommandContext();
		let idle = true;
		await command.handler("board", { ...context, isIdle: () => idle });
		const url = notices
			.join("\\n")
			.match(/http:\/\/127\.0\.0\.1:\d+\/board\.html\?k=[\w-]+/)?.[0];
		expect(url).toBeDefined();
		const authenticated = await fetch(url!, { redirect: "manual" });
		expect(authenticated.status).toBe(303);
		const cookie = authenticated.headers.get("set-cookie")?.split(";")[0];
		expect(cookie).toBeDefined();
		const origin = new URL(url!).origin;
		const exec = await fetch(`${origin}/api/exec`, {
			method: "POST",
			headers: { cookie: cookie!, origin, "content-type": "application/json" },
			body: JSON.stringify({ id: "PX-SPEC-SPEC0001" }),
		});
		expect(exec.status).toBe(200);
		expect(send).toHaveBeenCalledWith(
			expect.stringContaining("# /system exec: PX-SPEC-SPEC0001"),
			undefined,
		);
		send.mockClear();
		idle = false;
		const busy = await fetch(`${origin}/api/exec`, {
			method: "POST",
			headers: { cookie: cookie!, origin, "content-type": "application/json" },
			body: JSON.stringify({ id: "PX-SPEC-SPEC0001" }),
		});
		expect(busy.status).toBe(400);
		expect(await busy.json()).toEqual({
			error: expect.stringContaining("Wait for the current Pi run"),
		});
		expect(send).not.toHaveBeenCalled();
		for (const handler of extension.handlers.get("session_shutdown") ?? [])
			await handler({ type: "session_shutdown" }, context);
	});

	it("serializes codemode board batches and returns bundle slots as structured data", async () => {
		session = await createPiSession({
			extensionPaths: [ENTRY],
			extensionFactories: [createCodemodeExtension()],
			userSettings: { defaultTools: ["+codemode"] },
			responses: [
				fauxAssistantMessage(
					fauxToolCall("codemode", {
						code: `
							await Promise.all([
								tools.system_board_update({ items: [{ id: "PX-SPEC-SPEC0001", state: "todo", note: "Spec reviewed." }] }),
								tools.system_board_update({ items: [{ id: "PX-ISSUE-ISSUE001", state: "done", note: "Issue verified." }] }),
							]);
							let rejected = false;
							try {
								await tools.system_board_update({ items: [{ id: "PX-SPEC-MISSING", state: "done", note: "Missing item." }] });
							} catch { rejected = true; }
							await tools.system_board_update({ items: [{ id: "PX-ISSUE-ISSUE001", state: "done", note: "Queue recovered." }] });
							const slot = await tools.system_bundle_slot({ type: "plan", title: "Concurrent board updates" });
						text(JSON.stringify({ rejected, slot }));
						`,
					}),
					{ stopReason: "toolUse" },
				),
				fauxAssistantMessage("DONE"),
			],
		});
		applyInitialization(session.cwd, "PX");
		for (const [relative, source] of [
			[
				".system/specs/PX-SPEC-SPEC0001-contract/index.md",
				"---\nid: PX-SPEC-SPEC0001\ntype: spec\ntitle: Contract\n---\n\n# Contract\n",
			],
			[
				".system/plans/PX-PLAN-PLAN0001-work/index.md",
				"---\nid: PX-PLAN-PLAN0001\ntype: plan\ntitle: Work\nspec: PX-SPEC-SPEC0001\n---\n\n# Work\n",
			],
			[
				".system/issues/PX-ISSUE-ISSUE001-bug/index.md",
				"---\nid: PX-ISSUE-ISSUE001\ntype: issue\ntitle: Bug\nspecs: []\n---\n\n# Bug\n",
			],
		] as const) {
			const target = path.join(session.cwd, relative);
			mkdirSync(path.dirname(target), { recursive: true });
			writeFileSync(target, source);
		}
		await session.session.prompt(
			"Run concurrent board updates and reserve a plan bundle.",
		);
		const result = codemodeResult(session);
		expect(result.isError).toBe(false);
		const output = result.content
			.filter((part) => part.type === "text")
			.map((part) => part.text)
			.join("\n");
		const jsonStart = output.indexOf("{");
		const jsonEnd = output.lastIndexOf("}") + 1;
		const { rejected, slot } = JSON.parse(output.slice(jsonStart, jsonEnd)) as {
			rejected: boolean;
			slot: Record<string, unknown>;
		};
		expect(rejected).toBe(true);
		const board = JSON.parse(
			readFileSync(path.join(session.cwd, ".system/board.json"), "utf8"),
		);
		expect(board.items["PX-SPEC-SPEC0001"]).toMatchObject({
			state: "todo",
			source: "agent",
			note: "Spec reviewed.",
		});
		expect(board.items["PX-ISSUE-ISSUE001"]).toMatchObject({
			state: "done",
			source: "agent",
			note: "Queue recovered.",
		});
		const extension = session.extensions.find((item) => item.path === ENTRY);
		const definition = extension?.tools.get("system_bundle_slot")?.definition;
		if (!definition?.outputSchema)
			throw new Error("Missing bundle slot output schema");
		expect(Value.Check(definition.outputSchema, slot)).toBe(true);
		expect(slot).toMatchObject({
			type: "plan",
			title: "Concurrent board updates",
			id: expect.stringMatching(/^PX-PLAN-[A-Z0-9_]{8}$/),
			path: `.system/plans/${slot["id"]}-concurrent-board-updates/index.md`,
		});
		expect(session.errors).toEqual([]);
	});

	it.each(["system_board_update", "system_board_propose"])(
		"cancels queued %s before any mutation",
		async (name) => {
			session = await createPiSession({ extensionPaths: [ENTRY] });
			applyInitialization(session.cwd, "PX");
			const board = path.join(session.cwd, ".system/board.json");
			const before = existsSync(board)
				? readFileSync(board, "utf8")
				: undefined;
			let release!: () => void;
			let entered!: () => void;
			const held = new Promise<void>((resolve) => {
				release = resolve;
			});
			const started = new Promise<void>((resolve) => {
				entered = resolve;
			});
			const locked = withFileMutationQueue(board, async () => {
				entered();
				await held;
			});
			await started;
			const tool = session.extensions[0]!.tools.get(name)!.definition;
			const controller = new AbortController();
			const reason = new Error("cancelled while queued");
			const pending = tool.execute(
				"queued",
				{ items: [] },
				controller.signal,
				undefined,
				session.session.extensionRunner.createToolContext(
					"queued",
					controller.signal,
				),
			);
			controller.abort(reason);
			const rejected = expect(pending).rejects.toThrow(/cancel/);
			release();
			await Promise.all([locked, rejected]);
			expect(existsSync(board) ? readFileSync(board, "utf8") : undefined).toBe(
				before,
			);
		},
	);

	it("uses replacement proposals in an already-open board server", async () => {
		session = await createPiSession({});
		applyInitialization(session.cwd, "PX");
		const issue = path.join(
			session.cwd,
			".system/issues/PX-ISSUE-ISSUE001-bug/index.md",
		);
		mkdirSync(path.dirname(issue), { recursive: true });
		writeFileSync(
			issue,
			"---\nid: PX-ISSUE-ISSUE001\ntype: issue\ntitle: Bug\nspecs: []\n---\n\n# Bug\n",
		);
		const runtime = createBoardRuntime();
		const ctx = session.session.extensionRunner.createToolContext(
			"proposal",
			undefined,
		);
		const propose = (state: string) =>
			runtime.tool.execute(
				"proposal",
				{
					items: [
						{ id: "PX-ISSUE-ISSUE001", state, rationale: "Reviewed evidence." },
					],
				},
				undefined,
				undefined,
				ctx,
			);
		try {
			await propose("open");
			const opened = await runtime.open(session.cwd);
			const url = opened.match(/http:\/\/127\.0\.0\.1:[^\s]+/)?.[0];
			if (!url) throw new Error("Missing local board URL");
			const page = await fetch(url, { redirect: "manual" });
			const cookie = page.headers.get("set-cookie")?.split(";")[0];
			if (!cookie) throw new Error("Missing board cookie");
			const origin = new URL(url).origin;
			await propose("done");
			const accept = (state: string) =>
				fetch(`${origin}/api/proposal`, {
					method: "POST",
					headers: {
						"Content-Type": "application/json",
						Cookie: cookie,
						Origin: origin,
					},
					body: JSON.stringify({ id: "PX-ISSUE-ISSUE001", state }),
				});
			expect((await accept("open")).status).toBe(400);
			expect((await accept("done")).status).toBe(200);
		} finally {
			runtime.shutdown();
		}
	});

	it("registers scoped tools deferred and callable without a System stage", async () => {
		session = await createPiSession({ extensionPaths: [ENTRY] });
		applyInitialization(session.cwd, "PX");
		const extension = session.extensions.find((item) => item.path === ENTRY);
		if (!extension) throw new Error("the-system extension not loaded");
		for (const name of [
			"system_bundle_slot",
			"system_board_propose",
			"system_board_update",
		]) {
			const tool = extension.tools.get(name)?.definition;
			expect(tool?.exposure).toBe("deferred");
			expect(tool?.annotations?.readOnlyHint).toBe(false);
			expect(session.session.getCallableToolNames()).toContain(name);
			expect(session.session.getActiveToolNames()).not.toContain(name);
			expect(session.session.systemPrompt).not.toContain(`- ${name}:`);
		}
	});

	it.each([true, false])(
		"preserves nested codemode mutation approval: %s",
		async (approve) => {
			session = await createPiSession({
				extensionPaths: [ENTRY],
				extensionFactories: [createCodemodeExtension()],
				userSettings: { defaultTools: ["+codemode"] },
				responses: [
					fauxAssistantMessage(
						fauxToolCall("codemode", {
							code: 'await tools.edit({ path: ".system/SYSTEM.md", edits: [{ oldText: "Approved.", newText: "Human-approved." }] });',
						}),
					),
					fauxAssistantMessage("approval requested"),
				],
			});
			applyInitialization(session.cwd, "PX");
			const extension = session.extensions.find((item) => item.path === ENTRY);
			if (!extension) throw new Error("the-system extension not loaded");
			const systemCommand = extension.commands.get("system");
			if (!systemCommand) throw new Error("system command not registered");
			vi.spyOn(session.session, "sendUserMessage").mockResolvedValue();
			await systemCommand.handler(
				"spec nested confirmation",
				session.session.extensionRunner.createCommandContext(),
			);
			writeFileSync(
				path.join(session.cwd, ".system", "SYSTEM.md"),
				"Approved.\n",
			);
			const nestedStarts: {
				toolName: string;
				parentToolCallId?: string | undefined;
			}[] = [];
			const unsubscribe = session.session.subscribe((event) => {
				if (event.type === "tool_execution_start" && event.toolName === "edit")
					nestedStarts.push({
						toolName: event.toolName,
						parentToolCallId: event.parentToolCallId,
					});
			});
			const selectCalls: { args: [string, string[]] }[] = [];
			session.session.extensionRunner.setUIContext(
				recordingUi((title, items) => {
					selectCalls.push({ args: [title, items] });
					return items[approve ? 0 : 1];
				}),
				"tui",
			);

			await session.session.prompt("run the scripted nested edit");
			unsubscribe();
			const [approval] = selectCalls;
			expect(approval?.args).toEqual([
				"The System ownership approval",
				[
					"Approve once: edit .system/SYSTEM.md",
					"Deny: keep .system/SYSTEM.md unchanged",
				],
			]);
			expect(nestedStarts).toEqual([
				{ toolName: "edit", parentToolCallId: expect.any(String) },
			]);
			expect(
				readFileSync(path.join(session.cwd, ".system", "SYSTEM.md"), "utf8"),
			).toBe(approve ? "Human-approved.\n" : "Approved.\n");
		},
	);
});