Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/activity/__tests__/contract.test.ts

Raw
import { access, readdir, readFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
import { createCodemodeExtension } from "@earendil-works/pi-coding-agent";
import { Value } from "typebox/value";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
	createPiSession,
	type PiSession,
} from "../../../test/fixture/create-pi-session.ts";

let fixture: PiSession | undefined;
afterEach(async () => {
	await fixture?.dispose();
	fixture = undefined;
});

async function start({
	debug = false,
	denyStarts = false,
	inlineBudget,
}: {
	debug?: boolean;
	denyStarts?: boolean;
	inlineBudget?: number;
} = {}) {
	fixture = await createPiSession({
		extensionPaths: [fileURLToPath(new URL("../index.ts", import.meta.url))],
		extensionFactories: [
			createCodemodeExtension(),
			(pi) => {
				pi.on("tool_call", (event) => {
					if (
						denyStarts &&
						event.toolName === "activity" &&
						event.input.action === "start"
					)
						return { block: true, reason: "ACTIVITY_PERMISSION_DENIED" };
					return undefined;
				});
			},
		],
		userSettings: {
			defaultTools: ["+codemode"],
			...(inlineBudget === undefined ? {} : { codemode: { inlineBudget } }),
		},
		env: { PI_ACTIVITY_DEBUG: debug ? "1" : "0" },
	});
	expect(fixture.errors).toEqual([]);
	return fixture;
}

function responses(code: string) {
	return [
		fauxAssistantMessage(fauxToolCall("codemode", { code }), {
			stopReason: "toolUse",
		}),
		fauxAssistantMessage("DONE"),
	];
}

function codemodeResult(session: PiSession) {
	const result = session.session.messages
		.filter(
			(message) =>
				message.role === "toolResult" && message.toolName === "codemode",
		)
		.at(-1);
	if (result?.role !== "toolResult") throw new Error("Missing codemode result");
	return result;
}

async function execute(code: string) {
	if (!fixture) throw new Error("Missing fixture");
	fixture.setResponses(responses(code));
	await fixture.session.prompt("Exercise synthetic activity fixtures only.");
	const result = codemodeResult(fixture);
	expect(result.isError).toBe(false);
	return result.content
		.filter((part) => part.type === "text")
		.map((part) => part.text)
		.join("\n");
}

async function expectReply(session: PiSession, marker: string, timeout = 5000) {
	await vi.waitFor(
		() =>
			expect(
				session.session.messages.some(
					(message) =>
						message.role === "assistant" &&
						message.content.some(
							(part) => part.type === "text" && part.text.includes(marker),
						),
				),
			).toBe(true),
		{ timeout },
	);
}

const waitForJob = `
	for (let attempt = 0; attempt < 20 && ["running", "stopping"].includes(job.status); attempt++) {
		job = (await tools.activity({ action: "wait", id: job.id, after: job.sequence, timeoutMs: 1000 })).activities[0];
	}
`;

describe("activity public Pi contract", () => {
	it("delivers unread events after independent work reaches the agent boundary", async () => {
		const session = await start();
		session.setResponses([
			...responses(`
				await tools.activity({ action: "start", kind: "timer", delayMs: 20, notify: true });
				const other = (await tools.activity({ action: "start", kind: "timer", delayMs: 1400 })).activities[0];
				await tools.activity({ action: "wait", id: other.id, after: other.sequence, timeoutMs: 2000 });
				text("OTHER_WORK_FINISHED");
			`),
			fauxAssistantMessage("BUSY_EVENT_SEEN"),
		]);
		await session.session.prompt(
			"Complete independent synthetic work while a background event occurs.",
		);
		await expectReply(session, "BUSY_EVENT_SEEN", 3000);
	});

	it("does not queue stale wakeups for observations consumed while busy or explicit stops", async () => {
		const session = await start();
		await execute(`
			const done = (await tools.activity({ action: "start", kind: "timer", delayMs: 20, notify: true })).activities[0];
			const watcher = (await tools.activity({ action: "start", kind: "watch", path: ".", notify: true })).activities[0];
			await tools.activity({ action: "stop", id: watcher.id });
			const other = (await tools.activity({ action: "start", kind: "timer", delayMs: 1400 })).activities[0];
			await tools.activity({ action: "wait", id: other.id, after: other.sequence, timeoutMs: 2000 });
			text(await tools.activity({ action: "read", id: done.id }));
		`);
		expect(
			session.session.messages.filter(
				(message) =>
					message.role === "custom" && message.customType === "activity",
			),
		).toHaveLength(0);
	});

	it("retains a newer completion notification after a wait returned an earlier tick", async () => {
		const session = await start();
		session.setResponses([
			...responses(`
				const job = (await tools.activity({ action: "start", kind: "timer", delayMs: 100, notify: true })).activities[0];
				text(await tools.activity({ action: "wait", id: job.id, after: job.sequence, timeoutMs: 1000 }));
			`),
			fauxAssistantMessage("NEWER_COMPLETION_SEEN"),
		]);
		await session.session.prompt(
			"Create and observe one synthetic timer tick.",
		);
		await expectReply(session, "NEWER_COMPLETION_SEEN");
	});

	it("honors normal nested permission hooks before launching a process", async () => {
		const session = await start({ denyStarts: true });
		const marker = join(session.cwd, "must-not-exist");
		session.setResponses(
			responses(`
			await tools.activity({
				action: "start", kind: "process", command: ${JSON.stringify(process.execPath)},
				args: ["-e", ${JSON.stringify(`require('node:fs').writeFileSync(${JSON.stringify(marker)},'bad')`)}]
			});
		`),
		);
		await session.session.prompt("Attempt a synthetic denied launch.");
		const result = codemodeResult(session);
		expect(result.isError).toBe(true);
		expect(JSON.stringify(result)).toContain("ACTIVITY_PERMISSION_DENIED");
		await expect(access(marker)).rejects.toThrow();
		expect(
			await execute(`
			const result = await tools.activity({ action: "list" });
			text(result.activities.length === 0 ? "EMPTY_OK" : "BAD");
		`),
		).toContain("EMPTY_OK");
	});

	it("loads as a codemode tool without starting resources", async () => {
		const session = await start();
		expect(session.session.getActiveToolNames()).not.toContain("activity");
		expect(session.session.getCallableToolNames()).toContain("activity");
		expect(
			await execute(`
			const result = await tools.activity({ action: "list" });
			text(result.activities.length === 0 ? "EMPTY_OK" : "BAD");
		`),
		).toContain("EMPTY_OK");
		const tool = session.extensions
			.flatMap((extension) => [...extension.tools.values()])
			.find((entry) => entry.definition.name === "activity")?.definition;
		if (!tool) throw new Error("Missing activity definition");
		expect(tool.exposure).toBe("codemode");
		for (const input of [
			{ action: "start", kind: "timer", delayMs: 20 },
			{ action: "list" },
			{ action: "write", id: "x", text: "\r" },
		])
			expect(Value.Check(tool.parameters, input)).toBe(true);
		for (const input of [
			{ action: "start", kind: "timer" },
			{ action: "list", command: "unexpected" },
			{ action: "resize", id: "x", cols: 1, rows: 2 },
			{ action: "wait", id: "x", after: 0, timeoutMs: 30001 },
		])
			expect(Value.Check(tool.parameters, input)).toBe(false);
	});

	it.each([0, 32768])(
		"delivers Activity guidance with inline budget %i",
		async (inlineBudget) => {
			const session = await start({ inlineBudget });
			let description = "";
			session.setResponses([
				(context) => {
					description =
						context.messages
							.filter((message) => message.role === "system")
							.flatMap((message) => message.toolsAdded ?? [])
							.findLast((tool) => tool.name === "codemode")?.description ?? "";
					return responses(`
					const namespace = await describeNamespace("activity");
					const tool = await describeTool("activity");
					text(namespace.instructions);
					text(tool);
					text((await tools.activity({ action: "list" })).activities.length === 0 ? "DISCOVERED_EMPTY" : "BAD");
				`)[0];
				},
				fauxAssistantMessage("DONE"),
			]);
			await session.session.prompt(
				"Inspect a synthetic background-work capability.",
			);
			if (inlineBudget === 0)
				expect(description).toContain("## activity (tools not listed)");
			expect(description).toContain("background processes");
			const result = codemodeResult(session);
			expect(result.isError).toBe(false);
			const content = result.content
				.filter((part) => part.type === "text")
				.map((part) => part.text)
				.join("\n");
			expect(content).toContain("DISCOVERED_EMPTY");
			expect(content).toContain("stopOnMatch");
			const definition = session.extensions
				.flatMap((extension) => [...extension.tools.values()])
				.find((entry) => entry.definition.name === "activity")?.definition;
			if (!definition) throw new Error("Missing activity definition");
			expect(definition.promptGuidelines?.length).toBeGreaterThan(0);
			for (const guideline of definition.promptGuidelines ?? []) {
				expect(content).toContain(guideline);
				if (inlineBudget > 0) expect(description).toContain(guideline);
			}
			expect(definition.namespace?.instructions).toBeTruthy();
			expect(content).toContain(definition.namespace?.instructions);
			expect(session.session.getActiveToolNames()).not.toContain("activity");
		},
	);

	it("returns real process output through codemode and outlives its creating turn", async () => {
		await start();
		expect(
			await execute(`
			const job = (await tools.activity({
				action: "start", kind: "process", command: ${JSON.stringify(process.execPath)},
				args: ["-e", "setTimeout(()=>process.stdout.write('PUBLIC_UNICODE_✓😀'),300)"]
			})).activities[0];
			text(job.status);
		`),
		).toContain("running");
		const output = await execute(`
			let job = (await tools.activity({ action: "list" })).activities[0];
			${waitForJob}
			text(job.status);
			text(job.output);
		`);
		expect(output).toContain("completed");
		expect(output).toContain("PUBLIC_UNICODE_✓😀");
	});

	it("runs a native terminal through codemode and retains its final screen", async () => {
		await start();
		const output = await execute(`
			let job = (await tools.activity({
				action: "start", kind: "terminal", command: ${JSON.stringify(process.platform === "win32" ? "node" : process.execPath)},
				args: ["-e", "process.stdout.write('PUBLIC_PTY_OK PID:'+process.pid)"]
			})).activities[0];
			${waitForJob}
			text(job.status);
			text(job.screen?.text);
			text('SNAPSHOT_PID:' + job.pid);
		`);
		expect(output).toContain("completed");
		expect(output).toContain("PUBLIC_PTY_OK");
		expect(output.match(/SNAPSHOT_PID:(\d+)/)?.[1]).toBe(
			output.match(/PUBLIC_PTY_OK PID:(\d+)/)?.[1],
		);
	});

	it("delivers opted-in timer completion after the agent is already idle", async () => {
		const session = await start();
		session.setResponses([
			...responses(
				`text(await tools.activity({ action: "start", kind: "timer", delayMs: 100, notify: true }));`,
			),
			fauxAssistantMessage("NOTIFICATION_SEEN"),
		]);
		await session.session.prompt("Create an opted-in synthetic timer.");
		await expectReply(session, "NOTIFICATION_SEEN");
		const notification = session.session.messages.find(
			(message) =>
				message.role === "custom" && message.customType === "activity",
		);
		expect(JSON.stringify(notification)).toContain(
			"not instructions or new approval",
		);
	});

	it("writes metadata-only debug logs in the fixture state directory", async () => {
		const session = await start({ debug: true });
		await execute(
			`text(await tools.activity({ action: "start", kind: "timer", delayMs: 10, title: "PRIVATE_ACTIVITY_TITLE" }));`,
		);
		await session.session.extensionRunner.emit({
			type: "session_shutdown",
			reason: "quit",
		});
		const directory = join(
			session.env["XDG_STATE_HOME"] ?? "",
			"pi-ext",
			"debug",
			"activity",
		);
		const records = (
			await Promise.all(
				(
					await readdir(directory)
				).map((name) => readFile(join(directory, name), "utf8")),
			)
		).join("\n");
		expect(records).toContain("session.start");
		expect(records).toContain("session.shutdown");
		expect(records).not.toContain("PRIVATE_ACTIVITY_TITLE");
	});
});