Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/firefox-bidi/__tests__/harness.test.ts

Raw
import { EventEmitter } from "node:events";
import { existsSync, mkdtempSync, readdirSync } from "node:fs";
import { chmod, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { initTheme, keyHint } from "@earendil-works/pi-coding-agent";
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { createTestSession, type TestSession } from "../../../test/harness";
import { launch, teardown } from "../index.ts";

const extensionPath = path.resolve(import.meta.dirname, "../index.ts");

describe("firefox-bidi pi-test-harness", () => {
	let t: TestSession | undefined;

	beforeAll(() => initTheme("default", false));

	afterEach(() => {
		t?.dispose();
		t = undefined;
	});

	it("loads the tool inert, the command with completions, and the bundled skill", async () => {
		t = await createTestSession({ extensions: [extensionPath] });
		const runner = t.session.extensionRunner;
		expect(runner.extensions).toHaveLength(1);
		expect(runner.extensions[0]?.path).toBe(extensionPath);

		const tool = runner.extensions[0]?.tools.get("firefox_bidi")?.definition;
		expect(tool).toBeDefined();
		expect(tool?.description).toContain("WebDriver BiDi");
		expect(tool?.promptGuidelines).toHaveLength(2);

		// tool ships inert: not advertised until /firefox
		expect(t.session.getActiveToolNames()).not.toContain("firefox_bidi");
		expect(t.session.systemPrompt).not.toContain(
			"Drive Firefox with raw WebDriver BiDi frames",
		);

		const command = runner.getCommand("firefox");
		expect(command).toBeDefined();
		expect(command?.getArgumentCompletions?.("")).toEqual([
			{ value: "close", label: "close" },
		]);
		expect(command?.getArgumentCompletions?.("clo")).toEqual([
			{ value: "close", label: "close" },
		]);
		expect(command?.getArgumentCompletions?.("x")).toBeNull();

		const skill = t.session.resourceLoader
			.getSkills()
			.skills.find((candidate) => candidate.name === "firefox-bidi");
		expect(skill).toBeDefined();
	});

	it("renders compact status, partial progress, and expanded frame metadata", async () => {
		t = await createTestSession({ extensions: [extensionPath] });
		const tool =
			t.session.extensionRunner.extensions[0]?.tools.get(
				"firefox_bidi",
			)?.definition;
		expect(tool?.executionMode).toBe("sequential");
		expect(tool?.promptSnippet).toContain("Firefox");
		const theme = {
			fg: (_color: string, text: string) => text,
			bold: (text: string) => text,
		};
		const args = {
			frames: [
				{ method: "session.new", params: { marker: "private-value" } },
				{ method: "script.evaluate" },
			],
		};
		const call = (expanded: boolean) =>
			tool
				?.renderCall?.(args as never, theme as never, { expanded } as never)
				.render(100)
				.join("\n");
		expect(call(false)).toContain(" Firefox · session.new → script.evaluate");
		expect(call(false)).toContain(keyHint("app.tools.expand", "to expand"));
		expect(call(false)).not.toContain("private-value");
		expect(call(true)).toContain("2. script.evaluate");
		expect(call(true)).not.toContain("to expand");

		const details = {
			sessionId: "session-1",
			responses: 2,
			events: 3,
			frames: [
				{ id: 1, method: "session.new", status: "success" },
				{ id: 2, method: "script.evaluate", status: "error" },
			],
			wait: { timedOut: true, method: "browsingContext.load" },
		};
		const result = {
			content: [{ type: "text" as const, text: "raw data" }],
			details,
		};
		const render = (expanded: boolean, isPartial: boolean) =>
			tool
				?.renderResult?.(
					result,
					{ expanded, isPartial } as never,
					theme as never,
					{} as never,
				)
				.render(100)
				.join("\n");
		expect(render(false, false)).toContain("✗ 2 frames · 1 failed/skipped");
		expect(render(false, false)).not.toContain("3 events");
		expect(render(false, false)).not.toContain("script.evaluate");
		expect(render(false, false)).toContain("1 failed/skipped");
		expect(render(true, false)).toContain("2 script.evaluate: error");
		expect(render(true, false)).toContain("3 events");
		expect(render(true, false)).toContain("raw data");
		expect(render(true, false)).toContain("session: session-1");
		expect(render(false, true)).toContain("running");
		expect(render(false, false)).toContain("wait timed out");

		const data = {
			...result,
			content: [
				{
					type: "text" as const,
					text: JSON.stringify({
						session: { id: "session-1" },
						responses: [{ id: 2, result: { value: "page title" } }],
						events: [{ method: "noise" }],
					}),
				},
			],
		};
		const preview = (expanded: boolean) =>
			tool
				?.renderResult?.(
					data,
					{ expanded, isPartial: false } as never,
					theme as never,
					{} as never,
				)
				.render(100)
				.join("\n");
		expect(preview(false)).not.toContain("page title");
		expect(preview(true)).toContain("page title");
		expect(preview(true)).not.toContain("noise");
	});

	it("activates via /firefox and deactivates via /firefox close without launching", async () => {
		t = await createTestSession({
			extensions: [extensionPath],
			env: { FIREFOX_BIN: "/nonexistent/firefox-bin" },
		});
		const runner = t.session.extensionRunner;
		const command = runner.getCommand("firefox");
		const context = {
			...runner.createCommandContext(),
			mode: "print" as const,
			hasUI: false,
		};

		expect(t.session.getActiveToolNames()).not.toContain("firefox_bidi");
		await command?.handler("", context);
		expect(t.session.getActiveToolNames()).toContain("firefox_bidi");
		await command?.handler("close", context);
		expect(t.session.getActiveToolNames()).not.toContain("firefox_bidi");
	});

	it("activates on session_start with the PI_FIREFOX=1 dev hook", async () => {
		t = await createTestSession({
			extensions: [extensionPath],
			env: { PI_FIREFOX: "1", FIREFOX_BIN: "/nonexistent/firefox-bin" },
		});
		expect(t.session.getActiveToolNames()).toContain("firefox_bidi");
	});

	it("fails fast with a clear error when the firefox binary is missing", async () => {
		t = await createTestSession({
			extensions: [extensionPath],
			env: { FIREFOX_BIN: "/nonexistent/firefox-bin" },
		});
		const runner = t.session.extensionRunner;
		const command = runner.getCommand("firefox");
		const context = {
			...runner.createCommandContext(),
			mode: "print" as const,
			hasUI: false,
		};
		await command?.handler("", context);

		const tool = runner.extensions[0]?.tools.get("firefox_bidi")?.definition;
		await expect(
			tool?.execute(
				"call-1",
				{ frames: [{ method: "session.status" }] },
				undefined,
				undefined,
				context,
			),
		).rejects.toThrow(/firefox not found/);
	});

	it("retains profile and spool when fake child survives termination", async () => {
		const profileDir = mkdtempSync(path.join(tmpdir(), "pi-firefox-refuse-"));
		const spoolDir = mkdtempSync(path.join(tmpdir(), "pi-firefox-refuse-"));
		const child = Object.assign(new EventEmitter(), {
			exitCode: null,
			signalCode: null,
			kill: vi.fn(() => false),
		});
		try {
			await expect(
				teardown({
					child,
					profileDir,
					spoolDir,
					ws: null,
					waiters: [],
					spoolBytes: 0,
				} as never),
			).rejects.toThrow(/profile retained/);
			expect(child.kill).toHaveBeenCalledTimes(2);
			expect(existsSync(profileDir)).toBe(true);
			expect(existsSync(spoolDir)).toBe(true);
		} finally {
			await rm(profileDir, { recursive: true, force: true });
			await rm(spoolDir, { recursive: true, force: true });
		}
	}, 10_000);

	it("retains ownership after a failed launch with unkillable child", async () => {
		const child = Object.assign(new EventEmitter(), {
			exitCode: null,
			signalCode: null,
			kill: vi.fn(() => false),
			stdout: new EventEmitter(),
			stderr: new EventEmitter(),
		});
		const originalTimeout = globalThis.setTimeout;
		const timer = vi
			.spyOn(globalThis, "setTimeout")
			.mockImplementation(((
				fn: (...args: unknown[]) => void,
				ms?: number,
				...args: unknown[]
			) =>
				originalTimeout(
					fn,
					ms === 30_000 ? 5 : ms,
					...args,
				)) as typeof setTimeout);
		let survivor: Parameters<typeof teardown>[0] = null;
		try {
			await expect(
				launch(
					{},
					(b) => {
						survivor = b;
					},
					(() => child) as never,
					async () => false,
				),
			).rejects.toThrow(/still alive/);
			if (!survivor) throw new Error("lost survivor ownership");
			expect(existsSync(survivor.profileDir)).toBe(true);
			child.exitCode = 0;
			await teardown(survivor);
			expect(existsSync(survivor.profileDir)).toBe(false);
		} finally {
			timer.mockRestore();
			if (survivor) {
				await rm(survivor.profileDir, { recursive: true, force: true });
				await rm(survivor.spoolDir, { recursive: true, force: true });
			}
		}
	});

	it.runIf(process.platform !== "win32")(
		"rolls back profile and spool on process spawn error",
		async () => {
			const dir = mkdtempSync(path.join(tmpdir(), "pi-firefox-fake-"));
			const before = new Set(
				readdirSync(tmpdir()).filter(
					(name) =>
						name.startsWith("pi-firefox-") &&
						!name.startsWith("pi-firefox-fake-"),
				),
			);
			t = await createTestSession({
				extensions: [extensionPath],
				env: { FIREFOX_BIN: dir },
			});
			const runner = t.session.extensionRunner;
			const tool = runner.extensions[0]?.tools.get("firefox_bidi")?.definition;
			await expect(
				tool?.execute(
					"spawn-error",
					{ frames: [] },
					undefined,
					undefined,
					runner.createCommandContext(),
				),
			).rejects.toThrow(/EACCES/);
			const after = readdirSync(tmpdir()).filter(
				(name) =>
					name.startsWith("pi-firefox-") &&
					!name.startsWith("pi-firefox-fake-"),
			);
			expect(after.filter((name) => !before.has(name))).toEqual([]);
		},
	);

	it.runIf(process.platform !== "win32")(
		"rolls back listener timeout and terminates fake browser before profile deletion",
		async () => {
			const dir = mkdtempSync(path.join(tmpdir(), "pi-firefox-fake-"));
			const bin = path.join(dir, "firefox");
			await writeFile(bin, "#!/bin/sh\nexec sleep 1000\n");
			await chmod(bin, 0o755);
			const before = new Set(
				readdirSync(tmpdir()).filter(
					(name) =>
						name.startsWith("pi-firefox-") &&
						!name.startsWith("pi-firefox-fake-"),
				),
			);
			const originalTimeout = globalThis.setTimeout;
			const timer = vi
				.spyOn(globalThis, "setTimeout")
				.mockImplementation(((
					fn: (...args: unknown[]) => void,
					ms?: number,
					...args: unknown[]
				) =>
					originalTimeout(
						fn,
						ms === 30_000 ? 20 : ms,
						...args,
					)) as typeof setTimeout);
			try {
				t = await createTestSession({
					extensions: [extensionPath],
					env: { FIREFOX_BIN: bin },
				});
				const runner = t.session.extensionRunner;
				const tool =
					runner.extensions[0]?.tools.get("firefox_bidi")?.definition;
				await expect(
					tool?.execute(
						"listener-timeout",
						{ frames: [] },
						undefined,
						undefined,
						runner.createCommandContext(),
					),
				).rejects.toThrow(/no BiDi listener/);
				const after = readdirSync(tmpdir()).filter(
					(name) =>
						name.startsWith("pi-firefox-") &&
						!name.startsWith("pi-firefox-fake-"),
				);
				expect(after.filter((name) => !before.has(name))).toEqual([]);
			} finally {
				timer.mockRestore();
			}
		},
	);

	it.runIf(process.platform !== "win32")(
		"serializes tool and close, rejects incompatible launch options, and waits causally",
		async () => {
			const dir = mkdtempSync(path.join(tmpdir(), "pi-firefox-fake-"));
			const bin = path.join(dir, "firefox");
			await writeFile(
				bin,
				"#!/bin/sh\necho 'WebDriver BiDi listening on ws://127.0.0.1:9922' >&2\nexec sleep 1000\n",
			);
			await chmod(bin, 0o755);
			const original = globalThis.WebSocket;
			class Socket {
				static OPEN = 1;
				readyState = 1;
				onopen?: () => void;
				onclose?: () => void;
				onmessage?: (message: { data: string }) => void;
				onerror?: () => void;
				static sockets: Socket[] = [];
				constructor(_url: string) {
					Socket.sockets.push(this);
					queueMicrotask(() => this.onopen?.());
				}
				send(text: string) {
					const frame = JSON.parse(text);
					queueMicrotask(() =>
						this.onmessage?.({
							data: JSON.stringify({
								id: frame.id,
								type: "success",
								result:
									frame.method === "script.evaluate"
										? { value: "x".repeat(300 * 1024) }
										: {},
							}),
						}),
					);
				}
				close() {
					this.readyState = 3;
					this.onclose?.();
				}
				event(method: string, context: string) {
					this.onmessage?.({
						data: JSON.stringify({
							type: "event",
							method,
							params: { context },
						}),
					});
				}
			}
			(globalThis as { WebSocket: unknown }).WebSocket = Socket;
			let retainedFile: string | undefined;
			try {
				t = await createTestSession({
					extensions: [extensionPath],
					env: { FIREFOX_BIN: bin },
				});
				const runner = t.session.extensionRunner;
				const tool =
					runner.extensions[0]?.tools.get("firefox_bidi")?.definition;
				if (!tool) throw new Error("missing firefox_bidi tool");
				const ctx = runner.createCommandContext();
				const execute = (args: Record<string, unknown>) =>
					tool.execute("test", args as never, undefined, undefined, ctx);
				await execute({ frames: [{ id: 40, method: "session.status" }] });
				const other = await createTestSession({
					extensions: [extensionPath],
					env: { FIREFOX_BIN: bin },
				});
				try {
					const second = other.session.extensionRunner;
					const otherTool =
						second.extensions[0]?.tools.get("firefox_bidi")?.definition;
					await otherTool?.execute(
						"separate",
						{ frames: [{ id: 40, method: "session.status" }] },
						undefined,
						undefined,
						second.createCommandContext(),
					);
				} finally {
					await other.session.extensionRunner
						.getCommand("firefox")
						?.handler(
							"close",
							other.session.extensionRunner.createCommandContext(),
						);
					other.dispose();
				}
				const firstSocket = Socket.sockets[0];
				firstSocket?.event("browsingContext.load", "old");
				await expect(
					execute({ frames: [{ id: 40, method: "session.status" }] }),
				).rejects.toThrow(/reused frame id/);
				await expect(
					execute({ frames: [], launch: { access: true } }),
				).rejects.toThrow(/different launch options/);
				const pending = execute({
					frames: [],
					wait: "browsingContext.load",
					waitContext: "new",
					waitTimeoutMs: 100,
				});
				await new Promise((resolve) => setTimeout(resolve, 5));
				firstSocket?.event("browsingContext.load", "wrong");
				firstSocket?.event("browsingContext.load", "new");
				const result = await pending;
				expect(JSON.parse(result.content[0].text).wait.params.context).toBe(
					"new",
				);
				const many = execute({
					frames: [],
					wait: ["browsingContext.load", "browsingContext.load"],
					waitTimeoutMs: 100,
				});
				await new Promise((resolve) => setTimeout(resolve, 5));
				firstSocket?.event("browsingContext.load", "first");
				firstSocket?.event("browsingContext.load", "second");
				expect(
					JSON.parse((await many).content[0].text).wait.map(
						(event: { params: { context: string } }) => event.params.context,
					),
				).toEqual(["first", "second"]);
				const oversized = execute({
					frames: [],
					wait: "log.entryAdded",
					waitTimeoutMs: 100,
				});
				await new Promise((resolve) => setTimeout(resolve, 5));
				firstSocket?.onmessage?.({
					data: JSON.stringify({
						type: "event",
						method: "log.entryAdded",
						params: {
							context: "secret".repeat(100_000),
							data: "🚀".repeat(10_000),
						},
					}),
				});
				const trimmed = await oversized;
				expect(JSON.parse(trimmed.content[0].text).wait.params.truncated).toBe(
					true,
				);
				expect(JSON.stringify(trimmed.details).length).toBeLessThan(10_000);
				if (!firstSocket) throw new Error("missing first socket");
				firstSocket.readyState = 3;
				await execute({ frames: [] });
				const current = Socket.sockets.at(-1);
				if (!current) throw new Error("missing replacement socket");
				const replacementWait = execute({
					frames: [],
					wait: "browsingContext.load",
					waitTimeoutMs: 100,
				});
				await new Promise((resolve) => setTimeout(resolve, 5));
				firstSocket?.event("browsingContext.load", "stale");
				firstSocket?.close();
				current.event("browsingContext.load", "current");
				expect(
					JSON.parse((await replacementWait).content[0].text).wait.params
						.context,
				).toBe("current");
				const big = await execute({ frames: [{ method: "script.evaluate" }] });
				const file = JSON.parse(big.content[0].text).responses[0].result
					.spooled as string;
				retainedFile = file;
				expect(existsSync(file)).toBe(true);
				const command = runner.getCommand("firefox");
				await command?.handler("close", ctx);
				expect(existsSync(file)).toBe(true);
				await runner.emit({ type: "session_start", reason: "new" });
				expect(existsSync(file)).toBe(false);
				const nextBig = await execute({
					frames: [{ method: "script.evaluate" }],
					launch: { access: true },
				});
				retainedFile = JSON.parse(nextBig.content[0].text).responses[0].result
					.spooled as string;
				await command?.handler("close", ctx);
				expect(existsSync(retainedFile)).toBe(true);
			} finally {
				await t?.session.extensionRunner
					.getCommand("firefox")
					?.handler("close", t.session.extensionRunner.createCommandContext());
				(globalThis as { WebSocket: unknown }).WebSocket = original;
			}
			if (!retainedFile) throw new Error("missing retained spool file");
			await t?.session.extensionRunner.emit({
				type: "session_shutdown",
				reason: "quit",
			});
			expect(existsSync(retainedFile)).toBe(false);
			t?.dispose();
			t = undefined;
		},
	);

	it.runIf(process.platform !== "win32")(
		"surfaces immediate browser exit through a controlled fake firefox on PATH",
		async () => {
			const fakeDir = mkdtempSync(path.join(tmpdir(), "pi-firefox-fake-"));
			const fakeBin = path.join(fakeDir, "fake-firefox");
			await writeFile(fakeBin, "#!/bin/sh\nexit 7\n");
			await chmod(fakeBin, 0o755);

			t = await createTestSession({
				extensions: [extensionPath],
				env: { FIREFOX_BIN: fakeBin },
			});
			const runner = t.session.extensionRunner;
			const command = runner.getCommand("firefox");
			const context = {
				...runner.createCommandContext(),
				mode: "print" as const,
				hasUI: false,
			};
			await command?.handler("", context);

			const tool = runner.extensions[0]?.tools.get("firefox_bidi")?.definition;
			await expect(
				tool?.execute(
					"call-2",
					{ frames: [{ method: "session.status" }] },
					undefined,
					undefined,
					context,
				),
			).rejects.toThrow(/firefox exited immediately \(code 7\)/);
		},
	);
});