Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/chrome-cdp/__tests__/codemode.test.ts

Raw
import { mkdir, readFile } from "node:fs/promises";
import { join } from "node:path";
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
import { createCodemodeExtension } from "@earendil-works/pi-coding-agent";
import { Type } from "typebox";
import { Value } from "typebox/value";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
	createPiSession,
	type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import { Browser } from "../browser.ts";
import chromeExtension from "../index.ts";

const png =
	"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
let fixture: PiSession | undefined;
let captured: unknown;
let browser: Browser;
let nativeError = false;

afterEach(async () => {
	await fixture?.dispose();
	fixture = undefined;
	vi.restoreAllMocks();
});

async function start() {
	captured = undefined;
	nativeError = false;
	fixture = await createPiSession({
		extensionFactories: [
			chromeExtension,
			createCodemodeExtension(),
			(pi) => {
				pi.registerTool({
					name: "capture",
					label: "Capture fixture",
					description: "Capture test data",
					exposure: "codemode",
					parameters: Type.Object({ value: Type.Unknown() }),
					async execute(_id, { value }) {
						captured = value;
						return { content: [], details: {} };
					},
				});
			},
		],
		userSettings: { defaultTools: ["+codemode"] },
	});
	browser = new Browser();
	browser.initialSession = "fixture-session";
	browser.initialTarget = "fixture-target";
	browser.sessions.set(browser.initialSession, browser.initialTarget);
	browser.version = "Fixture Chrome";
	browser.endpoint = "http://127.0.0.1:9999";
	browser.spoolDir = join(fixture.cwd, "spool");
	browser.artifactDir = join(fixture.cwd, "captures");
	await mkdir(browser.spoolDir);
	vi.spyOn(Browser, "launch").mockResolvedValue(browser);
	vi.spyOn(browser, "close").mockImplementation(async () => {
		browser.closed = true;
	});
	vi.spyOn(browser, "send").mockImplementation(async (method) => ({
		id: 1,
		result:
			method === "Page.captureScreenshot" ? { data: png } : { value: method },
	}));
	fixture.session.subscribe((event) => {
		if (event.type === "tool_execution_end" && event.toolName === "chrome_cdp")
			nativeError = event.isError;
	});
	expect(fixture.errors).toEqual([]);
}

async function execute(code: string): Promise<Record<string, any>> {
	if (!fixture) throw new Error("Missing fixture");
	fixture.setResponses([
		fauxAssistantMessage(fauxToolCall("codemode", { code }), {
			stopReason: "toolUse",
		}),
		fauxAssistantMessage("DONE"),
	]);
	await fixture.session.prompt("Exercise controlled browser fixture only.");
	const result = fixture.session.messages.findLast(
		(message) =>
			message.role === "toolResult" && message.toolName === "codemode",
	);
	if (result?.role !== "toolResult") throw new Error("Missing codemode result");
	expect(result.isError, JSON.stringify(result.content)).toBe(false);
	expect(captured, JSON.stringify(result.content)).toBeDefined();
	return captured as Record<string, any>;
}

describe("Chrome structured codemode contract", () => {
	it("filters raw command replies and retains first-session metadata without parsing text", async () => {
		await start();
		const data = await execute(
			'const r=await tools.chrome_cdp({operations:[{kind:"cdp",method:"Browser.getVersion"},{kind:"cdp",method:"Target.getTargets"}]}); await tools.capture({value:{reply:r,selected:r.output.map(item=>item.raw.result.value)}});',
		);
		expect(data["selected"]).toEqual([
			"Browser.getVersion",
			"Target.getTargets",
		]);
		expect(data["reply"]).toMatchObject({
			outcome: "ok",
			status: {
				initialSession: "fixture-session",
				protocol: "http://127.0.0.1:9999/json/protocol",
			},
		});
		const schema =
			fixture!.extensions[0]!.tools.get("chrome_cdp")!.definition.outputSchema!;
		expect(Value.Check(schema, data["reply"])).toBe(true);
	});

	it.each([24, 160_000])(
		"preserves partial errors as bounded data and stops later effects (%s)",
		async (size) => {
			await start();
			const send = vi.spyOn(browser, "send").mockResolvedValue({
				id: 1,
				error: {
					code: -1,
					message:
						"deliberate protocol failure" + "x".repeat(size) + "END-OF-ERROR",
				},
			});
			const data = await execute(
				'const r=await tools.chrome_cdp({operations:[{kind:"cdp",method:"Bad.command"},{kind:"cdp",method:"Never.sent"}]}); await tools.capture({value:r});',
			);
			expect(data).toMatchObject({
				outcome: "partial",
				failure: expect.stringContaining("deliberate"),
				failedOperation: { operation: 0 },
			});
			expect(data["skipped"]).toHaveLength(1);
			expect(send).toHaveBeenCalledTimes(1);
			expect(nativeError).toBe(true);
			if (size > 128_000) {
				expect(JSON.stringify(data).length).toBeLessThan(4096);
				expect(data["failure"]).not.toContain("END-OF-ERROR");
				const full = JSON.parse(
					await readFile(String(data["resultSpool"]), "utf8"),
				);
				expect(full.failure).toContain("END-OF-ERROR");
			}
		},
	);

	it("recovers all omitted replies and oversized status from bounded spool references", async () => {
		await start();
		browser.observedTargets = [
			{ id: "large-target", title: "t".repeat(200_000) },
		];
		vi.spyOn(browser, "send").mockImplementation(async (method) => ({
			id: 1,
			result: {
				value: method === "Large.first" ? "x".repeat(80_000) : "later reply",
			},
		}));
		const data = await execute(
			'const r=await tools.chrome_cdp({operations:[{kind:"cdp",method:"Large.first"},{kind:"cdp",method:"Later.second"}]}); await tools.capture({value:r});',
		);
		expect(data).toMatchObject({
			outcome: "ok",
			truncated: true,
			status: { initialSession: "fixture-session" },
		});
		expect(JSON.stringify(data).length).toBeLessThan(4096);
		const full = JSON.parse(
			await readFile(String(data["resultSpool"]), "utf8"),
		);
		const first = full.output[0].raw.path
			? JSON.parse(await readFile(full.output[0].raw.path, "utf8"))
			: full.output[0].raw;
		expect(first.result.value).toHaveLength(80_000);
		expect(full.output[1].raw.result.value).toBe("later reply");
		expect(full.status.targets[0].title).toHaveLength(200_000);
	});

	it("returns an exact image artifact locator without base64 in machine data", async () => {
		await start();
		const data = await execute(
			'const r=await tools.chrome_cdp({operations:[{kind:"screenshot",sessionId:"fixture-session",filename:"fixture.png"}]}); await tools.capture({value:r});',
		);
		const file = data["output"][0].file;
		expect(await readFile(file)).toEqual(Buffer.from(png, "base64"));
		expect(JSON.stringify(data)).not.toContain(png);
	});

	it("propagates codemode session abort into the held command", async () => {
		await start();
		let entered!: () => void;
		const started = new Promise<void>((resolve) => {
			entered = resolve;
		});
		let signal: AbortSignal | undefined;
		vi.spyOn(browser, "send").mockImplementation(
			async (_method, _params, _sessionId, operationSignal) => {
				signal = operationSignal;
				entered();
				return new Promise((_resolve, reject) => {
					signal!.addEventListener("abort", () => reject(signal!.reason), {
						once: true,
					});
				});
			},
		);
		fixture!.setResponses([
			fauxAssistantMessage(
				fauxToolCall("codemode", {
					code: 'return await tools.chrome_cdp({operations:[{kind:"cdp",method:"Held.command"}]});',
				}),
				{ stopReason: "toolUse" },
			),
			fauxAssistantMessage("DONE"),
		]);
		const running = fixture!.session.prompt(
			"Cancel controlled browser request.",
		);
		await Promise.race([
			started,
			running.then(() => {
				throw new Error("Command never entered");
			}),
		]);
		await fixture!.session.abort();
		await running;
		expect(signal?.aborted).toBe(true);
	});
});