Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/afk/__tests__/contract.test.ts

Raw
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import {
	type FauxResponseStep,
	fauxAssistantMessage,
	fauxToolCall,
	type TranscriptContext,
} from "@earendil-works/pi-ai";
import {
	defineTool,
	type ExtensionFactory,
} from "@earendil-works/pi-coding-agent";
import type { Component, TUI } from "@earendil-works/pi-tui";
import { Type } from "typebox";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
	createPiSession,
	type PiSession,
} from "../../../test/fixture/create-pi-session.ts";

import afk from "../index.ts";
import { catTheme } from "./theme.ts";

let session: PiSession | undefined;
const path = fileURLToPath(new URL("../index.ts", import.meta.url));
function command(name: "afk" | "back") {
	const result = session?.extensions
		.flatMap((extension) => [...extension.commands])
		.find(([key]) => key === name)?.[1];
	if (!result) throw new Error(`Missing ${name}`);
	return result;
}
function text(context: TranscriptContext) {
	return context.messages
		.map((message) =>
			[
				message.role === "system"
					? Object.values(message.sections ?? {}).join("\n")
					: "",
				typeof message.content === "string"
					? message.content
					: message.content
							.flatMap((block) => (block.type === "text" ? [block.text] : []))
							.join("\n"),
			].join("\n"),
		)
		.join("\n");
}
function script(
	options: {
		objections?: number;
		executor?: FauxResponseStep;
		initial?: () => Promise<void>;
		review?: () => Promise<void>;
		invalid?: boolean;
	} = {},
) {
	const observed = { reviews: 0, executions: 0, intents: 0, reviewText: "" };
	const step: FauxResponseStep = async (
		context,
		modelOptions,
		state,
		model,
	) => {
		const input = text(context);
		if (input.includes("AFK_JUDGE:")) {
			const review = input.includes('"phase":"review"');
			if (review) {
				observed.reviews++;
				observed.reviewText = input;
				await options.review?.();
			} else {
				observed.intents++;
				await options.initial?.();
			}
			if (options.invalid && review) return fauxAssistantMessage("not JSON");
			return fauxAssistantMessage(
				JSON.stringify({
					goal: "Finish OAuth acceptance",
					intent:
						"Finish all requested acceptance, not just scoped green checks. Cite latest user messages.",
					objections:
						review && observed.reviews <= (options.objections ?? 0)
							? [
									"Acceptance remains unfinished: verify publication faults before stopping.",
								]
							: [],
				}),
			);
		}
		observed.executions++;
		return typeof options.executor === "function"
			? options.executor(context, modelOptions, state, model)
			: (options.executor ??
					fauxAssistantMessage(
						"Implemented scoped work; publication acceptance remains unfinished.",
					));
	};
	return { responses: Array.from({ length: 40 }, () => step), observed };
}
async function start() {
	await command("afk").handler(
		"",
		session!.session.extensionRunner.createContext() as never,
	);
	await session!.session.waitForIdle();
}
function entries(type: string) {
	return session!.session.sessionManager
		.getBranch()
		.filter(
			(entry) => entry.type === "custom_message" && entry.customType === type,
		);
}
const humanExecution = vi.fn();
const human: ExtensionFactory = (pi) => {
	pi.registerTool(
		defineTool({
			name: "ask_user_question",
			label: "Ask",
			description: "Ask a human",
			parameters: Type.Object({}),
			async execute() {
				humanExecution();
				return {
					content: [{ type: "text", text: "Human answered" }],
					details: undefined,
				};
			},
		}),
	);
};
afterEach(async () => {
	await session?.dispose();
	session = undefined;
});

describe("AFK judged continuation through real Pi", () => {
	it("has no completion tool and naturally settles after one clean review", async () => {
		const run = script();
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		expect(session.extensions[0]!.tools.has("afk_complete")).toBe(false);
		await start();
		expect(run.observed).toMatchObject({
			executions: 1,
			intents: 1,
			reviews: 1,
		});
		expect(entries("afk-review")).toHaveLength(0);
		expect(entries("afk-review-error")).toHaveLength(0);
		expect(session.errors).toEqual([]);
	});
	it("delivers an objection and subsequent executor evidence to the next review", async () => {
		let pass = 0;
		const run = script({
			objections: 1,
			executor: () =>
				fauxAssistantMessage(
					++pass === 1
						? "Scoped checks green; publication acceptance remains unfinished."
						: "Publication fault acceptance verified: 12 fault-injection cases pass.",
				),
		});
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await start();
		expect(run.observed).toMatchObject({
			executions: 2,
			intents: 1,
			reviews: 2,
		});
		expect(entries("afk-review")).toHaveLength(1);
		expect(run.observed.reviewText).toContain(
			"publication acceptance remains unfinished",
		);
		expect(run.observed.reviewText).toContain("12 fault-injection cases pass");
		expect(entries("afk-review-error")).toHaveLength(0);
		expect(session.errors).toEqual([]);
	});
	it("allows exactly five objections plus a final executor pass; /afk rearms the budget", async () => {
		const run = script({ objections: 99 });
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await start();
		expect(run.observed).toMatchObject({
			executions: 6,
			intents: 1,
			reviews: 5,
		});
		expect(entries("afk-review")).toHaveLength(5);
		expect(entries("afk-limit")).toHaveLength(1);
		await start();
		expect(run.observed).toMatchObject({
			executions: 12,
			intents: 2,
			reviews: 10,
		});
		expect(entries("afk-limit")).toHaveLength(2);
	});
	it("reconstructs intent asynchronously while the executor works", async () => {
		let release!: () => void;
		let executed!: () => void;
		const initial = new Promise<void>((resolve) => {
			release = resolve;
		});
		const execution = new Promise<void>((resolve) => {
			executed = resolve;
		});
		const run = script({
			initial: () => initial,
			executor: () => {
				executed();
				return fauxAssistantMessage("Worked while intent was pending.");
			},
		});
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await command("afk").handler(
			"",
			session.session.extensionRunner.createContext() as never,
		);
		try {
			await execution;
			expect(run.observed.reviews).toBe(0);
		} finally {
			release();
		}
		await session.session.waitForIdle();
		expect(run.observed.reviews).toBe(1);
	});
	it("does not let /back cancel executor work or publish a stale intent", async () => {
		let release!: () => void;
		const initial = new Promise<void>((resolve) => {
			release = resolve;
		});
		const run = script({
			initial: () => initial,
			executor: async () => {
				await command("back").handler(
					"",
					session!.session.extensionRunner.createContext() as never,
				);
				release();
				return fauxAssistantMessage("Current work finished normally.");
			},
		});
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await start();
		expect(run.observed.executions).toBe(1);
		expect(run.observed.reviews).toBe(0);
		expect(entries("afk-intent")).toHaveLength(0);
		expect(entries("afk-review-error")).toHaveLength(0);
	});
	it("cancels a held review through real SDK abort without stale objections or compaction restart", async () => {
		let entered!: () => void, release!: () => void;
		const reviewing = new Promise<void>((resolve) => {
			entered = resolve;
		});
		const held = new Promise<void>((resolve) => {
			release = resolve;
		});
		const run = script({
			objections: 1,
			review: async () => {
				entered();
				await held;
			},
		});
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await command("afk").handler(
			"",
			session.session.extensionRunner.createContext() as never,
		);
		await reviewing;
		try {
			await session.session.abort();
			expect(entries("afk-review")).toHaveLength(0);
			expect(entries("afk-review-error")).toHaveLength(0);
			const send = vi.spyOn(session.session, "sendCustomMessage");
			await session.session.extensionRunner.emit({
				type: "session_compact",
				compactionEntry: { id: "aborted" },
				fromExtension: false,
				reason: "manual",
				willRetry: false,
			} as never);
			await new Promise((resolve) => setTimeout(resolve, 10));
			expect(send).not.toHaveBeenCalled();
		} finally {
			release();
		}
	}, 5000);
	it("reports malformed judge output instead of pretending completion", async () => {
		const run = script({ invalid: true });
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await start();
		expect(entries("afk-review-error")).toHaveLength(1);
		expect(run.observed.executions).toBe(1);
	});
	it("drains queued user follow-ups in order and includes them in review evidence", async () => {
		let queued = false;
		const queue: ExtensionFactory = (pi) => {
			pi.on("agent_start", () => {
				if (queued) return;
				queued = true;
				pi.sendUserMessage("Follow-up A", { deliverAs: "followUp" });
				pi.sendUserMessage("Follow-up B", { deliverAs: "followUp" });
			});
		};
		const run = script();
		session = await createPiSession({
			extensionPaths: [path],
			extensionFactories: [queue],
			responses: run.responses,
		});
		await start();
		const users = session.session.messages.filter(
			(message) => message.role === "user",
		);
		expect(users).toHaveLength(2);
		expect(run.observed.reviewText).toContain("Follow-up A");
		expect(run.observed.reviewText.indexOf("Follow-up A")).toBeLessThan(
			run.observed.reviewText.indexOf("Follow-up B"),
		);
		expect(run.observed.reviews).toBe(1);
	});
	it("blocks routine human tools but lets the executor use an alternative", async () => {
		let calls = 0;
		humanExecution.mockClear();
		const boundaries: unknown[] = [];
		const observe: ExtensionFactory = (pi) => {
			pi.on("turn_end", (event) => {
				boundaries.push({
					stop:
						event.message.role === "assistant"
							? event.message.stopReason
							: event.message.role,
					continue: event.continue,
					pending: event.context.pendingMessages,
				});
			});
		};
		const run = script({
			executor: () =>
				++calls === 1
					? fauxAssistantMessage(fauxToolCall("ask_user_question", {}), {
							stopReason: "toolUse",
						})
					: fauxAssistantMessage("Used documented defaults."),
		});
		session = await createPiSession({
			extensionPaths: [path],
			extensionFactories: [human, observe],
			responses: run.responses,
		});
		await start();
		expect(
			session.session.messages.some(
				(message) =>
					message.role === "toolResult" &&
					message.toolName === "ask_user_question" &&
					message.isError,
			),
		).toBe(true);
		expect(run.observed.executions).toBe(2);
		expect(
			run.observed.reviews,
			JSON.stringify({
				boundaries,
				errors: session.errors,
				reviewErrors: entries("afk-review-error"),
			}),
		).toBe(1);
		expect(run.observed.reviewText).toContain("Used documented defaults");
		expect(humanExecution).not.toHaveBeenCalled();
		expect(entries("afk-review-error")).toHaveLength(0);
	});
	it.each(["aborted", "error"] as const)(
		"respects executor %s without reviewing or continuing",
		async (stopReason) => {
			const run = script({
				executor: fauxAssistantMessage("Stopped", { stopReason }),
			});
			session = await createPiSession({
				extensionPaths: [path],
				responses: run.responses,
			});
			await start();
			expect(run.observed.executions).toBe(1);
			expect(run.observed.reviews).toBe(0);
		},
	);
	it("reviews custom follow-up work queued during a held clean verdict", async () => {
		let queued = false;
		const run = script({
			review: async () => {
				if (queued) return;
				queued = true;
				await session!.session.sendCustomMessage(
					{
						customType: "queued-work",
						content: "Finish independent follow-up",
						display: false,
					},
					{ triggerTurn: true, deliverAs: "followUp" },
				);
			},
		});
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await start();
		expect(run.observed).toMatchObject({ executions: 2, reviews: 2 });
		expect(run.observed.reviewText).toContain("Finish independent follow-up");
		expect(entries("afk-review-error")).toHaveLength(0);
	});
	it("includes passive context from other extensions without skipping review", async () => {
		const run = script({
			executor: async () => {
				await session!.session.sendCustomMessage(
					{
						customType: "ephemlet-build-result",
						content: "Preview ready",
						display: false,
					},
					{ triggerTurn: false },
				);
				return fauxAssistantMessage("Used completed preview.");
			},
		});
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		await start();
		expect(run.observed).toMatchObject({ executions: 1, reviews: 1 });
		expect(run.observed.reviewText).toContain("Preview ready");
		expect(entries("afk-review-error")).toHaveLength(0);
	});
	it("reviews work continued by a later turn-boundary handler", async () => {
		let continued = false;
		const later: ExtensionFactory = (pi) => {
			pi.on("turn_end", (event) => {
				if (
					continued ||
					event.message.role !== "assistant" ||
					event.message.stopReason !== "stop"
				)
					return;
				continued = true;
				return { continue: true };
			});
		};
		const run = script();
		session = await createPiSession({
			extensionFactories: [afk, later],
			responses: run.responses,
		});
		await start();
		expect(run.observed).toMatchObject({ executions: 2, reviews: 2 });
	});
	it("preserves earlier boundary drafts alongside reviewer objections", async () => {
		let added = false;
		const earlier: ExtensionFactory = (pi) => {
			pi.on("turn_end", (event) => {
				if (added) return;
				added = true;
				return {
					entries: [
						...event.entries,
						{
							type: "custom_message",
							customType: "earlier-draft",
							content: "Evidence from another extension",
							display: false,
						},
					],
				};
			});
		};
		const run = script({ objections: 1 });
		session = await createPiSession({
			extensionFactories: [earlier, afk],
			responses: run.responses,
		});
		await start();
		expect(entries("earlier-draft")).toHaveLength(1);
		expect(entries("afk-review")).toHaveLength(1);
		expect(run.observed.reviewText).toContain(
			"Evidence from another extension",
		);
	});
	it("keeps AFK active through native retry of a transient executor error", async () => {
		let passes = 0;
		const run = script({
			executor: () =>
				++passes === 1
					? fauxAssistantMessage("", {
							stopReason: "error",
							errorMessage: "503 Service Unavailable",
						})
					: fauxAssistantMessage("Recovered executor result."),
		});
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
			userSettings: {
				retry: {
					enabled: true,
					maxRetries: 1,
					baseDelayMs: 1,
					maxAgentDelayMs: 1,
				},
			},
		});
		await start();
		expect(run.observed).toMatchObject({ executions: 2, reviews: 1 });
		expect(run.observed.reviewText).toContain("Recovered executor result");
	});
	it("renders native widget goal, pleased reaction, nap and disposes on /back", async () => {
		const run = script({ objections: 1 });
		session = await createPiSession({
			extensionPaths: [path],
			responses: run.responses,
		});
		let creature: (Component & { dispose?(): void }) | undefined;
		await session.session.bindExtensions({
			mode: "tui",
			uiContext: {
				...session.session.extensionRunner.getUIContext(),
				setWidget: (_key, content) => {
					creature =
						typeof content === "function"
							? content({ requestRender() {} } as TUI, catTheme())
							: undefined;
				},
			},
		});
		await start();
		expect(creature!.render(80)[0]).toContain("Finish OAuth acceptance");
		expect(creature!.render(80)[1]).toContain("objection addressed");
		const dispose = vi.spyOn(creature!, "dispose");
		await command("back").handler(
			"",
			session.session.extensionRunner.createContext() as never,
		);
		expect(dispose).toHaveBeenCalledOnce();
		expect(creature).toBeUndefined();
	});
	it.each(["alone", "first", "last"])(
		"recovers idle compaction with AFK %s and leaves naps settled",
		async (order) => {
			const paths = [path];
			if (order !== "alone")
				paths.push(
					fileURLToPath(new URL("../../continuity/index.ts", import.meta.url)),
				);
			if (order === "last") paths.reverse();
			const run = script();
			session = await createPiSession({
				extensionPaths: paths,
				responses: run.responses,
			});
			const startMessage = vi
				.spyOn(session.session, "sendCustomMessage")
				.mockImplementationOnce(async () => {});
			await command("afk").handler(
				"",
				session.session.extensionRunner.createContext() as never,
			);
			startMessage.mockRestore();
			const compact = () =>
				session!.session.extensionRunner.emit({
					type: "session_compact",
					compactionEntry: { id: "test-compact" },
					fromExtension: false,
					reason: "manual",
					willRetry: false,
				} as never);
			await compact();
			await vi.waitFor(() => expect(run.observed.reviews).toBe(1));
			await session.session.waitForIdle();
			expect(entries("afk-continue")).toHaveLength(1);
			expect(entries("continuity-resume")).toHaveLength(0);
			const send = vi.spyOn(session.session, "sendCustomMessage");
			await compact();
			await new Promise((resolve) => setTimeout(resolve, 10));
			expect(send).not.toHaveBeenCalled();
			expect(session.errors).toEqual([]);
		},
	);
	it("writes schema-bound lifecycle diagnostics only when enabled", async () => {
		session = await createPiSession({
			extensionPaths: [path],
			env: { PI_AFK_DEBUG: "1" },
		});
		await session.session.extensionRunner.emit({
			type: "session_shutdown",
			reason: "quit",
		});
		const directory = join(
			session.env["XDG_STATE_HOME"]!,
			"pi-ext",
			"debug",
			"afk",
		);
		await vi.waitFor(() => {
			const events = readdirSync(directory).flatMap((file) =>
				readFileSync(join(directory, file), "utf8")
					.trim()
					.split("\n")
					.filter(Boolean)
					.map((line) => JSON.parse(line).event),
			);
			expect(events).toContain("session.start");
			expect(events).toContain("session.shutdown");
		});
	});
});