Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/strata/__tests__/harness.test.ts

Raw
import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import path from "node:path";
import {
	type AssistantMessage,
	fauxAssistantMessage,
} from "@earendil-works/pi-ai";
import { aroundEach, describe, expect, it, vi } from "vitest";
import { createTestSession } from "../../../test/harness";
import {
	fixturePlan,
	git,
	nativeOperation,
	nativeTest,
	openFixture,
	testSignal,
	withFakeGh,
} from "./fixture.js";

let fixture: Awaited<ReturnType<typeof openFixture>> | undefined;
aroundEach(async (run, context) => {
	try {
		await nativeTest(run, context);
	} finally {
		fixture = undefined;
		vi.restoreAllMocks();
	}
});

async function connect(url: string) {
	const response = await fetch(url, {
		redirect: "manual",
		signal: testSignal(),
	});
	const cookie = response.headers.get("set-cookie")?.split(";")[0] ?? "";
	const origin = new URL(url).origin;
	let version = await (
		await fetch(`${origin}/api/state`, {
			headers: { Cookie: cookie },
			signal: testSignal(),
		})
	).json();
	return (route: string, body?: unknown) =>
		nativeOperation(async () => {
			const response = await fetch(`${origin}/api/${route}`, {
				signal: testSignal(),
				method: body === undefined ? "GET" : "POST",
				headers: {
					Cookie: cookie,
					Origin: origin,
					"Content-Type": "application/json",
					"If-Match": String(version.revision),
					"X-Strata-Snapshot": version.snapshotId,
				},
				body: body === undefined ? undefined : JSON.stringify(body),
			});
			const payload = await response.clone().json();
			if (payload.revision !== undefined) version = payload;
			return response;
		});
}

function planResponse(context: unknown): AssistantMessage {
	const messages = (
		context as {
			messages: Array<{
				role: string;
				content: string | Array<{ type: string; text?: string }>;
			}>;
		}
	).messages;
	const message = messages.findLast((item) => item.role === "user");
	const text =
		typeof message?.content === "string"
			? message.content
			: (message?.content ?? [])
					.filter((part) => part.type === "text")
					.map((part) => part.text ?? "")
					.join("");
	const hunks = [
		...text.matchAll(
			/<file path="(contract\.ts|consumer\.ts)">\s*<hunk id="([^"]+)"/g,
		),
	].map(([, filePath, id]) => ({ id, path: filePath }));
	return fauxAssistantMessage(JSON.stringify(fixturePlan(hunks)));
}

function expectPhase(value: string): void {
	expect(fixture?.widgetText()).toContain(`● ${value}`);
}

describe("Strata real Pi runtime", () => {
	it("warns about missing Git using the real startup path", async () => {
		fixture = await openFixture();
		fixture.t.env.PATH = path.join(fixture.root, "missing-bin");
		await fixture.runner.emit({ type: "session_start", reason: "reload" });
		await vi.waitFor(() =>
			expect(
				fixture?.t.events.ui.some(
					(call) =>
						call.method === "notify" &&
						call.args[0] === "Strata requires Git on PATH." &&
						call.args[1] === "warning",
				),
			).toBe(true),
		);
	});

	it("writes its opt-in lifecycle trace inside the harness sandbox", async () => {
		const root = await mkdtemp(path.join(process.cwd(), ".strata-debug-"));
		const t = await createTestSession({
			cwd: root,
			extensions: [path.resolve(import.meta.dirname, "../index.ts")],
			env: { PI_STRATA_DEBUG: "1" },
		});
		try {
			const runner = t.session.extensionRunner;
			await runner.emit({ type: "session_start", reason: "startup" });
			await runner.extensions[0].commands
				.get("strata")
				?.handler("--invalid", runner.createCommandContext());
			await runner.emit({ type: "session_shutdown", reason: "quit" });
			const stateHome = t.env.XDG_STATE_HOME;
			if (!stateHome) throw new Error("missing sandbox state home");
			const directory = path.join(stateHome, "pi-ext", "debug", "strata");
			const [file] = await readdir(directory);
			const records = (await readFile(path.join(directory, file), "utf8"))
				.trim()
				.split("\n")
				.map((line) => JSON.parse(line));
			expect(records.map((record) => record.event)).toEqual(
				expect.arrayContaining(["session.start", "session.shutdown"]),
			);
			expect(records).toContainEqual(
				expect.objectContaining({
					event: "review.open.finish",
					outcome: "failed",
				}),
			);
		} finally {
			t.dispose();
			await rm(root, { recursive: true, force: true });
		}
	});

	it("loads optional gh only for --pr and reports a missing CLI", async () => {
		fixture = await openFixture();
		const originalPath = fixture.t.env.PATH;
		fixture.t.env.PATH = path.join(fixture.root, "missing-bin");
		await fixture.run("--pr 7");
		expectPhase("Error");
		expect(fixture.widgetText()).toContain("optional gh CLI on PATH");
		await fixture.run("cancel");
		fixture.t.env.PATH = originalPath;
	});

	it("reports gh authentication failure without raw CLI output", async () => {
		fixture = await openFixture();
		await withFakeGh(
			`process.stderr.write("authentication failed: secret-token-value"); process.exitCode = 1;`,
			async () => {
				await fixture?.run("--pr 7");
			},
		);
		expectPhase("Error");
		expect(fixture.widgetText()).toContain("GitHub authentication failed");
		expect(fixture.widgetText()).not.toContain("secret-token-value");
		expect(fixture.calls).toBe(0);
	});

	it("refuses a dirty PR checkout through the real extension path", async () => {
		fixture = await openFixture();
		const commit = await git(fixture.root, "rev-parse", "HEAD");
		await git(
			fixture.root,
			"remote",
			"add",
			"origin",
			"https://github.com/acme/project.git",
		);
		const metadata = {
			number: 7,
			url: "https://github.com/acme/project/pull/7",
			title: "Fixture PR",
			body: "",
			author: { login: "author" },
			baseRefName: "main",
			baseRefOid: commit,
			headRefName: "feature",
			headRefOid: commit,
			headRepository: {
				name: "project",
				nameWithOwner: "acme/project",
			},
			headRepositoryOwner: { login: "acme" },
		};
		await withFakeGh(
			`if (args[0] === "pr") process.stdout.write(${JSON.stringify(JSON.stringify(metadata))}); else process.stdout.write("[]");`,
			async () => {
				await fixture?.run("--pr 7");
			},
		);
		expectPhase("Error");
		expect(fixture.widgetText()).toContain("requires a clean worktree");
		expect(fixture.calls).toBe(0);
	});

	it("places the link widget above the editor without footer or notification noise", async () => {
		fixture = await openFixture();
		expect(fixture.runner.extensions[0].path).toBe(
			path.resolve(import.meta.dirname, "../index.ts"),
		);
		expect(fixture.command.getArgumentCompletions("--")).toHaveLength(3);
		expect(fixture.calls).toBe(0);
		await fixture.run();
		expect(fixture.calls).toBe(1);
		expectPhase("Ready");
		expect(fixture.widgetText()).toContain("0/2 reviewed");
		expect(fixture.widgetText()).toContain("2 layers");
		expect(fixture.widgets.at(-1)?.options).toEqual({
			placement: "aboveEditor",
		});
		expect(
			fixture.t.events.ui.filter((call) => call.method === "setStatus"),
		).toEqual([]);
		expect(
			fixture.notifications.some((item) =>
				item.text.startsWith("Strata review:"),
			),
		).toBe(false);
		const url = fixture.url;
		const widgetCalls = fixture.widgets.length;
		await fixture.run();
		expect(fixture.url).toBe(url);
		expect(fixture.calls).toBe(1);
		expect(fixture.widgets.length).toBe(widgetCalls + 1);
		expect(fixture.notifications).toEqual([]);
	});

	it("persists draft progress, Ask history, close state, and snapshot-bound reopen", async () => {
		fixture = await openFixture();
		await fixture.run();
		const firstUrl = fixture.url;
		const api = await connect(firstUrl);
		const { review } = await (await api("state")).json();
		const draft = {
			reviewed: [review.snapshot.hunks[0].id],
			findings: [
				{
					id: "finding-1",
					hunkId: review.snapshot.hunks[0].id,
					side: "new",
					line: 1,
					severity: "minor",
					text: "Check the bound",
				},
			],
			notes: "Check retry bounds",
		};
		expect((await api("draft", draft)).status).toBe(200);
		expect(fixture.widgetText()).toContain("1/2 reviewed");
		expect(fixture.widgetText()).toContain("1 comment");
		expectPhase("Ready");
		const answer = await api("ask", {
			layerId: "layer-1",
			hunkId: review.snapshot.hunks[0].id,
			question: "How many attempts?",
		});
		expect((await answer.json()).answer).toBe(
			"Three attempts include the first call.",
		);
		expect(fixture.calls).toBe(2);
		await fixture.run("cancel");
		expectPhase("Closed");
		expect(fixture.widgetText()).toContain("Run /strata to reopen");
		expect(fixture.widgetText()).not.toContain("http://127.0.0.1");
		await fixture.run();
		expect(fixture.calls).toBe(2);
		expect(fixture.url).not.toBe(firstUrl);
		const resumed = await connect(fixture.url);
		const resumedState = await (await resumed("state")).json();
		expect(resumedState.review.draft).toEqual(draft);
		expect(resumedState.threads).toEqual([]);
		expect(fixture.widgetText()).toContain("1/2 reviewed");
	});

	it("reflects pending Ask work and returns to Ready", async () => {
		fixture = await openFixture();
		await fixture.run();
		const url = fixture.url;
		const api = await connect(url);
		const { review } = await (await api("state")).json();
		let releaseAsk!: (message: AssistantMessage) => void;
		let askStarted!: () => void;
		const didAskStart = new Promise<void>((resolve) => {
			askStarted = resolve;
		});
		vi.spyOn(fixture.ctx.modelRegistry, "complete").mockImplementationOnce(
			async (_model, _context, options) =>
				new Promise<AssistantMessage>((resolve, reject) => {
					options?.signal?.addEventListener(
						"abort",
						() => reject(options.signal?.reason),
						{ once: true },
					);
					releaseAsk = resolve;
					askStarted();
				}),
		);
		const asking = api("ask", {
			layerId: "layer-1",
			hunkId: review.snapshot.hunks[0].id,
			question: "Wait for this answer",
		});
		await didAskStart;
		expectPhase("Answering");
		releaseAsk(fauxAssistantMessage("Done"));
		expect((await asking).status).toBe(200);
		expectPhase("Ready");
	});

	it("preserves the active link and counts after an Ask error", async () => {
		fixture = await openFixture();
		await fixture.run();
		const url = fixture.url;
		const api = await connect(url);
		vi.spyOn(fixture.ctx.modelRegistry, "complete").mockRejectedValueOnce(
			new Error("model\x1b]8;;https://evil.example\x07 unavailable"),
		);
		expect(
			(
				await api("ask", {
					layerId: "layer-1",
					hunkId: null,
					question: "Fail",
				})
			).status,
		).toBe(500);
		expectPhase("Error");
		expect(fixture.widgetText()).toContain(
			"model ]8;;https://evil.example unavailable",
		);
		expect(fixture.url).toBe(url);
		expect(fixture.widgetText()).toContain("0/2 reviewed");
	});

	it("reflects pending refresh work and returns to Ready", async () => {
		fixture = await openFixture();
		await fixture.run();
		const api = await connect(fixture.url);
		const { review } = await (await api("state")).json();
		let releaseRefresh!: (message: AssistantMessage) => void;
		let refreshStarted!: () => void;
		const didRefreshStart = new Promise<void>((resolve) => {
			refreshStarted = resolve;
		});
		vi.spyOn(fixture.ctx.modelRegistry, "complete").mockImplementationOnce(
			async (_model, _context, options) =>
				new Promise<AssistantMessage>((resolve, reject) => {
					options?.signal?.addEventListener(
						"abort",
						() => reject(options.signal?.reason),
						{ once: true },
					);
					releaseRefresh = resolve;
					refreshStarted();
				}),
		);
		const refreshing = api("refresh", { confirm: true });
		await didRefreshStart;
		expectPhase("Refreshing");
		releaseRefresh(fauxAssistantMessage(JSON.stringify(review.plan)));
		expect((await refreshing).status).toBe(200);
		expectPhase("Ready");
	});

	it("preserves the active link after a refresh error", async () => {
		fixture = await openFixture();
		await fixture.run();
		const url = fixture.url;
		const api = await connect(url);
		vi.spyOn(fixture.ctx.modelRegistry, "complete").mockRejectedValueOnce(
			new Error("refresh provider failed"),
		);
		expect((await api("refresh", { confirm: true })).status).toBe(500);
		expectPhase("Error");
		expect(fixture.widgetText()).toContain("refresh provider failed");
		expect(fixture.url).toBe(url);
	});

	it("keeps stale until refresh clears the old draft", async () => {
		fixture = await openFixture();
		await fixture.run();
		const api = await connect(fixture.url);
		const { review } = await (await api("state")).json();
		const draft = {
			reviewed: [review.snapshot.hunks[0].id],
			findings: [],
			notes: "Verify the retry contract.",
		};
		await writeFile(
			path.join(fixture.root, "contract.ts"),
			"export const attempts = 4;\n",
		);
		expect((await api("submit", { draft })).status).toBe(409);
		expectPhase("Stale");
		expect(fixture.widgetText()).toContain("1/2 reviewed");
		expect((await api("draft", draft)).status).toBe(200);
		expectPhase("Stale");
		expect((await api("refresh", { confirm: true })).status).toBe(200);
		expectPhase("Ready");
		expect(fixture.widgetText()).toContain("0/2 reviewed");
	});

	it("keeps Feedback sent through the empty draft save", async () => {
		fixture = await openFixture();
		await writeFile(
			path.join(fixture.root, "contract.ts"),
			"export const attempts = 4;\n",
		);
		await fixture.run();
		const api = await connect(fixture.url);
		const { review: refreshedReview } = await (await api("state")).json();
		const send = vi
			.spyOn(fixture.t.session, "prompt")
			.mockResolvedValue(undefined);
		const feedback = {
			reviewed: [refreshedReview.snapshot.hunks[0].id],
			findings: [],
			notes: "Four attempts are intentional.",
		};
		expect((await api("submit", { draft: feedback })).status).toBe(200);
		await vi.waitFor(() => expect(send).toHaveBeenCalled());
		expect(send.mock.calls[0][0]).toContain(refreshedReview.snapshot.id);
		expectPhase("Feedback sent");
		expect(fixture.widgetText()).toContain("0/2 reviewed");
		expect((await (await api("state")).json()).review.draft.notes).toBe("");
	});

	it("retains URL and error discovery in RPC mode without component factories", async () => {
		fixture = await openFixture("rpc");
		await fixture.run();
		const url = fixture.url;
		expect(fixture.widgets).toEqual([]);
		expect(fixture.notifications).toEqual([
			{ text: `Strata review: ${url}`, severity: "info" },
		]);
		await fixture.run();
		expect(fixture.notifications).toHaveLength(1);
		await fixture.run("cancel");
		await writeFile(
			path.join(fixture.root, "contract.ts"),
			"export const attempts = 5;\n",
		);
		fixture.faux.setResponses([fauxAssistantMessage("not json")]);
		await fixture.run();
		expect(fixture.notifications.at(-1)).toMatchObject({ severity: "error" });
	});

	it("emits URL and error custom messages without terminal writes or LLM turns when no UI exists", async () => {
		const stdout = vi.spyOn(process.stdout, "write").mockReturnValue(true);
		const stderr = vi.spyOn(process.stderr, "write").mockReturnValue(true);
		fixture = await openFixture("json");
		await fixture.run();
		const statusMessages = () =>
			fixture?.t.events.messages.filter(
				(message) =>
					message.role === "custom" && message.customType === "strata-status",
			) ?? [];
		expect(statusMessages()).toHaveLength(1);
		expect(statusMessages()[0]).toMatchObject({
			content: expect.stringMatching(
				/Strata review: http:\/\/127\.0\.0\.1:\d+\/\?token=\S+/,
			),
			display: true,
			details: { severity: "info" },
		});
		expect(fixture.calls).toBe(1);
		await fixture.run();
		expect(statusMessages()).toHaveLength(1);
		expect(fixture.calls).toBe(1);
		await fixture.run("cancel");
		await writeFile(
			path.join(fixture.root, "contract.ts"),
			"export const attempts = 5;\n",
		);
		fixture.faux.setResponses([fauxAssistantMessage("not json")]);
		await fixture.run();
		expect(statusMessages().at(-1)).toMatchObject({
			content: expect.stringContaining("Strata failed:"),
			display: true,
			details: { severity: "error" },
		});
		expect(fixture.calls).toBe(1);
		expect(stdout).not.toHaveBeenCalled();
		expect(stderr).not.toHaveBeenCalled();
	});

	it("keeps initial failures visible and resets them on reopen", async () => {
		fixture = await openFixture();
		fixture.faux.setResponses([fauxAssistantMessage("not json")]);
		await fixture.run();
		expectPhase("Error");
		expect(fixture.widgetText()).toContain("Strata failed:");
		expect(fixture.widgetText(240)).toContain("Run /strata to retry");
		expect(fixture.notifications).toEqual([]);
		await fixture.run("--unknown");
		expectPhase("Error");
		expect(fixture.widgetText()).toContain("usage:");
		expect(fixture.widgetText()).not.toContain("model response is not JSON");
	});

	it("guards cancelled and replaced generations against late callbacks", async () => {
		fixture = await openFixture();
		const pendingCalls: Array<{
			context: unknown;
			resolve(message: AssistantMessage): void;
		}> = [];
		const started = Array.from({ length: 2 }, () =>
			Promise.withResolvers<void>(),
		);
		const complete = vi
			.spyOn(fixture.ctx.modelRegistry, "complete")
			.mockImplementation(async (_model, context) => {
				return new Promise<AssistantMessage>((resolve) => {
					pendingCalls.push({ context, resolve });
					started[pendingCalls.length - 1].resolve();
				});
			});
		testSignal().addEventListener(
			"abort",
			() => {
				for (const call of pendingCalls)
					call.resolve(planResponse(call.context));
			},
			{ once: true },
		);
		const oldOpen = fixture.run();
		await started[0].promise;
		expect(pendingCalls).toHaveLength(1);
		await fixture.run("cancel");
		expectPhase("Closed");
		const replacement = fixture.run();
		await started[1].promise;
		expect(pendingCalls).toHaveLength(2);
		pendingCalls[1].resolve(planResponse(pendingCalls[1].context));
		await replacement;
		expectPhase("Ready");
		const replacementUrl = fixture.url;
		pendingCalls[0].resolve(planResponse(pendingCalls[0].context));
		await oldOpen;
		expectPhase("Ready");
		expect(fixture.url).toBe(replacementUrl);
		expect(complete).toHaveBeenCalledTimes(2);
	});

	it("drains pending generation on shutdown and ignores its late callback", async () => {
		fixture = await openFixture();
		await writeFile(
			path.join(fixture.root, "contract.ts"),
			"export const attempts = 6;\n",
		);
		const started = Promise.withResolvers<void>();
		const pending = Promise.withResolvers<AssistantMessage>();
		let context: unknown;
		const complete = vi
			.spyOn(fixture.ctx.modelRegistry, "complete")
			.mockImplementation(async (_model, value) => {
				context = value;
				started.resolve();
				return pending.promise;
			});
		testSignal().addEventListener(
			"abort",
			() =>
				pending.resolve(
					context ? planResponse(context) : fauxAssistantMessage("cancelled"),
				),
			{ once: true },
		);
		const shutdownOpen = fixture.run();
		await started.promise;
		expect(complete).toHaveBeenCalledOnce();
		let settled = false;
		const shutdown = fixture.runner
			.emit({ type: "session_shutdown", reason: "reload" })
			.then(() => {
				settled = true;
			});
		expect(fixture.widgetLines()).toEqual([]);
		await Promise.resolve();
		expect(settled).toBe(false);
		pending.resolve(planResponse(context));
		await shutdown;
		await shutdownOpen;
		expect(settled).toBe(true);
		expect(fixture.widgetLines()).toEqual([]);
		expect(fixture.notifications).toEqual([]);
	});
});