repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/strata/__tests__/harness.test.ts
Rawimport { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import path from "node:path";
import {
type AssistantMessage,
fauxAssistantMessage,
} from "@earendil-works/pi-ai";
import { aroundEach, describe, expect, it, vi } from "vitest";
import { createTestSession } from "../../../test/harness";
import {
fixturePlan,
git,
nativeOperation,
nativeTest,
openFixture,
testSignal,
withFakeGh,
} from "./fixture.js";
let fixture: Awaited<ReturnType<typeof openFixture>> | undefined;
aroundEach(async (run, context) => {
try {
await nativeTest(run, context);
} finally {
fixture = undefined;
vi.restoreAllMocks();
}
});
async function connect(url: string) {
const response = await fetch(url, {
redirect: "manual",
signal: testSignal(),
});
const cookie = response.headers.get("set-cookie")?.split(";")[0] ?? "";
const origin = new URL(url).origin;
let version = await (
await fetch(`${origin}/api/state`, {
headers: { Cookie: cookie },
signal: testSignal(),
})
).json();
return (route: string, body?: unknown) =>
nativeOperation(async () => {
const response = await fetch(`${origin}/api/${route}`, {
signal: testSignal(),
method: body === undefined ? "GET" : "POST",
headers: {
Cookie: cookie,
Origin: origin,
"Content-Type": "application/json",
"If-Match": String(version.revision),
"X-Strata-Snapshot": version.snapshotId,
},
body: body === undefined ? undefined : JSON.stringify(body),
});
const payload = await response.clone().json();
if (payload.revision !== undefined) version = payload;
return response;
});
}
function planResponse(context: unknown): AssistantMessage {
const messages = (
context as {
messages: Array<{
role: string;
content: string | Array<{ type: string; text?: string }>;
}>;
}
).messages;
const message = messages.findLast((item) => item.role === "user");
const text =
typeof message?.content === "string"
? message.content
: (message?.content ?? [])
.filter((part) => part.type === "text")
.map((part) => part.text ?? "")
.join("");
const hunks = [
...text.matchAll(
/<file path="(contract\.ts|consumer\.ts)">\s*<hunk id="([^"]+)"/g,
),
].map(([, filePath, id]) => ({ id, path: filePath }));
return fauxAssistantMessage(JSON.stringify(fixturePlan(hunks)));
}
function expectPhase(value: string): void {
expect(fixture?.widgetText()).toContain(`● ${value}`);
}
describe("Strata real Pi runtime", () => {
it("warns about missing Git using the real startup path", async () => {
fixture = await openFixture();
fixture.t.env.PATH = path.join(fixture.root, "missing-bin");
await fixture.runner.emit({ type: "session_start", reason: "reload" });
await vi.waitFor(() =>
expect(
fixture?.t.events.ui.some(
(call) =>
call.method === "notify" &&
call.args[0] === "Strata requires Git on PATH." &&
call.args[1] === "warning",
),
).toBe(true),
);
});
it("writes its opt-in lifecycle trace inside the harness sandbox", async () => {
const root = await mkdtemp(path.join(process.cwd(), ".strata-debug-"));
const t = await createTestSession({
cwd: root,
extensions: [path.resolve(import.meta.dirname, "../index.ts")],
env: { PI_STRATA_DEBUG: "1" },
});
try {
const runner = t.session.extensionRunner;
await runner.emit({ type: "session_start", reason: "startup" });
await runner.extensions[0].commands
.get("strata")
?.handler("--invalid", runner.createCommandContext());
await runner.emit({ type: "session_shutdown", reason: "quit" });
const stateHome = t.env.XDG_STATE_HOME;
if (!stateHome) throw new Error("missing sandbox state home");
const directory = path.join(stateHome, "pi-ext", "debug", "strata");
const [file] = await readdir(directory);
const records = (await readFile(path.join(directory, file), "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(records.map((record) => record.event)).toEqual(
expect.arrayContaining(["session.start", "session.shutdown"]),
);
expect(records).toContainEqual(
expect.objectContaining({
event: "review.open.finish",
outcome: "failed",
}),
);
} finally {
t.dispose();
await rm(root, { recursive: true, force: true });
}
});
it("loads optional gh only for --pr and reports a missing CLI", async () => {
fixture = await openFixture();
const originalPath = fixture.t.env.PATH;
fixture.t.env.PATH = path.join(fixture.root, "missing-bin");
await fixture.run("--pr 7");
expectPhase("Error");
expect(fixture.widgetText()).toContain("optional gh CLI on PATH");
await fixture.run("cancel");
fixture.t.env.PATH = originalPath;
});
it("reports gh authentication failure without raw CLI output", async () => {
fixture = await openFixture();
await withFakeGh(
`process.stderr.write("authentication failed: secret-token-value"); process.exitCode = 1;`,
async () => {
await fixture?.run("--pr 7");
},
);
expectPhase("Error");
expect(fixture.widgetText()).toContain("GitHub authentication failed");
expect(fixture.widgetText()).not.toContain("secret-token-value");
expect(fixture.calls).toBe(0);
});
it("refuses a dirty PR checkout through the real extension path", async () => {
fixture = await openFixture();
const commit = await git(fixture.root, "rev-parse", "HEAD");
await git(
fixture.root,
"remote",
"add",
"origin",
"https://github.com/acme/project.git",
);
const metadata = {
number: 7,
url: "https://github.com/acme/project/pull/7",
title: "Fixture PR",
body: "",
author: { login: "author" },
baseRefName: "main",
baseRefOid: commit,
headRefName: "feature",
headRefOid: commit,
headRepository: {
name: "project",
nameWithOwner: "acme/project",
},
headRepositoryOwner: { login: "acme" },
};
await withFakeGh(
`if (args[0] === "pr") process.stdout.write(${JSON.stringify(JSON.stringify(metadata))}); else process.stdout.write("[]");`,
async () => {
await fixture?.run("--pr 7");
},
);
expectPhase("Error");
expect(fixture.widgetText()).toContain("requires a clean worktree");
expect(fixture.calls).toBe(0);
});
it("places the link widget above the editor without footer or notification noise", async () => {
fixture = await openFixture();
expect(fixture.runner.extensions[0].path).toBe(
path.resolve(import.meta.dirname, "../index.ts"),
);
expect(fixture.command.getArgumentCompletions("--")).toHaveLength(3);
expect(fixture.calls).toBe(0);
await fixture.run();
expect(fixture.calls).toBe(1);
expectPhase("Ready");
expect(fixture.widgetText()).toContain("0/2 reviewed");
expect(fixture.widgetText()).toContain("2 layers");
expect(fixture.widgets.at(-1)?.options).toEqual({
placement: "aboveEditor",
});
expect(
fixture.t.events.ui.filter((call) => call.method === "setStatus"),
).toEqual([]);
expect(
fixture.notifications.some((item) =>
item.text.startsWith("Strata review:"),
),
).toBe(false);
const url = fixture.url;
const widgetCalls = fixture.widgets.length;
await fixture.run();
expect(fixture.url).toBe(url);
expect(fixture.calls).toBe(1);
expect(fixture.widgets.length).toBe(widgetCalls + 1);
expect(fixture.notifications).toEqual([]);
});
it("persists draft progress, Ask history, close state, and snapshot-bound reopen", async () => {
fixture = await openFixture();
await fixture.run();
const firstUrl = fixture.url;
const api = await connect(firstUrl);
const { review } = await (await api("state")).json();
const draft = {
reviewed: [review.snapshot.hunks[0].id],
findings: [
{
id: "finding-1",
hunkId: review.snapshot.hunks[0].id,
side: "new",
line: 1,
severity: "minor",
text: "Check the bound",
},
],
notes: "Check retry bounds",
};
expect((await api("draft", draft)).status).toBe(200);
expect(fixture.widgetText()).toContain("1/2 reviewed");
expect(fixture.widgetText()).toContain("1 comment");
expectPhase("Ready");
const answer = await api("ask", {
layerId: "layer-1",
hunkId: review.snapshot.hunks[0].id,
question: "How many attempts?",
});
expect((await answer.json()).answer).toBe(
"Three attempts include the first call.",
);
expect(fixture.calls).toBe(2);
await fixture.run("cancel");
expectPhase("Closed");
expect(fixture.widgetText()).toContain("Run /strata to reopen");
expect(fixture.widgetText()).not.toContain("http://127.0.0.1");
await fixture.run();
expect(fixture.calls).toBe(2);
expect(fixture.url).not.toBe(firstUrl);
const resumed = await connect(fixture.url);
const resumedState = await (await resumed("state")).json();
expect(resumedState.review.draft).toEqual(draft);
expect(resumedState.threads).toEqual([]);
expect(fixture.widgetText()).toContain("1/2 reviewed");
});
it("reflects pending Ask work and returns to Ready", async () => {
fixture = await openFixture();
await fixture.run();
const url = fixture.url;
const api = await connect(url);
const { review } = await (await api("state")).json();
let releaseAsk!: (message: AssistantMessage) => void;
let askStarted!: () => void;
const didAskStart = new Promise<void>((resolve) => {
askStarted = resolve;
});
vi.spyOn(fixture.ctx.modelRegistry, "complete").mockImplementationOnce(
async (_model, _context, options) =>
new Promise<AssistantMessage>((resolve, reject) => {
options?.signal?.addEventListener(
"abort",
() => reject(options.signal?.reason),
{ once: true },
);
releaseAsk = resolve;
askStarted();
}),
);
const asking = api("ask", {
layerId: "layer-1",
hunkId: review.snapshot.hunks[0].id,
question: "Wait for this answer",
});
await didAskStart;
expectPhase("Answering");
releaseAsk(fauxAssistantMessage("Done"));
expect((await asking).status).toBe(200);
expectPhase("Ready");
});
it("preserves the active link and counts after an Ask error", async () => {
fixture = await openFixture();
await fixture.run();
const url = fixture.url;
const api = await connect(url);
vi.spyOn(fixture.ctx.modelRegistry, "complete").mockRejectedValueOnce(
new Error("model\x1b]8;;https://evil.example\x07 unavailable"),
);
expect(
(
await api("ask", {
layerId: "layer-1",
hunkId: null,
question: "Fail",
})
).status,
).toBe(500);
expectPhase("Error");
expect(fixture.widgetText()).toContain(
"model ]8;;https://evil.example unavailable",
);
expect(fixture.url).toBe(url);
expect(fixture.widgetText()).toContain("0/2 reviewed");
});
it("reflects pending refresh work and returns to Ready", async () => {
fixture = await openFixture();
await fixture.run();
const api = await connect(fixture.url);
const { review } = await (await api("state")).json();
let releaseRefresh!: (message: AssistantMessage) => void;
let refreshStarted!: () => void;
const didRefreshStart = new Promise<void>((resolve) => {
refreshStarted = resolve;
});
vi.spyOn(fixture.ctx.modelRegistry, "complete").mockImplementationOnce(
async (_model, _context, options) =>
new Promise<AssistantMessage>((resolve, reject) => {
options?.signal?.addEventListener(
"abort",
() => reject(options.signal?.reason),
{ once: true },
);
releaseRefresh = resolve;
refreshStarted();
}),
);
const refreshing = api("refresh", { confirm: true });
await didRefreshStart;
expectPhase("Refreshing");
releaseRefresh(fauxAssistantMessage(JSON.stringify(review.plan)));
expect((await refreshing).status).toBe(200);
expectPhase("Ready");
});
it("preserves the active link after a refresh error", async () => {
fixture = await openFixture();
await fixture.run();
const url = fixture.url;
const api = await connect(url);
vi.spyOn(fixture.ctx.modelRegistry, "complete").mockRejectedValueOnce(
new Error("refresh provider failed"),
);
expect((await api("refresh", { confirm: true })).status).toBe(500);
expectPhase("Error");
expect(fixture.widgetText()).toContain("refresh provider failed");
expect(fixture.url).toBe(url);
});
it("keeps stale until refresh clears the old draft", async () => {
fixture = await openFixture();
await fixture.run();
const api = await connect(fixture.url);
const { review } = await (await api("state")).json();
const draft = {
reviewed: [review.snapshot.hunks[0].id],
findings: [],
notes: "Verify the retry contract.",
};
await writeFile(
path.join(fixture.root, "contract.ts"),
"export const attempts = 4;\n",
);
expect((await api("submit", { draft })).status).toBe(409);
expectPhase("Stale");
expect(fixture.widgetText()).toContain("1/2 reviewed");
expect((await api("draft", draft)).status).toBe(200);
expectPhase("Stale");
expect((await api("refresh", { confirm: true })).status).toBe(200);
expectPhase("Ready");
expect(fixture.widgetText()).toContain("0/2 reviewed");
});
it("keeps Feedback sent through the empty draft save", async () => {
fixture = await openFixture();
await writeFile(
path.join(fixture.root, "contract.ts"),
"export const attempts = 4;\n",
);
await fixture.run();
const api = await connect(fixture.url);
const { review: refreshedReview } = await (await api("state")).json();
const send = vi
.spyOn(fixture.t.session, "prompt")
.mockResolvedValue(undefined);
const feedback = {
reviewed: [refreshedReview.snapshot.hunks[0].id],
findings: [],
notes: "Four attempts are intentional.",
};
expect((await api("submit", { draft: feedback })).status).toBe(200);
await vi.waitFor(() => expect(send).toHaveBeenCalled());
expect(send.mock.calls[0][0]).toContain(refreshedReview.snapshot.id);
expectPhase("Feedback sent");
expect(fixture.widgetText()).toContain("0/2 reviewed");
expect((await (await api("state")).json()).review.draft.notes).toBe("");
});
it("retains URL and error discovery in RPC mode without component factories", async () => {
fixture = await openFixture("rpc");
await fixture.run();
const url = fixture.url;
expect(fixture.widgets).toEqual([]);
expect(fixture.notifications).toEqual([
{ text: `Strata review: ${url}`, severity: "info" },
]);
await fixture.run();
expect(fixture.notifications).toHaveLength(1);
await fixture.run("cancel");
await writeFile(
path.join(fixture.root, "contract.ts"),
"export const attempts = 5;\n",
);
fixture.faux.setResponses([fauxAssistantMessage("not json")]);
await fixture.run();
expect(fixture.notifications.at(-1)).toMatchObject({ severity: "error" });
});
it("emits URL and error custom messages without terminal writes or LLM turns when no UI exists", async () => {
const stdout = vi.spyOn(process.stdout, "write").mockReturnValue(true);
const stderr = vi.spyOn(process.stderr, "write").mockReturnValue(true);
fixture = await openFixture("json");
await fixture.run();
const statusMessages = () =>
fixture?.t.events.messages.filter(
(message) =>
message.role === "custom" && message.customType === "strata-status",
) ?? [];
expect(statusMessages()).toHaveLength(1);
expect(statusMessages()[0]).toMatchObject({
content: expect.stringMatching(
/Strata review: http:\/\/127\.0\.0\.1:\d+\/\?token=\S+/,
),
display: true,
details: { severity: "info" },
});
expect(fixture.calls).toBe(1);
await fixture.run();
expect(statusMessages()).toHaveLength(1);
expect(fixture.calls).toBe(1);
await fixture.run("cancel");
await writeFile(
path.join(fixture.root, "contract.ts"),
"export const attempts = 5;\n",
);
fixture.faux.setResponses([fauxAssistantMessage("not json")]);
await fixture.run();
expect(statusMessages().at(-1)).toMatchObject({
content: expect.stringContaining("Strata failed:"),
display: true,
details: { severity: "error" },
});
expect(fixture.calls).toBe(1);
expect(stdout).not.toHaveBeenCalled();
expect(stderr).not.toHaveBeenCalled();
});
it("keeps initial failures visible and resets them on reopen", async () => {
fixture = await openFixture();
fixture.faux.setResponses([fauxAssistantMessage("not json")]);
await fixture.run();
expectPhase("Error");
expect(fixture.widgetText()).toContain("Strata failed:");
expect(fixture.widgetText(240)).toContain("Run /strata to retry");
expect(fixture.notifications).toEqual([]);
await fixture.run("--unknown");
expectPhase("Error");
expect(fixture.widgetText()).toContain("usage:");
expect(fixture.widgetText()).not.toContain("model response is not JSON");
});
it("guards cancelled and replaced generations against late callbacks", async () => {
fixture = await openFixture();
const pendingCalls: Array<{
context: unknown;
resolve(message: AssistantMessage): void;
}> = [];
const started = Array.from({ length: 2 }, () =>
Promise.withResolvers<void>(),
);
const complete = vi
.spyOn(fixture.ctx.modelRegistry, "complete")
.mockImplementation(async (_model, context) => {
return new Promise<AssistantMessage>((resolve) => {
pendingCalls.push({ context, resolve });
started[pendingCalls.length - 1].resolve();
});
});
testSignal().addEventListener(
"abort",
() => {
for (const call of pendingCalls)
call.resolve(planResponse(call.context));
},
{ once: true },
);
const oldOpen = fixture.run();
await started[0].promise;
expect(pendingCalls).toHaveLength(1);
await fixture.run("cancel");
expectPhase("Closed");
const replacement = fixture.run();
await started[1].promise;
expect(pendingCalls).toHaveLength(2);
pendingCalls[1].resolve(planResponse(pendingCalls[1].context));
await replacement;
expectPhase("Ready");
const replacementUrl = fixture.url;
pendingCalls[0].resolve(planResponse(pendingCalls[0].context));
await oldOpen;
expectPhase("Ready");
expect(fixture.url).toBe(replacementUrl);
expect(complete).toHaveBeenCalledTimes(2);
});
it("drains pending generation on shutdown and ignores its late callback", async () => {
fixture = await openFixture();
await writeFile(
path.join(fixture.root, "contract.ts"),
"export const attempts = 6;\n",
);
const started = Promise.withResolvers<void>();
const pending = Promise.withResolvers<AssistantMessage>();
let context: unknown;
const complete = vi
.spyOn(fixture.ctx.modelRegistry, "complete")
.mockImplementation(async (_model, value) => {
context = value;
started.resolve();
return pending.promise;
});
testSignal().addEventListener(
"abort",
() =>
pending.resolve(
context ? planResponse(context) : fauxAssistantMessage("cancelled"),
),
{ once: true },
);
const shutdownOpen = fixture.run();
await started.promise;
expect(complete).toHaveBeenCalledOnce();
let settled = false;
const shutdown = fixture.runner
.emit({ type: "session_shutdown", reason: "reload" })
.then(() => {
settled = true;
});
expect(fixture.widgetLines()).toEqual([]);
await Promise.resolve();
expect(settled).toBe(false);
pending.resolve(planResponse(context));
await shutdown;
await shutdownOpen;
expect(settled).toBe(true);
expect(fixture.widgetLines()).toEqual([]);
expect(fixture.notifications).toEqual([]);
});
});