repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/the-system/__tests__/contract.test.ts
Rawimport {
existsSync,
mkdirSync,
readdirSync,
readFileSync,
writeFileSync,
} from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
import {
createCodemodeExtension,
type ExtensionUIContext,
withFileMutationQueue,
} from "@earendil-works/pi-coding-agent";
import { Value } from "typebox/value";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
createPiSession,
type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import { createBoardRuntime } from "../board-runtime.ts";
import { applyInitialization } from "../init.ts";
const ENTRY = fileURLToPath(new URL("../index.ts", import.meta.url));
function codemodeResult(session: PiSession) {
const result = session.session.messages
.filter(
(message) =>
message.role === "toolResult" && message.toolName === "codemode",
)
.at(-1);
if (result?.role !== "toolResult") throw new Error("Missing codemode result");
return result;
}
const recordingUi = (
select: (title: string, items: string[]) => string | undefined,
): ExtensionUIContext =>
({
select: async (title: string, items: string[]) => select(title, items),
confirm: async () => false,
input: async () => undefined,
notify: () => {},
onTerminalInput: () => () => {},
setStatus: () => {},
setWorkingMessage: () => {},
setWorkingVisible: () => {},
setWorkingIndicator: () => {},
setHiddenThinkingLabel: () => {},
setWidget: () => {},
setFooter: () => {},
setHeader: () => {},
setTitle: () => {},
custom: async () => undefined,
pasteToEditor: () => {},
setEditorText: () => {},
getEditorText: () => "",
editor: async () => undefined,
addAutocompleteProvider: () => {},
setEditorComponent: () => {},
getEditorComponent: () => undefined,
getAllThemes: () => [],
getTheme: () => undefined,
setTheme: () => ({ success: false, error: "UI not available" }),
getToolsExpanded: () => false,
setToolsExpanded: () => {},
}) as unknown as ExtensionUIContext;
describe("the-system pi-test-harness", () => {
let session: PiSession | undefined;
afterEach(async () => {
await session?.dispose();
session = undefined;
});
it("loads in real Pi extension runtime", async () => {
session = await createPiSession({ extensionPaths: [ENTRY] });
expect(session.extensions).toHaveLength(1);
expect(session.extensions[0]!.path).toBe(ENTRY);
expect(session.session.getActiveToolNames()).not.toContain("mockup");
});
it("writes debug lifecycle records in its sandbox when enabled", async () => {
session = await createPiSession({
env: { PI_THE_SYSTEM_DEBUG: "1" },
extensionPaths: [ENTRY],
});
const context = session.session.extensionRunner.createCommandContext();
const [lifecycleExtension] = session.extensions;
if (!lifecycleExtension) throw new Error("the-system extension not loaded");
for (const handler of lifecycleExtension.handlers.get("session_start") ??
[])
await handler({ type: "session_start" }, context);
for (const handler of lifecycleExtension.handlers.get("session_shutdown") ??
[])
await handler({ type: "session_shutdown" }, context);
const directory = path.join(
session.cwd,
".test-state",
"pi-ext",
"debug",
"the-system",
);
const records = readFileSync(
path.join(directory, readdirSync(directory)[0]!),
"utf8",
)
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(records.map((record) => record.event)).toEqual(
expect.arrayContaining(["session.start", "session.shutdown"]),
);
});
it("routes authenticated board exec into this Pi session and shuts down cleanly", async () => {
session = await createPiSession({
extensionPaths: [ENTRY],
env: { NODE_ENV: "test" },
});
applyInitialization(session.cwd, "PX");
const directory = path.join(
session.cwd,
".system",
"specs",
"PX-SPEC-SPEC0001-contract",
);
mkdirSync(directory, { recursive: true });
writeFileSync(
path.join(directory, "index.md"),
`---\nid: PX-SPEC-SPEC0001\ntype: spec\ntitle: Contract\n---\n\n# Contract\n`,
);
const extension = session.extensions.find((item) => item.path === ENTRY);
if (!extension) throw new Error("the-system extension not loaded");
const command = extension.commands.get("system");
if (!command) throw new Error("system command not registered");
const notices: string[] = [];
const ui = recordingUi(() => undefined);
session.session.extensionRunner.setUIContext(
{ ...ui, notify: (message: string) => notices.push(message) },
"tui",
);
const send = vi
.spyOn(session.session, "sendUserMessage")
.mockResolvedValue();
const context = session.session.extensionRunner.createCommandContext();
let idle = true;
await command.handler("board", { ...context, isIdle: () => idle });
const url = notices
.join("\\n")
.match(/http:\/\/127\.0\.0\.1:\d+\/board\.html\?k=[\w-]+/)?.[0];
expect(url).toBeDefined();
const authenticated = await fetch(url!, { redirect: "manual" });
expect(authenticated.status).toBe(303);
const cookie = authenticated.headers.get("set-cookie")?.split(";")[0];
expect(cookie).toBeDefined();
const origin = new URL(url!).origin;
const exec = await fetch(`${origin}/api/exec`, {
method: "POST",
headers: { cookie: cookie!, origin, "content-type": "application/json" },
body: JSON.stringify({ id: "PX-SPEC-SPEC0001" }),
});
expect(exec.status).toBe(200);
expect(send).toHaveBeenCalledWith(
expect.stringContaining("# /system exec: PX-SPEC-SPEC0001"),
undefined,
);
send.mockClear();
idle = false;
const busy = await fetch(`${origin}/api/exec`, {
method: "POST",
headers: { cookie: cookie!, origin, "content-type": "application/json" },
body: JSON.stringify({ id: "PX-SPEC-SPEC0001" }),
});
expect(busy.status).toBe(400);
expect(await busy.json()).toEqual({
error: expect.stringContaining("Wait for the current Pi run"),
});
expect(send).not.toHaveBeenCalled();
for (const handler of extension.handlers.get("session_shutdown") ?? [])
await handler({ type: "session_shutdown" }, context);
});
it("serializes codemode board batches and returns bundle slots as structured data", async () => {
session = await createPiSession({
extensionPaths: [ENTRY],
extensionFactories: [createCodemodeExtension()],
userSettings: { defaultTools: ["+codemode"] },
responses: [
fauxAssistantMessage(
fauxToolCall("codemode", {
code: `
await Promise.all([
tools.system_board_update({ items: [{ id: "PX-SPEC-SPEC0001", state: "todo", note: "Spec reviewed." }] }),
tools.system_board_update({ items: [{ id: "PX-ISSUE-ISSUE001", state: "done", note: "Issue verified." }] }),
]);
let rejected = false;
try {
await tools.system_board_update({ items: [{ id: "PX-SPEC-MISSING", state: "done", note: "Missing item." }] });
} catch { rejected = true; }
await tools.system_board_update({ items: [{ id: "PX-ISSUE-ISSUE001", state: "done", note: "Queue recovered." }] });
const slot = await tools.system_bundle_slot({ type: "plan", title: "Concurrent board updates" });
text(JSON.stringify({ rejected, slot }));
`,
}),
{ stopReason: "toolUse" },
),
fauxAssistantMessage("DONE"),
],
});
applyInitialization(session.cwd, "PX");
for (const [relative, source] of [
[
".system/specs/PX-SPEC-SPEC0001-contract/index.md",
"---\nid: PX-SPEC-SPEC0001\ntype: spec\ntitle: Contract\n---\n\n# Contract\n",
],
[
".system/plans/PX-PLAN-PLAN0001-work/index.md",
"---\nid: PX-PLAN-PLAN0001\ntype: plan\ntitle: Work\nspec: PX-SPEC-SPEC0001\n---\n\n# Work\n",
],
[
".system/issues/PX-ISSUE-ISSUE001-bug/index.md",
"---\nid: PX-ISSUE-ISSUE001\ntype: issue\ntitle: Bug\nspecs: []\n---\n\n# Bug\n",
],
] as const) {
const target = path.join(session.cwd, relative);
mkdirSync(path.dirname(target), { recursive: true });
writeFileSync(target, source);
}
await session.session.prompt(
"Run concurrent board updates and reserve a plan bundle.",
);
const result = codemodeResult(session);
expect(result.isError).toBe(false);
const output = result.content
.filter((part) => part.type === "text")
.map((part) => part.text)
.join("\n");
const jsonStart = output.indexOf("{");
const jsonEnd = output.lastIndexOf("}") + 1;
const { rejected, slot } = JSON.parse(output.slice(jsonStart, jsonEnd)) as {
rejected: boolean;
slot: Record<string, unknown>;
};
expect(rejected).toBe(true);
const board = JSON.parse(
readFileSync(path.join(session.cwd, ".system/board.json"), "utf8"),
);
expect(board.items["PX-SPEC-SPEC0001"]).toMatchObject({
state: "todo",
source: "agent",
note: "Spec reviewed.",
});
expect(board.items["PX-ISSUE-ISSUE001"]).toMatchObject({
state: "done",
source: "agent",
note: "Queue recovered.",
});
const extension = session.extensions.find((item) => item.path === ENTRY);
const definition = extension?.tools.get("system_bundle_slot")?.definition;
if (!definition?.outputSchema)
throw new Error("Missing bundle slot output schema");
expect(Value.Check(definition.outputSchema, slot)).toBe(true);
expect(slot).toMatchObject({
type: "plan",
title: "Concurrent board updates",
id: expect.stringMatching(/^PX-PLAN-[A-Z0-9_]{8}$/),
path: `.system/plans/${slot["id"]}-concurrent-board-updates/index.md`,
});
expect(session.errors).toEqual([]);
});
it.each(["system_board_update", "system_board_propose"])(
"cancels queued %s before any mutation",
async (name) => {
session = await createPiSession({ extensionPaths: [ENTRY] });
applyInitialization(session.cwd, "PX");
const board = path.join(session.cwd, ".system/board.json");
const before = existsSync(board)
? readFileSync(board, "utf8")
: undefined;
let release!: () => void;
let entered!: () => void;
const held = new Promise<void>((resolve) => {
release = resolve;
});
const started = new Promise<void>((resolve) => {
entered = resolve;
});
const locked = withFileMutationQueue(board, async () => {
entered();
await held;
});
await started;
const tool = session.extensions[0]!.tools.get(name)!.definition;
const controller = new AbortController();
const reason = new Error("cancelled while queued");
const pending = tool.execute(
"queued",
{ items: [] },
controller.signal,
undefined,
session.session.extensionRunner.createToolContext(
"queued",
controller.signal,
),
);
controller.abort(reason);
const rejected = expect(pending).rejects.toThrow(/cancel/);
release();
await Promise.all([locked, rejected]);
expect(existsSync(board) ? readFileSync(board, "utf8") : undefined).toBe(
before,
);
},
);
it("uses replacement proposals in an already-open board server", async () => {
session = await createPiSession({});
applyInitialization(session.cwd, "PX");
const issue = path.join(
session.cwd,
".system/issues/PX-ISSUE-ISSUE001-bug/index.md",
);
mkdirSync(path.dirname(issue), { recursive: true });
writeFileSync(
issue,
"---\nid: PX-ISSUE-ISSUE001\ntype: issue\ntitle: Bug\nspecs: []\n---\n\n# Bug\n",
);
const runtime = createBoardRuntime();
const ctx = session.session.extensionRunner.createToolContext(
"proposal",
undefined,
);
const propose = (state: string) =>
runtime.tool.execute(
"proposal",
{
items: [
{ id: "PX-ISSUE-ISSUE001", state, rationale: "Reviewed evidence." },
],
},
undefined,
undefined,
ctx,
);
try {
await propose("open");
const opened = await runtime.open(session.cwd);
const url = opened.match(/http:\/\/127\.0\.0\.1:[^\s]+/)?.[0];
if (!url) throw new Error("Missing local board URL");
const page = await fetch(url, { redirect: "manual" });
const cookie = page.headers.get("set-cookie")?.split(";")[0];
if (!cookie) throw new Error("Missing board cookie");
const origin = new URL(url).origin;
await propose("done");
const accept = (state: string) =>
fetch(`${origin}/api/proposal`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Cookie: cookie,
Origin: origin,
},
body: JSON.stringify({ id: "PX-ISSUE-ISSUE001", state }),
});
expect((await accept("open")).status).toBe(400);
expect((await accept("done")).status).toBe(200);
} finally {
runtime.shutdown();
}
});
it("registers scoped tools deferred and callable without a System stage", async () => {
session = await createPiSession({ extensionPaths: [ENTRY] });
applyInitialization(session.cwd, "PX");
const extension = session.extensions.find((item) => item.path === ENTRY);
if (!extension) throw new Error("the-system extension not loaded");
for (const name of [
"system_bundle_slot",
"system_board_propose",
"system_board_update",
]) {
const tool = extension.tools.get(name)?.definition;
expect(tool?.exposure).toBe("deferred");
expect(tool?.annotations?.readOnlyHint).toBe(false);
expect(session.session.getCallableToolNames()).toContain(name);
expect(session.session.getActiveToolNames()).not.toContain(name);
expect(session.session.systemPrompt).not.toContain(`- ${name}:`);
}
});
it.each([true, false])(
"preserves nested codemode mutation approval: %s",
async (approve) => {
session = await createPiSession({
extensionPaths: [ENTRY],
extensionFactories: [createCodemodeExtension()],
userSettings: { defaultTools: ["+codemode"] },
responses: [
fauxAssistantMessage(
fauxToolCall("codemode", {
code: 'await tools.edit({ path: ".system/SYSTEM.md", edits: [{ oldText: "Approved.", newText: "Human-approved." }] });',
}),
),
fauxAssistantMessage("approval requested"),
],
});
applyInitialization(session.cwd, "PX");
const extension = session.extensions.find((item) => item.path === ENTRY);
if (!extension) throw new Error("the-system extension not loaded");
const systemCommand = extension.commands.get("system");
if (!systemCommand) throw new Error("system command not registered");
vi.spyOn(session.session, "sendUserMessage").mockResolvedValue();
await systemCommand.handler(
"spec nested confirmation",
session.session.extensionRunner.createCommandContext(),
);
writeFileSync(
path.join(session.cwd, ".system", "SYSTEM.md"),
"Approved.\n",
);
const nestedStarts: {
toolName: string;
parentToolCallId?: string | undefined;
}[] = [];
const unsubscribe = session.session.subscribe((event) => {
if (event.type === "tool_execution_start" && event.toolName === "edit")
nestedStarts.push({
toolName: event.toolName,
parentToolCallId: event.parentToolCallId,
});
});
const selectCalls: { args: [string, string[]] }[] = [];
session.session.extensionRunner.setUIContext(
recordingUi((title, items) => {
selectCalls.push({ args: [title, items] });
return items[approve ? 0 : 1];
}),
"tui",
);
await session.session.prompt("run the scripted nested edit");
unsubscribe();
const [approval] = selectCalls;
expect(approval?.args).toEqual([
"The System ownership approval",
[
"Approve once: edit .system/SYSTEM.md",
"Deny: keep .system/SYSTEM.md unchanged",
],
]);
expect(nestedStarts).toEqual([
{ toolName: "edit", parentToolCallId: expect.any(String) },
]);
expect(
readFileSync(path.join(session.cwd, ".system", "SYSTEM.md"), "utf8"),
).toBe(approve ? "Human-approved.\n" : "Approved.\n");
},
);
});