repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/afk/__tests__/contract.test.ts
Rawimport { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import {
type FauxResponseStep,
fauxAssistantMessage,
fauxToolCall,
type TranscriptContext,
} from "@earendil-works/pi-ai";
import {
defineTool,
type ExtensionFactory,
} from "@earendil-works/pi-coding-agent";
import type { Component, TUI } from "@earendil-works/pi-tui";
import { Type } from "typebox";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
createPiSession,
type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import afk from "../index.ts";
import { catTheme } from "./theme.ts";
let session: PiSession | undefined;
const path = fileURLToPath(new URL("../index.ts", import.meta.url));
function command(name: "afk" | "back") {
const result = session?.extensions
.flatMap((extension) => [...extension.commands])
.find(([key]) => key === name)?.[1];
if (!result) throw new Error(`Missing ${name}`);
return result;
}
function text(context: TranscriptContext) {
return context.messages
.map((message) =>
[
message.role === "system"
? Object.values(message.sections ?? {}).join("\n")
: "",
typeof message.content === "string"
? message.content
: message.content
.flatMap((block) => (block.type === "text" ? [block.text] : []))
.join("\n"),
].join("\n"),
)
.join("\n");
}
function script(
options: {
objections?: number;
executor?: FauxResponseStep;
initial?: () => Promise<void>;
review?: () => Promise<void>;
invalid?: boolean;
} = {},
) {
const observed = { reviews: 0, executions: 0, intents: 0, reviewText: "" };
const step: FauxResponseStep = async (
context,
modelOptions,
state,
model,
) => {
const input = text(context);
if (input.includes("AFK_JUDGE:")) {
const review = input.includes('"phase":"review"');
if (review) {
observed.reviews++;
observed.reviewText = input;
await options.review?.();
} else {
observed.intents++;
await options.initial?.();
}
if (options.invalid && review) return fauxAssistantMessage("not JSON");
return fauxAssistantMessage(
JSON.stringify({
goal: "Finish OAuth acceptance",
intent:
"Finish all requested acceptance, not just scoped green checks. Cite latest user messages.",
objections:
review && observed.reviews <= (options.objections ?? 0)
? [
"Acceptance remains unfinished: verify publication faults before stopping.",
]
: [],
}),
);
}
observed.executions++;
return typeof options.executor === "function"
? options.executor(context, modelOptions, state, model)
: (options.executor ??
fauxAssistantMessage(
"Implemented scoped work; publication acceptance remains unfinished.",
));
};
return { responses: Array.from({ length: 40 }, () => step), observed };
}
async function start() {
await command("afk").handler(
"",
session!.session.extensionRunner.createContext() as never,
);
await session!.session.waitForIdle();
}
function entries(type: string) {
return session!.session.sessionManager
.getBranch()
.filter(
(entry) => entry.type === "custom_message" && entry.customType === type,
);
}
const humanExecution = vi.fn();
const human: ExtensionFactory = (pi) => {
pi.registerTool(
defineTool({
name: "ask_user_question",
label: "Ask",
description: "Ask a human",
parameters: Type.Object({}),
async execute() {
humanExecution();
return {
content: [{ type: "text", text: "Human answered" }],
details: undefined,
};
},
}),
);
};
afterEach(async () => {
await session?.dispose();
session = undefined;
});
describe("AFK judged continuation through real Pi", () => {
it("has no completion tool and naturally settles after one clean review", async () => {
const run = script();
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
expect(session.extensions[0]!.tools.has("afk_complete")).toBe(false);
await start();
expect(run.observed).toMatchObject({
executions: 1,
intents: 1,
reviews: 1,
});
expect(entries("afk-review")).toHaveLength(0);
expect(entries("afk-review-error")).toHaveLength(0);
expect(session.errors).toEqual([]);
});
it("delivers an objection and subsequent executor evidence to the next review", async () => {
let pass = 0;
const run = script({
objections: 1,
executor: () =>
fauxAssistantMessage(
++pass === 1
? "Scoped checks green; publication acceptance remains unfinished."
: "Publication fault acceptance verified: 12 fault-injection cases pass.",
),
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await start();
expect(run.observed).toMatchObject({
executions: 2,
intents: 1,
reviews: 2,
});
expect(entries("afk-review")).toHaveLength(1);
expect(run.observed.reviewText).toContain(
"publication acceptance remains unfinished",
);
expect(run.observed.reviewText).toContain("12 fault-injection cases pass");
expect(entries("afk-review-error")).toHaveLength(0);
expect(session.errors).toEqual([]);
});
it("allows exactly five objections plus a final executor pass; /afk rearms the budget", async () => {
const run = script({ objections: 99 });
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await start();
expect(run.observed).toMatchObject({
executions: 6,
intents: 1,
reviews: 5,
});
expect(entries("afk-review")).toHaveLength(5);
expect(entries("afk-limit")).toHaveLength(1);
await start();
expect(run.observed).toMatchObject({
executions: 12,
intents: 2,
reviews: 10,
});
expect(entries("afk-limit")).toHaveLength(2);
});
it("reconstructs intent asynchronously while the executor works", async () => {
let release!: () => void;
let executed!: () => void;
const initial = new Promise<void>((resolve) => {
release = resolve;
});
const execution = new Promise<void>((resolve) => {
executed = resolve;
});
const run = script({
initial: () => initial,
executor: () => {
executed();
return fauxAssistantMessage("Worked while intent was pending.");
},
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await command("afk").handler(
"",
session.session.extensionRunner.createContext() as never,
);
try {
await execution;
expect(run.observed.reviews).toBe(0);
} finally {
release();
}
await session.session.waitForIdle();
expect(run.observed.reviews).toBe(1);
});
it("does not let /back cancel executor work or publish a stale intent", async () => {
let release!: () => void;
const initial = new Promise<void>((resolve) => {
release = resolve;
});
const run = script({
initial: () => initial,
executor: async () => {
await command("back").handler(
"",
session!.session.extensionRunner.createContext() as never,
);
release();
return fauxAssistantMessage("Current work finished normally.");
},
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await start();
expect(run.observed.executions).toBe(1);
expect(run.observed.reviews).toBe(0);
expect(entries("afk-intent")).toHaveLength(0);
expect(entries("afk-review-error")).toHaveLength(0);
});
it("cancels a held review through real SDK abort without stale objections or compaction restart", async () => {
let entered!: () => void, release!: () => void;
const reviewing = new Promise<void>((resolve) => {
entered = resolve;
});
const held = new Promise<void>((resolve) => {
release = resolve;
});
const run = script({
objections: 1,
review: async () => {
entered();
await held;
},
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await command("afk").handler(
"",
session.session.extensionRunner.createContext() as never,
);
await reviewing;
try {
await session.session.abort();
expect(entries("afk-review")).toHaveLength(0);
expect(entries("afk-review-error")).toHaveLength(0);
const send = vi.spyOn(session.session, "sendCustomMessage");
await session.session.extensionRunner.emit({
type: "session_compact",
compactionEntry: { id: "aborted" },
fromExtension: false,
reason: "manual",
willRetry: false,
} as never);
await new Promise((resolve) => setTimeout(resolve, 10));
expect(send).not.toHaveBeenCalled();
} finally {
release();
}
}, 5000);
it("reports malformed judge output instead of pretending completion", async () => {
const run = script({ invalid: true });
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await start();
expect(entries("afk-review-error")).toHaveLength(1);
expect(run.observed.executions).toBe(1);
});
it("drains queued user follow-ups in order and includes them in review evidence", async () => {
let queued = false;
const queue: ExtensionFactory = (pi) => {
pi.on("agent_start", () => {
if (queued) return;
queued = true;
pi.sendUserMessage("Follow-up A", { deliverAs: "followUp" });
pi.sendUserMessage("Follow-up B", { deliverAs: "followUp" });
});
};
const run = script();
session = await createPiSession({
extensionPaths: [path],
extensionFactories: [queue],
responses: run.responses,
});
await start();
const users = session.session.messages.filter(
(message) => message.role === "user",
);
expect(users).toHaveLength(2);
expect(run.observed.reviewText).toContain("Follow-up A");
expect(run.observed.reviewText.indexOf("Follow-up A")).toBeLessThan(
run.observed.reviewText.indexOf("Follow-up B"),
);
expect(run.observed.reviews).toBe(1);
});
it("blocks routine human tools but lets the executor use an alternative", async () => {
let calls = 0;
humanExecution.mockClear();
const boundaries: unknown[] = [];
const observe: ExtensionFactory = (pi) => {
pi.on("turn_end", (event) => {
boundaries.push({
stop:
event.message.role === "assistant"
? event.message.stopReason
: event.message.role,
continue: event.continue,
pending: event.context.pendingMessages,
});
});
};
const run = script({
executor: () =>
++calls === 1
? fauxAssistantMessage(fauxToolCall("ask_user_question", {}), {
stopReason: "toolUse",
})
: fauxAssistantMessage("Used documented defaults."),
});
session = await createPiSession({
extensionPaths: [path],
extensionFactories: [human, observe],
responses: run.responses,
});
await start();
expect(
session.session.messages.some(
(message) =>
message.role === "toolResult" &&
message.toolName === "ask_user_question" &&
message.isError,
),
).toBe(true);
expect(run.observed.executions).toBe(2);
expect(
run.observed.reviews,
JSON.stringify({
boundaries,
errors: session.errors,
reviewErrors: entries("afk-review-error"),
}),
).toBe(1);
expect(run.observed.reviewText).toContain("Used documented defaults");
expect(humanExecution).not.toHaveBeenCalled();
expect(entries("afk-review-error")).toHaveLength(0);
});
it.each(["aborted", "error"] as const)(
"respects executor %s without reviewing or continuing",
async (stopReason) => {
const run = script({
executor: fauxAssistantMessage("Stopped", { stopReason }),
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await start();
expect(run.observed.executions).toBe(1);
expect(run.observed.reviews).toBe(0);
},
);
it("reviews custom follow-up work queued during a held clean verdict", async () => {
let queued = false;
const run = script({
review: async () => {
if (queued) return;
queued = true;
await session!.session.sendCustomMessage(
{
customType: "queued-work",
content: "Finish independent follow-up",
display: false,
},
{ triggerTurn: true, deliverAs: "followUp" },
);
},
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await start();
expect(run.observed).toMatchObject({ executions: 2, reviews: 2 });
expect(run.observed.reviewText).toContain("Finish independent follow-up");
expect(entries("afk-review-error")).toHaveLength(0);
});
it("includes passive context from other extensions without skipping review", async () => {
const run = script({
executor: async () => {
await session!.session.sendCustomMessage(
{
customType: "ephemlet-build-result",
content: "Preview ready",
display: false,
},
{ triggerTurn: false },
);
return fauxAssistantMessage("Used completed preview.");
},
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
await start();
expect(run.observed).toMatchObject({ executions: 1, reviews: 1 });
expect(run.observed.reviewText).toContain("Preview ready");
expect(entries("afk-review-error")).toHaveLength(0);
});
it("reviews work continued by a later turn-boundary handler", async () => {
let continued = false;
const later: ExtensionFactory = (pi) => {
pi.on("turn_end", (event) => {
if (
continued ||
event.message.role !== "assistant" ||
event.message.stopReason !== "stop"
)
return;
continued = true;
return { continue: true };
});
};
const run = script();
session = await createPiSession({
extensionFactories: [afk, later],
responses: run.responses,
});
await start();
expect(run.observed).toMatchObject({ executions: 2, reviews: 2 });
});
it("preserves earlier boundary drafts alongside reviewer objections", async () => {
let added = false;
const earlier: ExtensionFactory = (pi) => {
pi.on("turn_end", (event) => {
if (added) return;
added = true;
return {
entries: [
...event.entries,
{
type: "custom_message",
customType: "earlier-draft",
content: "Evidence from another extension",
display: false,
},
],
};
});
};
const run = script({ objections: 1 });
session = await createPiSession({
extensionFactories: [earlier, afk],
responses: run.responses,
});
await start();
expect(entries("earlier-draft")).toHaveLength(1);
expect(entries("afk-review")).toHaveLength(1);
expect(run.observed.reviewText).toContain(
"Evidence from another extension",
);
});
it("keeps AFK active through native retry of a transient executor error", async () => {
let passes = 0;
const run = script({
executor: () =>
++passes === 1
? fauxAssistantMessage("", {
stopReason: "error",
errorMessage: "503 Service Unavailable",
})
: fauxAssistantMessage("Recovered executor result."),
});
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
userSettings: {
retry: {
enabled: true,
maxRetries: 1,
baseDelayMs: 1,
maxAgentDelayMs: 1,
},
},
});
await start();
expect(run.observed).toMatchObject({ executions: 2, reviews: 1 });
expect(run.observed.reviewText).toContain("Recovered executor result");
});
it("renders native widget goal, pleased reaction, nap and disposes on /back", async () => {
const run = script({ objections: 1 });
session = await createPiSession({
extensionPaths: [path],
responses: run.responses,
});
let creature: (Component & { dispose?(): void }) | undefined;
await session.session.bindExtensions({
mode: "tui",
uiContext: {
...session.session.extensionRunner.getUIContext(),
setWidget: (_key, content) => {
creature =
typeof content === "function"
? content({ requestRender() {} } as TUI, catTheme())
: undefined;
},
},
});
await start();
expect(creature!.render(80)[0]).toContain("Finish OAuth acceptance");
expect(creature!.render(80)[1]).toContain("objection addressed");
const dispose = vi.spyOn(creature!, "dispose");
await command("back").handler(
"",
session.session.extensionRunner.createContext() as never,
);
expect(dispose).toHaveBeenCalledOnce();
expect(creature).toBeUndefined();
});
it.each(["alone", "first", "last"])(
"recovers idle compaction with AFK %s and leaves naps settled",
async (order) => {
const paths = [path];
if (order !== "alone")
paths.push(
fileURLToPath(new URL("../../continuity/index.ts", import.meta.url)),
);
if (order === "last") paths.reverse();
const run = script();
session = await createPiSession({
extensionPaths: paths,
responses: run.responses,
});
const startMessage = vi
.spyOn(session.session, "sendCustomMessage")
.mockImplementationOnce(async () => {});
await command("afk").handler(
"",
session.session.extensionRunner.createContext() as never,
);
startMessage.mockRestore();
const compact = () =>
session!.session.extensionRunner.emit({
type: "session_compact",
compactionEntry: { id: "test-compact" },
fromExtension: false,
reason: "manual",
willRetry: false,
} as never);
await compact();
await vi.waitFor(() => expect(run.observed.reviews).toBe(1));
await session.session.waitForIdle();
expect(entries("afk-continue")).toHaveLength(1);
expect(entries("continuity-resume")).toHaveLength(0);
const send = vi.spyOn(session.session, "sendCustomMessage");
await compact();
await new Promise((resolve) => setTimeout(resolve, 10));
expect(send).not.toHaveBeenCalled();
expect(session.errors).toEqual([]);
},
);
it("writes schema-bound lifecycle diagnostics only when enabled", async () => {
session = await createPiSession({
extensionPaths: [path],
env: { PI_AFK_DEBUG: "1" },
});
await session.session.extensionRunner.emit({
type: "session_shutdown",
reason: "quit",
});
const directory = join(
session.env["XDG_STATE_HOME"]!,
"pi-ext",
"debug",
"afk",
);
await vi.waitFor(() => {
const events = readdirSync(directory).flatMap((file) =>
readFileSync(join(directory, file), "utf8")
.trim()
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line).event),
);
expect(events).toContain("session.start");
expect(events).toContain("session.shutdown");
});
});
});