repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/ephemlets/__tests__/contract.test.ts
Rawimport { readdir, readFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import {
fauxAssistantMessage,
fauxToolCall,
getCurrentSystemPrompt,
getCurrentTools,
} from "@earendil-works/pi-ai";
import { convertResponsesTools } from "@earendil-works/pi-ai/api/openai-responses-shared";
import {
type AgentToolResult,
createAgentSession,
DefaultResourceLoader,
type ExtensionUIContext,
SessionManager,
SettingsManager,
} from "@earendil-works/pi-coding-agent";
import type { Component, TUI } from "@earendil-works/pi-tui";
import { Value } from "typebox/value";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
createPiSession,
type PiSession,
} from "../../../test/fixture/create-pi-session.ts";
import type { EphemletParams } from "../runtime.ts";
let fixture: PiSession | undefined;
const source =
"export default (ctx) => ({render: () => [String(ctx.state)], invalidate() {}})";
afterEach(async () => {
await fixture?.dispose();
fixture = undefined;
});
async function setup(debug = false) {
fixture = await createPiSession({
extensionPaths: [fileURLToPath(new URL("../index.ts", import.meta.url))],
...(debug ? { env: { PI_EPHEMLETS_DEBUG: "1" } } : {}),
});
expect(fixture.errors).toEqual([]);
await fixture.session.bindExtensions({
mode: "tui",
uiContext: {
...fixture.session.extensionRunner.getUIContext(),
custom: async () => undefined,
} as ExtensionUIContext,
});
expect(fixture.session.getActiveToolNames()).toContain("ephemlet");
const definition = fixture.extensions[0]!.tools.get("ephemlet")!.definition;
expect(definition.annotations?.readOnlyHint).toBe(false);
const session = fixture.session;
const call = async (
params: EphemletParams,
): Promise<AgentToolResult<unknown>> =>
definition.execute(
"ephemlet-test",
params,
undefined,
undefined,
session.extensionRunner.createToolContext("ephemlet-test", undefined),
);
return { fixture, call, session };
}
function data(result: AgentToolResult<unknown>): Record<string, unknown> {
const content = result.content[0];
if (content?.type !== "text") throw new Error("Missing text result");
return JSON.parse(content.text) as Record<string, unknown>;
}
describe("ephemlets public Pi runtime", () => {
it("passes the authoring guidance file to the delegated builder", async () => {
const { call, fixture } = await setup();
fixture.setResponses([
(context) => {
expect(getCurrentSystemPrompt(context.messages)).toContain(
"Persisted source is loaded by jiti",
);
return fauxAssistantMessage(
fauxToolCall("submit_ephemlet", { source }),
{ stopReason: "toolUse" },
);
},
]);
const built = data(
await call({
action: "build",
title: "Guidance",
purpose: "Verify builder guidance",
brief: "Build a component",
wait: true,
}),
);
expect(built["status"]).toBe("ready");
});
it("declares ephemlet to the main model and handles a real tool turn", async () => {
const { call, session, fixture } = await setup();
fixture.setResponses([
(context) => {
expect(
getCurrentTools(context.messages).map((tool) => tool.name),
).toContain("ephemlet");
return fauxAssistantMessage(
fauxToolCall("ephemlet", {
action: "put",
title: "Progress",
purpose: "Track actual work",
source: "# Verified progress",
}),
{ stopReason: "toolUse" },
);
},
fauxAssistantMessage("Report created"),
]);
await session.prompt("Create a living progress report");
expect(data(await call({ action: "list" }))["ephemlets"]).toHaveLength(1);
});
it("declares provider-compatible recursive JSON state without narrowing stored values", async () => {
const { call, fixture, session } = await setup();
const tool = fixture.extensions[0]!.tools.get("ephemlet")!.definition;
const declaration = convertResponsesTools([tool])[0]!;
if (declaration.type !== "function")
throw new Error("Expected function tool");
expect(declaration.strict).toBe(false);
expect(convertResponsesTools([tool], { strict: null })[0]).toMatchObject({
type: "function",
strict: null,
parameters: declaration.parameters,
});
expect(declaration.parameters).toMatchObject({
type: "object",
properties: { state: { $ref: "#/$defs/json" } },
$defs: {
json: {
anyOf: [
{ type: "null" },
{ type: "boolean" },
{ type: "number" },
{ type: "string" },
{ type: "array", items: { $ref: "#/$defs/json" } },
{ type: "object", additionalProperties: { $ref: "#/$defs/json" } },
],
},
},
});
const wireSchema = declaration.parameters as typeof tool.parameters;
const created = data(
await call({
action: "put",
title: "JSON",
purpose: "Schema contract",
source: "# JSON",
}),
);
const id = created["id"] as string;
for (const state of [
null,
false,
0,
"",
[],
{},
[1, { nested: [true, null] }],
{ nested: { values: ["A", false, null] } },
]) {
const params = {
action: "state" as const,
id,
expectedRevision: 1,
state,
};
expect(Value.Check(wireSchema, params)).toBe(true);
const updated = await call(params);
expect(Value.Check(tool.outputSchema!, updated.structuredContent)).toBe(
true,
);
const snapshot = await call({ action: "get", id, include: ["state"] });
expect(data(snapshot)["state"]).toEqual(state);
expect(Value.Check(tool.outputSchema!, snapshot.structuredContent)).toBe(
true,
);
}
for (const state of [
{ nested: undefined },
{ nested: () => 1 },
{ nested: 1n },
]) {
expect(
Value.Check(wireSchema, {
action: "state",
id,
expectedRevision: 1,
state,
}),
).toBe(false);
}
const state = { code: "001", nested: [null, false, 0, { value: "true" }] };
fixture.setResponses([
fauxAssistantMessage(
fauxToolCall("ephemlet", {
action: "state",
id,
expectedRevision: 1,
state,
}),
{ stopReason: "toolUse" },
),
fauxAssistantMessage("State updated"),
]);
await session.prompt("Persist the JSON state");
expect(
data(await call({ action: "get", id, include: ["state"] }))["state"],
).toEqual(state);
});
it("uses native sequential execution, structured outputs, and interactive exposure", async () => {
const { call, fixture } = await setup();
const tool = fixture.extensions[0]!.tools.get("ephemlet")!.definition;
expect(tool.exposure).toBe("model-only");
expect(tool.executionMode).toBe("sequential");
expect(tool.constrainedSampling).toEqual({
type: "json_schema",
strict: "prefer",
});
expect(tool.renderCall).toBeTypeOf("function");
expect(tool.renderResult).toBeTypeOf("function");
const checked = async (params: EphemletParams) => {
const result = await call(params);
expect(result.structuredContent).toEqual(data(result));
expect(Value.Check(tool.outputSchema!, result.structuredContent)).toBe(
true,
);
return data(result);
};
const created = await checked({
action: "put",
title: "Report",
purpose: "Track",
source: "# Report",
});
const id = created["id"] as string;
await checked({ action: "list" });
await checked({ action: "get", id });
await checked({ action: "get", id, include: ["source", "state"] });
await checked({
action: "state",
id,
expectedRevision: 1,
state: { selected: "A" },
});
await checked({ action: "hide" });
await checked({ action: "show", id });
await checked({ action: "delete", id });
});
it("bounds default data reads and keeps persisted content out of automatic model context", async () => {
const { call, session, fixture } = await setup();
const oldSource = `private-original${"x".repeat(60000)}`;
const newSource = `private-source${"y".repeat(60000)}`;
const state = { choice: "A", payload: `private-state${"z".repeat(60000)}` };
const created = data(
await call({
action: "put",
title: "Bounded",
purpose: "p".repeat(4096),
source: oldSource,
state,
}),
);
const id = created["id"] as string;
await call({ action: "put", id, expectedRevision: 1, source: newSource });
const summary = data(await call({ action: "get", id }));
expect(summary).toMatchObject({ id, revision: 2, status: "ready" });
expect(summary).not.toHaveProperty("source");
expect(summary).not.toHaveProperty("state");
expect(summary).not.toHaveProperty("fallback");
expect(summary["purpose"]).toBe(`${"p".repeat(159)}…`);
expect(JSON.stringify(summary).length).toBeLessThan(1000);
const list = data(await call({ action: "list" }));
expect(JSON.stringify(list).length).toBeLessThan(2000);
expect(list["ephemlets"]).toEqual([summary]);
const withSource = data(
await call({ action: "get", id, include: ["source"] }),
);
expect(withSource["source"]).toBe(newSource);
expect(withSource).not.toHaveProperty("state");
expect(withSource).not.toHaveProperty("fallback");
const withState = data(
await call({ action: "get", id, include: ["state"] }),
);
expect(withState["state"]).toEqual(state);
expect(withState).not.toHaveProperty("source");
expect(withState).not.toHaveProperty("fallback");
fixture.setResponses([
(context) => {
const prompt = getCurrentSystemPrompt(context.messages);
expect(prompt).toContain("agent cannot see the rendered panel");
expect(prompt).toContain('include: ["state"]');
expect(prompt).toContain("do not start a new turn");
expect(prompt).toContain("exact snapshot");
const transcript = JSON.stringify(context.messages);
for (const marker of [
"private-original",
"private-source",
"private-state",
])
expect(transcript).not.toContain(marker);
return fauxAssistantMessage("OK");
},
]);
await session.prompt("Continue the task");
});
it.each(["ready", "failed"] as const)(
"delivers background %s notices without starting a main-agent turn",
async (status) => {
const { call, session, fixture } = await setup();
const send = vi.spyOn(session, "sendCustomMessage");
fixture.setResponses([
status === "ready"
? fauxAssistantMessage(fauxToolCall("submit_ephemlet", { source }), {
stopReason: "toolUse",
})
: fauxAssistantMessage("No structured result"),
]);
const pending = data(
await call({
action: "build",
title: "Background",
purpose: "Monitor",
brief: "Native counter",
}),
);
await vi.waitFor(async () => {
expect(
data(await call({ action: "get", id: pending["id"] as string }))[
"status"
],
).toBe(status);
expect(send).toHaveBeenCalledWith(
expect.objectContaining({
customType: "ephemlet-build",
display: false,
details: expect.objectContaining({
id: pending["id"],
jobId: pending["jobId"],
status,
}),
}),
{ triggerTurn: false },
);
});
expect(fixture.providerState.callCount).toBe(1);
const notice = send.mock.calls[0]![0];
expect(JSON.stringify(notice)).not.toContain(source);
fixture.setResponses([
(context) => {
expect(JSON.stringify(context.messages)).toContain(
status === "ready" ? "Ephemlet ready" : "Ephemlet build failed",
);
return fauxAssistantMessage("Status received");
},
]);
await session.prompt("Check the background build");
send.mockRestore();
},
);
it("hides tools and leaves session data untouched in non-TUI modes", async () => {
const { call, fixture, session } = await setup();
await call({
action: "put",
title: "Report",
purpose: "Track",
source: "# Report",
});
const before = session.sessionManager.getBranch().length;
await session.bindExtensions({ mode: "print" });
expect(
fixture.extensions[0]!.tools.get("ephemlet")!.definition.exposure,
).toBe("hidden");
expect(session.getActiveToolNames()).not.toContain("ephemlet");
const hidden = fixture.extensions[0]!.tools.get("ephemlet")!.definition;
await expect(
hidden.execute(
"hidden",
{ action: "list" },
undefined,
undefined,
session.extensionRunner.createToolContext("hidden", undefined),
),
).rejects.toThrow("TUI");
expect(session.sessionManager.getBranch().length).toBe(before);
});
it("registers tool/commands and persists reports through restore and deletion", async () => {
const { call, session, fixture } = await setup();
expect(fixture.extensions[0]!.commands.has("ephemlets")).toBe(true);
const first = data(
await call({
action: "put",
title: "Progress",
purpose: "Track original request",
source: "# Progress\n- Attempted\n- Verified",
}),
);
const id = first["id"] as string;
await call({ action: "hide" });
await session.extensionRunner.emit({
type: "session_start",
reason: "resume",
});
expect(
data(await call({ action: "get", id, include: ["source"] }))["source"],
).toContain("Verified");
expect(data(await call({ action: "list" }))["ephemlets"]).toHaveLength(1);
await call({ action: "delete", id });
await session.extensionRunner.emit({
type: "session_start",
reason: "resume",
});
expect(data(await call({ action: "list" }))["ephemlets"]).toEqual([]);
});
it("restores persisted native source/state into a new Pi session", async () => {
const { call, fixture } = await setup();
const created = data(
await call({
action: "put",
title: "Counter",
purpose: "Persist",
kind: "interactive",
source,
state: 7,
}),
);
await call({ action: "hide" });
const manager = SessionManager.create(
fixture.cwd,
join(fixture.cwd, "saved-sessions"),
);
manager.appendMessage({
role: "user",
content: "Persistence fixture",
timestamp: Date.now(),
});
for (const entry of fixture.session.sessionManager.getBranch())
if (entry.type === "custom")
manager.appendCustomEntry(entry.customType, entry.data);
const path = manager.getSessionFile();
if (!path) throw new Error("No persisted file");
const settings = SettingsManager.inMemory({});
const loader = new DefaultResourceLoader({
cwd: fixture.cwd,
agentDir: fixture.agentDir,
settingsManager: settings,
noExtensions: true,
noSkills: true,
noThemes: true,
noPromptTemplates: true,
noContextFiles: true,
additionalExtensionPaths: [
fileURLToPath(new URL("../index.ts", import.meta.url)),
],
});
await loader.reload();
const model = fixture.session.model;
if (!model) throw new Error("Fixture model is unavailable");
const { session: reopened } = await createAgentSession({
cwd: fixture.cwd,
modelRuntime: fixture.session.modelRuntime,
model,
settingsManager: settings,
resourceLoader: loader,
sessionManager: SessionManager.open(path),
});
try {
await reopened.bindExtensions({
mode: "tui",
uiContext: {
...fixture.session.extensionRunner.getUIContext(),
custom: async () => undefined,
} as ExtensionUIContext,
});
const tool = loader
.getExtensions()
.extensions[0]!.tools.get("ephemlet")!.definition;
const restored = data(
await tool.execute(
"restored",
{ action: "get", id: created["id"], include: ["source", "state"] },
undefined,
undefined,
reopened.extensionRunner.createToolContext("restored", undefined),
),
);
expect(restored).toMatchObject({ source, state: 7, revision: 1 });
fixture.setResponses([
(_context, options) => {
expect(options?.apiKey).toBe("test-key");
return fauxAssistantMessage(
fauxToolCall("submit_ephemlet", { source }),
{ stopReason: "toolUse" },
);
},
]);
await tool.execute(
"linked-build",
{
action: "build",
title: "Linked",
purpose: "Native tracking",
brief: "Native counter",
wait: true,
},
undefined,
undefined,
reopened.extensionRunner.createToolContext("linked-build", undefined),
);
const files = await readdir(manager.getSessionDir());
const children = files.filter(
(file) =>
file.endsWith(".jsonl") &&
join(manager.getSessionDir(), file) !== path,
);
expect(children).toHaveLength(1);
const entries = (
await readFile(join(manager.getSessionDir(), children[0]!), "utf8")
)
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(entries[0]).toMatchObject({
type: "session",
parentSession: path,
});
expect(
entries.some(
(entry) =>
entry.type === "message" && entry.message.role === "assistant",
),
).toBe(true);
} finally {
await reopened.extensionRunner.emit({
type: "session_shutdown",
reason: "quit",
});
reopened.dispose();
}
});
it("Save queues a snapshot follow-up without interrupting ongoing main-agent work", async () => {
const { call, session, fixture } = await setup();
let panel: Component | undefined;
const tui = {
terminal: { columns: 120, rows: 32 },
requestRender() {},
showOverlay(component: Component) {
panel = component;
return { hide() {}, setHidden() {}, focus() {} };
},
} as unknown as TUI;
await session.bindExtensions({
mode: "tui",
uiContext: {
...session.extensionRunner.getUIContext(),
setWidget(_key: string, factory: unknown) {
if (typeof factory === "function")
factory(tui, { fg: (_color: string, text: string) => text });
},
} as ExtensionUIContext,
});
const created = data(
await call({
action: "put",
title: "Report",
purpose: "Persist",
source: "# Report",
state: 0,
}),
);
let release = () => {};
const gate = new Promise<void>((resolve) => {
release = resolve;
});
let started = false;
let savePrompt = "";
fixture.setResponses([
async () => {
started = true;
await gate;
return fauxAssistantMessage("Finished original work");
},
(context) => {
const latest = context.messages
.filter((message) => message.role === "user")
.at(-1);
savePrompt = JSON.stringify(latest?.content);
return fauxAssistantMessage("Saved");
},
]);
const running = session.prompt("Long-running work");
try {
await vi.waitFor(() => expect(started).toBe(true));
if (!panel?.handleInput) throw new Error("Panel not mounted");
panel.handleInput("\x13");
await call({
action: "state",
id: created["id"] as string,
expectedRevision: 1,
state: 9,
});
expect(fixture.providerState.callCount).toBe(1);
release();
await running;
await vi.waitFor(() =>
expect(savePrompt).toContain("ephemlet-save-snapshot"),
);
expect(savePrompt).toContain('\\"state\\":0');
expect(savePrompt).not.toContain('\\"state\\":9');
expect(
data(
await call({
action: "get",
id: created["id"] as string,
include: ["state"],
}),
)["state"],
).toBe(9);
} finally {
release();
await running;
}
});
it("stores interactive source/state and preserves revision on invalid source update", async () => {
const { call } = await setup();
const first = data(
await call({
action: "put",
title: "Counter",
purpose: "Explore",
kind: "interactive",
source,
state: 0,
}),
);
const id = first["id"] as string;
await expect(
call({
action: "put",
id,
expectedRevision: 1,
source: "export default function (",
}),
).rejects.toThrow();
expect(
data(await call({ action: "get", id, include: ["source"] })),
).toMatchObject({
source,
revision: 1,
});
await call({ action: "state", id, expectedRevision: 1, state: 8 });
expect(
data(await call({ action: "get", id, include: ["state"] }))["state"],
).toBe(8);
});
it("delegates to real child AgentSession via parent provider and returns usage", async () => {
const { call, fixture } = await setup();
fixture.setResponses([
fauxAssistantMessage(fauxToolCall("submit_ephemlet", { source }), {
stopReason: "toolUse",
}),
]);
const builtResult = await call({
action: "build",
title: "Counter",
purpose: "Interactive explanation",
brief: "Native counter",
state: 0,
wait: true,
});
const built = data(builtResult);
expect(built["status"]).toBe("ready");
expect(built["revision"]).toBe(2);
expect(builtResult.usage).toBeDefined();
expect(fixture.providerState.callCount).toBe(1);
expect(
data(
await call({
action: "get",
id: built["id"] as string,
include: ["source"],
}),
)["source"],
).toBe(source);
});
it("routes virtual builder models through the parent with real provider auth", async () => {
const { call, session, fixture } = await setup();
const physical = session.model;
if (!physical) throw new Error("No fixture model");
session.modelRuntime.registerVirtualModel({
provider: physical.provider,
id: "ephemlet-router",
name: "Ephemlet router",
route: () => ({ model: physical, thinkingLevel: "off" }),
});
const virtual = session.modelRuntime.getModel(
physical.provider,
"ephemlet-router",
);
if (!virtual) throw new Error("Virtual fixture model missing");
await session.setModel(virtual);
fixture.setResponses([
(_context, options) => {
expect(options?.apiKey).toBe("test-key");
return fauxAssistantMessage(
fauxToolCall("submit_ephemlet", { source }),
{ stopReason: "toolUse" },
);
},
]);
expect(
data(
await call({
action: "build",
title: "Virtual",
purpose: "Provider routing",
brief: "Native counter",
wait: true,
}),
)["status"],
).toBe("ready");
});
it("respects explicit tool deactivation across TUI rebind", async () => {
const { session } = await setup();
session.setActiveToolsByName(
session.getActiveToolNames().filter((name) => name !== "ephemlet"),
);
await session.bindExtensions({
mode: "tui",
uiContext: {
...session.extensionRunner.getUIContext(),
custom: async () => undefined,
} as ExtensionUIContext,
});
expect(session.getActiveToolNames()).not.toContain("ephemlet");
});
it("background build returns stable ID before completion and Close prevents resurrection", async () => {
const { call, fixture } = await setup();
let release = () => {};
const gate = new Promise<void>((resolve) => {
release = resolve;
});
let started = false;
let finished = false;
let wasAborted = false;
fixture.setResponses([
async (_context, options) => {
started = true;
await gate;
wasAborted = options?.signal?.aborted ?? false;
finished = true;
return fauxAssistantMessage(
fauxToolCall("submit_ephemlet", { source }),
{ stopReason: "toolUse" },
);
},
]);
const pending = data(
await call({
action: "build",
title: "Background",
purpose: "Monitor",
brief: "Create native monitor",
}),
);
expect(pending["status"]).toBe("building");
expect(typeof pending["jobId"]).toBe("string");
await vi.waitFor(() => expect(started).toBe(true));
await call({ action: "delete", id: pending["id"] as string });
release();
await vi.waitFor(() => expect(finished).toBe(true));
expect(wasAborted).toBe(true);
expect(data(await call({ action: "list" }))["ephemlets"]).toEqual([]);
});
it("failed rebuild retains prior source and exposes failure", async () => {
const { call, fixture } = await setup();
const first = data(
await call({
action: "put",
title: "Counter",
purpose: "Explore",
kind: "interactive",
source,
}),
);
fixture.setResponses([fauxAssistantMessage("No structured result")]);
await expect(
call({
action: "build",
id: first["id"] as string,
expectedRevision: 1,
brief: "Adapt it",
wait: true,
}),
).rejects.toThrow("without submitting");
expect(
data(
await call({
action: "get",
id: first["id"] as string,
include: ["source"],
}),
),
).toMatchObject({ source, status: "failed" });
});
it("marks interrupted restored build failed rather than leaving permanent spinner", async () => {
const { call, session } = await setup();
const first = data(
await call({
action: "put",
title: "Pending",
purpose: "Monitor",
kind: "interactive",
source,
}),
);
const existing = data(
await call({
action: "get",
id: first["id"] as string,
include: ["source", "state"],
}),
);
session.sessionManager.appendCustomEntry("ephemlets", {
op: "put",
ephemlet: { ...existing, status: "building", jobId: "lost" },
});
await session.extensionRunner.emit({
type: "session_start",
reason: "resume",
});
expect(
data(await call({ action: "get", id: first["id"] as string })),
).toMatchObject({ status: "failed", jobId: null });
});
it("enabled diagnostics use one owned writer and omit task content", async () => {
const { call, fixture, session } = await setup(true);
await call({
action: "put",
title: "private-marker-do-not-log",
purpose: "private-marker-do-not-log",
source: "private-marker-do-not-log",
});
await session.extensionRunner.emit({
type: "session_shutdown",
reason: "quit",
});
const directory = join(
fixture.cwd,
".test-state",
"pi-ext",
"debug",
"ephemlets",
);
const files = await readdir(directory);
expect(files).toHaveLength(1);
const text = await readFile(join(directory, files[0]!), "utf8");
const records = text
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(records.map((record) => record.event)).toEqual([
"session.start",
"ephemlet.put",
"session.shutdown",
]);
expect(text).not.toContain("private-marker-do-not-log");
});
it("has schema-bound diagnostics and defaults to no files", async () => {
const { call, fixture } = await setup();
await call({
action: "put",
title: "Report",
purpose: "Report",
source: "done",
});
expect(fixture.errors).toEqual([]);
const { readdir } = await import("node:fs/promises");
await expect(
readdir(`${fixture.cwd}/.test-state/pi-ext/debug/ephemlets`),
).rejects.toMatchObject({ code: "ENOENT" });
});
});