repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/pipi/__acceptance__/live.ts
Rawimport assert from "node:assert/strict";
import { createHash, randomUUID } from "node:crypto";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
type Credential,
type CredentialStore,
InMemoryModelsStore,
} from "@earendil-works/pi-ai";
import {
type CreateAgentSessionRuntimeFactory,
createAgentSessionFromServices,
createAgentSessionRuntime,
createAgentSessionServices,
createCodemodeExtension,
createToolSearchExtension,
InteractiveMode,
ModelRuntime,
readStoredCredential,
type SessionEntry,
SessionManager,
SettingsManager,
} from "@earendil-works/pi-coding-agent";
import type { TaskResult } from "../runtime.ts";
const authPath = process.argv[2];
const ref = process.argv[3] ?? "openai-codex/gpt-6-luna";
assert.ok(
authPath && isAbsolute(authPath),
"Explicit absolute read-only auth.json path required",
);
const slash = ref.indexOf("/");
assert.ok(slash > 0, "Use provider/model");
const provider = ref.slice(0, slash),
modelId = ref.slice(slash + 1);
const original = await readFile(authPath);
const fingerprint = createHash("sha256").update(original).digest("hex");
const credential = readStoredCredential(provider, authPath);
assert.ok(credential, "Selected provider has no stored credential");
if (credential.type === "oauth")
assert.ok(
credential.expires > Date.now() + 600_000,
"OAuth near expiry; refresh forbidden in live tests",
);
const secrets = Object.values(credential).filter(
(value): value is string => typeof value === "string" && value.length > 12,
);
const safeError = (error: unknown): string =>
secrets.reduce(
(text, secret) => text.replaceAll(secret, "[redacted]"),
error instanceof Error ? error.message : String(error),
);
let mutations = 0;
const check = (): Credential => {
if (credential.type === "oauth" && credential.expires <= Date.now() + 300_000)
throw new Error("OAuth expired/near expiry; refresh forbidden");
return credential;
};
const credentials: CredentialStore = {
read: async (id) => (id === provider ? check() : undefined),
list: async () => [{ providerId: provider, type: credential.type }],
modify: async () => {
mutations++;
throw new Error("Credential mutation/refresh forbidden");
},
delete: async () => {
mutations++;
throw new Error("Credential deletion forbidden");
},
};
const root = await mkdtemp(join(tmpdir(), "pipi-live-")),
agentDir = join(root, "agent");
for (const [key, value] of Object.entries({
HOME: join(root, "home"),
USERPROFILE: join(root, "home"),
XDG_CONFIG_HOME: join(root, "config"),
XDG_STATE_HOME: join(root, "state"),
XDG_CACHE_HOME: join(root, "cache"),
XDG_DATA_HOME: join(root, "data"),
PI_CODING_AGENT_DIR: agentDir,
PI_OFFLINE: "1",
PI_PIPI_DEBUG: "0",
}))
process.env[key] = value;
await mkdir(join(agentDir, "pipi", "agents", "reader"), { recursive: true });
await mkdir(join(agentDir, "pipi", "dynamic", "sum"), { recursive: true });
const nonce = randomUUID();
await writeFile(
join(root, "evidence.json"),
JSON.stringify({ nonce, answer: 42 }),
);
await writeFile(
join(agentDir, "pipi", "agents", "reader", "config.json"),
JSON.stringify({
description:
"Read workspace evidence files; return requested facts without editing.",
tools: ["read"],
}),
);
await writeFile(
join(agentDir, "pipi", "agents", "reader", "instructions.md"),
"Read evidence from actual files when requested. Do not change files. Use supplied output contract when present.\n",
);
const sumPath = join(agentDir, "pipi", "dynamic", "sum", "index.ts");
await writeFile(
sumPath,
"export default pi=>{pi.registerTool({name:'sum_numbers',label:'Sum',description:'Add two integers.',parameters:{type:'object',properties:{a:{type:'integer'},b:{type:'integer'}},required:['a','b']},async execute(id,args){return {content:[{type:'text',text:String(args.a+args.b)}],details:{}};}});};\n",
);
const settings = {
defaultTools: ["pipi", "codemode", "read", "write"],
retry: { enabled: false, provider: { timeoutMs: 45_000, maxRetries: 0 } },
compaction: { enabled: false },
cacheWarming: "off" as const,
};
await writeFile(join(agentDir, "settings.json"), JSON.stringify(settings));
const runtime = await ModelRuntime.create({
credentials,
modelsPath: null,
modelsStore: new InMemoryModelsStore(),
allowModelNetwork: false,
});
const model = runtime.getModel(provider, modelId);
assert.ok(model, "Model not in installed chat catalog");
const built = fileURLToPath(
new URL("../../../dist/pipi/index.ts", import.meta.url),
);
const factory: CreateAgentSessionRuntimeFactory = async ({
cwd,
sessionManager,
sessionStartEvent,
}) => {
const services = await createAgentSessionServices({
cwd,
agentDir,
modelRuntime: runtime,
settingsManager: SettingsManager.inMemory(settings),
resourceLoaderOptions: {
additionalExtensionPaths: [built],
extensionFactories: [
{ name: "codemode", builtin: true, factory: createCodemodeExtension() },
{
name: "tool-search",
builtin: true,
factory: createToolSearchExtension(),
},
],
noThemes: true,
},
});
return {
...(await createAgentSessionFromServices({
services,
sessionManager,
...(sessionStartEvent ? { sessionStartEvent } : {}),
model,
thinkingLevel: "low",
})),
services,
diagnostics: services.diagnostics,
};
};
const host = await createAgentSessionRuntime(factory, {
cwd: root,
agentDir,
sessionManager: SessionManager.create(root, join(root, "sessions")),
});
function last(values: TaskResult[]): TaskResult {
const value = values.at(-1);
assert.ok(value);
return value;
}
const interactive = process.argv.includes("--interactive"),
keep = interactive || process.argv.includes("--keep");
const reports: Array<{
name: string;
ok: boolean;
calls: number;
cost: number;
detail?: string;
evidence?: Record<string, string | number>;
}> = [];
interface ObservedTool {
name: string;
id: string;
parent?: string;
}
async function transcript(file: string | null): Promise<SessionEntry[]> {
assert.ok(file);
return (await readFile(file, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as SessionEntry);
}
async function actualTool(result: TaskResult, name: string) {
const entries = await transcript(result.sessionFile);
const calls = entries.flatMap((entry) =>
entry.type === "message" && entry.message.role === "assistant"
? entry.message.content.filter(
(part) => part.type === "toolCall" && part.name === name,
)
: [],
);
const call = calls.find((part) => part.type === "toolCall");
assert.ok(call && call.type === "toolCall", `Child did not call ${name}`);
const response = entries.find(
(entry) =>
entry.type === "message" &&
entry.message.role === "toolResult" &&
entry.message.toolCallId === call.id,
);
assert.ok(
response?.type === "message" &&
response.message.role === "toolResult" &&
!response.message.isError,
`Missing successful ${name} result`,
);
return { call, response: response.message };
}
let failed = false;
try {
if (interactive) {
process.stderr.write(
`Live isolated Pi: ${root}; model ${ref}; credentials read-only.\n`,
);
await new InteractiveMode(host, {
tuiMode: "fullscreen",
initialThemeSetting: "dark",
}).run();
} else {
await host.session.bindExtensions({ mode: "print" });
const values: TaskResult[] = [];
const tools: ObservedTool[] = [];
host.session.subscribe((event) => {
if (event.type === "tool_execution_start") {
const parent =
"parentToolCallId" in event &&
typeof event.parentToolCallId === "string"
? event.parentToolCallId
: undefined;
tools.push({
name: event.toolName,
id: event.toolCallId,
...(parent ? { parent } : {}),
});
}
if (event.type === "tool_execution_end" && event.toolName === "pipi") {
const value = (event.result as { details?: { result?: TaskResult } })
.details?.result;
if (value) values.push(value);
}
});
const execute = async (
name: string,
prompt: string,
verify: (
calls: TaskResult[],
tools: ObservedTool[],
) =>
| Record<string, string | number>
| Promise<Record<string, string | number>>,
) => {
const before = values.length,
toolStart = tools.length,
billing = host.session.getSessionStats().cost;
let expired = false;
const timer = setTimeout(() => {
expired = true;
void host.session.abort();
}, 120_000);
try {
await host.session.prompt(prompt);
assert.equal(expired, false, "Live case deadline");
const calls = values.slice(before);
assert.ok(
calls.length > 0,
"Real orchestrator must actually call pipi",
);
const evidence = await verify(calls, tools.slice(toolStart));
reports.push({
name,
ok: true,
calls: calls.length,
cost: host.session.getSessionStats().cost - billing,
evidence,
});
process.stdout.write(`PASS live ${name}\n`);
} catch (error) {
failed = true;
const detail = safeError(error);
reports.push({
name,
ok: false,
calls: values.length - before,
cost: host.session.getSessionStats().cost - billing,
detail,
});
process.stderr.write(`FAIL live ${name}: ${detail}\n`);
} finally {
clearTimeout(timer);
}
};
const schema = {
type: "object",
properties: {
role: { const: "assistant" },
nonce: { type: "string" },
answer: { type: "integer" },
},
required: ["role", "nonce", "answer"],
additionalProperties: false,
};
await execute(
"preset reads real evidence and submits schema",
`Use pipi with agent reader to read evidence.json. Require outputSchema ${JSON.stringify(schema)}. Child must return role assistant and actual nonce and answer from file. Do not read the file yourself. After success summarize briefly.`,
async (calls) => {
const result = last(calls);
assert.equal(result.outcome, "completed");
assert.deepEqual(result.output, {
role: "assistant",
nonce,
answer: 42,
});
const observed = await actualTool(result, "read");
assert.ok(JSON.stringify(observed.response.content).includes(nonce));
return { tool: "read", result: "actual nonce observed" };
},
);
const first = values.find(
(value) =>
value.outcome === "completed" && value.outputMode === "structured",
);
if (first) {
await execute(
"continuation recalls prior evidence without tools",
`Continue pipi sessionId ${first.sessionId}. Give tools [] (no tools). Ask child to recall nonce from previous task without reading files. Require outputSchema {"type":"object","properties":{"nonce":{"type":"string"}},"required":["nonce"],"additionalProperties":false}. Use returned data, do not supply nonce in prompt.`,
(calls) => {
const result = last(calls);
assert.equal(result?.outcome, "completed");
assert.equal(result.sessionId, first.sessionId);
assert.deepEqual(result.output, { nonce });
return { sessionId: result.sessionId };
},
);
await execute(
"native fork preserves context with independent session",
`Use pipi with sessionId ${first.sessionId}, fork true, tools []. Ask child to recall prior nonce. Require the same nonce-only outputSchema as last task. Do not supply nonce yourself.`,
(calls) => {
const result = last(calls);
assert.equal(result?.outcome, "completed");
assert.notEqual(result.sessionId, first.sessionId);
assert.deepEqual(result.output, { nonce });
return { sessionId: result.sessionId };
},
);
}
await execute(
"scripted parallel structured children",
`Use the scripting tool to run two pipi calls concurrently with Promise.all. Each call: tools [], outputSchema {"type":"integer"}, prompt asking respectively for 17 and 25. Print their numeric sum. Do not replace delegation with local arithmetic.`,
(calls, observed) => {
const scripts = new Set(
observed
.filter((tool) => tool.name === "codemode")
.map((tool) => tool.id),
);
assert.equal(
observed.filter(
(tool) =>
tool.name === "pipi" && tool.parent && scripts.has(tool.parent),
).length,
2,
"Both child calls must originate inside scripting invocation",
);
assert.equal(calls.length, 2);
assert.ok(calls.every((value) => value.outcome === "completed"));
assert.deepEqual(
calls
.map((value) => value.output)
.sort((a, b) => Number(a) - Number(b)),
[17, 25],
);
const windows = calls.map((value) => {
const message = value.assistantMessage;
assert.ok(
message &&
typeof message.durationMs === "number" &&
message.durationMs > 0,
"Provider request timing required",
);
return {
start: message.timestamp,
end: message.timestamp + message.durationMs,
};
});
const overlapMs =
Math.min(...windows.map((value) => value.end)) -
Math.max(...windows.map((value) => value.start));
assert.ok(overlapMs > 0, "Actual child provider requests must overlap");
return { scriptingInvocations: scripts.size, overlapMs };
},
);
await execute(
"dynamic extension tool used by real child",
`Use pipi with extensions [${JSON.stringify(sumPath)}], tools ["sum_numbers"], outputSchema {"type":"integer"}. Ask child to call sum_numbers with a=19 and b=23 and submit result. Do not calculate it yourself.`,
async (calls) => {
const result = last(calls);
assert.equal(result.outcome, "completed");
assert.equal(result.output, 42);
const observed = await actualTool(result, "sum_numbers");
assert.deepEqual(observed.call.arguments, { a: 19, b: 23 });
assert.deepEqual(observed.response.content, [
{ type: "text", text: "42" },
]);
return { tool: "sum_numbers", a: 19, b: 23, result: 42 };
},
);
await host.session.extensionRunner.emit({
type: "session_shutdown",
reason: "quit",
});
}
assert.equal(
mutations,
0,
"Read-only credential store must not mutate or refresh",
);
assert.equal(
createHash("sha256")
.update(await readFile(authPath))
.digest("hex"),
fingerprint,
"Original credential file changed during test",
);
const local = await readFile(join(agentDir, "auth.json"), "utf8").catch(
(error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT") return "{}";
throw error;
},
);
assert.deepEqual(
JSON.parse(local),
{},
"No credential may be copied to test agent directory",
);
} catch (error) {
failed = true;
process.stderr.write(safeError(error) + "\n");
} finally {
const totalCost = host.session.getSessionStats().cost;
await host.dispose();
const report = {
model: ref,
passed: reports.filter((value) => value.ok).length,
failed: reports.filter((value) => !value.ok).length,
credentialMutations: mutations,
cost: totalCost,
cases: reports,
artifacts: keep || failed ? root : null,
};
if (keep || failed)
await writeFile(join(root, "report.json"), JSON.stringify(report, null, 2));
else await rm(root, { recursive: true, force: true });
process.stdout.write(JSON.stringify(report, null, 2) + "\n");
if (failed) process.exitCode = 1;
}