import assert from "node:assert/strict"; import { createHash, randomUUID } from "node:crypto"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { isAbsolute, join } from "node:path"; import { fileURLToPath } from "node:url"; import { type Credential, type CredentialStore, InMemoryModelsStore, } from "@earendil-works/pi-ai"; import { type CreateAgentSessionRuntimeFactory, createAgentSessionFromServices, createAgentSessionRuntime, createAgentSessionServices, createCodemodeExtension, createToolSearchExtension, InteractiveMode, ModelRuntime, readStoredCredential, type SessionEntry, SessionManager, SettingsManager, } from "@earendil-works/pi-coding-agent"; import type { TaskResult } from "../runtime.ts"; const authPath = process.argv[2]; const ref = process.argv[3] ?? "openai-codex/gpt-6-luna"; assert.ok( authPath && isAbsolute(authPath), "Explicit absolute read-only auth.json path required", ); const slash = ref.indexOf("/"); assert.ok(slash > 0, "Use provider/model"); const provider = ref.slice(0, slash), modelId = ref.slice(slash + 1); const original = await readFile(authPath); const fingerprint = createHash("sha256").update(original).digest("hex"); const credential = readStoredCredential(provider, authPath); assert.ok(credential, "Selected provider has no stored credential"); if (credential.type === "oauth") assert.ok( credential.expires > Date.now() + 600_000, "OAuth near expiry; refresh forbidden in live tests", ); const secrets = Object.values(credential).filter( (value): value is string => typeof value === "string" && value.length > 12, ); const safeError = (error: unknown): string => secrets.reduce( (text, secret) => text.replaceAll(secret, "[redacted]"), error instanceof Error ? error.message : String(error), ); let mutations = 0; const check = (): Credential => { if (credential.type === "oauth" && credential.expires <= Date.now() + 300_000) throw new Error("OAuth expired/near expiry; refresh forbidden"); return credential; }; const credentials: CredentialStore = { read: async (id) => (id === provider ? check() : undefined), list: async () => [{ providerId: provider, type: credential.type }], modify: async () => { mutations++; throw new Error("Credential mutation/refresh forbidden"); }, delete: async () => { mutations++; throw new Error("Credential deletion forbidden"); }, }; const root = await mkdtemp(join(tmpdir(), "pipi-live-")), agentDir = join(root, "agent"); for (const [key, value] of Object.entries({ HOME: join(root, "home"), USERPROFILE: join(root, "home"), XDG_CONFIG_HOME: join(root, "config"), XDG_STATE_HOME: join(root, "state"), XDG_CACHE_HOME: join(root, "cache"), XDG_DATA_HOME: join(root, "data"), PI_CODING_AGENT_DIR: agentDir, PI_OFFLINE: "1", PI_PIPI_DEBUG: "0", })) process.env[key] = value; await mkdir(join(agentDir, "pipi", "agents", "reader"), { recursive: true }); await mkdir(join(agentDir, "pipi", "dynamic", "sum"), { recursive: true }); const nonce = randomUUID(); await writeFile( join(root, "evidence.json"), JSON.stringify({ nonce, answer: 42 }), ); await writeFile( join(agentDir, "pipi", "agents", "reader", "config.json"), JSON.stringify({ description: "Read workspace evidence files; return requested facts without editing.", tools: ["read"], }), ); await writeFile( join(agentDir, "pipi", "agents", "reader", "instructions.md"), "Read evidence from actual files when requested. Do not change files. Use supplied output contract when present.\n", ); const sumPath = join(agentDir, "pipi", "dynamic", "sum", "index.ts"); await writeFile( sumPath, "export default pi=>{pi.registerTool({name:'sum_numbers',label:'Sum',description:'Add two integers.',parameters:{type:'object',properties:{a:{type:'integer'},b:{type:'integer'}},required:['a','b']},async execute(id,args){return {content:[{type:'text',text:String(args.a+args.b)}],details:{}};}});};\n", ); const settings = { defaultTools: ["pipi", "codemode", "read", "write"], retry: { enabled: false, provider: { timeoutMs: 45_000, maxRetries: 0 } }, compaction: { enabled: false }, cacheWarming: "off" as const, }; await writeFile(join(agentDir, "settings.json"), JSON.stringify(settings)); const runtime = await ModelRuntime.create({ credentials, modelsPath: null, modelsStore: new InMemoryModelsStore(), allowModelNetwork: false, }); const model = runtime.getModel(provider, modelId); assert.ok(model, "Model not in installed chat catalog"); const built = fileURLToPath( new URL("../../../dist/pipi/index.ts", import.meta.url), ); const factory: CreateAgentSessionRuntimeFactory = async ({ cwd, sessionManager, sessionStartEvent, }) => { const services = await createAgentSessionServices({ cwd, agentDir, modelRuntime: runtime, settingsManager: SettingsManager.inMemory(settings), resourceLoaderOptions: { additionalExtensionPaths: [built], extensionFactories: [ { name: "codemode", builtin: true, factory: createCodemodeExtension() }, { name: "tool-search", builtin: true, factory: createToolSearchExtension(), }, ], noThemes: true, }, }); return { ...(await createAgentSessionFromServices({ services, sessionManager, ...(sessionStartEvent ? { sessionStartEvent } : {}), model, thinkingLevel: "low", })), services, diagnostics: services.diagnostics, }; }; const host = await createAgentSessionRuntime(factory, { cwd: root, agentDir, sessionManager: SessionManager.create(root, join(root, "sessions")), }); function last(values: TaskResult[]): TaskResult { const value = values.at(-1); assert.ok(value); return value; } const interactive = process.argv.includes("--interactive"), keep = interactive || process.argv.includes("--keep"); const reports: Array<{ name: string; ok: boolean; calls: number; cost: number; detail?: string; evidence?: Record; }> = []; interface ObservedTool { name: string; id: string; parent?: string; } async function transcript(file: string | null): Promise { assert.ok(file); return (await readFile(file, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line) as SessionEntry); } async function actualTool(result: TaskResult, name: string) { const entries = await transcript(result.sessionFile); const calls = entries.flatMap((entry) => entry.type === "message" && entry.message.role === "assistant" ? entry.message.content.filter( (part) => part.type === "toolCall" && part.name === name, ) : [], ); const call = calls.find((part) => part.type === "toolCall"); assert.ok(call && call.type === "toolCall", `Child did not call ${name}`); const response = entries.find( (entry) => entry.type === "message" && entry.message.role === "toolResult" && entry.message.toolCallId === call.id, ); assert.ok( response?.type === "message" && response.message.role === "toolResult" && !response.message.isError, `Missing successful ${name} result`, ); return { call, response: response.message }; } let failed = false; try { if (interactive) { process.stderr.write( `Live isolated Pi: ${root}; model ${ref}; credentials read-only.\n`, ); await new InteractiveMode(host, { tuiMode: "fullscreen", initialThemeSetting: "dark", }).run(); } else { await host.session.bindExtensions({ mode: "print" }); const values: TaskResult[] = []; const tools: ObservedTool[] = []; host.session.subscribe((event) => { if (event.type === "tool_execution_start") { const parent = "parentToolCallId" in event && typeof event.parentToolCallId === "string" ? event.parentToolCallId : undefined; tools.push({ name: event.toolName, id: event.toolCallId, ...(parent ? { parent } : {}), }); } if (event.type === "tool_execution_end" && event.toolName === "pipi") { const value = (event.result as { details?: { result?: TaskResult } }) .details?.result; if (value) values.push(value); } }); const execute = async ( name: string, prompt: string, verify: ( calls: TaskResult[], tools: ObservedTool[], ) => | Record | Promise>, ) => { const before = values.length, toolStart = tools.length, billing = host.session.getSessionStats().cost; let expired = false; const timer = setTimeout(() => { expired = true; void host.session.abort(); }, 120_000); try { await host.session.prompt(prompt); assert.equal(expired, false, "Live case deadline"); const calls = values.slice(before); assert.ok( calls.length > 0, "Real orchestrator must actually call pipi", ); const evidence = await verify(calls, tools.slice(toolStart)); reports.push({ name, ok: true, calls: calls.length, cost: host.session.getSessionStats().cost - billing, evidence, }); process.stdout.write(`PASS live ${name}\n`); } catch (error) { failed = true; const detail = safeError(error); reports.push({ name, ok: false, calls: values.length - before, cost: host.session.getSessionStats().cost - billing, detail, }); process.stderr.write(`FAIL live ${name}: ${detail}\n`); } finally { clearTimeout(timer); } }; const schema = { type: "object", properties: { role: { const: "assistant" }, nonce: { type: "string" }, answer: { type: "integer" }, }, required: ["role", "nonce", "answer"], additionalProperties: false, }; await execute( "preset reads real evidence and submits schema", `Use pipi with agent reader to read evidence.json. Require outputSchema ${JSON.stringify(schema)}. Child must return role assistant and actual nonce and answer from file. Do not read the file yourself. After success summarize briefly.`, async (calls) => { const result = last(calls); assert.equal(result.outcome, "completed"); assert.deepEqual(result.output, { role: "assistant", nonce, answer: 42, }); const observed = await actualTool(result, "read"); assert.ok(JSON.stringify(observed.response.content).includes(nonce)); return { tool: "read", result: "actual nonce observed" }; }, ); const first = values.find( (value) => value.outcome === "completed" && value.outputMode === "structured", ); if (first) { await execute( "continuation recalls prior evidence without tools", `Continue pipi sessionId ${first.sessionId}. Give tools [] (no tools). Ask child to recall nonce from previous task without reading files. Require outputSchema {"type":"object","properties":{"nonce":{"type":"string"}},"required":["nonce"],"additionalProperties":false}. Use returned data, do not supply nonce in prompt.`, (calls) => { const result = last(calls); assert.equal(result?.outcome, "completed"); assert.equal(result.sessionId, first.sessionId); assert.deepEqual(result.output, { nonce }); return { sessionId: result.sessionId }; }, ); await execute( "native fork preserves context with independent session", `Use pipi with sessionId ${first.sessionId}, fork true, tools []. Ask child to recall prior nonce. Require the same nonce-only outputSchema as last task. Do not supply nonce yourself.`, (calls) => { const result = last(calls); assert.equal(result?.outcome, "completed"); assert.notEqual(result.sessionId, first.sessionId); assert.deepEqual(result.output, { nonce }); return { sessionId: result.sessionId }; }, ); } await execute( "scripted parallel structured children", `Use the scripting tool to run two pipi calls concurrently with Promise.all. Each call: tools [], outputSchema {"type":"integer"}, prompt asking respectively for 17 and 25. Print their numeric sum. Do not replace delegation with local arithmetic.`, (calls, observed) => { const scripts = new Set( observed .filter((tool) => tool.name === "codemode") .map((tool) => tool.id), ); assert.equal( observed.filter( (tool) => tool.name === "pipi" && tool.parent && scripts.has(tool.parent), ).length, 2, "Both child calls must originate inside scripting invocation", ); assert.equal(calls.length, 2); assert.ok(calls.every((value) => value.outcome === "completed")); assert.deepEqual( calls .map((value) => value.output) .sort((a, b) => Number(a) - Number(b)), [17, 25], ); const windows = calls.map((value) => { const message = value.assistantMessage; assert.ok( message && typeof message.durationMs === "number" && message.durationMs > 0, "Provider request timing required", ); return { start: message.timestamp, end: message.timestamp + message.durationMs, }; }); const overlapMs = Math.min(...windows.map((value) => value.end)) - Math.max(...windows.map((value) => value.start)); assert.ok(overlapMs > 0, "Actual child provider requests must overlap"); return { scriptingInvocations: scripts.size, overlapMs }; }, ); await execute( "dynamic extension tool used by real child", `Use pipi with extensions [${JSON.stringify(sumPath)}], tools ["sum_numbers"], outputSchema {"type":"integer"}. Ask child to call sum_numbers with a=19 and b=23 and submit result. Do not calculate it yourself.`, async (calls) => { const result = last(calls); assert.equal(result.outcome, "completed"); assert.equal(result.output, 42); const observed = await actualTool(result, "sum_numbers"); assert.deepEqual(observed.call.arguments, { a: 19, b: 23 }); assert.deepEqual(observed.response.content, [ { type: "text", text: "42" }, ]); return { tool: "sum_numbers", a: 19, b: 23, result: 42 }; }, ); await host.session.extensionRunner.emit({ type: "session_shutdown", reason: "quit", }); } assert.equal( mutations, 0, "Read-only credential store must not mutate or refresh", ); assert.equal( createHash("sha256") .update(await readFile(authPath)) .digest("hex"), fingerprint, "Original credential file changed during test", ); const local = await readFile(join(agentDir, "auth.json"), "utf8").catch( (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return "{}"; throw error; }, ); assert.deepEqual( JSON.parse(local), {}, "No credential may be copied to test agent directory", ); } catch (error) { failed = true; process.stderr.write(safeError(error) + "\n"); } finally { const totalCost = host.session.getSessionStats().cost; await host.dispose(); const report = { model: ref, passed: reports.filter((value) => value.ok).length, failed: reports.filter((value) => !value.ok).length, credentialMutations: mutations, cost: totalCost, cases: reports, artifacts: keep || failed ? root : null, }; if (keep || failed) await writeFile(join(root, "report.json"), JSON.stringify(report, null, 2)); else await rm(root, { recursive: true, force: true }); process.stdout.write(JSON.stringify(report, null, 2) + "\n"); if (failed) process.exitCode = 1; }