Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/pipi/__acceptance__/live.ts

Raw
import assert from "node:assert/strict";
import { createHash, randomUUID } from "node:crypto";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
	type Credential,
	type CredentialStore,
	InMemoryModelsStore,
} from "@earendil-works/pi-ai";
import {
	type CreateAgentSessionRuntimeFactory,
	createAgentSessionFromServices,
	createAgentSessionRuntime,
	createAgentSessionServices,
	createCodemodeExtension,
	createToolSearchExtension,
	InteractiveMode,
	ModelRuntime,
	readStoredCredential,
	type SessionEntry,
	SessionManager,
	SettingsManager,
} from "@earendil-works/pi-coding-agent";
import type { TaskResult } from "../runtime.ts";

const authPath = process.argv[2];
const ref = process.argv[3] ?? "openai-codex/gpt-6-luna";
assert.ok(
	authPath && isAbsolute(authPath),
	"Explicit absolute read-only auth.json path required",
);
const slash = ref.indexOf("/");
assert.ok(slash > 0, "Use provider/model");
const provider = ref.slice(0, slash),
	modelId = ref.slice(slash + 1);
const original = await readFile(authPath);
const fingerprint = createHash("sha256").update(original).digest("hex");
const credential = readStoredCredential(provider, authPath);
assert.ok(credential, "Selected provider has no stored credential");
if (credential.type === "oauth")
	assert.ok(
		credential.expires > Date.now() + 600_000,
		"OAuth near expiry; refresh forbidden in live tests",
	);
const secrets = Object.values(credential).filter(
	(value): value is string => typeof value === "string" && value.length > 12,
);
const safeError = (error: unknown): string =>
	secrets.reduce(
		(text, secret) => text.replaceAll(secret, "[redacted]"),
		error instanceof Error ? error.message : String(error),
	);
let mutations = 0;
const check = (): Credential => {
	if (credential.type === "oauth" && credential.expires <= Date.now() + 300_000)
		throw new Error("OAuth expired/near expiry; refresh forbidden");
	return credential;
};
const credentials: CredentialStore = {
	read: async (id) => (id === provider ? check() : undefined),
	list: async () => [{ providerId: provider, type: credential.type }],
	modify: async () => {
		mutations++;
		throw new Error("Credential mutation/refresh forbidden");
	},
	delete: async () => {
		mutations++;
		throw new Error("Credential deletion forbidden");
	},
};
const root = await mkdtemp(join(tmpdir(), "pipi-live-")),
	agentDir = join(root, "agent");
for (const [key, value] of Object.entries({
	HOME: join(root, "home"),
	USERPROFILE: join(root, "home"),
	XDG_CONFIG_HOME: join(root, "config"),
	XDG_STATE_HOME: join(root, "state"),
	XDG_CACHE_HOME: join(root, "cache"),
	XDG_DATA_HOME: join(root, "data"),
	PI_CODING_AGENT_DIR: agentDir,
	PI_OFFLINE: "1",
	PI_PIPI_DEBUG: "0",
}))
	process.env[key] = value;
await mkdir(join(agentDir, "pipi", "agents", "reader"), { recursive: true });
await mkdir(join(agentDir, "pipi", "dynamic", "sum"), { recursive: true });
const nonce = randomUUID();
await writeFile(
	join(root, "evidence.json"),
	JSON.stringify({ nonce, answer: 42 }),
);
await writeFile(
	join(agentDir, "pipi", "agents", "reader", "config.json"),
	JSON.stringify({
		description:
			"Read workspace evidence files; return requested facts without editing.",
		tools: ["read"],
	}),
);
await writeFile(
	join(agentDir, "pipi", "agents", "reader", "instructions.md"),
	"Read evidence from actual files when requested. Do not change files. Use supplied output contract when present.\n",
);
const sumPath = join(agentDir, "pipi", "dynamic", "sum", "index.ts");
await writeFile(
	sumPath,
	"export default pi=>{pi.registerTool({name:'sum_numbers',label:'Sum',description:'Add two integers.',parameters:{type:'object',properties:{a:{type:'integer'},b:{type:'integer'}},required:['a','b']},async execute(id,args){return {content:[{type:'text',text:String(args.a+args.b)}],details:{}};}});};\n",
);
const settings = {
	defaultTools: ["pipi", "codemode", "read", "write"],
	retry: { enabled: false, provider: { timeoutMs: 45_000, maxRetries: 0 } },
	compaction: { enabled: false },
	cacheWarming: "off" as const,
};
await writeFile(join(agentDir, "settings.json"), JSON.stringify(settings));
const runtime = await ModelRuntime.create({
	credentials,
	modelsPath: null,
	modelsStore: new InMemoryModelsStore(),
	allowModelNetwork: false,
});
const model = runtime.getModel(provider, modelId);
assert.ok(model, "Model not in installed chat catalog");
const built = fileURLToPath(
	new URL("../../../dist/pipi/index.ts", import.meta.url),
);
const factory: CreateAgentSessionRuntimeFactory = async ({
	cwd,
	sessionManager,
	sessionStartEvent,
}) => {
	const services = await createAgentSessionServices({
		cwd,
		agentDir,
		modelRuntime: runtime,
		settingsManager: SettingsManager.inMemory(settings),
		resourceLoaderOptions: {
			additionalExtensionPaths: [built],
			extensionFactories: [
				{ name: "codemode", builtin: true, factory: createCodemodeExtension() },
				{
					name: "tool-search",
					builtin: true,
					factory: createToolSearchExtension(),
				},
			],
			noThemes: true,
		},
	});
	return {
		...(await createAgentSessionFromServices({
			services,
			sessionManager,
			...(sessionStartEvent ? { sessionStartEvent } : {}),
			model,
			thinkingLevel: "low",
		})),
		services,
		diagnostics: services.diagnostics,
	};
};
const host = await createAgentSessionRuntime(factory, {
	cwd: root,
	agentDir,
	sessionManager: SessionManager.create(root, join(root, "sessions")),
});
function last(values: TaskResult[]): TaskResult {
	const value = values.at(-1);
	assert.ok(value);
	return value;
}
const interactive = process.argv.includes("--interactive"),
	keep = interactive || process.argv.includes("--keep");
const reports: Array<{
	name: string;
	ok: boolean;
	calls: number;
	cost: number;
	detail?: string;
	evidence?: Record<string, string | number>;
}> = [];
interface ObservedTool {
	name: string;
	id: string;
	parent?: string;
}
async function transcript(file: string | null): Promise<SessionEntry[]> {
	assert.ok(file);
	return (await readFile(file, "utf8"))
		.trim()
		.split("\n")
		.map((line) => JSON.parse(line) as SessionEntry);
}
async function actualTool(result: TaskResult, name: string) {
	const entries = await transcript(result.sessionFile);
	const calls = entries.flatMap((entry) =>
		entry.type === "message" && entry.message.role === "assistant"
			? entry.message.content.filter(
					(part) => part.type === "toolCall" && part.name === name,
				)
			: [],
	);
	const call = calls.find((part) => part.type === "toolCall");
	assert.ok(call && call.type === "toolCall", `Child did not call ${name}`);
	const response = entries.find(
		(entry) =>
			entry.type === "message" &&
			entry.message.role === "toolResult" &&
			entry.message.toolCallId === call.id,
	);
	assert.ok(
		response?.type === "message" &&
			response.message.role === "toolResult" &&
			!response.message.isError,
		`Missing successful ${name} result`,
	);
	return { call, response: response.message };
}
let failed = false;
try {
	if (interactive) {
		process.stderr.write(
			`Live isolated Pi: ${root}; model ${ref}; credentials read-only.\n`,
		);
		await new InteractiveMode(host, {
			tuiMode: "fullscreen",
			initialThemeSetting: "dark",
		}).run();
	} else {
		await host.session.bindExtensions({ mode: "print" });
		const values: TaskResult[] = [];
		const tools: ObservedTool[] = [];
		host.session.subscribe((event) => {
			if (event.type === "tool_execution_start") {
				const parent =
					"parentToolCallId" in event &&
					typeof event.parentToolCallId === "string"
						? event.parentToolCallId
						: undefined;
				tools.push({
					name: event.toolName,
					id: event.toolCallId,
					...(parent ? { parent } : {}),
				});
			}
			if (event.type === "tool_execution_end" && event.toolName === "pipi") {
				const value = (event.result as { details?: { result?: TaskResult } })
					.details?.result;
				if (value) values.push(value);
			}
		});
		const execute = async (
			name: string,
			prompt: string,
			verify: (
				calls: TaskResult[],
				tools: ObservedTool[],
			) =>
				| Record<string, string | number>
				| Promise<Record<string, string | number>>,
		) => {
			const before = values.length,
				toolStart = tools.length,
				billing = host.session.getSessionStats().cost;
			let expired = false;
			const timer = setTimeout(() => {
				expired = true;
				void host.session.abort();
			}, 120_000);
			try {
				await host.session.prompt(prompt);
				assert.equal(expired, false, "Live case deadline");
				const calls = values.slice(before);
				assert.ok(
					calls.length > 0,
					"Real orchestrator must actually call pipi",
				);
				const evidence = await verify(calls, tools.slice(toolStart));
				reports.push({
					name,
					ok: true,
					calls: calls.length,
					cost: host.session.getSessionStats().cost - billing,
					evidence,
				});
				process.stdout.write(`PASS live ${name}\n`);
			} catch (error) {
				failed = true;
				const detail = safeError(error);
				reports.push({
					name,
					ok: false,
					calls: values.length - before,
					cost: host.session.getSessionStats().cost - billing,
					detail,
				});
				process.stderr.write(`FAIL live ${name}: ${detail}\n`);
			} finally {
				clearTimeout(timer);
			}
		};
		const schema = {
			type: "object",
			properties: {
				role: { const: "assistant" },
				nonce: { type: "string" },
				answer: { type: "integer" },
			},
			required: ["role", "nonce", "answer"],
			additionalProperties: false,
		};
		await execute(
			"preset reads real evidence and submits schema",
			`Use pipi with agent reader to read evidence.json. Require outputSchema ${JSON.stringify(schema)}. Child must return role assistant and actual nonce and answer from file. Do not read the file yourself. After success summarize briefly.`,
			async (calls) => {
				const result = last(calls);
				assert.equal(result.outcome, "completed");
				assert.deepEqual(result.output, {
					role: "assistant",
					nonce,
					answer: 42,
				});
				const observed = await actualTool(result, "read");
				assert.ok(JSON.stringify(observed.response.content).includes(nonce));
				return { tool: "read", result: "actual nonce observed" };
			},
		);
		const first = values.find(
			(value) =>
				value.outcome === "completed" && value.outputMode === "structured",
		);
		if (first) {
			await execute(
				"continuation recalls prior evidence without tools",
				`Continue pipi sessionId ${first.sessionId}. Give tools [] (no tools). Ask child to recall nonce from previous task without reading files. Require outputSchema {"type":"object","properties":{"nonce":{"type":"string"}},"required":["nonce"],"additionalProperties":false}. Use returned data, do not supply nonce in prompt.`,
				(calls) => {
					const result = last(calls);
					assert.equal(result?.outcome, "completed");
					assert.equal(result.sessionId, first.sessionId);
					assert.deepEqual(result.output, { nonce });
					return { sessionId: result.sessionId };
				},
			);
			await execute(
				"native fork preserves context with independent session",
				`Use pipi with sessionId ${first.sessionId}, fork true, tools []. Ask child to recall prior nonce. Require the same nonce-only outputSchema as last task. Do not supply nonce yourself.`,
				(calls) => {
					const result = last(calls);
					assert.equal(result?.outcome, "completed");
					assert.notEqual(result.sessionId, first.sessionId);
					assert.deepEqual(result.output, { nonce });
					return { sessionId: result.sessionId };
				},
			);
		}
		await execute(
			"scripted parallel structured children",
			`Use the scripting tool to run two pipi calls concurrently with Promise.all. Each call: tools [], outputSchema {"type":"integer"}, prompt asking respectively for 17 and 25. Print their numeric sum. Do not replace delegation with local arithmetic.`,
			(calls, observed) => {
				const scripts = new Set(
					observed
						.filter((tool) => tool.name === "codemode")
						.map((tool) => tool.id),
				);
				assert.equal(
					observed.filter(
						(tool) =>
							tool.name === "pipi" && tool.parent && scripts.has(tool.parent),
					).length,
					2,
					"Both child calls must originate inside scripting invocation",
				);
				assert.equal(calls.length, 2);
				assert.ok(calls.every((value) => value.outcome === "completed"));
				assert.deepEqual(
					calls
						.map((value) => value.output)
						.sort((a, b) => Number(a) - Number(b)),
					[17, 25],
				);
				const windows = calls.map((value) => {
					const message = value.assistantMessage;
					assert.ok(
						message &&
							typeof message.durationMs === "number" &&
							message.durationMs > 0,
						"Provider request timing required",
					);
					return {
						start: message.timestamp,
						end: message.timestamp + message.durationMs,
					};
				});
				const overlapMs =
					Math.min(...windows.map((value) => value.end)) -
					Math.max(...windows.map((value) => value.start));
				assert.ok(overlapMs > 0, "Actual child provider requests must overlap");
				return { scriptingInvocations: scripts.size, overlapMs };
			},
		);
		await execute(
			"dynamic extension tool used by real child",
			`Use pipi with extensions [${JSON.stringify(sumPath)}], tools ["sum_numbers"], outputSchema {"type":"integer"}. Ask child to call sum_numbers with a=19 and b=23 and submit result. Do not calculate it yourself.`,
			async (calls) => {
				const result = last(calls);
				assert.equal(result.outcome, "completed");
				assert.equal(result.output, 42);
				const observed = await actualTool(result, "sum_numbers");
				assert.deepEqual(observed.call.arguments, { a: 19, b: 23 });
				assert.deepEqual(observed.response.content, [
					{ type: "text", text: "42" },
				]);
				return { tool: "sum_numbers", a: 19, b: 23, result: 42 };
			},
		);
		await host.session.extensionRunner.emit({
			type: "session_shutdown",
			reason: "quit",
		});
	}
	assert.equal(
		mutations,
		0,
		"Read-only credential store must not mutate or refresh",
	);
	assert.equal(
		createHash("sha256")
			.update(await readFile(authPath))
			.digest("hex"),
		fingerprint,
		"Original credential file changed during test",
	);
	const local = await readFile(join(agentDir, "auth.json"), "utf8").catch(
		(error: NodeJS.ErrnoException) => {
			if (error.code === "ENOENT") return "{}";
			throw error;
		},
	);
	assert.deepEqual(
		JSON.parse(local),
		{},
		"No credential may be copied to test agent directory",
	);
} catch (error) {
	failed = true;
	process.stderr.write(safeError(error) + "\n");
} finally {
	const totalCost = host.session.getSessionStats().cost;
	await host.dispose();
	const report = {
		model: ref,
		passed: reports.filter((value) => value.ok).length,
		failed: reports.filter((value) => !value.ok).length,
		credentialMutations: mutations,
		cost: totalCost,
		cases: reports,
		artifacts: keep || failed ? root : null,
	};
	if (keep || failed)
		await writeFile(join(root, "report.json"), JSON.stringify(report, null, 2));
	else await rm(root, { recursive: true, force: true });
	process.stdout.write(JSON.stringify(report, null, 2) + "\n");
	if (failed) process.exitCode = 1;
}