pi-ext

import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { runWorkflow } from "../engine.ts";
import { resolveSelector } from "../interp.ts";
import { compileSchema, discoverWorkflows, parseWorkflow } from "../spec.ts";

// The shipped example lives beside the runtime's bundled-workflows lookup.
const here = path.dirname(fileURLToPath(import.meta.url));
const EXAMPLE_PATH = path.join(here, "..", "workflows", "review.json");

// Read-only initial tool set for the v1 `fanout` isolation rule. Every
// `fanout` step in the shipped example must stay within this set (this example
// IS the rule's reference implementation).
const READ_ONLY_TOOLS = new Set(["read", "grep", "find", "ls", "bash"]);

function loadRaw(): string {
	return fs.readFileSync(EXAMPLE_PATH, "utf8");
}

describe("review shipped example", () => {
	it("parses through parseWorkflow without error (validates the actual shipped file)", () => {
		// Pass the raw string so JSON.parse also checks the shipped syntax.
		const spec = parseWorkflow(loadRaw());
		expect(spec.name).toBe("review");
		expect(spec.phases).toHaveLength(3);
		expect(spec.phases.map((phase) => phase.id)).toEqual([
			"review",
			"verify",
			"report",
		]);
	});

	it("keeps review and synthesis on the reasoning tier without pinning thinking", () => {
		const spec = parseWorkflow(loadRaw());
		expect(
			Object.fromEntries(
				spec.phases.map((phase) => [phase.id, phase.step.model]),
			),
		).toEqual({
			review: "large",
			verify: "large",
			report: "large",
		});
		for (const phase of spec.phases)
			expect(phase.step.thinkingLevel).toBeUndefined();
	});

	it("declares a general review target positional and consumes {args.target}", () => {
		const spec = parseWorkflow(loadRaw());
		// A single positional param soaks the whole tail: `/ultra run review PR 123`,
		// `/ultra run review src/foo.ts`, `/ultra run review last 3 commits`, etc.
		expect(spec.args?.params).toEqual(["target"]);
		const review = spec.phases.find((p) => p.id === "review");
		expect(review?.step.prompt).toContain("{args.target}");
		expect(review?.step.prompt).toMatch(/PR|branch|folder|file|recent/i);
	});

	it("requires evidence-rich tri-state verdicts correlated by id, file and title", () => {
		const spec = parseWorkflow(loadRaw());

		// Producer + consumer side of the join key both present.
		const verdict = spec.schemas?.Verdict as {
			required?: string[];
			properties?: Record<string, unknown>;
		};
		expect(verdict.required).toEqual(
			expect.arrayContaining(["id", "file", "title", "status", "why"]),
		);
		expect(verdict.properties).not.toHaveProperty("real");
		const validate = compileSchema(verdict);
		const value = {
			id: "correctness:1",
			file: "src/a.ts",
			title: "Boundary",
			status: "unresolved",
			why: "Cannot inspect target ref",
		};
		for (const status of ["confirmed", "refuted", "unresolved"])
			expect(validate.Check({ ...value, status })).toBe(true);
		expect(validate.Check({ ...value, status: "unknown" })).toBe(false);
		expect(validate.Check({ ...value, real: false })).toBe(false);
		expect(validate.Check({ ...value, why: "" })).toBe(false);
		const withoutWhy: Record<string, unknown> = { ...value };
		delete withoutWhy.why;
		expect(validate.Check(withoutWhy)).toBe(false);

		const verify = spec.phases.find((p) => p.id === "verify");
		// The echo instruction ties filtered verdicts back to their findings.
		expect(verify?.step.prompt).toMatch(/echo/i);
		expect(verify?.step.prompt).toMatch(/file/);
		expect(verify?.step.prompt).toContain("{args.target}");
		expect(verify?.step.prompt).toContain("{item}");
		expect(verify?.step.prompt).toContain("Uncertainty is never refutation");
		expect(verify?.step.tools).toEqual(
			expect.arrayContaining(["read", "bash", "grep", "find", "ls"]),
		);
	});

	it("every fanout step uses only read-only tools (the v1 read-only-fanout isolation rule)", () => {
		const spec = parseWorkflow(loadRaw());
		for (const phase of spec.phases) {
			if (phase.kind !== "fanout") continue;
			expect(phase.step.prompt).toContain("AGENTS.md");
			expect(phase.step.prompt).toMatch(/do not modify/i);
			const tools = phase.step.tools ?? [];
			for (const t of tools) {
				expect(READ_ONLY_TOOLS.has(t)).toBe(true);
			}
		}
	});

	it("returns the synthesized analysis, findings, and recommendations report", () => {
		const spec = parseWorkflow(loadRaw());
		expect(spec.return).toBe("{report.results}");
		const report = spec.phases.find((phase) => phase.id === "report");
		expect(report?.step.prompt).toContain("{review.results}");
		expect(report?.step.prompt).toContain("{verify.results}");
		expect(report?.step.prompt).toContain("{review.failures}");
		expect(report?.step.prompt).toContain("{verify.failures}");
		expect(report?.when).toBeUndefined();
		expect(report?.step.prompt).toMatch(/analysis/i);
		expect(report?.step.prompt).toMatch(/recommendations/i);
		expect(report?.step.schema).toBe("ReviewReport");

		const value = {
			status: "completed",
			summary: "One confirmed defect",
			analysis: "The change is sound except for one confirmed defect.",
			blockers: [],
			unresolved: [],
			findings: [
				{
					id: "correctness:1",
					title: "off-by-one in loop",
					file: "src/a.ts",
					why: "confirmed",
					severity: "high",
				},
			],
			recommendations: ["Fix the loop bound and rerun focused tests."],
		};
		const validate = compileSchema(spec.schemas?.ReviewReport ?? {});
		expect(validate.Check(value)).toBe(true);
		const withoutUnresolved: Record<string, unknown> = { ...value };
		delete withoutUnresolved.unresolved;
		expect(validate.Check(withoutUnresolved)).toBe(false);
		expect(
			validate.Check({
				...value,
				unresolved: [
					{ id: "correctness:2", title: "Unknown", file: "src/a.ts" },
				],
			}),
		).toBe(false);
		expect(
			resolveSelector(spec.return ?? "", { results: { report: [value] } }),
		).toEqual([value]);
	});

	it("requires dimension-prefixed globally unique ids and precise finding packets", () => {
		const spec = parseWorkflow(loadRaw());
		const scan = spec.phases.find((phase) => phase.id === "review");
		expect(scan?.step.prompt).toContain("globally unique id");
		expect(scan?.step.prompt).toContain("locally unique identifier");
		expect(scan?.step.prompt).toContain("never reuse an id");
		const finding = {
			id: "correctness:1",
			title: "Boundary",
			file: "src/a.ts",
			claim: "Zero input throws",
			evidence: "src/a.ts:10 rejects zero",
			scope: "Input zero on target ref",
			instructions: ["AGENTS.md"],
		};
		const validate = compileSchema(spec.schemas?.Findings ?? {});
		const packet = {
			dimension: "correctness",
			findings: [finding],
			blockers: [],
			gaps: [],
		};
		expect(validate.Check(packet)).toBe(true);
		for (const key of ["id", "claim", "evidence", "scope", "instructions"]) {
			const incomplete: Record<string, unknown> = { ...finding };
			delete incomplete[key];
			expect(validate.Check({ ...packet, findings: [incomplete] })).toBe(false);
		}
		expect(
			validate.Check({ ...packet, findings: [{ ...finding, id: "1" }] }),
		).toBe(false);
		expect(
			validate.Check({ ...packet, findings: Array(13).fill(finding) }),
		).toBe(false);
	});

	it.each([false, true])(
		"preserves same-title/file independent packets and unresolved verification (failure=%s)",
		async (failure) => {
			const spec = parseWorkflow(loadRaw());
			const verifiedIds: string[] = [];
			const target = "feature-branch against main";
			const unresolved = {
				id: "performance:1",
				title: "Boundary",
				file: "src/a.ts",
				why: "Target evidence inaccessible",
			};
			const report = {
				status: "unresolved",
				summary: "One confirmed defect, one unresolved claim",
				analysis:
					"Correctness confirmed; security refuted; performance unresolved.",
				blockers: [],
				findings: [
					{
						id: "correctness:1",
						title: "Boundary",
						file: "src/a.ts",
						why: "src/a.ts:10 rejects valid input",
						severity: "high",
					},
				],
				unresolved: [unresolved],
				recommendations: ["Inspect missing performance evidence"],
			};
			const result = await runWorkflow(
				spec,
				{ target },
				{
					concurrency: 3,
					stepRunner: async ({ phase, step, item }) => {
						if (phase === "review")
							return {
								ok: true,
								value: {
									dimension: item,
									blockers: [],
									gaps: [],
									findings: [
										{
											id: `${item}:1`,
											title: "Boundary",
											file: "src/a.ts",
											claim: `${item} claim`,
											evidence: "src/a.ts:10",
											scope: target,
											instructions: ["AGENTS.md: keep API"],
										},
									],
								},
							};
						if (phase === "verify") {
							const candidate = item as {
								id: string;
								title: string;
								file: string;
							};
							verifiedIds.push(candidate.id);
							expect(step.prompt).toContain(target);
							expect(step.prompt).toContain("src/a.ts:10");
							expect(step.prompt).toContain("AGENTS.md: keep API");
							if (candidate.id === unresolved.id && failure)
								return {
									ok: false,
									value: null,
									failure: {
										code: "transport",
										message: unresolved.why,
										retryable: true,
										attempts: 1,
									},
								};
							const value = {
								id: candidate.id,
								file: candidate.file,
								title: candidate.title,
								status:
									candidate.id === unresolved.id
										? "unresolved"
										: candidate.id === "correctness:1"
											? "confirmed"
											: "refuted",
								why:
									candidate.id === unresolved.id
										? unresolved.why
										: "Inspected src/a.ts:10",
							};
							expect(
								compileSchema(spec.schemas?.Verdict ?? {}).Check(value),
							).toBe(true);
							return { ok: true, value };
						}
						expect(phase).toBe("report");
						expect(step.prompt).toContain(unresolved.id);
						expect(step.prompt).toContain(unresolved.why);
						expect(step.prompt).toContain('"status":"refuted"');
						expect(step.prompt).toContain('"status":"confirmed"');
						if (failure) expect(step.prompt).toContain('"code":"transport"');
						expect(
							compileSchema(spec.schemas?.ReviewReport ?? {}).Check(report),
						).toBe(true);
						return { ok: true, value: report };
					},
				},
			);
			expect(verifiedIds.sort()).toEqual([
				"correctness:1",
				"performance:1",
				"security:1",
			]);
			expect(result.result).toEqual([report]);
			if (failure)
				expect(result.phaseFailures.verify[0].item).toMatchObject({
					id: unresolved.id,
					file: unresolved.file,
				});
		},
	);

	it.each([false, true])(
		"reports blocked scans rather than an empty successful review (failure=%s)",
		async (failure) => {
			const spec = parseWorkflow(loadRaw());
			const reason = "Requested ref does not exist";
			const report = {
				status: "blocked",
				summary: reason,
				analysis: "No target was inspected",
				blockers: [reason],
				findings: [],
				unresolved: [
					{
						id: "coverage:target",
						title: "Target unavailable",
						file: "",
						why: reason,
					},
				],
				recommendations: ["Supply an accessible ref"],
			};
			const calls: string[] = [];
			const result = await runWorkflow(
				spec,
				{ target: "missing-ref" },
				{
					concurrency: 3,
					stepRunner: async ({ phase, step, item }) => {
						calls.push(phase);
						if (phase === "review") {
							if (failure)
								return {
									ok: false,
									value: null,
									failure: {
										code: "session",
										message: reason,
										retryable: false,
										attempts: 1,
									},
								};
							return {
								ok: true,
								value: {
									dimension: item,
									findings: [],
									blockers: [reason],
									gaps: [],
								},
							};
						}
						expect(phase).toBe("report");
						expect(step.prompt).toContain(reason);
						expect(
							compileSchema(spec.schemas?.ReviewReport ?? {}).Check(report),
						).toBe(true);
						return { ok: true, value: report };
					},
				},
			);
			expect(calls).not.toContain("verify");
			expect(result.phaseResults.verify).toEqual([]);
			expect(result.result).toEqual([report]);
		},
	);

	it.each(["bug", "feature", "review", "research"])(
		"bounds all strings and arrays in shipped %s schemas",
		(name) => {
			const spec = parseWorkflow(
				fs.readFileSync(
					path.join(here, "..", "workflows", `${name}.json`),
					"utf8",
				),
			);
			function checkBounds(value: unknown): void {
				if (!value || typeof value !== "object") return;
				const node = value as Record<string, unknown>;
				if (node.type === "string" && !Array.isArray(node.enum))
					expect(node.maxLength).toEqual(expect.any(Number));
				if (node.type === "array")
					expect(node.maxItems).toEqual(expect.any(Number));
				for (const child of Object.values(node)) checkBounds(child);
			}
			for (const schema of Object.values(spec.schemas ?? {})) {
				expect(compileSchema(schema)).toBeDefined();
				checkBounds(schema);
			}
			for (const phase of spec.phases)
				expect(phase.step.prompt).toContain("overflow");
		},
	);

	it("is discoverable as a bundled workflow named `review`", () => {
		// Empty project/global dirs => only the bundled dir (the real shipped file)
		// contributes. `discoverWorkflows` keys on the `name` field.
		const emptyCwd = fs.mkdtempSync(path.join(os.tmpdir(), "ultra-cwd-"));
		const emptyHome = fs.mkdtempSync(path.join(os.tmpdir(), "ultra-home-"));
		try {
			const found = discoverWorkflows(emptyCwd, { home: emptyHome });
			const review = found.find((w) => w.name === "review");
			expect(review).toBeDefined();
			expect(review?.source).toBe("bundled");
		} finally {
			fs.rmSync(emptyCwd, { recursive: true, force: true });
			fs.rmSync(emptyHome, { recursive: true, force: true });
		}
	});
});