import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { runWorkflow } from "../engine.ts"; import { resolveSelector } from "../interp.ts"; import { compileSchema, discoverWorkflows, parseWorkflow } from "../spec.ts"; // The shipped `feature` workflow: research → plan → critique → refine → // gated implement → conditional review → report. const here = path.dirname(fileURLToPath(import.meta.url)); const FEATURE_PATH = path.join(here, "..", "workflows", "feature.json"); // Read-only initial tool set for the v1 `fanout` rule. The single `implement` // phase is exempt (it writes via edit/write); only `fanout` steps are checked. const READ_ONLY_TOOLS = new Set(["read", "grep", "find", "ls", "bash"]); function loadRaw(): string { return fs.readFileSync(FEATURE_PATH, "utf8"); } describe("feature shipped workflow", () => { it("parses through parseWorkflow (validates the actual shipped file)", () => { const spec = parseWorkflow(loadRaw()); expect(spec.name).toBe("feature"); expect(spec.phases.map((p) => p.id)).toEqual([ "research", "plan", "critique", "refine", "implement", "review", "report", ]); }); it("selects tiers by phase role and leaves thinking to settings", () => { const spec = parseWorkflow(loadRaw()); expect( Object.fromEntries( spec.phases.map((phase) => [phase.id, phase.step.model]), ), ).toEqual({ research: "small", plan: "large", critique: "large", refine: "large", implement: "medium", review: "large", report: "medium", }); for (const phase of spec.phases) expect(phase.step.thinkingLevel).toBeUndefined(); }); it("takes a free-text task as a positional arg consumed by every phase", () => { const spec = parseWorkflow(loadRaw()); expect(spec.args?.params).toEqual(["task"]); // Every reasoning/writing phase remains scoped to the task. for (const id of ["research", "critique", "refine", "implement"]) { const p = spec.phases.find((ph) => ph.id === id); expect(p?.step.prompt).toContain("{args.task}"); } }); it("chains the phases: each consumes its predecessors' results", () => { const spec = parseWorkflow(loadRaw()); const get = (id: string) => spec.phases.find((p) => p.id === id)?.step.prompt ?? ""; expect(get("plan")).toContain("{research.results}"); expect(get("critique")).toContain("{plan.results}"); expect(get("refine")).toContain("{plan.results}"); expect(get("refine")).toContain("{critique.results}"); expect(get("implement")).toContain("{refine.results}"); expect(get("review")).toContain("{implement.results}"); }); it("only `implement` (a single phase) may write; the fanout review stays read-only", () => { const spec = parseWorkflow(loadRaw()); const implement = spec.phases.find((p) => p.id === "implement"); expect(implement?.kind).toBe("single"); expect(implement?.step.tools).toEqual( expect.arrayContaining(["edit", "write"]), ); for (const phase of spec.phases) { if (phase.kind !== "fanout") continue; // the rule only binds fanout steps for (const t of phase.step.tools ?? []) { expect(READ_ONLY_TOOLS.has(t)).toBe(true); } } }); it("strengthens the plan with critique feedback and reviews only completed implementations", () => { const spec = parseWorkflow(loadRaw()); const refine = spec.phases.find((p) => p.id === "refine"); const implement = spec.phases.find((p) => p.id === "implement"); const review = spec.phases.find((p) => p.id === "review"); expect(refine?.step.prompt).toContain("resolves EVERY blocking item"); expect(implement?.when).toBe("{refine.results | where ready}"); expect(implement?.step.prompt).toContain("{refine.results}"); expect(review?.when).toBe("{implement.results | where implemented}"); const implementation = spec.schemas?.Implementation as { required?: string[]; }; expect(implementation.required).toContain("implemented"); }); it("reviews across security + quality, each echoing its dimension as the join key", () => { const spec = parseWorkflow(loadRaw()); const review = spec.phases.find((p) => p.id === "review"); expect(review?.kind).toBe("fanout"); expect(review?.over).toEqual(["security", "quality"]); // the verdict echoes `dimension` so a result ties back to its lens expect(review?.step.prompt).toMatch(/dimension/); const verdict = spec.schemas?.Verdict as { required?: string[] }; expect(verdict.required).toContain("dimension"); }); it("returns the explicit report, including unresolved evidence", () => { const spec = parseWorkflow(loadRaw()); expect(spec.return).toBe("{report.results}"); const value = { status: "unresolved", summary: "Review incomplete", blockers: [], unresolved: ["quality: checks unavailable"], checks: [], findings: [], recommendations: ["Run missing quality checks"], }; expect(compileSchema(spec.schemas?.FeatureReport ?? {}).Check(value)).toBe( true, ); expect( resolveSelector(spec.return ?? "", { results: { report: [value] } }), ).toEqual([value]); }); it("bounds scouting to facts and requires a self-contained ready implementation handoff", () => { const spec = parseWorkflow(loadRaw()); const get = (id: string) => spec.phases.find((phase) => phase.id === id); expect(get("research")?.step.prompt).toContain("at most 8"); expect(get("research")?.step.prompt).toContain( "Do not decide the implementation approach", ); expect(get("plan")?.step.prompt).toContain("You own approach decisions"); expect(get("refine")?.step.prompt).toContain("complete replacement"); expect(get("implement")?.step.prompt).toContain( "do not choose a new approach", ); const plan = spec.schemas?.Plan as { required: string[] }; expect(plan.required).toEqual( expect.arrayContaining([ "ready", "tasks", "approach", "rationale", "resolved_issues", "instructions", "evidence_paths", "completion_checks", "blockers", "gaps", ]), ); for (const phase of spec.phases.filter((phase) => phase.id !== "report")) { expect(phase.step.prompt).toContain("AGENTS.md"); expect(phase.step.prompt).toMatch(/user instructions/); expect(phase.step.prompt).toContain("overflow"); } const report = get("report"); expect(report?.when).toBeUndefined(); expect(report?.step.prompt).toContain("exact source field and issue text"); for (const id of [ "research", "plan", "critique", "refine", "implement", "review", ]) { expect(report?.step.prompt).toContain(`{${id}.results}`); expect(report?.step.prompt).toContain(`{${id}.failures}`); } }); it.each([ ["refine", false, "blocked"], ["refine", true, "blocked"], ["implement", false, "blocked"], ["implement", true, "blocked"], ["review", false, "unresolved"], ["review", true, "unresolved"], ["none", false, "completed"], ])( "delivers %s (failure=%s) evidence to an unconditional simulated %s report", async (stoppedAt, failure, status) => { const spec = parseWorkflow(loadRaw()); const reason = `${stoppedAt}: required constraint unavailable`; const calls: string[] = []; const plan = { ready: true, tasks: ["Update src/a.ts"], approach: "Reuse shared route", rationale: "Critique concerns resolved by existing validation", resolved_issues: [], instructions: ["AGENTS.md: preserve public API"], evidence_paths: ["src/a.ts:10"], completion_checks: ["focused test must pass"], risks: [], blockers: [], gaps: [], }; const fixtures: Record> = { research: { findings: "Shared route exists", files: ["src/a.ts"], patterns: [], instructions: plan.instructions, evidence_paths: plan.evidence_paths, blockers: [], gaps: [], }, plan, critique: { approved: true, blocking: [], feedback: "Ready", evidence_paths: plan.evidence_paths, gaps: [], }, refine: plan, implement: { implemented: true, summary: "Feature added", modified_files: ["src/a.ts"], completed: plan.tasks, checks: ["focused test passed"], evidence_paths: plan.evidence_paths, blockers: [], gaps: [], }, }; const report = { status, summary: status === "completed" ? "Feature verified" : reason, blockers: status === "blocked" ? [reason] : [], unresolved: status === "unresolved" ? [reason] : [], checks: [], findings: [], recommendations: [], }; const result = await runWorkflow( spec, { task: "Add feature without changing API" }, { concurrency: 2, stepRunner: async ({ phase, step, item }) => { calls.push(phase); if (phase === "report") { if (stoppedAt !== "none") expect(step.prompt).toContain(reason); expect(step.prompt).toContain("Refinement failures:"); expect( compileSchema(spec.schemas?.FeatureReport ?? {}).Check(report), ).toBe(true); return { ok: true, value: report }; } const stopped = phase === stoppedAt && (phase !== "review" || item === "quality"); if (stopped && failure) return { ok: false, value: null, failure: { code: "transport", message: reason, retryable: true, attempts: 1, }, }; const value = phase === "review" ? { dimension: item, passed: !stopped, findings: [], checks: [], unresolved: stopped ? [reason] : [], recommendations: [], } : { ...fixtures[phase] }; if (stopped && phase !== "review") Object.assign(value, { [phase === "implement" ? "implemented" : "ready"]: false, blockers: [reason], }); if (phase === "implement") { expect(step.prompt).toContain("AGENTS.md: preserve public API"); expect(step.prompt).toContain("src/a.ts:10"); expect(step.prompt).toContain("focused test must pass"); } expect( compileSchema(spec.schemas?.[String(step.schema)] ?? {}).Check( value, ), ).toBe(true); return { ok: true, value }; }, }, ); expect(result.result).toEqual([report]); expect(calls.at(-1)).toBe("report"); if (stoppedAt === "refine") { expect(calls).not.toContain("implement"); expect(result.phaseResults.implement).toEqual([]); } if (stoppedAt === "refine" || stoppedAt === "implement") expect(calls).not.toContain("review"); }, ); it("is discoverable as a bundled workflow named `feature`", () => { const emptyCwd = fs.mkdtempSync(path.join(os.tmpdir(), "ultra-cwd-")); const emptyHome = fs.mkdtempSync(path.join(os.tmpdir(), "ultra-home-")); try { const found = discoverWorkflows(emptyCwd, { home: emptyHome }); const feature = found.find((w) => w.name === "feature"); expect(feature).toBeDefined(); expect(feature?.source).toBe("bundled"); } finally { fs.rmSync(emptyCwd, { recursive: true, force: true }); fs.rmSync(emptyHome, { recursive: true, force: true }); } }); });