import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { compileSchema, discoverWorkflows, missingWorkflowTools, parseWorkflow, } from "../spec"; const FINDINGS_SCHEMA = { type: "object", properties: { findings: { type: "array", items: { type: "object", properties: { title: { type: "string" }, file: { type: "string" }, detail: { type: "string" }, }, required: ["title", "file"], }, }, }, required: ["findings"], }; const VERDICT_SCHEMA = { type: "object", properties: { title: { type: "string" }, real: { type: "boolean" }, why: { type: "string" }, file: { type: "string", description: "echo of the reviewed finding's file", }, }, required: ["title", "real", "file"], }; // Mirrors the design's review example (design lines 104-164). Both // referenced schema names (Findings + Verdict) are defined. function validSpec(): any { return { name: "review", description: "Review the diff across dimensions, then adversarially verify each finding.", args: { hint: "[base-ref]" }, schemas: { Findings: FINDINGS_SCHEMA, Verdict: VERDICT_SCHEMA }, phases: [ { id: "review", kind: "fanout", over: ["correctness", "security", "performance"], step: { summary: "Review diff for {item} issues", prompt: "Review the working diff for {item} issues. Return JSON matching Findings.", tools: ["read", "bash", "grep", "find", "ls"], schema: "Findings", }, }, { id: "verify", kind: "fanout", over: "{review.results[].findings[]}", step: { summary: "Verify one finding", prompt: "Adversarially try to refute this finding.\n\nFinding: {item}", schema: "Verdict", }, }, ], return: "{verify.results | where real}", }; } describe("parseWorkflow", () => { it("accepts an independent report selector", () => { const spec = { ...validSpec(), report: "{verify.results}" }; expect(parseWorkflow(spec).report).toBe("{verify.results}"); }); it("parses a valid review-shaped spec (object input)", () => { const parsed = parseWorkflow(validSpec()); expect(parsed.name).toBe("review"); expect(parsed.phases).toHaveLength(2); expect(parsed.phases[0].kind).toBe("fanout"); expect(parsed.return).toBe("{verify.results | where real}"); }); it("parses a valid spec from a JSON string", () => { const parsed = parseWorkflow(JSON.stringify(validSpec())); expect(parsed.name).toBe("review"); }); it("requires a progress summary for every step", () => { const spec = validSpec(); delete spec.phases[0].step.summary; expect(() => parseWorkflow(spec)).toThrow(/summary/i); }); it("accepts optional phase `when` selectors", () => { const s = validSpec(); s.phases = [ { id: "only", kind: "single", when: "{args.enabled}", step: { summary: "go", prompt: "go" }, }, ]; expect(parseWorkflow(s).phases[0].when).toBe("{args.enabled}"); }); it("rejects non-string phase `when` values", () => { const s = validSpec(); s.phases[0].when = true; expect(() => parseWorkflow(s)).toThrow(/when/i); }); it("accepts a single phase without `over`", () => { const s = validSpec(); s.phases = [ { id: "only", kind: "single", step: { summary: "go", prompt: "go" } }, ]; expect(() => parseWorkflow(s)).not.toThrow(); }); it("requires interpolation-safe phase identifiers", () => { const s = validSpec(); s.phases[0].id = "first-pass"; expect(() => parseWorkflow(s)).toThrow(/id/i); }); it("rejects duplicate phase identifiers", () => { const s = validSpec(); s.phases[1].id = s.phases[0].id; expect(() => parseWorkflow(s)).toThrow(/duplicate phase id.*review/i); }); it("reserves args but allows item as a phase identifier", () => { const reserved = validSpec(); reserved.phases[0].id = "args"; expect(() => parseWorkflow(reserved)).toThrow(/reserved phase id.*args/i); const allowed = validSpec(); allowed.phases[0].id = "item"; expect(() => parseWorkflow(allowed)).not.toThrow(); }); it("throws (not returns) on missing `name`, surfacing the TypeBox message", () => { const s = validSpec(); delete s.name; // throws rather than returning an error sentinel expect(() => parseWorkflow(s)).toThrow(); // surfaces the underlying TypeBox detail (the failing property name) expect(() => parseWorkflow(s)).toThrow(/name/i); }); it("throws on a bad phase `kind`", () => { const s = validSpec(); s.phases[0].kind = "parallel"; expect(() => parseWorkflow(s)).toThrow(); }); it("throws on a `fanout` phase without `over`", () => { const s = validSpec(); delete s.phases[0].over; expect(() => parseWorkflow(s)).toThrow(); }); it("accepts thinkingLevel 'low' (correction #8)", () => { const s = validSpec(); s.phases[0].step.thinkingLevel = "low"; expect(() => parseWorkflow(s)).not.toThrow(); expect(parseWorkflow(s).phases[0].step.thinkingLevel).toBe("low"); }); it("accepts every thinkingLevel in off|minimal|low|medium|high|xhigh|max", () => { for (const level of [ "off", "minimal", "low", "medium", "high", "xhigh", "max", ]) { const s = validSpec(); s.phases[0].step.thinkingLevel = level; expect(() => parseWorkflow(s)).not.toThrow(); } }); it("rejects a thinkingLevel outside the union", () => { const s = validSpec(); s.phases[0].step.thinkingLevel = "ultra"; expect(() => parseWorkflow(s)).toThrow(); }); it("rejects unknown step fields", () => { const s = validSpec(); s.phases[0].step.unknown = true; expect(() => parseWorkflow(s)).toThrow(/additional properties/i); }); it("throws a clear message when a step references an unknown schema name", () => { const s = validSpec(); s.phases[1].step.schema = "Nonexistent"; expect(() => parseWorkflow(s)).toThrow(/Nonexistent/); }); it("throws (not returns) on malformed JSON input", () => { expect(() => parseWorkflow("{ this is not json")).toThrow(); }); }); describe("fanout initial tool policy", () => { const spec = () => ({ name: "research", phases: [ { id: "shop", kind: "fanout", over: ["one", "two"], step: { summary: "Research", prompt: "Do not mutate.", tools: ["web"] }, }, ], }); it("rejects names outside the default list without labeling them mutating", () => { expect(() => parseWorkflow(spec())).toThrow( /fanout phase "shop" selects tools outside the fanout allowlist without writeIsolation: web\. Allowed initial tools: read, grep, find, ls, bash/, ); }); it("accepts configured names but still rejects other unlisted names", () => { expect(() => parseWorkflow(spec(), ["read", "web"])).not.toThrow(); const unlisted = spec(); unlisted.phases[0].step.tools = ["web", "other"]; expect(() => parseWorkflow(unlisted, ["read", "web"])).toThrow( /outside the fanout allowlist without writeIsolation: other/, ); }); it("accepts other names with genuine writeIsolation or in a single phase", () => { const isolated = { ...spec(), phases: [{ ...spec().phases[0], writeIsolation: "Separate owners." }], }; expect(() => parseWorkflow(isolated)).not.toThrow(); const single = { ...spec(), phases: [{ ...spec().phases[0], kind: "single" }], }; expect(() => parseWorkflow(single)).not.toThrow(); }); }); describe("missingWorkflowTools", () => { it("returns every distinct unavailable step tool in stable order", () => { const spec = parseWorkflow(validSpec()); expect(missingWorkflowTools(spec, ["read", "bash", "grep"])).toEqual([ "find", "ls", ]); expect( missingWorkflowTools(spec, ["read", "bash", "grep", "find", "ls"]), ).toEqual([]); }); }); describe("compileSchema", () => { it("compiles a named schema from the schemas map into a working validator", () => { const v = compileSchema(FINDINGS_SCHEMA); expect(v.Check({ findings: [{ title: "t", file: "f" }] })).toBe(true); expect(v.Check({ findings: [{ title: "t" }] })).toBe(false); // missing required `file` expect(v.Check({})).toBe(false); // missing required `findings` }); it("compiles an inline JSON-schema object into a working validator", () => { const inline = { type: "object", properties: { ok: { type: "boolean" } }, required: ["ok"], }; const v = compileSchema(inline); expect(v.Check({ ok: true })).toBe(true); expect(v.Check({ ok: "yes" })).toBe(false); expect(v.Check({})).toBe(false); }); it("parses a step that inlines a schema object, which then compiles", () => { const s = validSpec(); const inline = { type: "object", properties: { score: { type: "number" } }, required: ["score"], }; s.phases[1].step.schema = inline; const parsed = parseWorkflow(s); const stepSchema = parsed.phases[1].step.schema; expect(typeof stepSchema).toBe("object"); const v = compileSchema(stepSchema as object); expect(v.Check({ score: 1 })).toBe(true); expect(v.Check({ score: "x" })).toBe(false); }); }); describe("discoverWorkflows", () => { let tmp: string; let projectDir: string; let homeDir: string; let missingBundled: string; beforeAll(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), "ultra-spec-")); projectDir = path.join(tmp, "project"); homeDir = path.join(tmp, "home"); missingBundled = path.join(tmp, "no-bundled-dir-here"); const projWf = path.join(projectDir, ".pi", "workflows"); const globalWf = path.join(homeDir, ".pi", "agent", "workflows"); fs.mkdirSync(projWf, { recursive: true }); fs.mkdirSync(globalWf, { recursive: true }); // `foo` exists in BOTH project and global -> project must win. fs.writeFileSync( path.join(projWf, "foo.json"), JSON.stringify({ name: "foo", phases: [ { id: "p", kind: "single", step: { summary: "from-project", prompt: "from-project" }, }, ], }), ); fs.writeFileSync( path.join(globalWf, "foo.json"), JSON.stringify({ name: "foo", phases: [ { id: "p", kind: "single", step: { summary: "from-global", prompt: "from-global" }, }, ], }), ); // `bar` only in global. fs.writeFileSync( path.join(globalWf, "bar.json"), JSON.stringify({ name: "bar", phases: [ { id: "p", kind: "single", step: { summary: "x", prompt: "x" } }, ], }), ); }); afterAll(() => { fs.rmSync(tmp, { recursive: true, force: true }); }); it("discovers project + global workflows and tolerates a missing bundled dir", () => { const found = discoverWorkflows(projectDir, { home: homeDir, bundledDir: missingBundled, }); const names = found.map((w) => w.name).sort(); expect(names).toContain("foo"); expect(names).toContain("bar"); }); it("applies project-over-global precedence and tags the source", () => { const found = discoverWorkflows(projectDir, { home: homeDir, bundledDir: missingBundled, }); const foo = found.find((w) => w.name === "foo"); expect(foo?.source).toBe("project"); expect(foo?.spec.phases[0].step.prompt).toBe("from-project"); const bar = found.find((w) => w.name === "bar"); expect(bar?.source).toBe("global"); expect(bar?.path.endsWith("bar.json")).toBe(true); }); });