pi-ext

import * as fs from "node:fs";
import * as path from "node:path";
import { fileURLToPath } from "node:url";
import { loadSkillsFromDir } from "@earendil-works/pi-coding-agent";
import { describe, expect, it } from "vitest";
import { resolveSelector, resolveTemplate } from "../interp.ts";
import { DEFAULT_FANOUT_TOOLS } from "../settings.ts";
import {
	compileSchema,
	DYNAMIC_EXTENSION_TOOL_NAMES,
	parseWorkflow,
	WorkflowSpecSchema,
} from "../spec.ts";

const PROMPT_PATH = fileURLToPath(
	new URL("../prompts/ultra-authoring/SKILL.md", import.meta.url),
);
const SPEC_FORMAT_PATH = fileURLToPath(
	new URL("../prompts/references/spec-format.md", import.meta.url),
);
const EXAMPLES_DIR = fileURLToPath(
	new URL("../prompts/references/examples/", import.meta.url),
);
const README_PATH = fileURLToPath(new URL("../README.md", import.meta.url));
const BUILDER_PROMPT_PATH = fileURLToPath(
	new URL("../prompts/dynamic-extension-builder.md", import.meta.url),
);

function referenceMarkdown(): string {
	const index = fs.readFileSync(SPEC_FORMAT_PATH, "utf8");
	const links = [...index.matchAll(/\]\((examples\/[^)]+\.md)\)/g)].map(
		([, link]) => link,
	);
	return [
		index,
		...links.map((link) =>
			fs.readFileSync(path.join(EXAMPLES_DIR, path.basename(link)), "utf8"),
		),
	].join("\n");
}

function referenceWorkflow(name: string) {
	const markdown = referenceMarkdown();
	const workflows = [...markdown.matchAll(/```json\n([\s\S]*?)```/g)].map(
		([, block]) => parseWorkflow(block),
	);
	const workflow = workflows.find((spec) => spec.name === name);
	if (!workflow) throw new Error(`Missing reference workflow: ${name}`);
	return workflow;
}

describe("workflow authoring skill", () => {
	it("loads and resolves its references", () => {
		const packageJson = JSON.parse(
			fs.readFileSync(
				fileURLToPath(new URL("../package.json", import.meta.url)),
				"utf8",
			),
		);
		expect(packageJson.pi).toEqual({ extensions: ["index.ts"] });
		const result = loadSkillsFromDir({
			dir: fileURLToPath(
				new URL("../prompts/ultra-authoring", import.meta.url),
			),
			source: "test",
		});
		expect(result.diagnostics).toEqual([]);
		expect(result.skills.map((skill) => skill.name)).toEqual([
			"ultra-authoring",
		]);
		for (const [, link] of fs
			.readFileSync(PROMPT_PATH, "utf8")
			.matchAll(/\]\(([^)]+)\)/g)) {
			expect(
				fs.existsSync(
					new URL(
						link,
						new URL("../prompts/ultra-authoring/SKILL.md", import.meta.url),
					),
				),
			).toBe(true);
		}
	});

	it("documents command, child, phase, and result mechanics", () => {
		const prompt = fs.readFileSync(PROMPT_PATH, "utf8");
		const reference = fs.readFileSync(SPEC_FORMAT_PATH, "utf8");
		for (const term of [
			"/ultra exec <instructions>",
			"/ultra run <name> [args]",
			"inline spec",
			"single",
			"fanout",
			"children run in separate sessions",
			"do not inherit the parent conversation",
			"automatically wraps fresh assignments",
			"immediate producer identity",
			"writeIsolation",
			"structured_output",
			"fullOutputPath",
			"thinkingLevel",
		])
			expect(prompt).toContain(term);
		for (const policy of [
			"materially help",
			"least-compute",
			"token economics",
			"Delegation contract",
		])
			expect(prompt).not.toContain(policy);
		expect(reference).not.toContain("least-compute tier");
		const readme = fs.readFileSync(README_PATH, "utf8");
		expect(readme).toContain("registers the [Ultra authoring skill]");
		expect(readme).toContain("`/ultra exec` sends only the user's request");
		expect(readme).not.toContain("selection rules live in the authoring skill");
	});

	it("keeps workflow mechanics synchronized with runtime validation", () => {
		const prompt = fs.readFileSync(PROMPT_PATH, "utf8");
		const reference = fs.readFileSync(SPEC_FORMAT_PATH, "utf8");
		const schema = JSON.stringify(WorkflowSpecSchema);
		for (const term of [
			"{PHASE.results[].FIELD}",
			"{PHASE.results[].FIELD[]}",
			'{EXPR | where FIELD == "VALUE"}',
			"{EXPR | groupBy FIELD}",
			"single-selector grammar",
			"every listed tool must be available",
			"loaded extensions remain trusted to activate additional tools",
			"prompt must instruct the agent not to mutate anything",
		])
			expect(reference).toContain(term);
		for (const tool of DEFAULT_FANOUT_TOOLS)
			expect(reference).toContain(`\`${tool}\``);
		expect(reference).toContain("ultra.fanoutToolAllowlist");
		expect(schema).toContain('"required":["id","kind","step"]');
		expect(schema).toContain('"required":["id","kind","over","step"]');
		for (const tool of DYNAMIC_EXTENSION_TOOL_NAMES) {
			expect(prompt).toContain(tool);
			expect(reference).toContain(tool);
		}
		expect(schema).toContain("dynamicExtensions");
		expect(schema).toContain("writeIsolation");
		expect(prompt).toContain("builder failure blocks later phases");
		expect(reference).toContain("dynamic_extension_builder_failure");
	});

	it("keeps dynamic builder guidance outside normal workflow guidance", () => {
		const prompt = fs.readFileSync(PROMPT_PATH, "utf8");
		const builder = fs.readFileSync(BUILDER_PROMPT_PATH, "utf8");
		expect(prompt).toContain("dynamic-extension builder contract");
		expect(builder).toContain("dynamic-extension builder guidance");
		expect(builder).toContain("failures are reported but do not block");
		expect(prompt).not.toContain("manifest.json");
	});

	it("keeps the reference doc and validator in lockstep", () => {
		const md = referenceMarkdown();
		const blocks = [...md.matchAll(/```json\n([\s\S]*?)```/g)].map((m) => m[1]);
		expect(blocks.length).toBeGreaterThan(0);

		let completeSpecs = 0;
		for (const block of blocks) {
			let parsed: unknown;
			expect(() => {
				parsed = JSON.parse(block);
			}).not.toThrow();

			const isCompleteSpec =
				typeof parsed === "object" &&
				parsed !== null &&
				!Array.isArray(parsed) &&
				"name" in parsed &&
				"phases" in parsed;
			if (!isCompleteSpec) continue;
			completeSpecs += 1;
			expect(compileSchema(WorkflowSpecSchema).Check(parsed)).toBe(true);
			const spec = parseWorkflow(parsed);
			const context = {
				args: { paths: ["README.md"], instructions: "Read AGENTS.md" },
				item: "README.md",
				results: {} as Record<string, unknown[]>,
				failures: {} as Record<string, unknown[]>,
			};
			for (const phase of spec.phases) {
				expect(() => resolveTemplate(phase.step.prompt, context)).not.toThrow();
				expect(() =>
					resolveTemplate(phase.step.summary, context),
				).not.toThrow();
				for (const selector of [phase.when, phase.over]) {
					if (typeof selector === "string")
						expect(() => resolveSelector(selector, context)).not.toThrow();
				}
				context.results[phase.id] = [];
				context.failures[phase.id] = [];
			}
			for (const selector of [spec.return, spec.report])
				if (selector)
					expect(() => resolveSelector(selector, context)).not.toThrow();
		}
		expect(completeSpecs).toBeGreaterThanOrEqual(20);
	});

	it("curates examples for the full workflow feature surface", () => {
		for (const name of [
			"single-input",
			"literal-fanout",
			"input-fanout",
			"conditional-single",
			"conditional-fanout",
			"project-reports",
			"flatten-findings",
			"truthy-filter",
			"source-filter",
			"source-groups",
			"named-schema",
			"inline-schema",
			"bounded-fanout",
			"routed-review",
			"concrete-model",
			"read-only-review",
			"isolated-edits",
			"dynamic-consumer",
			"dynamic-builder",
			"separate-report",
			"recover-failure",
			"three-stage",
			"complex-mixed",
		])
			expect(() => referenceWorkflow(name)).not.toThrow();

		const markdown = referenceMarkdown();
		const specs = [...markdown.matchAll(/```json\n([\s\S]*?)```/g)].map(
			([, block]) => parseWorkflow(block),
		);
		for (const spec of specs)
			for (const phase of spec.phases)
				expect(phase).not.toHaveProperty("tools");

		const index = fs.readFileSync(SPEC_FORMAT_PATH, "utf8");
		const links = [...index.matchAll(/\]\((examples\/[^)]+\.md)\)/g)].map(
			([, link]) => path.basename(link),
		);
		for (const link of links)
			expect(fs.existsSync(path.join(EXAMPLES_DIR, link))).toBe(true);
		expect(
			fs
				.readdirSync(EXAMPLES_DIR)
				.filter((name) => name.endsWith(".md"))
				.sort(),
		).toEqual(links.sort());

		const builder = referenceWorkflow("dynamic-builder");
		const build = builder.phases.find((phase) => phase.id === "build");
		const use = builder.phases.find((phase) => phase.id === "use");
		expect(build?.step.tools).toEqual([...DYNAMIC_EXTENSION_TOOL_NAMES]);
		expect(build?.step.schema).toBe("BuilderResult");
		expect(use?.when).toBe('{build.results | where status == "validated"}');
		expect(use?.step.dynamicExtensions).toBe(
			"{build.results[].extensionNames[]}",
		);
		expect(builder.return).not.toBe(builder.report);
	});

	it("keeps canonical workflow examples addressable", () => {
		for (const name of ["review", "read-package-license"]) {
			const spec = referenceWorkflow(name);
			expect(spec.name).toBe(name);
			expect(spec.phases.length).toBeGreaterThan(0);
		}
	});
});