Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/skillz/__tests__/harness.test.ts

Raw
import {
	chmodSync,
	mkdirSync,
	mkdtempSync,
	readdirSync,
	readFileSync,
	rmSync,
	symlinkSync,
	writeFileSync,
} from "node:fs";
import os from "node:os";
import path from "node:path";
import {
	type ExtensionCommandContext,
	initTheme,
} from "@earendil-works/pi-coding-agent";
import { visibleWidth } from "@earendil-works/pi-tui";
import { afterEach, describe, expect, it } from "vitest";
import { createTestSession, type TestSession } from "../../../test/harness.ts";
import { inspectCandidates, showInspection } from "../inspect.ts";
import { matchSkill } from "../match.ts";

const extension = path.resolve(import.meta.dirname, "../index.ts");

function skill(agentDir: string, name: string, conditions: string): void {
	const dir = path.join(agentDir, "skillz", name);
	mkdirSync(dir, { recursive: true });
	writeFileSync(
		path.join(dir, "SKILL.md"),
		`---\nname: ${name}\ndescription: Skill ${name}\n${conditions}---\n\n# ${name}\n`,
	);
}

describe("skillz conditions", () => {
	const cwd = path.join(os.tmpdir(), "skillz-cwd");
	it("combines fields with AND and values within fields with OR", () => {
		expect(
			matchSkill({ os: ["mac", "lnx"], host: ["a", "b"] }, cwd, "linux", "b")
				.matches,
		).toBe(true);
		expect(
			matchSkill({ os: "win32", host: "b" }, cwd, "linux", "b").matches,
		).toBe(false);
		expect(matchSkill({ host: "B" }, cwd, "linux", "b").matches).toBe(false);
	});

	it("rejects missing and malformed conditions", () => {
		for (const value of [
			{},
			{ os: "linux" },
			{ os: [] },
			{ host: 4 },
			{ path: "../up" },
			{ path: "C:/x" },
			{ path: "foo\\bar" },
			{ path: "." },
			{ which: [] },
			{ which: "gh --version" },
			{ which: "../gh" },
		])
			expect(matchSkill(value, cwd).error).toBeDefined();
	});

	it("validates every present condition before matching", () => {
		expect(
			matchSkill(
				{ os: "mac", host: [], path: "../outside" },
				cwd,
				"linux",
				"x",
			),
		).toEqual({ matches: false, error: "invalid host condition" });
		expect(
			matchSkill(
				{ os: "mac", host: "x", path: "../outside" },
				cwd,
				"linux",
				"x",
			),
		).toEqual({ matches: false, error: "invalid path condition" });
		expect(
			matchSkill({ os: "mac", host: "x", path: ".jj" }, cwd, "linux", "x")
				.matches,
		).toBe(false);
	});

	it("supports explicit any and all for every field", () => {
		const any = matchSkill(
			{ os: { any: ["mac", "lnx"] }, host: { all: ["desktop", "other"] } },
			cwd,
			"linux",
			"desktop",
		);
		expect(any.matches).toBe(false);
		expect(any.checks?.map((check) => [check.mode, check.matches])).toEqual([
			["any", true],
			["all", false],
		]);
		expect(
			matchSkill({ os: { all: ["lnx", "lnx"] } }, cwd, "linux").matches,
		).toBe(true);
		expect(
			matchSkill({ os: { all: ["mac", "lnx"] } }, cwd, "linux").matches,
		).toBe(false);
		expect(
			matchSkill(
				{ host: { any: ["other", "desktop"] } },
				cwd,
				"linux",
				"desktop",
			).matches,
		).toBe(true);
		for (const condition of [
			{ os: { any: [] } },
			{ host: { all: [] } },
			{ path: { any: ["x", 5] } },
			{ which: { any: "gh" } },
			{ env: { all: ["OK"], any: ["NO"] } },
			{ os: { nope: ["lnx"] } },
		])
			expect(matchSkill(condition, cwd).error).toBeDefined();
	});

	it("matches nonempty env vars and retains names, never values", () => {
		const secret = ["skillz-secret", "DO-NOT-PRINT"].join("-");
		const names = [
			"SKILLZ_TEST_PRESENT",
			"SKILLZ_TEST_EMPTY",
			"SKILLZ_TEST_MISSING",
		];
		const previous = names.map((name) => process.env[name]);
		try {
			process.env.SKILLZ_TEST_PRESENT = secret;
			process.env.SKILLZ_TEST_EMPTY = "";
			delete process.env.SKILLZ_TEST_MISSING;
			const present = matchSkill({ env: "SKILLZ_TEST_PRESENT" }, cwd);
			expect(present.matches).toBe(true);
			expect(JSON.stringify(present)).not.toContain(secret);
			expect(matchSkill({ env: "SKILLZ_TEST_EMPTY" }, cwd).matches).toBe(false);
			expect(matchSkill({ env: "SKILLZ_TEST_MISSING" }, cwd).matches).toBe(
				false,
			);
			expect(
				matchSkill({ env: ["SKILLZ_TEST_EMPTY", "SKILLZ_TEST_PRESENT"] }, cwd)
					.matches,
			).toBe(true);
			expect(
				matchSkill(
					{ env: { any: ["SKILLZ_TEST_EMPTY", "SKILLZ_TEST_PRESENT"] } },
					cwd,
				).matches,
			).toBe(true);
			expect(
				matchSkill(
					{ env: { all: ["SKILLZ_TEST_EMPTY", "SKILLZ_TEST_PRESENT"] } },
					cwd,
				).matches,
			).toBe(false);
			expect(
				matchSkill(
					{ env: { all: ["SKILLZ_TEST_PRESENT"] }, os: "mac" },
					cwd,
					"linux",
				).matches,
			).toBe(false);
			for (const value of [
				"",
				"1INVALID",
				"A=B",
				["SKILLZ_TEST_PRESENT", "bad-name"],
			])
				expect(matchSkill({ env: value }, cwd).error).toBe(
					"invalid env condition",
				);
		} finally {
			for (const [index, name] of names.entries()) {
				if (previous[index] === undefined) delete process.env[name];
				else process.env[name] = previous[index];
			}
		}
	});

	it.skipIf(process.platform === "win32")(
		"checks executable files on PATH without running them",
		() => {
			const root = mkdtempSync(path.join(os.tmpdir(), "skillz-which-"));
			const first = path.join(root, "first");
			const second = path.join(root, "second");
			const originalPath = process.env.PATH;
			try {
				mkdirSync(first);
				mkdirSync(second);
				writeFileSync(path.join(first, "gh"), "");
				writeFileSync(path.join(second, "gh"), "");
				chmodSync(path.join(first, "gh"), 0o644);
				chmodSync(path.join(second, "gh"), 0o755);
				symlinkSync("gh", path.join(second, "gh-link"));
				mkdirSync(path.join(second, "directory"));
				process.env.PATH = [first, second].join(path.delimiter);
				expect(matchSkill({ which: "gh" }, root).matches).toBe(true);
				expect(matchSkill({ which: "gh-link" }, root).matches).toBe(true);
				expect(
					matchSkill({ which: { all: ["gh", "gh-link"] } }, root).matches,
				).toBe(true);
				expect(
					matchSkill({ which: { all: ["gh", "absent"] } }, root).matches,
				).toBe(false);
				expect(matchSkill({ which: "directory" }, root).matches).toBe(false);
				expect(matchSkill({ which: ["missing", "gh"] }, root).matches).toBe(
					true,
				);
				expect(matchSkill({ which: "gh", path: "missing" }, root).matches).toBe(
					false,
				);
				process.env.PATH = first;
				expect(matchSkill({ which: "gh" }, root).matches).toBe(false);
			} finally {
				if (originalPath === undefined) delete process.env.PATH;
				else process.env.PATH = originalPath;
				rmSync(root, { recursive: true, force: true });
			}
		},
	);

	it("resolves Windows PATHEXT and rejects directories", () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-win-which-"));
		const originalPaths = Object.entries(process.env).filter(
			([key]) => key.toLowerCase() === "path",
		);
		const originalPathext = process.env.PATHEXT;
		try {
			writeFileSync(path.join(root, "gh.exe"), "");
			mkdirSync(path.join(root, "directory.EXE"));
			for (const [key] of originalPaths) delete process.env[key];
			process.env.Path = `;"${root}";`;
			process.env.PATHEXT = ".EXE;.CMD";
			expect(matchSkill({ which: "gh" }, root, "win32").matches).toBe(true);
			expect(
				matchSkill({ which: path.join(root, "gh.exe") }, root, "win32").matches,
			).toBe(false);
			expect(matchSkill({ which: "directory" }, root, "win32").matches).toBe(
				false,
			);
			process.env.PATHEXT = ".CMD";
			expect(matchSkill({ which: "gh" }, root, "win32").matches).toBe(false);
		} finally {
			for (const [key] of Object.entries(process.env))
				if (key.toLowerCase() === "path") delete process.env[key];
			for (const [key, value] of originalPaths) process.env[key] = value;
			if (originalPathext === undefined) delete process.env.PATHEXT;
			else process.env.PATHEXT = originalPathext;
			rmSync(root, { recursive: true, force: true });
		}
	});

	it("matches exact cwd markers and rejects symlink components", () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-markers-"));
		try {
			writeFileSync(path.join(root, ".jj"), "");
			mkdirSync(path.join(root, "nested"));
			writeFileSync(path.join(root, "nested", "marker"), "");
			symlinkSync("nested", path.join(root, "linked"));
			expect(matchSkill({ path: ".jj" }, root).matches).toBe(true);
			expect(
				matchSkill({ path: { all: [".jj", "nested/marker"] } }, root).matches,
			).toBe(true);
			expect(
				matchSkill({ path: { all: [".jj", "absent"] } }, root).matches,
			).toBe(false);
			expect(
				matchSkill({ path: ["missing", "nested/marker"] }, root).matches,
			).toBe(true);
			expect(
				matchSkill({ path: ".jj" }, path.join(root, "nested")).matches,
			).toBe(false);
			expect(matchSkill({ path: "linked/marker" }, root).matches).toBe(false);
		} finally {
			rmSync(root, { recursive: true, force: true });
		}
	});
});

describe("skillz inspection", () => {
	it("renders a bounded navigable TUI view without writing to chat", async () => {
		initTheme("default", false);
		let closed = false;
		const rows = Array.from({ length: 25 }, (_, index) => ({
			name: `skill-${index}`,
			status: "excluded" as const,
			detail: `host mismatch ${index}`,
		}));
		const context = {
			mode: "tui",
			ui: {
				custom: async (factory: (...args: unknown[]) => unknown) => {
					const view = (await factory(
						{ requestRender: () => {} },
						{
							fg: (_color: string, text: string) => text,
							bold: (text: string) => text,
						},
						{},
						() => {
							closed = true;
						},
					)) as {
						render(width: number): string[];
						handleInput(data: string): void;
					};
					for (const width of [24, 80]) {
						const lines = view.render(width);
						expect(lines.length).toBeLessThan(20);
						expect(lines.every((line) => visibleWidth(line) <= width)).toBe(
							true,
						);
					}
					view.handleInput("\u001b[B");
					expect(view.render(80).join("\n")).toContain("host mismatch 1");
					view.handleInput("\u001b");
				},
			},
		} as unknown as ExtensionCommandContext;
		await showInspection(context, rows, true);
		expect(closed).toBe(true);
	});

	it("reports registry status by path, evidence, and env presence only", () => {
		const secret = ["skillz-secret", "DO-NOT-PRINT"].join("-");
		const previous = process.env.SKILLZ_TEST_PRESENT;
		try {
			process.env.SKILLZ_TEST_PRESENT = secret;
			const result = matchSkill(
				{ env: { all: ["SKILLZ_TEST_PRESENT"] } },
				os.tmpdir(),
			);
			const candidates = [
				{
					name: "loaded",
					path: "/skillz/loaded/SKILL.md",
					submitted: true,
					checks: result.checks,
				},
				{ name: "missing", path: "/skillz/missing/SKILL.md", submitted: true },
				{
					name: "stale",
					path: "/skillz/stale/SKILL.md",
					submitted: false,
					reason: "host mismatch",
				},
				{
					name: "collision",
					path: "/skillz/collision/SKILL.md",
					submitted: false,
					reason: "name collision",
				},
			];
			const commands = [
				{
					name: "skill:loaded",
					source: "skill" as const,
					sourceInfo: {
						path: candidates[0].path,
						source: "user",
						scope: "user" as const,
						origin: "top-level" as const,
					},
				},
				{
					name: "skill:missing",
					source: "skill" as const,
					sourceInfo: {
						path: "/native/missing/SKILL.md",
						source: "user",
						scope: "user" as const,
						origin: "top-level" as const,
					},
				},
				{
					name: "skill:stale",
					source: "skill" as const,
					sourceInfo: {
						path: "/skillz/stale/SKILL.md",
						source: "user",
						scope: "user" as const,
						origin: "top-level" as const,
					},
				},
			];
			const rows = inspectCandidates(candidates, commands);
			expect(rows.map((row) => [row.name, row.status])).toEqual([
				["collision", "excluded"],
				["loaded", "loaded"],
				["missing", "excluded"],
				["stale", "loaded"],
			]);
			expect(rows.find((row) => row.name === "loaded")?.detail).toContain(
				"SKILLZ_TEST_PRESENT present",
			);
			expect(rows.find((row) => row.name === "missing")?.detail).toContain(
				"name claimed",
			);
			expect(rows.find((row) => row.name === "stale")?.detail).toContain(
				"already registered by Pi despite skillz exclusion",
			);
			expect(JSON.stringify(rows)).not.toContain(secret);
		} finally {
			if (previous === undefined) delete process.env.SKILLZ_TEST_PRESENT;
			else process.env.SKILLZ_TEST_PRESENT = previous;
		}
	});
});

describe("skillz runtime", () => {
	let t: TestSession | undefined;
	const roots: string[] = [];
	afterEach(() => {
		t?.dispose();
		t = undefined;
		for (const root of roots.splice(0))
			rmSync(root, { recursive: true, force: true });
	});

	it("loads eligible global skills into native registry and /skill commands", async () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-runtime-"));
		roots.push(root);
		const agentDir = path.join(root, "agent");
		mkdirSync(path.join(root, ".jj"));
		skill(agentDir, "matched", "path: .jj\n");
		skill(agentDir, "wrong-host", "host: definitely-not-this-host\n");
		skill(
			agentDir,
			"manual",
			"os: [lnx, mac, win32]\ndisable-model-invocation: true\n",
		);
		skill(agentDir, "unconditional", "");
		t = await createTestSession({
			cwd: root,
			extensions: [extension],
			env: { PI_CODING_AGENT_DIR: agentDir },
		});
		const names = t.session.resourceLoader
			.getSkills()
			.skills.map((entry) => entry.name);
		expect(names).toContain("matched");
		expect(names).toContain("manual");
		expect(t.session.systemPrompt).not.toContain("<name>manual</name>");
		expect(names).not.toContain("wrong-host");
		expect(names).not.toContain("unconditional");
		expect(
			t.session.extensionRunner.extensions.map((entry) => entry.path),
		).toContain(extension);
		const expand = t.session as unknown as {
			_expandSkillCommand(text: string): string;
		};
		expect(expand._expandSkillCommand("/skill:matched")).toContain(
			'<skill name="matched"',
		);
		expect(expand._expandSkillCommand("/skill:manual")).toContain(
			'<skill name="manual"',
		);
		expect(expand._expandSkillCommand("/skill:wrong-host")).toBe(
			"/skill:wrong-host",
		);
	});

	it("writes its opt-in lifecycle trace inside the harness sandbox", async () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "pi-ext-skillz-debug-"));
		roots.push(root);
		t = await createTestSession({
			cwd: root,
			extensions: [extension],
			env: { PI_SKILLZ_DEBUG: "1" },
		});
		const runner = t.session.extensionRunner;
		await runner.emit({ type: "session_start", reason: "startup" });
		await runner.emit({ type: "session_shutdown", reason: "quit" });
		const stateHome = t.env.XDG_STATE_HOME;
		if (!stateHome) throw new Error("missing sandbox state home");
		const directory = path.join(stateHome, "pi-ext", "debug", "skillz");
		const [file] = readdirSync(directory);
		const events = readFileSync(path.join(directory, file), "utf8")
			.trim()
			.split("\n")
			.map((line) => JSON.parse(line).event);
		expect(events).toEqual(
			expect.arrayContaining(["session.start", "session.shutdown"]),
		);
	});

	it("adds no prompt text without matching skills", async () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-context-"));
		roots.push(root);
		const agentDir = path.join(root, "agent");
		skill(agentDir, "conditional", "path: marker\n");
		const env = { PI_CODING_AGENT_DIR: agentDir };
		const baseline = await createTestSession({ cwd: root, env });
		const without = baseline.session.systemPrompt;
		baseline.dispose();
		t = await createTestSession({ cwd: root, extensions: [extension], env });
		expect(t.session.systemPrompt).toBe(without);
		writeFileSync(path.join(root, "marker"), "");
		await t.session.reload();
		expect(t.session.systemPrompt.length).toBeGreaterThan(without.length);
	});

	it("reports eligible skills at startup but only warnings on reload", async () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-notify-"));
		roots.push(root);
		const agentDir = path.join(root, "agent");
		skill(agentDir, "invalid", "os: linux\n");
		t = await createTestSession({
			cwd: root,
			extensions: [extension],
			env: { PI_CODING_AGENT_DIR: agentDir },
		});
		skill(agentDir, "first", "os: [lnx, mac, win32]\n");
		await t.session.bindExtensions({ mode: "tui" });
		expect(
			t.events.uiCallsFor("notify").map((notice) => notice.args),
		).toContainEqual(["skillz: eligible: first", "info"]);
		skill(agentDir, "second", "os: [lnx, mac, win32]\n");
		t.events.ui.length = 0;
		await t.session.reload();
		const notices = t.events.uiCallsFor("notify");
		expect(notices.some((notice) => notice.args[1] === "info")).toBe(false);
		expect(notices.some((notice) => notice.args[1] === "warning")).toBe(true);
		expect(
			t.session.resourceLoader.getSkills().skills.map((skill) => skill.name),
		).toContain("second");
	});

	it("/skillz uses discovery snapshot until reload without adding model context", async () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-inspect-"));
		roots.push(root);
		const agentDir = path.join(root, "agent");
		skill(
			agentDir,
			"secret",
			"env: { all: [SKILLZ_TEST_TOKEN, SKILLZ_TEST_ENDPOINT] }\n",
		);
		skill(agentDir, "excluded", "host: never-this-host\n");
		skill(agentDir, "invalid", "env: invalid-name\n");
		const secret = ["private-token", "DO-NOT-PRINT"].join("-");
		t = await createTestSession({
			cwd: root,
			extensions: [extension],
			env: {
				PI_CODING_AGENT_DIR: agentDir,
				SKILLZ_TEST_TOKEN: secret,
				SKILLZ_TEST_ENDPOINT: "",
			},
		});
		const before = t.session.systemPrompt;
		async function report(): Promise<string> {
			const messages: string[] = [];
			const runner = t?.session.extensionRunner;
			const command = runner?.extensions[0].commands.get("skillz");
			if (!command || !runner) throw new Error("missing skillz command");
			const ctx = runner.createCommandContext();
			Object.defineProperties(ctx, {
				mode: { value: "rpc" },
				hasUI: { value: true },
				ui: { value: { notify: (message: string) => messages.push(message) } },
			});
			await command.handler("", ctx);
			return messages.join("\n");
		}
		const initial = await report();
		expect(initial).toContain(
			"excluded: secret: env all (SKILLZ_TEST_TOKEN present, SKILLZ_TEST_ENDPOINT absent)",
		);
		expect(initial).toContain("excluded: invalid: invalid env condition");
		expect(initial).toContain("excluded: excluded: host (never-this-host ✗)");
		expect(initial).not.toContain(secret);
		process.env.SKILLZ_TEST_ENDPOINT = "https://sensitive-endpoint.example";
		expect(await report()).toBe(initial);
		await t.session.reload();
		const updated = await report();
		expect(updated).toContain(
			"loaded: secret: env all (SKILLZ_TEST_TOKEN present, SKILLZ_TEST_ENDPOINT present)",
		);
		expect(updated).not.toContain(secret);
		expect(updated).not.toContain("https://sensitive-endpoint.example");
		expect(t.session.systemPrompt).not.toBe(before);
	});

	it("reevaluates markers after reload", async () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-reload-"));
		roots.push(root);
		const agentDir = path.join(root, "agent");
		skill(agentDir, "conditional", "path: marker\n");
		t = await createTestSession({
			cwd: root,
			extensions: [extension],
			env: { PI_CODING_AGENT_DIR: agentDir },
		});
		expect(
			t.session.resourceLoader
				.getSkills()
				.skills.some((entry) => entry.name === "conditional"),
		).toBe(false);
		writeFileSync(path.join(root, "marker"), "");
		await t.session.reload();
		expect(
			t.session.resourceLoader
				.getSkills()
				.skills.some((entry) => entry.name === "conditional"),
		).toBe(true);
		rmSync(path.join(root, "marker"));
		await t.session.reload();
		expect(
			t.session.resourceLoader
				.getSkills()
				.skills.some((entry) => entry.name === "conditional"),
		).toBe(false);
	});

	it("preserves native skill on name collision and follows linked skill files", async () => {
		const root = mkdtempSync(path.join(os.tmpdir(), "skillz-collision-"));
		roots.push(root);
		const agentDir = path.join(root, "agent");
		const native = path.join(root, "skills", "duplicate");
		mkdirSync(native, { recursive: true });
		writeFileSync(
			path.join(native, "SKILL.md"),
			"---\nname: duplicate\ndescription: Native skill\n---\n\n# Native\n",
		);
		skill(agentDir, "duplicate", "os: [lnx, mac, win32]\n");
		const outside = path.join(root, "outside");
		mkdirSync(outside);
		writeFileSync(
			path.join(outside, "SKILL.md"),
			"---\nname: linked\ndescription: Linked skill\nos: [lnx, mac, win32]\n---\n\n# Linked\n",
		);
		const linked = path.join(agentDir, "skillz", "linked");
		mkdirSync(linked);
		symlinkSync(path.join(outside, "SKILL.md"), path.join(linked, "SKILL.md"));
		t = await createTestSession({
			cwd: root,
			extensions: [extension],
			env: { PI_CODING_AGENT_DIR: agentDir },
		});
		const result = t.session.resourceLoader.getSkills();
		const loaded = result.skills;
		expect(result.diagnostics.some((entry) => entry.type === "collision")).toBe(
			false,
		);
		expect(loaded.filter((entry) => entry.name === "duplicate")).toHaveLength(
			1,
		);
		expect(
			loaded.find((entry) => entry.name === "duplicate")?.description,
		).toBe("Native skill");
		expect(loaded.some((entry) => entry.name === "linked")).toBe(true);
	});
});