import { mkdir, mkdtemp, readFile, rename, rm, stat, symlink, utimes, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; import { afterEach, describe, expect, it, vi } from "vitest"; import { hashArtifacts } from "../dynamic-artifacts.ts"; import { createDynamicExtensionCoordinator, type DynamicCatalogPaths, dynamicCatalogPaths, } from "../dynamic-catalog.ts"; import type { DynamicExtensionResolver, UsedDynamicExtension, } from "../dynamic-types.ts"; import { runWorkflow } from "../engine.ts"; import type { StepRunner } from "../runner.ts"; import { DYNAMIC_EXTENSION_TOOL_NAMES, parseWorkflow, type WorkflowSpec, } from "../spec.ts"; const roots: string[] = []; afterEach(async () => { await Promise.all( roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), ); }); function fakePaths(root: string): DynamicCatalogPaths { const cache = join(root, "cache"); return { catalog: join(root, "agent", "ultra", "dynamic-extensions", "catalog"), cache, runs: join(cache, "runs"), locks: join(cache, "locks"), }; } function requiredTool( tools: Map, name: string, ): ToolDefinition { const tool = tools.get(name); if (!tool) throw new Error(`missing test tool: ${name}`); return tool; } async function invoke( tool: ToolDefinition, params: unknown, signal?: AbortSignal, ): Promise { const result = await tool.execute( "call", params, signal, undefined, {} as never, ); return result.details; } describe("dynamic-extension workflow contract", () => { it("accepts dynamic selectors and isolated mutating fanout", () => { const spec = parseWorkflow({ name: "builders", phases: [ { id: "build", kind: "fanout", over: ["a", "b"], writeIsolation: "Each item owns one distinct extension topic.", step: { summary: "Build {item}", prompt: "Build only the distinct extension topic {item}.", tools: [...DYNAMIC_EXTENSION_TOOL_NAMES, "edit", "write"], dynamicExtensions: "{args.known}", }, }, ], }); expect(spec.phases[0].writeIsolation).toContain("distinct"); expect(spec.phases[0].step.dynamicExtensions).toBe("{args.known}"); }); it("rejects partial builder sets and mutating fanout without isolation", () => { const base = { name: "bad", phases: [ { id: "p", kind: "fanout", over: [1], step: { summary: "Bad", prompt: "Bad", tools: ["search_dynamic_extensions"], }, }, ], }; expect(() => parseWorkflow(base)).toThrow(/all four or none/u); base.phases[0].step.tools = ["edit"]; expect(() => parseWorkflow(base)).toThrow(/without writeIsolation/u); }); it("hashes resolved revisions into checkpoints and reports exact loaded metadata", async () => { const used: UsedDynamicExtension = { name: "probe-capability", description: "Handles probe values.", selector: `probe-capability@${"a".repeat(64)}`, sourcePath: "/snapshot/probe-capability/revision", debugLogPaths: ["/logs/child/probe-capability.jsonl"], }; const resolver: DynamicExtensionResolver = { resolve: vi.fn(async () => [ { name: used.name, description: used.description, revision: "a".repeat(64), selector: used.selector, canonicalPath: "/catalog/probe-capability/revision", sourcePath: used.sourcePath, loadPath: "/snapshot/index.ts", overrides: [], }, ]), builder: vi.fn(async () => ({ tools: [], prompt: "", activeTools: [] })), loaded: vi.fn(() => [used]), complete: vi.fn(async () => {}), }; const calls: Parameters[0][] = []; const runner: StepRunner = async (args) => { calls.push(args); return { ok: true, value: { done: true }, dynamicExtensions: [used] }; }; const spec: WorkflowSpec = { name: "consumer", phases: [ { id: "consume", kind: "single", step: { summary: "Consume", prompt: "Consume", dynamicExtensions: ["probe-capability"], }, }, ], }; const result = await runWorkflow(spec, undefined, { stepRunner: runner, concurrency: 1, dynamicExtensions: resolver, }); expect(calls[0].dynamicExtensions?.[0].revision).toBe("a".repeat(64)); expect(JSON.parse(calls[0].sessionKey).dynamicExtensions).toEqual([ { name: "probe-capability", revision: "a".repeat(64) }, ]); expect(result.dynamicExtensions).toEqual([used]); expect(resolver.complete).toHaveBeenCalledOnce(); }); it("preserves dynamic metadata from cached steps without charging usage", async () => { const used: UsedDynamicExtension = { name: "cached-capability", description: "Handles cached values.", selector: "cached-capability@revision", sourcePath: "/snapshot/cached-capability/revision", debugLogPaths: ["/logs/child/cached-capability.jsonl"], }; const resolver: DynamicExtensionResolver = { resolve: async () => [ { name: used.name, description: used.description, revision: "revision", selector: used.selector, canonicalPath: "/catalog/cached-capability/revision", sourcePath: used.sourcePath, loadPath: "/snapshot/index.ts", overrides: [], }, ], builder: async () => ({ tools: [], prompt: "", activeTools: [] }), loaded: () => [used], complete: async () => {}, }; const runner = vi.fn(); const result = await runWorkflow( { name: "cached", phases: [ { id: "consume", kind: "single", step: { summary: "Consume", prompt: "Consume", dynamicExtensions: ["cached-capability"], }, }, ], }, undefined, { stepRunner: runner, concurrency: 1, dynamicExtensions: resolver, journal: { get: async () => ({ ok: true, value: { cached: true }, dynamicExtensions: [used], usage: { input: 100, output: 20, total: 120, cacheRead: 0, cacheWrite: 0, cost: 1, }, }), set: async () => {}, }, }, ); expect(runner).not.toHaveBeenCalled(); expect(result.dynamicExtensions).toEqual([used]); expect(result.tokenUsage.total).toBe(0); }); it("turns every later phase into a typed blocker after builder failure", async () => { const calls: string[] = []; const runner: StepRunner = async (args) => { calls.push(args.phase); return { ok: false, value: null, failure: { code: "session", message: "builder failed", retryable: true, attempts: 1, }, }; }; const result = await runWorkflow( { name: "blocked", phases: [ { id: "build", kind: "single", step: { summary: "Build", prompt: "Build", tools: [...DYNAMIC_EXTENSION_TOOL_NAMES], }, }, { id: "consume", kind: "single", step: { summary: "Consume", prompt: "Consume" }, }, ], }, undefined, { stepRunner: runner, concurrency: 1 }, ); expect(calls).toEqual(["build"]); expect(result.phaseFailures.consume[0].code).toBe( "dynamic_extension_builder_failure", ); expect(result.phases[1]).toEqual({ id: "consume", ran: 0, ok: 0, dropped: 1, }); }); }); describe("dynamic-extension catalog", () => { it("uses the specified platform cache roots", () => { expect( dynamicCatalogPaths({ agentDir: "/agent", platform: "win32", env: { LOCALAPPDATA: "C:/cache" }, home: "C:/home", }).cache.replaceAll("\\", "/"), ).toBe("C:/cache/pi/ultra/dynamic-extensions"); expect( dynamicCatalogPaths({ agentDir: "/agent", platform: "darwin", env: {}, home: "/home/oliver", }).cache, ).toBe("/home/oliver/Library/Caches/pi/ultra/dynamic-extensions"); expect( dynamicCatalogPaths({ agentDir: "/agent", platform: "linux", env: { XDG_CACHE_HOME: "/xdg" }, home: "/home/oliver", }).cache, ).toBe("/xdg/pi/ultra/dynamic-extensions"); }); it("ranks search deterministically and returns at most five matches", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-search-test-")); roots.push(root); const paths = fakePaths(root); for (const suffix of ["zeta", "beta", "alpha", "delta", "gamma", "eta"]) { const name = `probe-${suffix}-capability`; const staging = join(paths.catalog, name, "staging"); await mkdir(staging, { recursive: true }); const manifest = { schemaVersion: 1, name, description: `Handles ${suffix} probe values.`, topic: "probe value handling", scope: "Probe values.", nonGoals: ["Network access."], tags: ["probe"], capabilities: [`${name}-core`], requiredExecutables: [], optionalExecutables: [], overrides: [], benchmarkBudgets: { [process.platform]: { hotPathMs: 10 } }, }; await Promise.all([ writeFile( join(staging, "manifest.json"), `${JSON.stringify(manifest)}\n`, ), writeFile( join(staging, "README.md"), `# ${name}\n\n## Purpose\n\nProbe values.\n\n## Scope\n\nProbe values.\n\n## Usage\n\nUse probe values.\n`, ), ]); const revision = hashArtifacts(staging); const source = join(paths.catalog, name, revision); await rename(staging, source); await Promise.all([ writeFile( join(source, ".ultra-validation.json"), JSON.stringify({ schemaVersion: 1, validatorContractVersion: 2, sourceHash: revision, validatedAt: new Date().toISOString(), platform: process.platform, testedPlatforms: [process.platform], nodeVersion: process.version, warningTargetMs: 5_000, gates: [], }), ), writeFile( join(paths.catalog, name, "current.json"), JSON.stringify({ schemaVersion: 1, name, revision, description: manifest.description, topic: manifest.topic, publishedAt: new Date().toISOString(), }), ), ]); } const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "search-run", availableTools: ["read"], paths, }); const builder = await coordinator.builder("search#0"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); const details = (await invoke( requiredTool(tools, "search_dynamic_extensions"), { query: "probe" }, )) as { matches: Array<{ name: string; integrityStatus: string }>; }; expect(details.matches).toHaveLength(5); expect(details.matches[0].integrityStatus).toBe("intact"); expect(details.matches.map((match) => match.name)).toEqual([ "probe-alpha-capability", "probe-beta-capability", "probe-delta-capability", "probe-eta-capability", "probe-gamma-capability", ]); }); it("aborts the validator process tree and retains its draft", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-abort-test-")); roots.push(root); const workerPath = join(root, "hanging-validator.mjs"); await writeFile( workerPath, `import { spawn } from "node:child_process";\nimport { writeFileSync } from "node:fs";\nimport { dirname, join } from "node:path";\nconst child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });\nwriteFileSync(join(dirname(process.argv[2]), "tree-child.pid"), String(child.pid));\nsetInterval(() => {}, 1000);\n`, "utf8", ); const paths = fakePaths(root); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "abort-run", availableTools: ["read", "edit"], paths, workerPath, }); const builder = await coordinator.builder("build#0"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); await invoke(requiredTool(tools, "search_dynamic_extensions"), { query: "abort probe", }); const created = (await invoke( requiredTool(tools, "create_dynamic_extension"), { name: "abort-probe-capability", description: "Handles deterministic abort probe values.", topic: "abort probe value handling", scope: "Transform abort probe values deterministically.", nonGoals: ["Network access."], tags: ["abort", "probe"], consideredCandidates: [], noMatchRationale: "The catalog has no abort probe capability.", }, )) as { workingCopyPath: string }; const controller = new AbortController(); const validationPromise = invoke( requiredTool(tools, "validate_dynamic_extension"), { path: created.workingCopyPath }, controller.signal, ); const pidPath = join(created.workingCopyPath, "..", "tree-child.pid"); for (let attempt = 0; attempt < 100; attempt++) { try { await readFile(pidPath, "utf8"); break; } catch { await new Promise((resolveDelay) => setTimeout(resolveDelay, 20)); } } const childPid = Number(await readFile(pidPath, "utf8")); controller.abort(); const validation = (await validationPromise) as { status: string; workingCopyPath: string; }; expect(validation).toMatchObject({ status: "aborted", workingCopyPath: created.workingCopyPath, }); await new Promise((resolveDelay) => setTimeout(resolveDelay, 100)); expect(() => process.kill(childPid, 0)).toThrow(); expect(await stat(created.workingCopyPath)).toBeDefined(); }, 10_000); it("rejects a malformed validator result without crashing publication", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-protocol-test-")); roots.push(root); const workerPath = join(root, "malformed-validator.mjs"); await writeFile( workerPath, 'console.log("ULTRA_VALIDATION_RESULT=null");\n', "utf8", ); const paths = fakePaths(root); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "protocol-run", availableTools: ["read"], paths, workerPath, }); const builder = await coordinator.builder("build#0"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); await invoke(requiredTool(tools, "search_dynamic_extensions"), { query: "protocol probe", }); const created = (await invoke( requiredTool(tools, "create_dynamic_extension"), { name: "protocol-probe-capability", description: "Handles malformed validator protocol probes.", topic: "malformed validator protocol handling", scope: "Reject malformed validator protocol results safely.", nonGoals: ["Network access."], tags: ["protocol", "probe"], consideredCandidates: [], noMatchRationale: "The catalog has no protocol probe capability.", }, )) as { workingCopyPath: string }; const validation = (await invoke( requiredTool(tools, "validate_dynamic_extension"), { path: created.workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(validation.status).toBe("failed"); expect(validation.diagnostics).toEqual( expect.arrayContaining([ expect.objectContaining({ code: "validator_protocol" }), ]), ); const hash = hashArtifacts(created.workingCopyPath); const manifest = JSON.parse( await readFile(join(created.workingCopyPath, "manifest.json"), "utf8"), ); await writeFile( workerPath, `import { writeFileSync } from "node:fs";\nwriteFileSync(process.env.ULTRA_VALIDATOR_RESULT_PATH + ".preflight", ${JSON.stringify(JSON.stringify({ ok: true, diagnostics: [], manifest, hash, attestation: { sourceHash: hash }, durationMs: 0 }))});\n`, "utf8", ); const interrupted = (await invoke( requiredTool(tools, "validate_dynamic_extension"), { path: created.workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(interrupted.status).toBe("failed"); expect(interrupted.diagnostics.map((item) => item.code)).toContain( "validator_protocol", ); }); it("searches, scaffolds, validates, publishes, and resolves an exact snapshot", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-dynamic-test-")); roots.push(root); const paths = fakePaths(root); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "run", availableTools: ["read", "edit", "run_workflow"], paths, }); const builder = await coordinator.builder("build#0"); expect(builder.activeTools).toContain("read"); expect(builder.activeTools).toContain("edit"); expect(builder.activeTools).not.toContain("run_workflow"); expect(builder.prompt).toContain("dynamic-extension builder guidance"); const byName = new Map(builder.tools.map((tool) => [tool.name, tool])); const search = await invoke( requiredTool(byName, "search_dynamic_extensions"), { query: "probe values", }, ); expect(search).toEqual({ matches: [] }); const created = (await invoke( requiredTool(byName, "create_dynamic_extension"), { name: "probe-capability", description: "Handles deterministic probe values.", topic: "deterministic probe value handling", scope: "Transform one probe value into deterministic structured output.", nonGoals: ["Network access."], tags: ["probe", "deterministic"], consideredCandidates: [], noMatchRationale: "The catalog contains no probe capability.", }, )) as { name: string; workingCopyPath: string }; expect(created.name).toBe("probe-capability"); await expect( stat(join(created.workingCopyPath, "debug.ts")), ).rejects.toThrow(); await expect( stat(join(created.workingCopyPath, "benchmark.ts")), ).rejects.toThrow(); const validation = (await invoke( requiredTool(byName, "validate_dynamic_extension"), { path: created.workingCopyPath }, )) as { status: string; name: string; diagnostics: Array<{ code: string; status: string }>; }; expect(validation, JSON.stringify(validation, null, 2)).toMatchObject({ status: "validated", name: "probe-capability", }); expect(validation.diagnostics.some((item) => item.status === "error")).toBe( false, ); const resolved = await coordinator.resolve(["probe-capability"], { phase: "consume", agentId: "consume#0", }); expect(resolved).toHaveLength(1); expect(resolved[0].selector).toBe( `probe-capability@${resolved[0].revision}`, ); expect(hashPath(resolved[0].loadPath)).toBe(resolved[0].revision); const used = coordinator.loaded(resolved, "child-session"); expect(used[0].debugLogPaths).toEqual([]); const canonicalReadme = join(resolved[0].canonicalPath, "README.md"); await writeFile( canonicalReadme, `${await readFile(canonicalReadme, "utf8")}\nManual repair evidence.\n`, "utf8", ); await expect( coordinator.resolve(["probe-capability"], { phase: "consume", agentId: "consume#1", }), ).rejects.toThrow(/call validate_dynamic_extension/u); const repair = await coordinator.builder("repair#0"); const repairTools = new Map(repair.tools.map((tool) => [tool.name, tool])); const validateRepair = requiredTool( repairTools, "validate_dynamic_extension", ); const repaired = (await invoke(validateRepair, { path: resolved[0].canonicalPath, })) as Record; expect(repaired, JSON.stringify(repaired, null, 2)).toMatchObject({ status: "validated", name: "probe-capability", }); expect(repaired.revision).not.toBe(resolved[0].revision); const brokenBuilder = await coordinator.builder("broken#0"); const brokenTools = new Map( brokenBuilder.tools.map((tool) => [tool.name, tool]), ); const brokenCheckout = (await invoke( requiredTool(brokenTools, "copy_dynamic_extension"), { name: "probe-capability" }, )) as { workingCopyPath: string }; await Promise.all([ writeFile( join(brokenCheckout.workingCopyPath, "README.md"), "# Broken\n", ), writeFile( join(brokenCheckout.workingCopyPath, "package.json"), JSON.stringify({ private: true, type: "module", main: "index.ts", dependencies: { dependency: "1.0.0" }, peerDependencies: { "@earendil-works/pi-coding-agent": "*", }, }), ), writeFile( join(brokenCheckout.workingCopyPath, "index.test.ts"), 'import test from "node:test";\ntest("probe-capability-core: broken", () => { throw new Error("broken"); });\n', ), ]); const changedManifest = JSON.parse( await readFile( join(brokenCheckout.workingCopyPath, "manifest.json"), "utf8", ), ); changedManifest.topic = "new responsibility"; changedManifest.benchmarkBudgets = { [process.platform]: { hotPathMs: 1_000 }, }; await writeFile( join(brokenCheckout.workingCopyPath, "manifest.json"), JSON.stringify(changedManifest), ); const broken = (await invoke( requiredTool(brokenTools, "validate_dynamic_extension"), { path: brokenCheckout.workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }>; workingCopyPath: string; }; expect(broken.status).toBe("validated"); expect(broken.diagnostics.map((item) => item.code)).toContain( "tests_failed", ); await writeFile( join(brokenCheckout.workingCopyPath, "core.ts"), "export const fixed = true;\n", ); const retried = (await invoke( requiredTool(brokenTools, "validate_dynamic_extension"), { path: brokenCheckout.workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(retried.status).toBe("validated"); expect(retried.diagnostics.map((item) => item.code)).toContain( "tests_failed", ); await coordinator.complete(); await expect(stat(resolved[0].sourcePath)).rejects.toThrow(); expect(await stat(brokenCheckout.workingCopyPath)).toBeDefined(); }, 60_000); it("preserves copies, advances only a published base, and exposes a bounded recovery handoff", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-copies-test-")); roots.push(root); const options = { agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "run", availableTools: ["read", "write"], paths: fakePaths(root), }; const coordinator = createDynamicExtensionCoordinator(options); const builder = await coordinator.builder("builder#0"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); const create = requiredTool(tools, "create_dynamic_extension"); const copy = requiredTool(tools, "copy_dynamic_extension"); const validate = requiredTool(tools, "validate_dynamic_extension"); const input = { name: "probe-capability", description: "Probe capability.", }; const first = (await invoke(create, input)) as { workingCopyPath: string }; await writeFile( join(first.workingCopyPath, "core.ts"), "export const original = true;\n", ); const second = (await invoke(create, input)) as { workingCopyPath: string }; expect(second.workingCopyPath).not.toBe(first.workingCopyPath); expect( await readFile(join(first.workingCopyPath, "core.ts"), "utf8"), ).toContain("original"); const published = (await invoke(validate, { path: first.workingCopyPath, })) as { status: string; revision: string }; expect(published.status).toBe("validated"); const stale = (await invoke(validate, { path: second.workingCopyPath, })) as { status: string; workingCopyPath: string }; expect(stale).toMatchObject({ status: "stale", workingCopyPath: second.workingCopyPath, }); const third = (await invoke(copy, { name: input.name })) as { workingCopyPath: string; }; await writeFile( join(third.workingCopyPath, "core.ts"), "export const independent = true;\n", ); const fourth = (await invoke(copy, { name: input.name })) as { workingCopyPath: string; }; expect(fourth.workingCopyPath).not.toBe(third.workingCopyPath); expect( await readFile(join(third.workingCopyPath, "core.ts"), "utf8"), ).toContain("independent"); await writeFile( join(first.workingCopyPath, "core.ts"), "export const next = true;\n", ); const republished = (await invoke(validate, { path: first.workingCopyPath, })) as { status: string; revision: string }; expect(republished.status).toBe("validated"); expect(republished.revision).not.toBe(published.revision); const competing = (await invoke(validate, { path: third.workingCopyPath, })) as { status: string; workingCopyPath: string }; expect(competing).toMatchObject({ status: "stale", workingCopyPath: third.workingCopyPath, }); const recovered = (await invoke(copy, { name: input.name })) as { workingCopyPath: string; }; expect(recovered.workingCopyPath).not.toBe(third.workingCopyPath); const toolResult = await validate.execute( "call", { path: third.workingCopyPath }, undefined, undefined, {} as never, ); const handoff = JSON.parse( (toolResult.content[0] as { text: string }).text, ); expect(handoff).toMatchObject({ status: "stale", name: input.name, workingCopyPath: third.workingCopyPath, }); expect(handoff.diagnostics[0]).toHaveProperty("nextAction"); await coordinator.complete(); const restarted = createDynamicExtensionCoordinator(options); const again = await restarted.builder("builder#0"); expect(again.tools.map((tool) => tool.name)).toContain( "copy_dynamic_extension", ); expect( await readFile(join(third.workingCopyPath, "core.ts"), "utf8"), ).toContain("independent"); expect( await readFile(join(first.workingCopyPath, "core.ts"), "utf8"), ).toContain("next"); const old = new Date(0); await utimes(join(options.paths.runs, "parent"), old, old); const later = createDynamicExtensionCoordinator({ ...options, workflowRunId: "later", }); await later.builder("later#0"); expect( await readFile(join(first.workingCopyPath, "core.ts"), "utf8"), ).toContain("next"); }, 60_000); it("keeps a blocked structured builder's working copy after workflow completion", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-blocked-copy-test-")); roots.push(root); const options = { agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "blocked", availableTools: ["read"], paths: fakePaths(root), }; const coordinator = createDynamicExtensionCoordinator(options); let path = ""; const result = await runWorkflow( { name: "blocked-copy", phases: [ { id: "build", kind: "single", step: { summary: "Build", prompt: "Build", tools: [...DYNAMIC_EXTENSION_TOOL_NAMES], }, }, ], }, undefined, { stepRunner: async (args) => { const builder = await coordinator.builder(args.agentId); const tool = builder.tools.find( (item) => item.name === "create_dynamic_extension", ); if (!tool) throw new Error("missing builder tool"); path = ( (await invoke(tool, { name: "blocked-capability", description: "Unfinished work.", })) as { workingCopyPath: string } ).workingCopyPath; return { ok: true, value: { status: "blocked", workingCopyPath: path }, }; }, concurrency: 1, dynamicExtensions: coordinator, }, ); expect(result.phases[0].ok).toBe(1); expect(result.phaseResults.build[0]).toMatchObject({ ok: true, value: { status: "blocked", workingCopyPath: path }, }); expect(await stat(path)).toBeDefined(); const reconstructed = createDynamicExtensionCoordinator(options); await reconstructed.builder("build#0"); expect(await stat(path)).toBeDefined(); }, 15_000); it("resolves old catalog artifacts regardless of validator metadata", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-v2-test-")); roots.push(root); const paths = fakePaths(root); const name = "existing-capability"; const staging = join(paths.catalog, name, "staging"); await mkdir(staging, { recursive: true }); const manifest = { schemaVersion: 1, name, description: "Existing capability.", topic: "old topic", scope: "old scope", nonGoals: ["Other tasks"], tags: ["existing"], capabilities: ["existing-capability-core"], requiredExecutables: [], optionalExecutables: [], overrides: [], benchmarkBudgets: { [process.platform]: { hotPathMs: 10 } }, }; await writeFile(join(staging, "manifest.json"), JSON.stringify(manifest)); await writeFile( join(staging, "index.ts"), "export default function extension() {}\n", ); await writeFile( join(staging, "debug.ts"), "// Existing debug content is not rewritten.\n", ); const revision = hashArtifacts(staging); const canonical = join(paths.catalog, name, revision); await rename(staging, canonical); await writeFile( join(canonical, ".ultra-validation.json"), JSON.stringify({ schemaVersion: 1, validatorContractVersion: 999, sourceHash: revision, validatedAt: new Date().toISOString(), platform: process.platform, testedPlatforms: [process.platform], nodeVersion: process.version, warningTargetMs: 5_000, gates: [ "manifest", "readme", "package", "debug-template", "artifacts", "tests", "benchmark", "real-loader", "hash", ], }), ); await writeFile( join(paths.catalog, name, "current.json"), JSON.stringify({ schemaVersion: 1, name, revision, description: manifest.description, topic: manifest.topic, publishedAt: new Date().toISOString(), }), ); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "v2", availableTools: [], paths, }); const resolved = await coordinator.resolve([name], { phase: "consume", agentId: "child", }); expect(resolved[0].revision).toBe(revision); expect(await readFile(join(canonical, "debug.ts"), "utf8")).toBe( "// Existing debug content is not rewritten.\n", ); expect( await readFile(join(resolved[0].sourcePath, "debug.ts"), "utf8"), ).toBe("// Existing debug content is not rewritten.\n"); await rm(join(canonical, ".ultra-validation.json")); const stillAvailable = await coordinator.resolve([name], { phase: "consume", agentId: "next-child", }); expect(stillAvailable[0].revision).toBe(revision); const builder = await coordinator.builder("update#0"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); await invoke(requiredTool(tools, "search_dynamic_extensions"), { query: name, }); const checkout = (await invoke( requiredTool(tools, "copy_dynamic_extension"), { name }, )) as { workingCopyPath: string }; expect( await readFile(join(checkout.workingCopyPath, "debug.ts"), "utf8"), ).toContain("Existing debug content"); const reimagined = { ...manifest, description: "Reimagined capability.", topic: "new purpose", }; await writeFile( join(checkout.workingCopyPath, "manifest.json"), JSON.stringify(reimagined), ); const updated = (await invoke( requiredTool(tools, "validate_dynamic_extension"), { path: checkout.workingCopyPath }, )) as { status: string }; expect(updated.status).toBe("validated"); const next = await coordinator.resolve([name], { phase: "consume", agentId: "after-update", }); expect(next[0].description).toBe(reimagined.description); }); it("publishes minimal metadata without README or tests and reports failed load/tests", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-minimal-test-")); roots.push(root); const paths = fakePaths(root); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "minimal", availableTools: ["read"], paths, }); const builder = await coordinator.builder("build#0"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); const { workingCopyPath } = (await invoke( requiredTool(tools, "create_dynamic_extension"), { name: "minimal", description: "Minimal extension" }, )) as { workingCopyPath: string }; await rm(join(workingCopyPath, "README.md")); await rm(join(workingCopyPath, "package.json")); await rm(join(workingCopyPath, "core.ts")); await writeFile( join(workingCopyPath, "manifest.json"), JSON.stringify({ name: "minimal", description: "Minimal extension" }), ); await writeFile( join(workingCopyPath, "index.ts"), "export default function extension() {}\n", ); const validation = (await invoke( requiredTool(tools, "validate_dynamic_extension"), { path: workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(validation.status, JSON.stringify(validation)).toBe("validated"); const matches = (await invoke( requiredTool(tools, "search_dynamic_extensions"), { query: "minimal" }, )) as { matches: Array<{ name: string }> }; expect(matches.matches[0].name).toBe("minimal"); const resolved = await coordinator.resolve(["minimal"], { phase: "use", agentId: "use#0", }); expect(resolved[0].overrides).toEqual([]); expect(coordinator.loaded(resolved, "child")[0].debugLogPaths).toEqual([]); const checkout = await coordinator.builder("repair#0"); const repairTools = new Map( checkout.tools.map((tool) => [tool.name, tool]), ); const { workingCopyPath: brokenPath } = (await invoke( requiredTool(repairTools, "copy_dynamic_extension"), { name: "minimal" }, )) as { workingCopyPath: string }; await writeFile( join(brokenPath, "index.ts"), 'import "./missing.ts"; export default function extension() {}\n', ); await writeFile( join(brokenPath, "other.test.ts"), 'import test from "node:test"; test("broken", () => { throw new Error("failed") });\n', ); const broken = (await invoke( requiredTool(repairTools, "validate_dynamic_extension"), { path: brokenPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(broken.status).toBe("validated"); expect(broken.diagnostics.map((item) => item.code)).toEqual( expect.arrayContaining(["extension_load", "tests_failed"]), ); }); it("reports a hanging load without losing structural publication", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-timeout-test-")); roots.push(root); const paths = fakePaths(root); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "p", workflowRunId: "timeout", availableTools: [], paths, }); const builder = await coordinator.builder("b"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); await invoke(requiredTool(tools, "search_dynamic_extensions"), { query: "hang", }); const { workingCopyPath } = (await invoke( requiredTool(tools, "create_dynamic_extension"), { name: "hang", description: "Hanging load", consideredCandidates: [], noMatchRationale: "None", }, )) as { workingCopyPath: string }; await writeFile( join(workingCopyPath, "index.ts"), "while (true) {}\nexport default function extension() {}\n", ); const result = (await invoke( requiredTool(tools, "validate_dynamic_extension"), { path: workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(result.status, JSON.stringify(result)).toBe("validated"); expect(result.diagnostics.map((item) => item.code)).toContain( "validator_timeout", ); }, 25_000); it("rejects symlink artifacts and reserved tool registrations", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-safety-test-")); roots.push(root); const paths = fakePaths(root); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "p", workflowRunId: "r", availableTools: [], paths, }); const builder = await coordinator.builder("b"); const tools = new Map(builder.tools.map((tool) => [tool.name, tool])); await invoke(requiredTool(tools, "search_dynamic_extensions"), { query: "safety", }); const { workingCopyPath } = (await invoke( requiredTool(tools, "create_dynamic_extension"), { name: "safety", description: "Safety extension", consideredCandidates: [], noMatchRationale: "None", }, )) as { workingCopyPath: string }; await symlink( join(workingCopyPath, "index.ts"), join(workingCopyPath, "linked.ts"), ); const linked = (await invoke( requiredTool(tools, "validate_dynamic_extension"), { path: workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(linked.status).toBe("failed"); expect(linked.diagnostics.map((item) => item.code)).toContain( "artifact_symlink", ); await rm(join(workingCopyPath, "linked.ts")); await writeFile( join(workingCopyPath, "index.ts"), `export default function extension(pi) { pi.registerTool({ name: "run_workflow", label: "bad", description: "bad", parameters: { type: "object", properties: {} }, execute: async () => ({ content: [], details: {} }) }); }\n`, ); const reserved = (await invoke( requiredTool(tools, "validate_dynamic_extension"), { path: workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(reserved.status).toBe("failed"); expect(reserved.diagnostics.map((item) => item.code)).toContain( "generated_tool_reserved", ); await writeFile( join(workingCopyPath, "index.ts"), "export default function extension() {}\n", ); const manifest = JSON.parse( await readFile(join(workingCopyPath, "manifest.json"), "utf8"), ); manifest.name = "different-name"; await writeFile( join(workingCopyPath, "manifest.json"), JSON.stringify(manifest), ); const otherBuilder = await coordinator.builder("other"); const otherTools = new Map( otherBuilder.tools.map((tool) => [tool.name, tool]), ); const identity = (await invoke( requiredTool(otherTools, "validate_dynamic_extension"), { path: workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(identity.status).toBe("failed"); expect(identity.diagnostics.map((item) => item.code)).toContain( "extension_identity_changed", ); const fakeWorker = join(root, "wrong-hash.mjs"); await writeFile( fakeWorker, `import { writeFileSync } from "node:fs"; const hash = "f".repeat(64); writeFileSync(process.env.ULTRA_VALIDATOR_RESULT_PATH, JSON.stringify({ ok: true, diagnostics: [], manifest: { schemaVersion: 1, name: "different-name", description: "Safety extension" }, hash, attestation: { schemaVersion: 1, validatorContractVersion: 2, sourceHash: hash, testedPlatforms: [process.platform] }, durationMs: 0 }));`, ); const fake = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "p", workflowRunId: "fake", availableTools: [], paths, workerPath: fakeWorker, }); const fakeBuilder = await fake.builder("fake"); const fakeTools = new Map( fakeBuilder.tools.map((tool) => [tool.name, tool]), ); const mismatch = (await invoke( requiredTool(fakeTools, "validate_dynamic_extension"), { path: workingCopyPath }, )) as { status: string; diagnostics: Array<{ code: string }> }; expect(mismatch.status).toBe("failed"); expect(mismatch.diagnostics.map((item) => item.code)).toContain( "validation_identity_or_hash", ); }); it("serializes concurrent publication and rejects the stale builder without clobbering", async () => { const root = await mkdtemp(join(tmpdir(), "ultra-stale-test-")); roots.push(root); const paths = fakePaths(root); const workerPath = join(root, "successful-validator.mjs"); await writeFile( workerPath, `import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { hashArtifacts } from ${JSON.stringify(new URL("../dynamic-artifacts.ts", import.meta.url).href)}; const workingCopyPath = process.argv[2]; const manifest = JSON.parse(await readFile(join(workingCopyPath, "manifest.json"), "utf8")); const hash = hashArtifacts(workingCopyPath); const result = { ok: true, diagnostics: [], manifest, hash, attestation: { sourceHash: hash, validatedAt: new Date().toISOString(), platform: process.platform, testedPlatforms: [process.platform], nodeVersion: process.version, warningTargetMs: 5_000, gates: [], }, durationMs: 0, }; await writeFile(process.env.ULTRA_VALIDATOR_RESULT_PATH, JSON.stringify(result), "utf8"); `, "utf8", ); const coordinator = createDynamicExtensionCoordinator({ agentDir: join(root, "agent"), parentSessionId: "parent", workflowRunId: "stale-run", availableTools: ["read", "edit"], paths, workerPath, }); const initial = await coordinator.builder("initial#0"); const initialTools = new Map( initial.tools.map((tool) => [tool.name, tool]), ); await invoke(requiredTool(initialTools, "search_dynamic_extensions"), { query: "stale probe", }); const created = (await invoke( requiredTool(initialTools, "create_dynamic_extension"), { name: "stale-probe-capability", description: "Handles deterministic stale probe values.", topic: "stale probe value handling", scope: "Transform stale probe values deterministically.", nonGoals: ["Network access."], tags: ["stale", "probe"], consideredCandidates: [], noMatchRationale: "The catalog has no stale probe capability.", }, )) as { workingCopyPath: string }; const initialPublication = (await invoke( requiredTool(initialTools, "validate_dynamic_extension"), { path: created.workingCopyPath }, )) as Record; expect(initialPublication.status).toBe("validated"); const left = await coordinator.builder("left#0"); const right = await coordinator.builder("right#0"); const leftTools = new Map(left.tools.map((tool) => [tool.name, tool])); const rightTools = new Map(right.tools.map((tool) => [tool.name, tool])); const [leftCheckout, rightCheckout] = (await Promise.all([ invoke(requiredTool(leftTools, "copy_dynamic_extension"), { name: "stale-probe-capability", }), invoke(requiredTool(rightTools, "copy_dynamic_extension"), { name: "stale-probe-capability", }), ])) as Array<{ workingCopyPath: string }>; for (const [checkout, marker] of [ [leftCheckout, "left"], [rightCheckout, "right"], ] as const) { const readme = join(checkout.workingCopyPath, "README.md"); await writeFile( readme, `${await readFile(readme, "utf8")}\n${marker} publication.\n`, "utf8", ); } const publications = (await Promise.all([ invoke(requiredTool(leftTools, "validate_dynamic_extension"), { path: leftCheckout.workingCopyPath, }), invoke(requiredTool(rightTools, "validate_dynamic_extension"), { path: rightCheckout.workingCopyPath, }), ])) as Array>; expect( publications.map((value) => value.status).sort(), JSON.stringify(publications, null, 2), ).toEqual(["stale", "validated"]); const canonical = await coordinator.resolve(["stale-probe-capability"], { phase: "consume", agentId: "consume#0", }); expect(canonical[0].revision).toBe( publications.find((value) => value.status === "validated")?.revision, ); }, 20_000); }); function hashPath(indexPath: string): string { return indexPath.split(/[\\/]/u).at(-2) ?? ""; }