repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
scripts/context-extensions.test.ts
Rawimport {
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { getCurrentTools } from "@earendil-works/pi-ai";
import {
DefaultResourceLoader,
SettingsManager,
} from "@earendil-works/pi-coding-agent";
import { Type } from "typebox";
import { afterEach, expect, it, vi } from "vitest";
import { createCodemodeDescription } from "../node_modules/@earendil-works/pi-coding-agent/dist/extensions/codemode/tool.js";
import {
formatExtensionOverview,
measureExtensions,
registrationSignals,
} from "./context-extensions.ts";
import {
inspectCodemodeCatalog,
inspectExtensionRegistrations,
resourceOptions,
} from "./context-startup-session.ts";
let directory: string | undefined;
afterEach(() => {
vi.unstubAllEnvs();
if (directory) rmSync(directory, { recursive: true, force: true });
directory = undefined;
});
const emptyCatalogs = [false, true].map((models) =>
createCodemodeDescription([], { models }),
);
const catalogTool = (description: string) => ({
name: "codemode",
description,
parameters: Type.Object({}),
});
it("reads actual Pi catalogs, distinguishing absent, empty, budget-hidden, and unsupported formats", () => {
expect(inspectCodemodeCatalog([], emptyCatalogs)).toEqual([]);
for (const description of emptyCatalogs)
expect(
inspectCodemodeCatalog([catalogTool(description)], emptyCatalogs),
).toEqual([]);
expect(
inspectCodemodeCatalog([catalogTool("Unknown renderer")], emptyCatalogs),
).toBeUndefined();
const tool = {
name: "raw-name",
label: "Example",
description: "Example tool",
parameters: Type.Object({}),
async execute() {
return { content: [], details: undefined };
},
};
const description = createCodemodeDescription([tool]);
// Replay the request's final declaration, not its stale initial loadout.
const request = [
{
role: "system",
content: "",
toolsAdded: [catalogTool(emptyCatalogs[0]!)],
},
{ role: "system", content: "", toolsAdded: [catalogTool(description)] },
];
const declarations = getCurrentTools(request);
expect(inspectCodemodeCatalog(declarations, emptyCatalogs)).toEqual([
"raw-name",
]);
expect(
inspectCodemodeCatalog(
[catalogTool(createCodemodeDescription([tool], { inlineBudget: 0 }))],
emptyCatalogs,
),
).toEqual([]);
});
it("observes budget-driven replacements from Pi-rendered descriptions without simulating selection", () => {
const tools = [
["clipboard", "c"],
["activity", "a".repeat(800)],
["bak", "b".repeat(1200)],
].map(([name, description]) => ({
name: name!,
description: description!,
label: name!,
parameters: Type.Object({}),
async execute() {
return { content: [], details: undefined };
},
}));
const options = {
inlineBudget: 600,
namespaces: new Map([["activity", { name: "activity" }]]),
};
const read = (omitted?: string) =>
inspectCodemodeCatalog(
[
catalogTool(
createCodemodeDescription(
tools.filter((tool) => tool.name !== omitted),
options,
),
),
],
emptyCatalogs,
);
expect(read()).toEqual(["activity", "clipboard"]);
expect(read("activity")).toEqual(["bak", "clipboard"]);
expect(read("clipboard")).toEqual(["activity", "bak"]);
});
it("inventories actual maps, provider ownership, and future registrations without configs or tool bodies", () => {
const extension = (path: string) => ({
path,
tools: new Map(),
handlers: new Map(),
});
const one = extension("/extensions/one/index.ts");
one.tools.set("example", { description: "PRIVATE BODY" });
one.handlers.set("future_event", [() => {}]);
const result = {
extensions: [
one,
extension("/extensions/new.ts"),
extension("<inline:measurement>"),
],
runtime: {
pendingProviderRegistrations: [
{ name: "custom", extensionPath: one.path, config: "PRIVATE CONFIG" },
],
pendingNativeProviderRegistrations: [
{
provider: { id: "native", other: "PRIVATE CONFIG" },
extensionPath: one.path,
},
],
},
};
const inventory = inspectExtensionRegistrations(
result as unknown as Parameters<typeof inspectExtensionRegistrations>[0],
);
expect(inventory).toEqual([
{
path: one.path,
name: "one",
tools: ["example"],
hooks: ["future_event"],
providers: ["custom", "native"],
},
{
path: "/extensions/new.ts",
name: "new",
tools: [],
hooks: [],
providers: [],
},
]);
expect(JSON.stringify(inventory)).not.toContain("PRIVATE");
expect(
inspectExtensionRegistrations(
{ ...result, runtime: {} } as unknown as Parameters<
typeof inspectExtensionRegistrations
>[0],
inventory,
),
).toEqual(inventory);
expect(registrationSignals(inventory[0]!)).toEqual(["tools", "provider"]);
expect(registrationSignals(inventory[1]!)).toEqual(["unknown"]);
expect(
registrationSignals({
tools: [],
providers: [],
hooks: ["future_event"],
} as unknown as Parameters<typeof registrationSignals>[0]),
).toEqual(["other-hooks"]);
expect(
registrationSignals({
tools: [],
providers: [],
hooks: [
"before_agent_start",
"context",
"before_provider_request",
"resources_discover",
"input",
"session_start",
],
} as unknown as Parameters<typeof registrationSignals>[0]),
).toEqual(["prompt", "context", "payload", "resources", "input", "startup"]);
});
it.each([true, false])(
"omits modules before evaluation, retaining package skills and inline factories (keep file: %s)",
async (keep) => {
directory = mkdtempSync(join(tmpdir(), "pi-extension-profile-test-"));
const cwd = join(directory, "project");
const agentDir = join(directory, "agent");
const packageDir = join(directory, "package");
for (const path of [cwd, agentDir, packageDir]) mkdirSync(path);
writeFileSync(
join(packageDir, "package.json"),
JSON.stringify({
name: "fixture",
pi: { extensions: ["omit.mjs"], skills: ["skill.md"] },
}),
);
writeFileSync(
join(packageDir, "omit.mjs"),
"throw new Error('Excluded module executed'); export default () => {}; ",
);
writeFileSync(
join(packageDir, "skill.md"),
"---\nname: fixture\ndescription: Fixture skill\n---\nBody",
);
const kept = join(directory, "keep.mjs");
writeFileSync(
kept,
"export default pi => pi.registerFlag('kept', { type: 'boolean', description: 'kept' });",
);
const settingsManager = SettingsManager.inMemory(
{ packages: [packageDir] },
{ projectTrusted: false },
);
const loader = new DefaultResourceLoader({
cwd,
agentDir,
settingsManager,
...resourceOptions(
"headless+1msg",
[
{
name: "builtin",
factory(pi) {
pi.registerFlag("inline", {
type: "boolean",
description: "inline",
});
},
},
],
{ extensionPaths: keep ? [kept] : [] } as Parameters<
typeof resourceOptions
>[2],
),
});
await loader.reload();
expect(loader.getExtensions().errors).toEqual([]);
expect(
loader.getExtensions().extensions.map((extension) => extension.path),
).toEqual([...(keep ? [kept] : []), "<inline:builtin>"]);
expect(loader.getExtensions().extensions.at(-1)?.flags.has("inline")).toBe(
true,
);
expect(loader.getSkills().skills.map((skill) => skill.name)).toContain(
"fixture",
);
},
);
function fixture() {
directory = mkdtempSync(join(tmpdir(), "pi-extension-profile-test-"));
const cwd = join(directory, "project with spaces");
mkdirSync(cwd);
const inventory = [
{
path: "/extensions/writer/index.ts",
name: "writer",
tools: ["write"],
hooks: [],
providers: [],
},
{
path: "/extensions/observer/index.ts",
name: "observer",
tools: [],
hooks: ["message_end"],
providers: [],
},
{
path: "/extensions/brand-new/index.ts",
name: "brand-new",
tools: [],
hooks: [],
providers: [],
},
{
path: "/extensions/new-hook/index.ts",
name: "new-hook",
tools: [],
hooks: ["future_event"],
providers: [],
},
];
const log = join(directory, "calls.jsonl");
const worker = join(directory, "worker.mjs");
writeFileSync(
worker,
`
import fs from 'node:fs';
import assert from 'node:assert/strict';
const options = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify(options) + '\\n');
assert.equal(process.cwd(), options.cwd);
assert.equal(options.agentDir, options.sourceAgentDir);
assert.equal(options.inspectExtensions, true);
const inventory = ${JSON.stringify(inventory)};
const mode = process.env.EXTENSION_PROFILE_TEST;
const calls = fs.readFileSync(${JSON.stringify(log)}, 'utf8').trim().split('\\n').length;
const omitted = inventory.find(entry => options.extensionPaths && !options.extensionPaths.includes(entry.path));
const result = { kind: options.kind, cwd: options.cwd, version: 'test', provider: 'test', model: 'model', thinking: 'low', contextWindow: 100000,
extensions: inventory.filter(entry => !options.extensionPaths || options.extensionPaths.includes(entry.path)),
local: { system: 80, tools: omitted?.name === 'writer' ? 0 : 20, messages: 8, toolCount: 1 },
codemodeCatalog: omitted?.name === 'brand-new' ? ['replacement'] : ['original'],
...(options.kind === 'headless+0msg' ? {} : { inputTokens: omitted?.name === 'writer' ? 80 : omitted?.name === 'brand-new' ? 105 : 100 }),
};
if (mode === 'startup' && options.kind === 'headless+0msg') result.error = 'controlled startup failure';
if (mode === 'inventory') delete result.extensions;
if (mode === 'omitted' && omitted?.name === 'writer') result.error = 'controlled dependency failure';
if (mode === 'mismatch' && omitted?.name === 'writer') result.model = 'changed';
if (mode === 'baseline' && calls === 2) result.error = 'controlled baseline failure';
if (mode === 'drift' && calls === inventory.length + 3) result.inputTokens++;
if (mode === 'repeat' && calls === inventory.length + 3) result.error = 'controlled repeat failure';
if (mode === 'catalog-drift' && calls === inventory.length + 3) result.codemodeCatalog = ['changed'];
if (mode === 'no-catalog') delete result.codemodeCatalog;
fs.writeFileSync(options.resultPath, JSON.stringify(result));
`,
);
return {
cwd,
worker,
inventory,
calls: () =>
readFileSync(log, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line)),
};
}
it("automatically measures every new extension, exactly one omission each, with pinned controls and cleanup", async () => {
const { cwd, worker, inventory, calls } = fixture();
const progress = vi.fn();
const report = await measureExtensions(cwd, worker, progress);
expect(report.stable).toBe(true);
expect(report.inventory).toEqual(inventory);
expect(report.runs.map((run) => run.extension.path).sort()).toEqual(
inventory.map((entry) => entry.path).sort(),
);
const jobs = calls();
expect(jobs).toHaveLength(inventory.length + 3);
expect(jobs[0].extensionPaths).toBeUndefined();
expect(jobs[1].extensionPaths).toEqual(inventory.map((entry) => entry.path));
expect(jobs.at(-1).extensionPaths).toEqual(jobs[1].extensionPaths);
for (const job of jobs.slice(1)) expect(job.selection.model).toBe("model");
for (const [i, run] of report.runs.entries())
expect(jobs[i + 2].extensionPaths).toEqual(
inventory
.filter((entry) => entry.path !== run.extension.path)
.map((entry) => entry.path),
);
for (const job of jobs) expect(() => readFileSync(job.resultPath)).toThrow();
expect(progress).toHaveBeenCalledTimes(inventory.length + 4);
const text = formatExtensionOverview(report);
expect(text).toMatch(/writer\s+\+20\s+\+0\s+\+20/);
expect(text).toMatch(/brand-new\s+-5.*\+replacement, -original/);
expect(text).toContain("Net removal effect, not standalone extension cost");
expect(text).toMatch(/observer\s+\+0/);
expect(text).toContain("Deltas are not additive");
});
it.each(["startup", "inventory", "baseline"])(
"blocks dependent requests after %s failure",
async (mode) => {
const { cwd, worker, calls } = fixture();
vi.stubEnv("EXTENSION_PROFILE_TEST", mode);
const report = await measureExtensions(cwd, worker);
expect(report.error).toBeTruthy();
expect(report.runs).toEqual([]);
expect(calls()).toHaveLength(mode === "baseline" ? 2 : 1);
expect(formatExtensionOverview(report)).toContain("failed");
},
);
it.each(["omitted", "mismatch"])(
"reports %s failure without zero savings or blocking independent omissions",
async (mode) => {
const { cwd, worker } = fixture();
vi.stubEnv("EXTENSION_PROFILE_TEST", mode);
const report = await measureExtensions(cwd, worker);
expect(report.stable).toBe(true);
expect(report.runs).toHaveLength(4);
expect(formatExtensionOverview(report)).toMatch(/writer\s+FAIL/);
},
);
it("keeps token results but reports unavailable catalog metadata explicitly", async () => {
const { cwd, worker } = fixture();
vi.stubEnv("EXTENSION_PROFILE_TEST", "no-catalog");
const report = await measureExtensions(cwd, worker);
expect(report.stable).toBe(true);
expect(formatExtensionOverview(report)).toMatch(
/brand-new\s+-5.*unavailable/,
);
});
it.each(["drift", "repeat", "catalog-drift"])(
"withholds savings when final control has %s",
async (mode) => {
const { cwd, worker } = fixture();
vi.stubEnv("EXTENSION_PROFILE_TEST", mode);
const report = await measureExtensions(cwd, worker);
expect(report.stable).toBe(false);
expect(report.error).toContain("savings withheld");
expect(formatExtensionOverview(report)).toMatch(/writer\s+n\/a/);
},
);