repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

test: stabilize goal harness and vitest selection

3c795d685466ee866777f608ab27cb0a85ec1325

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (7)

extensions/cost/index.ts

Modified
414 414 void syncBestEffort(ctx);
415 415 }, 0);
417 + const sessionDir = ctx.sessionManager.getSessionDir();
417 418 setTimeout(() => {
418 419 void (async () => {
419 - const total = await calculateMonthlyCost(
420 - ctx.sessionManager.getSessionDir(),
421 - );
420 + const total = await calculateMonthlyCost(sessionDir);
422 421 if (ticket !== generation) return;
423 422 monthlyCost = total;
424 423 lastSignature = "";

extensions/goal/__tests__/harness.test.ts

Modified
1 -import {
2 - createAssistantMessageEventStream,
3 - getApiProvider,
4 - registerApiProvider,
5 -} from "@earendil-works/pi-ai";
1 +import { createAssistantMessageEventStream } from "@earendil-works/pi-ai";
6 2 import {
7 3 createTestSession,
8 4 type TestSession,
15 11 afterEach(() => {
16 12 t?.dispose();
17 13 t = undefined;
14 + __goalTest.resetCompleteJudgeForTest();
18 15 delete process.env.PI_GOAL_PROBE;
19 16 delete process.env.PI_GOAL_LOG;
153 150 function installJudgeProvider(replies: string[]) {
154 - const previousProvider = getApiProvider("openai-responses");
155 151 const queue = [...replies];
156 - registerApiProvider(
157 - {
158 - api: "openai-responses",
159 - stream: () => {
160 - const stream = createAssistantMessageEventStream();
161 - queueMicrotask(() =>
162 - stream.push(
163 - doneEvent(
164 - queue.shift() ??
165 - '{"done":false,"blocked":false,"reason":"exhausted"}',
166 - ),
167 - ),
168 - );
169 - return stream;
170 - },
171 - streamSimple: () => {
172 - throw new Error("not used");
173 - },
174 - },
175 - "goal-test",
152 + __goalTest.setCompleteJudgeForTest(
153 + async () =>
154 + doneEvent(
155 + queue.shift() ?? '{"done":false,"blocked":false,"reason":"exhausted"}',
156 + ).message,
177 - return () => {
178 - if (previousProvider)
179 - registerApiProvider(previousProvider, "goal-test-restore");
180 - };
158 + return () => __goalTest.resetCompleteJudgeForTest();
183 161 function installJudgeErrorProvider(message: string) {
184 - const previousProvider = getApiProvider("openai-responses");
185 - registerApiProvider(
186 - {
187 - api: "openai-responses",
188 - stream: () => {
189 - const stream = createAssistantMessageEventStream();
190 - queueMicrotask(() => stream.push(errorEvent(message)));
191 - return stream;
192 - },
193 - streamSimple: () => {
194 - throw new Error("not used");
195 - },
196 - },
197 - "goal-test",
198 - );
199 - return () => {
200 - if (previousProvider)
201 - registerApiProvider(previousProvider, "goal-test-restore");
202 - };
162 + __goalTest.setCompleteJudgeForTest(async () => {
163 + throw new Error(message);
164 + });
165 + return () => __goalTest.resetCompleteJudgeForTest();
205 168 async function runProbe(
808 771 it("ignores stale judge result after goal is cleared during await", async () => {
809 - const previousProvider = getApiProvider("openai-responses");
810 772 let resolveJudge: ((text: string) => void) | undefined;
811 773 let judgeCalled = false;
812 - registerApiProvider(
813 - {
814 - api: "openai-responses",
815 - stream: () => {
816 - judgeCalled = true;
817 - const stream = createAssistantMessageEventStream();
818 - new Promise<string>((resolve) => {
819 - resolveJudge = resolve;
820 - }).then((text) => stream.push(doneEvent(text)));
821 - return stream;
822 - },
823 - streamSimple: () => {
824 - throw new Error("not used");
825 - },
826 - },
827 - "goal-test",
828 - );
774 + __goalTest.setCompleteJudgeForTest(async () => {
775 + judgeCalled = true;
776 + const text = await new Promise<string>((resolve) => {
777 + resolveJudge = resolve;
778 + });
779 + return doneEvent(text).message;
780 + });
830 - try {
831 - t = await createTestSession({ extensions: [goalPath()] });
832 - installSaysStream(t, ["still working"]);
782 + t = await createTestSession({ extensions: [goalPath()] });
783 + installSaysStream(t, ["still working"]);
834 - await t.session.prompt("/goal finish the task");
835 - await waitFor(() => judgeCalled);
836 - await t.session.prompt("/goal clear");
837 - resolveJudge?.('{"done":true,"blocked":false,"reason":"finished"}');
838 - await t.session.agent.waitForIdle();
839 - await new Promise((resolve) => setTimeout(resolve, 10));
785 + await t.session.prompt("/goal finish the task");
786 + await waitFor(() => judgeCalled);
787 + await t.session.prompt("/goal clear");
788 + resolveJudge?.('{"done":true,"blocked":false,"reason":"finished"}');
789 + await t.session.agent.waitForIdle();
790 + await new Promise((resolve) => setTimeout(resolve, 10));
841 - expect(
842 - customEvents("goal-state").map((entry) => entry.status),
843 - ).not.toContain("complete");
844 - expect(latestGoal()).toBeUndefined();
845 - } finally {
846 - if (previousProvider)
847 - registerApiProvider(previousProvider, "goal-test-restore");
848 - }
792 + expect(
793 + customEvents("goal-state").map((entry) => entry.status),
794 + ).not.toContain("complete");
795 + expect(latestGoal()).toBeUndefined();

extensions/goal/index.ts

Modified
1 -import { complete, StringEnum } from "@earendil-works/pi-ai";
1 +import { complete as piComplete, StringEnum } from "@earendil-works/pi-ai";
2 2 import type {
3 3 ExtensionAPI,
4 4 ExtensionContext,
13 13 } from "@earendil-works/pi-tui";
14 14 import { Type } from "typebox";
16 +type JudgeComplete = typeof piComplete;
17 +const JUDGE_COMPLETE_FOR_TEST = Symbol.for(
18 + "@bugabinga/pi-ext-goal/completeJudge",
19 +);
20 +type GoalTestGlobals = typeof globalThis & {
21 + [JUDGE_COMPLETE_FOR_TEST]?: JudgeComplete;
22 +};
24 +function judgeComplete(): JudgeComplete {
25 + return (globalThis as GoalTestGlobals)[JUDGE_COMPLETE_FOR_TEST] ?? piComplete;
16 28 type GoalStatus = "active" | "paused" | "complete" | "blocked";
17 29 type GoalVerdictLabel = "done" | "blocked" | "continue" | "error";
18 30 type ProbeMode = "direct" | "microtask" | "timeout";
482 494 return judgeVerdictLabel(verdict);
497 +function appendVisibleCustomMessage(
498 + ctx: ExtensionContext,
499 + customType: string,
500 + content: string,
501 + details: Record<string, unknown>,
502 +): void {
503 + ctx.sessionManager.appendCustomMessageEntry(
504 + customType,
505 + content,
506 + true,
507 + details,
508 + );
485 511 function appendGoalEvent(
486 512 pi: ExtensionAPI,
513 + ctx: ExtensionContext,
487 514 event: string,
488 515 state: GoalState | undefined,
489 516 data: Record<string, unknown> = {},
496 523 ...data,
498 525 pi.appendEntry(GOAL_EVENT_ENTRY, payload);
499 - try {
500 - pi.sendMessage(
502 - customType: GOAL_EVENT_ENTRY,
503 - content: `goal ${event}${state?.lastReason ? ` · ${shortText(state.lastReason)}` : ""}`,
504 - display: true,
505 - details: payload,
506 - },
507 - { triggerTurn: false },
508 - );
509 - } catch (error) {
510 - if (!isStaleContextError(error)) throw error;
526 + appendVisibleCustomMessage(
527 + ctx,
528 + GOAL_EVENT_ENTRY,
529 + `goal ${event}${state?.lastReason ? ` · ${shortText(state.lastReason)}` : ""}`,
530 + payload,
531 + );
514 534 function debugGoal(
1086 1106 function appendJudge(
1087 1107 pi: ExtensionAPI,
1108 + ctx: ExtensionContext,
1088 1109 event: "start" | "verdict",
1089 1110 state: GoalState,
1090 1111 verdict?: JudgeVerdict,
1101 1122 pi.appendEntry(GOAL_JUDGE_ENTRY, data);
1102 1123 setTimeout(() => {
1104 - pi.sendMessage(
1106 - customType: GOAL_JUDGE_ENTRY,
1107 - content: verdict
1108 - ? `goal judge ${judgeVerdictLabel(verdict)} · turn ${data.turn}/${data.maxTurns} · ${judgeDisplayReason(verdict.reason)}`
1109 - : `goal judge running · turn ${data.turn}/${data.maxTurns}`,
1110 - display: true,
1111 - details: data,
1112 - },
1113 - { triggerTurn: false },
1125 + appendVisibleCustomMessage(
1126 + ctx,
1127 + GOAL_JUDGE_ENTRY,
1128 + verdict
1129 + ? `goal judge ${judgeVerdictLabel(verdict)} · turn ${data.turn}/${data.maxTurns} · ${judgeDisplayReason(verdict.reason)}`
1130 + : `goal judge running · turn ${data.turn}/${data.maxTurns}`,
1131 + data,
1115 1133 } catch (error) {
1116 1134 if (!isStaleContextError(error)) throw error;
1170 1188 let lastError: unknown;
1171 1189 for (let attempt = 0; attempt < JUDGE_ATTEMPTS; attempt++) {
1173 - const response = await complete(model, judgeContext, {
1191 + const response = await judgeComplete()(model, judgeContext, {
1174 1192 apiKey: auth.apiKey,
1175 1193 headers: auth.headers,
1176 1194 signal: ctx.signal,
1292 1310 !ownsGoal(ctx, latest)
1294 1312 debugGoal(pi, "continuation-stale", { goalId: state.id });
1295 - appendGoalEvent(pi, "continuation skipped", latest, { goalId: state.id });
1313 + appendGoalEvent(pi, ctx, "continuation skipped", latest, {
1314 + goalId: state.id,
1315 + });
1296 1316 continuation.queued = false;
1297 1317 return;
1435 1455 criteria: [...state.criteria, text],
1436 1456 lastReason: `steering added as criterion: ${shortText(text)}`,
1438 - appendGoalEvent(pi, "steering-added", state, { text });
1458 + appendGoalEvent(pi, ctx, "steering-added", state, { text });
1439 1459 updateUI(pi, ctx, state);
1441 1461 pi.sendMessage(
1569 1589 lastVerdict: "error",
1570 1590 lastReason: `retry ${retries}/${MAX_ERROR_RETRIES}: ${errorMessage}`,
1572 - appendGoalEvent(pi, "error-retry", state, {
1592 + appendGoalEvent(pi, ctx, "error-retry", state, {
1573 1593 reason: errorMessage,
1574 1594 retries,
1591 1611 lastVerdict: "error",
1592 1612 lastReason: errorMessage,
1594 - appendGoalEvent(pi, "paused", state, { reason: state.lastReason });
1614 + appendGoalEvent(pi, ctx, "paused", state, { reason: state.lastReason });
1595 1615 updateUI(pi, ctx, state);
1596 1616 syncUpdateGoalTool();
1597 1617 if (ctx.hasUI)
1599 1619 return;
1601 1621 const judged = state;
1602 - appendJudge(pi, "start", judged);
1622 + appendJudge(pi, ctx, "start", judged);
1603 1623 updateUI(pi, ctx, judged, "judging");
1604 1624 let verdict: JudgeVerdict;
1614 1634 if (state !== judged || state.status !== "active") return;
1615 - appendJudge(pi, "verdict", judged, verdict);
1635 + appendJudge(pi, ctx, "verdict", judged, verdict);
1616 1636 const commonPatch: Partial<GoalState> = {
1617 1637 lastReason: verdict.reason,
1618 1638 lastVerdict: goalVerdictLabel(verdict),
1636 1656 updateUI(pi, ctx, state);
1637 1657 syncUpdateGoalTool();
1638 1658 if (state.status === "paused") {
1639 - appendGoalEvent(pi, "paused", state, { reason: verdict.reason });
1659 + appendGoalEvent(pi, ctx, "paused", state, { reason: verdict.reason });
1640 1660 if (ctx.hasUI)
1641 1661 ctx.ui.notify(`Goal paused — ${verdict.reason}`, "warning");
1642 1662 } else scheduleContinuation(pi, ctx, state, continuation);
1649 1669 judgeFailures: 0,
1650 1670 errorRetries: 0,
1652 - appendGoalEvent(pi, "complete", state, { reason: verdict.reason });
1672 + appendGoalEvent(pi, ctx, "complete", state, { reason: verdict.reason });
1653 1673 updateUI(pi, ctx, state);
1654 1674 syncUpdateGoalTool();
1655 1675 if (ctx.hasUI)
1671 1691 updateUI(pi, ctx, state);
1672 1692 syncUpdateGoalTool();
1673 1693 if (blocked) {
1674 - appendGoalEvent(pi, "blocked", state, { reason: verdict.reason });
1694 + appendGoalEvent(pi, ctx, "blocked", state, { reason: verdict.reason });
1675 1695 if (ctx.hasUI)
1676 1696 ctx.ui.notify(`Goal blocked: ${verdict.reason}`, "warning");
1677 1697 } else scheduleContinuation(pi, ctx, state, continuation);
1696 1716 errorRetries: 0,
1697 1717 lastReason: `${kind} budget exhausted: ${verdict.reason}`,
1699 - appendGoalEvent(pi, "paused", state, { reason: state.lastReason });
1719 + appendGoalEvent(pi, ctx, "paused", state, { reason: state.lastReason });
1700 1720 updateUI(pi, ctx, state);
1701 1721 syncUpdateGoalTool();
1702 1722 if (ctx.hasUI)
1927 1947 export const GOAL_PROBE_MESSAGE = PROBE_MESSAGE;
1928 1948 export const GOAL_STATE_CUSTOM_TYPE = GOAL_STATE_ENTRY;
1949 +function setCompleteJudgeForTest(fn: JudgeComplete) {
1950 + (globalThis as GoalTestGlobals)[JUDGE_COMPLETE_FOR_TEST] = fn;
1953 +function resetCompleteJudgeForTest() {
1954 + delete (globalThis as GoalTestGlobals)[JUDGE_COMPLETE_FOR_TEST];
1929 1957 export const __goalTest = {
1930 1958 activeGoalPrompt,
1931 1959 continuationPrompt,
1981 2009 canUseUpdateGoal,
1982 2010 parseLooseBoolean,
1983 2011 JUDGE_PARSE_FAILURE_LIMIT,
2012 + setCompleteJudgeForTest,
2013 + resetCompleteJudgeForTest,

extensions/llmiterate/__tests__/core.test.ts

Modified
265 265 saveState(storeDir, state);
266 266 expect(loadState(storeDir)).toEqual(state);
267 - expect(readFileSync(join(storeDir, STATE_FILE), "utf-8").endsWith("\n")).toBe(true);
267 + expect(
268 + readFileSync(join(storeDir, STATE_FILE), "utf-8").endsWith("\n"),
269 + ).toBe(true);
269 271 writeFileSync(
270 272 join(storeDir, STATE_FILE),

extensions/superpowers/__tests__/helpers.test.ts

Modified
199 199 it("package manifest ships extension, prompts, and skills", () => {
200 200 const pkg = JSON.parse(
201 - readFileSync(path.resolve(import.meta.dirname, "../package.json"), "utf8"),
201 + readFileSync(
202 + path.resolve(import.meta.dirname, "../package.json"),
203 + "utf8",
204 + ),
203 206 expect(pkg.pi).toEqual({
204 207 extensions: ["./index.ts"],

vitest.config.ts

Added
new file mode 100644
1 +import { defineConfig } from "vitest/config";
3 +export default defineConfig({
4 + test: {
5 + include: ["extensions/**/__tests__/**/*.test.ts", "skills/**/*.test.ts"],
6 + exclude: ["**/__e2e__/**", "**/node_modules/**"],
7 + },
8 +});

vitest.workspace.ts

Deleted
deleted file mode 100644
1 -import { defineWorkspace } from "vitest/config";
3 -export default defineWorkspace([
4 - {
5 - test: {
6 - include: [
7 - "extensions/**/__tests__/**/*.test.ts",
8 - "skills/**/*.test.{ts,mjs}",
9 - ],
10 - exclude: ["**/__e2e__/**", "**/node_modules/**", "**/*.integration.test.mjs"],
11 - },
12 - },
13 - {
14 - test: {
15 - include: ["extensions/**/__benches__/**/*.bench.ts"],
16 - exclude: ["**/node_modules/**"],
17 - benchmark: true,
18 - },
19 - },
20 -]);