pi-ext
bugabingas pi extensions
owned by bugabinga
test: stabilize goal harness and vitest selection
3c795d685466ee866777f608ab27cb0a85ec1325
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (7)
extensions/cost/index.ts
Modified414 414 void syncBestEffort(ctx);
417 + const sessionDir
= ctx.sessionManager.
getSessionDir();
417 418 setTimeout(()
=> {
418 419 void (
async ()
=> {
419 - const total
= await calculateMonthlyCost(
420 - ctx.sessionManager.
getSessionDir(),
420 + const total
= await calculateMonthlyCost(sessionDir);
422 421 if (ticket
!== generation)
return;
423 422 monthlyCost
= total;
424 423 lastSignature
= "";
extensions/goal/__tests__/harness.test.ts
Modified 2 - createAssistantMessageEventStream,
5 -}
from "@earendil-works/pi-ai";
1 +import { createAssistantMessageEventStream }
from "@earendil-works/pi-ai";
14 + __goalTest.
resetCompleteJudgeForTest();
18 15 delete process.
env.PI_GOAL_PROBE;
19 16 delete process.
env.PI_GOAL_LOG;
153 150 function installJudgeProvider(replies
: string[]) {
154 - const previousProvider
= getApiProvider(
"openai-responses");
155 151 const queue
= [
...replies];
156 - registerApiProvider(
158 - api:
"openai-responses",
160 - const stream
= createAssistantMessageEventStream();
161 - queueMicrotask(()
=> 165 - '{"done":false,"blocked":false,"reason":"exhausted"}',
171 - streamSimple: ()
=> {
172 - throw new Error(
"not used");
152 + __goalTest.
setCompleteJudgeForTest(
155 + queue.
shift()
?? '{"done":false,"blocked":false,"reason":"exhausted"}',
178 - if (previousProvider)
179 - registerApiProvider(previousProvider,
"goal-test-restore");
158 + return ()
=> __goalTest.
resetCompleteJudgeForTest();
183 161 function installJudgeErrorProvider(message
: string) {
184 - const previousProvider
= getApiProvider(
"openai-responses");
185 - registerApiProvider(
187 - api:
"openai-responses",
189 - const stream
= createAssistantMessageEventStream();
190 - queueMicrotask(()
=> stream.
push(
errorEvent(message)));
193 - streamSimple: ()
=> {
194 - throw new Error(
"not used");
200 - if (previousProvider)
201 - registerApiProvider(previousProvider,
"goal-test-restore");
162 + __goalTest.
setCompleteJudgeForTest(
async ()
=> {
163 + throw new Error(message);
165 + return ()
=> __goalTest.
resetCompleteJudgeForTest();
205 168 async function runProbe(
808 771 it(
"ignores stale judge result after goal is cleared during await",
async ()
=> {
809 - const previousProvider
= getApiProvider(
"openai-responses");
810 772 let resolveJudge
: ((text
: string)
=> void)
| undefined;
811 773 let judgeCalled
= false;
812 - registerApiProvider(
814 - api:
"openai-responses",
816 - judgeCalled
= true;
817 - const stream
= createAssistantMessageEventStream();
818 - new Promise<
string>((resolve)
=> {
819 - resolveJudge
= resolve;
820 - }).
then((text)
=> stream.
push(
doneEvent(text)));
823 - streamSimple: ()
=> {
824 - throw new Error(
"not used");
774 + __goalTest.
setCompleteJudgeForTest(
async ()
=> {
775 + judgeCalled
= true;
776 + const text
= await new Promise<
string>((resolve)
=> {
777 + resolveJudge
= resolve;
779 + return doneEvent(text).message;
831 - t
= await createTestSession({ extensions: [
goalPath()] });
832 - installSaysStream(t, [
"still working"]);
782 + t
= await createTestSession({ extensions: [
goalPath()] });
783 + installSaysStream(t, [
"still working"]);
834 - await t.session.
prompt(
"/goal finish the task");
835 - await waitFor(()
=> judgeCalled);
836 - await t.session.
prompt(
"/goal clear");
837 - resolveJudge?.(
'{"done":true,"blocked":false,"reason":"finished"}');
838 - await t.session.agent.
waitForIdle();
839 - await new Promise((resolve)
=> setTimeout(resolve,
10));
785 + await t.session.
prompt(
"/goal finish the task");
786 + await waitFor(()
=> judgeCalled);
787 + await t.session.
prompt(
"/goal clear");
788 + resolveJudge?.(
'{"done":true,"blocked":false,"reason":"finished"}');
789 + await t.session.agent.
waitForIdle();
790 + await new Promise((resolve)
=> setTimeout(resolve,
10));
842 - customEvents(
"goal-state").
map((entry)
=> entry.
status),
843 - ).not.
toContain(
"complete");
844 - expect(
latestGoal()).
toBeUndefined();
846 - if (previousProvider)
847 - registerApiProvider(previousProvider,
"goal-test-restore");
793 + customEvents(
"goal-state").
map((entry)
=> entry.
status),
794 + ).not.
toContain(
"complete");
795 + expect(
latestGoal()).
toBeUndefined();
extensions/goal/index.ts
Modified 1 -import { complete, StringEnum }
from "@earendil-works/pi-ai";
1 +import { complete
as piComplete, StringEnum }
from "@earendil-works/pi-ai";
13 13 } from
"@earendil-works/pi-tui";
14 14 import { Type }
from "typebox";
16 +type JudgeComplete = typeof piComplete;
17 +const JUDGE_COMPLETE_FOR_TEST
= Symbol.
for(
18 + "@bugabinga/pi-ext-goal/completeJudge",
20 +type GoalTestGlobals = typeof globalThis
& {
21 + [JUDGE_COMPLETE_FOR_TEST]
?: JudgeComplete;
24 +function judgeComplete()
: JudgeComplete {
25 + return (globalThis
as GoalTestGlobals)[JUDGE_COMPLETE_FOR_TEST]
?? piComplete;
16 28 type GoalStatus = "active" | "paused" | "complete" | "blocked";
17 29 type GoalVerdictLabel = "done" | "blocked" | "continue" | "error";
18 30 type ProbeMode = "direct" | "microtask" | "timeout";
482 494 return judgeVerdictLabel(verdict);
497 +function appendVisibleCustomMessage(
498 + ctx
: ExtensionContext,
499 + customType
: string,
501 + details
: Record<
string,
unknown>,
503 + ctx.sessionManager.
appendCustomMessageEntry(
485 511 function appendGoalEvent(
486 512 pi
: ExtensionAPI,
513 + ctx
: ExtensionContext,
488 515 state
: GoalState | undefined,
489 516 data
: Record<
string,
unknown>
= {},
498 525 pi.
appendEntry(GOAL_EVENT_ENTRY, payload);
502 - customType: GOAL_EVENT_ENTRY,
503 - content:
`goal ${event}${state?.lastReason ? ` · ${shortText(state.lastReason)}` : ""}`,
507 - { triggerTurn:
false },
510 - if (
!isStaleContextError(error))
throw error;
526 + appendVisibleCustomMessage(
529 + `goal ${event}${state?.lastReason ? ` · ${shortText(state.lastReason)}` : ""}`,
514 534 function debugGoal(
1086 1106 function appendJudge(
1087 1107 pi
: ExtensionAPI,
1108 + ctx
: ExtensionContext,
1088 1109 event
: "start" | "verdict",
1089 1110 state
: GoalState,
1090 1111 verdict
?: JudgeVerdict,
1101 1122 pi.
appendEntry(GOAL_JUDGE_ENTRY, data);
1102 1123 setTimeout(()
=> {
1106 - customType: GOAL_JUDGE_ENTRY,
1108 - ? `goal judge ${judgeVerdictLabel(verdict)} · turn ${data.turn}/${data.maxTurns} · ${judgeDisplayReason(verdict.reason)}` 1109 - : `goal judge running · turn ${data.turn}/${data.maxTurns}`,
1113 - { triggerTurn:
false },
1125 + appendVisibleCustomMessage(
1129 + ? `goal judge ${judgeVerdictLabel(verdict)} · turn ${data.turn}/${data.maxTurns} · ${judgeDisplayReason(verdict.reason)}` 1130 + : `goal judge running · turn ${data.turn}/${data.maxTurns}`,
1115 1133 }
catch (error) {
1116 1134 if (
!isStaleContextError(error))
throw error;
1170 1188 let lastError
: unknown;
1171 1189 for (
let attempt
= 0; attempt
< JUDGE_ATTEMPTS; attempt
++) {
1173 - const response
= await complete(model, judgeContext, {
1191 + const response
= await judgeComplete()(model, judgeContext, {
1174 1192 apiKey: auth.apiKey,
1175 1193 headers: auth.
headers,
1176 1194 signal: ctx.signal,
1292 1310 !ownsGoal(ctx, latest)
1294 1312 debugGoal(pi,
"continuation-stale", { goalId: state.
id });
1295 - appendGoalEvent(pi,
"continuation skipped", latest, { goalId: state.
id });
1313 + appendGoalEvent(pi, ctx,
"continuation skipped", latest, {
1296 1316 continuation.queued
= false;
1435 1455 criteria: [
...state.criteria, text],
1436 1456 lastReason:
`steering added as criterion: ${shortText(text)}`,
1438 - appendGoalEvent(pi,
"steering-added", state, { text });
1458 + appendGoalEvent(pi, ctx,
"steering-added", state, { text });
1439 1459 updateUI(pi, ctx, state);
1441 1461 pi.
sendMessage(
1569 1589 lastVerdict:
"error",
1570 1590 lastReason:
`retry ${retries}/${MAX_ERROR_RETRIES}: ${errorMessage}`,
1572 - appendGoalEvent(pi,
"error-retry", state, {
1592 + appendGoalEvent(pi, ctx,
"error-retry", state, {
1573 1593 reason: errorMessage,
1591 1611 lastVerdict:
"error",
1592 1612 lastReason: errorMessage,
1594 - appendGoalEvent(pi,
"paused", state, { reason: state.lastReason });
1614 + appendGoalEvent(pi, ctx,
"paused", state, { reason: state.lastReason });
1595 1615 updateUI(pi, ctx, state);
1596 1616 syncUpdateGoalTool();
1601 1621 const judged
= state;
1602 - appendJudge(pi,
"start", judged);
1622 + appendJudge(pi, ctx,
"start", judged);
1603 1623 updateUI(pi, ctx, judged,
"judging");
1604 1624 let verdict
: JudgeVerdict;
1614 1634 if (state
!== judged
|| state.
status !== "active")
return;
1615 - appendJudge(pi,
"verdict", judged, verdict);
1635 + appendJudge(pi, ctx,
"verdict", judged, verdict);
1616 1636 const commonPatch
: Partial<
GoalState>
= {
1617 1637 lastReason: verdict.reason,
1618 1638 lastVerdict:
goalVerdictLabel(verdict),
1636 1656 updateUI(pi, ctx, state);
1637 1657 syncUpdateGoalTool();
1638 1658 if (state.
status === "paused") {
1639 - appendGoalEvent(pi,
"paused", state, { reason: verdict.reason });
1659 + appendGoalEvent(pi, ctx,
"paused", state, { reason: verdict.reason });
1641 1661 ctx.ui.
notify(
`Goal paused — ${verdict.reason}`,
"warning");
1642 1662 }
else scheduleContinuation(pi, ctx, state, continuation);
1649 1669 judgeFailures:
0,
1650 1670 errorRetries:
0,
1652 - appendGoalEvent(pi,
"complete", state, { reason: verdict.reason });
1672 + appendGoalEvent(pi, ctx,
"complete", state, { reason: verdict.reason });
1653 1673 updateUI(pi, ctx, state);
1654 1674 syncUpdateGoalTool();
1671 1691 updateUI(pi, ctx, state);
1672 1692 syncUpdateGoalTool();
1674 - appendGoalEvent(pi,
"blocked", state, { reason: verdict.reason });
1694 + appendGoalEvent(pi, ctx,
"blocked", state, { reason: verdict.reason });
1676 1696 ctx.ui.
notify(
`Goal blocked: ${verdict.reason}`,
"warning");
1677 1697 }
else scheduleContinuation(pi, ctx, state, continuation);
1696 1716 errorRetries:
0,
1697 1717 lastReason:
`${kind} budget exhausted: ${verdict.reason}`,
1699 - appendGoalEvent(pi,
"paused", state, { reason: state.lastReason });
1719 + appendGoalEvent(pi, ctx,
"paused", state, { reason: state.lastReason });
1700 1720 updateUI(pi, ctx, state);
1701 1721 syncUpdateGoalTool();
1927 1947 export const GOAL_PROBE_MESSAGE
= PROBE_MESSAGE;
1928 1948 export const GOAL_STATE_CUSTOM_TYPE
= GOAL_STATE_ENTRY;
1949 +function setCompleteJudgeForTest(fn
: JudgeComplete) {
1950 + (globalThis
as GoalTestGlobals)[JUDGE_COMPLETE_FOR_TEST]
= fn;
1953 +function resetCompleteJudgeForTest() {
1954 + delete (globalThis
as GoalTestGlobals)[JUDGE_COMPLETE_FOR_TEST];
1929 1957 export const __goalTest
= {
1930 1958 activeGoalPrompt,
1931 1959 continuationPrompt,
1981 2009 canUseUpdateGoal,
1982 2010 parseLooseBoolean,
1983 2011 JUDGE_PARSE_FAILURE_LIMIT,
2012 + setCompleteJudgeForTest,
2013 + resetCompleteJudgeForTest,
extensions/llmiterate/__tests__/core.test.ts
Modified265 265 saveState(storeDir, state);
266 266 expect(
loadState(storeDir)).
toEqual(state);
267 - expect(
readFileSync(
join(storeDir, STATE_FILE),
"utf-8").
endsWith(
"\n")).
toBe(
true);
268 + readFileSync(
join(storeDir, STATE_FILE),
"utf-8").
endsWith(
"\n"),
270 272 join(storeDir, STATE_FILE),
extensions/superpowers/__tests__/helpers.test.ts
Modified199 199 it(
"package manifest ships extension, prompts, and skills", ()
=> {
200 200 const pkg
= JSON.
parse(
201 - readFileSync(path.
resolve(
import.
meta.dirname,
"../package.json"),
"utf8"),
202 + path.
resolve(
import.
meta.dirname,
"../package.json"),
203 206 expect(pkg.pi).
toEqual({
204 207 extensions: [
"./index.ts"],
new file mode 100644 1 +import { defineConfig }
from "vitest/config";
3 +export default defineConfig({
5 + include: [
"extensions/**/__tests__/**/*.test.ts",
"skills/**/*.test.ts"],
6 + exclude: [
"**/__e2e__/**",
"**/node_modules/**"],
vitest.workspace.ts
Deleteddeleted file mode 100644 1 -import { defineWorkspace }
from "vitest/config";
3 -export default defineWorkspace([
7 - "extensions/**/__tests__/**/*.test.ts",
8 - "skills/**/*.test.{ts,mjs}",
10 - exclude: [
"**/__e2e__/**",
"**/node_modules/**",
"**/*.integration.test.mjs"],
15 - include: [
"extensions/**/__benches__/**/*.bench.ts"],
16 - exclude: [
"**/node_modules/**"],