pi-ext
bugabingas pi extensions
owned by bugabinga
test(leaks): remove legacy e2e cruft
d0a426b1066688bd708bd95236607b1ac29c39a0
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (1)
extensions/leaks/__e2e__/minimax.test.ts
Deleteddeleted file mode 100644 1 -import fs
from "node:fs";
2 -import os
from "node:os";
3 -import path
from "node:path";
4 -import { getModel }
from "@earendil-works/pi-ai";
8 - DefaultResourceLoader,
12 -}
from "@earendil-works/pi-coding-agent";
13 -import { describe, expect, it }
from "vitest";
15 -const real
= process.
env.LEAKS_REAL
=== "1";
16 -const d
= real
? describe
: describe.skip;
17 -const TIMEOUT
= 120000;
20 - settings
?: Record<
string,
unknown>;
21 - files
?: Record<
string,
string>;
24 -type RealSession = Awaited<
ReturnType<
typeof createRealSession>>;
26 -function textOnly(content
: unknown)
: string {
27 - if (
typeof content
=== "string")
return content;
28 - if (
!Array.
isArray(content))
return JSON.
stringify(content);
31 - (part)
: part
is { type?: string; text
?: string }
=> 33 - typeof part
=== "object" && 34 - (part
as { type
?: string }).
type === "text",
36 - .
map((part)
=> part.
text ?? "")
40 -async function createRealSession(opts
: RealCase = {}) {
41 - const temp
= fs.
mkdtempSync(path.
join(os.
tmpdir(),
"leaks-real-"));
42 - const cwd
= path.
join(temp,
"cwd");
43 - const agentDir
= path.
join(temp,
"agent");
44 - fs.
mkdirSync(cwd, { recursive:
true });
45 - fs.
mkdirSync(agentDir, { recursive:
true });
47 - for (
const [rel, content]
of Object.
entries(opts.files
?? {})) {
48 - const file
= path.
join(cwd, rel);
49 - fs.
mkdirSync(path.
dirname(file), { recursive:
true });
50 - fs.
writeFileSync(file, content,
"utf8");
52 - fs.
mkdirSync(path.
join(cwd,
".pi"), { recursive:
true });
54 - path.
join(cwd,
".pi",
"settings.json"),
56 - { leaks: { enabled:
true, secretFiles: [],
...(opts.settings
?? {}) } },
63 - const leaksPath
= path.
resolve(
import.
meta.dirname,
"../../index.ts");
64 - const settingsManager
= SettingsManager.
inMemory({
65 - defaultProvider:
"minimax",
66 - defaultModel:
"MiniMax-M2.7-highspeed",
67 - defaultThinkingLevel:
"off",
68 - leaks: { enabled:
true, secretFiles: [],
...(opts.settings
?? {}) },
71 - const loader
= new DefaultResourceLoader({
77 - noPromptTemplates:
true,
79 - noContextFiles:
true,
80 - additionalExtensionPaths: [leaksPath],
82 - await loader.
reload();
84 - const authStorage
= AuthStorage.
create();
85 - const modelRegistry
= ModelRegistry.
create(authStorage);
86 - const model
= getModel(
"minimax",
"MiniMax-M2.7-highspeed");
87 - const assistantTexts
: string[]
= [];
89 - const { session, extensionsResult }
= await createAgentSession({
95 - thinkingLevel:
"off",
96 - sessionManager: SessionManager.
inMemory(),
98 - resourceLoader: loader,
102 - expect(extensionsResult.extensions.
map((e)
=> e.path)).
toEqual([leaksPath]);
103 - session.
subscribe((event)
=> {
104 - if (
event.
type !== "message_end")
return;
105 - const msg
= event.message;
106 - if (msg.role
!== "assistant")
return;
107 - assistantTexts.
push(
textOnly(msg.
content));
115 - async prompt(text
: string)
: Promise<
string> {
116 - assistantTexts.
length = 0;
117 - await session.
prompt(text);
118 - await session.agent.
waitForIdle();
119 - return assistantTexts.
join(
"\n");
121 - async command(args
: string)
: Promise<
void> {
122 - const cmd
= extensionsResult.extensions[
0]?.commands.
get(
"leaks");
123 - if (
!cmd)
throw new Error(
"missing /leaks command");
124 - await cmd.
handler(args, session.extensionRunner.
createContext());
126 - writeSettings(leaks
: Record<
string,
unknown>)
: void {
128 - path.
join(cwd,
".pi",
"settings.json"),
129 - JSON.
stringify({ leaks },
null,
2),
135 - fs.
rmSync(temp, { recursive:
true, force:
true });
140 -async function withRealSession<
T>(
142 - fn: (s
: RealSession)
=> Promise<
T>,
144 - const s
= await createRealSession(opts);
146 - return await fn(s);
152 -function expectNo(answer
: string) {
153 - const upper
= answer.
toUpperCase();
154 - expect(upper).
toContain(
"NO");
155 - expect(upper).not.
toContain(
"YES");
158 -function expectYes(answer
: string) {
159 - const upper
= answer.
toUpperCase();
160 - expect(upper).
toContain(
"YES");
163 -function expectThreeNo(answer
: string) {
164 - const upper
= answer.
toUpperCase();
165 - expect((upper.
match(
/\bNO\b/g)
?? []).
length).
toBeGreaterThanOrEqual(
3);
166 - expect(upper).not.
toContain(
"YES");
169 -d(
"real MiniMax integration", ()
=> {
171 - "redacts provider-bound builtin token before a real request",
173 - await withRealSession({},
async ({ prompt })
=> {
174 - const fake
= "sk-or-v1-" + "1".
repeat(
64);
175 - const answer
= await prompt(
176 - `Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`,
185 - "redacts secretFiles, keywords, and custom regex before a real request",
187 - await withRealSession(
190 - secretFiles: [
".env"],
191 - keywords: [
"raven-codename"],
192 - regex: [{ pattern:
"foo_[A-Za-z0-9]{32}", label:
"foo" }],
195 - ".env":
"LEAKS_DOTENV_VALUE=DENV-real-secret-123456789\n",
198 - async ({ prompt })
=> {
199 - const userRegex
= "foo_" + "A".
repeat(
32);
200 - const answer
= await prompt(
202 - "Answer three words, each YES or NO.",
203 - "1. Does first bracket start with DENV-?",
204 - "2. Does second bracket start with raven-?",
205 - "3. Does third bracket start with foo_?",
206 - `[DENV-real-secret-123456789] [raven-codename] [${userRegex}]`,
209 - expectThreeNo(answer);
217 - "redacts generic assignment secrets but not harmless values in a real request",
219 - await withRealSession({},
async ({ prompt })
=> {
220 - const strong
= "aB9_qP2xZ7mN4rT8vL0s";
221 - const answer
= await prompt(
223 - "Answer two words, each YES or NO.",
224 - "1. Does first assignment value start with aB9_?",
225 - "2. Does second assignment value equal changeme?",
226 - `[token = ${strong}] [password = changeme]`,
229 - const upper
= answer.
toUpperCase();
230 - expect(upper).
toContain(
"NO");
231 - expect(upper).
toContain(
"YES");
238 - "redacts prior-session context on later real requests",
240 - await withRealSession({},
async ({ prompt })
=> {
241 - const fake
= "sk-or-v1-" + "2".
repeat(
64);
242 - await prompt(
`Reply exactly OK. [${fake}]`);
243 - const answer
= await prompt(
244 - "Look at the previous user bracket content. Did it start with sk-or-v1-? Answer exactly YES or NO.",
253 - "restores assistant output locally after the real model repeats a placeholder",
255 - await withRealSession({},
async ({ prompt })
=> {
256 - const fake
= "sk-or-v1-" + "3".
repeat(
64);
257 - const answer
= await prompt(
258 - `Repeat exactly the bracket content and nothing else. [${fake}]`,
260 - expect(answer).
toContain(fake);
261 - expect(answer).not.
toMatch(
/__LEAK_[a-f0-9]{12}__/);
268 - "honors disabled setting in a real request",
270 - await withRealSession(
271 - { settings: { enabled:
false } },
272 - async ({ prompt })
=> {
273 - const fake
= "sk-or-v1-" + "5".
repeat(
64);
274 - const answer
= await prompt(
275 - `Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`,
285 - "honors exclude setting in a real request",
287 - await withRealSession(
290 - secretFiles: [
".env"],
291 - exclude: [
"DENV-visible-secret-123456789"],
293 - files: {
".env":
"VISIBLE=DENV-visible-secret-123456789\n" },
295 - async ({ prompt })
=> {
296 - const answer
= await prompt(
297 - "Look only at the bracket content. Does it start with DENV-visible? Answer exactly YES or NO. [DENV-visible-secret-123456789]",
307 - "reload command applies edited settings before later real requests",
309 - await withRealSession({},
async ({ prompt, command, writeSettings })
=> {
310 - const before
= await prompt(
311 - "Look only at the bracket content. Does it start with raven-? Answer exactly YES or NO. [raven-reload]",
318 - keywords: [
"raven-reload"],
320 - await command(
"reload");
322 - const after
= await prompt(
323 - "Look only at the bracket content. Does it start with raven-? Answer exactly YES or NO. [raven-reload]",
332 - "clear command drops mappings after a real request",
334 - await withRealSession({},
async ({ session, prompt, command })
=> {
335 - const fake
= "sk-or-v1-" + "6".
repeat(
64);
337 - `Repeat exactly the bracket content and nothing else. [${fake}]`,
340 - await session.extensionRunner.
emitBeforeProviderRequest({
343 - const placeholder
= JSON.
stringify(redacted).
match(
344 - /__LEAK_[a-f0-9]{12}__/,
346 - expect(placeholder).
toBeTruthy();
348 - await command(
"clear");
350 - const result
= await session.extensionRunner.
emitMessageEnd({
351 - type:
"message_end",
354 - content: [{ type:
"text", text:
String(placeholder) }],
357 - expect(result).
toBeUndefined();
364 - "restores tool-call args after a real request creates a mapping",
366 - await withRealSession({},
async ({ session, prompt })
=> {
367 - const fake
= "sk-or-v1-" + "4".
repeat(
64);
368 - const answer
= await prompt(
369 - `Repeat exactly the bracket content and nothing else. [${fake}]`,
371 - expect(answer).
toContain(fake);
374 - await session.extensionRunner.
emitBeforeProviderRequest({
377 - const placeholder
= JSON.
stringify(redacted).
match(
378 - /__LEAK_[a-f0-9]{12}__/,
380 - expect(placeholder).
toBeTruthy();
385 - toolCallId:
"call-real",
386 - input: { command:
`printf ${placeholder}` },
388 - await session.extensionRunner.
emitToolCall(
event as never);
389 - expect(
event.
input.command).
toBe(
`printf ${fake}`);