pi-ext
bugabingas pi extensions
owned by bugabinga
test(leaks): expand real provider cases
f4aed80fa71bc632522548de7b20386c71d95cf0
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (1)
extensions/leaks/__tests__/real/minimax.test.ts
Modified15 15 const real
= process.
env.LEAKS_REAL
=== "1";
16 16 const d
= real
? describe
: describe.skip;
17 +const TIMEOUT
= 120000;
18 -d(
"real MiniMax integration", ()
=> {
19 - it(
"redacts provider-bound token before a real MiniMax request",
async ()
=> {
20 - const temp
= fs.
mkdtempSync(path.
join(os.
tmpdir(),
"leaks-real-"));
21 - const cwd
= path.
join(temp,
"cwd");
22 - const agentDir
= path.
join(temp,
"agent");
23 - fs.
mkdirSync(cwd, { recursive:
true });
24 - fs.
mkdirSync(agentDir, { recursive:
true });
20 + settings
?: Record<
string,
unknown>;
21 + files
?: Record<
string,
string>;
26 - const leaksPath
= path.
resolve(
import.
meta.
dir,
"../../index.ts");
27 - const settingsManager
= SettingsManager.
inMemory({
28 - defaultProvider:
"minimax",
29 - defaultModel:
"MiniMax-M2.7-highspeed",
30 - defaultThinkingLevel:
"off",
31 - leaks: { enabled:
true, secretFiles: [] },
24 +type RealSession = Awaited<
ReturnType<
typeof createRealSession>>;
34 - const loader
= new DefaultResourceLoader({
40 - noPromptTemplates:
true,
42 - noContextFiles:
true,
43 - additionalExtensionPaths: [leaksPath],
45 - await loader.
reload();
26 +async function createRealSession(opts
: RealCase = {}) {
27 + const temp
= fs.
mkdtempSync(path.
join(os.
tmpdir(),
"leaks-real-"));
28 + const cwd
= path.
join(temp,
"cwd");
29 + const agentDir
= path.
join(temp,
"agent");
30 + fs.
mkdirSync(cwd, { recursive:
true });
31 + fs.
mkdirSync(agentDir, { recursive:
true });
47 - const authStorage
= AuthStorage.
create();
48 - const modelRegistry
= ModelRegistry.
create(authStorage);
49 - const model
= getModel(
"minimax",
"MiniMax-M2.7-highspeed");
50 - const assistantTexts
: string[]
= [];
33 + for (
const [rel, content]
of Object.
entries(opts.files
?? {})) {
34 + const file
= path.
join(cwd, rel);
35 + fs.
mkdirSync(path.
dirname(file), { recursive:
true });
36 + fs.
writeFileSync(file, content,
"utf8");
52 - const { session, extensionsResult }
= await createAgentSession({
58 - thinkingLevel:
"off",
59 - sessionManager: SessionManager.
inMemory(),
61 - resourceLoader: loader,
39 + const leaksPath
= path.
resolve(
import.
meta.
dir,
"../../index.ts");
40 + const settingsManager
= SettingsManager.
inMemory({
41 + defaultProvider:
"minimax",
42 + defaultModel:
"MiniMax-M2.7-highspeed",
43 + defaultThinkingLevel:
"off",
44 + leaks: { enabled:
true, secretFiles: [],
...(opts.settings
?? {}) },
66 - expect(extensionsResult.extensions.
map((e)
=> e.path)).
toEqual([leaksPath]);
67 - session.
subscribe((event)
=> {
68 - if (
event.
type !== "message_end")
return;
69 - const msg
= event.message;
70 - if (msg.role
!== "assistant")
return;
71 - assistantTexts.
push(
JSON.
stringify(msg.
content));
47 + const loader
= new DefaultResourceLoader({
53 + noPromptTemplates:
true,
55 + noContextFiles:
true,
56 + additionalExtensionPaths: [leaksPath],
58 + await loader.
reload();
74 - const fake
= "sk-or-v1-" + "1".
repeat(
64);
75 - await session.
prompt(
`Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`);
60 + const authStorage
= AuthStorage.
create();
61 + const modelRegistry
= ModelRegistry.
create(authStorage);
62 + const model
= getModel(
"minimax",
"MiniMax-M2.7-highspeed");
63 + const assistantTexts
: string[]
= [];
65 + const { session, extensionsResult }
= await createAgentSession({
71 + thinkingLevel:
"off",
72 + sessionManager: SessionManager.
inMemory(),
74 + resourceLoader: loader,
78 + expect(extensionsResult.extensions.
map((e)
=> e.path)).
toEqual([leaksPath]);
79 + session.
subscribe((event)
=> {
80 + if (
event.
type !== "message_end")
return;
81 + const msg
= event.message;
82 + if (msg.role
!== "assistant")
return;
83 + assistantTexts.
push(
JSON.
stringify(msg.
content));
91 + async prompt(text
: string)
: Promise<
string> {
92 + assistantTexts.
length = 0;
93 + await session.
prompt(text);
76 94 await session.agent.
waitForIdle();
78 - const answer
= assistantTexts.
join(
"\n").
toUpperCase();
79 - expect(answer).
toContain(
"NO");
80 - expect(answer).not.
toContain(
"YES");
95 + return assistantTexts.
join(
"\n");
83 99 fs.
rmSync(temp, { recursive:
true, force:
true });
104 +async function withRealSession<
T>(opts
: RealCase,
fn: (s
: RealSession)
=> Promise<
T>)
: Promise<
T> {
105 + const s
= await createRealSession(opts);
107 + return await fn(s);
113 +function expectNo(answer
: string) {
114 + const upper
= answer.
toUpperCase();
115 + expect(upper).
toContain(
"NO");
116 + expect(upper).not.
toContain(
"YES");
119 +function expectThreeNo(answer
: string) {
120 + const upper
= answer.
toUpperCase();
121 + expect((upper.
match(
/\bNO\b/g)
?? []).
length).
toBeGreaterThanOrEqual(
3);
122 + expect(upper).not.
toContain(
"YES");
125 +d(
"real MiniMax integration", ()
=> {
126 + it(
"redacts provider-bound builtin token before a real request",
async ()
=> {
127 + await withRealSession({},
async ({ prompt })
=> {
128 + const fake
= "sk-or-v1-" + "1".
repeat(
64);
129 + const answer
= await prompt(
`Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`);
134 + it(
"redacts secretFiles, keywords, and custom regex before a real request",
async ()
=> {
135 + await withRealSession({
137 + secretFiles: [
".env"],
138 + keywords: [
"raven-codename"],
139 + regex: [{ pattern:
"foo_[A-Za-z0-9]{32}", label:
"foo" }],
142 + ".env":
"LEAKS_DOTENV_VALUE=DENV-real-secret-123456789\n",
144 + },
async ({ prompt })
=> {
145 + const userRegex
= "foo_" + "A".
repeat(
32);
146 + const answer
= await prompt([
147 + "Answer three words, each YES or NO.",
148 + "1. Does first bracket start with DENV-?",
149 + "2. Does second bracket start with raven-?",
150 + "3. Does third bracket start with foo_?",
151 + `[DENV-real-secret-123456789] [raven-codename] [${userRegex}]`,
153 + expectThreeNo(answer);
157 + it(
"redacts generic assignment secrets but not harmless values in a real request",
async ()
=> {
158 + await withRealSession({},
async ({ prompt })
=> {
159 + const strong
= "aB9_qP2xZ7mN4rT8vL0s";
160 + const answer
= await prompt([
161 + "Answer two words, each YES or NO.",
162 + "1. Does first assignment value start with aB9_?",
163 + "2. Does second assignment value equal changeme?",
164 + `[token = ${strong}] [password = changeme]`,
166 + const upper
= answer.
toUpperCase();
167 + expect(upper).
toContain(
"NO");
168 + expect(upper).
toContain(
"YES");
172 + it(
"redacts prior-session context on later real requests",
async ()
=> {
173 + await withRealSession({},
async ({ prompt })
=> {
174 + const fake
= "sk-or-v1-" + "2".
repeat(
64);
175 + await prompt(
`Reply exactly OK. [${fake}]`);
176 + const answer
= await prompt(
"Look at the previous user bracket content. Did it start with sk-or-v1-? Answer exactly YES or NO.");
181 + it(
"restores assistant output locally after the real model repeats a placeholder",
async ()
=> {
182 + await withRealSession({},
async ({ prompt })
=> {
183 + const fake
= "sk-or-v1-" + "3".
repeat(
64);
184 + const answer
= await prompt(
`Repeat exactly the bracket content and nothing else. [${fake}]`);
185 + expect(answer).
toContain(fake);
186 + expect(answer).not.
toMatch(
/__LEAK_[a-f0-9]{12}__/);
190 + it(
"restores tool-call args after a real request creates a mapping",
async ()
=> {
191 + await withRealSession({},
async ({ session, prompt })
=> {
192 + const fake
= "sk-or-v1-" + "4".
repeat(
64);
193 + const answer
= await prompt(
`Repeat exactly the bracket content and nothing else. [${fake}]`);
194 + expect(answer).
toContain(fake);
196 + const redacted
= await session.extensionRunner.
emitBeforeProviderRequest({ text: fake });
197 + const placeholder
= JSON.
stringify(redacted).
match(
/__LEAK_[a-f0-9]{12}__/)?.[
0];
198 + expect(placeholder).
toBeTruthy();
200 + const event
= { type:
"tool_call", toolName:
"bash", toolCallId:
"call-real", input: { command:
`printf ${placeholder}` } };
201 + await session.extensionRunner.
emitToolCall(
event as never);
202 + expect(
event.
input.command).
toBe(
`printf ${fake}`);