repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

test(leaks): expand real provider cases

f4aed80fa71bc632522548de7b20386c71d95cf0

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (1)

extensions/leaks/__tests__/real/minimax.test.ts

Modified
15 15 const real = process.env.LEAKS_REAL === "1";
16 16 const d = real ? describe : describe.skip;
17 +const TIMEOUT = 120000;
18 -d("real MiniMax integration", () => {
19 - it("redacts provider-bound token before a real MiniMax request", async () => {
20 - const temp = fs.mkdtempSync(path.join(os.tmpdir(), "leaks-real-"));
21 - const cwd = path.join(temp, "cwd");
22 - const agentDir = path.join(temp, "agent");
23 - fs.mkdirSync(cwd, { recursive: true });
24 - fs.mkdirSync(agentDir, { recursive: true });
19 +type RealCase = {
20 + settings?: Record<string, unknown>;
21 + files?: Record<string, string>;
22 +};
26 - const leaksPath = path.resolve(import.meta.dir, "../../index.ts");
27 - const settingsManager = SettingsManager.inMemory({
28 - defaultProvider: "minimax",
29 - defaultModel: "MiniMax-M2.7-highspeed",
30 - defaultThinkingLevel: "off",
31 - leaks: { enabled: true, secretFiles: [] },
32 - });
24 +type RealSession = Awaited<ReturnType<typeof createRealSession>>;
34 - const loader = new DefaultResourceLoader({
35 - cwd,
36 - agentDir,
37 - settingsManager,
38 - noExtensions: true,
39 - noSkills: true,
40 - noPromptTemplates: true,
41 - noThemes: true,
42 - noContextFiles: true,
43 - additionalExtensionPaths: [leaksPath],
44 - });
45 - await loader.reload();
26 +async function createRealSession(opts: RealCase = {}) {
27 + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "leaks-real-"));
28 + const cwd = path.join(temp, "cwd");
29 + const agentDir = path.join(temp, "agent");
30 + fs.mkdirSync(cwd, { recursive: true });
31 + fs.mkdirSync(agentDir, { recursive: true });
47 - const authStorage = AuthStorage.create();
48 - const modelRegistry = ModelRegistry.create(authStorage);
49 - const model = getModel("minimax", "MiniMax-M2.7-highspeed");
50 - const assistantTexts: string[] = [];
33 + for (const [rel, content] of Object.entries(opts.files ?? {})) {
34 + const file = path.join(cwd, rel);
35 + fs.mkdirSync(path.dirname(file), { recursive: true });
36 + fs.writeFileSync(file, content, "utf8");
37 + }
52 - const { session, extensionsResult } = await createAgentSession({
53 - cwd,
54 - agentDir,
55 - authStorage,
56 - modelRegistry,
57 - model,
58 - thinkingLevel: "off",
59 - sessionManager: SessionManager.inMemory(),
60 - settingsManager,
61 - resourceLoader: loader,
62 - noTools: "all",
63 - });
39 + const leaksPath = path.resolve(import.meta.dir, "../../index.ts");
40 + const settingsManager = SettingsManager.inMemory({
41 + defaultProvider: "minimax",
42 + defaultModel: "MiniMax-M2.7-highspeed",
43 + defaultThinkingLevel: "off",
44 + leaks: { enabled: true, secretFiles: [], ...(opts.settings ?? {}) },
45 + });
65 - try {
66 - expect(extensionsResult.extensions.map((e) => e.path)).toEqual([leaksPath]);
67 - session.subscribe((event) => {
68 - if (event.type !== "message_end") return;
69 - const msg = event.message;
70 - if (msg.role !== "assistant") return;
71 - assistantTexts.push(JSON.stringify(msg.content));
72 - });
47 + const loader = new DefaultResourceLoader({
48 + cwd,
49 + agentDir,
50 + settingsManager,
51 + noExtensions: true,
52 + noSkills: true,
53 + noPromptTemplates: true,
54 + noThemes: true,
55 + noContextFiles: true,
56 + additionalExtensionPaths: [leaksPath],
57 + });
58 + await loader.reload();
74 - const fake = "sk-or-v1-" + "1".repeat(64);
75 - await session.prompt(`Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`);
60 + const authStorage = AuthStorage.create();
61 + const modelRegistry = ModelRegistry.create(authStorage);
62 + const model = getModel("minimax", "MiniMax-M2.7-highspeed");
63 + const assistantTexts: string[] = [];
65 + const { session, extensionsResult } = await createAgentSession({
66 + cwd,
67 + agentDir,
68 + authStorage,
69 + modelRegistry,
70 + model,
71 + thinkingLevel: "off",
72 + sessionManager: SessionManager.inMemory(),
73 + settingsManager,
74 + resourceLoader: loader,
75 + noTools: "all",
76 + });
78 + expect(extensionsResult.extensions.map((e) => e.path)).toEqual([leaksPath]);
79 + session.subscribe((event) => {
80 + if (event.type !== "message_end") return;
81 + const msg = event.message;
82 + if (msg.role !== "assistant") return;
83 + assistantTexts.push(JSON.stringify(msg.content));
84 + });
86 + return {
87 + cwd,
88 + temp,
89 + session,
90 + assistantTexts,
91 + async prompt(text: string): Promise<string> {
92 + assistantTexts.length = 0;
93 + await session.prompt(text);
76 94 await session.agent.waitForIdle();
78 - const answer = assistantTexts.join("\n").toUpperCase();
79 - expect(answer).toContain("NO");
80 - expect(answer).not.toContain("YES");
81 - } finally {
95 + return assistantTexts.join("\n");
96 + },
97 + dispose() {
82 98 session.dispose();
83 99 fs.rmSync(temp, { recursive: true, force: true });
84 - }
85 - }, 120000);
100 + },
101 + };
104 +async function withRealSession<T>(opts: RealCase, fn: (s: RealSession) => Promise<T>): Promise<T> {
105 + const s = await createRealSession(opts);
106 + try {
107 + return await fn(s);
108 + } finally {
109 + s.dispose();
113 +function expectNo(answer: string) {
114 + const upper = answer.toUpperCase();
115 + expect(upper).toContain("NO");
116 + expect(upper).not.toContain("YES");
119 +function expectThreeNo(answer: string) {
120 + const upper = answer.toUpperCase();
121 + expect((upper.match(/\bNO\b/g) ?? []).length).toBeGreaterThanOrEqual(3);
122 + expect(upper).not.toContain("YES");
125 +d("real MiniMax integration", () => {
126 + it("redacts provider-bound builtin token before a real request", async () => {
127 + await withRealSession({}, async ({ prompt }) => {
128 + const fake = "sk-or-v1-" + "1".repeat(64);
129 + const answer = await prompt(`Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`);
130 + expectNo(answer);
131 + });
132 + }, TIMEOUT);
134 + it("redacts secretFiles, keywords, and custom regex before a real request", async () => {
135 + await withRealSession({
136 + settings: {
137 + secretFiles: [".env"],
138 + keywords: ["raven-codename"],
139 + regex: [{ pattern: "foo_[A-Za-z0-9]{32}", label: "foo" }],
140 + },
141 + files: {
142 + ".env": "LEAKS_DOTENV_VALUE=DENV-real-secret-123456789\n",
143 + },
144 + }, async ({ prompt }) => {
145 + const userRegex = "foo_" + "A".repeat(32);
146 + const answer = await prompt([
147 + "Answer three words, each YES or NO.",
148 + "1. Does first bracket start with DENV-?",
149 + "2. Does second bracket start with raven-?",
150 + "3. Does third bracket start with foo_?",
151 + `[DENV-real-secret-123456789] [raven-codename] [${userRegex}]`,
152 + ].join("\n"));
153 + expectThreeNo(answer);
154 + });
155 + }, TIMEOUT);
157 + it("redacts generic assignment secrets but not harmless values in a real request", async () => {
158 + await withRealSession({}, async ({ prompt }) => {
159 + const strong = "aB9_qP2xZ7mN4rT8vL0s";
160 + const answer = await prompt([
161 + "Answer two words, each YES or NO.",
162 + "1. Does first assignment value start with aB9_?",
163 + "2. Does second assignment value equal changeme?",
164 + `[token = ${strong}] [password = changeme]`,
165 + ].join("\n"));
166 + const upper = answer.toUpperCase();
167 + expect(upper).toContain("NO");
168 + expect(upper).toContain("YES");
169 + });
170 + }, TIMEOUT);
172 + it("redacts prior-session context on later real requests", async () => {
173 + await withRealSession({}, async ({ prompt }) => {
174 + const fake = "sk-or-v1-" + "2".repeat(64);
175 + await prompt(`Reply exactly OK. [${fake}]`);
176 + const answer = await prompt("Look at the previous user bracket content. Did it start with sk-or-v1-? Answer exactly YES or NO.");
177 + expectNo(answer);
178 + });
179 + }, TIMEOUT);
181 + it("restores assistant output locally after the real model repeats a placeholder", async () => {
182 + await withRealSession({}, async ({ prompt }) => {
183 + const fake = "sk-or-v1-" + "3".repeat(64);
184 + const answer = await prompt(`Repeat exactly the bracket content and nothing else. [${fake}]`);
185 + expect(answer).toContain(fake);
186 + expect(answer).not.toMatch(/__LEAK_[a-f0-9]{12}__/);
187 + });
188 + }, TIMEOUT);
190 + it("restores tool-call args after a real request creates a mapping", async () => {
191 + await withRealSession({}, async ({ session, prompt }) => {
192 + const fake = "sk-or-v1-" + "4".repeat(64);
193 + const answer = await prompt(`Repeat exactly the bracket content and nothing else. [${fake}]`);
194 + expect(answer).toContain(fake);
196 + const redacted = await session.extensionRunner.emitBeforeProviderRequest({ text: fake });
197 + const placeholder = JSON.stringify(redacted).match(/__LEAK_[a-f0-9]{12}__/)?.[0];
198 + expect(placeholder).toBeTruthy();
200 + const event = { type: "tool_call", toolName: "bash", toolCallId: "call-real", input: { command: `printf ${placeholder}` } };
201 + await session.extensionRunner.emitToolCall(event as never);
202 + expect(event.input.command).toBe(`printf ${fake}`);
203 + });
204 + }, TIMEOUT);