repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

test(leaks): remove legacy e2e cruft

d0a426b1066688bd708bd95236607b1ac29c39a0

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (1)

extensions/leaks/__e2e__/minimax.test.ts

Deleted
deleted file mode 100644
1 -import fs from "node:fs";
2 -import os from "node:os";
3 -import path from "node:path";
4 -import { getModel } from "@earendil-works/pi-ai";
5 -import {
6 - AuthStorage,
7 - createAgentSession,
8 - DefaultResourceLoader,
9 - ModelRegistry,
10 - SessionManager,
11 - SettingsManager,
12 -} from "@earendil-works/pi-coding-agent";
13 -import { describe, expect, it } from "vitest";
15 -const real = process.env.LEAKS_REAL === "1";
16 -const d = real ? describe : describe.skip;
17 -const TIMEOUT = 120000;
19 -type RealCase = {
20 - settings?: Record<string, unknown>;
21 - files?: Record<string, string>;
22 -};
24 -type RealSession = Awaited<ReturnType<typeof createRealSession>>;
26 -function textOnly(content: unknown): string {
27 - if (typeof content === "string") return content;
28 - if (!Array.isArray(content)) return JSON.stringify(content);
29 - return content
30 - .filter(
31 - (part): part is { type?: string; text?: string } =>
32 - !!part &&
33 - typeof part === "object" &&
34 - (part as { type?: string }).type === "text",
35 - )
36 - .map((part) => part.text ?? "")
37 - .join("\n");
40 -async function createRealSession(opts: RealCase = {}) {
41 - const temp = fs.mkdtempSync(path.join(os.tmpdir(), "leaks-real-"));
42 - const cwd = path.join(temp, "cwd");
43 - const agentDir = path.join(temp, "agent");
44 - fs.mkdirSync(cwd, { recursive: true });
45 - fs.mkdirSync(agentDir, { recursive: true });
47 - for (const [rel, content] of Object.entries(opts.files ?? {})) {
48 - const file = path.join(cwd, rel);
49 - fs.mkdirSync(path.dirname(file), { recursive: true });
50 - fs.writeFileSync(file, content, "utf8");
51 - }
52 - fs.mkdirSync(path.join(cwd, ".pi"), { recursive: true });
53 - fs.writeFileSync(
54 - path.join(cwd, ".pi", "settings.json"),
55 - JSON.stringify(
56 - { leaks: { enabled: true, secretFiles: [], ...(opts.settings ?? {}) } },
57 - null,
58 - 2,
59 - ),
60 - "utf8",
61 - );
63 - const leaksPath = path.resolve(import.meta.dirname, "../../index.ts");
64 - const settingsManager = SettingsManager.inMemory({
65 - defaultProvider: "minimax",
66 - defaultModel: "MiniMax-M2.7-highspeed",
67 - defaultThinkingLevel: "off",
68 - leaks: { enabled: true, secretFiles: [], ...(opts.settings ?? {}) },
69 - });
71 - const loader = new DefaultResourceLoader({
72 - cwd,
73 - agentDir,
74 - settingsManager,
75 - noExtensions: true,
76 - noSkills: true,
77 - noPromptTemplates: true,
78 - noThemes: true,
79 - noContextFiles: true,
80 - additionalExtensionPaths: [leaksPath],
81 - });
82 - await loader.reload();
84 - const authStorage = AuthStorage.create();
85 - const modelRegistry = ModelRegistry.create(authStorage);
86 - const model = getModel("minimax", "MiniMax-M2.7-highspeed");
87 - const assistantTexts: string[] = [];
89 - const { session, extensionsResult } = await createAgentSession({
90 - cwd,
91 - agentDir,
92 - authStorage,
93 - modelRegistry,
94 - model,
95 - thinkingLevel: "off",
96 - sessionManager: SessionManager.inMemory(),
97 - settingsManager,
98 - resourceLoader: loader,
99 - noTools: "all",
100 - });
102 - expect(extensionsResult.extensions.map((e) => e.path)).toEqual([leaksPath]);
103 - session.subscribe((event) => {
104 - if (event.type !== "message_end") return;
105 - const msg = event.message;
106 - if (msg.role !== "assistant") return;
107 - assistantTexts.push(textOnly(msg.content));
108 - });
110 - return {
111 - cwd,
112 - temp,
113 - session,
114 - assistantTexts,
115 - async prompt(text: string): Promise<string> {
116 - assistantTexts.length = 0;
117 - await session.prompt(text);
118 - await session.agent.waitForIdle();
119 - return assistantTexts.join("\n");
120 - },
121 - async command(args: string): Promise<void> {
122 - const cmd = extensionsResult.extensions[0]?.commands.get("leaks");
123 - if (!cmd) throw new Error("missing /leaks command");
124 - await cmd.handler(args, session.extensionRunner.createContext());
125 - },
126 - writeSettings(leaks: Record<string, unknown>): void {
127 - fs.writeFileSync(
128 - path.join(cwd, ".pi", "settings.json"),
129 - JSON.stringify({ leaks }, null, 2),
130 - "utf8",
131 - );
132 - },
133 - dispose() {
134 - session.dispose();
135 - fs.rmSync(temp, { recursive: true, force: true });
136 - },
137 - };
140 -async function withRealSession<T>(
141 - opts: RealCase,
142 - fn: (s: RealSession) => Promise<T>,
143 -): Promise<T> {
144 - const s = await createRealSession(opts);
145 - try {
146 - return await fn(s);
147 - } finally {
148 - s.dispose();
149 - }
152 -function expectNo(answer: string) {
153 - const upper = answer.toUpperCase();
154 - expect(upper).toContain("NO");
155 - expect(upper).not.toContain("YES");
158 -function expectYes(answer: string) {
159 - const upper = answer.toUpperCase();
160 - expect(upper).toContain("YES");
163 -function expectThreeNo(answer: string) {
164 - const upper = answer.toUpperCase();
165 - expect((upper.match(/\bNO\b/g) ?? []).length).toBeGreaterThanOrEqual(3);
166 - expect(upper).not.toContain("YES");
169 -d("real MiniMax integration", () => {
170 - it(
171 - "redacts provider-bound builtin token before a real request",
172 - async () => {
173 - await withRealSession({}, async ({ prompt }) => {
174 - const fake = "sk-or-v1-" + "1".repeat(64);
175 - const answer = await prompt(
176 - `Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`,
177 - );
178 - expectNo(answer);
179 - });
180 - },
181 - TIMEOUT,
182 - );
184 - it(
185 - "redacts secretFiles, keywords, and custom regex before a real request",
186 - async () => {
187 - await withRealSession(
188 - {
189 - settings: {
190 - secretFiles: [".env"],
191 - keywords: ["raven-codename"],
192 - regex: [{ pattern: "foo_[A-Za-z0-9]{32}", label: "foo" }],
193 - },
194 - files: {
195 - ".env": "LEAKS_DOTENV_VALUE=DENV-real-secret-123456789\n",
196 - },
197 - },
198 - async ({ prompt }) => {
199 - const userRegex = "foo_" + "A".repeat(32);
200 - const answer = await prompt(
201 - [
202 - "Answer three words, each YES or NO.",
203 - "1. Does first bracket start with DENV-?",
204 - "2. Does second bracket start with raven-?",
205 - "3. Does third bracket start with foo_?",
206 - `[DENV-real-secret-123456789] [raven-codename] [${userRegex}]`,
207 - ].join("\n"),
208 - );
209 - expectThreeNo(answer);
210 - },
211 - );
212 - },
213 - TIMEOUT,
214 - );
216 - it(
217 - "redacts generic assignment secrets but not harmless values in a real request",
218 - async () => {
219 - await withRealSession({}, async ({ prompt }) => {
220 - const strong = "aB9_qP2xZ7mN4rT8vL0s";
221 - const answer = await prompt(
222 - [
223 - "Answer two words, each YES or NO.",
224 - "1. Does first assignment value start with aB9_?",
225 - "2. Does second assignment value equal changeme?",
226 - `[token = ${strong}] [password = changeme]`,
227 - ].join("\n"),
228 - );
229 - const upper = answer.toUpperCase();
230 - expect(upper).toContain("NO");
231 - expect(upper).toContain("YES");
232 - });
233 - },
234 - TIMEOUT,
235 - );
237 - it(
238 - "redacts prior-session context on later real requests",
239 - async () => {
240 - await withRealSession({}, async ({ prompt }) => {
241 - const fake = "sk-or-v1-" + "2".repeat(64);
242 - await prompt(`Reply exactly OK. [${fake}]`);
243 - const answer = await prompt(
244 - "Look at the previous user bracket content. Did it start with sk-or-v1-? Answer exactly YES or NO.",
245 - );
246 - expectNo(answer);
247 - });
248 - },
249 - TIMEOUT,
250 - );
252 - it(
253 - "restores assistant output locally after the real model repeats a placeholder",
254 - async () => {
255 - await withRealSession({}, async ({ prompt }) => {
256 - const fake = "sk-or-v1-" + "3".repeat(64);
257 - const answer = await prompt(
258 - `Repeat exactly the bracket content and nothing else. [${fake}]`,
259 - );
260 - expect(answer).toContain(fake);
261 - expect(answer).not.toMatch(/__LEAK_[a-f0-9]{12}__/);
262 - });
263 - },
264 - TIMEOUT,
265 - );
267 - it(
268 - "honors disabled setting in a real request",
269 - async () => {
270 - await withRealSession(
271 - { settings: { enabled: false } },
272 - async ({ prompt }) => {
273 - const fake = "sk-or-v1-" + "5".repeat(64);
274 - const answer = await prompt(
275 - `Look only at the bracket content. Does it start with sk-or-v1-? Answer exactly YES or NO. [${fake}]`,
276 - );
277 - expectYes(answer);
278 - },
279 - );
280 - },
281 - TIMEOUT,
282 - );
284 - it(
285 - "honors exclude setting in a real request",
286 - async () => {
287 - await withRealSession(
288 - {
289 - settings: {
290 - secretFiles: [".env"],
291 - exclude: ["DENV-visible-secret-123456789"],
292 - },
293 - files: { ".env": "VISIBLE=DENV-visible-secret-123456789\n" },
294 - },
295 - async ({ prompt }) => {
296 - const answer = await prompt(
297 - "Look only at the bracket content. Does it start with DENV-visible? Answer exactly YES or NO. [DENV-visible-secret-123456789]",
298 - );
299 - expectYes(answer);
300 - },
301 - );
302 - },
303 - TIMEOUT,
304 - );
306 - it(
307 - "reload command applies edited settings before later real requests",
308 - async () => {
309 - await withRealSession({}, async ({ prompt, command, writeSettings }) => {
310 - const before = await prompt(
311 - "Look only at the bracket content. Does it start with raven-? Answer exactly YES or NO. [raven-reload]",
312 - );
313 - expectYes(before);
315 - writeSettings({
316 - enabled: true,
317 - secretFiles: [],
318 - keywords: ["raven-reload"],
319 - });
320 - await command("reload");
322 - const after = await prompt(
323 - "Look only at the bracket content. Does it start with raven-? Answer exactly YES or NO. [raven-reload]",
324 - );
325 - expectNo(after);
326 - });
327 - },
328 - TIMEOUT,
329 - );
331 - it(
332 - "clear command drops mappings after a real request",
333 - async () => {
334 - await withRealSession({}, async ({ session, prompt, command }) => {
335 - const fake = "sk-or-v1-" + "6".repeat(64);
336 - await prompt(
337 - `Repeat exactly the bracket content and nothing else. [${fake}]`,
338 - );
339 - const redacted =
340 - await session.extensionRunner.emitBeforeProviderRequest({
341 - text: fake,
342 - });
343 - const placeholder = JSON.stringify(redacted).match(
344 - /__LEAK_[a-f0-9]{12}__/,
345 - )?.[0];
346 - expect(placeholder).toBeTruthy();
348 - await command("clear");
350 - const result = await session.extensionRunner.emitMessageEnd({
351 - type: "message_end",
352 - message: {
353 - role: "assistant",
354 - content: [{ type: "text", text: String(placeholder) }],
355 - },
356 - } as never);
357 - expect(result).toBeUndefined();
358 - });
359 - },
360 - TIMEOUT,
361 - );
363 - it(
364 - "restores tool-call args after a real request creates a mapping",
365 - async () => {
366 - await withRealSession({}, async ({ session, prompt }) => {
367 - const fake = "sk-or-v1-" + "4".repeat(64);
368 - const answer = await prompt(
369 - `Repeat exactly the bracket content and nothing else. [${fake}]`,
370 - );
371 - expect(answer).toContain(fake);
373 - const redacted =
374 - await session.extensionRunner.emitBeforeProviderRequest({
375 - text: fake,
376 - });
377 - const placeholder = JSON.stringify(redacted).match(
378 - /__LEAK_[a-f0-9]{12}__/,
379 - )?.[0];
380 - expect(placeholder).toBeTruthy();
382 - const event = {
383 - type: "tool_call",
384 - toolName: "bash",
385 - toolCallId: "call-real",
386 - input: { command: `printf ${placeholder}` },
387 - };
388 - await session.extensionRunner.emitToolCall(event as never);
389 - expect(event.input.command).toBe(`printf ${fake}`);
390 - });
391 - },
392 - TIMEOUT,
393 - );
394 -});