repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

feat(leaks): redact secret structure components

1e89b3b867727620f45f5cb5c4c7c05f2ff59963

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (4)

extensions/leaks/__tests__/structures.test.ts

Added
new file mode 100644
1 +import { describe, expect, it } from "vitest";
2 +import { __test } from "../index.ts";
4 +const secret = "structure-fixture-secret-123456789";
5 +const jwt = ["eyJhbGciOiJIUzI1NiJ9", "eyJzdWIiOiIxMjM0NTY3ODkwIn0", "c2lnbmF0dXJl"].join(".");
6 +const privateKey = "-----BEGIN OPENSSH PRIVATE KEY-----\nfixture-private-key-body\n-----END OPENSSH PRIVATE KEY-----";
8 +describe("leaks structured component parsers", () => {
9 + it("redacts credentialed URL passwords while preserving safe URL structure", () => {
10 + const engine = new __test.LeaksEngine();
11 + const text = `DATABASE_URL=postgres://alice:${secret}@db/app`;
12 + const result = engine.redactText(text, "provider");
14 + expect(result.text).toContain("DATABASE_URL=postgres://alice:");
15 + expect(result.text).toContain("@db/app");
16 + expect(result.text.includes(secret)).toBe(false);
17 + });
19 + it("redacts sensitive query values and preserves names", () => {
20 + const engine = new __test.LeaksEngine();
21 + const text = `https://api.example.test/callback?token=${secret}&user=me`;
22 + const result = engine.redactText(text, "provider");
24 + expect(result.text).toContain("token=");
25 + expect(result.text).toContain("&user=me");
26 + expect(result.text.includes(secret)).toBe(false);
27 + });
29 + it("does not redact URLs without credentials even for database keys", () => {
30 + const engine = new __test.LeaksEngine();
31 + const text = "DATABASE_URL=postgres://db/app REDIS_URL=redis://cache MONGO_URL=mongodb://mongo/app";
32 + expect(engine.redactText(text, "provider").text).toBe(text);
33 + });
35 + it("redacts header values only", () => {
36 + const engine = new __test.LeaksEngine();
37 + const text = `Authorization: Bearer ${secret}\nAuthorization: Basic ${secret}\nX-API-Key: ${secret}`;
38 + const result = engine.redactText(text, "provider");
40 + expect(result.text).toContain("Authorization: Bearer ");
41 + expect(result.text).toContain("Authorization: Basic ");
42 + expect(result.text).toContain("X-API-Key: ");
43 + expect(result.text.includes(secret)).toBe(false);
44 + expect(result.count).toBe(3);
45 + });
47 + it("redacts sensitive cookie values only", () => {
48 + const engine = new __test.LeaksEngine();
49 + const result = engine.redactText(`Cookie: session=${secret}; theme=dark`, "provider");
51 + expect(result.text).toContain("Cookie: session=");
52 + expect(result.text).toContain("; theme=dark");
53 + expect(result.text.includes(secret)).toBe(false);
54 + });
56 + it("redacts assignment values across common one-line syntaxes", () => {
57 + const engine = new __test.LeaksEngine();
58 + const samples = [
59 + `TOKEN=${secret}`,
60 + `token=${secret}`,
61 + `token: ${secret}`,
62 + `"token": "${secret}"`,
63 + `token = "${secret}"`,
64 + `<token>${secret}</token>`,
65 + `export TOKEN=${secret}`,
66 + ];
68 + for (const sample of samples) {
69 + const result = engine.redactText(sample, "provider");
70 + expect(result.text.includes(secret)).toBe(false);
71 + expect(result.text.toLowerCase(), sample).toContain("token".slice(0, 3));
72 + }
73 + });
75 + it("redacts shell header value without redacting whole command", () => {
76 + const engine = new __test.LeaksEngine();
77 + const result = engine.redactText(`curl -H 'Authorization: Bearer ${secret}' https://example.test`, "provider");
79 + expect(result.text).toContain("curl -H '");
80 + expect(result.text).toContain("https://example.test");
81 + expect(result.text.includes(secret)).toBe(false);
82 + });
84 + it("redacts JWTs and private keys as whole spans", () => {
85 + const engine = new __test.LeaksEngine();
86 + const jwtResult = engine.redactText(`jwt=${jwt}`, "provider");
87 + const keyResult = engine.redactText(privateKey, "provider");
89 + expect(jwtResult.text).toMatch(/^jwt=__LEAK_JWT_[a-f0-9]{12}__$/);
90 + expect(keyResult.text).toMatch(/^__LEAK_OPENSSH_PRIVATE_KEY_[a-f0-9]{12}__$/);
91 + });
92 +});

extensions/leaks/index.ts

Modified
17 17 isUnsafeRegex,
18 18 loadLeaksSettings,
19 19 } from "./src/settings.ts";
20 +import {
21 + findAssignmentSpans,
22 + findCookieSpans,
23 + findCredentialedUrlSpans,
24 + findHeaderSpans,
25 + findJwtSpans,
26 + findPrivateKeySpans,
27 + findSensitiveQuerySpans,
28 + findStructureSpans,
29 +} from "./src/structures.ts";
20 30 import {
21 31 isExistingPlaceholder,
22 32 isPlainObject,
44 54 customRegexSpans,
45 55 countCaptures,
46 56 isUnsafeRegex,
57 + findStructureSpans,
58 + findCredentialedUrlSpans,
59 + findSensitiveQuerySpans,
60 + findHeaderSpans,
61 + findCookieSpans,
62 + findAssignmentSpans,
63 + findJwtSpans,
64 + findPrivateKeySpans,
47 65 isExistingPlaceholder,
48 66 isPlainObject,
49 67 isWeakValue,

extensions/leaks/src/engine.ts

Modified
6 6 } from "./dotenv.ts";
7 7 import { PlaceholderStore } from "./placeholders.ts";
8 8 import { selectSecretSpans } from "./selector.ts";
9 +import { findStructureSpans } from "./structures.ts";
9 10 import {
10 11 compileCustomRegex,
11 12 customRegexSpans,
60 61 redactText(text: string, origin: RedactionOrigin = "provider"): RedactionResult {
61 62 let spans: SecretSpan[] = [
63 + ...findStructureSpans(text),
62 64 ...customRegexSpans(text, this.customRules),
63 65 ...learnedValueSpans(text, this.learned),

extensions/leaks/src/structures.ts

Added
new file mode 100644
1 +import { highConfidenceSecretShape, isSensitiveKey } from "./dotenv.ts";
2 +import { isWeakValue } from "./safety.ts";
3 +import type { SecretSpan, SpanLabel } from "./types.ts";
5 +const URL_CREDENTIAL_RE = /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]+):([^\s@/]+)@[^\s]+/dgi;
6 +const QUERY_RE = /([?&])([A-Za-z0-9_-]*(?:token|key|secret|password|auth)[A-Za-z0-9_-]*)=([^&#\s'"`]+)/dgi;
7 +const AUTH_RE = /Authorization:\s*(Bearer|Basic)\s+([^\s'"`]+)/dgi;
8 +const X_API_RE = /\b(X-API-Key|X-Api-Key|Api-Key):\s*([^\s'"`]+)/dg;
9 +const COOKIE_LINE_RE = /Cookie:\s*([^\n]+)/dgi;
10 +const ASSIGN_RE = /(?:export\s+)?["']?([A-Za-z_][A-Za-z0-9_-]*)["']?\s*(?:=|:|=>)\s*["']?([^\s'",;}]+)["']?/dgi;
11 +const XML_RE = /<([A-Za-z_][A-Za-z0-9_-]*)>([^<]+)<\/\1>/dgi;
12 +const JWT_RE = /\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{8,}\b/dg;
13 +const PRIVATE_KEY_RE = /-----BEGIN ([A-Z0-9 ]*PRIVATE KEY)-----[\s\S]*?-----END \1-----/dg;
15 +export function findStructureSpans(text: string): SecretSpan[] {
16 + return [
17 + ...findPrivateKeySpans(text),
18 + ...findJwtSpans(text),
19 + ...findCredentialedUrlSpans(text),
20 + ...findSensitiveQuerySpans(text),
21 + ...findHeaderSpans(text),
22 + ...findCookieSpans(text),
23 + ...findAssignmentSpans(text),
24 + ];
27 +export function findCredentialedUrlSpans(text: string): SecretSpan[] {
28 + return matches(URL_CREDENTIAL_RE, text, 2, "DB_PASSWORD");
31 +export function findSensitiveQuerySpans(text: string): SecretSpan[] {
32 + const spans: SecretSpan[] = [];
33 + for (const match of text.matchAll(QUERY_RE)) {
34 + const value = match[3] ?? "";
35 + if (isWeakValue(value)) continue;
36 + spans.push(makeSpan(match, 3, value, [label(match[2] ?? "QUERY_SECRET", "structure")], "structure"));
37 + }
38 + return spans;
41 +export function findHeaderSpans(text: string): SecretSpan[] {
42 + return [
43 + ...matches(AUTH_RE, text, 2, "AUTHORIZATION", (match) => `AUTHORIZATION_${match[1]?.toUpperCase()}`),
44 + ...matches(X_API_RE, text, 2, "X_API_KEY"),
45 + ];
48 +export function findCookieSpans(text: string): SecretSpan[] {
49 + const spans: SecretSpan[] = [];
50 + for (const line of text.matchAll(COOKIE_LINE_RE)) {
51 + const body = line[1] ?? "";
52 + let offset = line.indices?.[1]?.[0] ?? 0;
53 + for (const part of body.split(";")) {
54 + const trimmed = part.trim();
55 + const eq = trimmed.indexOf("=");
56 + if (eq === -1) {
57 + offset += part.length + 1;
58 + continue;
59 + }
60 + const name = trimmed.slice(0, eq);
61 + const value = trimmed.slice(eq + 1);
62 + const localStart = part.indexOf(value);
63 + if (isSensitiveCookie(name) && !isWeakValue(value)) {
64 + const start = offset + Math.max(localStart, 0);
65 + spans.push({
66 + start,
67 + end: start + value.length,
68 + value,
69 + labels: [label(`COOKIE_${name}`, "structure")],
70 + kind: "structure",
71 + confidence: 85,
72 + wholeSpan: false,
73 + });
74 + }
75 + offset += part.length + 1;
76 + }
77 + }
78 + return spans;
81 +export function findAssignmentSpans(text: string): SecretSpan[] {
82 + const spans: SecretSpan[] = [];
83 + for (const regex of [ASSIGN_RE, XML_RE]) {
84 + regex.lastIndex = 0;
85 + for (const match of text.matchAll(regex)) {
86 + const key = match[1] ?? "";
87 + const value = match[2] ?? "";
88 + if (!isSensitiveKey(key)) continue;
89 + if (isWeakValue(value) && !highConfidenceSecretShape(value)) continue;
90 + spans.push(makeSpan(match, 2, value, [label(key, "structure")], "structure"));
91 + }
92 + }
93 + return spans;
96 +export function findJwtSpans(text: string): SecretSpan[] {
97 + return [...text.matchAll(JWT_RE)].map((match) => {
98 + const start = match.index ?? 0;
99 + return {
100 + start,
101 + end: start + match[0].length,
102 + value: match[0],
103 + labels: [label("JWT", "builtin")],
104 + kind: "jwt",
105 + confidence: 100,
106 + wholeSpan: true,
107 + };
108 + });
111 +export function findPrivateKeySpans(text: string): SecretSpan[] {
112 + return [...text.matchAll(PRIVATE_KEY_RE)].map((match) => {
113 + const start = match.index ?? 0;
114 + return {
115 + start,
116 + end: start + match[0].length,
117 + value: match[0],
118 + labels: [label((match[1] ?? "PRIVATE KEY").replace(/ /g, "_"), "builtin")],
119 + kind: "private-key",
120 + confidence: 100,
121 + wholeSpan: true,
122 + };
123 + });
126 +function matches(
127 + regex: RegExp,
128 + text: string,
129 + group: number,
130 + fallback: string,
131 + tag?: (match: RegExpMatchArray) => string,
132 +): SecretSpan[] {
133 + regex.lastIndex = 0;
134 + const spans: SecretSpan[] = [];
135 + for (const match of text.matchAll(regex)) {
136 + const value = match[group] ?? "";
137 + if (isWeakValue(value)) continue;
138 + spans.push(makeSpan(match, group, value, [label(tag?.(match) ?? fallback, "structure")], "structure"));
140 + return spans;
143 +function makeSpan(match: RegExpMatchArray, group: number, value: string, labels: SpanLabel[], kind: SecretSpan["kind"]): SecretSpan {
144 + const range = match.indices?.[group];
145 + const start = range?.[0] ?? 0;
146 + return {
147 + start,
148 + end: range?.[1] ?? start + value.length,
149 + value,
150 + labels,
151 + kind,
152 + confidence: 85,
153 + wholeSpan: false,
154 + };
157 +function label(tag: string, source: SpanLabel["source"]): SpanLabel {
158 + return { tag, priority: source === "builtin" ? 40 : 20, source };
161 +function isSensitiveCookie(name: string) {
162 + return /(?:session|token|auth|secret|key|jwt)/i.test(name);