pi-ext
bugabingas pi extensions
owned by bugabinga
feat(leaks): redact secret structure components
1e89b3b867727620f45f5cb5c4c7c05f2ff59963
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (4)
extensions/leaks/__tests__/structures.test.ts
Addednew file mode 100644 1 +import { describe, expect, it }
from "vitest";
2 +import { __test }
from "../index.ts";
4 +const secret
= "structure-fixture-secret-123456789";
5 +const jwt
= [
"eyJhbGciOiJIUzI1NiJ9",
"eyJzdWIiOiIxMjM0NTY3ODkwIn0",
"c2lnbmF0dXJl"].
join(
".");
6 +const privateKey
= "-----BEGIN OPENSSH PRIVATE KEY-----\nfixture-private-key-body\n-----END OPENSSH PRIVATE KEY-----";
8 +describe(
"leaks structured component parsers", ()
=> {
9 + it(
"redacts credentialed URL passwords while preserving safe URL structure", ()
=> {
10 + const engine
= new __test.
LeaksEngine();
11 + const text
= `DATABASE_URL=postgres://alice:${secret}@db/app`;
12 + const result
= engine.
redactText(text,
"provider");
14 + expect(result.
text).
toContain(
"DATABASE_URL=postgres://alice:");
15 + expect(result.
text).
toContain(
"@db/app");
16 + expect(result.
text.
includes(secret)).
toBe(
false);
19 + it(
"redacts sensitive query values and preserves names", ()
=> {
20 + const engine
= new __test.
LeaksEngine();
21 + const text
= `https://api.example.test/callback?token=${secret}&user=me`;
22 + const result
= engine.
redactText(text,
"provider");
24 + expect(result.
text).
toContain(
"token=");
25 + expect(result.
text).
toContain(
"&user=me");
26 + expect(result.
text.
includes(secret)).
toBe(
false);
29 + it(
"does not redact URLs without credentials even for database keys", ()
=> {
30 + const engine
= new __test.
LeaksEngine();
31 + const text
= "DATABASE_URL=postgres://db/app REDIS_URL=redis://cache MONGO_URL=mongodb://mongo/app";
32 + expect(engine.
redactText(text,
"provider").
text).
toBe(text);
35 + it(
"redacts header values only", ()
=> {
36 + const engine
= new __test.
LeaksEngine();
37 + const text
= `Authorization: Bearer ${secret}\nAuthorization: Basic ${secret}\nX-API-Key: ${secret}`;
38 + const result
= engine.
redactText(text,
"provider");
40 + expect(result.
text).
toContain(
"Authorization: Bearer ");
41 + expect(result.
text).
toContain(
"Authorization: Basic ");
42 + expect(result.
text).
toContain(
"X-API-Key: ");
43 + expect(result.
text.
includes(secret)).
toBe(
false);
44 + expect(result.count).
toBe(
3);
47 + it(
"redacts sensitive cookie values only", ()
=> {
48 + const engine
= new __test.
LeaksEngine();
49 + const result
= engine.
redactText(
`Cookie: session=${secret}; theme=dark`,
"provider");
51 + expect(result.
text).
toContain(
"Cookie: session=");
52 + expect(result.
text).
toContain(
"; theme=dark");
53 + expect(result.
text.
includes(secret)).
toBe(
false);
56 + it(
"redacts assignment values across common one-line syntaxes", ()
=> {
57 + const engine
= new __test.
LeaksEngine();
62 + `"token": "${secret}"`,
63 + `token = "${secret}"`,
64 + `<token>${secret}</token>`,
65 + `export TOKEN=${secret}`,
68 + for (
const sample
of samples) {
69 + const result
= engine.
redactText(sample,
"provider");
70 + expect(result.
text.
includes(secret)).
toBe(
false);
71 + expect(result.
text.
toLowerCase(), sample).
toContain(
"token".
slice(
0,
3));
75 + it(
"redacts shell header value without redacting whole command", ()
=> {
76 + const engine
= new __test.
LeaksEngine();
77 + const result
= engine.
redactText(
`curl -H 'Authorization: Bearer ${secret}' https://example.test`,
"provider");
79 + expect(result.
text).
toContain(
"curl -H '");
80 + expect(result.
text).
toContain(
"https://example.test");
81 + expect(result.
text.
includes(secret)).
toBe(
false);
84 + it(
"redacts JWTs and private keys as whole spans", ()
=> {
85 + const engine
= new __test.
LeaksEngine();
86 + const jwtResult
= engine.
redactText(
`jwt=${jwt}`,
"provider");
87 + const keyResult
= engine.
redactText(privateKey,
"provider");
89 + expect(jwtResult.
text).
toMatch(
/^jwt=__LEAK_JWT_[a-f0-9]{12}__$/);
90 + expect(keyResult.
text).
toMatch(
/^__LEAK_OPENSSH_PRIVATE_KEY_[a-f0-9]{12}__$/);
extensions/leaks/index.ts
Modified19 19 } from
"./src/settings.ts";
21 + findAssignmentSpans,
23 + findCredentialedUrlSpans,
26 + findPrivateKeySpans,
27 + findSensitiveQuerySpans,
29 +}
from "./src/structures.ts";
21 31 isExistingPlaceholder,
58 + findCredentialedUrlSpans,
59 + findSensitiveQuerySpans,
62 + findAssignmentSpans,
64 + findPrivateKeySpans,
47 65 isExistingPlaceholder,
extensions/leaks/src/engine.ts
Modified 6 6 } from
"./dotenv.ts";
7 7 import { PlaceholderStore }
from "./placeholders.ts";
8 8 import { selectSecretSpans }
from "./selector.ts";
9 +import { findStructureSpans }
from "./structures.ts";
10 11 compileCustomRegex,
60 61 redactText(text: string, origin: RedactionOrigin
= "provider"): RedactionResult {
61 62 let spans
: SecretSpan[]
= [
63 + ...findStructureSpans(text),
62 64 ...customRegexSpans(text, this.customRules),
63 65 ...learnedValueSpans(text, this.learned),
extensions/leaks/src/structures.ts
Addednew file mode 100644 1 +import { highConfidenceSecretShape, isSensitiveKey }
from "./dotenv.ts";
2 +import { isWeakValue }
from "./safety.ts";
3 +import type { SecretSpan, SpanLabel }
from "./types.ts";
5 +const URL_CREDENTIAL_RE
= /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]+):([^\s@/]+)@[^\s]+/dgi;
6 +const QUERY_RE
= /([?&])([A-Za-z0-9_-]*(?:token|key|secret|password|auth)[A-Za-z0-9_-]*)=([^&#\s'"`]+)/dgi;
7 +const AUTH_RE
= /Authorization:\s*(Bearer|Basic)\s+([^\s'"`]+)/dgi;
8 +const X_API_RE
= /\b(X-API-Key|X-Api-Key|Api-Key):\s*([^\s'"`]+)/dg;
9 +const COOKIE_LINE_RE
= /Cookie:\s*([^\n]+)/dgi;
10 +const ASSIGN_RE
= /(?:export\s+)?["']?([A-Za-z_][A-Za-z0-9_-]*)["']?\s*(?:=|:|=>)\s*["']?([^\s'",;}]+)["']?/dgi;
11 +const XML_RE
= /<([A-Za-z_][A-Za-z0-9_-]*)>([^<]+)<\/\1>/dgi;
12 +const JWT_RE
= /\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{8,}\b/dg;
13 +const PRIVATE_KEY_RE
= /-----BEGIN ([A-Z0-9 ]*PRIVATE KEY)-----[\s\S]*?-----END \1-----/dg;
15 +export function findStructureSpans(text
: string)
: SecretSpan[] {
17 + ...findPrivateKeySpans(text),
18 + ...findJwtSpans(text),
19 + ...findCredentialedUrlSpans(text),
20 + ...findSensitiveQuerySpans(text),
21 + ...findHeaderSpans(text),
22 + ...findCookieSpans(text),
23 + ...findAssignmentSpans(text),
27 +export function findCredentialedUrlSpans(text
: string)
: SecretSpan[] {
28 + return matches(URL_CREDENTIAL_RE, text,
2,
"DB_PASSWORD");
31 +export function findSensitiveQuerySpans(text
: string)
: SecretSpan[] {
32 + const spans
: SecretSpan[]
= [];
33 + for (
const match
of text.
matchAll(QUERY_RE)) {
34 + const value
= match[
3]
?? "";
35 + if (
isWeakValue(value))
continue;
36 + spans.
push(
makeSpan(match,
3, value, [
label(match[
2]
?? "QUERY_SECRET",
"structure")],
"structure"));
41 +export function findHeaderSpans(text
: string)
: SecretSpan[] {
43 + ...matches(AUTH_RE, text,
2,
"AUTHORIZATION", (match)
=> `AUTHORIZATION_${match[1]?.toUpperCase()}`),
44 + ...matches(X_API_RE, text,
2,
"X_API_KEY"),
48 +export function findCookieSpans(text
: string)
: SecretSpan[] {
49 + const spans
: SecretSpan[]
= [];
50 + for (
const line
of text.
matchAll(COOKIE_LINE_RE)) {
51 + const body
= line[
1]
?? "";
52 + let offset
= line.indices?.[
1]?.[
0]
?? 0;
53 + for (
const part
of body.
split(
";")) {
54 + const trimmed
= part.
trim();
55 + const eq
= trimmed.
indexOf(
"=");
57 + offset
+= part.
length + 1;
60 + const name
= trimmed.
slice(
0, eq);
61 + const value
= trimmed.
slice(eq
+ 1);
62 + const localStart
= part.
indexOf(value);
63 + if (
isSensitiveCookie(name)
&& !isWeakValue(value)) {
64 + const start
= offset
+ Math.
max(localStart,
0);
67 + end: start
+ value.
length,
69 + labels: [
label(
`COOKIE_${name}`,
"structure")],
75 + offset
+= part.
length + 1;
81 +export function findAssignmentSpans(text
: string)
: SecretSpan[] {
82 + const spans
: SecretSpan[]
= [];
83 + for (
const regex
of [ASSIGN_RE, XML_RE]) {
84 + regex.
lastIndex = 0;
85 + for (
const match
of text.
matchAll(regex)) {
86 + const key
= match[
1]
?? "";
87 + const value
= match[
2]
?? "";
88 + if (
!isSensitiveKey(key))
continue;
89 + if (
isWeakValue(value)
&& !highConfidenceSecretShape(value))
continue;
90 + spans.
push(
makeSpan(match,
2, value, [
label(key,
"structure")],
"structure"));
96 +export function findJwtSpans(text
: string)
: SecretSpan[] {
97 + return [
...text.
matchAll(JWT_RE)].
map((match)
=> {
98 + const start
= match.
index ?? 0;
101 + end: start
+ match[
0].
length,
103 + labels: [
label(
"JWT",
"builtin")],
111 +export function findPrivateKeySpans(text
: string)
: SecretSpan[] {
112 + return [
...text.
matchAll(PRIVATE_KEY_RE)].
map((match)
=> {
113 + const start
= match.
index ?? 0;
116 + end: start
+ match[
0].
length,
118 + labels: [
label((match[
1]
?? "PRIVATE KEY").
replace(
/ /g,
"_"),
"builtin")],
119 + kind:
"private-key",
131 + tag?: (match
: RegExpMatchArray)
=> string,
133 + regex.
lastIndex = 0;
134 + const spans
: SecretSpan[]
= [];
135 + for (
const match
of text.
matchAll(regex)) {
136 + const value
= match[group]
?? "";
137 + if (
isWeakValue(value))
continue;
138 + spans.
push(
makeSpan(match, group, value, [
label(
tag?.(match)
?? fallback,
"structure")],
"structure"));
143 +function makeSpan(match
: RegExpMatchArray, group
: number, value
: string, labels
: SpanLabel[], kind
: SecretSpan[
"kind"])
: SecretSpan {
144 + const range
= match.indices?.[group];
145 + const start
= range?.[
0]
?? 0;
148 + end: range?.[
1]
?? start
+ value.
length,
157 +function label(tag
: string, source
: SpanLabel[
"source"])
: SpanLabel {
158 + return { tag, priority: source
=== "builtin" ? 40 : 20, source };
161 +function isSensitiveCookie(name
: string) {
162 + return /(?:session|token|auth|secret|key|jwt)/i.
test(name);