pi-ext
bugabingas pi extensions
owned by bugabinga
feat(leaks): add high-confidence built-in detectors
9c0ff87877f7d478bd8507a0b3105fdf418720e2
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (5)
extensions/leaks/__tests__/detectors.test.ts
Addednew file mode 100644 1 +import { describe, expect, it }
from "vitest";
2 +import { __test }
from "../index.ts";
4 +function expectRedacts(text
: string, label
: string) {
5 + const engine
= new __test.
LeaksEngine();
6 + const result
= engine.
redactText(text,
"provider");
7 + expect(result.count, label).
toBeGreaterThan(
0);
8 + expect(result.
text, label).not.
toBe(text);
11 +function expectVisible(text
: string, label
: string) {
12 + const engine
= new __test.
LeaksEngine();
13 + expect(engine.
redactText(text,
"provider").
text, label).
toBe(text);
16 +describe(
"leaks built-in detectors", ()
=> {
17 + it(
"covers minimum positive detector families", ()
=> {
19 + [
"ai-openrouter",
"sk-or-v1-" + "a".
repeat(
64)],
20 + [
"ai-openai",
"sk-proj-" + "b".
repeat(
48)],
21 + [
"vcs-github",
"ghp_" + "A".
repeat(
36)],
22 + [
"package-npm",
"npm_" + "c".
repeat(
36)],
23 + [
"cloud-aws-access",
"AKIA" + "A".
repeat(
16)],
24 + [
"cloud-google",
"AIza" + "B".
repeat(
35)],
25 + [
"payment-stripe",
"sk_live_" + "c".
repeat(
32)],
26 + [
"messaging-slack",
"xoxb-" + "d".
repeat(
24)],
27 + [
"webhook",
"https://hooks.slack.com/services/T00000000/B00000000/" + "e".
repeat(
24)],
28 + [
"mail",
"SG." + "f".
repeat(
22)
+ "." + "g".
repeat(
43)],
29 + [
"auth-cookie",
"Cookie: session=session-fixture-secret-123456789"],
30 + [
"oauth",
"ya29." + "h".
repeat(
80)],
31 + [
"db",
"postgres://alice:db-fixture-secret-123456789@db/app"],
32 + [
"redis",
"redis://:redis-fixture-secret-123456789@cache:6379/0"],
35 + for (
const [label, value]
of positives)
expectRedacts(value, label);
38 + it(
"redacts supported private-key material", ()
=> {
40 + [
"-----BEGIN PGP ",
"PRIVATE KEY BLOCK-----\nbody\n-----END PGP ",
"PRIVATE KEY BLOCK-----"].
join(
""),
41 + [
"-----BEGIN ENCRYPTED COSIGN ",
"PRIVATE KEY-----\nbody\n-----END ENCRYPTED COSIGN ",
"PRIVATE KEY-----"].
join(
""),
42 + "PuTTY-User-Key-File-3: ssh-rsa\nPrivate-Lines: 1\nbody",
43 + "AGE-SECRET-KEY-1" + "Q".
repeat(
58),
44 + "untrusted comment: minisign secret key\nRW" + "R".
repeat(
72),
47 + for (
const block
of blocks)
expectRedacts(block, block.
slice(
0,
16));
50 + it(
"redacts generic sensitive-key assignments listed in the spec", ()
=> {
68 + for (
const key
of keys)
expectRedacts(
`${key}=assignment-fixture-secret-123456789`, key);
71 + it(
"keeps explicit negative controls visible", ()
=> {
73 + [
"account",
"account_id=123456789012"],
74 + [
"customer",
"customer_id=cus_1234567890"],
75 + [
"org",
"org_id=org_1234567890"],
76 + [
"project",
"project_id=proj_1234567890"],
77 + [
"hostname",
"db.internal.example.test"],
78 + [
"domain",
"example.com"],
79 + [
"lineage",
"lineage=alpha-beta-gamma-delta"],
80 + [
"value",
"value=opaque-but-not-secret-123456789"],
81 + [
"ssh-public",
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMetadataOnlyKey user@example"],
82 + [
"certificate",
"-----BEGIN CERTIFICATE-----\nbody\n-----END CERTIFICATE-----"],
85 + for (
const [label, value]
of negatives)
expectVisible(value, label);
88 + it(
"exports a stable built-in rule count", ()
=> {
89 + expect(__test.builtinRules.
length).
toBeGreaterThanOrEqual(
12);
90 + expect(
new __test.
LeaksEngine().
stats().builtInRuleCount).
toBe(__test.builtinRules.
length);
extensions/leaks/index.ts
Modified 1 +import { builtinRules, findBuiltinSpans }
from "./src/detectors.ts";
3 4 highConfidenceSecretShape,
40 41 export const __test
= {
43 46 discoverDotenvFiles,
extensions/leaks/src/detectors.ts
Addednew file mode 100644 1 +import type { SecretSpan, SpanLabel }
from "./types.ts";
3 +export type BuiltinRule = {
9 + kind
?: SecretSpan[
"kind"];
10 + wholeSpan
?: boolean;
13 +export const builtinRules
: BuiltinRule[]
= [
14 + rule(
"openrouter",
"OPENROUTER_TOKEN",
"ai",
/\bsk-or-v1-[A-Za-z0-9]{32,}\b/dg),
15 + rule(
"openai",
"OPENAI_TOKEN",
"ai",
/\bsk-proj-[A-Za-z0-9_-]{32,}\b/dg),
16 + rule(
"anthropic",
"ANTHROPIC_TOKEN",
"ai",
/\bsk-ant-api03-[A-Za-z0-9_-]{32,}\b/dg),
17 + rule(
"huggingface",
"HUGGINGFACE_TOKEN",
"ai",
/\bhf_[A-Za-z0-9]{32,}\b/dg),
18 + rule(
"github",
"GITHUB_TOKEN",
"vcs",
/\bgh[pousr]_[A-Za-z0-9]{30,}\b/dg),
19 + rule(
"npm",
"NPM_TOKEN",
"package",
/\bnpm_[A-Za-z0-9]{30,}\b/dg),
20 + rule(
"aws-access-key",
"AWS_ACCESS_KEY_ID",
"cloud",
/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/dg),
21 + rule(
"google-api-key",
"GOOGLE_API_KEY",
"cloud",
/\bAIza[A-Za-z0-9_-]{35}\b/dg),
22 + rule(
"stripe-live",
"STRIPE_LIVE_SECRET_KEY",
"payment",
/\bsk_live_[A-Za-z0-9]{24,}\b/dg),
23 + rule(
"slack",
"SLACK_TOKEN",
"messaging",
/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/dg),
24 + rule(
"slack-webhook",
"SLACK_WEBHOOK",
"webhook",
/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]{20,}/dg),
25 + rule(
"sendgrid",
"SENDGRID_TOKEN",
"mail",
/\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{32,}\b/dg),
26 + rule(
"oauth-google",
"OAUTH_TOKEN",
"auth",
/\bya29\.[A-Za-z0-9_-]{32,}\b/dg),
27 + rule(
"pgp-private",
"PGP_PRIVATE_KEY",
"private-key",
armorRegex(
"PGP",
"PRIVATE KEY BLOCK"),
0,
"private-key",
true),
28 + rule(
"cosign-private",
"COSIGN_PRIVATE_KEY",
"private-key",
armorRegex(
"ENCRYPTED COSIGN",
"PRIVATE KEY"),
0,
"private-key",
true),
29 + rule(
"putty-private",
"PUTTY_PRIVATE_KEY",
"private-key",
/PuTTY-User-Key-File-[\s\S]*?Private-Lines:\s*\d+[\s\S]*/dg,
0,
"private-key",
true),
30 + rule(
"age-secret",
"AGE_SECRET_KEY",
"private-key",
/\bAGE-SECRET-KEY-1[0-9A-Z]{40,}\b/dg,
0,
"private-key",
true),
31 + rule(
"minisign-secret",
"MINISIGN_SECRET_KEY",
"private-key",
/untrusted comment:\s*minisign secret key\s+RW[A-Za-z0-9+\/]{40,}/dgi,
0,
"private-key",
true),
34 +export function findBuiltinSpans(text
: string)
: SecretSpan[] {
35 + const spans
: SecretSpan[]
= [];
36 + for (
const item
of builtinRules) {
37 + item.regex.
lastIndex = 0;
38 + for (
const match
of text.
matchAll(item.regex)) {
39 + const group
= item.group
?? 0;
40 + const value
= match[group]
?? "";
41 + const range
= match.indices?.[group];
42 + if (
!range
|| value.
length === 0)
continue;
47 + labels: [
label(item.tag)],
48 + kind: item.kind
?? "builtin",
49 + confidence: item.wholeSpan
? 100 : 90,
50 + wholeSpan: item.wholeSpan
?? false,
63 + kind
: SecretSpan[
"kind"]
= "builtin",
66 + return { id, tag, family, regex, group, kind, wholeSpan };
69 +function label(tag
: string)
: SpanLabel {
70 + return { tag, priority:
40, source:
"builtin" };
73 +function armorRegex(prefix
: string, name
: string) {
74 + const marker
= `${prefix} ${name}`;
75 + return new RegExp(
`-----BEGIN ${marker}-----[\\s\\S]*?-----END ${marker}-----`,
"dg");
extensions/leaks/src/engine.ts
Modified 1 +import { builtinRules, findBuiltinSpans }
from "./detectors.ts";
2 3 learnEnvironmentValues,
61 62 redactText(text: string, origin: RedactionOrigin
= "provider"): RedactionResult {
62 63 let spans
: SecretSpan[]
= [
63 64 ...findStructureSpans(text),
65 + ...findBuiltinSpans(text),
64 66 ...customRegexSpans(text, this.customRules),
65 67 ...learnedValueSpans(text, this.learned),
129 131 stats(): EngineStats {
132 - builtInRuleCount:
0,
134 + builtInRuleCount: builtinRules.
length,
133 135 customRegexCount: this.customRules.
length,
134 136 regexWarningCount: this.warnings.
filter((warning)
=> warning.
code.
startsWith(
"settings.regex.")).
length,
135 137 warningCount: this.warnings.
length,
extensions/leaks/src/structures.ts
Modified 2 2 import { highConfidenceSecretShape, isSensitiveKey }
from "./dotenv.ts";
3 3 import { isWeakValue }
from "./safety.ts";
4 4 import type { SecretSpan, SpanLabel }
from "./types.ts";
6 -const URL_CREDENTIAL_RE
= /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]+):([^\s@/]+)@[^\s]+/dgi;
6 +const URL_CREDENTIAL_RE
= /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]*)?:([^\s@/]+)@[^\s]+/dgi;
7 7 const QUERY_RE
= /([?&])([A-Za-z0-9_-]*(?:token|key|secret|password|auth)[A-Za-z0-9_-]*)=([^&#\s'"`]+)/dgi;
8 8 const AUTH_RE
= /Authorization:\s*(Bearer|Basic)\s+([^\s'"`]+)/dgi;
9 9 const X_API_RE
= /\b(X-API-Key|X-Api-Key|Api-Key):\s*([^\s'"`]+)/dg;