repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

feat(leaks): add high-confidence built-in detectors

9c0ff87877f7d478bd8507a0b3105fdf418720e2

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (5)

extensions/leaks/__tests__/detectors.test.ts

Added
new file mode 100644
1 +import { describe, expect, it } from "vitest";
2 +import { __test } from "../index.ts";
4 +function expectRedacts(text: string, label: string) {
5 + const engine = new __test.LeaksEngine();
6 + const result = engine.redactText(text, "provider");
7 + expect(result.count, label).toBeGreaterThan(0);
8 + expect(result.text, label).not.toBe(text);
9 +}
11 +function expectVisible(text: string, label: string) {
12 + const engine = new __test.LeaksEngine();
13 + expect(engine.redactText(text, "provider").text, label).toBe(text);
16 +describe("leaks built-in detectors", () => {
17 + it("covers minimum positive detector families", () => {
18 + const positives = [
19 + ["ai-openrouter", "sk-or-v1-" + "a".repeat(64)],
20 + ["ai-openai", "sk-proj-" + "b".repeat(48)],
21 + ["vcs-github", "ghp_" + "A".repeat(36)],
22 + ["package-npm", "npm_" + "c".repeat(36)],
23 + ["cloud-aws-access", "AKIA" + "A".repeat(16)],
24 + ["cloud-google", "AIza" + "B".repeat(35)],
25 + ["payment-stripe", "sk_live_" + "c".repeat(32)],
26 + ["messaging-slack", "xoxb-" + "d".repeat(24)],
27 + ["webhook", "https://hooks.slack.com/services/T00000000/B00000000/" + "e".repeat(24)],
28 + ["mail", "SG." + "f".repeat(22) + "." + "g".repeat(43)],
29 + ["auth-cookie", "Cookie: session=session-fixture-secret-123456789"],
30 + ["oauth", "ya29." + "h".repeat(80)],
31 + ["db", "postgres://alice:db-fixture-secret-123456789@db/app"],
32 + ["redis", "redis://:redis-fixture-secret-123456789@cache:6379/0"],
33 + ];
35 + for (const [label, value] of positives) expectRedacts(value, label);
36 + });
38 + it("redacts supported private-key material", () => {
39 + const blocks = [
40 + ["-----BEGIN PGP ", "PRIVATE KEY BLOCK-----\nbody\n-----END PGP ", "PRIVATE KEY BLOCK-----"].join(""),
41 + ["-----BEGIN ENCRYPTED COSIGN ", "PRIVATE KEY-----\nbody\n-----END ENCRYPTED COSIGN ", "PRIVATE KEY-----"].join(""),
42 + "PuTTY-User-Key-File-3: ssh-rsa\nPrivate-Lines: 1\nbody",
43 + "AGE-SECRET-KEY-1" + "Q".repeat(58),
44 + "untrusted comment: minisign secret key\nRW" + "R".repeat(72),
45 + ];
47 + for (const block of blocks) expectRedacts(block, block.slice(0, 16));
48 + });
50 + it("redacts generic sensitive-key assignments listed in the spec", () => {
51 + const keys = [
52 + "KEY",
53 + "KEY_DATA",
54 + "PASS",
55 + "PAT",
56 + "PASSPHRASE",
57 + "PEPPER",
58 + "LICENSE_KEY",
59 + "MASTER_KEY",
60 + "ROUTING_KEY",
61 + "ENCRYPTION_KEY",
62 + "SECRET_KEY_BASE",
63 + "SEED",
64 + "UNSEAL_KEY",
65 + "MNEMONIC",
66 + ];
68 + for (const key of keys) expectRedacts(`${key}=assignment-fixture-secret-123456789`, key);
69 + });
71 + it("keeps explicit negative controls visible", () => {
72 + const negatives = [
73 + ["account", "account_id=123456789012"],
74 + ["customer", "customer_id=cus_1234567890"],
75 + ["org", "org_id=org_1234567890"],
76 + ["project", "project_id=proj_1234567890"],
77 + ["hostname", "db.internal.example.test"],
78 + ["domain", "example.com"],
79 + ["lineage", "lineage=alpha-beta-gamma-delta"],
80 + ["value", "value=opaque-but-not-secret-123456789"],
81 + ["ssh-public", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMetadataOnlyKey user@example"],
82 + ["certificate", "-----BEGIN CERTIFICATE-----\nbody\n-----END CERTIFICATE-----"],
83 + ];
85 + for (const [label, value] of negatives) expectVisible(value, label);
86 + });
88 + it("exports a stable built-in rule count", () => {
89 + expect(__test.builtinRules.length).toBeGreaterThanOrEqual(12);
90 + expect(new __test.LeaksEngine().stats().builtInRuleCount).toBe(__test.builtinRules.length);
91 + });
92 +});

extensions/leaks/index.ts

Modified
1 +import { builtinRules, findBuiltinSpans } from "./src/detectors.ts";
1 2 import {
2 3 discoverDotenvFiles,
3 4 highConfidenceSecretShape,
40 41 export const __test = {
41 42 LeaksEngine,
42 43 PlaceholderStore,
44 + builtinRules,
45 + findBuiltinSpans,
43 46 discoverDotenvFiles,
44 47 parseDotenv,
45 48 loadDotenvValues,

extensions/leaks/src/detectors.ts

Added
new file mode 100644
1 +import type { SecretSpan, SpanLabel } from "./types.ts";
3 +export type BuiltinRule = {
4 + id: string;
5 + tag: string;
6 + family: string;
7 + regex: RegExp;
8 + group?: number;
9 + kind?: SecretSpan["kind"];
10 + wholeSpan?: boolean;
11 +};
13 +export const builtinRules: BuiltinRule[] = [
14 + rule("openrouter", "OPENROUTER_TOKEN", "ai", /\bsk-or-v1-[A-Za-z0-9]{32,}\b/dg),
15 + rule("openai", "OPENAI_TOKEN", "ai", /\bsk-proj-[A-Za-z0-9_-]{32,}\b/dg),
16 + rule("anthropic", "ANTHROPIC_TOKEN", "ai", /\bsk-ant-api03-[A-Za-z0-9_-]{32,}\b/dg),
17 + rule("huggingface", "HUGGINGFACE_TOKEN", "ai", /\bhf_[A-Za-z0-9]{32,}\b/dg),
18 + rule("github", "GITHUB_TOKEN", "vcs", /\bgh[pousr]_[A-Za-z0-9]{30,}\b/dg),
19 + rule("npm", "NPM_TOKEN", "package", /\bnpm_[A-Za-z0-9]{30,}\b/dg),
20 + rule("aws-access-key", "AWS_ACCESS_KEY_ID", "cloud", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/dg),
21 + rule("google-api-key", "GOOGLE_API_KEY", "cloud", /\bAIza[A-Za-z0-9_-]{35}\b/dg),
22 + rule("stripe-live", "STRIPE_LIVE_SECRET_KEY", "payment", /\bsk_live_[A-Za-z0-9]{24,}\b/dg),
23 + rule("slack", "SLACK_TOKEN", "messaging", /\bxox[baprs]-[A-Za-z0-9-]{20,}\b/dg),
24 + rule("slack-webhook", "SLACK_WEBHOOK", "webhook", /https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]{20,}/dg),
25 + rule("sendgrid", "SENDGRID_TOKEN", "mail", /\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{32,}\b/dg),
26 + rule("oauth-google", "OAUTH_TOKEN", "auth", /\bya29\.[A-Za-z0-9_-]{32,}\b/dg),
27 + rule("pgp-private", "PGP_PRIVATE_KEY", "private-key", armorRegex("PGP", "PRIVATE KEY BLOCK"), 0, "private-key", true),
28 + rule("cosign-private", "COSIGN_PRIVATE_KEY", "private-key", armorRegex("ENCRYPTED COSIGN", "PRIVATE KEY"), 0, "private-key", true),
29 + rule("putty-private", "PUTTY_PRIVATE_KEY", "private-key", /PuTTY-User-Key-File-[\s\S]*?Private-Lines:\s*\d+[\s\S]*/dg, 0, "private-key", true),
30 + rule("age-secret", "AGE_SECRET_KEY", "private-key", /\bAGE-SECRET-KEY-1[0-9A-Z]{40,}\b/dg, 0, "private-key", true),
31 + rule("minisign-secret", "MINISIGN_SECRET_KEY", "private-key", /untrusted comment:\s*minisign secret key\s+RW[A-Za-z0-9+\/]{40,}/dgi, 0, "private-key", true),
32 +];
34 +export function findBuiltinSpans(text: string): SecretSpan[] {
35 + const spans: SecretSpan[] = [];
36 + for (const item of builtinRules) {
37 + item.regex.lastIndex = 0;
38 + for (const match of text.matchAll(item.regex)) {
39 + const group = item.group ?? 0;
40 + const value = match[group] ?? "";
41 + const range = match.indices?.[group];
42 + if (!range || value.length === 0) continue;
43 + spans.push({
44 + start: range[0],
45 + end: range[1],
46 + value,
47 + labels: [label(item.tag)],
48 + kind: item.kind ?? "builtin",
49 + confidence: item.wholeSpan ? 100 : 90,
50 + wholeSpan: item.wholeSpan ?? false,
51 + });
52 + }
53 + }
54 + return spans;
57 +function rule(
58 + id: string,
59 + tag: string,
60 + family: string,
61 + regex: RegExp,
62 + group = 0,
63 + kind: SecretSpan["kind"] = "builtin",
64 + wholeSpan = false,
65 +): BuiltinRule {
66 + return { id, tag, family, regex, group, kind, wholeSpan };
69 +function label(tag: string): SpanLabel {
70 + return { tag, priority: 40, source: "builtin" };
73 +function armorRegex(prefix: string, name: string) {
74 + const marker = `${prefix} ${name}`;
75 + return new RegExp(`-----BEGIN ${marker}-----[\\s\\S]*?-----END ${marker}-----`, "dg");

extensions/leaks/src/engine.ts

Modified
1 +import { builtinRules, findBuiltinSpans } from "./detectors.ts";
1 2 import {
2 3 learnEnvironmentValues,
3 4 learnedValueSpans,
61 62 redactText(text: string, origin: RedactionOrigin = "provider"): RedactionResult {
62 63 let spans: SecretSpan[] = [
63 64 ...findStructureSpans(text),
65 + ...findBuiltinSpans(text),
64 66 ...customRegexSpans(text, this.customRules),
65 67 ...learnedValueSpans(text, this.learned),
129 131 stats(): EngineStats {
130 132 return {
131 133 active: true,
132 - builtInRuleCount: 0,
134 + builtInRuleCount: builtinRules.length,
133 135 customRegexCount: this.customRules.length,
134 136 regexWarningCount: this.warnings.filter((warning) => warning.code.startsWith("settings.regex.")).length,
135 137 warningCount: this.warnings.length,

extensions/leaks/src/structures.ts

Modified
2 2 import { highConfidenceSecretShape, isSensitiveKey } from "./dotenv.ts";
3 3 import { isWeakValue } from "./safety.ts";
4 4 import type { SecretSpan, SpanLabel } from "./types.ts";
6 -const URL_CREDENTIAL_RE = /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]+):([^\s@/]+)@[^\s]+/dgi;
6 +const URL_CREDENTIAL_RE = /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]*)?:([^\s@/]+)@[^\s]+/dgi;
7 7 const QUERY_RE = /([?&])([A-Za-z0-9_-]*(?:token|key|secret|password|auth)[A-Za-z0-9_-]*)=([^&#\s'"`]+)/dgi;
8 8 const AUTH_RE = /Authorization:\s*(Bearer|Basic)\s+([^\s'"`]+)/dgi;
9 9 const X_API_RE = /\b(X-API-Key|X-Api-Key|Api-Key):\s*([^\s'"`]+)/dg;