repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

feat(leaks): learn dotenv and env secret values

a6d00f3c5f892777cd1555af74507c26fc53a040

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (4)

extensions/leaks/__tests__/dotenv.test.ts

Added
new file mode 100644
1 +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
2 +import { tmpdir } from "node:os";
3 +import path from "node:path";
4 +import { describe, expect, it } from "vitest";
5 +import { __test } from "../index.ts";
7 +const dotenvSecret = "dotenv-fixture-secret-123456789";
8 +const envSecret = "env-fixture-secret-123456789";
10 +function temp() {
11 + return mkdtempSync(path.join(tmpdir(), "leaks-dotenv-"));
14 +describe("leaks dotenv and environment learning", () => {
15 + it("discovers only fixed cwd dotenv files", () => {
16 + const cwd = temp();
17 + mkdirSync(path.join(cwd, "nested"));
18 + for (const file of [".env", ".env.local", ".env.prod.local", ".env.prod", "nested/.env"]) {
19 + writeFileSync(path.join(cwd, file), "TOKEN=value\n", "utf8");
20 + }
22 + expect(__test.discoverDotenvFiles(cwd).map((file) => path.basename(file))).toEqual([
23 + ".env",
24 + ".env.local",
25 + ".env.prod.local",
26 + ]);
27 + });
29 + it("parses export, quotes, and inline comments", () => {
30 + expect(
31 + __test.parseDotenv(`export API_KEY=${dotenvSecret}\nSINGLE='${dotenvSecret}'\nDOUBLE="${dotenvSecret}" # local\nPLAIN=value # comment`),
32 + ).toEqual([
33 + { key: "API_KEY", value: dotenvSecret },
34 + { key: "SINGLE", value: dotenvSecret },
35 + { key: "DOUBLE", value: dotenvSecret },
36 + { key: "PLAIN", value: "value" },
37 + ]);
38 + });
40 + it("learns sensitive non-weak dotenv values and ignores weak values", () => {
41 + const cwd = temp();
42 + writeFileSync(path.join(cwd, ".env"), `API_KEY=${dotenvSecret}\nPASSWORD=true\nTOKEN=short\n`, "utf8");
43 + const engine = new __test.LeaksEngine();
44 + engine.reload(cwd, engine.settings, {});
46 + const result = engine.redactText(`send ${dotenvSecret} true short`, "provider");
48 + expect(result.text).not.toContain(dotenvSecret);
49 + expect(result.text).toContain("true short");
50 + expect(engine.stats().learnedCount).toBe(1);
51 + });
53 + it("public prefixes suppress key-name sensitivity only", () => {
54 + const weak = "public-visible-fixture-value";
55 + const high = "sk-or-v1-" + "a".repeat(64);
56 + const learned = __test.learnEnvironmentValues({
57 + NEXT_PUBLIC_TOKEN: weak,
58 + VITE_API_KEY: high,
59 + });
61 + expect(learned.values.map((item) => item.value)).toEqual([high]);
62 + expect(learned.values[0].labels.map((label) => label.tag)).toContain("ENV_KEY");
63 + });
65 + it("process env uses same classifier and collapses labels to ENV_KEY", () => {
66 + const engine = new __test.LeaksEngine({ env: { PRIVATE_TOKEN: envSecret } });
67 + engine.reload(temp(), engine.settings, { PRIVATE_TOKEN: envSecret });
69 + const result = engine.redactText(`send ${envSecret}`, "provider");
71 + expect(result.text).not.toContain(envSecret);
72 + expect(result.labels).toEqual(["ENV_KEY"]);
73 + expect(JSON.stringify(engine.stats())).not.toContain("PRIVATE_TOKEN");
74 + expect(JSON.stringify(engine.stats())).not.toContain(envSecret);
75 + });
77 + it("sensitive key matching ignores case, separators, and numeric suffixes", () => {
78 + expect(__test.isSensitiveKey("client-secret_1")).toBe(true);
79 + expect(__test.isSensitiveKey("CLIENT_SECRET_2")).toBe(true);
80 + expect(__test.isSensitiveKey("lineage")).toBe(false);
81 + });
83 + it("enforces learned exact value cap and warns once", () => {
84 + const env: Record<string, string> = {};
85 + for (let index = 0; index < 8194; index++) env[`TOKEN_${index}`] = `cap-fixture-secret-${index}-123456`;
86 + const learned = __test.learnEnvironmentValues(env);
88 + expect(learned.values).toHaveLength(8192);
89 + expect(learned.warnings.map((warning) => warning.message)).toEqual([
90 + "learned value cap reached; further dotenv/env values ignored",
91 + ]);
92 + });
93 +});

extensions/leaks/index.ts

Modified
1 +import {
2 + discoverDotenvFiles,
3 + highConfidenceSecretShape,
4 + isSensitiveKey,
5 + learnEnvironmentValues,
6 + learnedValueSpans,
7 + loadDotenvValues,
8 + parseDotenv,
9 +} from "./src/dotenv.ts";
1 10 import { LeaksEngine } from "./src/engine.ts";
2 11 import { PlaceholderStore } from "./src/placeholders.ts";
3 12 import { selectSecretSpans } from "./src/selector.ts";
21 30 export const __test = {
22 31 LeaksEngine,
23 32 PlaceholderStore,
33 + discoverDotenvFiles,
34 + parseDotenv,
35 + loadDotenvValues,
36 + learnEnvironmentValues,
37 + learnedValueSpans,
38 + isSensitiveKey,
39 + highConfidenceSecretShape,
24 40 RUNTIME_SETTING_KEYS,
25 41 selectSecretSpans,
26 42 loadLeaksSettings,

extensions/leaks/src/dotenv.ts

Added
new file mode 100644
1 +import { existsSync, readdirSync, readFileSync } from "node:fs";
2 +import path from "node:path";
3 +import { isWeakValue, sanitizeTag } from "./safety.ts";
4 +import type { SecretSpan, SpanLabel, WarningRecord } from "./types.ts";
6 +const CAP = 8192;
7 +const PUBLIC_PREFIX_RE = /^(?:PUBLIC|NEXT_PUBLIC|VITE)[_-]/i;
8 +const SENSITIVE = new Set([
9 + "TOKEN",
10 + "SECRET",
11 + "PASSWORD",
12 + "PASSWD",
13 + "PASS",
14 + "PEPPER",
15 + "API_KEY",
16 + "ACCESS_KEY",
17 + "PRIVATE_KEY",
18 + "KEY",
19 + "KEY_DATA",
20 + "PAT",
21 + "PASSPHRASE",
22 + "LICENSE_KEY",
23 + "MASTER_KEY",
24 + "ROUTING_KEY",
25 + "ENCRYPTION_KEY",
26 + "SECRET_KEY_BASE",
27 + "SEED",
28 + "UNSEAL_KEY",
29 + "MNEMONIC",
30 + "CLIENT_SECRET",
31 + "AUTH",
32 + "CREDENTIAL",
33 + "SESSION",
34 + "COOKIE",
35 + "JWT",
36 + "DATABASE_URL",
37 + "MONGO_URL",
38 + "REDIS_URL",
39 +]);
41 +export type LearnedValue = {
42 + value: string;
43 + labels: SpanLabel[];
44 +};
46 +export type LearnedValues = {
47 + values: LearnedValue[];
48 + warnings: WarningRecord[];
49 +};
51 +export function discoverDotenvFiles(cwd: string): string[] {
52 + const direct = [".env", ".env.local"]
53 + .map((file) => path.join(cwd, file))
54 + .filter((file) => existsSync(file));
55 + const local = readdirSync(cwd, { withFileTypes: true })
56 + .filter((entry) => entry.isFile())
57 + .map((entry) => entry.name)
58 + .filter((name) => /^\.env\..+\.local$/.test(name))
59 + .sort()
60 + .map((file) => path.join(cwd, file));
61 + return [...direct, ...local];
64 +export function parseDotenv(text: string): Array<{ key: string; value: string }> {
65 + const rows: Array<{ key: string; value: string }> = [];
66 + for (const raw of text.split(/\r?\n/)) {
67 + let line = raw.trim();
68 + if (!line || line.startsWith("#")) continue;
69 + line = line.replace(/^export\s+/, "");
70 + const match = /^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*(.*)$/.exec(line);
71 + if (!match) continue;
72 + let value = match[2] ?? "";
73 + if ((value.startsWith("'") && value.includes("'", 1)) || (value.startsWith('"') && value.includes('"', 1))) {
74 + const quote = value[0];
75 + const end = value.indexOf(quote, 1);
76 + value = value.slice(1, end);
77 + } else {
78 + value = value.replace(/\s+#.*$/, "").trim();
79 + }
80 + rows.push({ key: match[1], value });
81 + }
82 + return rows;
85 +export function loadDotenvValues(cwd: string): LearnedValues {
86 + const pairs: Record<string, string> = {};
87 + for (const file of discoverDotenvFiles(cwd)) {
88 + for (const item of parseDotenv(readFileSync(file, "utf8"))) pairs[item.key] = item.value;
89 + }
90 + return learnKeyValues(pairs, "dotenv");
93 +export function learnEnvironmentValues(env: Record<string, string | undefined>): LearnedValues {
94 + const pairs: Record<string, string> = {};
95 + for (const [key, value] of Object.entries(env)) if (typeof value === "string") pairs[key] = value;
96 + return learnKeyValues(pairs, "env");
99 +export function isSensitiveKey(key: string): boolean {
100 + const normalized = normalizeKey(key.replace(PUBLIC_PREFIX_RE, ""));
101 + if (SENSITIVE.has(normalized)) return true;
102 + return [...SENSITIVE].some((sensitive) => normalized.endsWith(`_${sensitive}`) || normalized.startsWith(`${sensitive}_`));
105 +export function highConfidenceSecretShape(value: string): boolean {
106 + return /^(?:sk-or-v1-[A-Za-z0-9]{32,}|sk-proj-[A-Za-z0-9_-]{32,}|sk-ant-api03-[A-Za-z0-9_-]{32,}|ghp_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{32,})$/.test(value);
109 +export function learnedValueSpans(text: string, values: LearnedValue[]): SecretSpan[] {
110 + const spans: SecretSpan[] = [];
111 + for (const learned of values) {
112 + let start = text.indexOf(learned.value);
113 + while (start !== -1) {
114 + spans.push({
115 + start,
116 + end: start + learned.value.length,
117 + value: learned.value,
118 + labels: learned.labels,
119 + kind: "exact",
120 + confidence: 70,
121 + wholeSpan: false,
122 + });
123 + start = text.indexOf(learned.value, start + learned.value.length);
126 + return spans;
129 +function learnKeyValues(pairs: Record<string, string>, source: "dotenv" | "env"): LearnedValues {
130 + const values: LearnedValue[] = [];
131 + const warnings: WarningRecord[] = [];
132 + let warned = false;
133 + for (const [key, value] of Object.entries(pairs)) {
134 + if (values.length >= CAP) {
135 + if (!warned) {
136 + warnings.push({
137 + code: "learned.cap",
138 + message: "learned value cap reached; further dotenv/env values ignored",
139 + severity: "warning",
140 + });
141 + warned = true;
143 + continue;
145 + const publicPrefixed = PUBLIC_PREFIX_RE.test(key);
146 + const secretShape = highConfidenceSecretShape(value);
147 + if ((!publicPrefixed && isSensitiveKey(key) && !isWeakValue(value)) || secretShape) {
148 + values.push({
149 + value,
150 + labels: [
152 + tag: source === "env" ? "ENV_KEY" : sanitizeTag(key),
153 + priority: 10,
154 + source,
155 + },
156 + ],
157 + });
160 + return { values, warnings };
163 +function normalizeKey(key: string) {
164 + return key
165 + .toUpperCase()
166 + .replace(/[-]+/g, "_")
167 + .replace(/_\d+$/, "");

extensions/leaks/src/engine.ts

Modified
1 +import {
2 + learnEnvironmentValues,
3 + learnedValueSpans,
4 + loadDotenvValues,
5 + type LearnedValue,
6 +} from "./dotenv.ts";
1 7 import { PlaceholderStore } from "./placeholders.ts";
2 8 import { selectSecretSpans } from "./selector.ts";
3 9 import {
33 39 private customRules: CompiledCustomRule[] = [];
34 40 private invalidRegexCount = 0;
41 + private learned: LearnedValue[] = [];
35 42 private lastProviderRedactionCount = 0;
36 43 private totalProviderRedactionCount = 0;
37 44 private recentLabels: string[] = [];
40 47 constructor(options: LeaksEngineOptions = {}) {
41 48 this.settings = { regex: options.settings?.regex ?? [] };
42 49 this.detectors = options.detectors ?? [];
43 - void options.env;
44 50 this.compileSettings();
51 + if (options.env) this.learn(process.cwd(), options.env);
47 - reload(_cwd = process.cwd(), settings = this.settings, _env = process.env) {
54 + reload(cwd = process.cwd(), settings = this.settings, env = process.env) {
48 55 this.settings = { regex: settings.regex ?? [] };
49 56 this.compileSettings();
57 + this.learn(cwd, env);
52 60 redactText(text: string, origin: RedactionOrigin = "provider"): RedactionResult {
53 - let spans: SecretSpan[] = customRegexSpans(text, this.customRules);
61 + let spans: SecretSpan[] = [
62 + ...customRegexSpans(text, this.customRules),
63 + ...learnedValueSpans(text, this.learned),
64 + ];
54 65 for (const detector of this.detectors) {
55 66 try {
56 67 spans = spans.concat(detector(text));
121 132 regexWarningCount: this.warnings.filter((warning) => warning.code.startsWith("settings.regex.")).length,
122 133 warningCount: this.warnings.length,
123 134 invalidRegexCount: this.invalidRegexCount,
124 - learnedCount: 0,
135 + learnedCount: this.learned.length,
125 136 mappingCount: this.placeholders.stats().mappingCount,
126 137 lastProviderRedactionCount: this.lastProviderRedactionCount,
127 138 totalProviderRedactionCount: this.totalProviderRedactionCount,
138 149 for (const warning of compiled.warnings) this.addWarning(warning);
152 + private learn(cwd: string, env: Record<string, string | undefined>) {
153 + const dotenv = loadDotenvValues(cwd);
154 + const processEnv = learnEnvironmentValues(env);
155 + this.learned = [...dotenv.values, ...processEnv.values].slice(0, 8192);
156 + for (const warning of [...dotenv.warnings, ...processEnv.warnings]) this.addWarning(warning);
141 159 private warnOnce(code: string, message: string) {
142 160 if (this.warningKeys.has(code)) return;
143 161 this.warningKeys.add(code);