pi-ext
bugabingas pi extensions
owned by bugabinga
feat(leaks): learn dotenv and env secret values
a6d00f3c5f892777cd1555af74507c26fc53a040
Oliver Jan Krylow <oliver@bugabinga.net> ·
unsigned
Browse filesCompare with HEAD
Changed files (4)
extensions/leaks/__tests__/dotenv.test.ts
Addednew file mode 100644 1 +import { mkdtempSync, mkdirSync, writeFileSync }
from "node:fs";
2 +import { tmpdir }
from "node:os";
3 +import path
from "node:path";
4 +import { describe, expect, it }
from "vitest";
5 +import { __test }
from "../index.ts";
7 +const dotenvSecret
= "dotenv-fixture-secret-123456789";
8 +const envSecret
= "env-fixture-secret-123456789";
11 + return mkdtempSync(path.
join(
tmpdir(),
"leaks-dotenv-"));
14 +describe(
"leaks dotenv and environment learning", ()
=> {
15 + it(
"discovers only fixed cwd dotenv files", ()
=> {
17 + mkdirSync(path.
join(cwd,
"nested"));
18 + for (
const file
of [
".env",
".env.local",
".env.prod.local",
".env.prod",
"nested/.env"]) {
19 + writeFileSync(path.
join(cwd, file),
"TOKEN=value\n",
"utf8");
22 + expect(__test.
discoverDotenvFiles(cwd).
map((file)
=> path.
basename(file))).
toEqual([
29 + it(
"parses export, quotes, and inline comments", ()
=> {
31 + __test.
parseDotenv(
`export API_KEY=${dotenvSecret}\nSINGLE='${dotenvSecret}'\nDOUBLE="${dotenvSecret}" # local\nPLAIN=value # comment`),
33 + { key:
"API_KEY", value: dotenvSecret },
34 + { key:
"SINGLE", value: dotenvSecret },
35 + { key:
"DOUBLE", value: dotenvSecret },
36 + { key:
"PLAIN", value:
"value" },
40 + it(
"learns sensitive non-weak dotenv values and ignores weak values", ()
=> {
42 + writeFileSync(path.
join(cwd,
".env"),
`API_KEY=${dotenvSecret}\nPASSWORD=true\nTOKEN=short\n`,
"utf8");
43 + const engine
= new __test.
LeaksEngine();
44 + engine.
reload(cwd, engine.settings, {});
46 + const result
= engine.
redactText(
`send ${dotenvSecret} true short`,
"provider");
48 + expect(result.
text).not.
toContain(dotenvSecret);
49 + expect(result.
text).
toContain(
"true short");
50 + expect(engine.
stats().learnedCount).
toBe(
1);
53 + it(
"public prefixes suppress key-name sensitivity only", ()
=> {
54 + const weak
= "public-visible-fixture-value";
55 + const high
= "sk-or-v1-" + "a".
repeat(
64);
56 + const learned
= __test.
learnEnvironmentValues({
57 + NEXT_PUBLIC_TOKEN: weak,
61 + expect(learned.values.
map((item)
=> item.
value)).
toEqual([high]);
62 + expect(learned.values[
0].labels.
map((label)
=> label.tag)).
toContain(
"ENV_KEY");
65 + it(
"process env uses same classifier and collapses labels to ENV_KEY", ()
=> {
66 + const engine
= new __test.
LeaksEngine({ env: { PRIVATE_TOKEN: envSecret } });
67 + engine.
reload(
temp(), engine.settings, { PRIVATE_TOKEN: envSecret });
69 + const result
= engine.
redactText(
`send ${envSecret}`,
"provider");
71 + expect(result.
text).not.
toContain(envSecret);
72 + expect(result.labels).
toEqual([
"ENV_KEY"]);
73 + expect(
JSON.
stringify(engine.
stats())).not.
toContain(
"PRIVATE_TOKEN");
74 + expect(
JSON.
stringify(engine.
stats())).not.
toContain(envSecret);
77 + it(
"sensitive key matching ignores case, separators, and numeric suffixes", ()
=> {
78 + expect(__test.
isSensitiveKey(
"client-secret_1")).
toBe(
true);
79 + expect(__test.
isSensitiveKey(
"CLIENT_SECRET_2")).
toBe(
true);
80 + expect(__test.
isSensitiveKey(
"lineage")).
toBe(
false);
83 + it(
"enforces learned exact value cap and warns once", ()
=> {
84 + const env
: Record<
string,
string>
= {};
85 + for (
let index
= 0; index
< 8194; index
++) env[
`TOKEN_${index}`]
= `cap-fixture-secret-${index}-123456`;
86 + const learned
= __test.
learnEnvironmentValues(env);
88 + expect(learned.values).
toHaveLength(
8192);
89 + expect(learned.warnings.
map((warning)
=> warning.message)).
toEqual([
90 + "learned value cap reached; further dotenv/env values ignored",
extensions/leaks/index.ts
Modified 3 + highConfidenceSecretShape,
5 + learnEnvironmentValues,
9 +}
from "./src/dotenv.ts";
1 10 import { LeaksEngine }
from "./src/engine.ts";
2 11 import { PlaceholderStore }
from "./src/placeholders.ts";
3 12 import { selectSecretSpans }
from "./src/selector.ts";
21 30 export const __test
= {
33 + discoverDotenvFiles,
36 + learnEnvironmentValues,
39 + highConfidenceSecretShape,
24 40 RUNTIME_SETTING_KEYS,
extensions/leaks/src/dotenv.ts
Addednew file mode 100644 1 +import { existsSync, readdirSync, readFileSync }
from "node:fs";
2 +import path
from "node:path";
3 +import { isWeakValue, sanitizeTag }
from "./safety.ts";
4 +import type { SecretSpan, SpanLabel, WarningRecord }
from "./types.ts";
7 +const PUBLIC_PREFIX_RE
= /^(?:PUBLIC|NEXT_PUBLIC|VITE)[_-]/i;
8 +const SENSITIVE
= new Set([
41 +export type LearnedValue = {
43 + labels
: SpanLabel[];
46 +export type LearnedValues = {
47 + values
: LearnedValue[];
48 + warnings
: WarningRecord[];
51 +export function discoverDotenvFiles(cwd
: string)
: string[] {
52 + const direct
= [
".env",
".env.local"]
53 + .
map((file)
=> path.
join(cwd, file))
54 + .
filter((file)
=> existsSync(file));
55 + const local
= readdirSync(cwd, { withFileTypes:
true })
56 + .
filter((entry)
=> entry.
isFile())
57 + .
map((entry)
=> entry.
name)
58 + .
filter((name)
=> /^\.env\..+\.local$/.
test(name))
60 + .
map((file)
=> path.
join(cwd, file));
61 + return [
...direct,
...local];
64 +export function parseDotenv(text
: string)
: Array<{ key
: string; value
: string }> {
65 + const rows
: Array<{ key
: string; value
: string }>
= [];
66 + for (
const raw
of text.
split(
/\r?\n/)) {
67 + let line
= raw.
trim();
68 + if (
!line
|| line.
startsWith(
"#"))
continue;
69 + line
= line.
replace(
/^export\s+/,
"");
70 + const match
= /^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*(.*)$/.
exec(line);
71 + if (
!match)
continue;
72 + let value
= match[
2]
?? "";
73 + if ((value.
startsWith(
"'")
&& value.
includes(
"'",
1))
|| (value.
startsWith(
'"')
&& value.
includes(
'"',
1))) {
74 + const quote
= value[
0];
75 + const end
= value.
indexOf(quote,
1);
76 + value
= value.
slice(
1, end);
78 + value
= value.
replace(
/\s+#.*$/,
"").
trim();
80 + rows.
push({ key: match[
1], value });
85 +export function loadDotenvValues(cwd
: string)
: LearnedValues {
86 + const pairs
: Record<
string,
string>
= {};
87 + for (
const file
of discoverDotenvFiles(cwd)) {
88 + for (
const item
of parseDotenv(
readFileSync(file,
"utf8"))) pairs[item.key]
= item.
value;
90 + return learnKeyValues(pairs,
"dotenv");
93 +export function learnEnvironmentValues(env
: Record<
string,
string | undefined>)
: LearnedValues {
94 + const pairs
: Record<
string,
string>
= {};
95 + for (
const [key, value]
of Object.
entries(env))
if (
typeof value
=== "string") pairs[key]
= value;
96 + return learnKeyValues(pairs,
"env");
99 +export function isSensitiveKey(key
: string)
: boolean {
100 + const normalized
= normalizeKey(key.
replace(PUBLIC_PREFIX_RE,
""));
101 + if (SENSITIVE.
has(normalized))
return true;
102 + return [
...SENSITIVE].
some((sensitive)
=> normalized.
endsWith(
`_${sensitive}`)
|| normalized.
startsWith(
`${sensitive}_`));
105 +export function highConfidenceSecretShape(value
: string)
: boolean {
106 + return /^(?:sk-or-v1-[A-Za-z0-9]{32,}|sk-proj-[A-Za-z0-9_-]{32,}|sk-ant-api03-[A-Za-z0-9_-]{32,}|ghp_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{32,})$/.
test(value);
109 +export function learnedValueSpans(text
: string, values
: LearnedValue[])
: SecretSpan[] {
110 + const spans
: SecretSpan[]
= [];
111 + for (
const learned
of values) {
112 + let start
= text.
indexOf(learned.
value);
113 + while (start
!== -1) {
116 + end: start
+ learned.
value.
length,
117 + value: learned.
value,
118 + labels: learned.labels,
123 + start
= text.
indexOf(learned.
value, start
+ learned.
value.
length);
129 +function learnKeyValues(pairs
: Record<
string,
string>, source
: "dotenv" | "env")
: LearnedValues {
130 + const values
: LearnedValue[]
= [];
131 + const warnings
: WarningRecord[]
= [];
132 + let warned
= false;
133 + for (
const [key, value]
of Object.
entries(pairs)) {
134 + if (values.
length >= CAP) {
137 + code:
"learned.cap",
138 + message:
"learned value cap reached; further dotenv/env values ignored",
139 + severity:
"warning",
145 + const publicPrefixed
= PUBLIC_PREFIX_RE.
test(key);
146 + const secretShape
= highConfidenceSecretShape(value);
147 + if ((
!publicPrefixed
&& isSensitiveKey(key)
&& !isWeakValue(value))
|| secretShape) {
152 + tag: source
=== "env" ? "ENV_KEY" : sanitizeTag(key),
160 + return { values, warnings };
163 +function normalizeKey(key
: string) {
166 + .
replace(
/[-]+/g,
"_")
167 + .
replace(
/_\d+$/,
"");
extensions/leaks/src/engine.ts
Modified 2 + learnEnvironmentValues,
1 7 import { PlaceholderStore }
from "./placeholders.ts";
2 8 import { selectSecretSpans }
from "./selector.ts";
33 39 private
customRules: CompiledCustomRule[]
= [];
34 40 private invalidRegexCount
= 0;
41 + private
learned: LearnedValue[]
= [];
35 42 private lastProviderRedactionCount
= 0;
36 43 private totalProviderRedactionCount
= 0;
37 44 private
recentLabels: string[]
= [];
40 47 constructor(options: LeaksEngineOptions
= {}) {
41 48 this.settings
= { regex: options.settings?.regex
?? [] };
42 49 this.detectors
= options.detectors
?? [];
44 50 this.
compileSettings();
51 + if (options.env) this.
learn(
process.
cwd(), options.env);
47 - reload(_cwd
= process.
cwd(), settings
= this.settings, _env
= process.
env) {
54 + reload(cwd
= process.
cwd(), settings
= this.settings, env
= process.
env) {
48 55 this.settings
= { regex: settings.regex
?? [] };
49 56 this.
compileSettings();
57 + this.
learn(cwd, env);
52 60 redactText(text: string, origin: RedactionOrigin
= "provider"): RedactionResult {
53 - let spans
: SecretSpan[]
= customRegexSpans(text, this.customRules);
61 + let spans
: SecretSpan[]
= [
62 + ...customRegexSpans(text, this.customRules),
63 + ...learnedValueSpans(text, this.learned),
54 65 for (
const detector
of this.detectors) {
56 67 spans
= spans.
concat(
detector(text));
121 132 regexWarningCount: this.warnings.
filter((warning)
=> warning.
code.
startsWith(
"settings.regex.")).
length,
122 133 warningCount: this.warnings.
length,
123 134 invalidRegexCount: this.invalidRegexCount,
135 + learnedCount: this.learned.
length,
125 136 mappingCount: this.placeholders.
stats().mappingCount,
126 137 lastProviderRedactionCount: this.lastProviderRedactionCount,
127 138 totalProviderRedactionCount: this.totalProviderRedactionCount,
138 149 for (
const warning
of compiled.warnings) this.
addWarning(warning);
152 + private
learn(cwd: string, env: Record
<string, string
| undefined>) {
153 + const dotenv
= loadDotenvValues(cwd);
154 + const processEnv
= learnEnvironmentValues(env);
155 + this.learned
= [
...dotenv.values,
...processEnv.values].
slice(
0,
8192);
156 + for (
const warning
of [
...dotenv.warnings,
...processEnv.warnings]) this.
addWarning(warning);
141 159 private
warnOnce(code: string, message: string) {
142 160 if (this.warningKeys.
has(code))
return;
143 161 this.warningKeys.
add(code);