import { builtinRules, findBuiltinSpans } from "./detectors.ts"; import { type LearnedValue, learnEnvironmentValues, learnedValueSpans, loadDotenvValues, } from "./dotenv.ts"; import { type PersistedEntry, PlaceholderStore } from "./placeholders.ts"; import { isPlainObject } from "./safety.ts"; import { selectSecretSpans } from "./selector.ts"; import { type CompiledCustomRule, compileCustomRegex, customRegexSpans, } from "./settings.ts"; import { findStructureAmbiguities, findStructureSpans } from "./structures.ts"; import type { DeepRedactionResult, Detector, EngineStats, LeaksSettings, RedactionOrigin, RedactionRecord, RedactionResult, SecretSpan, WarningRecord, } from "./types.ts"; export type LeaksEngineOptions = { settings?: Partial; env?: Record; detectors?: Detector[]; }; export class LeaksEngine { settings: LeaksSettings; private readonly detectors: Detector[]; private readonly warnings: WarningRecord[] = []; private readonly warningKeys = new Set(); private readonly ephemeralEntries: PersistedEntry[] = []; private readonly placeholders = new PlaceholderStore({ onWarning: (warning) => this.addWarning(warning), }); private customRules: CompiledCustomRule[] = []; private invalidRegexCount = 0; private learned: LearnedValue[] = []; private lastProviderRedactionCount = 0; private totalProviderRedactionCount = 0; private totalProtectedRedactionCount = 0; private lastProviderAmbiguityCount = 0; private recentLabels: string[] = []; private latestTags: string[] = []; readonly notifiedPlaceholders = new Set(); constructor(options: LeaksEngineOptions = {}) { this.placeholders.configure(this.ephemeralEntries, (customType, data) => this.ephemeralEntries.push({ type: "custom", customType, data }), ); this.settings = { regex: options.settings?.regex ?? [] }; this.detectors = options.detectors ?? []; this.compileSettings(); if (options.env) this.learn(process.cwd(), options.env); } reload(cwd = process.cwd(), settings = this.settings, env = process.env) { this.settings = { regex: settings.regex }; this.compileSettings(); this.learn(cwd, env); this.notifiedPlaceholders.clear(); } redactText( text: string, origin: RedactionOrigin = "provider", ): RedactionResult { const ambiguities = findStructureAmbiguities(text).map((item) => ({ path: "", family: item.family, })); let spans: SecretSpan[] = [ ...findStructureSpans(text), ...findBuiltinSpans(text), ...customRegexSpans(text, this.customRules), ...learnedValueSpans(text, this.learned), ]; for (const detector of this.detectors) { try { spans = spans.concat(detector(text)); } catch { this.warnOnce( "detector.failure", "detector failure; redaction pass left unchanged", ); if (origin === "provider") this.lastProviderRedactionCount = 0; return { text, count: 0, labels: [], placeholders: [], redactions: [], ambiguities, }; } } const selected = selectSecretSpans(text, spans); let redacted = text; const placeholders: string[] = []; const labels: string[] = []; const redactions: RedactionRecord[] = []; for (const span of selected) { const spanLabels = span.labels.map((label) => label.tag); const placeholder = this.placeholders.create(span.value, span.labels); if (!placeholder) { ambiguities.push({ path: "", family: this.placeholders.takeMiss() ?? "REDACTION_MISS", }); continue; } redacted = `${redacted.slice(0, span.start)}${placeholder}${redacted.slice(span.end)}`; placeholders.push(placeholder); labels.push(...spanLabels); redactions.push({ path: "", labels: spanLabels, placeholders: [placeholder], }); } const count = placeholders.length; if (origin === "context" || origin === "provider") this.totalProtectedRedactionCount += count; if (origin === "provider") { this.lastProviderRedactionCount = count; this.lastProviderAmbiguityCount = ambiguities.length; this.totalProviderRedactionCount += count; } if (labels.length > 0) this.recentLabels = labels.slice(-12); if (placeholders.length > 0) { this.latestTags = placeholders .map((placeholder) => placeholder .replace(/^__LEAK_V1_/, "") .replace(/_[a-f0-9]{32}__$/, ""), ) .slice(-12); } return { text: redacted, count, labels, placeholders, redactions, ambiguities, }; } redactDeep( value: T, origin: RedactionOrigin = "provider", ): DeepRedactionResult { return this.walk(value, origin, false); } configurePersistence( entries: PersistedEntry[], append: (customType: string, data: unknown) => void, ) { this.placeholders.configure(entries, append); } taintPersistence() { this.placeholders.taint(); } restoreText(text: string) { return this.placeholders.restore(text); } hydrateToolInput(value: T): { value: T; issue?: { path: string; label: string }; } { const keyIssue = this.inspectHydratedKeys(value); if (keyIssue) return { value, issue: keyIssue }; const restored = this.restoreDeep(value).value; const seen = new WeakSet(); const inspect = ( item: unknown, depth: number, path: string, ): { path: string; label: string } | undefined => { if (typeof item === "string") { const unresolved = this.placeholders.unresolved(item)[0]; return unresolved ? { path: path || "$", label: placeholderLabel(unresolved) } : undefined; } if (item === null || typeof item !== "object") return undefined; if (depth >= 64) return { path: path || "$", label: "TRAVERSAL_DEPTH" }; if (seen.has(item)) return { path: path || "$", label: "TRAVERSAL_CYCLE" }; seen.add(item); if (Array.isArray(item)) { for (let index = 0; index < item.length; index++) { const issue = inspect(item[index], depth + 1, `${path}[${index}]`); if (issue) { seen.delete(item); return issue; } } seen.delete(item); return undefined; } if (!isPlainObject(item)) { seen.delete(item); return { path: path || "$", label: "UNSUPPORTED_OBJECT" }; } for (const [index, [key, child]] of Object.entries(item).entries()) { const keyPath = childPath(path, index); const keyIssue = inspect(key, depth + 1, `${keyPath}.key`); if (keyIssue) { seen.delete(item); return keyIssue; } const issue = inspect(child, depth + 1, keyPath); if (issue) { seen.delete(item); return issue; } } seen.delete(item); return undefined; }; return { value: restored, issue: inspect(restored, 0, "") }; } restoreDeep(value: T): DeepRedactionResult { return this.walk(value, "provider", true); } private inspectHydratedKeys(value: unknown) { const stack = new WeakSet(); const visit = ( item: unknown, depth: number, path: string, ): { path: string; label: string } | undefined => { if (item === null || typeof item !== "object") return undefined; if (depth >= 64) return { path: path || "$", label: "TRAVERSAL_DEPTH" }; if (stack.has(item)) return { path: path || "$", label: "TRAVERSAL_CYCLE" }; stack.add(item); if (Array.isArray(item)) { for (let index = 0; index < item.length; index++) { const issue = visit(item[index], depth + 1, `${path}[${index}]`); if (issue) { stack.delete(item); return issue; } } stack.delete(item); return undefined; } if (!isPlainObject(item)) { stack.delete(item); return { path: path || "$", label: "UNSUPPORTED_OBJECT" }; } const keys = new Set(); for (const [index, [key, child]] of Object.entries(item).entries()) { const keyPath = childPath(path, index); const hydrated = this.restoreText(key); const unresolved = this.placeholders.unresolved(hydrated)[0]; if (unresolved) { stack.delete(item); return { path: `${keyPath}.key`, label: placeholderLabel(unresolved), }; } if (keys.has(hydrated)) { stack.delete(item); return { path: `${keyPath}.key`, label: "KEY_COLLISION" }; } keys.add(hydrated); const issue = visit(child, depth + 1, keyPath); if (issue) { stack.delete(item); return issue; } } stack.delete(item); return undefined; }; return visit(value, 0, ""); } clear() { const cleared = this.placeholders.clear(); this.lastProviderRedactionCount = 0; this.lastProviderAmbiguityCount = 0; this.recentLabels = []; this.latestTags = []; this.notifiedPlaceholders.clear(); return cleared; } stats(): EngineStats { return { active: true, builtInRuleCount: builtinRules.length, customRegexCount: this.customRules.length, regexWarningCount: this.warnings.filter((warning) => warning.code.startsWith("settings.regex."), ).length, warningCount: this.warnings.length, invalidRegexCount: this.invalidRegexCount, learnedCount: this.learned.length, mappingCount: this.placeholders.stats().mappingCount, persistenceEnabled: this.placeholders.stats().persistenceEnabled, lastProviderRedactionCount: this.lastProviderRedactionCount, totalProviderRedactionCount: this.totalProviderRedactionCount, totalProtectedRedactionCount: this.totalProtectedRedactionCount, lastProviderAmbiguityCount: this.lastProviderAmbiguityCount, recentLabels: [...this.recentLabels], latestTags: [...this.latestTags], warnings: [...this.warnings], }; } private compileSettings() { const compiled = compileCustomRegex(this.settings); this.customRules = compiled.rules; this.invalidRegexCount = compiled.invalidRegexCount; for (const warning of compiled.warnings) this.addWarning(warning); } private learn(cwd: string, env: Record) { const dotenv = loadDotenvValues(cwd); const processEnv = learnEnvironmentValues(env); const learned = [...dotenv.values, ...processEnv.values]; this.learned = learned.slice(0, 8192); if (learned.length > this.learned.length) this.addWarning({ code: "learned.cap", message: "combined learned value cap reached; further values ignored", severity: "warning", }); for (const warning of [...dotenv.warnings, ...processEnv.warnings]) this.addWarning(warning); } private walk( input: T, origin: RedactionOrigin, restore: boolean, ): DeepRedactionResult { const seen = new WeakMap(); const redactions: RedactionRecord[] = []; const ambiguities: Array<{ path: string; family: string }> = []; let count = 0; let changed = false; const visit = (value: unknown, depth: number, trace: string): unknown => { if (typeof value === "string") { if (restore) { const next = this.restoreText(value); if (next !== value) { changed = true; count++; } return next; } const result = this.redactText(value, origin); if (result.text !== value) changed = true; count += result.count; for (const item of result.redactions) redactions.push({ ...item, path: trace || "$" }); for (const item of result.ambiguities) ambiguities.push({ ...item, path: trace || "$" }); return result.text; } if (value === null || typeof value !== "object") return value; if (depth >= 64) { this.warnOnce( "traversal.depth", "maximum redaction traversal depth reached", ); return value; } const cached = seen.get(value); if (cached) { this.warnOnce( "traversal.cycle", "cyclic value left unchanged during redaction traversal", ); return cached; } if (Array.isArray(value)) { const copy: unknown[] = []; seen.set(value, copy); for (let index = 0; index < value.length; index++) copy.push(visit(value[index], depth + 1, `${trace}[${index}]`)); if (copy.some((item, index) => item !== value[index])) changed = true; return copy; } if (!isPlainObject(value)) { this.warnOnce( "traversal.unsupported", "unsupported object left unchanged during redaction traversal", ); return value; } const source = value as Record; const copy: Record = {}; seen.set(value, copy); for (const [index, [key, item]] of Object.entries(source).entries()) { const keyPath = `${childPath(trace, index)}.key`; let keyResult = this.restoreText(key); if (!restore) { const result = this.redactText(key, origin); keyResult = result.text; count += result.count; for (const redaction of result.redactions) redactions.push({ ...redaction, path: keyPath }); for (const ambiguity of result.ambiguities) ambiguities.push({ ...ambiguity, path: keyPath }); } Object.defineProperty(copy, keyResult, { value: visit(item, depth + 1, childPath(trace, index)), enumerable: true, configurable: true, writable: true, }); } if (Object.keys(copy).some((key) => copy[key] !== source[key])) changed = true; return copy; }; const value = visit(input, 0, "") as T; if (origin === "provider" && !restore) { this.lastProviderRedactionCount = count; this.lastProviderAmbiguityCount = ambiguities.length; } return { value, changed, count, redactions, ambiguities }; } private warnOnce(code: string, message: string) { if (this.warningKeys.has(code)) return; this.warningKeys.add(code); this.addWarning({ code, message, severity: "warning" }); } private addWarning(warning: WarningRecord) { this.warnings.push(warning); } } function placeholderLabel(placeholder: string) { const current = /^__LEAK_V1_(.+)_[a-f0-9]{32}__$/.exec(placeholder); if (current) return current[1]; const legacy = /^__LEAK_(.+)_[a-f0-9]{12}(?:_\d+)?__$/.exec(placeholder); return legacy?.[1] ?? "UNKNOWN"; } function childPath(parent: string, index: number) { return `${parent || "$"}{${index}}`; }