repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

extensions/leaks/src/engine.ts

Raw
import { builtinRules, findBuiltinSpans } from "./detectors.ts";
import {
	type LearnedValue,
	learnEnvironmentValues,
	learnedValueSpans,
	loadDotenvValues,
} from "./dotenv.ts";
import { type PersistedEntry, PlaceholderStore } from "./placeholders.ts";
import { isPlainObject } from "./safety.ts";
import { selectSecretSpans } from "./selector.ts";
import {
	type CompiledCustomRule,
	compileCustomRegex,
	customRegexSpans,
} from "./settings.ts";
import { findStructureAmbiguities, findStructureSpans } from "./structures.ts";
import type {
	DeepRedactionResult,
	Detector,
	EngineStats,
	LeaksSettings,
	RedactionOrigin,
	RedactionRecord,
	RedactionResult,
	SecretSpan,
	WarningRecord,
} from "./types.ts";

export type LeaksEngineOptions = {
	settings?: Partial<LeaksSettings>;
	env?: Record<string, string | undefined>;
	detectors?: Detector[];
};

export class LeaksEngine {
	settings: LeaksSettings;
	private readonly detectors: Detector[];
	private readonly warnings: WarningRecord[] = [];
	private readonly warningKeys = new Set<string>();
	private readonly ephemeralEntries: PersistedEntry[] = [];
	private readonly placeholders = new PlaceholderStore({
		onWarning: (warning) => this.addWarning(warning),
	});
	private customRules: CompiledCustomRule[] = [];
	private invalidRegexCount = 0;
	private learned: LearnedValue[] = [];
	private lastProviderRedactionCount = 0;
	private totalProviderRedactionCount = 0;
	private totalProtectedRedactionCount = 0;
	private lastProviderAmbiguityCount = 0;
	private recentLabels: string[] = [];
	private latestTags: string[] = [];
	readonly notifiedPlaceholders = new Set<string>();

	constructor(options: LeaksEngineOptions = {}) {
		this.placeholders.configure(this.ephemeralEntries, (customType, data) =>
			this.ephemeralEntries.push({ type: "custom", customType, data }),
		);
		this.settings = { regex: options.settings?.regex ?? [] };
		this.detectors = options.detectors ?? [];
		this.compileSettings();
		if (options.env) this.learn(process.cwd(), options.env);
	}

	reload(cwd = process.cwd(), settings = this.settings, env = process.env) {
		this.settings = { regex: settings.regex };
		this.compileSettings();
		this.learn(cwd, env);
		this.notifiedPlaceholders.clear();
	}

	redactText(
		text: string,
		origin: RedactionOrigin = "provider",
	): RedactionResult {
		const ambiguities = findStructureAmbiguities(text).map((item) => ({
			path: "",
			family: item.family,
		}));
		let spans: SecretSpan[] = [
			...findStructureSpans(text),
			...findBuiltinSpans(text),
			...customRegexSpans(text, this.customRules),
			...learnedValueSpans(text, this.learned),
		];
		for (const detector of this.detectors) {
			try {
				spans = spans.concat(detector(text));
			} catch {
				this.warnOnce(
					"detector.failure",
					"detector failure; redaction pass left unchanged",
				);
				if (origin === "provider") this.lastProviderRedactionCount = 0;
				return {
					text,
					count: 0,
					labels: [],
					placeholders: [],
					redactions: [],
					ambiguities,
				};
			}
		}

		const selected = selectSecretSpans(text, spans);
		let redacted = text;
		const placeholders: string[] = [];
		const labels: string[] = [];
		const redactions: RedactionRecord[] = [];
		for (const span of selected) {
			const spanLabels = span.labels.map((label) => label.tag);
			const placeholder = this.placeholders.create(span.value, span.labels);
			if (!placeholder) {
				ambiguities.push({
					path: "",
					family: this.placeholders.takeMiss() ?? "REDACTION_MISS",
				});
				continue;
			}
			redacted = `${redacted.slice(0, span.start)}${placeholder}${redacted.slice(span.end)}`;
			placeholders.push(placeholder);
			labels.push(...spanLabels);
			redactions.push({
				path: "",
				labels: spanLabels,
				placeholders: [placeholder],
			});
		}

		const count = placeholders.length;
		if (origin === "context" || origin === "provider")
			this.totalProtectedRedactionCount += count;
		if (origin === "provider") {
			this.lastProviderRedactionCount = count;
			this.lastProviderAmbiguityCount = ambiguities.length;
			this.totalProviderRedactionCount += count;
		}
		if (labels.length > 0) this.recentLabels = labels.slice(-12);
		if (placeholders.length > 0) {
			this.latestTags = placeholders
				.map((placeholder) =>
					placeholder
						.replace(/^__LEAK_V1_/, "")
						.replace(/_[a-f0-9]{32}__$/, ""),
				)
				.slice(-12);
		}
		return {
			text: redacted,
			count,
			labels,
			placeholders,
			redactions,
			ambiguities,
		};
	}

	redactDeep<T>(
		value: T,
		origin: RedactionOrigin = "provider",
	): DeepRedactionResult<T> {
		return this.walk(value, origin, false);
	}

	configurePersistence(
		entries: PersistedEntry[],
		append: (customType: string, data: unknown) => void,
	) {
		this.placeholders.configure(entries, append);
	}

	taintPersistence() {
		this.placeholders.taint();
	}

	restoreText(text: string) {
		return this.placeholders.restore(text);
	}

	hydrateToolInput<T>(value: T): {
		value: T;
		issue?: { path: string; label: string };
	} {
		const keyIssue = this.inspectHydratedKeys(value);
		if (keyIssue) return { value, issue: keyIssue };
		const restored = this.restoreDeep(value).value;
		const seen = new WeakSet<object>();
		const inspect = (
			item: unknown,
			depth: number,
			path: string,
		): { path: string; label: string } | undefined => {
			if (typeof item === "string") {
				const unresolved = this.placeholders.unresolved(item)[0];
				return unresolved
					? { path: path || "$", label: placeholderLabel(unresolved) }
					: undefined;
			}
			if (item === null || typeof item !== "object") return undefined;
			if (depth >= 64) return { path: path || "$", label: "TRAVERSAL_DEPTH" };
			if (seen.has(item))
				return { path: path || "$", label: "TRAVERSAL_CYCLE" };
			seen.add(item);
			if (Array.isArray(item)) {
				for (let index = 0; index < item.length; index++) {
					const issue = inspect(item[index], depth + 1, `${path}[${index}]`);
					if (issue) {
						seen.delete(item);
						return issue;
					}
				}
				seen.delete(item);
				return undefined;
			}
			if (!isPlainObject(item)) {
				seen.delete(item);
				return { path: path || "$", label: "UNSUPPORTED_OBJECT" };
			}
			for (const [index, [key, child]] of Object.entries(item).entries()) {
				const keyPath = childPath(path, index);
				const keyIssue = inspect(key, depth + 1, `${keyPath}.key`);
				if (keyIssue) {
					seen.delete(item);
					return keyIssue;
				}
				const issue = inspect(child, depth + 1, keyPath);
				if (issue) {
					seen.delete(item);
					return issue;
				}
			}
			seen.delete(item);
			return undefined;
		};
		return { value: restored, issue: inspect(restored, 0, "") };
	}

	restoreDeep<T>(value: T): DeepRedactionResult<T> {
		return this.walk(value, "provider", true);
	}

	private inspectHydratedKeys(value: unknown) {
		const stack = new WeakSet<object>();
		const visit = (
			item: unknown,
			depth: number,
			path: string,
		): { path: string; label: string } | undefined => {
			if (item === null || typeof item !== "object") return undefined;
			if (depth >= 64) return { path: path || "$", label: "TRAVERSAL_DEPTH" };
			if (stack.has(item))
				return { path: path || "$", label: "TRAVERSAL_CYCLE" };
			stack.add(item);
			if (Array.isArray(item)) {
				for (let index = 0; index < item.length; index++) {
					const issue = visit(item[index], depth + 1, `${path}[${index}]`);
					if (issue) {
						stack.delete(item);
						return issue;
					}
				}
				stack.delete(item);
				return undefined;
			}
			if (!isPlainObject(item)) {
				stack.delete(item);
				return { path: path || "$", label: "UNSUPPORTED_OBJECT" };
			}
			const keys = new Set<string>();
			for (const [index, [key, child]] of Object.entries(item).entries()) {
				const keyPath = childPath(path, index);
				const hydrated = this.restoreText(key);
				const unresolved = this.placeholders.unresolved(hydrated)[0];
				if (unresolved) {
					stack.delete(item);
					return {
						path: `${keyPath}.key`,
						label: placeholderLabel(unresolved),
					};
				}
				if (keys.has(hydrated)) {
					stack.delete(item);
					return { path: `${keyPath}.key`, label: "KEY_COLLISION" };
				}
				keys.add(hydrated);
				const issue = visit(child, depth + 1, keyPath);
				if (issue) {
					stack.delete(item);
					return issue;
				}
			}
			stack.delete(item);
			return undefined;
		};
		return visit(value, 0, "");
	}

	clear() {
		const cleared = this.placeholders.clear();
		this.lastProviderRedactionCount = 0;
		this.lastProviderAmbiguityCount = 0;
		this.recentLabels = [];
		this.latestTags = [];
		this.notifiedPlaceholders.clear();
		return cleared;
	}

	stats(): EngineStats {
		return {
			active: true,
			builtInRuleCount: builtinRules.length,
			customRegexCount: this.customRules.length,
			regexWarningCount: this.warnings.filter((warning) =>
				warning.code.startsWith("settings.regex."),
			).length,
			warningCount: this.warnings.length,
			invalidRegexCount: this.invalidRegexCount,
			learnedCount: this.learned.length,
			mappingCount: this.placeholders.stats().mappingCount,
			persistenceEnabled: this.placeholders.stats().persistenceEnabled,
			lastProviderRedactionCount: this.lastProviderRedactionCount,
			totalProviderRedactionCount: this.totalProviderRedactionCount,
			totalProtectedRedactionCount: this.totalProtectedRedactionCount,
			lastProviderAmbiguityCount: this.lastProviderAmbiguityCount,
			recentLabels: [...this.recentLabels],
			latestTags: [...this.latestTags],
			warnings: [...this.warnings],
		};
	}

	private compileSettings() {
		const compiled = compileCustomRegex(this.settings);
		this.customRules = compiled.rules;
		this.invalidRegexCount = compiled.invalidRegexCount;
		for (const warning of compiled.warnings) this.addWarning(warning);
	}

	private learn(cwd: string, env: Record<string, string | undefined>) {
		const dotenv = loadDotenvValues(cwd);
		const processEnv = learnEnvironmentValues(env);
		const learned = [...dotenv.values, ...processEnv.values];
		this.learned = learned.slice(0, 8192);
		if (learned.length > this.learned.length)
			this.addWarning({
				code: "learned.cap",
				message: "combined learned value cap reached; further values ignored",
				severity: "warning",
			});
		for (const warning of [...dotenv.warnings, ...processEnv.warnings])
			this.addWarning(warning);
	}

	private walk<T>(
		input: T,
		origin: RedactionOrigin,
		restore: boolean,
	): DeepRedactionResult<T> {
		const seen = new WeakMap<object, unknown>();
		const redactions: RedactionRecord[] = [];
		const ambiguities: Array<{ path: string; family: string }> = [];
		let count = 0;
		let changed = false;
		const visit = (value: unknown, depth: number, trace: string): unknown => {
			if (typeof value === "string") {
				if (restore) {
					const next = this.restoreText(value);
					if (next !== value) {
						changed = true;
						count++;
					}
					return next;
				}
				const result = this.redactText(value, origin);
				if (result.text !== value) changed = true;
				count += result.count;
				for (const item of result.redactions)
					redactions.push({ ...item, path: trace || "$" });
				for (const item of result.ambiguities)
					ambiguities.push({ ...item, path: trace || "$" });
				return result.text;
			}
			if (value === null || typeof value !== "object") return value;
			if (depth >= 64) {
				this.warnOnce(
					"traversal.depth",
					"maximum redaction traversal depth reached",
				);
				return value;
			}
			const cached = seen.get(value);
			if (cached) {
				this.warnOnce(
					"traversal.cycle",
					"cyclic value left unchanged during redaction traversal",
				);
				return cached;
			}
			if (Array.isArray(value)) {
				const copy: unknown[] = [];
				seen.set(value, copy);
				for (let index = 0; index < value.length; index++)
					copy.push(visit(value[index], depth + 1, `${trace}[${index}]`));
				if (copy.some((item, index) => item !== value[index])) changed = true;
				return copy;
			}
			if (!isPlainObject(value)) {
				this.warnOnce(
					"traversal.unsupported",
					"unsupported object left unchanged during redaction traversal",
				);
				return value;
			}
			const source = value as Record<string, unknown>;
			const copy: Record<string, unknown> = {};
			seen.set(value, copy);
			for (const [index, [key, item]] of Object.entries(source).entries()) {
				const keyPath = `${childPath(trace, index)}.key`;
				let keyResult = this.restoreText(key);
				if (!restore) {
					const result = this.redactText(key, origin);
					keyResult = result.text;
					count += result.count;
					for (const redaction of result.redactions)
						redactions.push({ ...redaction, path: keyPath });
					for (const ambiguity of result.ambiguities)
						ambiguities.push({ ...ambiguity, path: keyPath });
				}
				Object.defineProperty(copy, keyResult, {
					value: visit(item, depth + 1, childPath(trace, index)),
					enumerable: true,
					configurable: true,
					writable: true,
				});
			}
			if (Object.keys(copy).some((key) => copy[key] !== source[key]))
				changed = true;
			return copy;
		};
		const value = visit(input, 0, "") as T;
		if (origin === "provider" && !restore) {
			this.lastProviderRedactionCount = count;
			this.lastProviderAmbiguityCount = ambiguities.length;
		}
		return { value, changed, count, redactions, ambiguities };
	}

	private warnOnce(code: string, message: string) {
		if (this.warningKeys.has(code)) return;
		this.warningKeys.add(code);
		this.addWarning({ code, message, severity: "warning" });
	}

	private addWarning(warning: WarningRecord) {
		this.warnings.push(warning);
	}
}

function placeholderLabel(placeholder: string) {
	const current = /^__LEAK_V1_(.+)_[a-f0-9]{32}__$/.exec(placeholder);
	if (current) return current[1];
	const legacy = /^__LEAK_(.+)_[a-f0-9]{12}(?:_\d+)?__$/.exec(placeholder);
	return legacy?.[1] ?? "UNKNOWN";
}

function childPath(parent: string, index: number) {
	return `${parent || "$"}{${index}}`;
}