repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by bugabinga
extensions/leaks/src/engine.ts
Rawimport { builtinRules, findBuiltinSpans } from "./detectors.ts";
import {
type LearnedValue,
learnEnvironmentValues,
learnedValueSpans,
loadDotenvValues,
} from "./dotenv.ts";
import { type PersistedEntry, PlaceholderStore } from "./placeholders.ts";
import { isPlainObject } from "./safety.ts";
import { selectSecretSpans } from "./selector.ts";
import {
type CompiledCustomRule,
compileCustomRegex,
customRegexSpans,
} from "./settings.ts";
import { findStructureAmbiguities, findStructureSpans } from "./structures.ts";
import type {
DeepRedactionResult,
Detector,
EngineStats,
LeaksSettings,
RedactionOrigin,
RedactionRecord,
RedactionResult,
SecretSpan,
WarningRecord,
} from "./types.ts";
export type LeaksEngineOptions = {
settings?: Partial<LeaksSettings>;
env?: Record<string, string | undefined>;
detectors?: Detector[];
};
export class LeaksEngine {
settings: LeaksSettings;
private readonly detectors: Detector[];
private readonly warnings: WarningRecord[] = [];
private readonly warningKeys = new Set<string>();
private readonly ephemeralEntries: PersistedEntry[] = [];
private readonly placeholders = new PlaceholderStore({
onWarning: (warning) => this.addWarning(warning),
});
private customRules: CompiledCustomRule[] = [];
private invalidRegexCount = 0;
private learned: LearnedValue[] = [];
private lastProviderRedactionCount = 0;
private totalProviderRedactionCount = 0;
private totalProtectedRedactionCount = 0;
private lastProviderAmbiguityCount = 0;
private recentLabels: string[] = [];
private latestTags: string[] = [];
readonly notifiedPlaceholders = new Set<string>();
constructor(options: LeaksEngineOptions = {}) {
this.placeholders.configure(this.ephemeralEntries, (customType, data) =>
this.ephemeralEntries.push({ type: "custom", customType, data }),
);
this.settings = { regex: options.settings?.regex ?? [] };
this.detectors = options.detectors ?? [];
this.compileSettings();
if (options.env) this.learn(process.cwd(), options.env);
}
reload(cwd = process.cwd(), settings = this.settings, env = process.env) {
this.settings = { regex: settings.regex };
this.compileSettings();
this.learn(cwd, env);
this.notifiedPlaceholders.clear();
}
redactText(
text: string,
origin: RedactionOrigin = "provider",
): RedactionResult {
const ambiguities = findStructureAmbiguities(text).map((item) => ({
path: "",
family: item.family,
}));
let spans: SecretSpan[] = [
...findStructureSpans(text),
...findBuiltinSpans(text),
...customRegexSpans(text, this.customRules),
...learnedValueSpans(text, this.learned),
];
for (const detector of this.detectors) {
try {
spans = spans.concat(detector(text));
} catch {
this.warnOnce(
"detector.failure",
"detector failure; redaction pass left unchanged",
);
if (origin === "provider") this.lastProviderRedactionCount = 0;
return {
text,
count: 0,
labels: [],
placeholders: [],
redactions: [],
ambiguities,
};
}
}
const selected = selectSecretSpans(text, spans);
let redacted = text;
const placeholders: string[] = [];
const labels: string[] = [];
const redactions: RedactionRecord[] = [];
for (const span of selected) {
const spanLabels = span.labels.map((label) => label.tag);
const placeholder = this.placeholders.create(span.value, span.labels);
if (!placeholder) {
ambiguities.push({
path: "",
family: this.placeholders.takeMiss() ?? "REDACTION_MISS",
});
continue;
}
redacted = `${redacted.slice(0, span.start)}${placeholder}${redacted.slice(span.end)}`;
placeholders.push(placeholder);
labels.push(...spanLabels);
redactions.push({
path: "",
labels: spanLabels,
placeholders: [placeholder],
});
}
const count = placeholders.length;
if (origin === "context" || origin === "provider")
this.totalProtectedRedactionCount += count;
if (origin === "provider") {
this.lastProviderRedactionCount = count;
this.lastProviderAmbiguityCount = ambiguities.length;
this.totalProviderRedactionCount += count;
}
if (labels.length > 0) this.recentLabels = labels.slice(-12);
if (placeholders.length > 0) {
this.latestTags = placeholders
.map((placeholder) =>
placeholder
.replace(/^__LEAK_V1_/, "")
.replace(/_[a-f0-9]{32}__$/, ""),
)
.slice(-12);
}
return {
text: redacted,
count,
labels,
placeholders,
redactions,
ambiguities,
};
}
redactDeep<T>(
value: T,
origin: RedactionOrigin = "provider",
): DeepRedactionResult<T> {
return this.walk(value, origin, false);
}
configurePersistence(
entries: PersistedEntry[],
append: (customType: string, data: unknown) => void,
) {
this.placeholders.configure(entries, append);
}
taintPersistence() {
this.placeholders.taint();
}
restoreText(text: string) {
return this.placeholders.restore(text);
}
hydrateToolInput<T>(value: T): {
value: T;
issue?: { path: string; label: string };
} {
const keyIssue = this.inspectHydratedKeys(value);
if (keyIssue) return { value, issue: keyIssue };
const restored = this.restoreDeep(value).value;
const seen = new WeakSet<object>();
const inspect = (
item: unknown,
depth: number,
path: string,
): { path: string; label: string } | undefined => {
if (typeof item === "string") {
const unresolved = this.placeholders.unresolved(item)[0];
return unresolved
? { path: path || "$", label: placeholderLabel(unresolved) }
: undefined;
}
if (item === null || typeof item !== "object") return undefined;
if (depth >= 64) return { path: path || "$", label: "TRAVERSAL_DEPTH" };
if (seen.has(item))
return { path: path || "$", label: "TRAVERSAL_CYCLE" };
seen.add(item);
if (Array.isArray(item)) {
for (let index = 0; index < item.length; index++) {
const issue = inspect(item[index], depth + 1, `${path}[${index}]`);
if (issue) {
seen.delete(item);
return issue;
}
}
seen.delete(item);
return undefined;
}
if (!isPlainObject(item)) {
seen.delete(item);
return { path: path || "$", label: "UNSUPPORTED_OBJECT" };
}
for (const [index, [key, child]] of Object.entries(item).entries()) {
const keyPath = childPath(path, index);
const keyIssue = inspect(key, depth + 1, `${keyPath}.key`);
if (keyIssue) {
seen.delete(item);
return keyIssue;
}
const issue = inspect(child, depth + 1, keyPath);
if (issue) {
seen.delete(item);
return issue;
}
}
seen.delete(item);
return undefined;
};
return { value: restored, issue: inspect(restored, 0, "") };
}
restoreDeep<T>(value: T): DeepRedactionResult<T> {
return this.walk(value, "provider", true);
}
private inspectHydratedKeys(value: unknown) {
const stack = new WeakSet<object>();
const visit = (
item: unknown,
depth: number,
path: string,
): { path: string; label: string } | undefined => {
if (item === null || typeof item !== "object") return undefined;
if (depth >= 64) return { path: path || "$", label: "TRAVERSAL_DEPTH" };
if (stack.has(item))
return { path: path || "$", label: "TRAVERSAL_CYCLE" };
stack.add(item);
if (Array.isArray(item)) {
for (let index = 0; index < item.length; index++) {
const issue = visit(item[index], depth + 1, `${path}[${index}]`);
if (issue) {
stack.delete(item);
return issue;
}
}
stack.delete(item);
return undefined;
}
if (!isPlainObject(item)) {
stack.delete(item);
return { path: path || "$", label: "UNSUPPORTED_OBJECT" };
}
const keys = new Set<string>();
for (const [index, [key, child]] of Object.entries(item).entries()) {
const keyPath = childPath(path, index);
const hydrated = this.restoreText(key);
const unresolved = this.placeholders.unresolved(hydrated)[0];
if (unresolved) {
stack.delete(item);
return {
path: `${keyPath}.key`,
label: placeholderLabel(unresolved),
};
}
if (keys.has(hydrated)) {
stack.delete(item);
return { path: `${keyPath}.key`, label: "KEY_COLLISION" };
}
keys.add(hydrated);
const issue = visit(child, depth + 1, keyPath);
if (issue) {
stack.delete(item);
return issue;
}
}
stack.delete(item);
return undefined;
};
return visit(value, 0, "");
}
clear() {
const cleared = this.placeholders.clear();
this.lastProviderRedactionCount = 0;
this.lastProviderAmbiguityCount = 0;
this.recentLabels = [];
this.latestTags = [];
this.notifiedPlaceholders.clear();
return cleared;
}
stats(): EngineStats {
return {
active: true,
builtInRuleCount: builtinRules.length,
customRegexCount: this.customRules.length,
regexWarningCount: this.warnings.filter((warning) =>
warning.code.startsWith("settings.regex."),
).length,
warningCount: this.warnings.length,
invalidRegexCount: this.invalidRegexCount,
learnedCount: this.learned.length,
mappingCount: this.placeholders.stats().mappingCount,
persistenceEnabled: this.placeholders.stats().persistenceEnabled,
lastProviderRedactionCount: this.lastProviderRedactionCount,
totalProviderRedactionCount: this.totalProviderRedactionCount,
totalProtectedRedactionCount: this.totalProtectedRedactionCount,
lastProviderAmbiguityCount: this.lastProviderAmbiguityCount,
recentLabels: [...this.recentLabels],
latestTags: [...this.latestTags],
warnings: [...this.warnings],
};
}
private compileSettings() {
const compiled = compileCustomRegex(this.settings);
this.customRules = compiled.rules;
this.invalidRegexCount = compiled.invalidRegexCount;
for (const warning of compiled.warnings) this.addWarning(warning);
}
private learn(cwd: string, env: Record<string, string | undefined>) {
const dotenv = loadDotenvValues(cwd);
const processEnv = learnEnvironmentValues(env);
const learned = [...dotenv.values, ...processEnv.values];
this.learned = learned.slice(0, 8192);
if (learned.length > this.learned.length)
this.addWarning({
code: "learned.cap",
message: "combined learned value cap reached; further values ignored",
severity: "warning",
});
for (const warning of [...dotenv.warnings, ...processEnv.warnings])
this.addWarning(warning);
}
private walk<T>(
input: T,
origin: RedactionOrigin,
restore: boolean,
): DeepRedactionResult<T> {
const seen = new WeakMap<object, unknown>();
const redactions: RedactionRecord[] = [];
const ambiguities: Array<{ path: string; family: string }> = [];
let count = 0;
let changed = false;
const visit = (value: unknown, depth: number, trace: string): unknown => {
if (typeof value === "string") {
if (restore) {
const next = this.restoreText(value);
if (next !== value) {
changed = true;
count++;
}
return next;
}
const result = this.redactText(value, origin);
if (result.text !== value) changed = true;
count += result.count;
for (const item of result.redactions)
redactions.push({ ...item, path: trace || "$" });
for (const item of result.ambiguities)
ambiguities.push({ ...item, path: trace || "$" });
return result.text;
}
if (value === null || typeof value !== "object") return value;
if (depth >= 64) {
this.warnOnce(
"traversal.depth",
"maximum redaction traversal depth reached",
);
return value;
}
const cached = seen.get(value);
if (cached) {
this.warnOnce(
"traversal.cycle",
"cyclic value left unchanged during redaction traversal",
);
return cached;
}
if (Array.isArray(value)) {
const copy: unknown[] = [];
seen.set(value, copy);
for (let index = 0; index < value.length; index++)
copy.push(visit(value[index], depth + 1, `${trace}[${index}]`));
if (copy.some((item, index) => item !== value[index])) changed = true;
return copy;
}
if (!isPlainObject(value)) {
this.warnOnce(
"traversal.unsupported",
"unsupported object left unchanged during redaction traversal",
);
return value;
}
const source = value as Record<string, unknown>;
const copy: Record<string, unknown> = {};
seen.set(value, copy);
for (const [index, [key, item]] of Object.entries(source).entries()) {
const keyPath = `${childPath(trace, index)}.key`;
let keyResult = this.restoreText(key);
if (!restore) {
const result = this.redactText(key, origin);
keyResult = result.text;
count += result.count;
for (const redaction of result.redactions)
redactions.push({ ...redaction, path: keyPath });
for (const ambiguity of result.ambiguities)
ambiguities.push({ ...ambiguity, path: keyPath });
}
Object.defineProperty(copy, keyResult, {
value: visit(item, depth + 1, childPath(trace, index)),
enumerable: true,
configurable: true,
writable: true,
});
}
if (Object.keys(copy).some((key) => copy[key] !== source[key]))
changed = true;
return copy;
};
const value = visit(input, 0, "") as T;
if (origin === "provider" && !restore) {
this.lastProviderRedactionCount = count;
this.lastProviderAmbiguityCount = ambiguities.length;
}
return { value, changed, count, redactions, ambiguities };
}
private warnOnce(code: string, message: string) {
if (this.warningKeys.has(code)) return;
this.warningKeys.add(code);
this.addWarning({ code, message, severity: "warning" });
}
private addWarning(warning: WarningRecord) {
this.warnings.push(warning);
}
}
function placeholderLabel(placeholder: string) {
const current = /^__LEAK_V1_(.+)_[a-f0-9]{32}__$/.exec(placeholder);
if (current) return current[1];
const legacy = /^__LEAK_(.+)_[a-f0-9]{12}(?:_\d+)?__$/.exec(placeholder);
return legacy?.[1] ?? "UNKNOWN";
}
function childPath(parent: string, index: number) {
return `${parent || "$"}{${index}}`;
}