repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

extensions/leaks/src/dotenv.ts

Raw
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
import path from "node:path";
import { isWeakValue, sanitizeTag } from "./safety.ts";
import type { SecretSpan, SpanLabel, WarningRecord } from "./types.ts";

const CAP = 8192;
const PUBLIC_PREFIX_RE = /^(?:PUBLIC|NEXT_PUBLIC|VITE)[_-]/i;
const METADATA_SUFFIX_RE =
	/_(?:ID|NAME|PATH|FILE|TYPE|HOST|PORT|MANAGER|SOCKET|ENDPOINT)$/;
const CONTEXT_ONLY = new Set([
	"KEY",
	"AUTH",
	"SESSION",
	"COOKIE",
	"JWT",
	"SEED",
	"DATABASE_URL",
	"MONGO_URL",
	"REDIS_URL",
]);
const SENSITIVE = new Set([
	"TOKEN",
	"SECRET",
	"PASSWORD",
	"PASSWD",
	"PASS",
	"PEPPER",
	"API_KEY",
	"ACCESS_KEY",
	"PRIVATE_KEY",
	"KEY",
	"KEY_DATA",
	"PAT",
	"PASSPHRASE",
	"LICENSE_KEY",
	"MASTER_KEY",
	"ROUTING_KEY",
	"ENCRYPTION_KEY",
	"SECRET_KEY_BASE",
	"SEED",
	"UNSEAL_KEY",
	"MNEMONIC",
	"CLIENT_SECRET",
	"AUTH",
	"CREDENTIAL",
	"SESSION",
	"COOKIE",
	"JWT",
	"DATABASE_URL",
	"MONGO_URL",
	"REDIS_URL",
]);
const LEARNABLE = new Set(
	[...SENSITIVE].filter((sensitive) => !CONTEXT_ONLY.has(sensitive)),
);

export type LearnedValue = {
	value: string;
	labels: SpanLabel[];
};

export type LearnedValues = {
	values: LearnedValue[];
	warnings: WarningRecord[];
};

export function discoverDotenvFiles(cwd: string): string[] {
	const direct = [".env", ".env.local"]
		.map((file) => path.join(cwd, file))
		.filter((file) => existsSync(file));
	const local = readdirSync(cwd, { withFileTypes: true })
		.filter((entry) => entry.isFile())
		.map((entry) => entry.name)
		.filter((name) => /^\.env\..+\.local$/.test(name))
		.sort()
		.map((file) => path.join(cwd, file));
	return [...direct, ...local];
}

export function parseDotenv(
	text: string,
): Array<{ key: string; value: string }> {
	const rows: Array<{ key: string; value: string }> = [];
	for (const raw of text.split(/\r?\n/)) {
		let line = raw.trim();
		if (!line || line.startsWith("#")) continue;
		line = line.replace(/^export\s+/, "");
		const match = /^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*(.*)$/.exec(line);
		if (!match) continue;
		let value = match[2] as string;
		if (value.startsWith("'") || value.startsWith('"')) {
			if (value.includes("\\")) continue;
			const quote = value[0];
			const closing = value.indexOf(quote, 1);
			if (closing < 0 || !/^(?:\s+#.*)?$/.test(value.slice(closing + 1)))
				continue;
			value = value.slice(1, closing);
		} else value = value.replace(/\s+#.*$/, "").trim();
		rows.push({ key: match[1], value });
	}
	return rows;
}

export function loadDotenvValues(cwd: string): LearnedValues {
	const pairs: Record<string, string> = {};
	const warnings: WarningRecord[] = [];
	let files: string[];
	try {
		files = discoverDotenvFiles(cwd);
	} catch {
		return {
			values: [],
			warnings: [dotenvWarning("dotenv directory is unreadable")],
		};
	}
	for (const file of files) {
		try {
			if (!statSync(file).isFile()) throw new Error("not a file");
			for (const item of parseDotenv(readFileSync(file, "utf8")))
				pairs[item.key] = item.value;
		} catch {
			warnings.push(
				dotenvWarning(`dotenv file is unreadable: ${path.basename(file)}`),
			);
		}
	}
	const learned = learnKeyValues(pairs, "dotenv");
	return {
		values: learned.values,
		warnings: [...warnings, ...learned.warnings],
	};
}

export function learnEnvironmentValues(
	env: Record<string, string | undefined>,
	cap = CAP,
): LearnedValues {
	const pairs: Record<string, string> = {};
	for (const [key, value] of Object.entries(env))
		if (typeof value === "string") pairs[key] = value;
	return learnKeyValues(pairs, "env", cap);
}

export function isSensitiveKey(key: string): boolean {
	return hasTokens(
		normalizeKey(key.replace(PUBLIC_PREFIX_RE, "")).split("_"),
		SENSITIVE,
	);
}

export function highConfidenceSecretShape(value: string): boolean {
	return /^(?:sk-or-v1-[A-Za-z0-9]{32,}|sk-proj-[A-Za-z0-9_-]{32,}|sk-ant-api03-[A-Za-z0-9_-]{32,}|ghp_[A-Za-z0-9]{36}|hf_[A-Za-z0-9]{32,})$/.test(
		value,
	);
}

export function learnedValueSpans(
	text: string,
	values: LearnedValue[],
): SecretSpan[] {
	const spans: SecretSpan[] = [];
	for (const learned of values) {
		let start = text.indexOf(learned.value);
		while (start !== -1) {
			spans.push({
				start,
				end: start + learned.value.length,
				value: learned.value,
				labels: learned.labels,
				kind: "exact",
				confidence: 70,
				wholeSpan: false,
			});
			start = text.indexOf(learned.value, start + learned.value.length);
		}
	}
	return spans;
}

function learnKeyValues(
	pairs: Record<string, string>,
	source: "dotenv" | "env",
	cap = CAP,
): LearnedValues {
	const values: LearnedValue[] = [];
	const warnings: WarningRecord[] = [];
	let warned = false;
	const atCap = () => {
		if (values.length < cap) return false;
		if (!warned) {
			warnings.push({
				code: "learned.cap",
				message: "learned value cap reached; further dotenv/env values ignored",
				severity: "warning",
			});
			warned = true;
		}
		return true;
	};
	for (const [key, value] of Object.entries(pairs)) {
		if (atCap()) continue;
		const components = urlSecretComponents(value);
		if (components.length > 0) {
			for (const component of components) {
				if (atCap()) break;
				pushLearned(values, component, key, source);
			}
			continue;
		}
		const secretShape = highConfidenceSecretShape(value);
		if (!secretShape && (!isLearnableKey(key) || isWeakValue(value))) continue;
		pushLearned(values, value, key, source);
	}
	return { values, warnings };
}

function pushLearned(
	values: LearnedValue[],
	value: string,
	key: string,
	source: "dotenv" | "env",
) {
	if (isWeakValue(value)) return;
	values.push({
		value,
		labels: [
			{
				tag: source === "env" ? "ENV_KEY" : sanitizeTag(key),
				priority: 10,
				source,
			},
		],
	});
}

function urlSecretComponents(value: string) {
	try {
		const url = new URL(value);
		const components = url.password ? [url.password] : [];
		for (const [name, item] of url.searchParams)
			if (isSensitiveKey(name) && !isWeakValue(item)) components.push(item);
		return components;
	} catch {
		return [];
	}
}

function isLearnableKey(key: string) {
	if (PUBLIC_PREFIX_RE.test(key)) return false;
	const normalized = normalizeKey(key);
	if (METADATA_SUFFIX_RE.test(normalized)) return false;
	return hasTokens(normalized.split("_"), LEARNABLE);
}

function hasTokens(tokens: string[], names: Set<string>) {
	return [...names].some((name) => {
		const expected = name.split("_");
		return tokens.some((_, start) =>
			expected.every((token, offset) => tokens[start + offset] === token),
		);
	});
}

function dotenvWarning(message: string): WarningRecord {
	return { code: "dotenv.read", message, severity: "warning" };
}

function normalizeKey(key: string) {
	return key.toUpperCase().replace(/[-]+/g, "_").replace(/_\d+$/, "");
}