repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

extensions/leaks/src/structures.ts

Raw
import { highConfidenceSecretShape, isSensitiveKey } from "./dotenv.ts";
import { isWeakValue } from "./safety.ts";
import type { SecretSpan, SpanLabel } from "./types.ts";

const URL_CANDIDATE_RE = /\b[a-z][a-z0-9+.-]*:\/\/[^\s'"`]+/dgi;
const URL_CREDENTIAL_RE =
	/\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]*)?:([^\s@/]+)@[^\s'"`]+/dgi;
const QUERY_RE = /([?&])([^=&#\s'"`]+)=([^&#\s'"`]+)/dgi;
const AUTH_RE =
	/(?:^|[\n\r\s'"`])Authorization:\s*(Bearer|Basic)\s+([^\s'"`]+)/dgi;
const X_API_RE = /(?:^|[\n\r\s'"`])(X-API-Key|Api-Key):\s*([^\s'"`]+)/dgi;
const COOKIE_LINE_RE = /(?:^|[\n\r])\s*Cookie:\s*([^\r\n]+)/dgi;
// Keys that look like assignments but are common programming nouns, not secrets.
// Scoped to findAssignmentSpans only — does NOT affect dotenv/env learning
// or structure non-assignment detectors. Excludes any key in dotenv.SENSITIVE
// (TOKEN, KEY, SECRET, …) since those remain real secret indicators. 'seed' is
// the lone SENSITIVE key in this list: it dominates as a programming term in
// tool-arg JSON, and users with real SEED secrets can use a custom regex.
const ASSIGNMENT_STOP_KEYS = new Set([
	"seed",
	"id",
	"name",
	"count",
	"type",
	"value",
	"path",
	"ts",
	"time",
	"date",
	"idx",
	"mode",
	"flag",
	"opts",
	"params",
	"args",
	"options",
	"label",
	"field",
	"tag",
	"node",
	"host",
	"port",
	"version",
]);

const ASSIGN_RE =
	/(?:^|[\n\r\s{,;])(?:export\s+)?["']?([A-Za-z_][A-Za-z0-9_-]*)["']?\s*(?:=|:|=>)\s*["']?([^\s'",;}]{1,512})["']?/dgi;
const XML_RE = /<([A-Za-z_][A-Za-z0-9_-]*)>([^<]{1,512})<\/\1>/dgi;
const JWT_RE =
	/(?<![A-Za-z0-9_-])[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)+(?![A-Za-z0-9_-])/dg;
const PRIVATE_KEY_RE =
	/-----BEGIN ([A-Z0-9 ]*PRIVATE KEY)-----[\s\S]*?-----END \1-----/dg;

export const structureRegexFuzzTargets = [
	["url-credential", URL_CREDENTIAL_RE],
	["query", QUERY_RE],
	["auth-header", AUTH_RE],
	["api-header", X_API_RE],
	["cookie-line", COOKIE_LINE_RE],
	["assignment", ASSIGN_RE],
	["xml", XML_RE],
	["jwt", JWT_RE],
	["private-key", PRIVATE_KEY_RE],
].map(([name, regex]) => ({
	name: `structure:${name}`,
	regex: regex as RegExp,
}));

export function findStructureAmbiguities(text: string) {
	const ambiguities: Array<{ family: string; start: number }> = [];
	for (const match of text.matchAll(JWT_RE)) {
		const header = decodeJsonObject(match[0].split(".")[0] ?? "");
		if (
			header &&
			typeof header.alg === "string" &&
			header.alg.length > 0 &&
			findJwtSpans(match[0]).length === 0
		)
			ambiguities.push({ family: "JWT", start: match.index as number });
	}
	for (const match of text.matchAll(AUTH_RE))
		if (
			match[1]?.toLowerCase() === "basic" &&
			findHeaderSpans(match[0]).length === 0
		)
			ambiguities.push({ family: "BASIC_AUTH", start: match.index as number });
	for (const match of text.matchAll(URL_CANDIDATE_RE)) {
		if (!candidateHasSensitiveQueryName(match[0])) continue;
		if (findSensitiveQuerySpans(match[0]).length === 0)
			ambiguities.push({ family: "URL_QUERY", start: match.index as number });
	}
	return ambiguities;
}

export function findStructureSpans(text: string): SecretSpan[] {
	return [
		...findPrivateKeySpans(text),
		...findJwtSpans(text),
		...findCredentialedUrlSpans(text),
		...findSensitiveQuerySpans(text),
		...findHeaderSpans(text),
		...findCookieSpans(text),
		...findAssignmentSpans(text),
	];
}

export function findCredentialedUrlSpans(text: string): SecretSpan[] {
	const spans: SecretSpan[] = [];
	for (const match of text.matchAll(URL_CREDENTIAL_RE)) {
		try {
			new URL(match[0]);
		} catch {
			continue;
		}
		const value = match[2] as string;
		if (isWeakValue(value)) continue;
		spans.push(
			makeSpan(
				match,
				2,
				value,
				[label("DB_PASSWORD", "structure")],
				"structure",
			),
		);
	}
	return spans;
}

export function findSensitiveQuerySpans(text: string): SecretSpan[] {
	const spans: SecretSpan[] = [];
	for (const candidate of text.matchAll(URL_CANDIDATE_RE)) {
		const rawCandidate = trimUrlCandidate(candidate[0]);
		try {
			new URL(rawCandidate);
		} catch {
			continue;
		}
		const question = rawCandidate.indexOf("?");
		if (question < 0) continue;
		const fragment = rawCandidate.indexOf("#", question);
		const end = fragment < 0 ? rawCandidate.length : fragment;
		spans.push(
			...parsedQuerySpans(
				rawCandidate.slice(question + 1, end),
				(candidate.index as number) + question + 1,
			),
		);
	}
	const trimmedStart = text.search(/\S/);
	if (trimmedStart >= 0 && text[trimmedStart] === "?") {
		const endMatch = /[\s'"`]/.exec(text.slice(trimmedStart));
		const end = endMatch
			? trimmedStart + (endMatch.index as number)
			: text.length;
		const fragment = text.indexOf("#", trimmedStart);
		const queryEnd = fragment >= 0 && fragment < end ? fragment : end;
		spans.push(
			...parsedQuerySpans(
				text.slice(trimmedStart + 1, queryEnd),
				trimmedStart + 1,
			),
		);
	}
	return spans;
}

export function findHeaderSpans(text: string): SecretSpan[] {
	return [
		...matches(
			AUTH_RE,
			text,
			2,
			"AUTHORIZATION",
			(match) => `AUTHORIZATION_${match[1]?.toUpperCase()}`,
			(match) => match[1]?.toLowerCase() !== "basic" || validBasic(match[2]),
		),
		...matches(X_API_RE, text, 2, "X_API_KEY"),
	];
}

export function findCookieSpans(text: string): SecretSpan[] {
	const spans: SecretSpan[] = [];
	for (const line of text.matchAll(COOKIE_LINE_RE)) {
		const body = line[1] as string;
		const bodyIndices = line.indices?.[1];
		if (!bodyIndices) throw new Error("Cookie regex missing body indices");
		let offset = bodyIndices[0];
		for (const part of body.split(";")) {
			const trimmed = part.trim();
			const eq = trimmed.indexOf("=");
			if (eq === -1) {
				offset += part.length + 1;
				continue;
			}
			const name = trimmed.slice(0, eq);
			const value = trimmed.slice(eq + 1);
			const localStart = part.indexOf(trimmed) + eq + 1;
			if (isSensitiveCookie(name) && !isWeakValue(value)) {
				const start = offset + Math.max(localStart, 0);
				spans.push({
					start,
					end: start + value.length,
					value,
					labels: [label(`COOKIE_${name}`, "structure")],
					kind: "structure",
					confidence: 85,
					wholeSpan: false,
				});
			}
			offset += part.length + 1;
		}
	}
	return spans;
}

export function findAssignmentSpans(text: string): SecretSpan[] {
	const spans: SecretSpan[] = [];
	for (const regex of [ASSIGN_RE, XML_RE]) {
		regex.lastIndex = 0;
		for (const match of text.matchAll(regex)) {
			const key = match[1] as string;
			const value = match[2] as string;
			if (ASSIGNMENT_STOP_KEYS.has(key.toLowerCase())) continue;
			if (!isSensitiveKey(key)) continue;
			const hcloudToken = isHCloudTokenAssignment(key, value);
			if (isCodeReference(value) && !hcloudToken) continue;
			if (!highConfidenceSecretShape(value) && !hcloudToken) continue;
			spans.push(
				makeSpan(match, 2, value, [label(key, "structure")], "structure"),
			);
		}
	}
	return spans;
}

export function findJwtSpans(text: string): SecretSpan[] {
	const spans: SecretSpan[] = [];
	for (const match of text.matchAll(JWT_RE)) {
		const parts = match[0].split(".");
		if (parts.length !== 3) continue;
		const [headerPart, payloadPart, signaturePart] = parts;
		if (!headerPart || !payloadPart || !strictBase64Url(signaturePart))
			continue;
		const header = decodeJsonObject(headerPart);
		const payload = decodeJsonObject(payloadPart);
		if (!header || !payload || typeof header.alg !== "string" || !header.alg)
			continue;
		const start = match.index as number;
		spans.push({
			start,
			end: start + match[0].length,
			value: match[0],
			labels: [label("JWT", "builtin")],
			kind: "jwt",
			confidence: 100,
			wholeSpan: true,
		});
	}
	return spans;
}

function strictBase64Url(value: string | undefined) {
	if (!value || !/^[A-Za-z0-9_-]+$/.test(value)) return false;
	return Buffer.from(value, "base64url").toString("base64url") === value;
}

function decodeJsonObject(value: string) {
	if (!strictBase64Url(value)) return undefined;
	try {
		const bytes = Buffer.from(value, "base64url");
		const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
		const parsed: unknown = JSON.parse(text);
		if (!isPlainRecord(parsed)) return undefined;
		return parsed;
	} catch {
		return undefined;
	}
}

function isPlainRecord(value: unknown): value is Record<string, unknown> {
	return value !== null && typeof value === "object" && !Array.isArray(value);
}

export function findPrivateKeySpans(text: string): SecretSpan[] {
	return [...text.matchAll(PRIVATE_KEY_RE)].map((match) => {
		const start = match.index as number;
		const keyType = match[1] as string;
		return {
			start,
			end: start + match[0].length,
			value: match[0],
			labels: [label(keyType.replace(/ /g, "_"), "builtin")],
			kind: "private-key",
			confidence: 100,
			wholeSpan: true,
		};
	});
}

function isHCloudTokenAssignment(key: string, value: string) {
	const normalized = key.replaceAll(/[-_]/g, "").toLowerCase();
	return (
		(normalized === "hcloudtoken" || normalized === "tfvarhcloudtoken") &&
		/^[A-Za-z0-9]{64}$/.test(value)
	);
}

function isCodeReference(value: string) {
	const text = value.trim();
	return (
		text.startsWith("$") ||
		/[()[\]{}]/.test(text) ||
		/^[A-Z][A-Za-z0-9_$]*$/.test(text) ||
		/^(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+[A-Za-z_$][A-Za-z0-9_$]*$/.test(text)
	);
}

function matches(
	regex: RegExp,
	text: string,
	group: number,
	fallback: string,
	tag?: (match: RegExpMatchArray) => string,
	accept: (match: RegExpMatchArray) => boolean = () => true,
): SecretSpan[] {
	regex.lastIndex = 0;
	const spans: SecretSpan[] = [];
	for (const match of text.matchAll(regex)) {
		const value = match[group] as string;
		if (!accept(match) || isWeakValue(value)) continue;
		spans.push(
			makeSpan(
				match,
				group,
				value,
				[label(tag ? tag(match) : fallback, "structure")],
				"structure",
			),
		);
	}
	return spans;
}

function makeSpan(
	match: RegExpMatchArray,
	group: number,
	value: string,
	labels: SpanLabel[],
	kind: SecretSpan["kind"],
): SecretSpan {
	const range = match.indices?.[group] as [number, number];
	const start = range[0];
	return {
		start,
		end: range[1],
		value,
		labels,
		kind,
		confidence: 85,
		wholeSpan: false,
	};
}

function label(tag: string, source: SpanLabel["source"]): SpanLabel {
	return { tag, priority: source === "builtin" ? 40 : 20, source };
}

function trimUrlCandidate(candidate: string) {
	return candidate.replace(/[\])},.;!?]+$/, "");
}

function candidateHasSensitiveQueryName(candidate: string) {
	const question = candidate.indexOf("?");
	if (question < 0) return false;
	return [...new URLSearchParams(candidate.slice(question + 1))].some(
		([name]) => isSensitiveKey(name),
	);
}

function parsedQuerySpans(query: string, offset: number): SecretSpan[] {
	const parsed = [...new URLSearchParams(query)];
	const rawPairs: Array<{ pair: string; start: number }> = [];
	let cursor = 0;
	for (const pair of query.split("&")) {
		if (pair.length > 0) rawPairs.push({ pair, start: cursor });
		cursor += pair.length + 1;
	}
	if (parsed.length !== rawPairs.length) return [];
	const spans: SecretSpan[] = [];
	for (let index = 0; index < parsed.length; index++) {
		const [name, decodedValue] = parsed[index];
		const { pair, start } = rawPairs[index];
		const equals = pair.indexOf("=");
		if (equals <= 0) return [];
		const pairEntries = [...new URLSearchParams(pair)];
		if (
			pairEntries.length !== 1 ||
			pairEntries[0][0] !== name ||
			pairEntries[0][1] !== decodedValue
		)
			return [];
		const rawValue = pair.slice(equals + 1);
		if (!isSensitiveKey(name) || isWeakValue(rawValue)) continue;
		const valueStart = offset + start + equals + 1;
		spans.push({
			start: valueStart,
			end: valueStart + rawValue.length,
			value: rawValue,
			labels: [label(name, "structure")],
			kind: "structure",
			confidence: 85,
			wholeSpan: false,
		});
	}
	return spans;
}

function validBasic(value: string | undefined) {
	if (!value || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) return false;
	try {
		const bytes = Buffer.from(value, "base64");
		if (bytes.toString("base64") !== value) return false;
		return new TextDecoder("utf-8", { fatal: true })
			.decode(bytes)
			.includes(":");
	} catch {
		return false;
	}
}

function isSensitiveCookie(name: string) {
	return isSensitiveKey(name);
}