import { highConfidenceSecretShape, isSensitiveKey } from "./dotenv.ts"; import { isWeakValue } from "./safety.ts"; import type { SecretSpan, SpanLabel } from "./types.ts"; const URL_CANDIDATE_RE = /\b[a-z][a-z0-9+.-]*:\/\/[^\s'"`]+/dgi; const URL_CREDENTIAL_RE = /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]*)?:([^\s@/]+)@[^\s'"`]+/dgi; const QUERY_RE = /([?&])([^=&#\s'"`]+)=([^&#\s'"`]+)/dgi; const AUTH_RE = /(?:^|[\n\r\s'"`])Authorization:\s*(Bearer|Basic)\s+([^\s'"`]+)/dgi; const X_API_RE = /(?:^|[\n\r\s'"`])(X-API-Key|Api-Key):\s*([^\s'"`]+)/dgi; const COOKIE_LINE_RE = /(?:^|[\n\r])\s*Cookie:\s*([^\r\n]+)/dgi; // Keys that look like assignments but are common programming nouns, not secrets. // Scoped to findAssignmentSpans only — does NOT affect dotenv/env learning // or structure non-assignment detectors. Excludes any key in dotenv.SENSITIVE // (TOKEN, KEY, SECRET, …) since those remain real secret indicators. 'seed' is // the lone SENSITIVE key in this list: it dominates as a programming term in // tool-arg JSON, and users with real SEED secrets can use a custom regex. const ASSIGNMENT_STOP_KEYS = new Set([ "seed", "id", "name", "count", "type", "value", "path", "ts", "time", "date", "idx", "mode", "flag", "opts", "params", "args", "options", "label", "field", "tag", "node", "host", "port", "version", ]); const ASSIGN_RE = /(?:^|[\n\r\s{,;])(?:export\s+)?["']?([A-Za-z_][A-Za-z0-9_-]*)["']?\s*(?:=|:|=>)\s*["']?([^\s'",;}]{1,512})["']?/dgi; const XML_RE = /<([A-Za-z_][A-Za-z0-9_-]*)>([^<]{1,512})<\/\1>/dgi; const JWT_RE = /(? ({ name: `structure:${name}`, regex: regex as RegExp, })); export function findStructureAmbiguities(text: string) { const ambiguities: Array<{ family: string; start: number }> = []; for (const match of text.matchAll(JWT_RE)) { const header = decodeJsonObject(match[0].split(".")[0] ?? ""); if ( header && typeof header.alg === "string" && header.alg.length > 0 && findJwtSpans(match[0]).length === 0 ) ambiguities.push({ family: "JWT", start: match.index as number }); } for (const match of text.matchAll(AUTH_RE)) if ( match[1]?.toLowerCase() === "basic" && findHeaderSpans(match[0]).length === 0 ) ambiguities.push({ family: "BASIC_AUTH", start: match.index as number }); for (const match of text.matchAll(URL_CANDIDATE_RE)) { if (!candidateHasSensitiveQueryName(match[0])) continue; if (findSensitiveQuerySpans(match[0]).length === 0) ambiguities.push({ family: "URL_QUERY", start: match.index as number }); } return ambiguities; } export function findStructureSpans(text: string): SecretSpan[] { return [ ...findPrivateKeySpans(text), ...findJwtSpans(text), ...findCredentialedUrlSpans(text), ...findSensitiveQuerySpans(text), ...findHeaderSpans(text), ...findCookieSpans(text), ...findAssignmentSpans(text), ]; } export function findCredentialedUrlSpans(text: string): SecretSpan[] { const spans: SecretSpan[] = []; for (const match of text.matchAll(URL_CREDENTIAL_RE)) { try { new URL(match[0]); } catch { continue; } const value = match[2] as string; if (isWeakValue(value)) continue; spans.push( makeSpan( match, 2, value, [label("DB_PASSWORD", "structure")], "structure", ), ); } return spans; } export function findSensitiveQuerySpans(text: string): SecretSpan[] { const spans: SecretSpan[] = []; for (const candidate of text.matchAll(URL_CANDIDATE_RE)) { const rawCandidate = trimUrlCandidate(candidate[0]); try { new URL(rawCandidate); } catch { continue; } const question = rawCandidate.indexOf("?"); if (question < 0) continue; const fragment = rawCandidate.indexOf("#", question); const end = fragment < 0 ? rawCandidate.length : fragment; spans.push( ...parsedQuerySpans( rawCandidate.slice(question + 1, end), (candidate.index as number) + question + 1, ), ); } const trimmedStart = text.search(/\S/); if (trimmedStart >= 0 && text[trimmedStart] === "?") { const endMatch = /[\s'"`]/.exec(text.slice(trimmedStart)); const end = endMatch ? trimmedStart + (endMatch.index as number) : text.length; const fragment = text.indexOf("#", trimmedStart); const queryEnd = fragment >= 0 && fragment < end ? fragment : end; spans.push( ...parsedQuerySpans( text.slice(trimmedStart + 1, queryEnd), trimmedStart + 1, ), ); } return spans; } export function findHeaderSpans(text: string): SecretSpan[] { return [ ...matches( AUTH_RE, text, 2, "AUTHORIZATION", (match) => `AUTHORIZATION_${match[1]?.toUpperCase()}`, (match) => match[1]?.toLowerCase() !== "basic" || validBasic(match[2]), ), ...matches(X_API_RE, text, 2, "X_API_KEY"), ]; } export function findCookieSpans(text: string): SecretSpan[] { const spans: SecretSpan[] = []; for (const line of text.matchAll(COOKIE_LINE_RE)) { const body = line[1] as string; const bodyIndices = line.indices?.[1]; if (!bodyIndices) throw new Error("Cookie regex missing body indices"); let offset = bodyIndices[0]; for (const part of body.split(";")) { const trimmed = part.trim(); const eq = trimmed.indexOf("="); if (eq === -1) { offset += part.length + 1; continue; } const name = trimmed.slice(0, eq); const value = trimmed.slice(eq + 1); const localStart = part.indexOf(trimmed) + eq + 1; if (isSensitiveCookie(name) && !isWeakValue(value)) { const start = offset + Math.max(localStart, 0); spans.push({ start, end: start + value.length, value, labels: [label(`COOKIE_${name}`, "structure")], kind: "structure", confidence: 85, wholeSpan: false, }); } offset += part.length + 1; } } return spans; } export function findAssignmentSpans(text: string): SecretSpan[] { const spans: SecretSpan[] = []; for (const regex of [ASSIGN_RE, XML_RE]) { regex.lastIndex = 0; for (const match of text.matchAll(regex)) { const key = match[1] as string; const value = match[2] as string; if (ASSIGNMENT_STOP_KEYS.has(key.toLowerCase())) continue; if (!isSensitiveKey(key)) continue; const hcloudToken = isHCloudTokenAssignment(key, value); if (isCodeReference(value) && !hcloudToken) continue; if (!highConfidenceSecretShape(value) && !hcloudToken) continue; spans.push( makeSpan(match, 2, value, [label(key, "structure")], "structure"), ); } } return spans; } export function findJwtSpans(text: string): SecretSpan[] { const spans: SecretSpan[] = []; for (const match of text.matchAll(JWT_RE)) { const parts = match[0].split("."); if (parts.length !== 3) continue; const [headerPart, payloadPart, signaturePart] = parts; if (!headerPart || !payloadPart || !strictBase64Url(signaturePart)) continue; const header = decodeJsonObject(headerPart); const payload = decodeJsonObject(payloadPart); if (!header || !payload || typeof header.alg !== "string" || !header.alg) continue; const start = match.index as number; spans.push({ start, end: start + match[0].length, value: match[0], labels: [label("JWT", "builtin")], kind: "jwt", confidence: 100, wholeSpan: true, }); } return spans; } function strictBase64Url(value: string | undefined) { if (!value || !/^[A-Za-z0-9_-]+$/.test(value)) return false; return Buffer.from(value, "base64url").toString("base64url") === value; } function decodeJsonObject(value: string) { if (!strictBase64Url(value)) return undefined; try { const bytes = Buffer.from(value, "base64url"); const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); const parsed: unknown = JSON.parse(text); if (!isPlainRecord(parsed)) return undefined; return parsed; } catch { return undefined; } } function isPlainRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } export function findPrivateKeySpans(text: string): SecretSpan[] { return [...text.matchAll(PRIVATE_KEY_RE)].map((match) => { const start = match.index as number; const keyType = match[1] as string; return { start, end: start + match[0].length, value: match[0], labels: [label(keyType.replace(/ /g, "_"), "builtin")], kind: "private-key", confidence: 100, wholeSpan: true, }; }); } function isHCloudTokenAssignment(key: string, value: string) { const normalized = key.replaceAll(/[-_]/g, "").toLowerCase(); return ( (normalized === "hcloudtoken" || normalized === "tfvarhcloudtoken") && /^[A-Za-z0-9]{64}$/.test(value) ); } function isCodeReference(value: string) { const text = value.trim(); return ( text.startsWith("$") || /[()[\]{}]/.test(text) || /^[A-Z][A-Za-z0-9_$]*$/.test(text) || /^(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+[A-Za-z_$][A-Za-z0-9_$]*$/.test(text) ); } function matches( regex: RegExp, text: string, group: number, fallback: string, tag?: (match: RegExpMatchArray) => string, accept: (match: RegExpMatchArray) => boolean = () => true, ): SecretSpan[] { regex.lastIndex = 0; const spans: SecretSpan[] = []; for (const match of text.matchAll(regex)) { const value = match[group] as string; if (!accept(match) || isWeakValue(value)) continue; spans.push( makeSpan( match, group, value, [label(tag ? tag(match) : fallback, "structure")], "structure", ), ); } return spans; } function makeSpan( match: RegExpMatchArray, group: number, value: string, labels: SpanLabel[], kind: SecretSpan["kind"], ): SecretSpan { const range = match.indices?.[group] as [number, number]; const start = range[0]; return { start, end: range[1], value, labels, kind, confidence: 85, wholeSpan: false, }; } function label(tag: string, source: SpanLabel["source"]): SpanLabel { return { tag, priority: source === "builtin" ? 40 : 20, source }; } function trimUrlCandidate(candidate: string) { return candidate.replace(/[\])},.;!?]+$/, ""); } function candidateHasSensitiveQueryName(candidate: string) { const question = candidate.indexOf("?"); if (question < 0) return false; return [...new URLSearchParams(candidate.slice(question + 1))].some( ([name]) => isSensitiveKey(name), ); } function parsedQuerySpans(query: string, offset: number): SecretSpan[] { const parsed = [...new URLSearchParams(query)]; const rawPairs: Array<{ pair: string; start: number }> = []; let cursor = 0; for (const pair of query.split("&")) { if (pair.length > 0) rawPairs.push({ pair, start: cursor }); cursor += pair.length + 1; } if (parsed.length !== rawPairs.length) return []; const spans: SecretSpan[] = []; for (let index = 0; index < parsed.length; index++) { const [name, decodedValue] = parsed[index]; const { pair, start } = rawPairs[index]; const equals = pair.indexOf("="); if (equals <= 0) return []; const pairEntries = [...new URLSearchParams(pair)]; if ( pairEntries.length !== 1 || pairEntries[0][0] !== name || pairEntries[0][1] !== decodedValue ) return []; const rawValue = pair.slice(equals + 1); if (!isSensitiveKey(name) || isWeakValue(rawValue)) continue; const valueStart = offset + start + equals + 1; spans.push({ start: valueStart, end: valueStart + rawValue.length, value: rawValue, labels: [label(name, "structure")], kind: "structure", confidence: 85, wholeSpan: false, }); } return spans; } function validBasic(value: string | undefined) { if (!value || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) return false; try { const bytes = Buffer.from(value, "base64"); if (bytes.toString("base64") !== value) return false; return new TextDecoder("utf-8", { fatal: true }) .decode(bytes) .includes(":"); } catch { return false; } } function isSensitiveCookie(name: string) { return isSensitiveKey(name); }