repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by bugabinga
extensions/leaks/src/structures.ts
Rawimport { highConfidenceSecretShape, isSensitiveKey } from "./dotenv.ts";
import { isWeakValue } from "./safety.ts";
import type { SecretSpan, SpanLabel } from "./types.ts";
const URL_CANDIDATE_RE = /\b[a-z][a-z0-9+.-]*:\/\/[^\s'"`]+/dgi;
const URL_CREDENTIAL_RE =
/\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]*)?:([^\s@/]+)@[^\s'"`]+/dgi;
const QUERY_RE = /([?&])([^=&#\s'"`]+)=([^&#\s'"`]+)/dgi;
const AUTH_RE =
/(?:^|[\n\r\s'"`])Authorization:\s*(Bearer|Basic)\s+([^\s'"`]+)/dgi;
const X_API_RE = /(?:^|[\n\r\s'"`])(X-API-Key|Api-Key):\s*([^\s'"`]+)/dgi;
const COOKIE_LINE_RE = /(?:^|[\n\r])\s*Cookie:\s*([^\r\n]+)/dgi;
// Keys that look like assignments but are common programming nouns, not secrets.
// Scoped to findAssignmentSpans only — does NOT affect dotenv/env learning
// or structure non-assignment detectors. Excludes any key in dotenv.SENSITIVE
// (TOKEN, KEY, SECRET, …) since those remain real secret indicators. 'seed' is
// the lone SENSITIVE key in this list: it dominates as a programming term in
// tool-arg JSON, and users with real SEED secrets can use a custom regex.
const ASSIGNMENT_STOP_KEYS = new Set([
"seed",
"id",
"name",
"count",
"type",
"value",
"path",
"ts",
"time",
"date",
"idx",
"mode",
"flag",
"opts",
"params",
"args",
"options",
"label",
"field",
"tag",
"node",
"host",
"port",
"version",
]);
const ASSIGN_RE =
/(?:^|[\n\r\s{,;])(?:export\s+)?["']?([A-Za-z_][A-Za-z0-9_-]*)["']?\s*(?:=|:|=>)\s*["']?([^\s'",;}]{1,512})["']?/dgi;
const XML_RE = /<([A-Za-z_][A-Za-z0-9_-]*)>([^<]{1,512})<\/\1>/dgi;
const JWT_RE =
/(?<![A-Za-z0-9_-])[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)+(?![A-Za-z0-9_-])/dg;
const PRIVATE_KEY_RE =
/-----BEGIN ([A-Z0-9 ]*PRIVATE KEY)-----[\s\S]*?-----END \1-----/dg;
export const structureRegexFuzzTargets = [
["url-credential", URL_CREDENTIAL_RE],
["query", QUERY_RE],
["auth-header", AUTH_RE],
["api-header", X_API_RE],
["cookie-line", COOKIE_LINE_RE],
["assignment", ASSIGN_RE],
["xml", XML_RE],
["jwt", JWT_RE],
["private-key", PRIVATE_KEY_RE],
].map(([name, regex]) => ({
name: `structure:${name}`,
regex: regex as RegExp,
}));
export function findStructureAmbiguities(text: string) {
const ambiguities: Array<{ family: string; start: number }> = [];
for (const match of text.matchAll(JWT_RE)) {
const header = decodeJsonObject(match[0].split(".")[0] ?? "");
if (
header &&
typeof header.alg === "string" &&
header.alg.length > 0 &&
findJwtSpans(match[0]).length === 0
)
ambiguities.push({ family: "JWT", start: match.index as number });
}
for (const match of text.matchAll(AUTH_RE))
if (
match[1]?.toLowerCase() === "basic" &&
findHeaderSpans(match[0]).length === 0
)
ambiguities.push({ family: "BASIC_AUTH", start: match.index as number });
for (const match of text.matchAll(URL_CANDIDATE_RE)) {
if (!candidateHasSensitiveQueryName(match[0])) continue;
if (findSensitiveQuerySpans(match[0]).length === 0)
ambiguities.push({ family: "URL_QUERY", start: match.index as number });
}
return ambiguities;
}
export function findStructureSpans(text: string): SecretSpan[] {
return [
...findPrivateKeySpans(text),
...findJwtSpans(text),
...findCredentialedUrlSpans(text),
...findSensitiveQuerySpans(text),
...findHeaderSpans(text),
...findCookieSpans(text),
...findAssignmentSpans(text),
];
}
export function findCredentialedUrlSpans(text: string): SecretSpan[] {
const spans: SecretSpan[] = [];
for (const match of text.matchAll(URL_CREDENTIAL_RE)) {
try {
new URL(match[0]);
} catch {
continue;
}
const value = match[2] as string;
if (isWeakValue(value)) continue;
spans.push(
makeSpan(
match,
2,
value,
[label("DB_PASSWORD", "structure")],
"structure",
),
);
}
return spans;
}
export function findSensitiveQuerySpans(text: string): SecretSpan[] {
const spans: SecretSpan[] = [];
for (const candidate of text.matchAll(URL_CANDIDATE_RE)) {
const rawCandidate = trimUrlCandidate(candidate[0]);
try {
new URL(rawCandidate);
} catch {
continue;
}
const question = rawCandidate.indexOf("?");
if (question < 0) continue;
const fragment = rawCandidate.indexOf("#", question);
const end = fragment < 0 ? rawCandidate.length : fragment;
spans.push(
...parsedQuerySpans(
rawCandidate.slice(question + 1, end),
(candidate.index as number) + question + 1,
),
);
}
const trimmedStart = text.search(/\S/);
if (trimmedStart >= 0 && text[trimmedStart] === "?") {
const endMatch = /[\s'"`]/.exec(text.slice(trimmedStart));
const end = endMatch
? trimmedStart + (endMatch.index as number)
: text.length;
const fragment = text.indexOf("#", trimmedStart);
const queryEnd = fragment >= 0 && fragment < end ? fragment : end;
spans.push(
...parsedQuerySpans(
text.slice(trimmedStart + 1, queryEnd),
trimmedStart + 1,
),
);
}
return spans;
}
export function findHeaderSpans(text: string): SecretSpan[] {
return [
...matches(
AUTH_RE,
text,
2,
"AUTHORIZATION",
(match) => `AUTHORIZATION_${match[1]?.toUpperCase()}`,
(match) => match[1]?.toLowerCase() !== "basic" || validBasic(match[2]),
),
...matches(X_API_RE, text, 2, "X_API_KEY"),
];
}
export function findCookieSpans(text: string): SecretSpan[] {
const spans: SecretSpan[] = [];
for (const line of text.matchAll(COOKIE_LINE_RE)) {
const body = line[1] as string;
const bodyIndices = line.indices?.[1];
if (!bodyIndices) throw new Error("Cookie regex missing body indices");
let offset = bodyIndices[0];
for (const part of body.split(";")) {
const trimmed = part.trim();
const eq = trimmed.indexOf("=");
if (eq === -1) {
offset += part.length + 1;
continue;
}
const name = trimmed.slice(0, eq);
const value = trimmed.slice(eq + 1);
const localStart = part.indexOf(trimmed) + eq + 1;
if (isSensitiveCookie(name) && !isWeakValue(value)) {
const start = offset + Math.max(localStart, 0);
spans.push({
start,
end: start + value.length,
value,
labels: [label(`COOKIE_${name}`, "structure")],
kind: "structure",
confidence: 85,
wholeSpan: false,
});
}
offset += part.length + 1;
}
}
return spans;
}
export function findAssignmentSpans(text: string): SecretSpan[] {
const spans: SecretSpan[] = [];
for (const regex of [ASSIGN_RE, XML_RE]) {
regex.lastIndex = 0;
for (const match of text.matchAll(regex)) {
const key = match[1] as string;
const value = match[2] as string;
if (ASSIGNMENT_STOP_KEYS.has(key.toLowerCase())) continue;
if (!isSensitiveKey(key)) continue;
const hcloudToken = isHCloudTokenAssignment(key, value);
if (isCodeReference(value) && !hcloudToken) continue;
if (!highConfidenceSecretShape(value) && !hcloudToken) continue;
spans.push(
makeSpan(match, 2, value, [label(key, "structure")], "structure"),
);
}
}
return spans;
}
export function findJwtSpans(text: string): SecretSpan[] {
const spans: SecretSpan[] = [];
for (const match of text.matchAll(JWT_RE)) {
const parts = match[0].split(".");
if (parts.length !== 3) continue;
const [headerPart, payloadPart, signaturePart] = parts;
if (!headerPart || !payloadPart || !strictBase64Url(signaturePart))
continue;
const header = decodeJsonObject(headerPart);
const payload = decodeJsonObject(payloadPart);
if (!header || !payload || typeof header.alg !== "string" || !header.alg)
continue;
const start = match.index as number;
spans.push({
start,
end: start + match[0].length,
value: match[0],
labels: [label("JWT", "builtin")],
kind: "jwt",
confidence: 100,
wholeSpan: true,
});
}
return spans;
}
function strictBase64Url(value: string | undefined) {
if (!value || !/^[A-Za-z0-9_-]+$/.test(value)) return false;
return Buffer.from(value, "base64url").toString("base64url") === value;
}
function decodeJsonObject(value: string) {
if (!strictBase64Url(value)) return undefined;
try {
const bytes = Buffer.from(value, "base64url");
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
const parsed: unknown = JSON.parse(text);
if (!isPlainRecord(parsed)) return undefined;
return parsed;
} catch {
return undefined;
}
}
function isPlainRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
export function findPrivateKeySpans(text: string): SecretSpan[] {
return [...text.matchAll(PRIVATE_KEY_RE)].map((match) => {
const start = match.index as number;
const keyType = match[1] as string;
return {
start,
end: start + match[0].length,
value: match[0],
labels: [label(keyType.replace(/ /g, "_"), "builtin")],
kind: "private-key",
confidence: 100,
wholeSpan: true,
};
});
}
function isHCloudTokenAssignment(key: string, value: string) {
const normalized = key.replaceAll(/[-_]/g, "").toLowerCase();
return (
(normalized === "hcloudtoken" || normalized === "tfvarhcloudtoken") &&
/^[A-Za-z0-9]{64}$/.test(value)
);
}
function isCodeReference(value: string) {
const text = value.trim();
return (
text.startsWith("$") ||
/[()[\]{}]/.test(text) ||
/^[A-Z][A-Za-z0-9_$]*$/.test(text) ||
/^(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+[A-Za-z_$][A-Za-z0-9_$]*$/.test(text)
);
}
function matches(
regex: RegExp,
text: string,
group: number,
fallback: string,
tag?: (match: RegExpMatchArray) => string,
accept: (match: RegExpMatchArray) => boolean = () => true,
): SecretSpan[] {
regex.lastIndex = 0;
const spans: SecretSpan[] = [];
for (const match of text.matchAll(regex)) {
const value = match[group] as string;
if (!accept(match) || isWeakValue(value)) continue;
spans.push(
makeSpan(
match,
group,
value,
[label(tag ? tag(match) : fallback, "structure")],
"structure",
),
);
}
return spans;
}
function makeSpan(
match: RegExpMatchArray,
group: number,
value: string,
labels: SpanLabel[],
kind: SecretSpan["kind"],
): SecretSpan {
const range = match.indices?.[group] as [number, number];
const start = range[0];
return {
start,
end: range[1],
value,
labels,
kind,
confidence: 85,
wholeSpan: false,
};
}
function label(tag: string, source: SpanLabel["source"]): SpanLabel {
return { tag, priority: source === "builtin" ? 40 : 20, source };
}
function trimUrlCandidate(candidate: string) {
return candidate.replace(/[\])},.;!?]+$/, "");
}
function candidateHasSensitiveQueryName(candidate: string) {
const question = candidate.indexOf("?");
if (question < 0) return false;
return [...new URLSearchParams(candidate.slice(question + 1))].some(
([name]) => isSensitiveKey(name),
);
}
function parsedQuerySpans(query: string, offset: number): SecretSpan[] {
const parsed = [...new URLSearchParams(query)];
const rawPairs: Array<{ pair: string; start: number }> = [];
let cursor = 0;
for (const pair of query.split("&")) {
if (pair.length > 0) rawPairs.push({ pair, start: cursor });
cursor += pair.length + 1;
}
if (parsed.length !== rawPairs.length) return [];
const spans: SecretSpan[] = [];
for (let index = 0; index < parsed.length; index++) {
const [name, decodedValue] = parsed[index];
const { pair, start } = rawPairs[index];
const equals = pair.indexOf("=");
if (equals <= 0) return [];
const pairEntries = [...new URLSearchParams(pair)];
if (
pairEntries.length !== 1 ||
pairEntries[0][0] !== name ||
pairEntries[0][1] !== decodedValue
)
return [];
const rawValue = pair.slice(equals + 1);
if (!isSensitiveKey(name) || isWeakValue(rawValue)) continue;
const valueStart = offset + start + equals + 1;
spans.push({
start: valueStart,
end: valueStart + rawValue.length,
value: rawValue,
labels: [label(name, "structure")],
kind: "structure",
confidence: 85,
wholeSpan: false,
});
}
return spans;
}
function validBasic(value: string | undefined) {
if (!value || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) return false;
try {
const bytes = Buffer.from(value, "base64");
if (bytes.toString("base64") !== value) return false;
return new TextDecoder("utf-8", { fatal: true })
.decode(bytes)
.includes(":");
} catch {
return false;
}
}
function isSensitiveCookie(name: string) {
return isSensitiveKey(name);
}