import { setTimeout as delay } from "node:timers/promises"; import type { SearchResult } from "./constants.js"; import { extractBalanced } from "./extract.js"; import { fetchSearchHtml } from "./fetch.js"; const NOKIA_USER_AGENT = "Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0"; let nextRequestAt = 0; function decodeEntities(text: string): string { const named: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", }; return text.replace( /&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos|nbsp);/gi, (entity, code: string) => { if (!code.startsWith("#")) return named[code.toLowerCase()] ?? entity; const number = code[1].toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10); return number > 0 && number <= 0x10ffff && !(number >= 0xd800 && number <= 0xdfff) ? String.fromCodePoint(number) : "�"; }, ); } function plainText(html: string): string { return decodeEntities(html.replace(/<[^>]*>/g, " ")) .replace(/\s+/g, " ") .trim(); } function classPattern(tag: string, name: string): RegExp { return new RegExp( `<${tag}\\b[^>]*\\bclass=["'][^"']*\\b${name}\\b[^"']*["'][^>]*>`, "g", ); } function classContent(html: string, tag: string, name: string): string { const match = classPattern(tag, name).exec(html); return match ? (extractBalanced(html, match.index + match[0].length, tag) ?? "") : ""; } export function parseGoogleResults(html: string): SearchResult[] { if ( /unusual traffic|our systems have detected|g-recaptcha|\/sorry\/index/i.test( html, ) ) throw new Error("Google WML returned a CAPTCHA or traffic block"); if (/before you continue to google|enable javascript/i.test(html)) throw new Error("Google WML returned a consent or JavaScript page"); const results: SearchResult[] = []; const seen = new Set(); for (const match of html.matchAll(classPattern("div", "zMzFAb"))) { const block = extractBalanced(html, match.index + match[0].length, "div"); if (!block) continue; const anchor = classPattern("a", "fuLhoc").exec(block)?.[0]; const href = anchor?.match(/\bhref=["']([^"']+)["']/)?.[1]; const title = plainText(classContent(block, "span", "CVA68e")); if (!href || !title) continue; try { let url = new URL(decodeEntities(href), "https://www.google.com"); if (url.hostname === "www.google.com" && url.pathname === "/url") url = new URL( url.searchParams.get("q") ?? url.searchParams.get("url") ?? "", ); if ( !/^https?:$/.test(url.protocol) || url.username || url.password || /(^|\.)google\.com$/.test(url.hostname) || seen.has(url.href) ) continue; seen.add(url.href); results.push({ title, url: url.href, snippet: plainText(classContent(block, "span", "FrIlee")), }); } catch { /* Ignore malformed result links, never navigate them. */ } } if ( !results.length && !/did not match any documents|no results found/i.test(html) ) throw new Error( "Google WML returned no parseable results (layout may have changed)", ); return results; } export async function googleSearch( query: string, signal?: AbortSignal, count = 5, timeout = 30_000, ): Promise { const deadline = AbortSignal.timeout(timeout); const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; requestSignal.throwIfAborted(); const now = Date.now(); const wait = Math.max(0, nextRequestAt - now); nextRequestAt = Math.max(now, nextRequestAt) + 2000; if (wait) await delay(wait, undefined, { signal: requestSignal }); const url = new URL("https://www.google.com/wml/search"); url.search = new URLSearchParams({ q: query, hl: "en", ie: "utf8", oe: "utf8", sca_esv: "1", lr: "", cr: "", }).toString(); const html = await fetchSearchHtml(url.href, requestSignal, timeout, { headers: { "User-Agent": NOKIA_USER_AGENT, Accept: "*/*", Cookie: "CONSENT=YES+", }, }); return parseGoogleResults(html).slice(0, count); }