import { DEFAULT_TIMEOUT_MS } from "./constants.js"; import { fetchUrlStream } from "./fetch.js"; export interface MapUrlEntry { url: string; source: "sitemap" | "llms"; sourceUrl: string; } export interface SiteMapResult { seedUrl: string; urls: MapUrlEntry[]; sitemaps: string[]; tree: Record; partial?: boolean; truncated?: boolean; failures?: string[]; } export interface DiscoverSiteMapOptions { maxUrls?: number; maxSitemaps?: number; timeoutMs?: number; fetchText?: ( url: string, signal?: AbortSignal, ) => Promise; } export async function discoverSiteMap( seed: string, options: DiscoverSiteMapOptions = {}, signal?: AbortSignal, ): Promise { const deadline = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS); const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline; requestSignal.throwIfAborted(); const seedUrl = new URL(seed).toString(); const fetchText = options.fetchText ?? defaultFetchText; const maxUrls = Math.floor(options.maxUrls ?? 1_000); const maxSitemaps = Math.floor(options.maxSitemaps ?? 20); const urls = new Map(); const sitemapSet = new Set(); const sitemapQueue: string[] = []; const failures: string[] = []; let truncated = false; let metadataSucceeded = false; const readMetadata = async (url: string): Promise => { requestSignal.throwIfAborted(); try { const text = await fetchText(url, requestSignal); requestSignal.throwIfAborted(); metadataSucceeded ||= text !== undefined; return text; } catch (_error) { requestSignal.throwIfAborted(); failures.push(url); return undefined; } }; const enqueueSitemap = (url: string): void => { if (sitemapSet.has(url)) return; if (sitemapQueue.length >= maxSitemaps) { truncated = true; return; } sitemapSet.add(url); sitemapQueue.push(url); }; const addUrl = (entry: MapUrlEntry): void => { if (urls.has(entry.url)) return; if (urls.size >= maxUrls) { truncated = true; return; } urls.set(entry.url, entry); }; const robotsUrl = siteUrl(seedUrl, "/robots.txt"); const robots = await readMetadata(robotsUrl); for (const sitemap of parseRobotsSitemaps(robots ?? "")) enqueueSitemap(sitemap); enqueueSitemap(siteUrl(seedUrl, "/sitemap.xml")); for (const sitemapUrl of sitemapQueue) { const xml = await readMetadata(sitemapUrl); if (!xml) continue; for (const nested of parseSitemapIndexes(xml)) enqueueSitemap(nested); for (const url of parseSitemapXml(xml)) addUrl({ url, source: "sitemap", sourceUrl: sitemapUrl, }); } const llmsUrl = siteUrl(seedUrl, "/llms.txt"); const llms = await readMetadata(llmsUrl); for (const url of parseLlmsLinks(llms ?? "", llmsUrl)) addUrl({ url, source: "llms", sourceUrl: llmsUrl }); requestSignal.throwIfAborted(); if (!metadataSucceeded && failures.length > 0) throw new Error("Site metadata discovery failed"); const entries = [...urls.values()].toSorted((a, b) => a.url.localeCompare(b.url), ); return { seedUrl, urls: entries, sitemaps: [...sitemapSet], tree: buildTree(entries), ...(failures.length || truncated ? { partial: true, truncated, ...(failures.length ? { failures } : {}) } : {}), }; } export function parseRobotsSitemaps(text: string): string[] { return text .split(/\r?\n/u) .map((line) => line.match(/^\s*sitemap:\s*(\S+)\s*$/iu)?.[1]) .filter((url): url is string => Boolean(url && isHttpUrl(url))); } export function parseSitemapXml(xml: string): string[] { return locs(xml).filter( (url) => isHttpUrl(url) && !url.endsWith(".xml") && !url.endsWith(".xml.gz"), ); } function parseSitemapIndexes(xml: string): string[] { return locs(xml).filter( (url) => isHttpUrl(url) && (url.endsWith(".xml") || url.endsWith(".xml.gz")), ); } export function parseLlmsLinks(text: string, baseUrl: string): string[] { const out: string[] = []; for (const match of text.matchAll(/\[[^\]]*\]\(([^)]+)\)/gu)) { const raw = match[1]?.trim(); if (!raw) continue; try { const url = new URL(raw, baseUrl); if (isHttpUrl(url)) out.push(url.toString()); } catch {} } return out; } async function defaultFetchText( url: string, signal?: AbortSignal, ): Promise { try { const result = await fetchUrlStream(url, undefined, signal, { maxBytes: 2 * 1024 * 1024, }); return result.body.toString("utf8"); } catch (error) { if (error instanceof Error && /^HTTP 404:/u.test(error.message)) return undefined; throw error; } } function isHttpUrl(input: string | URL): boolean { try { const { protocol } = new URL(input); return protocol === "http:" || protocol === "https:"; } catch { return false; } } function locs(xml: string): string[] { return [...xml.matchAll(/\s*([^<]+?)\s*<\/loc>/giu)].map((x) => decodeXml(x[1] ?? ""), ); } function decodeXml(text: string): string { return text .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll(""", '"') .replaceAll("'", "'"); } function siteUrl(seed: string, pathname: string): string { const url = new URL(seed); url.pathname = pathname; url.search = ""; url.hash = ""; return url.toString(); } function buildTree(entries: MapUrlEntry[]): Record { const tree = Object.create(null) as Record; for (const entry of entries) { const section = new URL(entry.url).pathname.split("/").find(Boolean) ?? "/"; const sectionUrls = tree[section] ?? []; sectionUrls.push(entry.url); tree[section] = sectionUrls; } return tree; }