Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/web/map.ts

Raw
import { DEFAULT_TIMEOUT_MS } from "./constants.js";
import { fetchUrlStream } from "./fetch.js";

export interface MapUrlEntry {
	url: string;
	source: "sitemap" | "llms";
	sourceUrl: string;
}

export interface SiteMapResult {
	seedUrl: string;
	urls: MapUrlEntry[];
	sitemaps: string[];
	tree: Record<string, string[]>;
	partial?: boolean;
	truncated?: boolean;
	failures?: string[];
}

export interface DiscoverSiteMapOptions {
	maxUrls?: number;
	maxSitemaps?: number;
	timeoutMs?: number;
	fetchText?: (
		url: string,
		signal?: AbortSignal,
	) => Promise<string | undefined>;
}

export async function discoverSiteMap(
	seed: string,
	options: DiscoverSiteMapOptions = {},
	signal?: AbortSignal,
): Promise<SiteMapResult> {
	const deadline = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
	const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
	requestSignal.throwIfAborted();
	const seedUrl = new URL(seed).toString();
	const fetchText = options.fetchText ?? defaultFetchText;
	const maxUrls = Math.floor(options.maxUrls ?? 1_000);
	const maxSitemaps = Math.floor(options.maxSitemaps ?? 20);
	const urls = new Map<string, MapUrlEntry>();
	const sitemapSet = new Set<string>();
	const sitemapQueue: string[] = [];
	const failures: string[] = [];
	let truncated = false;
	let metadataSucceeded = false;

	const readMetadata = async (url: string): Promise<string | undefined> => {
		requestSignal.throwIfAborted();
		try {
			const text = await fetchText(url, requestSignal);
			requestSignal.throwIfAborted();
			metadataSucceeded ||= text !== undefined;
			return text;
		} catch (_error) {
			requestSignal.throwIfAborted();
			failures.push(url);
			return undefined;
		}
	};
	const enqueueSitemap = (url: string): void => {
		if (sitemapSet.has(url)) return;
		if (sitemapQueue.length >= maxSitemaps) {
			truncated = true;
			return;
		}
		sitemapSet.add(url);
		sitemapQueue.push(url);
	};
	const addUrl = (entry: MapUrlEntry): void => {
		if (urls.has(entry.url)) return;
		if (urls.size >= maxUrls) {
			truncated = true;
			return;
		}
		urls.set(entry.url, entry);
	};

	const robotsUrl = siteUrl(seedUrl, "/robots.txt");
	const robots = await readMetadata(robotsUrl);
	for (const sitemap of parseRobotsSitemaps(robots ?? ""))
		enqueueSitemap(sitemap);
	enqueueSitemap(siteUrl(seedUrl, "/sitemap.xml"));

	for (const sitemapUrl of sitemapQueue) {
		const xml = await readMetadata(sitemapUrl);
		if (!xml) continue;
		for (const nested of parseSitemapIndexes(xml)) enqueueSitemap(nested);
		for (const url of parseSitemapXml(xml))
			addUrl({
				url,
				source: "sitemap",
				sourceUrl: sitemapUrl,
			});
	}

	const llmsUrl = siteUrl(seedUrl, "/llms.txt");
	const llms = await readMetadata(llmsUrl);
	for (const url of parseLlmsLinks(llms ?? "", llmsUrl))
		addUrl({ url, source: "llms", sourceUrl: llmsUrl });

	requestSignal.throwIfAborted();
	if (!metadataSucceeded && failures.length > 0)
		throw new Error("Site metadata discovery failed");
	const entries = [...urls.values()].toSorted((a, b) =>
		a.url.localeCompare(b.url),
	);
	return {
		seedUrl,
		urls: entries,
		sitemaps: [...sitemapSet],
		tree: buildTree(entries),
		...(failures.length || truncated
			? { partial: true, truncated, ...(failures.length ? { failures } : {}) }
			: {}),
	};
}

export function parseRobotsSitemaps(text: string): string[] {
	return text
		.split(/\r?\n/u)
		.map((line) => line.match(/^\s*sitemap:\s*(\S+)\s*$/iu)?.[1])
		.filter((url): url is string => Boolean(url && isHttpUrl(url)));
}

export function parseSitemapXml(xml: string): string[] {
	return locs(xml).filter(
		(url) =>
			isHttpUrl(url) && !url.endsWith(".xml") && !url.endsWith(".xml.gz"),
	);
}

function parseSitemapIndexes(xml: string): string[] {
	return locs(xml).filter(
		(url) =>
			isHttpUrl(url) && (url.endsWith(".xml") || url.endsWith(".xml.gz")),
	);
}

export function parseLlmsLinks(text: string, baseUrl: string): string[] {
	const out: string[] = [];
	for (const match of text.matchAll(/\[[^\]]*\]\(([^)]+)\)/gu)) {
		const raw = match[1]?.trim();
		if (!raw) continue;
		try {
			const url = new URL(raw, baseUrl);
			if (isHttpUrl(url)) out.push(url.toString());
		} catch {}
	}
	return out;
}

async function defaultFetchText(
	url: string,
	signal?: AbortSignal,
): Promise<string | undefined> {
	try {
		const result = await fetchUrlStream(url, undefined, signal, {
			maxBytes: 2 * 1024 * 1024,
		});
		return result.body.toString("utf8");
	} catch (error) {
		if (error instanceof Error && /^HTTP 404:/u.test(error.message))
			return undefined;
		throw error;
	}
}

function isHttpUrl(input: string | URL): boolean {
	try {
		const { protocol } = new URL(input);
		return protocol === "http:" || protocol === "https:";
	} catch {
		return false;
	}
}

function locs(xml: string): string[] {
	return [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/giu)].map((x) =>
		decodeXml(x[1] ?? ""),
	);
}

function decodeXml(text: string): string {
	return text
		.replaceAll("&amp;", "&")
		.replaceAll("&lt;", "<")
		.replaceAll("&gt;", ">")
		.replaceAll("&quot;", '"')
		.replaceAll("&apos;", "'");
}

function siteUrl(seed: string, pathname: string): string {
	const url = new URL(seed);
	url.pathname = pathname;
	url.search = "";
	url.hash = "";
	return url.toString();
}

function buildTree(entries: MapUrlEntry[]): Record<string, string[]> {
	const tree = Object.create(null) as Record<string, string[]>;
	for (const entry of entries) {
		const section = new URL(entry.url).pathname.split("/").find(Boolean) ?? "/";
		const sectionUrls = tree[section] ?? [];
		sectionUrls.push(entry.url);
		tree[section] = sectionUrls;
	}
	return tree;
}