repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/web/map.ts
Rawimport { DEFAULT_TIMEOUT_MS } from "./constants.js";
import { fetchUrlStream } from "./fetch.js";
export interface MapUrlEntry {
url: string;
source: "sitemap" | "llms";
sourceUrl: string;
}
export interface SiteMapResult {
seedUrl: string;
urls: MapUrlEntry[];
sitemaps: string[];
tree: Record<string, string[]>;
partial?: boolean;
truncated?: boolean;
failures?: string[];
}
export interface DiscoverSiteMapOptions {
maxUrls?: number;
maxSitemaps?: number;
timeoutMs?: number;
fetchText?: (
url: string,
signal?: AbortSignal,
) => Promise<string | undefined>;
}
export async function discoverSiteMap(
seed: string,
options: DiscoverSiteMapOptions = {},
signal?: AbortSignal,
): Promise<SiteMapResult> {
const deadline = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
const requestSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
requestSignal.throwIfAborted();
const seedUrl = new URL(seed).toString();
const fetchText = options.fetchText ?? defaultFetchText;
const maxUrls = Math.floor(options.maxUrls ?? 1_000);
const maxSitemaps = Math.floor(options.maxSitemaps ?? 20);
const urls = new Map<string, MapUrlEntry>();
const sitemapSet = new Set<string>();
const sitemapQueue: string[] = [];
const failures: string[] = [];
let truncated = false;
let metadataSucceeded = false;
const readMetadata = async (url: string): Promise<string | undefined> => {
requestSignal.throwIfAborted();
try {
const text = await fetchText(url, requestSignal);
requestSignal.throwIfAborted();
metadataSucceeded ||= text !== undefined;
return text;
} catch (_error) {
requestSignal.throwIfAborted();
failures.push(url);
return undefined;
}
};
const enqueueSitemap = (url: string): void => {
if (sitemapSet.has(url)) return;
if (sitemapQueue.length >= maxSitemaps) {
truncated = true;
return;
}
sitemapSet.add(url);
sitemapQueue.push(url);
};
const addUrl = (entry: MapUrlEntry): void => {
if (urls.has(entry.url)) return;
if (urls.size >= maxUrls) {
truncated = true;
return;
}
urls.set(entry.url, entry);
};
const robotsUrl = siteUrl(seedUrl, "/robots.txt");
const robots = await readMetadata(robotsUrl);
for (const sitemap of parseRobotsSitemaps(robots ?? ""))
enqueueSitemap(sitemap);
enqueueSitemap(siteUrl(seedUrl, "/sitemap.xml"));
for (const sitemapUrl of sitemapQueue) {
const xml = await readMetadata(sitemapUrl);
if (!xml) continue;
for (const nested of parseSitemapIndexes(xml)) enqueueSitemap(nested);
for (const url of parseSitemapXml(xml))
addUrl({
url,
source: "sitemap",
sourceUrl: sitemapUrl,
});
}
const llmsUrl = siteUrl(seedUrl, "/llms.txt");
const llms = await readMetadata(llmsUrl);
for (const url of parseLlmsLinks(llms ?? "", llmsUrl))
addUrl({ url, source: "llms", sourceUrl: llmsUrl });
requestSignal.throwIfAborted();
if (!metadataSucceeded && failures.length > 0)
throw new Error("Site metadata discovery failed");
const entries = [...urls.values()].toSorted((a, b) =>
a.url.localeCompare(b.url),
);
return {
seedUrl,
urls: entries,
sitemaps: [...sitemapSet],
tree: buildTree(entries),
...(failures.length || truncated
? { partial: true, truncated, ...(failures.length ? { failures } : {}) }
: {}),
};
}
export function parseRobotsSitemaps(text: string): string[] {
return text
.split(/\r?\n/u)
.map((line) => line.match(/^\s*sitemap:\s*(\S+)\s*$/iu)?.[1])
.filter((url): url is string => Boolean(url && isHttpUrl(url)));
}
export function parseSitemapXml(xml: string): string[] {
return locs(xml).filter(
(url) =>
isHttpUrl(url) && !url.endsWith(".xml") && !url.endsWith(".xml.gz"),
);
}
function parseSitemapIndexes(xml: string): string[] {
return locs(xml).filter(
(url) =>
isHttpUrl(url) && (url.endsWith(".xml") || url.endsWith(".xml.gz")),
);
}
export function parseLlmsLinks(text: string, baseUrl: string): string[] {
const out: string[] = [];
for (const match of text.matchAll(/\[[^\]]*\]\(([^)]+)\)/gu)) {
const raw = match[1]?.trim();
if (!raw) continue;
try {
const url = new URL(raw, baseUrl);
if (isHttpUrl(url)) out.push(url.toString());
} catch {}
}
return out;
}
async function defaultFetchText(
url: string,
signal?: AbortSignal,
): Promise<string | undefined> {
try {
const result = await fetchUrlStream(url, undefined, signal, {
maxBytes: 2 * 1024 * 1024,
});
return result.body.toString("utf8");
} catch (error) {
if (error instanceof Error && /^HTTP 404:/u.test(error.message))
return undefined;
throw error;
}
}
function isHttpUrl(input: string | URL): boolean {
try {
const { protocol } = new URL(input);
return protocol === "http:" || protocol === "https:";
} catch {
return false;
}
}
function locs(xml: string): string[] {
return [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/giu)].map((x) =>
decodeXml(x[1] ?? ""),
);
}
function decodeXml(text: string): string {
return text
.replaceAll("&", "&")
.replaceAll("<", "<")
.replaceAll(">", ">")
.replaceAll(""", '"')
.replaceAll("'", "'");
}
function siteUrl(seed: string, pathname: string): string {
const url = new URL(seed);
url.pathname = pathname;
url.search = "";
url.hash = "";
return url.toString();
}
function buildTree(entries: MapUrlEntry[]): Record<string, string[]> {
const tree = Object.create(null) as Record<string, string[]>;
for (const entry of entries) {
const section = new URL(entry.url).pathname.split("/").find(Boolean) ?? "/";
const sectionUrls = tree[section] ?? [];
sectionUrls.push(entry.url);
tree[section] = sectionUrls;
}
return tree;
}