import { describe, expect, it } from "vitest"; import { discoverSiteMap } from "../map"; describe("site map limits and cancellation", () => { it.each([1, 1.5])( "limits URLs across sitemaps and llms metadata (%s)", async (maxUrls) => { const result = await discoverSiteMap("https://example.com", { maxUrls, fetchText: async (url) => { if (url.endsWith("robots.txt")) return "Sitemap: https://example.com/one.xml\nSitemap: https://example.com/two.xml"; if (url.endsWith("one.xml")) return "https://example.com/one"; if (url.endsWith("two.xml")) return "https://example.com/two"; if (url.endsWith("llms.txt")) return "[three](https://example.com/three)"; return undefined; }, }); expect(result.urls).toHaveLength(1); expect(result.partial).toBe(true); expect(result.truncated).toBe(true); }, ); it.each([1, 1.5])( "limits actual sitemap fetches, including nested candidates (%s)", async (maxSitemaps) => { const visited: string[] = []; const result = await discoverSiteMap("https://example.com", { maxSitemaps, fetchText: async (url) => { visited.push(url); if (url.endsWith("robots.txt")) return Array.from( { length: 10 }, (_, i) => `Sitemap: https://example.com/${i}.xml`, ).join("\n"); return "https://example.com/nested.xml"; }, }); expect( visited.filter((url) => url.endsWith(".xml")).length, ).toBeLessThanOrEqual(maxSitemaps); expect(result.sitemaps.length).toBeLessThanOrEqual(maxSitemaps); expect(result.truncated).toBe(true); expect(result.partial).toBe(true); }, ); it.each(["constructor", "__proto__", "toString"])( "builds a safe tree for /%s URLs", async (section) => { const result = await discoverSiteMap("https://example.com", { fetchText: async (url) => url.endsWith("robots.txt") ? "Sitemap: https://example.com/site.xml" : url.endsWith("site.xml") ? `https://example.com/${section}/page` : undefined, }); expect(result.tree[section]).toEqual([ `https://example.com/${section}/page`, ]); }, ); it("does not return a successful metadata read after cancellation", async () => { const controller = new AbortController(); await expect( discoverSiteMap( "https://example.com", { fetchText: async () => { controller.abort(new Error("late abort")); return ""; }, }, controller.signal, ), ).rejects.toThrow("late abort"); }); it("propagates cancellation instead of returning empty metadata", async () => { const controller = new AbortController(); await expect( discoverSiteMap( "https://example.com", { fetchText: async (_url, signal) => { controller.abort(new Error("stopped")); signal?.throwIfAborted(); return undefined; }, }, controller.signal, ), ).rejects.toThrow("stopped"); }); });