import { describe, expect, it } from "vitest";
import { discoverSiteMap } from "../map";
describe("site map limits and cancellation", () => {
it.each([1, 1.5])(
"limits URLs across sitemaps and llms metadata (%s)",
async (maxUrls) => {
const result = await discoverSiteMap("https://example.com", {
maxUrls,
fetchText: async (url) => {
if (url.endsWith("robots.txt"))
return "Sitemap: https://example.com/one.xml\nSitemap: https://example.com/two.xml";
if (url.endsWith("one.xml"))
return "https://example.com/one";
if (url.endsWith("two.xml"))
return "https://example.com/two";
if (url.endsWith("llms.txt"))
return "[three](https://example.com/three)";
return undefined;
},
});
expect(result.urls).toHaveLength(1);
expect(result.partial).toBe(true);
expect(result.truncated).toBe(true);
},
);
it.each([1, 1.5])(
"limits actual sitemap fetches, including nested candidates (%s)",
async (maxSitemaps) => {
const visited: string[] = [];
const result = await discoverSiteMap("https://example.com", {
maxSitemaps,
fetchText: async (url) => {
visited.push(url);
if (url.endsWith("robots.txt"))
return Array.from(
{ length: 10 },
(_, i) => `Sitemap: https://example.com/${i}.xml`,
).join("\n");
return "https://example.com/nested.xml";
},
});
expect(
visited.filter((url) => url.endsWith(".xml")).length,
).toBeLessThanOrEqual(maxSitemaps);
expect(result.sitemaps.length).toBeLessThanOrEqual(maxSitemaps);
expect(result.truncated).toBe(true);
expect(result.partial).toBe(true);
},
);
it.each(["constructor", "__proto__", "toString"])(
"builds a safe tree for /%s URLs",
async (section) => {
const result = await discoverSiteMap("https://example.com", {
fetchText: async (url) =>
url.endsWith("robots.txt")
? "Sitemap: https://example.com/site.xml"
: url.endsWith("site.xml")
? `https://example.com/${section}/page`
: undefined,
});
expect(result.tree[section]).toEqual([
`https://example.com/${section}/page`,
]);
},
);
it("does not return a successful metadata read after cancellation", async () => {
const controller = new AbortController();
await expect(
discoverSiteMap(
"https://example.com",
{
fetchText: async () => {
controller.abort(new Error("late abort"));
return "";
},
},
controller.signal,
),
).rejects.toThrow("late abort");
});
it("propagates cancellation instead of returning empty metadata", async () => {
const controller = new AbortController();
await expect(
discoverSiteMap(
"https://example.com",
{
fetchText: async (_url, signal) => {
controller.abort(new Error("stopped"));
signal?.throwIfAborted();
return undefined;
},
},
controller.signal,
),
).rejects.toThrow("stopped");
});
});