Luigit
repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

extensions/devdocs/__serial__/upstream.test.ts

Raw
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, it, vi } from "vitest";
import {
	type UpstreamFetch,
	type UpstreamOptions,
	upstreamAdapter,
} from "../src/adapters/upstream.ts";
import {
	type CatalogSource,
	formatReaders,
	localSourceCatalog,
	sourceCapabilities,
	sourceCatalog,
} from "../src/catalog.ts";
import { Devdocs } from "../src/engine.ts";
import { htmlMarkdown, readFormat } from "../src/formats.ts";
import type { Adapter, Request } from "../src/types.ts";

const roots: string[] = [];
const engines: Devdocs[] = [];
afterEach(async () => {
	await Promise.all(engines.splice(0).map((engine) => engine.close()));
	await Promise.all(
		roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
	);
});
const request = (target: string, query: string): Request => ({
	target,
	query,
	cwd: tmpdir(),
});
const absent = (id: string, rank: number): Adapter => ({
	id,
	rank,
	formatVersion: 1,
	coverage: "corpus",
	discover: async () => [],
	retrieve: async () => {
		throw new Error("unexpected retrieval from absent source");
	},
});
async function engine(options: UpstreamOptions, lowerRanks = false) {
	const root = await mkdtemp(join(tmpdir(), "devdocs-upstream-"));
	roots.push(root);
	const instance = new Devdocs(
		{
			databasePath: join(root, "cache.db"),
			enabledSources: ["local", "devdocs", "upstream"],
			sources: [],
			offline: false,
			cacheBytes: 4 * 1024 * 1024,
			ttlMs: 60000,
		},
		[
			...(lowerRanks ? [absent("local", 1), absent("devdocs", 2)] : []),
			upstreamAdapter(options),
		],
	);
	engines.push(instance);
	return instance;
}
function fixtureFetch(pages: Record<string, string>, type = "text/html") {
	return vi.fn<UpstreamFetch>(async (url) =>
		pages[url] === undefined
			? new Response("missing", { status: 404 })
			: new Response(pages[url], {
					headers: { "content-type": type, etag: '"fixture-v1"' },
				}),
	);
}
function source(
	id: string,
	overrides: Partial<CatalogSource> = {},
): CatalogSource {
	const base = sourceCatalog.find((entry) => entry.id === id);
	if (!base) throw new Error("missing fixture source");
	return { ...base, ...overrides };
}

it.each([
	{
		target: "mdn",
		url: "https://developer.mozilla.org/en-US/docs/Web/",
		query: "Array.prototype.map",
		locator: "map",
		html: '<nav>UNWANTED_NAV</nav><main><h1>Array</h1><h2 id="map">Array.prototype.map</h2><p>Creates a new array &amp; preserves values.</p><pre><code>const values = [1, 2];\n  values.map(x =&gt; x * 2);</code></pre><script>UNWANTED_SCRIPT</script><style>UNWANTED_STYLE</style></main>',
		expected: "  values.map(x => x * 2);",
		version: undefined,
	},
	{
		target: "java",
		url: "https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/List.html",
		query: "get(int)",
		locator: "get(int)",
		html: '<h1>java.util.List&lt;E&gt;</h1><section class="detail" id="get(int)"><h3>get(int)</h3><pre>public E get(int index)</pre><p>Returns the element at the specified position.</p></section>',
		expected: "public E get(int index)",
		version: "21",
	},
	{
		target: "man7",
		url: "https://man7.org/linux/man-pages/",
		query: "open(2)",
		locator: "DESCRIPTION",
		html: '<h1>open(2) Linux manual page</h1><h2><a id="DESCRIPTION"></a>DESCRIPTION</h2><pre>open(2) opens a file descriptor.\n    #include &lt;fcntl.h&gt;\n    int open(const char *path, int flags);</pre>',
		expected: "    #include <fcntl.h>",
		version: undefined,
	},
	{
		target: "win32",
		url: "https://learn.microsoft.com/en-us/windows/win32/",
		query: "CreateFileW",
		locator: "requirements",
		html: '<nav>UNWANTED_NAV</nav><h1>CreateFileW</h1><h2 id="requirements">CreateFileW requirements</h2><p>Minimum supported client: Windows XP.</p><table><tr><th>Header</th><td>fileapi.h</td></tr><tr><th>Library</th><td>Kernel32.lib</td></tr></table><pre>HANDLE CreateFileW(\n    LPCWSTR lpFileName\n);</pre>',
		expected: "Kernel32.lib",
		version: undefined,
	},
])(
	"search/read preserves $target domain identity, anchors and code",
	async ({ target, url, query, locator, html, expected, version }) => {
		const fetch = fixtureFetch({ [url]: html });
		const adapterOptions = {
			fetch,
			sources: [
				source(target, {
					home: url,
					...(version
						? {
								version,
								evidence: "Fixture publisher release metadata: JDK 21",
							}
						: {}),
				}),
			],
		};
		const docs = await engine(adapterOptions);
		const found = await docs.search({
			...request(target, query),
			...(version ? { version } : {}),
		});
		expect(found.status).toBe("ok");
		if (found.status !== "ok") throw new Error(JSON.stringify(found));
		expect(found.results[0]).toMatchObject({
			target,
			source: "upstream",
			origin: url,
			version: version ?? null,
		});
		const read = await docs.read(
			found.results.map((hit) => hit.ref),
			4000,
		);
		expect(
			read.sections.map((section) => section.content).join("\n"),
		).toContain(expected);
		expect(read.sections[0]?.attribution).toContain(target);
		expect(JSON.stringify(read)).not.toContain("UNWANTED_");
		const adapter = upstreamAdapter(adapterOptions);
		const candidates = await adapter.discover(request(target, query));
		const document = await adapter.retrieve(
			candidates[0]!,
			request(target, query),
		);
		if (document === "unchanged") throw new Error("unexpected 304");
		expect(
			document.sections.some((section) => section.locator === locator),
		).toBe(true);
	},
);

it("falls through absent local/DevDocs to query-relevant scoped llms pages without whole-site ingestion", async () => {
	const index = "https://mise.jdx.dev/llms.txt";
	const trust = "https://mise.jdx.dev/configuration/trust.md";
	const backends = "https://mise.jdx.dev/backends.md";
	const fetch = fixtureFetch(
		{
			[index]:
				"# Mise\n[Broken](./%E0%A4)\n[Backends](./backends.md)\n[Trust config](./configuration/trust.md)\n[Secret](./trust?token=private)\n[Other](https://other.example/trust.md)",
			[trust]:
				"# Trust\n\nTrusted configuration requires explicit trust approval.",
			[backends]: "# Backends\n\nBackend plugin reference.",
		},
		"text/markdown",
	);
	const docs = await engine({ fetch }, true);
	const found = await docs.search(request("mise", "trust configuration"));
	expect(found.status).toBe("ok");
	if (found.status !== "ok") throw new Error(JSON.stringify(found));
	expect(found.reasons.join(" ")).toMatch(/local/);
	expect(found.reasons.join(" ")).toMatch(/devdocs/);
	expect(found.results[0]?.origin).toBe(trust);
	const read = await docs.read([found.results[0]!.ref]);
	expect(read.sections[0]?.content).toContain("explicit trust approval");
	expect(fetch.mock.calls.map(([url]) => url)).toEqual([index, trust]);
	const second = await docs.search(request("mise", "backend plugin"));
	expect(second.status === "ok" && second.results[0]?.origin).toBe(backends);
	const adapter = upstreamAdapter({ fetch });
	expect(await adapter.discover(request("mise", "nonexistent-topic"))).toEqual(
		[],
	);
});

it("bounds llms link enumeration and supports registered roots plus explicit HTTPS targets", async () => {
	const root = source("mise", {
		id: "custom",
		aliases: ["custom docs"],
		home: "https://docs.example/",
		index: "https://docs.example/llms.txt",
	});
	const fetch = fixtureFetch(
		{
			"https://docs.example/llms.txt":
				Array.from({ length: 256 }, (_, n) => `[noise](./noise-${n}.md)`).join(
					"\n",
				) + "\n[Needle](./needle.md)",
			"https://docs.example/guide.md": "# Guide\n\nNeedle custom source.",
		},
		"text/markdown",
	);
	const adapter = upstreamAdapter({ fetch, sources: [root] });
	expect(await adapter.discover(request("custom docs", "needle"))).toEqual([]);
	const docs = await engine({ fetch, sources: [root] });
	const found = await docs.search(
		request("https://docs.example/guide.md", "needle"),
	);
	expect(found.status).toBe("ok");
	if (found.status !== "ok") throw new Error(JSON.stringify(found));
	expect(
		(await docs.read([found.results[0]!.ref])).sections[0]?.content,
	).toContain("custom source");
	expect(await adapter.discover(request("unknown-product", "needle"))).toEqual(
		[],
	);
});

it.each([
	{ name: "429 delta-seconds", status: 429, header: "3600", delay: 3600000 },
	{ name: "503 HTTP-date", status: 503, header: "date", delay: 3600000 },
	{ name: "short delay floor", status: 429, header: "1", delay: 60000 },
	{
		name: "invalid delay fallback",
		status: 503,
		header: "invalid",
		delay: 60000,
	},
	{ name: "absent delay fallback", status: 429, header: null, delay: 60000 },
])("honors upstream retry policy: $name", async ({ status, header, delay }) => {
	const start = Math.floor(Date.now() / 1000) * 1000;
	const retry =
		header === "date" ? new Date(start + delay).toUTCString() : header;
	const fetch = vi
		.fn<UpstreamFetch>()
		.mockResolvedValueOnce(
			new Response("PRIVATE_UPSTREAM_ERROR", {
				status,
				headers: retry === null ? {} : { "retry-after": retry },
			}),
		)
		.mockImplementation(
			async () =>
				new Response(
					"# Transactions\n\nTransactions resume after the server delay.",
					{
						headers: { "content-type": "text/markdown" },
					},
				),
		);
	const docs = await engine({ fetch });
	const req = request("https://docs.example/retry.md", "transactions");
	const clock = vi.spyOn(Date, "now").mockReturnValue(start);
	try {
		const first = await docs.search(req);
		expect(first.status).toBe("unavailable");
		expect(JSON.stringify(first)).not.toContain("PRIVATE_UPSTREAM_ERROR");
		expect(fetch).toHaveBeenCalledTimes(1);
		// A long Retry-After must not fall back to the ordinary one-minute retry.
		clock.mockReturnValue(start + Math.min(60001, delay - 1));
		await docs.search(req);
		expect(fetch).toHaveBeenCalledTimes(1);
		clock.mockReturnValue(start + delay - 1);
		await docs.search(req);
		expect(fetch).toHaveBeenCalledTimes(1);
		clock.mockReturnValue(start + delay);
		const resumed = await docs.search(req);
		expect(fetch).toHaveBeenCalledTimes(2);
		expect(resumed.status).toBe("ok");
		if (resumed.status !== "ok") throw new Error("retry did not resume");
		expect(
			(await docs.read([resumed.results[0]!.ref])).sections[0]?.content,
		).toContain("resume after the server delay");
	} finally {
		clock.mockRestore();
	}
});

it("does not infer explicit release compatibility from a URL or HTTP success", async () => {
	const fetch = fixtureFetch({
		"https://docs.example/v21/api.html": "<h1>Java</h1><p>List get(int)</p>",
	});
	const docs = await engine({
		fetch,
		sources: [source("java", { home: "https://docs.example/v21/api.html" })],
	});
	expect(
		(await docs.search({ ...request("java", "List"), version: "21" })).status,
	).toBe("unavailable");
	expect(fetch).not.toHaveBeenCalled();
});

it("keeps remote scripts inert, code fences literal, anchors reusable and entities decoded", () => {
	const html =
		'<h1 id="api">API &#x1F600;</h1><p>Use <code>a`b</code> &lt; &#169; &nbsp; &#x3bb;.</p><pre>  first\n````\n# literal {#keep}\n```\n\tlast</pre><h2 id="next">Next</h2><p>Done.</p><script>throw new Error("executed")</script>';
	const markdown = htmlMarkdown(html);
	expect(markdown).toContain("# API 😀 {#api}");
	expect(markdown).toContain("``a`b``");
	expect(markdown).toContain("< © λ");
	const sections = readFormat(html, "html");
	expect(sections.map((part) => part.locator)).toEqual(["api", "next"]);
	expect(sections[0]?.body).toContain(
		"  first\n````\n# literal {#keep}\n```\n\tlast",
	);
	expect(sections[0]?.body).not.toContain("executed");
	expect(
		readFormat(
			"# Guide\n\n~~~\n```\n# example\n~~~\n\n## Real\nText",
			"markdown",
		).map((part) => part.heading),
	).toEqual(["Guide", "Guide > Real"]);
	expect(() =>
		htmlMarkdown("<pre>" + "a`".repeat(150000) + "</pre>"),
	).not.toThrow();
	expect(() => htmlMarkdown("<a ".repeat(10000))).not.toThrow();
});

it("reports source identity separately from unsupported format capabilities", () => {
	expect(new Set(sourceCatalog.map((entry) => entry.id)).size).toBe(
		sourceCatalog.length,
	);
	expect(localSourceCatalog.every((entry) => entry.adapter === "local")).toBe(
		true,
	);
	expect(sourceCapabilities(source("tldr")).coverage).toBe("supplemental");
	expect(sourceCapabilities(source("nerd-fonts"))).toMatchObject({
		available: true,
		version: null,
	});
	expect(source("readthedocs").authority).toBe("host");
	expect(
		sourceCatalog.some((entry) =>
			/ci\.bugabinga|x230/i.test(JSON.stringify(entry)),
		),
	).toBe(false);
	for (const id of [
		"pdf",
		"javadoc",
		"sphinx",
		"json",
		"mdx",
		"rst",
	] as const) {
		expect(formatReaders.find((entry) => entry.id === id)?.available).toBe(
			true,
		);
	}
	expect(() => readFormat("unparsed", "docset")).toThrow(/unavailable/);
});

it("rejects secret/non-HTTPS targets before fetch and refuses unsafe redirects", async () => {
	const fetch = vi.fn<UpstreamFetch>(
		async () =>
			new Response(null, {
				status: 302,
				headers: { location: "https://other.example/page" },
			}),
	);
	const adapter = upstreamAdapter({ fetch });
	for (const url of [
		"http://docs.example/",
		"https://user:secret@docs.example/",
		"https://docs.example/?token=secret",
	]) {
		await expect(adapter.discover(request(url, "docs"))).rejects.toThrow(
			/HTTPS/,
		);
	}
	expect(fetch).not.toHaveBeenCalled();
	const req = request("https://docs.example/", "docs");
	const [candidate] = await adapter.discover(req);
	await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
		/unavailable/,
	);
	expect(fetch).toHaveBeenCalledTimes(1);
	expect(fetch.mock.calls[0]?.[1]).toMatchObject({
		redirect: "manual",
		credentials: "omit",
		referrerPolicy: "no-referrer",
	});
	for (const location of [
		"/page?key=secret",
		"http://docs.example/page",
		"https://user:secret@docs.example/page",
	]) {
		fetch.mockImplementation(
			async () => new Response(null, { status: 302, headers: { location } }),
		);
		await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
			/unavailable/,
		);
	}
});

it("bounds streamed bytes, redirects and deadlines; validates retained revisions conditionally", async () => {
	const req = request("https://docs.example/guide.md", "guide");
	const fetch = vi.fn<UpstreamFetch>(
		async () => new Response("# Guide\n\n" + "x".repeat(128)),
	);
	const adapter = upstreamAdapter({ fetch, maxBytes: 64, timeoutMs: 30 });
	const [candidate] = await adapter.discover(req);
	await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
		/unavailable/,
	);
	fetch.mockImplementation(
		async () => new Response("body", { headers: { "content-length": "1024" } }),
	);
	await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
		/unavailable/,
	);
	fetch.mockImplementation(
		async () =>
			new Response(null, { status: 302, headers: { location: "/again" } }),
	);
	const before = fetch.mock.calls.length;
	await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
		/unavailable/,
	);
	expect(fetch.mock.calls.length - before).toBe(4);
	fetch.mockImplementation(async () => new Response(null, { status: 304 }));
	expect(await adapter.retrieve(candidate!, req, '"revision-1"')).toBe(
		"unchanged",
	);
	expect(fetch.mock.lastCall?.[1].headers).toMatchObject({
		"if-none-match": '"revision-1"',
	});
	fetch.mockImplementation(
		async () =>
			new Response(
				new ReadableStream({ pull() {}, cancel: () => new Promise(() => {}) }),
			),
	);
	await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
		/unavailable/,
	);
	fetch.mockImplementation(() => new Promise(() => {}));
	await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
		/unavailable/,
	);
	const cancelled = new AbortController();
	cancelled.abort();
	await expect(
		adapter.discover({ ...req, signal: cancelled.signal }),
	).rejects.toThrow();
});