repositories / pi-ext
pi-ext
bugabingas pi extensions
owned by admin
extensions/devdocs/__serial__/upstream.test.ts
Rawimport { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, it, vi } from "vitest";
import {
type UpstreamFetch,
type UpstreamOptions,
upstreamAdapter,
} from "../src/adapters/upstream.ts";
import {
type CatalogSource,
formatReaders,
localSourceCatalog,
sourceCapabilities,
sourceCatalog,
} from "../src/catalog.ts";
import { Devdocs } from "../src/engine.ts";
import { htmlMarkdown, readFormat } from "../src/formats.ts";
import type { Adapter, Request } from "../src/types.ts";
const roots: string[] = [];
const engines: Devdocs[] = [];
afterEach(async () => {
await Promise.all(engines.splice(0).map((engine) => engine.close()));
await Promise.all(
roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
);
});
const request = (target: string, query: string): Request => ({
target,
query,
cwd: tmpdir(),
});
const absent = (id: string, rank: number): Adapter => ({
id,
rank,
formatVersion: 1,
coverage: "corpus",
discover: async () => [],
retrieve: async () => {
throw new Error("unexpected retrieval from absent source");
},
});
async function engine(options: UpstreamOptions, lowerRanks = false) {
const root = await mkdtemp(join(tmpdir(), "devdocs-upstream-"));
roots.push(root);
const instance = new Devdocs(
{
databasePath: join(root, "cache.db"),
enabledSources: ["local", "devdocs", "upstream"],
sources: [],
offline: false,
cacheBytes: 4 * 1024 * 1024,
ttlMs: 60000,
},
[
...(lowerRanks ? [absent("local", 1), absent("devdocs", 2)] : []),
upstreamAdapter(options),
],
);
engines.push(instance);
return instance;
}
function fixtureFetch(pages: Record<string, string>, type = "text/html") {
return vi.fn<UpstreamFetch>(async (url) =>
pages[url] === undefined
? new Response("missing", { status: 404 })
: new Response(pages[url], {
headers: { "content-type": type, etag: '"fixture-v1"' },
}),
);
}
function source(
id: string,
overrides: Partial<CatalogSource> = {},
): CatalogSource {
const base = sourceCatalog.find((entry) => entry.id === id);
if (!base) throw new Error("missing fixture source");
return { ...base, ...overrides };
}
it.each([
{
target: "mdn",
url: "https://developer.mozilla.org/en-US/docs/Web/",
query: "Array.prototype.map",
locator: "map",
html: '<nav>UNWANTED_NAV</nav><main><h1>Array</h1><h2 id="map">Array.prototype.map</h2><p>Creates a new array & preserves values.</p><pre><code>const values = [1, 2];\n values.map(x => x * 2);</code></pre><script>UNWANTED_SCRIPT</script><style>UNWANTED_STYLE</style></main>',
expected: " values.map(x => x * 2);",
version: undefined,
},
{
target: "java",
url: "https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/List.html",
query: "get(int)",
locator: "get(int)",
html: '<h1>java.util.List<E></h1><section class="detail" id="get(int)"><h3>get(int)</h3><pre>public E get(int index)</pre><p>Returns the element at the specified position.</p></section>',
expected: "public E get(int index)",
version: "21",
},
{
target: "man7",
url: "https://man7.org/linux/man-pages/",
query: "open(2)",
locator: "DESCRIPTION",
html: '<h1>open(2) Linux manual page</h1><h2><a id="DESCRIPTION"></a>DESCRIPTION</h2><pre>open(2) opens a file descriptor.\n #include <fcntl.h>\n int open(const char *path, int flags);</pre>',
expected: " #include <fcntl.h>",
version: undefined,
},
{
target: "win32",
url: "https://learn.microsoft.com/en-us/windows/win32/",
query: "CreateFileW",
locator: "requirements",
html: '<nav>UNWANTED_NAV</nav><h1>CreateFileW</h1><h2 id="requirements">CreateFileW requirements</h2><p>Minimum supported client: Windows XP.</p><table><tr><th>Header</th><td>fileapi.h</td></tr><tr><th>Library</th><td>Kernel32.lib</td></tr></table><pre>HANDLE CreateFileW(\n LPCWSTR lpFileName\n);</pre>',
expected: "Kernel32.lib",
version: undefined,
},
])(
"search/read preserves $target domain identity, anchors and code",
async ({ target, url, query, locator, html, expected, version }) => {
const fetch = fixtureFetch({ [url]: html });
const adapterOptions = {
fetch,
sources: [
source(target, {
home: url,
...(version
? {
version,
evidence: "Fixture publisher release metadata: JDK 21",
}
: {}),
}),
],
};
const docs = await engine(adapterOptions);
const found = await docs.search({
...request(target, query),
...(version ? { version } : {}),
});
expect(found.status).toBe("ok");
if (found.status !== "ok") throw new Error(JSON.stringify(found));
expect(found.results[0]).toMatchObject({
target,
source: "upstream",
origin: url,
version: version ?? null,
});
const read = await docs.read(
found.results.map((hit) => hit.ref),
4000,
);
expect(
read.sections.map((section) => section.content).join("\n"),
).toContain(expected);
expect(read.sections[0]?.attribution).toContain(target);
expect(JSON.stringify(read)).not.toContain("UNWANTED_");
const adapter = upstreamAdapter(adapterOptions);
const candidates = await adapter.discover(request(target, query));
const document = await adapter.retrieve(
candidates[0]!,
request(target, query),
);
if (document === "unchanged") throw new Error("unexpected 304");
expect(
document.sections.some((section) => section.locator === locator),
).toBe(true);
},
);
it("falls through absent local/DevDocs to query-relevant scoped llms pages without whole-site ingestion", async () => {
const index = "https://mise.jdx.dev/llms.txt";
const trust = "https://mise.jdx.dev/configuration/trust.md";
const backends = "https://mise.jdx.dev/backends.md";
const fetch = fixtureFetch(
{
[index]:
"# Mise\n[Broken](./%E0%A4)\n[Backends](./backends.md)\n[Trust config](./configuration/trust.md)\n[Secret](./trust?token=private)\n[Other](https://other.example/trust.md)",
[trust]:
"# Trust\n\nTrusted configuration requires explicit trust approval.",
[backends]: "# Backends\n\nBackend plugin reference.",
},
"text/markdown",
);
const docs = await engine({ fetch }, true);
const found = await docs.search(request("mise", "trust configuration"));
expect(found.status).toBe("ok");
if (found.status !== "ok") throw new Error(JSON.stringify(found));
expect(found.reasons.join(" ")).toMatch(/local/);
expect(found.reasons.join(" ")).toMatch(/devdocs/);
expect(found.results[0]?.origin).toBe(trust);
const read = await docs.read([found.results[0]!.ref]);
expect(read.sections[0]?.content).toContain("explicit trust approval");
expect(fetch.mock.calls.map(([url]) => url)).toEqual([index, trust]);
const second = await docs.search(request("mise", "backend plugin"));
expect(second.status === "ok" && second.results[0]?.origin).toBe(backends);
const adapter = upstreamAdapter({ fetch });
expect(await adapter.discover(request("mise", "nonexistent-topic"))).toEqual(
[],
);
});
it("bounds llms link enumeration and supports registered roots plus explicit HTTPS targets", async () => {
const root = source("mise", {
id: "custom",
aliases: ["custom docs"],
home: "https://docs.example/",
index: "https://docs.example/llms.txt",
});
const fetch = fixtureFetch(
{
"https://docs.example/llms.txt":
Array.from({ length: 256 }, (_, n) => `[noise](./noise-${n}.md)`).join(
"\n",
) + "\n[Needle](./needle.md)",
"https://docs.example/guide.md": "# Guide\n\nNeedle custom source.",
},
"text/markdown",
);
const adapter = upstreamAdapter({ fetch, sources: [root] });
expect(await adapter.discover(request("custom docs", "needle"))).toEqual([]);
const docs = await engine({ fetch, sources: [root] });
const found = await docs.search(
request("https://docs.example/guide.md", "needle"),
);
expect(found.status).toBe("ok");
if (found.status !== "ok") throw new Error(JSON.stringify(found));
expect(
(await docs.read([found.results[0]!.ref])).sections[0]?.content,
).toContain("custom source");
expect(await adapter.discover(request("unknown-product", "needle"))).toEqual(
[],
);
});
it.each([
{ name: "429 delta-seconds", status: 429, header: "3600", delay: 3600000 },
{ name: "503 HTTP-date", status: 503, header: "date", delay: 3600000 },
{ name: "short delay floor", status: 429, header: "1", delay: 60000 },
{
name: "invalid delay fallback",
status: 503,
header: "invalid",
delay: 60000,
},
{ name: "absent delay fallback", status: 429, header: null, delay: 60000 },
])("honors upstream retry policy: $name", async ({ status, header, delay }) => {
const start = Math.floor(Date.now() / 1000) * 1000;
const retry =
header === "date" ? new Date(start + delay).toUTCString() : header;
const fetch = vi
.fn<UpstreamFetch>()
.mockResolvedValueOnce(
new Response("PRIVATE_UPSTREAM_ERROR", {
status,
headers: retry === null ? {} : { "retry-after": retry },
}),
)
.mockImplementation(
async () =>
new Response(
"# Transactions\n\nTransactions resume after the server delay.",
{
headers: { "content-type": "text/markdown" },
},
),
);
const docs = await engine({ fetch });
const req = request("https://docs.example/retry.md", "transactions");
const clock = vi.spyOn(Date, "now").mockReturnValue(start);
try {
const first = await docs.search(req);
expect(first.status).toBe("unavailable");
expect(JSON.stringify(first)).not.toContain("PRIVATE_UPSTREAM_ERROR");
expect(fetch).toHaveBeenCalledTimes(1);
// A long Retry-After must not fall back to the ordinary one-minute retry.
clock.mockReturnValue(start + Math.min(60001, delay - 1));
await docs.search(req);
expect(fetch).toHaveBeenCalledTimes(1);
clock.mockReturnValue(start + delay - 1);
await docs.search(req);
expect(fetch).toHaveBeenCalledTimes(1);
clock.mockReturnValue(start + delay);
const resumed = await docs.search(req);
expect(fetch).toHaveBeenCalledTimes(2);
expect(resumed.status).toBe("ok");
if (resumed.status !== "ok") throw new Error("retry did not resume");
expect(
(await docs.read([resumed.results[0]!.ref])).sections[0]?.content,
).toContain("resume after the server delay");
} finally {
clock.mockRestore();
}
});
it("does not infer explicit release compatibility from a URL or HTTP success", async () => {
const fetch = fixtureFetch({
"https://docs.example/v21/api.html": "<h1>Java</h1><p>List get(int)</p>",
});
const docs = await engine({
fetch,
sources: [source("java", { home: "https://docs.example/v21/api.html" })],
});
expect(
(await docs.search({ ...request("java", "List"), version: "21" })).status,
).toBe("unavailable");
expect(fetch).not.toHaveBeenCalled();
});
it("keeps remote scripts inert, code fences literal, anchors reusable and entities decoded", () => {
const html =
'<h1 id="api">API 😀</h1><p>Use <code>a`b</code> < © λ.</p><pre> first\n````\n# literal {#keep}\n```\n\tlast</pre><h2 id="next">Next</h2><p>Done.</p><script>throw new Error("executed")</script>';
const markdown = htmlMarkdown(html);
expect(markdown).toContain("# API 😀 {#api}");
expect(markdown).toContain("``a`b``");
expect(markdown).toContain("< © λ");
const sections = readFormat(html, "html");
expect(sections.map((part) => part.locator)).toEqual(["api", "next"]);
expect(sections[0]?.body).toContain(
" first\n````\n# literal {#keep}\n```\n\tlast",
);
expect(sections[0]?.body).not.toContain("executed");
expect(
readFormat(
"# Guide\n\n~~~\n```\n# example\n~~~\n\n## Real\nText",
"markdown",
).map((part) => part.heading),
).toEqual(["Guide", "Guide > Real"]);
expect(() =>
htmlMarkdown("<pre>" + "a`".repeat(150000) + "</pre>"),
).not.toThrow();
expect(() => htmlMarkdown("<a ".repeat(10000))).not.toThrow();
});
it("reports source identity separately from unsupported format capabilities", () => {
expect(new Set(sourceCatalog.map((entry) => entry.id)).size).toBe(
sourceCatalog.length,
);
expect(localSourceCatalog.every((entry) => entry.adapter === "local")).toBe(
true,
);
expect(sourceCapabilities(source("tldr")).coverage).toBe("supplemental");
expect(sourceCapabilities(source("nerd-fonts"))).toMatchObject({
available: true,
version: null,
});
expect(source("readthedocs").authority).toBe("host");
expect(
sourceCatalog.some((entry) =>
/ci\.bugabinga|x230/i.test(JSON.stringify(entry)),
),
).toBe(false);
for (const id of [
"pdf",
"javadoc",
"sphinx",
"json",
"mdx",
"rst",
] as const) {
expect(formatReaders.find((entry) => entry.id === id)?.available).toBe(
true,
);
}
expect(() => readFormat("unparsed", "docset")).toThrow(/unavailable/);
});
it("rejects secret/non-HTTPS targets before fetch and refuses unsafe redirects", async () => {
const fetch = vi.fn<UpstreamFetch>(
async () =>
new Response(null, {
status: 302,
headers: { location: "https://other.example/page" },
}),
);
const adapter = upstreamAdapter({ fetch });
for (const url of [
"http://docs.example/",
"https://user:secret@docs.example/",
"https://docs.example/?token=secret",
]) {
await expect(adapter.discover(request(url, "docs"))).rejects.toThrow(
/HTTPS/,
);
}
expect(fetch).not.toHaveBeenCalled();
const req = request("https://docs.example/", "docs");
const [candidate] = await adapter.discover(req);
await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
/unavailable/,
);
expect(fetch).toHaveBeenCalledTimes(1);
expect(fetch.mock.calls[0]?.[1]).toMatchObject({
redirect: "manual",
credentials: "omit",
referrerPolicy: "no-referrer",
});
for (const location of [
"/page?key=secret",
"http://docs.example/page",
"https://user:secret@docs.example/page",
]) {
fetch.mockImplementation(
async () => new Response(null, { status: 302, headers: { location } }),
);
await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
/unavailable/,
);
}
});
it("bounds streamed bytes, redirects and deadlines; validates retained revisions conditionally", async () => {
const req = request("https://docs.example/guide.md", "guide");
const fetch = vi.fn<UpstreamFetch>(
async () => new Response("# Guide\n\n" + "x".repeat(128)),
);
const adapter = upstreamAdapter({ fetch, maxBytes: 64, timeoutMs: 30 });
const [candidate] = await adapter.discover(req);
await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
/unavailable/,
);
fetch.mockImplementation(
async () => new Response("body", { headers: { "content-length": "1024" } }),
);
await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
/unavailable/,
);
fetch.mockImplementation(
async () =>
new Response(null, { status: 302, headers: { location: "/again" } }),
);
const before = fetch.mock.calls.length;
await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
/unavailable/,
);
expect(fetch.mock.calls.length - before).toBe(4);
fetch.mockImplementation(async () => new Response(null, { status: 304 }));
expect(await adapter.retrieve(candidate!, req, '"revision-1"')).toBe(
"unchanged",
);
expect(fetch.mock.lastCall?.[1].headers).toMatchObject({
"if-none-match": '"revision-1"',
});
fetch.mockImplementation(
async () =>
new Response(
new ReadableStream({ pull() {}, cancel: () => new Promise(() => {}) }),
),
);
await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
/unavailable/,
);
fetch.mockImplementation(() => new Promise(() => {}));
await expect(adapter.retrieve(candidate!, req)).rejects.toThrow(
/unavailable/,
);
const cancelled = new AbortController();
cancelled.abort();
await expect(
adapter.discover({ ...req, signal: cancelled.signal }),
).rejects.toThrow();
});