import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; import { type UpstreamFetch, type UpstreamOptions, upstreamAdapter, } from "../src/adapters/upstream.ts"; import { type CatalogSource, formatReaders, localSourceCatalog, sourceCapabilities, sourceCatalog, } from "../src/catalog.ts"; import { Devdocs } from "../src/engine.ts"; import { htmlMarkdown, readFormat } from "../src/formats.ts"; import type { Adapter, Request } from "../src/types.ts"; const roots: string[] = []; const engines: Devdocs[] = []; afterEach(async () => { await Promise.all(engines.splice(0).map((engine) => engine.close())); await Promise.all( roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), ); }); const request = (target: string, query: string): Request => ({ target, query, cwd: tmpdir(), }); const absent = (id: string, rank: number): Adapter => ({ id, rank, formatVersion: 1, coverage: "corpus", discover: async () => [], retrieve: async () => { throw new Error("unexpected retrieval from absent source"); }, }); async function engine(options: UpstreamOptions, lowerRanks = false) { const root = await mkdtemp(join(tmpdir(), "devdocs-upstream-")); roots.push(root); const instance = new Devdocs( { databasePath: join(root, "cache.db"), enabledSources: ["local", "devdocs", "upstream"], sources: [], offline: false, cacheBytes: 4 * 1024 * 1024, ttlMs: 60000, }, [ ...(lowerRanks ? [absent("local", 1), absent("devdocs", 2)] : []), upstreamAdapter(options), ], ); engines.push(instance); return instance; } function fixtureFetch(pages: Record, type = "text/html") { return vi.fn(async (url) => pages[url] === undefined ? new Response("missing", { status: 404 }) : new Response(pages[url], { headers: { "content-type": type, etag: '"fixture-v1"' }, }), ); } function source( id: string, overrides: Partial = {}, ): CatalogSource { const base = sourceCatalog.find((entry) => entry.id === id); if (!base) throw new Error("missing fixture source"); return { ...base, ...overrides }; } it.each([ { target: "mdn", url: "https://developer.mozilla.org/en-US/docs/Web/", query: "Array.prototype.map", locator: "map", html: '

Array

Array.prototype.map

Creates a new array & preserves values.

const values = [1, 2];\n  values.map(x => x * 2);
', expected: " values.map(x => x * 2);", version: undefined, }, { target: "java", url: "https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/List.html", query: "get(int)", locator: "get(int)", html: '

java.util.List<E>

get(int)

public E get(int index)

Returns the element at the specified position.

', expected: "public E get(int index)", version: "21", }, { target: "man7", url: "https://man7.org/linux/man-pages/", query: "open(2)", locator: "DESCRIPTION", html: '

open(2) Linux manual page

DESCRIPTION

open(2) opens a file descriptor.\n    #include <fcntl.h>\n    int open(const char *path, int flags);
', expected: " #include ", version: undefined, }, { target: "win32", url: "https://learn.microsoft.com/en-us/windows/win32/", query: "CreateFileW", locator: "requirements", html: '

CreateFileW

CreateFileW requirements

Minimum supported client: Windows XP.

Headerfileapi.h
LibraryKernel32.lib
HANDLE CreateFileW(\n    LPCWSTR lpFileName\n);
', expected: "Kernel32.lib", version: undefined, }, ])( "search/read preserves $target domain identity, anchors and code", async ({ target, url, query, locator, html, expected, version }) => { const fetch = fixtureFetch({ [url]: html }); const adapterOptions = { fetch, sources: [ source(target, { home: url, ...(version ? { version, evidence: "Fixture publisher release metadata: JDK 21", } : {}), }), ], }; const docs = await engine(adapterOptions); const found = await docs.search({ ...request(target, query), ...(version ? { version } : {}), }); expect(found.status).toBe("ok"); if (found.status !== "ok") throw new Error(JSON.stringify(found)); expect(found.results[0]).toMatchObject({ target, source: "upstream", origin: url, version: version ?? null, }); const read = await docs.read( found.results.map((hit) => hit.ref), 4000, ); expect( read.sections.map((section) => section.content).join("\n"), ).toContain(expected); expect(read.sections[0]?.attribution).toContain(target); expect(JSON.stringify(read)).not.toContain("UNWANTED_"); const adapter = upstreamAdapter(adapterOptions); const candidates = await adapter.discover(request(target, query)); const document = await adapter.retrieve( candidates[0]!, request(target, query), ); if (document === "unchanged") throw new Error("unexpected 304"); expect( document.sections.some((section) => section.locator === locator), ).toBe(true); }, ); it("falls through absent local/DevDocs to query-relevant scoped llms pages without whole-site ingestion", async () => { const index = "https://mise.jdx.dev/llms.txt"; const trust = "https://mise.jdx.dev/configuration/trust.md"; const backends = "https://mise.jdx.dev/backends.md"; const fetch = fixtureFetch( { [index]: "# Mise\n[Broken](./%E0%A4)\n[Backends](./backends.md)\n[Trust config](./configuration/trust.md)\n[Secret](./trust?token=private)\n[Other](https://other.example/trust.md)", [trust]: "# Trust\n\nTrusted configuration requires explicit trust approval.", [backends]: "# Backends\n\nBackend plugin reference.", }, "text/markdown", ); const docs = await engine({ fetch }, true); const found = await docs.search(request("mise", "trust configuration")); expect(found.status).toBe("ok"); if (found.status !== "ok") throw new Error(JSON.stringify(found)); expect(found.reasons.join(" ")).toMatch(/local/); expect(found.reasons.join(" ")).toMatch(/devdocs/); expect(found.results[0]?.origin).toBe(trust); const read = await docs.read([found.results[0]!.ref]); expect(read.sections[0]?.content).toContain("explicit trust approval"); expect(fetch.mock.calls.map(([url]) => url)).toEqual([index, trust]); const second = await docs.search(request("mise", "backend plugin")); expect(second.status === "ok" && second.results[0]?.origin).toBe(backends); const adapter = upstreamAdapter({ fetch }); expect(await adapter.discover(request("mise", "nonexistent-topic"))).toEqual( [], ); }); it("bounds llms link enumeration and supports registered roots plus explicit HTTPS targets", async () => { const root = source("mise", { id: "custom", aliases: ["custom docs"], home: "https://docs.example/", index: "https://docs.example/llms.txt", }); const fetch = fixtureFetch( { "https://docs.example/llms.txt": Array.from({ length: 256 }, (_, n) => `[noise](./noise-${n}.md)`).join( "\n", ) + "\n[Needle](./needle.md)", "https://docs.example/guide.md": "# Guide\n\nNeedle custom source.", }, "text/markdown", ); const adapter = upstreamAdapter({ fetch, sources: [root] }); expect(await adapter.discover(request("custom docs", "needle"))).toEqual([]); const docs = await engine({ fetch, sources: [root] }); const found = await docs.search( request("https://docs.example/guide.md", "needle"), ); expect(found.status).toBe("ok"); if (found.status !== "ok") throw new Error(JSON.stringify(found)); expect( (await docs.read([found.results[0]!.ref])).sections[0]?.content, ).toContain("custom source"); expect(await adapter.discover(request("unknown-product", "needle"))).toEqual( [], ); }); it.each([ { name: "429 delta-seconds", status: 429, header: "3600", delay: 3600000 }, { name: "503 HTTP-date", status: 503, header: "date", delay: 3600000 }, { name: "short delay floor", status: 429, header: "1", delay: 60000 }, { name: "invalid delay fallback", status: 503, header: "invalid", delay: 60000, }, { name: "absent delay fallback", status: 429, header: null, delay: 60000 }, ])("honors upstream retry policy: $name", async ({ status, header, delay }) => { const start = Math.floor(Date.now() / 1000) * 1000; const retry = header === "date" ? new Date(start + delay).toUTCString() : header; const fetch = vi .fn() .mockResolvedValueOnce( new Response("PRIVATE_UPSTREAM_ERROR", { status, headers: retry === null ? {} : { "retry-after": retry }, }), ) .mockImplementation( async () => new Response( "# Transactions\n\nTransactions resume after the server delay.", { headers: { "content-type": "text/markdown" }, }, ), ); const docs = await engine({ fetch }); const req = request("https://docs.example/retry.md", "transactions"); const clock = vi.spyOn(Date, "now").mockReturnValue(start); try { const first = await docs.search(req); expect(first.status).toBe("unavailable"); expect(JSON.stringify(first)).not.toContain("PRIVATE_UPSTREAM_ERROR"); expect(fetch).toHaveBeenCalledTimes(1); // A long Retry-After must not fall back to the ordinary one-minute retry. clock.mockReturnValue(start + Math.min(60001, delay - 1)); await docs.search(req); expect(fetch).toHaveBeenCalledTimes(1); clock.mockReturnValue(start + delay - 1); await docs.search(req); expect(fetch).toHaveBeenCalledTimes(1); clock.mockReturnValue(start + delay); const resumed = await docs.search(req); expect(fetch).toHaveBeenCalledTimes(2); expect(resumed.status).toBe("ok"); if (resumed.status !== "ok") throw new Error("retry did not resume"); expect( (await docs.read([resumed.results[0]!.ref])).sections[0]?.content, ).toContain("resume after the server delay"); } finally { clock.mockRestore(); } }); it("does not infer explicit release compatibility from a URL or HTTP success", async () => { const fetch = fixtureFetch({ "https://docs.example/v21/api.html": "

Java

List get(int)

", }); const docs = await engine({ fetch, sources: [source("java", { home: "https://docs.example/v21/api.html" })], }); expect( (await docs.search({ ...request("java", "List"), version: "21" })).status, ).toBe("unavailable"); expect(fetch).not.toHaveBeenCalled(); }); it("keeps remote scripts inert, code fences literal, anchors reusable and entities decoded", () => { const html = '

API 😀

Use a`b < ©   λ.

  first\n````\n# literal {#keep}\n```\n\tlast

Next

Done.

'; const markdown = htmlMarkdown(html); expect(markdown).toContain("# API 😀 {#api}"); expect(markdown).toContain("``a`b``"); expect(markdown).toContain("< © λ"); const sections = readFormat(html, "html"); expect(sections.map((part) => part.locator)).toEqual(["api", "next"]); expect(sections[0]?.body).toContain( " first\n````\n# literal {#keep}\n```\n\tlast", ); expect(sections[0]?.body).not.toContain("executed"); expect( readFormat( "# Guide\n\n~~~\n```\n# example\n~~~\n\n## Real\nText", "markdown", ).map((part) => part.heading), ).toEqual(["Guide", "Guide > Real"]); expect(() => htmlMarkdown("
" + "a`".repeat(150000) + "
"), ).not.toThrow(); expect(() => htmlMarkdown(" { expect(new Set(sourceCatalog.map((entry) => entry.id)).size).toBe( sourceCatalog.length, ); expect(localSourceCatalog.every((entry) => entry.adapter === "local")).toBe( true, ); expect(sourceCapabilities(source("tldr")).coverage).toBe("supplemental"); expect(sourceCapabilities(source("nerd-fonts"))).toMatchObject({ available: true, version: null, }); expect(source("readthedocs").authority).toBe("host"); expect( sourceCatalog.some((entry) => /ci\.bugabinga|x230/i.test(JSON.stringify(entry)), ), ).toBe(false); for (const id of [ "pdf", "javadoc", "sphinx", "json", "mdx", "rst", ] as const) { expect(formatReaders.find((entry) => entry.id === id)?.available).toBe( true, ); } expect(() => readFormat("unparsed", "docset")).toThrow(/unavailable/); }); it("rejects secret/non-HTTPS targets before fetch and refuses unsafe redirects", async () => { const fetch = vi.fn( async () => new Response(null, { status: 302, headers: { location: "https://other.example/page" }, }), ); const adapter = upstreamAdapter({ fetch }); for (const url of [ "http://docs.example/", "https://user:secret@docs.example/", "https://docs.example/?token=secret", ]) { await expect(adapter.discover(request(url, "docs"))).rejects.toThrow( /HTTPS/, ); } expect(fetch).not.toHaveBeenCalled(); const req = request("https://docs.example/", "docs"); const [candidate] = await adapter.discover(req); await expect(adapter.retrieve(candidate!, req)).rejects.toThrow( /unavailable/, ); expect(fetch).toHaveBeenCalledTimes(1); expect(fetch.mock.calls[0]?.[1]).toMatchObject({ redirect: "manual", credentials: "omit", referrerPolicy: "no-referrer", }); for (const location of [ "/page?key=secret", "http://docs.example/page", "https://user:secret@docs.example/page", ]) { fetch.mockImplementation( async () => new Response(null, { status: 302, headers: { location } }), ); await expect(adapter.retrieve(candidate!, req)).rejects.toThrow( /unavailable/, ); } }); it("bounds streamed bytes, redirects and deadlines; validates retained revisions conditionally", async () => { const req = request("https://docs.example/guide.md", "guide"); const fetch = vi.fn( async () => new Response("# Guide\n\n" + "x".repeat(128)), ); const adapter = upstreamAdapter({ fetch, maxBytes: 64, timeoutMs: 30 }); const [candidate] = await adapter.discover(req); await expect(adapter.retrieve(candidate!, req)).rejects.toThrow( /unavailable/, ); fetch.mockImplementation( async () => new Response("body", { headers: { "content-length": "1024" } }), ); await expect(adapter.retrieve(candidate!, req)).rejects.toThrow( /unavailable/, ); fetch.mockImplementation( async () => new Response(null, { status: 302, headers: { location: "/again" } }), ); const before = fetch.mock.calls.length; await expect(adapter.retrieve(candidate!, req)).rejects.toThrow( /unavailable/, ); expect(fetch.mock.calls.length - before).toBe(4); fetch.mockImplementation(async () => new Response(null, { status: 304 })); expect(await adapter.retrieve(candidate!, req, '"revision-1"')).toBe( "unchanged", ); expect(fetch.mock.lastCall?.[1].headers).toMatchObject({ "if-none-match": '"revision-1"', }); fetch.mockImplementation( async () => new Response( new ReadableStream({ pull() {}, cancel: () => new Promise(() => {}) }), ), ); await expect(adapter.retrieve(candidate!, req)).rejects.toThrow( /unavailable/, ); fetch.mockImplementation(() => new Promise(() => {})); await expect(adapter.retrieve(candidate!, req)).rejects.toThrow( /unavailable/, ); const cancelled = new AbortController(); cancelled.abort(); await expect( adapter.discover({ ...req, signal: cancelled.signal }), ).rejects.toThrow(); });