repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

feat(web): persist capped fetch results

f2dd95e2f07ad6bfa5e20352f7b2d743877c214b

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (8)

extensions/web/__tests__/web-fetch-contract.test.ts

Added
new file mode 100644
1 +import { mkdtemp, readFile } from "node:fs/promises";
2 +import { tmpdir } from "node:os";
3 +import path from "node:path";
4 +import { afterEach, describe, expect, it, vi } from "vitest";
5 +import { createTestSession, type TestSession } from "../../../test/harness";
6 +import { fetchUrlStream } from "../fetch";
7 +import { filterLineMatches } from "../line-match";
8 +import {
9 + discoverSiteMap,
10 + parseLlmsLinks,
11 + parseRobotsSitemaps,
12 + parseSitemapXml,
13 +} from "../map";
14 +import {
15 + assertSafeHttpUrl,
16 + isPrivateOrReservedIp,
17 + resolveRedirectUrl,
18 +} from "../safety";
19 +import { platformStateDir, projectStoreDir, saveJsonResult } from "../store";
21 +const originalFetch = globalThis.fetch;
23 +describe("web_fetch context contract", () => {
24 + let t: TestSession | undefined;
25 + afterEach(() => {
26 + globalThis.fetch = originalFetch;
27 + t?.dispose();
28 + t = undefined;
29 + });
31 + it("stores full output but returns only capped visible text and metadata", async () => {
32 + globalThis.fetch = vi.fn(
33 + async () =>
34 + new Response("x".repeat(50_000), {
35 + status: 200,
36 + headers: { "content-type": "text/plain" },
37 + }),
38 + ) as typeof fetch;
40 + const webPath = path.resolve(import.meta.dirname, "../index.ts");
41 + t = await createTestSession({ extensions: [webPath] });
42 + const [extension] = t.session.extensionRunner.extensions;
43 + const tool = extension.tools.get("web_fetch").definition;
44 + const result = await tool.execute(
45 + "test",
46 + { url: "https://93.184.216.34/plain", maxChars: 1_000 },
47 + new AbortController().signal,
48 + undefined,
49 + { cwd: t.cwd },
50 + );
52 + const text = result.content[0].text;
53 + expect(text.length).toBeLessThan(1_500);
54 + expect(text).toContain("responseId:");
55 + expect(result.details.responseId).toBeTypeOf("string");
56 + expect(result.details.fullOutputPath).toBeTypeOf("string");
57 + expect(JSON.stringify(result.details)).not.toContain("x".repeat(200));
58 + const saved = JSON.parse(
59 + await readFile(result.details.fullOutputPath, "utf8"),
60 + );
61 + expect(saved.output).toContain("x".repeat(10_000));
62 + });
64 + it("line matching returns excerpts instead of full page", async () => {
65 + globalThis.fetch = vi.fn(
66 + async () =>
67 + new Response(
68 + ["one", "before", "install with mise", "after", "tail"].join("\n"),
69 + { status: 200, headers: { "content-type": "text/plain" } },
70 + ),
71 + ) as typeof fetch;
73 + const webPath = path.resolve(import.meta.dirname, "../index.ts");
74 + t = await createTestSession({ extensions: [webPath] });
75 + const [extension] = t.session.extensionRunner.extensions;
76 + const result = await extension.tools.get("web_fetch").definition.execute(
77 + "test",
78 + {
79 + url: "https://93.184.216.34/plain",
80 + linesMatching: ["install"],
81 + contextLines: 1,
82 + },
83 + new AbortController().signal,
84 + undefined,
85 + { cwd: t.cwd },
86 + );
88 + const text = result.content[0].text;
89 + expect(text).toContain("2: before");
90 + expect(text).toContain("3: install with mise");
91 + expect(text).toContain("4: after");
92 + expect(text).not.toContain("1: one");
93 + expect(text).not.toContain("5: tail");
94 + });
95 +});
97 +describe("web safety/store/map primitives", () => {
98 + it("blocks unsafe URLs and DNS answers", async () => {
99 + await expect(assertSafeHttpUrl("file:///etc/passwd")).rejects.toThrow(
100 + /Only http: and https:/,
101 + );
102 + await expect(assertSafeHttpUrl("http://127.0.0.1")).rejects.toThrow(
103 + /private address/,
104 + );
105 + await expect(
106 + assertSafeHttpUrl("https://public.test", {
107 + resolver: async () => ["10.0.0.2"],
108 + }),
109 + ).rejects.toThrow(/blocked private address/);
110 + await expect(
111 + resolveRedirectUrl("https://example.com/a", "http://127.0.0.1/b"),
112 + ).rejects.toThrow(/private address/);
113 + expect(isPrivateOrReservedIp("8.8.8.8")).toBe(false);
114 + });
116 + it("fetch validates redirects and max bytes", async () => {
117 + globalThis.fetch = vi.fn(
118 + async () =>
119 + new Response(null, {
120 + status: 302,
121 + headers: { location: "http://127.0.0.1/private" },
122 + }),
123 + ) as typeof fetch;
124 + await expect(
125 + fetchUrlStream("https://example.com", undefined, undefined, {
126 + resolver: async () => ["93.184.216.34"],
127 + }),
128 + ).rejects.toThrow(/private address/);
129 + globalThis.fetch = vi.fn(
130 + async () =>
131 + new Response("x".repeat(20), {
132 + status: 200,
133 + headers: { "content-type": "text/plain" },
134 + }),
135 + ) as typeof fetch;
136 + await expect(
137 + fetchUrlStream("https://example.com", undefined, undefined, {
138 + maxBytes: 10,
139 + resolver: async () => ["93.184.216.34"],
140 + }),
141 + ).rejects.toThrow(/exceeds maxBytes/);
142 + });
144 + it("stores full JSON under XDG state dir", async () => {
145 + const root = await mkdtemp(path.join(tmpdir(), "pi-web-store-"));
146 + expect(
147 + platformStateDir("linux", { XDG_STATE_HOME: "/state" }, "/home/me"),
148 + ).toBe("/state/pi");
149 + expect(
150 + projectStoreDir("web", "/work/project", {
151 + platform: "linux",
152 + env: { XDG_STATE_HOME: "/state" },
153 + home: "/home/me",
154 + }),
155 + ).toMatch(/^\/state\/pi\/web\/[a-f0-9]{64}$/);
156 + const saved = await saveJsonResult(
157 + "web",
158 + root,
159 + { body: "x".repeat(50_000) },
160 + { platform: "linux", env: { XDG_STATE_HOME: root }, home: root },
161 + );
162 + expect(
163 + JSON.parse(await readFile(saved.fullOutputPath, "utf8")).body,
164 + ).toHaveLength(50_000);
165 + expect(JSON.stringify(saved)).not.toContain("xxxxx");
166 + });
168 + it("filters matching lines with context", () => {
169 + const result = filterLineMatches(
170 + ["a", "before", "Install with mise", "after", "z"].join("\n"),
171 + { needles: ["install"], contextLines: 1 },
172 + );
173 + expect(result.text).toContain("2: before");
174 + expect(result.text).toContain("3: Install with mise");
175 + expect(result.text).toContain("4: after");
176 + expect(filterLineMatches("huge\nbody", { needles: ["missing"] }).text).toBe(
177 + "",
178 + );
179 + });
181 + it("maps robots/sitemap/llms without page bodies", async () => {
182 + expect(
183 + parseRobotsSitemaps("Sitemap: https://x.test/sitemap.xml\n"),
184 + ).toEqual(["https://x.test/sitemap.xml"]);
185 + expect(parseSitemapXml("<url><loc>https://x.test/a</loc></url>")).toEqual([
186 + "https://x.test/a",
187 + ]);
188 + expect(
189 + parseLlmsLinks(
190 + "[Docs](https://x.test/docs) [Rel](/rel)",
191 + "https://x.test/base",
192 + ),
193 + ).toEqual(["https://x.test/docs", "https://x.test/rel"]);
194 + const fetched: string[] = [];
195 + const result = await discoverSiteMap("https://x.test/docs", {
196 + fetchText: async (url) => {
197 + fetched.push(url);
198 + if (url.endsWith("robots.txt"))
199 + return "Sitemap: https://x.test/sitemap.xml";
200 + if (url.endsWith("sitemap.xml"))
201 + return "<url><loc>https://x.test/a</loc></url>";
202 + if (url.endsWith("llms.txt")) return "[LLM](https://x.test/llm)";
203 + throw new Error(`unexpected ${url}`);
204 + },
205 + });
206 + expect(fetched).toEqual([
207 + "https://x.test/robots.txt",
208 + "https://x.test/sitemap.xml",
209 + "https://x.test/llms.txt",
210 + ]);
211 + expect(result.urls.map((x) => x.url)).toEqual([
212 + "https://x.test/a",
213 + "https://x.test/llm",
214 + ]);
215 + });
216 +});

extensions/web/constants.ts

Modified
1 1 // Shared constants and types
3 3 export const DEFAULT_TIMEOUT_MS = 120_000; // 2 minutes for large downloads
4 -export const DEFAULT_MAX_CHARS = 100_000;
4 +export const DEFAULT_MAX_BYTES = 50 * 1024 * 1024;
5 +export const DEFAULT_MAX_CHARS = 20_000;
5 6 export const DDG_HTML_URL = "https://html.duckduckgo.com/html/";
7 8 export const USER_AGENT =
29 30 contentType: string;
30 31 contentLength: number;
31 32 status: number;
33 + finalUrl: string;

extensions/web/fetch.ts

Modified
1 1 // HTTP fetch helper — returns body as Buffer + content-type for routing
3 3 import type { FetchResult } from "./constants.js";
4 -import { BROWSER_HEADERS, DEFAULT_TIMEOUT_MS } from "./constants.js";
4 +import {
5 + BROWSER_HEADERS,
6 + DEFAULT_MAX_BYTES,
7 + DEFAULT_TIMEOUT_MS,
8 +} from "./constants.js";
9 +import {
10 + assertSafeHttpUrl,
11 + type DnsResolver,
12 + resolveRedirectUrl,
13 +} from "./safety.js";
15 +export interface FetchUrlOptions {
16 + maxBytes?: number;
17 + maxRedirects?: number;
18 + timeoutMs?: number;
19 + resolver?: DnsResolver;
6 22 export async function fetchUrlStream(
7 23 url: string,
8 24 onProgress?: (loaded: number, total: number) => void,
9 25 signal?: AbortSignal,
26 + options: FetchUrlOptions = {},
10 27 ): Promise<FetchResult> {
11 - const parsed = new URL(url);
12 - const headers: Record<string, string> = {
13 - ...BROWSER_HEADERS,
14 - Referer: `${parsed.protocol}//${parsed.host}/`,
15 - };
28 + let safe = await assertSafeHttpUrl(url, { resolver: options.resolver });
29 + const maxRedirects = options.maxRedirects ?? 5;
30 + const timeoutSignal =
31 + signal ?? AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
17 - const response = await fetch(url, {
18 - headers,
19 - signal: signal ?? AbortSignal.timeout(DEFAULT_TIMEOUT_MS),
20 - });
22 - if (!response.ok) {
23 - throw new Error(`HTTP ${response.status}: ${response.statusText}`);
33 + for (let redirects = 0; redirects <= maxRedirects; redirects++) {
34 + const response = await fetchOnce(safe.normalizedUrl, timeoutSignal);
35 + if (!isRedirect(response.status)) {
36 + return readResponse(
37 + response,
38 + safe.normalizedUrl,
39 + onProgress,
40 + options.maxBytes ?? DEFAULT_MAX_BYTES,
41 + );
42 + }
43 + const location = response.headers.get("location");
44 + await response.body?.cancel().catch(() => {});
45 + if (!location)
46 + throw new Error(`HTTP ${response.status}: redirect missing location`);
47 + if (redirects === maxRedirects)
48 + throw new Error(`Too many redirects (${maxRedirects})`);
49 + safe = await resolveRedirectUrl(safe.normalizedUrl, location, {
50 + resolver: options.resolver,
51 + });
53 + throw new Error(`Too many redirects (${maxRedirects})`);
56 +async function fetchOnce(url: string, signal: AbortSignal): Promise<Response> {
57 + const parsed = new URL(url);
58 + return await fetch(url, {
59 + redirect: "manual",
60 + headers: {
61 + ...BROWSER_HEADERS,
62 + Referer: `${parsed.protocol}//${parsed.host}/`,
63 + },
64 + signal,
65 + });
68 +async function readResponse(
69 + response: Response,
70 + finalUrl: string,
71 + onProgress: ((loaded: number, total: number) => void) | undefined,
72 + maxBytes: number,
73 +): Promise<FetchResult> {
74 + if (!response.ok)
75 + throw new Error(`HTTP ${response.status}: ${response.statusText}`);
76 + if (!response.body) throw new Error("No response body");
26 77 const contentType = response.headers.get("content-type") ?? "";
27 - const contentLength = parseInt(
28 - response.headers.get("content-length") ?? "0",
29 - 10,
30 - );
32 - if (!response.body) {
33 - throw new Error("No response body");
78 + const contentLength =
79 + Number.parseInt(response.headers.get("content-length") ?? "0", 10) || 0;
80 + if (contentLength > maxBytes) {
81 + throw new Error(
82 + `Response exceeds maxBytes (${contentLength} > ${maxBytes})`,
83 + );
36 86 const reader = response.body.getReader();
37 87 let loaded = 0;
38 88 const chunks: Uint8Array[] = [];
40 89 while (true) {
41 90 const { done, value } = await reader.read();
42 91 if (done) break;
43 92 loaded += value.length;
93 + if (loaded > maxBytes) {
94 + await reader.cancel().catch(() => {});
95 + throw new Error(`Response exceeds maxBytes (${loaded} > ${maxBytes})`);
96 + }
44 97 chunks.push(value);
45 98 onProgress?.(loaded, contentLength);
48 - const body = Buffer.concat(chunks);
50 101 return {
51 - body,
102 + body: Buffer.concat(chunks),
52 103 contentType,
53 104 contentLength,
54 105 status: response.status,
106 + finalUrl,
110 +function isRedirect(status: number): boolean {
111 + return status >= 300 && status < 400;
58 114 // DuckDuckGo search fetch (separate because it uses DDG-specific headers)
59 115 export async function fetchSearchHtml(
60 116 url: string,
61 117 signal?: AbortSignal,
62 118 timeout?: number,
63 119 ): Promise<string> {
120 + await assertSafeHttpUrl(url);
64 121 const response = await fetch(url, {
65 122 headers: BROWSER_HEADERS,
123 + redirect: "manual",
66 124 signal: signal ?? AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT_MS),
69 - if (!response.ok) {
127 + if (!response.ok)
70 128 throw new Error(`HTTP ${response.status}: ${response.statusText}`);
71 - }
73 129 return response.text();

extensions/web/index.ts

Modified
39 39 } from "./constants.js";
40 40 import { extractMainContent } from "./extract.js";
41 41 import { fetchUrlStream } from "./fetch.js";
42 +import { filterLineMatches } from "./line-match.js";
43 +import { discoverSiteMap } from "./map.js";
42 44 import { minimaxUnderstandImage } from "./minimax.js";
43 45 import { runPandoc, runPandocFile } from "./pandoc.js";
44 46 import { searchWeb } from "./search.js";
48 50 type SpotlightMarkers,
49 51 wrapUntrusted,
50 52 } from "./spotlight.js";
53 +import { saveJsonResult } from "./store.js";
52 55 // ── Shared formatters ────────────────────────────────────────────────────
472 475 body: Buffer,
473 476 contentType: string,
474 477 url: string,
475 - maxChars: number,
478 + _maxChars: number,
476 479 headerLine: (s: string) => string,
477 480 ): Promise<ContentOutput> {
478 481 const ext = extensionFromContentType(contentType, url);
489 492 "media",
491 494 const fullOutput = `${headerLine(`ffprobe JSON (${desc})`)}\n${desc}\n\n${meta.raw}`;
492 - const output =
493 - fullOutput.length > maxChars
494 - ? fullOutput.slice(0, maxChars) + "\n\u2026 (truncated)"
495 - : fullOutput;
496 495 return {
497 - output,
496 + output: fullOutput,
498 497 tmpPath,
499 498 preview: desc,
501 500 earlyReturn: {
502 - content: [{ type: "text", text: `${url}\n\n${output}` }],
501 + content: [{ type: "text", text: `${url}\n\n${fullOutput}` }],
503 502 details: {
505 504 kind: "media",
754 753 Parameters:
755 754 - url: URL to fetch
756 755 - format: Output format (markdown, text, html, json, raw)
757 -- maxChars: Truncate output to this many characters`,
756 +- maxChars: Limit text returned to the agent (full output is saved)
757 +- maxBytes: Refuse responses larger than this
758 +- linesMatching/contextLines: Return matching excerpts instead of full output`,
759 760 promptSnippet:
760 - "web_fetch(url, format?, maxChars?): fetch URL and convert to readable format",
761 + "web_fetch(url, format?, maxChars?, linesMatching?): fetch URL; keep agent context capped; full output saved",
762 763 parameters: Type.Object({
763 764 url: Type.String({ description: "URL to fetch" }),
779 780 maxChars: Type.Optional(
780 781 Type.Number({
781 - description: "Truncate output to this many characters",
782 + description:
783 + "Limit text returned to the agent; full converted output is saved",
782 784 minimum: 100,
783 - maximum: 1_000_000,
785 + maximum: DEFAULT_MAX_CHARS,
788 + maxBytes: Type.Optional(
789 + Type.Number({
790 + description: "Refuse responses larger than this many bytes",
791 + minimum: 1_000,
792 + maximum: 100_000_000,
793 + }),
794 + ),
795 + linesMatching: Type.Optional(
796 + Type.Array(Type.String(), {
797 + description: "Return only lines matching these strings",
798 + }),
799 + ),
800 + contextLines: Type.Optional(
801 + Type.Number({
802 + description: "Lines of context around each line match",
803 + minimum: 0,
804 + maximum: 20,
805 + }),
806 + ),
807 + caseSensitive: Type.Optional(
808 + Type.Boolean({ description: "Case-sensitive line matching" }),
809 + ),
786 810 timeout: Type.Optional(
787 811 Type.Number({
788 812 description: `Timeout in milliseconds (default: ${DEFAULT_TIMEOUT_MS}ms = ${DEFAULT_TIMEOUT_MS / 1000 / 60}min)`,
834 858 const ct = details?.contentType ?? "unknown";
835 859 const bodySize = details?.bodySize ?? 0;
836 860 const chars = details?.chars ?? 0;
861 + const visibleChars =
862 + (details as { visibleChars?: number } | undefined)?.visibleChars ??
863 + chars;
864 + const responseId = (details as { responseId?: string } | undefined)
865 + ?.responseId;
837 866 const isRaw = details?.format === "raw";
839 868 let header = theme.fg("success", "\u2713 ");
858 887 let llmDesc: string;
859 888 if (isRaw) {
860 - llmDesc = `${fmtChars(chars)} of raw content (no conversion)`;
889 + llmDesc = `${fmtChars(visibleChars)} of raw content (no conversion)`;
861 890 } else {
862 891 switch (kind) {
863 892 case "html": {
870 899 : fmt === "html"
871 900 ? "extracted HTML"
872 901 : "markdown";
873 - llmDesc = `${fmtChars(chars)} of ${typeLabel} (extracted via pandoc)`;
902 + llmDesc = `${fmtChars(visibleChars)} of ${typeLabel} (full saved)`;
874 903 break;
876 905 case "text": {
879 908 : ct.includes("tsv")
880 909 ? "TSV text"
881 910 : "plain text";
882 - llmDesc = `${fmtChars(chars)} of ${sub}`;
911 + llmDesc = `${fmtChars(visibleChars)} of ${sub} (full saved)`;
883 912 break;
885 914 case "json":
886 - llmDesc = `${fmtChars(chars)} of formatted JSON`;
915 + llmDesc = `${fmtChars(visibleChars)} of formatted JSON (full saved)`;
887 916 break;
888 917 case "pdf":
889 - llmDesc = `${fmtChars(chars)} of extracted text (pdftotext)`;
918 + llmDesc = `${fmtChars(visibleChars)} of extracted text (full saved)`;
890 919 break;
891 920 case "office":
892 - llmDesc = `${fmtChars(chars)} of extracted text (pandoc from ${(details?.officeFormat ?? "document").toUpperCase()})`;
921 + llmDesc = `${fmtChars(visibleChars)} of extracted text (full saved)`;
893 922 break;
894 923 case "image":
895 924 llmDesc = `image inline (${details?.imageInfo?.format ?? ct.split("/")[1] ?? "?"}) + ${fmtChars(chars)} metadata text`;
896 925 break;
897 926 case "media": {
898 927 const dur = details?.mediaInfo?.duration;
899 - llmDesc = `${fmtChars(chars)} of ffprobe JSON metadata${dur ? ` (${dur})` : ""}`;
928 + llmDesc = `${fmtChars(visibleChars)} of ffprobe JSON metadata${dur ? ` (${dur})` : ""}`;
900 929 break;
902 931 case "binary":
903 932 llmDesc = "file path + content-type only (binary, not decoded)";
904 933 break;
905 934 default:
906 - llmDesc = `${fmtChars(chars)} of text`;
935 + llmDesc = `${fmtChars(visibleChars)} of text (full saved)`;
910 - header += `\n ${theme.fg("muted", "\u21B3 LLM sees:")} ${theme.fg("dim", llmDesc)}`;
939 + header += `\n ${theme.fg("muted", "\u21B3 Agent sees:")} ${theme.fg("dim", llmDesc)}`;
940 + if (responseId)
941 + header += `\n ${theme.fg("muted", "\u21B3 Saved:")} ${theme.fg("dim", responseId)}`;
912 943 const mdTheme = getMarkdownTheme();
913 944 const box = new Box(1, 0, (t) => theme.bg("toolSuccessBg", t));
963 994 async execute(_toolCallId, params, signal, onUpdate, ctx) {
964 995 const format = params.format ?? "markdown";
965 - const maxChars = params.maxChars ?? DEFAULT_MAX_CHARS;
996 + const maxChars = Math.min(
997 + params.maxChars ?? DEFAULT_MAX_CHARS,
998 + DEFAULT_MAX_CHARS,
999 + );
967 1001 const setStatus = (msg: string) => ctx?.ui?.setStatus("web", msg);
968 1002 const clearStatus = () => ctx?.ui?.setStatus("web", undefined);
978 1012 // ── Fetch ────────────────────────────────────────────────────
979 1013 let body: Buffer;
980 1014 let contentType: string;
1015 + let finalUrl = params.url;
982 - const result = await fetchUrlStream(params.url, onProgress, signal);
1017 + const result = await fetchUrlStream(params.url, onProgress, signal, {
1018 + maxBytes: params.maxBytes,
1019 + timeoutMs: params.timeout,
1020 + });
983 1021 body = result.body;
984 1022 contentType = result.contentType;
1023 + finalUrl = result.finalUrl;
985 1024 } catch (err) {
986 1025 clearStatus();
987 1026 return errorFetchResult(
1002 1041 `[${ctShort} | ${fmtSize(bodyLen)} | \u2192 ${conversion}]`;
1005 - if (format === "raw") {
1006 - const output = truncateText(body.toString("utf-8"), maxChars);
1007 - clearStatus();
1008 - return {
1009 - content: [
1011 - type: "text",
1012 - text: wrapUntrusted(
1013 - `${params.url}\n\n${output}`,
1014 - spotlightMarkers,
1015 - ),
1016 - },
1017 - ],
1018 - details: {
1019 - url: params.url,
1020 - kind,
1021 - format,
1022 - chars: output.length,
1023 - contentType,
1024 - bodySize: bodyLen,
1025 - },
1026 - };
1029 - const processed = await convertContent({
1030 - kind,
1031 - body,
1032 - contentType,
1044 + const processed =
1045 + format === "raw"
1046 + ? { output: body.toString("utf-8") }
1047 + : await convertContent({
1048 + kind,
1049 + body,
1050 + contentType,
1051 + url: finalUrl,
1052 + format,
1053 + maxChars,
1054 + ctShort,
1055 + officeFormat,
1056 + headerLine,
1057 + });
1058 + const fullOutput = processed.output;
1059 + const stored = await saveJsonResult("web", ctx?.cwd ?? process.cwd(), {
1033 1060 url: params.url,
1061 + finalUrl,
1062 + kind,
1034 1063 format,
1035 - maxChars,
1036 - ctShort,
1037 - officeFormat,
1038 - headerLine,
1064 + contentType,
1065 + bodySize: bodyLen,
1066 + output: fullOutput,
1067 + ...(processed.tmpPath && { tmpPath: processed.tmpPath }),
1068 + ...(officeFormat && { officeFormat }),
1041 - if (processed.earlyReturn) {
1042 - clearStatus();
1043 - return { ...processed.earlyReturn };
1046 - const output = truncateText(processed.output, maxChars);
1070 + const matchSource =
1071 + "preview" in processed && processed.preview
1072 + ? processed.preview
1073 + : fullOutput;
1074 + const matched = params.linesMatching?.length
1075 + ? filterLineMatches(matchSource, {
1076 + needles: params.linesMatching,
1077 + contextLines: params.contextLines,
1078 + caseSensitive: params.caseSensitive,
1079 + maxChars,
1080 + })
1081 + : undefined;
1082 + const visible = matched
1083 + ? matched.text ||
1084 + `No lines matched: ${params.linesMatching.join(", ")}`
1085 + : truncateText(fullOutput, maxChars);
1047 1086 clearStatus();
1048 1087 return {
1049 1088 content: [
1051 1090 type: "text",
1052 1091 text: wrapUntrusted(
1053 - `${params.url}\n\n${output}`,
1092 + `${finalUrl}\n\n${visible}\n\nresponseId: ${stored.responseId}`,
1054 1093 spotlightMarkers,
1058 1097 details: {
1059 1098 url: params.url,
1099 + finalUrl,
1061 1101 format,
1062 - chars: output.length,
1102 + chars: fullOutput.length,
1103 + visibleChars: visible.length,
1104 + truncated: visible.length < fullOutput.length,
1063 1105 contentType,
1064 1106 bodySize: bodyLen,
1107 + responseId: stored.responseId,
1108 + fullOutputPath: stored.fullOutputPath,
1109 + storedBytes: stored.byteLength,
1065 1110 ...(processed.tmpPath && { tmpPath: processed.tmpPath }),
1066 1111 ...(officeFormat && { officeFormat }),
1067 - ...(processed.preview && { preview: processed.preview }),
1112 + ...(processed.meta && { meta: processed.meta }),
1113 + ...(matched && {
1114 + lineMatches: {
1115 + matchCount: matched.matchCount,
1116 + truncated: matched.truncated,
1117 + },
1118 + }),
1070 1121 } catch (err) {
1128 + // ── web_map ────────────────────────────────────────────────────────────
1130 + pi.registerTool({
1131 + name: "web_map",
1132 + label: "web_map",
1133 + description: `Inventory URLs from robots.txt, sitemap.xml, and llms.txt without fetching page bodies.
1135 +Parameters:
1136 +- url: Site URL
1137 +- maxUrls: Maximum URLs to keep (default 1000)
1138 +- maxSitemaps: Maximum sitemap files to inspect (default 20)`,
1139 + promptSnippet:
1140 + "web_map(url, maxUrls?, maxSitemaps?): discover site URLs without fetching page bodies",
1141 + parameters: Type.Object({
1142 + url: Type.String({ description: "Site URL" }),
1143 + maxUrls: Type.Optional(
1144 + Type.Number({
1145 + description: "Maximum URLs to keep",
1146 + minimum: 1,
1147 + maximum: 10_000,
1148 + }),
1149 + ),
1150 + maxSitemaps: Type.Optional(
1151 + Type.Number({
1152 + description: "Maximum sitemaps to inspect",
1153 + minimum: 1,
1154 + maximum: 100,
1155 + }),
1156 + ),
1157 + maxChars: Type.Optional(
1158 + Type.Number({
1159 + description:
1160 + "Limit text returned to the agent; full URL map is saved",
1161 + minimum: 100,
1162 + maximum: 100_000,
1163 + }),
1164 + ),
1165 + }),
1166 + renderCall(args, theme) {
1167 + let text = theme.fg("toolTitle", theme.bold("🗺 web_map "));
1168 + text += theme.fg("accent", safeDisplayText(args.url));
1169 + return new Text(text, 0, 0);
1170 + },
1171 + renderResult(result, { expanded, isPartial }, theme) {
1172 + if (isPartial)
1173 + return hintBox(theme, "warning", "🗺 Mapping site metadata…");
1174 + const details = result.details as
1175 + | {
1176 + urlCount?: number;
1177 + sitemapCount?: number;
1178 + responseId?: string;
1179 + fullOutputPath?: string;
1181 + | undefined;
1182 + if (result.isError) {
1183 + const content = result.content[0];
1184 + const msg = content?.type === "text" ? content.text : "Map failed";
1185 + return hintBox(theme, "error", `✗ ${safeDisplayText(msg)}`);
1187 + let text = theme.fg("success", "✓ ");
1188 + text += theme.fg("accent", `${details?.urlCount ?? 0} URL(s)`);
1189 + text += theme.fg(
1190 + "dim",
1191 + ` · ${details?.sitemapCount ?? 0} sitemap candidate(s)`,
1192 + );
1193 + if (details?.responseId)
1194 + text += theme.fg("dim", ` · saved ${details.responseId}`);
1195 + if (expanded && details?.fullOutputPath) {
1196 + text += `\n ${theme.fg("muted", "full:")} ${theme.fg("dim", details.fullOutputPath)}`;
1198 + return new Text(text, 0, 0);
1199 + },
1200 + async execute(_toolCallId, params, signal, onUpdate, ctx) {
1201 + onUpdate?.({
1202 + content: [{ type: "text", text: `Mapping ${params.url}…` }],
1203 + });
1204 + try {
1205 + const map = await discoverSiteMap(
1206 + params.url,
1207 + { maxUrls: params.maxUrls, maxSitemaps: params.maxSitemaps },
1208 + signal,
1209 + );
1210 + const stored = await saveJsonResult(
1211 + "web",
1212 + ctx?.cwd ?? process.cwd(),
1213 + map,
1214 + );
1215 + const listed = map.urls
1216 + .slice(0, 50)
1217 + .map((entry) => `- ${entry.url}`)
1218 + .join("\n");
1219 + const body = truncateText(
1220 + `Mapped ${map.urls.length} URL(s) from ${map.sitemaps.length} sitemap candidate(s).\n\n${listed}`,
1221 + params.maxChars ?? 8_000,
1222 + );
1223 + return {
1224 + content: [
1226 + type: "text",
1227 + text: wrapUntrusted(
1228 + `${body}\n\nresponseId: ${stored.responseId}`,
1229 + spotlightMarkers,
1230 + ),
1231 + },
1232 + ],
1233 + details: {
1234 + url: params.url,
1235 + kind: "map",
1236 + urlCount: map.urls.length,
1237 + sitemapCount: map.sitemaps.length,
1238 + responseId: stored.responseId,
1239 + fullOutputPath: stored.fullOutputPath,
1240 + },
1241 + };
1242 + } catch (err) {
1243 + return errorFetchResult(params.url, (err as Error).message);
1245 + },
1246 + });
1077 1248 // ── web_search (Codex → Z.AI → MiniMax → DDG) ─────────────────────────
1079 1250 pi.registerTool({

extensions/web/line-match.ts

Added
new file mode 100644
1 +export interface LineMatchOptions {
2 + needles: string[];
3 + contextLines?: number;
4 + caseSensitive?: boolean;
5 + maxChars?: number;
6 +}
8 +export interface LineMatchResult {
9 + text: string;
10 + matchCount: number;
11 + truncated: boolean;
14 +export function filterLineMatches(
15 + text: string,
16 + options: LineMatchOptions,
17 +): LineMatchResult {
18 + const needles = options.needles.filter((x) => x.length > 0);
19 + if (needles.length === 0)
20 + return { text: "", matchCount: 0, truncated: false };
21 + const lines = text.split(/\r?\n/u);
22 + const haystack = options.caseSensitive
23 + ? lines
24 + : lines.map((x) => x.toLowerCase());
25 + const wanted = options.caseSensitive
26 + ? needles
27 + : needles.map((x) => x.toLowerCase());
28 + const context = Math.max(0, options.contextLines ?? 0);
29 + const ranges: Array<[number, number]> = [];
30 + let matchCount = 0;
31 + for (let i = 0; i < haystack.length; i++) {
32 + if (!wanted.some((needle) => haystack[i].includes(needle))) continue;
33 + matchCount++;
34 + const start = Math.max(0, i - context);
35 + const end = Math.min(lines.length - 1, i + context);
36 + const last = ranges[ranges.length - 1];
37 + if (last && start <= last[1] + 1) last[1] = Math.max(last[1], end);
38 + else ranges.push([start, end]);
39 + }
40 + if (matchCount === 0) return { text: "", matchCount: 0, truncated: false };
41 + const out: string[] = [];
42 + for (const [start, end] of ranges) {
43 + if (out.length > 0) out.push("--");
44 + for (let i = start; i <= end; i++) out.push(`${i + 1}: ${lines[i]}`);
45 + }
46 + const full = out.join("\n");
47 + const max = options.maxChars ?? 8_000;
48 + if (full.length <= max) return { text: full, matchCount, truncated: false };
49 + return {
50 + text: `${full.slice(0, max)}\n… (matches truncated)`,
51 + matchCount,
52 + truncated: true,
53 + };

extensions/web/map.ts

Added
new file mode 100644
1 +import { fetchUrlStream } from "./fetch.js";
3 +export interface MapUrlEntry {
4 + url: string;
5 + source: "sitemap" | "llms";
6 + sourceUrl: string;
7 +}
9 +export interface SiteMapResult {
10 + seedUrl: string;
11 + urls: MapUrlEntry[];
12 + sitemaps: string[];
13 + tree: Record<string, string[]>;
16 +export interface DiscoverSiteMapOptions {
17 + maxUrls?: number;
18 + maxSitemaps?: number;
19 + fetchText?: (
20 + url: string,
21 + signal?: AbortSignal,
22 + ) => Promise<string | undefined>;
25 +export async function discoverSiteMap(
26 + seed: string,
27 + options: DiscoverSiteMapOptions = {},
28 + signal?: AbortSignal,
29 +): Promise<SiteMapResult> {
30 + const seedUrl = new URL(seed).toString();
31 + const fetchText = options.fetchText ?? defaultFetchText;
32 + const urls = new Map<string, MapUrlEntry>();
33 + const sitemapSet = new Set<string>();
34 + const robotsUrl = siteUrl(seedUrl, "/robots.txt");
35 + const robots = await fetchText(robotsUrl, signal).catch(() => undefined);
36 + for (const sitemap of parseRobotsSitemaps(robots ?? ""))
37 + sitemapSet.add(sitemap);
38 + sitemapSet.add(siteUrl(seedUrl, "/sitemap.xml"));
40 + const sitemapQueue = [...sitemapSet];
41 + for (
42 + let i = 0;
43 + i < sitemapQueue.length && i < (options.maxSitemaps ?? 20);
44 + i++
45 + ) {
46 + const sitemapUrl = sitemapQueue[i];
47 + const xml = await fetchText(sitemapUrl, signal).catch(() => undefined);
48 + if (!xml) continue;
49 + for (const nested of parseSitemapIndexes(xml)) {
50 + if (!sitemapSet.has(nested)) {
51 + sitemapSet.add(nested);
52 + sitemapQueue.push(nested);
53 + }
54 + }
55 + for (const url of parseSitemapXml(xml)) {
56 + urls.set(url, { url, source: "sitemap", sourceUrl: sitemapUrl });
57 + if (urls.size >= (options.maxUrls ?? 1_000)) break;
58 + }
59 + }
61 + const llmsUrl = siteUrl(seedUrl, "/llms.txt");
62 + const llms = await fetchText(llmsUrl, signal).catch(() => undefined);
63 + for (const url of parseLlmsLinks(llms ?? "", llmsUrl)) {
64 + urls.set(url, { url, source: "llms", sourceUrl: llmsUrl });
65 + if (urls.size >= (options.maxUrls ?? 1_000)) break;
66 + }
68 + const entries = [...urls.values()].toSorted((a, b) =>
69 + a.url.localeCompare(b.url),
70 + );
71 + return {
72 + seedUrl,
73 + urls: entries,
74 + sitemaps: [...sitemapSet],
75 + tree: buildTree(entries),
76 + };
79 +export function parseRobotsSitemaps(text: string): string[] {
80 + return text
81 + .split(/\r?\n/u)
82 + .map((line) => line.match(/^\s*sitemap:\s*(\S+)\s*$/iu)?.[1])
83 + .filter((x): x is string => Boolean(x));
86 +export function parseSitemapXml(xml: string): string[] {
87 + return locs(xml).filter(
88 + (url) => !url.endsWith(".xml") && !url.endsWith(".xml.gz"),
89 + );
92 +function parseSitemapIndexes(xml: string): string[] {
93 + return locs(xml).filter(
94 + (url) => url.endsWith(".xml") || url.endsWith(".xml.gz"),
95 + );
98 +export function parseLlmsLinks(text: string, baseUrl: string): string[] {
99 + const out: string[] = [];
100 + for (const match of text.matchAll(/\[[^\]]*\]\(([^)]+)\)/gu)) {
101 + const raw = match[1]?.trim();
102 + if (!raw) continue;
103 + try {
104 + out.push(new URL(raw, baseUrl).toString());
105 + } catch {}
107 + return out;
110 +async function defaultFetchText(
111 + url: string,
112 + signal?: AbortSignal,
113 +): Promise<string | undefined> {
114 + const result = await fetchUrlStream(url, undefined, signal, {
115 + maxBytes: 2 * 1024 * 1024,
116 + });
117 + return result.body.toString("utf8");
120 +function locs(xml: string): string[] {
121 + return [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/giu)].map((x) =>
122 + decodeXml(x[1]),
123 + );
126 +function decodeXml(text: string): string {
127 + return text
128 + .replaceAll("&amp;", "&")
129 + .replaceAll("&lt;", "<")
130 + .replaceAll("&gt;", ">")
131 + .replaceAll("&quot;", '"')
132 + .replaceAll("&apos;", "'");
135 +function siteUrl(seed: string, pathname: string): string {
136 + const url = new URL(seed);
137 + url.pathname = pathname;
138 + url.search = "";
139 + url.hash = "";
140 + return url.toString();
143 +function buildTree(entries: MapUrlEntry[]): Record<string, string[]> {
144 + const tree: Record<string, string[]> = {};
145 + for (const entry of entries) {
146 + const section = new URL(entry.url).pathname.split("/").find(Boolean) ?? "/";
147 + const sectionUrls = tree[section] ?? [];
148 + sectionUrls.push(entry.url);
149 + tree[section] = sectionUrls;
151 + return tree;

extensions/web/safety.ts

Added
new file mode 100644
1 +import { lookup } from "node:dns/promises";
2 +import { isIP } from "node:net";
4 +export type DnsResolver = (hostname: string) => Promise<string[]>;
6 +export interface SafeUrlOptions {
7 + resolver?: DnsResolver;
8 + allowPrivateNetwork?: boolean;
9 +}
11 +export interface SafeUrlResult {
12 + url: URL;
13 + normalizedUrl: string;
14 + addresses: string[];
17 +const BLOCKED_SUFFIXES = [".local", ".localhost", ".internal"];
19 +export async function assertSafeHttpUrl(
20 + input: string | URL,
21 + options: SafeUrlOptions = {},
22 +): Promise<SafeUrlResult> {
23 + const url = new URL(input.toString().trim());
24 + if (url.protocol !== "http:" && url.protocol !== "https:") {
25 + throw new Error(
26 + `Only http: and https: URLs are supported, got ${url.protocol}`,
27 + );
28 + }
29 + validateHostname(url, options.allowPrivateNetwork === true);
30 + if (options.allowPrivateNetwork === true) {
31 + return { url, normalizedUrl: url.toString(), addresses: [] };
32 + }
33 + const hostname = stripIpv6Brackets(url.hostname.toLowerCase());
34 + if (isIP(hostname) !== 0)
35 + return { url, normalizedUrl: url.toString(), addresses: [hostname] };
36 + const addresses = options.resolver
37 + ? await options.resolver(hostname)
38 + : (await lookup(hostname, { all: true, verbatim: true })).map(
39 + (x) => x.address,
40 + );
41 + for (const address of addresses) {
42 + if (isPrivateOrReservedIp(address)) {
43 + throw new Error(
44 + `Hostname resolves to blocked private address: ${address}`,
45 + );
46 + }
47 + }
48 + return { url, normalizedUrl: url.toString(), addresses };
51 +export async function resolveRedirectUrl(
52 + fromUrl: string,
53 + location: string,
54 + options: SafeUrlOptions = {},
55 +): Promise<SafeUrlResult> {
56 + return assertSafeHttpUrl(new URL(location, fromUrl), options);
59 +function validateHostname(url: URL, allowPrivate: boolean): void {
60 + const hostname = stripIpv6Brackets(url.hostname.toLowerCase());
61 + if (!hostname) throw new Error("URL must include a hostname");
62 + if (
63 + hostname === "localhost" ||
64 + BLOCKED_SUFFIXES.some((x) => hostname.endsWith(x))
65 + ) {
66 + throw new Error(`Blocked private hostname: ${hostname}`);
67 + }
68 + if (!allowPrivate && isPrivateOrReservedIp(hostname)) {
69 + throw new Error(`Blocked private address: ${hostname}`);
70 + }
73 +export function isPrivateOrReservedIp(address: string): boolean {
74 + const host = stripIpv6Brackets(address.toLowerCase());
75 + const family = isIP(host);
76 + if (family === 4) return isPrivateOrReservedIpv4(host);
77 + if (family === 6) return isPrivateOrReservedIpv6(host);
78 + return false;
81 +function isPrivateOrReservedIpv4(address: string): boolean {
82 + const [a = 0, b = 0] = address.split(".").map((x) => Number.parseInt(x, 10));
83 + return (
84 + a === 0 ||
85 + a === 10 ||
86 + a === 127 ||
87 + (a === 169 && b === 254) ||
88 + (a === 172 && b >= 16 && b <= 31) ||
89 + (a === 192 && b === 168) ||
90 + (a === 100 && b >= 64 && b <= 127) ||
91 + a >= 224
92 + );
95 +function isPrivateOrReservedIpv6(address: string): boolean {
96 + const embedded = address.match(
97 + /(?:::ffff:)?(\d{1,3}(?:\.\d{1,3}){3})$/u,
98 + )?.[1];
99 + if (embedded) return isPrivateOrReservedIpv4(embedded);
100 + const first = hextet(address, 0);
101 + const second = hextet(address, 1);
102 + const third = hextet(address, 2);
103 + const fourth = hextet(address, 3);
104 + return (
105 + address === "::" ||
106 + address === "::1" ||
107 + (first & 0xff00) === 0x0000 ||
108 + (first === 0x0100 && second === 0 && third === 0 && fourth === 0) ||
109 + (first & 0xfe00) === 0xfc00 ||
110 + (first & 0xffc0) === 0xfe80 ||
111 + (first & 0xff00) === 0xff00 ||
112 + first === 0x2002 ||
113 + (first === 0x2001 && second === 0x0db8)
114 + );
117 +function hextet(address: string, index: number): number {
118 + return Number.parseInt(address.split(":")[index] || "0", 16) || 0;
121 +function stripIpv6Brackets(hostname: string): string {
122 + return hostname.startsWith("[") && hostname.endsWith("]")
123 + ? hostname.slice(1, -1)
124 + : hostname;

extensions/web/store.ts

Added
new file mode 100644
1 +import { createHash, randomUUID } from "node:crypto";
2 +import { mkdir, writeFile } from "node:fs/promises";
3 +import { homedir } from "node:os";
4 +import path from "node:path";
6 +export interface StateDirOptions {
7 + platform?: NodeJS.Platform;
8 + env?: NodeJS.ProcessEnv;
9 + home?: string;
12 +export interface SavedJsonResult {
13 + namespace: string;
14 + responseId: string;
15 + fullOutputPath: string;
16 + byteLength: number;
19 +export function platformStateDir(
20 + platform: NodeJS.Platform = process.platform,
21 + env: NodeJS.ProcessEnv = process.env,
22 + home = homedir(),
23 +): string {
24 + const paths = platform === "win32" ? path.win32 : path.posix;
25 + if (platform === "win32")
26 + return paths.join(
27 + env.LOCALAPPDATA ?? paths.join(home, "AppData", "Local"),
28 + "pi",
29 + );
30 + if (env.XDG_STATE_HOME) return paths.join(env.XDG_STATE_HOME, "pi");
31 + if (platform === "darwin")
32 + return paths.join(home, "Library", "Application Support", "pi");
33 + return paths.join(home, ".local", "state", "pi");
36 +export function projectStoreDir(
37 + namespace: string,
38 + cwd: string,
39 + options: StateDirOptions = {},
40 +): string {
41 + return path.join(
42 + platformStateDir(options.platform, options.env, options.home),
43 + namespace,
44 + hash(cwd),
45 + );
48 +export async function saveJsonResult(
49 + namespace: string,
50 + cwd: string,
51 + value: unknown,
52 + options: StateDirOptions = {},
53 +): Promise<SavedJsonResult> {
54 + const responseId = randomUUID();
55 + const dir = path.join(projectStoreDir(namespace, cwd, options), "results");
56 + await mkdir(dir, { recursive: true, mode: 0o700 });
57 + const fullOutputPath = path.join(dir, `${responseId}.json`);
58 + const body = `${JSON.stringify(value, null, 2)}\n`;
59 + await writeFile(fullOutputPath, body, { mode: 0o600 });
60 + return {
61 + namespace,
62 + responseId,
63 + fullOutputPath,
64 + byteLength: Buffer.byteLength(body),
65 + };
68 +function hash(input: string): string {
69 + return createHash("sha256").update(input).digest("hex");