39 39 } from
"./constants.js";
40 40 import { extractMainContent }
from "./extract.js";
41 41 import { fetchUrlStream }
from "./fetch.js";
42 +import { filterLineMatches }
from "./line-match.js";
43 +import { discoverSiteMap }
from "./map.js";
42 44 import { minimaxUnderstandImage }
from "./minimax.js";
43 45 import { runPandoc, runPandocFile }
from "./pandoc.js";
44 46 import { searchWeb }
from "./search.js";
48 50 type SpotlightMarkers,
50 52 } from
"./spotlight.js";
53 +import { saveJsonResult }
from "./store.js";
473 476 contentType: string,
476 479 headerLine: (s
: string)
=> string,
477 480 ):
Promise<ContentOutput
> {
478 481 const ext
= extensionFromContentType(contentType, url);
491 494 const fullOutput
= `${headerLine(`ffprobe JSON (${desc})`)}\n${desc}\n\n${meta.raw}`;
493 - fullOutput.
length > maxChars
494 - ? fullOutput.
slice(
0, maxChars)
+ "\n\u2026 (truncated)" 496 + output: fullOutput,
502 - content: [{ type:
"text", text:
`${url}\n\n${output}` }],
501 + content: [{ type:
"text", text:
`${url}\n\n${fullOutput}` }],
755 754 - url: URL to fetch
756 755 - format: Output
format (markdown, text, html, json, raw)
757 -- maxChars: Truncate output to this many
characters`, 756 +- maxChars: Limit text returned to the
agent (full output is saved)
757 +- maxBytes: Refuse responses larger than this
758 +- linesMatching
/contextLines: Return matching excerpts instead
of full
output`, 760 - "web_fetch(url, format?, maxChars?): fetch URL and convert to readable format", 761 + "web_fetch(url, format?, maxChars?, linesMatching?): fetch URL; keep agent context capped; full output saved", 762 763 parameters: Type.Object({ 763 764 url: Type.String({ description: "URL to fetch" }), 779 780 maxChars: Type.
Optional(
781 - description:
"Truncate output to this many characters",
783 + "Limit text returned to the agent; full converted output is saved",
783 - maximum:
1_000_000,
785 + maximum: DEFAULT_MAX_CHARS,
788 + maxBytes: Type.
Optional(
790 + description:
"Refuse responses larger than this many bytes",
792 + maximum:
100_000_000,
795 + linesMatching: Type.
Optional(
796 + Type.
Array(Type.
String(), {
797 + description:
"Return only lines matching these strings",
800 + contextLines: Type.
Optional(
802 + description:
"Lines of context around each line match",
807 + caseSensitive: Type.
Optional(
808 + Type.
Boolean({ description:
"Case-sensitive line matching" }),
786 810 timeout: Type.
Optional(
788 812 description:
`Timeout in milliseconds (default: ${DEFAULT_TIMEOUT_MS}ms = ${DEFAULT_TIMEOUT_MS / 1000 / 60}min)`,
834 858 const ct
= details?.contentType
?? "unknown";
835 859 const bodySize
= details?.bodySize
?? 0;
836 860 const chars
= details?.chars
?? 0;
861 + const visibleChars
= 862 + (details
as { visibleChars
?: number }
| undefined)?.visibleChars
?? 864 + const responseId
= (details
as { responseId
?: string }
| undefined)
837 866 const isRaw
= details?.format
=== "raw";
839 868 let header
= theme.
fg(
"success",
"\u2713 ");
858 887 let llmDesc
: string;
860 - llmDesc
= `${fmtChars(chars)} of raw content (no conversion)`;
889 + llmDesc
= `${fmtChars(visibleChars)} of raw content (no conversion)`;
871 900 ? "extracted HTML" 873 - llmDesc
= `${fmtChars(chars)} of ${typeLabel} (extracted via pandoc)`;
902 + llmDesc
= `${fmtChars(visibleChars)} of ${typeLabel} (full saved)`;
879 908 : ct.
includes(
"tsv")
882 - llmDesc
= `${fmtChars(chars)} of ${sub}`;
911 + llmDesc
= `${fmtChars(visibleChars)} of ${sub} (full saved)`;
886 - llmDesc
= `${fmtChars(chars)} of formatted JSON`;
915 + llmDesc
= `${fmtChars(visibleChars)} of formatted JSON (full saved)`;
889 - llmDesc
= `${fmtChars(chars)} of extracted text (pdftotext)`;
918 + llmDesc
= `${fmtChars(visibleChars)} of extracted text (full saved)`;
892 - llmDesc
= `${fmtChars(chars)} of extracted text (pandoc from ${(details?.officeFormat ?? "document").toUpperCase()})`;
921 + llmDesc
= `${fmtChars(visibleChars)} of extracted text (full saved)`;
895 924 llmDesc
= `image inline (${details?.imageInfo?.format ?? ct.split("/")[1] ?? "?"}) + ${fmtChars(chars)} metadata text`;
898 927 const dur
= details?.mediaInfo?.duration;
899 - llmDesc
= `${fmtChars(chars)} of ffprobe JSON metadata${dur ? ` (${dur})` : ""}`;
928 + llmDesc
= `${fmtChars(visibleChars)} of ffprobe JSON metadata${dur ? ` (${dur})` : ""}`;
903 932 llmDesc
= "file path + content-type only (binary, not decoded)";
906 - llmDesc
= `${fmtChars(chars)} of text`;
935 + llmDesc
= `${fmtChars(visibleChars)} of text (full saved)`;
910 - header
+= `\n ${theme.fg("muted", "\u21B3 LLM sees:")} ${theme.fg("dim", llmDesc)}`;
939 + header
+= `\n ${theme.fg("muted", "\u21B3 Agent sees:")} ${theme.fg("dim", llmDesc)}`;
941 + header
+= `\n ${theme.fg("muted", "\u21B3 Saved:")} ${theme.fg("dim", responseId)}`;
912 943 const mdTheme
= getMarkdownTheme();
913 944 const box
= new Box(
1,
0, (t)
=> theme.
bg(
"toolSuccessBg", t));
963 994 async
execute(_toolCallId, params, signal, onUpdate, ctx) {
964 995 const format
= params.format
?? "markdown";
965 - const maxChars
= params.maxChars
?? DEFAULT_MAX_CHARS;
996 + const maxChars
= Math.
min(
997 + params.maxChars
?? DEFAULT_MAX_CHARS,
967 1001 const setStatus = (msg
: string)
=> ctx?.ui?.
setStatus(
"web", msg);
968 1002 const clearStatus = ()
=> ctx?.ui?.
setStatus(
"web",
undefined);
979 1013 let body
: Buffer;
980 1014 let contentType
: string;
1015 + let finalUrl
= params.url;
982 - const result
= await fetchUrlStream(params.url, onProgress, signal);
1017 + const result
= await fetchUrlStream(params.url, onProgress, signal, {
1018 + maxBytes: params.maxBytes,
1019 + timeoutMs: params.timeout,
983 1021 body
= result.
body;
984 1022 contentType
= result.contentType;
1023 + finalUrl
= result.finalUrl;
987 1026 return errorFetchResult(
1002 1041 `[${ctShort} | ${fmtSize(bodyLen)} | \u2192 ${conversion}]`;
1005 - if (format
=== "raw") {
1006 - const output
= truncateText(body.
toString(
"utf-8"), maxChars);
1012 - text:
wrapUntrusted(
1013 - `${params.url}\n\n${output}`,
1022 - chars: output.
length,
1024 - bodySize: bodyLen,
1029 - const processed
= await convertContent({
1046 + ? { output: body.
toString(
"utf-8") }
1047 + : await convertContent({
1058 + const fullOutput
= processed.output;
1059 + const stored
= await saveJsonResult(
"web", ctx?.cwd
?? process.
cwd(), {
1033 1060 url: params.url,
1065 + bodySize: bodyLen,
1066 + output: fullOutput,
1067 + ...(processed.tmpPath
&& { tmpPath: processed.tmpPath }),
1068 + ...(officeFormat
&& { officeFormat }),
1041 - if (processed.earlyReturn) {
1043 - return {
...processed.earlyReturn };
1046 - const output
= truncateText(processed.output, maxChars);
1070 + const matchSource
= 1071 + "preview" in processed
&& processed.preview
1072 + ? processed.preview
1074 + const matched
= params.linesMatching?.
length 1075 + ? filterLineMatches(matchSource, {
1076 + needles: params.linesMatching,
1077 + contextLines: params.contextLines,
1078 + caseSensitive: params.caseSensitive,
1082 + const visible
= matched
1084 + `No lines matched: ${params.linesMatching.join(", ")}` 1085 + : truncateText(fullOutput, maxChars);
1052 1091 text:
wrapUntrusted(
1053 - `${params.url}\n\n${output}`,
1092 + `${finalUrl}\n\n${visible}\n\nresponseId: ${stored.responseId}`,
1054 1093 spotlightMarkers,
1059 1098 url: params.url,
1062 - chars: output.
length,
1102 + chars: fullOutput.
length,
1103 + visibleChars: visible.
length,
1104 + truncated: visible.
length < fullOutput.
length,
1064 1106 bodySize: bodyLen,
1107 + responseId: stored.responseId,
1108 + fullOutputPath: stored.fullOutputPath,
1109 + storedBytes: stored.byteLength,
1065 1110 ...(processed.tmpPath
&& { tmpPath: processed.tmpPath }),
1066 1111 ...(officeFormat
&& { officeFormat }),
1067 - ...(processed.preview
&& { preview: processed.preview }),
1112 + ...(processed.meta
&& { meta: processed.meta }),
1115 + matchCount: matched.matchCount,
1116 + truncated: matched.truncated,
1070 1121 }
catch (err) {
1133 + description:
`Inventory URLs from robots.txt, sitemap.xml, and llms.txt without fetching page bodies. 1137 +- maxUrls: Maximum URLs to keep (default 1000) 1138 +- maxSitemaps: Maximum sitemap files to inspect (default 20)`,
1140 + "web_map(url, maxUrls?, maxSitemaps?): discover site URLs without fetching page bodies",
1141 + parameters: Type.
Object({
1142 + url: Type.
String({ description:
"Site URL" }),
1143 + maxUrls: Type.
Optional(
1145 + description:
"Maximum URLs to keep",
1150 + maxSitemaps: Type.
Optional(
1152 + description:
"Maximum sitemaps to inspect",
1157 + maxChars: Type.
Optional(
1160 + "Limit text returned to the agent; full URL map is saved",
1166 + renderCall(args, theme) {
1167 + let text
= theme.
fg(
"toolTitle", theme.
bold(
"🗺 web_map "));
1168 + text
+= theme.
fg(
"accent",
safeDisplayText(args.url));
1169 + return new Text(text,
0,
0);
1171 + renderResult(result, { expanded, isPartial }, theme) {
1173 + return hintBox(theme,
"warning",
"🗺 Mapping site metadata…");
1174 + const details
= result.details as
1176 + urlCount
?: number;
1177 + sitemapCount
?: number;
1178 + responseId
?: string;
1179 + fullOutputPath
?: string;
1182 + if (result.isError) {
1183 + const content
= result.
content[
0];
1184 + const msg
= content?.
type === "text" ? content.
text : "Map failed";
1185 + return hintBox(theme,
"error",
`✗ ${safeDisplayText(msg)}`);
1187 + let text
= theme.
fg(
"success",
"✓ ");
1188 + text
+= theme.
fg(
"accent",
`${details?.urlCount ?? 0} URL(s)`);
1191 + ` · ${details?.sitemapCount ?? 0} sitemap candidate(s)`,
1193 + if (details?.responseId)
1194 + text
+= theme.
fg(
"dim",
` · saved ${details.responseId}`);
1195 + if (expanded
&& details?.fullOutputPath) {
1196 + text
+= `\n ${theme.fg("muted", "full:")} ${theme.fg("dim", details.fullOutputPath)}`;
1198 + return new Text(text,
0,
0);
1200 + async execute(_toolCallId, params, signal, onUpdate, ctx) {
1202 + content: [{ type:
"text", text:
`Mapping ${params.url}…` }],
1205 + const map
= await discoverSiteMap(
1207 + { maxUrls: params.maxUrls, maxSitemaps: params.maxSitemaps },
1210 + const stored
= await saveJsonResult(
1212 + ctx?.cwd
?? process.
cwd(),
1215 + const listed
= map.urls
1217 + .
map((entry)
=> `- ${entry.url}`)
1219 + const body
= truncateText(
1220 + `Mapped ${map.urls.length} URL(s) from ${map.sitemaps.length} sitemap candidate(s).\n\n${listed}`,
1221 + params.maxChars
?? 8_000,
1227 + text:
wrapUntrusted(
1228 + `${body}\n\nresponseId: ${stored.responseId}`,
1236 + urlCount: map.urls.
length,
1237 + sitemapCount: map.sitemaps.
length,
1238 + responseId: stored.responseId,
1239 + fullOutputPath: stored.fullOutputPath,
1243 + return errorFetchResult(params.url, (err
as Error).message);
1079 1250 pi.
registerTool({