repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by admin

fix(web): align fetch output limit

734bbd0ceff17fc2dd7dbf11a0c93d4ac7a9c327

Oliver Jan Krylow <oliver@bugabinga.net> ·

SSH signature · not checked

Browse filesCompare with HEAD

Changed files (4)

extensions/web/README.md

Modified
25 25 - `format`:
26 26 optional; `markdown` default, or `text`, `html`, `json`, `raw`.
27 27 - `maxChars`:
28 - optional; default/max `20000`, min `100`.
28 + optional; default `20000`, max `100000`, min `100`.
29 29 - `timeout`:
30 30 optional; default `120000` ms, min `5000`, max `600000`.
31 31 Passed to `fetchUrlStream` alongside the Pi/tool abort signal.
41 41 - `maxSitemaps`:
42 42 optional; default `20`, min `1`, max `100`.
43 43 - `maxChars`:
44 - optional; default `8000`, min `100`, max `100000`.
44 + optional; default `8000`, max `100000`, min `100`.
45 45 - `search`
46 46 - `query`:
47 47 search query.

extensions/web/__tests__/harness.test.ts

Modified
11 11 } from "vitest";
12 12 import { createTestSession, type TestSession } from "../../../test/harness";
13 -import { DEFAULT_MAX_CHARS, DEFAULT_TIMEOUT_MS } from "../constants";
13 +import { DEFAULT_TIMEOUT_MS, MAX_VISIBLE_CHARS } from "../constants";
14 14 import webExtension from "../index";
15 15 import * as search from "../search";
87 87 expect(props.format.description).toContain("raw skips conversion");
88 88 expect(props.linesMatching.description).toContain("Literal substrings");
89 89 expect(props.contextLines.description).toContain("max 100");
90 - expect(props.maxChars.description).toContain(
91 - `default/max ${DEFAULT_MAX_CHARS}`,
92 - );
90 + expect(props.maxChars.description).toContain(`max ${MAX_VISIBLE_CHARS}`);
93 91 expect(props.timeout.description).toContain(
94 92 `default ${DEFAULT_TIMEOUT_MS}`,
139 137 it.each([
140 - ["maxChars", 100, 100_000],
138 + ["maxChars", 100, MAX_VISIBLE_CHARS],
141 139 ["maxBytes", 1000, 100_000_000],
142 140 ["contextLines", 0, 100],
143 141 ["timeout", 5000, 600_000],
165 163 action: "fetch",
166 164 url: "https://example.com",
167 - maxChars: DEFAULT_MAX_CHARS + 1,
165 + maxChars: MAX_VISIBLE_CHARS + 1,
169 167 { action: "search", query: "docs", format: "markdown" },
170 168 { action: "fetch", url: "https://example.com", count: 5 },

extensions/web/constants.ts

Modified
3 3 // Shared constants and types
5 5 export const DEFAULT_TIMEOUT_MS = 120_000; // 2 minutes for large downloads
6 6 export const DEFAULT_MAX_BYTES = 50 * 1024 * 1024;
7 7 export const DEFAULT_MAX_CHARS = 20_000;
8 +export const MAX_VISIBLE_CHARS = 100_000;
8 9 export const DDG_HTML_URL = "https://html.duckduckgo.com/html/";
10 11 export const USER_AGENT =

extensions/web/index.ts

Modified
30 30 import {
31 31 DEFAULT_MAX_CHARS,
32 32 DEFAULT_TIMEOUT_MS,
33 + MAX_VISIBLE_CHARS,
33 34 type SearchResult,
34 35 } from "./constants.js";
35 36 import { lazy } from "./lazy.js";
799 800 maxChars: Type.Optional(
800 801 Type.Number({
801 - description: `Visible characters (default ${DEFAULT_MAX_CHARS}).`,
802 + description: `Visible characters (default ${DEFAULT_MAX_CHARS}, max ${MAX_VISIBLE_CHARS}).`,
802 803 minimum: 100,
803 - maximum: DEFAULT_MAX_CHARS,
804 + maximum: MAX_VISIBLE_CHARS,
806 807 maxBytes: Type.Optional(
1020 1021 const format = params.format ?? "markdown";
1021 1022 const maxChars = Math.min(
1022 1023 params.maxChars ?? DEFAULT_MAX_CHARS,
1023 - DEFAULT_MAX_CHARS,
1024 + MAX_VISIBLE_CHARS,
1026 1027 const setStatus = (msg: string) => ctx?.ui?.setStatus("web", msg);
1184 1185 maxChars: Type.Optional(
1185 1186 Type.Number({
1186 - description: "Visible characters (default 8000).",
1187 + description: `Visible characters (default 8000, max ${MAX_VISIBLE_CHARS}).`,
1187 1188 minimum: 100,
1188 - maximum: 100_000,
1189 + maximum: MAX_VISIBLE_CHARS,
1475 1476 maxChars: Type.Optional(
1476 1477 Type.Number({
1477 - description: `Fetch: default/max ${DEFAULT_MAX_CHARS}. Map: default 8000, max 100000.`,
1478 + description: `Fetch: default ${DEFAULT_MAX_CHARS}, max ${MAX_VISIBLE_CHARS}. Map: default 8000, max ${MAX_VISIBLE_CHARS}.`,
1478 1479 minimum: 100,
1479 - maximum: 100_000,
1480 + maximum: MAX_VISIBLE_CHARS,
1491 1492 const { Check, Errors } = await import("typebox/value");
1492 1493 const schema = { ...handler.parameters, additionalProperties: false };
1493 1494 if (!Check(schema, params)) {
1494 - throw new Error(
1495 - `Invalid web ${action} arguments: ${JSON.stringify([...Errors(schema, params)])}`,
1496 - );
1495 + const details = [...Errors(schema, params)]
1496 + .map((error) => {
1497 + const path = "path" in error ? error.path : "arguments";
1498 + const value =
1499 + "value" in error ? JSON.stringify(error.value) : "unknown";
1500 + return `${path} ${error.message} (received ${value})`;
1501 + })
1502 + .join("; ");
1503 + throw new Error(`Invalid web ${action} arguments: ${details}`);
1498 1505 return handler.execute(id, params, signal, onUpdate, ctx);