repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

fix web platform tool lookup

f3b9d6fde4b2e9b08134b41fda3f607392bb6cfe

Oliver Krylow <o.krylow@isp-insoft.de> ·

unsigned

Browse filesCompare with HEAD

Resolve platform tools through Node instead of Unix which.

Warn about missing required web tools and info-notify optional metadata tools.

Add fake PATH binary coverage for web_fetch conversion and spawn helpers.

Changed files (9)

AGENTS.md

Modified
42 42 escalate to a human.
43 43 Do not work around `mise` or add ad-hoc scripts.
45 +## Cross-platform Requirements
47 +Extensions must work on Windows, macOS, and Linux unless their README explicitly scopes them narrower.
48 +Do not rely on shell-specific helpers such as `which`, `bash`, `sed`, `grep`, or `/tmp` when Node APIs can do the job.
49 +If an extension uses platform tools, check their availability when the extension loads.
50 +Notify missing required tools with warning severity.
51 +Notify missing optional tools with info severity.
52 +Tests for platform-tool behavior must exercise the real extension/tool path with controlled fake binaries on `PATH`, not only mocked helpers.
45 54 ## Markdown Style
47 56 Use one sentence per line for prose.

CHANGELOG.md

Modified
76 76 - Add `mise run clean` for generated caches and scratch artifacts, clearing
77 77 `.demo/` contents while keeping the directory and removing `node_modules`,
78 78 `__pycache__`, `.ruff_cache`, and `extensions/diff-review/static/`
79 +- Make web extension tool availability checks cross-platform without requiring
80 + Unix `which`, handle Windows `.cmd`/`.bat` shims with spaces, avoid stale
81 + negative PATH cache entries, notify missing required/optional tools at the
82 + right severity, and add coverage
80 84 ## 0.0.3

extensions/llmiterate/README.md

Modified
48 48 ```sql
49 49 -- llmiterate: add index for lookup query
53 52 Rules:
85 84 ```json
87 - "llmiterate": {
88 - "commentPrefixes": {
89 - "foo": ["COMMENT"],
90 - "lua": ["--"]
91 - }
92 - }
86 + "llmiterate": {
87 + "commentPrefixes": {
88 + "foo": ["COMMENT"],
89 + "lua": ["--"]
90 + }
91 + }
171 170 ```json
173 - "llmiterate": {
174 - "enabled": true,
175 - "include": ["src/**/*.{ts,tsx,rs,java,md}", "docs/**/*.md"],
176 - "exclude": ["**/target/**", "**/dist/**"],
177 - "debounceMs": 3000,
178 - "maxFileBytes": 1000000,
179 - "timeoutMs": 600000,
180 - "maxWatchDirs": 10000,
181 - "notify": "errors",
182 - "commentPrefixes": {
183 - "foo": ["COMMENT"],
184 - "lua": ["--"]
185 - }
186 - }
172 + "llmiterate": {
173 + "enabled": true,
174 + "include": ["src/**/*.{ts,tsx,rs,java,md}", "docs/**/*.md"],
175 + "exclude": ["**/target/**", "**/dist/**"],
176 + "debounceMs": 3000,
177 + "maxFileBytes": 1000000,
178 + "timeoutMs": 600000,
179 + "maxWatchDirs": 10000,
180 + "notify": "errors",
181 + "commentPrefixes": {
182 + "foo": ["COMMENT"],
183 + "lua": ["--"]

extensions/web/README.md

Modified
59 59 - `pandoc`:
60 60 HTML conversion; DOCX/ODT/EPUB/RTF extraction.
61 + Missing binary emits a warning notification on session start.
61 62 - `pdftotext`:
62 63 PDF extraction.
64 + Missing binary emits a warning notification on session start.
63 65 - `identify`:
64 66 image metadata.
67 + Missing binary emits an info notification on session start.
65 68 Missing/failing binary returns fallback metadata text.
66 69 - `ffprobe`:
67 70 audio/video metadata.
71 + Missing binary emits an info notification on session start.
68 72 Missing/failing binary returns fallback metadata text.
70 74 Provider/env config read by source:

extensions/web/__tests__/spawn.test.ts

Added
new file mode 100644
1 +import { randomUUID } from "node:crypto";
2 +import { mkdtemp, writeFile } from "node:fs/promises";
3 +import { tmpdir } from "node:os";
4 +import path from "node:path";
5 +import { afterEach, describe, expect, it } from "vitest";
6 +import { checkToolAvailability, spawnToText } from "../spawn";
8 +const originalPath = process.env.PATH;
9 +const originalPathAlt = process.env.Path;
11 +afterEach(() => {
12 + if (originalPath === undefined) delete process.env.PATH;
13 + else process.env.PATH = originalPath;
14 + if (originalPathAlt === undefined) delete process.env.Path;
15 + else process.env.Path = originalPathAlt;
16 +});
18 +async function makeFakeTool(dir: string, name: string) {
19 + if (process.platform === "win32") {
20 + const toolPath = path.join(dir, `${name}.cmd`);
21 + await writeFile(toolPath, "@echo off\r\nexit /b 0\r\n");
22 + return toolPath;
23 + }
25 + const toolPath = path.join(dir, name);
26 + await writeFile(toolPath, "#!/bin/sh\nexit 0\n", { mode: 0o755 });
27 + return toolPath;
30 +describe("checkToolAvailability", () => {
31 + it("finds executables on PATH without requiring Unix which", async () => {
32 + const dir = await mkdtemp(path.join(tmpdir(), "pi-web-path-"));
33 + const name = `fake-tool-${randomUUID()}`;
34 + await makeFakeTool(dir, name);
35 + process.env.PATH = dir;
37 + await expect(checkToolAvailability(name, "missing")).resolves.toBeNull();
38 + });
40 + it("does not keep stale negative cache entries after PATH changes", async () => {
41 + const dir = await mkdtemp(path.join(tmpdir(), "pi-web-path-"));
42 + const emptyDir = await mkdtemp(path.join(tmpdir(), "pi-web-empty-path-"));
43 + const name = `fake-tool-${randomUUID()}`;
45 + process.env.PATH = emptyDir;
46 + await expect(checkToolAvailability(name, "missing")).resolves.toBe(
47 + "missing",
48 + );
50 + await makeFakeTool(dir, name);
51 + process.env.PATH = dir;
53 + await expect(checkToolAvailability(name, "missing")).resolves.toBeNull();
54 + });
56 + it("runs PATH-resolved tools without a shell-specific lookup utility", async () => {
57 + const dir = await mkdtemp(path.join(tmpdir(), "pi web path "));
58 + const name = `fake-tool-${randomUUID()}`;
59 + await makeFakeTool(dir, name);
60 + process.env.PATH = dir;
62 + await expect(spawnToText(name, ["arg with spaces"])).resolves.toBe("");
63 + });
64 +});

extensions/web/__tests__/web-fetch-contract.test.ts

Modified
1 -import { mkdtemp, readFile } from "node:fs/promises";
1 +import { mkdtemp, readFile, writeFile } from "node:fs/promises";
2 2 import { tmpdir } from "node:os";
3 3 import path from "node:path";
4 4 import { afterEach, describe, expect, it, vi } from "vitest";
21 21 const originalFetch = globalThis.fetch;
23 +async function makeFakeCommand(
24 + dir: string,
25 + name: string,
26 + stdout: string,
27 +): Promise<void> {
28 + if (process.platform === "win32") {
29 + await writeFile(
30 + path.join(dir, `${name}.cmd`),
31 + `@echo off\r\necho ${stdout}\r\n`,
32 + );
33 + return;
34 + }
35 + await writeFile(
36 + path.join(dir, name),
37 + `#!/bin/sh\nprintf '%s\\n' '${stdout}'\n`,
38 + {
39 + mode: 0o755,
40 + },
41 + );
44 +function pathEnv(dir: string): Record<string, string> {
45 + return process.platform === "win32"
46 + ? { PATH: dir, Path: dir, PATHEXT: ".COM;.EXE;.BAT;.CMD" }
47 + : { PATH: dir };
23 50 describe("web_fetch context contract", () => {
24 51 let t: TestSession | undefined;
25 52 afterEach(() => {
61 88 expect(saved.output).toContain("x".repeat(10_000));
91 + it("converts HTML with PATH-resolved pandoc and no Unix which", async () => {
92 + globalThis.fetch = vi.fn(
93 + async () =>
94 + new Response("<html><main><h1>Hello</h1></main></html>", {
95 + status: 200,
96 + headers: { "content-type": "text/html" },
97 + }),
98 + ) as typeof fetch;
100 + const binDir = await mkdtemp(path.join(tmpdir(), "pi-web-bin-"));
101 + await makeFakeCommand(binDir, "pandoc", "converted markdown");
102 + const webPath = path.resolve(import.meta.dirname, "../index.ts");
103 + t = await createTestSession({
104 + extensions: [webPath],
105 + env: pathEnv(binDir),
106 + });
107 + const [extension] = t.session.extensionRunner.extensions;
108 + const result = await extension.tools
109 + .get("web_fetch")
110 + .definition.execute(
111 + "test",
112 + { url: "https://93.184.216.34/html" },
113 + new AbortController().signal,
114 + undefined,
115 + { cwd: t.cwd },
116 + );
118 + expect(result.isError).not.toBe(true);
119 + expect(result.details.kind).toBe("html");
120 + expect(result.content[0].text).toContain("converted markdown");
121 + });
123 + it("notifies missing required and optional platform tools on session start", async () => {
124 + const emptyDir = await mkdtemp(path.join(tmpdir(), "pi-web-empty-bin-"));
125 + const webPath = path.resolve(import.meta.dirname, "../index.ts");
126 + t = await createTestSession({
127 + extensions: [webPath],
128 + env: pathEnv(emptyDir),
129 + });
130 + await new Promise((resolve) => setTimeout(resolve, 0));
132 + const notify = t.events.uiCallsFor("notify");
133 + expect(notify.map((call) => call.args)).toEqual(
134 + expect.arrayContaining([
135 + expect.arrayContaining([expect.stringContaining("pandoc"), "warning"]),
136 + expect.arrayContaining([
137 + expect.stringContaining("pdftotext"),
138 + "warning",
139 + ]),
140 + expect.arrayContaining([expect.stringContaining("identify"), "info"]),
141 + expect.arrayContaining([expect.stringContaining("ffprobe"), "info"]),
142 + ]),
143 + );
144 + });
64 146 it("line matching returns excerpts instead of full page", async () => {
65 147 globalThis.fetch = vi.fn(
66 148 async () =>

extensions/web/index.ts

Modified
402 +const PLATFORM_TOOL_DEPENDENCIES = [
404 + cmd: "pandoc",
405 + hint: TOOL_HINTS.pandoc,
406 + level: "warning" as const,
407 + purpose: "HTML and Office document conversion unavailable",
408 + },
410 + cmd: "pdftotext",
411 + hint: TOOL_HINTS.pdftotext,
412 + level: "warning" as const,
413 + purpose: "PDF extraction unavailable",
414 + },
416 + cmd: "identify",
417 + hint: TOOL_HINTS.identify,
418 + level: "info" as const,
419 + purpose: "image metadata will use fallback text",
420 + },
422 + cmd: "ffprobe",
423 + hint: TOOL_HINTS.ffprobe,
424 + level: "info" as const,
425 + purpose: "audio/video metadata will use fallback text",
426 + },
429 +async function notifyMissingPlatformTools(
430 + ctx: ExtensionContext,
431 +): Promise<void> {
432 + if (!ctx.hasUI) return;
433 + for (const dep of PLATFORM_TOOL_DEPENDENCIES) {
434 + const hint = await checkToolAvailability(dep.cmd, dep.hint);
435 + if (hint) ctx.ui.notify(`web: ${dep.purpose}. ${hint}`, dep.level);
402 439 async function processBinary(
403 440 body: Buffer,
404 441 contentType: string,
721 758 // ── Session-scoped spotlight markers ───────────────────────────────────
722 759 const spotlightMarkers: SpotlightMarkers = createMarkers();
761 + pi.on("session_start", async (_event, ctx) => {
762 + await notifyMissingPlatformTools(ctx);
763 + });
724 765 // ── Cleanup temp files on session shutdown ─────────────────────────────
726 767 pi.on("session_shutdown", async () => {

extensions/web/pandoc.ts

Modified
2 2 // Pandoc HTML→markdown conversion with LLM-friendly content extraction
3 3 // Uses a lua filter to strip nav/sidebar/ads and fix code block languages
4 4 // Then strips residual raw-HTML tags from commonmark output
6 -import { spawn } from "node:child_process";
7 6 import { dirname, join } from "node:path";
8 7 import { fileURLToPath } from "node:url";
9 8 import { spawnToText } from "./spawn.js";
46 -export function runPandoc(
45 +export async function runPandoc(
47 46 html: string,
48 47 format: string,
49 48 useFilter = true,
50 49 ): Promise<string> {
51 - return new Promise((resolve, reject) => {
52 - const toFormat =
53 - format === "text"
54 - ? "plain"
55 - : format === "html"
56 - ? "html"
57 - : format === "json"
58 - ? "json"
59 - : "markdown-raw_html";
50 + const toFormat =
51 + format === "text"
52 + ? "plain"
53 + : format === "html"
54 + ? "html"
55 + : format === "json"
56 + ? "json"
57 + : "markdown-raw_html";
61 - const args = ["--from=html", "--to=" + toFormat, "--wrap=none"];
59 + const args = ["--from=html", "--to=" + toFormat, "--wrap=none"];
63 - if (useFilter) {
64 - args.push("--lua-filter=" + LUA_FILTER);
65 - }
61 + if (useFilter) {
62 + args.push("--lua-filter=" + LUA_FILTER);
63 + }
67 - const proc = spawn("pandoc", args);
69 - let stdout = "";
70 - let stderr = "";
72 - proc.stdout.on("data", (d: Buffer) => (stdout += d));
73 - proc.stderr.on("data", (d: Buffer) => (stderr += d));
74 - proc.on("close", (code) => {
75 - if (code === 0) {
76 - resolve(
77 - format === "html" || format === "json"
78 - ? stdout
79 - : stripResidualTags(stdout),
80 - );
81 - } else {
82 - reject(new Error(stderr || `pandoc: ${code}`));
83 - }
84 - });
85 - proc.on("error", reject);
87 - proc.stdin.write(cleanTableClasses(html));
88 - proc.stdin.end();
89 - });
65 + const stdout = await spawnToText("pandoc", args, cleanTableClasses(html));
66 + return format === "html" || format === "json"
67 + ? stdout
68 + : stripResidualTags(stdout);
92 71 // Convert non-HTML formats via pandoc (docx, odt, epub, rtf, etc.)

extensions/web/spawn.ts

Modified
1 1 // Shared spawn helper for running external commands
3 3 import { spawn } from "node:child_process";
4 +import { constants } from "node:fs";
5 +import { access } from "node:fs/promises";
6 +import { basename, delimiter, dirname, extname, join } from "node:path";
5 -export function spawnToText(
8 +export async function spawnToText(
6 9 cmd: string,
7 10 args: string[],
8 11 input?: string,
9 12 ): Promise<string> {
13 + const resolved = (await resolveCommand(cmd)) ?? cmd;
14 + const spawnArgs = commandArgs(resolved, args);
10 15 return new Promise((resolve, reject) => {
11 - const proc = spawn(cmd, args);
16 + const proc = spawn(spawnArgs.command, spawnArgs.args, {
17 + windowsHide: true,
18 + ...spawnArgs.options,
19 + });
12 20 let stdout = "";
13 21 let stderr = "";
22 + let settled = false;
23 + const fail = (err: Error) => {
24 + if (settled) return;
25 + settled = true;
26 + reject(err);
27 + };
14 28 proc.stdout.on("data", (d: Buffer) => (stdout += d));
15 29 proc.stderr.on("data", (d: Buffer) => (stderr += d));
16 30 proc.on("close", (code) => {
31 + if (settled) return;
32 + settled = true;
17 33 if (code === 0) resolve(stdout);
18 34 else reject(new Error(stderr || `${cmd}: ${code}`));
20 - proc.on("error", reject);
36 + proc.on("error", fail);
21 37 if (input !== undefined) {
22 38 proc.stdin.write(input);
23 39 proc.stdin.end();
29 45 * Check if a command is available on PATH.
30 46 * Returns null if available, or an install hint string if not.
32 -const availableCommands = new Map<string, string | null>();
48 +const availableCommands = new Set<string>();
50 +function pathEnv(): string {
51 + const key = Object.keys(process.env).find((k) => k.toLowerCase() === "path");
52 + return key ? (process.env[key] ?? "") : "";
55 +function stripPathQuotes(dir: string): string {
56 + return dir.length >= 2 && dir.startsWith('"') && dir.endsWith('"')
57 + ? dir.slice(1, -1)
58 + : dir;
61 +function pathextEnv(): string[] {
62 + if (process.platform !== "win32") return [""];
63 + const raw = process.env.PATHEXT || ".COM;.EXE;.BAT;.CMD";
64 + return raw
65 + .split(";")
66 + .map((ext) => ext.trim())
67 + .filter(Boolean);
70 +function candidateNames(cmd: string): string[] {
71 + if (process.platform !== "win32" || extname(cmd)) return [cmd];
72 + return [
73 + cmd,
74 + ...pathextEnv().flatMap((ext) => [cmd + ext, cmd + ext.toLowerCase()]),
75 + ];
78 +async function isExecutable(filePath: string): Promise<boolean> {
79 + try {
80 + await access(
81 + filePath,
82 + process.platform === "win32" ? constants.F_OK : constants.X_OK,
83 + );
84 + return true;
85 + } catch {
86 + return false;
87 + }
90 +export async function resolveCommand(cmd: string): Promise<string | null> {
91 + const hasPathSeparator = cmd.includes("/") || cmd.includes("\\");
92 + if (hasPathSeparator) {
93 + const dir = dirname(cmd);
94 + for (const name of candidateNames(basename(cmd))) {
95 + const candidate = join(dir, name);
96 + if (await isExecutable(candidate)) return candidate;
97 + }
98 + return null;
99 + }
101 + const names = candidateNames(cmd);
102 + for (const rawDir of pathEnv().split(delimiter).filter(Boolean)) {
103 + const dir = stripPathQuotes(rawDir);
104 + for (const name of names) {
105 + const candidate = join(dir, name);
106 + if (await isExecutable(candidate)) return candidate;
109 + return null;
112 +function commandArgs(
113 + resolved: string,
114 + args: string[],
115 +): {
116 + command: string;
117 + args: string[];
118 + options?: { windowsVerbatimArguments: boolean };
119 +} {
120 + if (process.platform === "win32" && /\.(?:cmd|bat)$/i.test(resolved)) {
121 + return {
122 + command: process.env.ComSpec || process.env.COMSPEC || "cmd.exe",
123 + args: [
124 + "/d",
125 + "/s",
126 + "/c",
127 + `"${[quoteCmdArg(resolved), ...args.map(quoteCmdArg)].join(" ")}"`,
128 + ],
129 + options: { windowsVerbatimArguments: true },
130 + };
132 + return { command: resolved, args };
135 +function quoteCmdArg(arg: string): string {
136 + return `"${arg.replace(/"/g, '""')}"`;
34 139 export async function checkToolAvailability(
35 140 cmd: string,
36 141 installHint: string,
37 142 ): Promise<string | null> {
38 - const cached = availableCommands.get(cmd);
39 - if (cached !== undefined) return cached;
143 + const cacheKey = `${cmd}\0${pathEnv()}\0${process.env.PATHEXT ?? ""}`;
144 + if (availableCommands.has(cacheKey)) return null;
41 - try {
42 - await spawnToText("which", [cmd]);
43 - availableCommands.set(cmd, null);
146 + if (await resolveCommand(cmd)) {
147 + availableCommands.add(cacheKey);
44 148 return null;
45 - } catch {
46 - availableCommands.set(cmd, installHint);
47 - return installHint;
150 + return installHint;
51 153 // Tool install hints