repositories / dotfiles

dotfiles

bugabingas dorkfiles

owned by bugabinga

fix(ctx): collapse oversized bash output

f3b678aec88ecb82e0041c9f5c6ed62841e334c0

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (6)

pi/agent/extensions/ctx/README.md

Modified
1 1 # ctx
3 -Off-context execution and large-output search for Pi.
3 +Pi context blowup fuse for oversized `bash` output.
5 -Runs JS, Python, and shell analysis outside the main conversation and searches stored outputs.
5 +`ctx` does one thing:
7 -## Tools
7 +1. watches `bash` tool results
8 +2. if output is too large, stores full text off-context in SQLite FTS
9 +3. replaces visible output with a small `ctx:stored ...` pointer
10 +4. exposes `ctx` search for that stored output
9 -- `ctx`
12 +It does **not** run shell/JS/Python jobs. It does **not** inspect files. Use normal Pi tools for normal work.
11 -## Demo
14 +## Tool
13 -<!-- demo:tools_suite:start -->
14 -![Tools suite](assets/tools_suite.gif)
15 -<!-- demo:tools_suite:end -->
16 +### `ctx`
18 +Search output previously stored by ctx.
20 +```json
22 + "query": "TARGET_EVENT",
23 + "source": "bash:178810b1",
24 + "limit": 3
26 +```
28 +`source` is optional, but use the exact label from `ctx:stored` when available.
30 +## Thresholds
32 +Defaults:
34 +```text
35 +PI_CTX_LARGE_OUTPUT_THRESHOLD=20000
36 +PI_CTX_ERROR_OUTPUT_THRESHOLD=5000
37 +```
39 +Meaning:
41 +- successful `bash` output > 20KB → stored
42 +- failed `bash` output > 5KB → stored + error-oriented preview
43 +- smaller output → unchanged
45 +## Storage
47 +Default DB path:
49 +```text
50 +${XDG_STATE_HOME:-~/.local/state}/pi/ctx/<session-file-basename>-<sha256(abs-cwd)[0..16]>.db
51 +```
53 +Platform fallbacks:
55 +```text
56 +Linux: ${XDG_STATE_HOME:-~/.local/state}/pi/ctx/
57 +macOS: ~/Library/Application Support/pi/ctx/
58 +Windows:%LOCALAPPDATA%\\pi\\ctx\\
59 +```
61 +`ctx` still requires the active Pi session file, but only to derive the session id. It does not store under `sessionManager.getSessionDir()`, because that can mix unrelated session runtime data.
63 +Override for tests/debug:
65 +```bash
66 +PI_CTX_DB_PATH=/tmp/ctx.db
67 +```
69 +In benchmark runs this is inside sandbox HOME:
71 +```text
72 +/home/bench/.local/state/pi/ctx/<session-id>-<cwd-hash>.db
73 +```
75 +The DB contains:
77 +- `sources` table: labels, byte counts, metadata
78 +- `chunks_fts` FTS5 table: chunked searchable output
80 +## Example flow
82 +```text
83 +bash ./dump.sh
84 +→ ctx:stored bash:178810b1 (2085.9KB, 135 chunks).
85 + Stored output is not in the workspace. Call the ctx tool with {"query":"...","source":"bash:178810b1"}.
87 +ctx {"query":"TARGET_EVENT","source":"bash:178810b1"}
88 +→ matching snippets only
89 +```

pi/agent/extensions/ctx/TODO.md

Modified
1 1 # ctx TODO
3 -- Real-LLM behavior eval: does model choose `ctx` naturally vs `bash`/`read`?
4 -- Tune thresholds/snippets after actual use.
5 3 - Replace git-pinned `@marcfargas/pi-test-harness` fork once upstream PR/package exists.

pi/agent/extensions/ctx/__tests__/ctx.test.ts

Modified
2 2 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
3 3 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
4 4 import { tmpdir } from "node:os";
5 5 import { join } from "node:path";
6 -import { CtxStore, defaultDbPath, executeCtxOperation, handleBashToolResult, registerCtxExtension, type CtxToolParams } from "../index.js";
6 +import { CtxStore, defaultDbPath, executeCtxSearch, handleBashToolResult, registerCtxExtension, stateRoot } from "../index.ts";
8 8 let dir: string;
9 9 let dbPath: string;
10 10 beforeEach(() => {
11 11 dir = mkdtempSync(join(tmpdir(), "pi-ctx-test-"));
12 12 dbPath = join(dir, "ctx.db");
13 + process.env.XDG_STATE_HOME = join(dir, "state");
13 14 process.env.PI_CTX_DB_PATH = dbPath;
16 17 afterEach(() => {
17 18 delete process.env.PI_CTX_DB_PATH;
19 + delete process.env.XDG_STATE_HOME;
18 20 rmSync(dir, { recursive: true, force: true });
21 23 describe("ctx DB path", () => {
22 - test("defaults under current Pi session dir", () => {
24 + test("defaults under current Pi session file", () => {
23 25 delete process.env.PI_CTX_DB_PATH;
24 26 const sessionFile = join(dir, "sessions", "session.jsonl");
25 27 const path = defaultDbPath("/repo/project", {
26 28 sessionManager: { getSessionFile: () => sessionFile },
28 - expect(path).toStartWith(join(dir, "sessions", "ctx"));
30 + expect(path).toStartWith(join(dir, "state", "pi", "ctx", "session-"));
29 31 expect(path).toEndWith(".db");
32 - test("uses SessionManager sessionDir when no session file exists", () => {
34 + test("uses XDG/native state root", () => {
35 + expect(stateRoot({ XDG_STATE_HOME: "/state" } as NodeJS.ProcessEnv, "linux", "/home/me")).toBe("/state/pi");
36 + expect(stateRoot({} as NodeJS.ProcessEnv, "linux", "/home/me")).toBe("/home/me/.local/state/pi");
37 + expect(stateRoot({} as NodeJS.ProcessEnv, "darwin", "/Users/me")).toBe("/Users/me/Library/Application Support/pi");
38 + expect(stateRoot({ LOCALAPPDATA: "C:\\Users\\me\\AppData\\Local" } as NodeJS.ProcessEnv, "win32", "C:\\Users\\me")).toBe("C:\\Users\\me\\AppData\\Local/pi");
39 + });
41 + test("rejects sessionDir-only contexts because storage must be session-scoped", () => {
33 42 delete process.env.PI_CTX_DB_PATH;
34 - const path = defaultDbPath("/repo/project", {
43 + expect(() => defaultDbPath("/repo/project", {
35 44 sessionManager: { getSessionDir: () => join(dir, "session-dir") },
36 - });
37 - expect(path).toStartWith(join(dir, "session-dir", "ctx"));
38 - expect(path).toEndWith(".db");
45 + })).toThrow("session file");
44 51 const store = new CtxStore(dbPath);
45 52 const indexed = store.indexText({ kind: "test", label: "sample", text: "alpha beta\npanic stack trace\nomega" });
46 53 expect(indexed.chunkCount).toBeGreaterThan(0);
47 - const hits = store.search(["panic trace"], { limit: 3 });
48 - expect(hits).toContain("panic stack trace");
49 - store.close();
50 - });
52 - test("purges project DB content", () => {
53 - const store = new CtxStore(dbPath);
54 - store.indexText({ kind: "test", label: "sample", text: "alpha beta" });
55 - expect(store.stats().sources).toBe(1);
56 - store.purge();
57 - expect(store.stats().sources).toBe(0);
58 - store.close();
59 - });
60 -});
62 -describe("ctx operations", () => {
63 - test("run-js returns printed findings", async () => {
64 - const store = new CtxStore(dbPath);
65 - const result = await executeCtxOperation(store, {
66 - op: "run-js",
67 - code: "console.log('answer', 41 + 1)",
68 - } satisfies CtxToolParams, dir);
69 - expect(result.isError).toBeFalsy();
70 - expect(result.content[0].text).toContain("answer 42");
71 - store.close();
72 - });
74 - test("file-py reads file as text", async () => {
75 - const file = join(dir, "data.txt");
76 - writeFileSync(file, "one\ntwo\nthree\n", "utf8");
77 - const store = new CtxStore(dbPath);
78 - const result = await executeCtxOperation(store, {
79 - op: "file-py",
80 - path: file,
81 - code: "print(len(text.splitlines()))",
82 - } satisfies CtxToolParams, dir);
83 - expect(result.content[0].text.trim()).toBe("3");
84 - store.close();
85 - });
87 - test("large run output is indexed and returns pointer", async () => {
88 - const store = new CtxStore(dbPath);
89 - const result = await executeCtxOperation(store, {
90 - op: "run-js",
91 - code: "console.log('x'.repeat(110_000))",
92 - } satisfies CtxToolParams, dir);
93 - expect(result.content[0].text).toContain("ctx:stored");
94 - expect(result.content[0].text).toContain("Use ctx search");
95 - expect(store.stats().sources).toBe(1);
54 + expect(store.search(["panic trace"], { limit: 3 })).toContain("panic stack trace");
96 55 store.close();
100 59 describe("bash auto-collapse", () => {
101 - test("uses fullOutputPath, stores raw, returns bounded replacement", () => {
60 + test("leaves small bash output unchanged", () => {
61 + const store = new CtxStore(dbPath);
62 + expect(handleBashToolResult(store, {
63 + toolName: "bash",
64 + input: { command: "echo small" },
65 + content: [{ type: "text", text: "small" }],
66 + details: {},
67 + isError: false,
68 + })).toBeUndefined();
69 + store.close();
70 + });
72 + test("uses fullOutputPath, stores raw, returns bounded replacement without leaking fullOutputPath", () => {
102 73 const full = join(dir, "bash.log");
103 74 const raw = "CTX_RAW_SENTINEL ".repeat(10_000);
104 75 writeFileSync(full, raw, "utf8");
113 84 expect(patch).toBeDefined();
114 85 const text = patch!.content[0].text;
115 86 expect(text).toContain("ctx:stored");
87 + expect(text).toContain("Call the ctx tool");
116 88 expect(text).not.toContain("CTX_RAW_SENTINEL");
117 - expect(store.search(["CTX_RAW_SENTINEL"], { limit: 1 })).toContain("CTX_RAW_SENTINEL");
89 + expect(JSON.stringify(patch)).not.toContain("fullOutputPath");
90 + expect(executeCtxSearch(store, { query: "CTX_RAW_SENTINEL", limit: 1 }).content[0].text).toContain("CTX_RAW_SENTINEL");
118 91 store.close();
121 - test("large error output is searched for failures", () => {
94 + test("large error output is stored and searchable", () => {
122 95 const full = join(dir, "bash-error.log");
123 96 writeFileSync(full, `${"noise\n".repeat(1000)}panic: exploded\nstack trace here\n`, "utf8");
124 97 const store = new CtxStore(dbPath);
129 102 details: { fullOutputPath: full },
130 103 isError: true,
132 - expect(patch?.content[0].text).toContain("matched");
133 - expect(patch?.content[0].text).toContain("panic");
105 + expect(patch?.isError).toBe(true);
106 + expect(patch?.content[0].text).toContain("ctx:stored");
107 + expect(executeCtxSearch(store, { query: "panic exploded", limit: 1 }).content[0].text).toContain("panic: exploded");
134 108 store.close();
138 112 describe("extension registration", () => {
139 - test("registers one tool and one command", () => {
113 + test("registers one search tool and a tool_result hook", () => {
140 114 const tools: any[] = [];
141 - const commands: Record<string, any> = {};
142 115 const handlers: Record<string, Function[]> = {};
143 116 registerCtxExtension({
144 117 registerTool: (tool: any) => tools.push(tool),
145 - registerCommand: (name: string, command: any) => { commands[name] = command; },
146 118 on: (name: string, handler: Function) => { (handlers[name] ??= []).push(handler); },
148 120 expect(tools.map(t => t.name)).toEqual(["ctx"]);
149 - expect(Object.keys(commands)).toEqual(["ctx"]);
150 - expect(Object.keys(handlers)).toContain("tool_result");
121 + expect(Object.keys(handlers)).toEqual(["tool_result"]);

pi/agent/extensions/ctx/__tests__/harness.test.ts

Modified
1 1 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2 -import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
2 +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3 3 import { tmpdir } from "node:os";
4 4 import path, { join } from "node:path";
5 5 import { createTestSession, type TestSession } from "@marcfargas/pi-test-harness";
38 38 return readdirSync(dir).filter((name) => name.endsWith(".db"));
41 +async function emitBash(params: { command?: string; text: string; fullPath?: boolean; isError?: boolean }) {
42 + const details: Record<string, unknown> = {};
43 + let contentText = params.text;
44 + if (params.fullPath) {
45 + const full = join(root, `bash-${crypto.randomUUID()}.log`);
46 + writeFileSync(full, params.text, "utf8");
47 + details.fullOutputPath = full;
48 + details.truncation = { truncated: true };
49 + contentText = "[pi truncated visible output]";
50 + }
51 + return await t!.session.extensionRunner.emitToolResult({
52 + type: "tool_result",
53 + toolName: "bash",
54 + input: { command: params.command ?? "generate massive report" },
55 + content: [{ type: "text", text: contentText }],
56 + details,
57 + isError: params.isError ?? false,
58 + });
41 61 beforeEach(async () => {
42 62 root = mkdtempSync(join(tmpdir(), "pi-ctx-harness-"));
43 63 sessionDir = join(root, "session");
44 64 process.env.PI_CODING_AGENT_SESSION_DIR = sessionDir;
45 - delete process.env.PI_CTX_DB_PATH;
65 + process.env.PI_CTX_DB_PATH = join(sessionDir, "ctx", "harness.db");
46 66 t = await createTestSession({ extensions: [ctxPath()] });
62 82 expect(lock).toContain(CTX_COMMIT);
65 - it("registers exactly one ctx tool and one ctx command", () => {
85 + it("registers exactly one ctx search tool and no command", () => {
66 86 const [extension] = t!.session.extensionRunner.extensions;
67 87 expect([...extension.tools.keys()]).toEqual(["ctx"]);
68 - expect([...extension.commands.keys()]).toEqual(["ctx"]);
69 - });
71 - it("runs JS off-context, stores oversized output, and retrieves exact needles later", async () => {
72 - const result = await ctxExec({
73 - op: "run-js",
74 - query: "ctxneedle12345",
75 - code: `
76 - for (let i = 0; i < 9000; i++) console.log('noise line ' + i + ' ' + 'x'.repeat(20));
77 - console.log('ctxneedle12345 fatal failure in generated report');
78 - for (let i = 0; i < 9000; i++) console.log('tail noise ' + i + ' ' + 'y'.repeat(20));
79 - `,
80 - });
82 - const text = textOf(result);
83 - expect(text).toContain("ctx:stored");
84 - expect(text).toContain("matched");
85 - expect(text.length).toBeLessThan(2_000);
86 - expect(text).not.toContain("tail noise 8999");
87 - expect(dbFiles()).toHaveLength(1);
89 - const search = await ctxExec({ op: "search", query: "ctxneedle12345", limit: 1 });
90 - const searchText = textOf(search);
91 - expect(searchText).toContain("ctxneedle12345 fatal failure");
92 - });
94 - it("analyzes a huge file without returning the file body or indexing small findings", async () => {
95 - const file = join(root, "huge.log");
96 - const lines = Array.from({ length: 25_000 }, (_, i) => `row ${i} boring payload ${"z".repeat(40)}`);
97 - lines[19_876] = "ctxfilemarker67890 ERROR customer=42 shard=west";
98 - writeFileSync(file, `${lines.join("\n")}\n`, "utf8");
100 - const result = await ctxExec({
101 - op: "file-js",
102 - path: file,
103 - code: `
104 - const hit = text.split(/\\n/).find(line => line.includes('ctxfilemarker67890'));
105 - console.log(hit);
106 - `,
107 - });
109 - const out = textOf(result);
110 - expect(out.trim()).toBe("ctxfilemarker67890 ERROR customer=42 shard=west");
111 - expect(out).not.toContain("row 0 boring payload");
112 - const [extension] = t!.session.extensionRunner.extensions;
113 - const stats = await extension.commands.get("ctx").handler("", t!.session.extensionRunner.createContext());
114 - expect(stats.text).toContain("ctx: 0 sources");
88 + expect([...extension.commands.keys()]).toEqual([]);
117 91 it("collapses huge bash tool_result using fullOutputPath before it reaches the transcript", async () => {
121 95 "bashneedle24680 terminal fact hidden in full log",
122 96 ...Array.from({ length: 7_500 }, (_, i) => `bash tail ${i} ${"a".repeat(30)}`),
123 97 ].join("\n");
124 - const full = join(root, "bash-full.log");
125 - writeFileSync(full, raw, "utf8");
127 - const patched = await t!.session.extensionRunner.emitToolResult({
128 - type: "tool_result",
129 - toolName: "bash",
130 - input: { command: "generate massive report" },
131 - content: [{ type: "text", text: "[pi truncated visible output]" }],
132 - details: { fullOutputPath: full, truncation: { truncated: true } },
133 - isError: false,
134 - });
99 + const patched = await emitBash({ text: raw, fullPath: true });
136 100 const text = textOf(patched);
137 101 expect(text).toContain("ctx:stored");
138 - expect(text).toContain("Use ctx search");
102 + expect(text).toContain("Call the ctx tool");
139 103 expect(text).not.toContain("bashneedle24680");
104 + expect(JSON.stringify(patched)).not.toContain("fullOutputPath");
140 105 expect(patched.details.ctxStored).toBe(true);
141 - expect(statSync(join(sessionDir, "ctx", dbFiles()[0])).size).toBeGreaterThan(0);
106 + expect(dbFiles()).toHaveLength(1);
143 - const search = await ctxExec({ op: "search", query: "bashneedle24680", limit: 1 });
108 + const search = await ctxExec({ query: "bashneedle24680", limit: 1 });
144 109 expect(textOf(search)).toContain("bashneedle24680 terminal fact");
147 112 it("extracts searchable failure context for huge failing bash output and keeps isError", async () => {
148 - const full = join(root, "bash-error.log");
149 - writeFileSync(full, `${"boring\n".repeat(5_000)}panic ctxboom999 exploded\nstack trace line\n`, "utf8");
151 - const patched = await t!.session.extensionRunner.emitToolResult({
152 - type: "tool_result",
153 - toolName: "bash",
154 - input: { command: "run failing suite" },
155 - content: [{ type: "text", text: "[truncated]" }],
156 - details: { fullOutputPath: full },
113 + const patched = await emitBash({
114 + text: `${"boring\n".repeat(5_000)}panic ctxboom999 exploded\nstack trace line\n`,
115 + fullPath: true,
157 116 isError: true,
117 + command: "run failing suite",
160 120 expect(patched.isError).toBe(true);
161 121 expect(textOf(patched)).toContain("ctx:stored");
162 - expect(textOf(patched)).toContain("matched");
164 - const search = await ctxExec({ op: "search", query: "ctxboom999", limit: 1 });
122 + const search = await ctxExec({ query: "ctxboom999", limit: 1 });
165 123 expect(textOf(search)).toContain("panic ctxboom999 exploded");
168 126 it("search returns the matching region when a hit is buried deep inside a chunk", async () => {
169 127 const filler = Array.from({ length: 180 }, (_, i) => `preface ${i} ${"p".repeat(80)}`).join("\n");
170 - await ctxExec({
171 - op: "run-js",
172 - query: "buriedneedle424242",
173 - code: `console.log(${JSON.stringify(`${filler}\nburiedneedle424242 exact match near end of chunk\n${"tail ".repeat(5_000)}`)})`,
128 + await emitBash({
129 + text: `${filler}\nburiedneedle424242 exact match near end of chunk\n${"tail ".repeat(5_000)}`,
130 + fullPath: false,
176 - const search = await ctxExec({ op: "search", query: "buriedneedle424242", limit: 1 });
133 + const search = await ctxExec({ query: "buriedneedle424242", limit: 1 });
177 134 const out = textOf(search);
178 135 expect(out).toContain("buriedneedle424242 exact match");
179 136 expect(out.length).toBeLessThan(3_000);
201 158 expect(textOf(patched)).toContain("ctx:stored");
202 159 expect(textOf(patched)).not.toContain("realbashneedle777 line 6999");
203 - const search = await ctxExec({ op: "search", query: "realbashneedle777", limit: 1 });
160 + expect(JSON.stringify(patched)).not.toContain("fullOutputPath");
161 + const search = await ctxExec({ query: "realbashneedle777", limit: 1 });
204 162 expect(textOf(search)).toContain("realbashneedle777 line");
205 163 } finally {
206 164 rmSync(raw.details?.fullOutputPath, { force: true });
210 168 it("falls back to visible bash content when fullOutputPath is absent", async () => {
211 169 const visible = `${"visible filler\n".repeat(9_000)}visibleonly333 final line`;
212 - const patched = await t!.session.extensionRunner.emitToolResult({
213 - type: "tool_result",
214 - toolCallId: "bash-visible-1",
215 - toolName: "bash",
216 - input: { command: "already truncated elsewhere" },
217 - content: [{ type: "text", text: visible }],
218 - details: { truncation: { truncated: true } },
219 - isError: false,
220 - });
170 + const patched = await emitBash({ text: visible });
222 172 expect(textOf(patched)).toContain("ctx:stored");
223 173 expect(textOf(patched)).not.toContain("visibleonly333");
224 - const search = await ctxExec({ op: "search", query: "visibleonly333", limit: 1 });
174 + const search = await ctxExec({ query: "visibleonly333", limit: 1 });
225 175 expect(textOf(search)).toContain("visibleonly333 final line");
228 - it("handles concurrent ctx writes without lock errors or lost sources", async () => {
178 + it("handles concurrent bash collapses without lock errors or lost sources", async () => {
229 179 const count = 8;
230 - const results = await Promise.all(Array.from({ length: count }, (_, i) => ctxExec({
231 - op: "run-js",
232 - code: `console.log('concurrentneedle${i} ' + '${"x".repeat(130_000)}')`,
180 + const results = await Promise.all(Array.from({ length: count }, (_, i) => emitBash({
181 + text: `concurrentneedle${i} ${"x".repeat(130_000)}`,
182 + command: `report ${i}`,
235 185 for (const result of results) expect(textOf(result)).toContain("ctx:stored");
236 - const [extension] = t!.session.extensionRunner.extensions;
237 - const stats = await extension.commands.get("ctx").handler("", t!.session.extensionRunner.createContext());
238 - expect(stats.text).toContain(`ctx: ${count} sources`);
240 186 for (let i = 0; i < count; i++) {
241 - const search = await ctxExec({ op: "search", query: `concurrentneedle${i}`, limit: 1 });
187 + const search = await ctxExec({ query: `concurrentneedle${i}`, limit: 1 });
242 188 expect(textOf(search)).toContain(`concurrentneedle${i}`);
246 - it("keeps unicode output valid while using byte thresholds conservatively", async () => {
192 + it("keeps unicode output searchable", async () => {
247 193 const marker = "unicodeNeedle555 測試 ✅";
248 - const result = await ctxExec({
249 - op: "run-js",
250 - code: `console.log('${"😀".repeat(40_000)} ${marker}')`,
251 - });
194 + await emitBash({ text: `${"😀".repeat(40_000)} ${marker}` });
253 - expect(textOf(result)).toContain("ctx:stored");
254 - const search = await ctxExec({ op: "search", query: "unicodeNeedle555", limit: 1 });
196 + const search = await ctxExec({ query: "unicodeNeedle555", limit: 1 });
255 197 expect(textOf(search)).toContain(marker);
256 198 expect(textOf(search)).not.toContain("�");
259 201 it("closes ctx resources so session directory can be removed after dispose", async () => {
260 - await ctxExec({ op: "run-js", code: "console.log('cleanupneedle ' + 'x'.repeat(120_000))" });
202 + await emitBash({ text: `cleanupneedle ${"x".repeat(120_000)}` });
261 203 expect(dbFiles()).toHaveLength(1);
262 204 t!.dispose();
263 205 t = undefined;
265 207 expect(existsSync(root)).toBe(false);
268 - it("does not touch small bash output, then /ctx reports, searches, and purges stored data", async () => {
269 - const unchanged = await t!.session.extensionRunner.emitToolResult({
270 - type: "tool_result",
271 - toolName: "bash",
272 - input: { command: "echo small" },
273 - content: [{ type: "text", text: "small output" }],
274 - details: {},
275 - isError: false,
276 - });
210 + it("does not touch small bash output", async () => {
211 + const unchanged = await emitBash({ text: "small output", command: "echo small" });
277 212 expect(unchanged).toBeUndefined();
279 - await ctxExec({ op: "run-js", code: "console.log('cmdneedle13579 ' + 'q'.repeat(120_000))" });
280 - const [extension] = t!.session.extensionRunner.extensions;
281 - const command = extension.commands.get("ctx");
283 - const stats = await command.handler("", t!.session.extensionRunner.createContext());
284 - expect(stats.text).toContain("ctx: 1 sources");
285 - expect(stats.text).toContain("recent:");
287 - const found = await command.handler("cmdneedle13579", t!.session.extensionRunner.createContext());
288 - expect(found.text).toContain("cmdneedle13579");
290 - const purged = await command.handler("--purge", t!.session.extensionRunner.createContext());
291 - expect(purged.text).toBe("ctx purged");
293 - const empty = await command.handler("", t!.session.extensionRunner.createContext());
294 - expect(empty.text).toContain("ctx: 0 sources");
213 + const search = await ctxExec({ query: "small", limit: 1 });
214 + expect(textOf(search)).toContain("no matches");

pi/agent/extensions/ctx/index.ts

Modified
1 -import { spawn } from "node:child_process";
2 1 import { createHash } from "node:crypto";
3 -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
4 -import { mkdtemp, rm, writeFile } from "node:fs/promises";
5 -import { tmpdir } from "node:os";
6 -import { dirname, isAbsolute, join, resolve } from "node:path";
2 +import { existsSync, mkdirSync, readFileSync } from "node:fs";
3 +import { basename, dirname, join, resolve } from "node:path";
4 +import { homedir, platform } from "node:os";
7 5 import { Type } from "typebox";
8 6 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
10 -const INTENT_SEARCH_THRESHOLD = 5_000;
11 -const LARGE_OUTPUT_THRESHOLD = 102_400;
12 -const MAX_RETURN_BYTES = 12_000;
8 +const TOOL_NAME = "ctx";
9 +const LARGE_OUTPUT_THRESHOLD = 20_000;
10 +const ERROR_OUTPUT_THRESHOLD = 5_000;
13 11 const MAX_CHUNK_BYTES = 16_000;
14 12 const MAX_CHUNK_LINES = 200;
15 -const ERROR_QUERY = "errors failures exceptions panic traceback assertion failed error";
13 +const MAX_SNIPPET_BYTES = 2_500;
14 +const ERROR_QUERY = "error failed failure exception panic traceback assertion";
17 16 type TextContent = { type: "text"; text: string };
18 17 type ToolPatch = { content: TextContent[]; details?: Record<string, unknown>; isError?: boolean };
20 -export type CtxToolParams =
21 - | { op: "run-sh"; script: string; query?: string | string[]; timeoutMs?: number }
22 - | { op: "run-js"; code: string; query?: string | string[]; timeoutMs?: number }
23 - | { op: "run-py"; code: string; query?: string | string[]; timeoutMs?: number }
24 - | { op: "file-sh"; path: string; script: string; query?: string | string[]; timeoutMs?: number }
25 - | { op: "file-js"; path: string; code: string; query?: string | string[]; timeoutMs?: number }
26 - | { op: "file-py"; path: string; code: string; query?: string | string[]; timeoutMs?: number }
27 - | { op: "search"; query: string | string[]; source?: string; limit?: number };
29 -interface SourceRow {
30 - id: number;
31 - kind: string;
32 - label: string;
33 - bytes: number;
34 - created_at: string;
18 +export type CtxSearchParams = { query: string | string[]; source?: string; limit?: number };
37 20 interface SearchRow {
38 21 source_id: number;
43 26 score: number;
46 -interface RunResult {
47 - stdout: string;
48 - stderr: string;
49 - exitCode: number | null;
50 - timedOut: boolean;
29 +function envThreshold(name: string, fallback: number): number {
30 + const raw = process.env[name];
31 + if (raw === undefined) return fallback;
32 + const n = Number(raw);
33 + return Number.isFinite(n) && n >= 0 ? n : fallback;
36 +function largeOutputThreshold(): number {
37 + return envThreshold("PI_CTX_LARGE_OUTPUT_THRESHOLD", LARGE_OUTPUT_THRESHOLD);
40 +function errorOutputThreshold(): number {
41 + return envThreshold("PI_CTX_ERROR_OUTPUT_THRESHOLD", ERROR_OUTPUT_THRESHOLD);
53 44 function byteLen(text: string): number {
54 45 return Buffer.byteLength(text, "utf8");
57 -function truncateBytes(text: string, max = MAX_RETURN_BYTES): string {
48 +function truncateBytes(text: string, max = MAX_SNIPPET_BYTES): string {
58 49 if (byteLen(text) <= max) return text;
59 50 let out = "";
60 51 let used = 0;
72 63 return content.map((c: any) => c?.type === "text" && typeof c.text === "string" ? c.text : "").join("\n");
66 +function shortHash(text: string): string {
67 + return createHash("sha256").update(text).digest("hex").slice(0, 8);
75 70 function queryTerms(query: string): string[] {
76 71 return [...query.matchAll(/[\p{L}\p{N}_]{2,}/gu)]
77 72 .map(match => match[0].toLowerCase())
78 73 .filter(term => !["and", "or", "not", "near"].includes(term));
81 -function snippetForQuery(text: string, query: string, max = 2500): string {
82 - if (byteLen(text) <= max) return text.trim();
76 +function ftsQuery(query: string): string {
77 + const terms = queryTerms(query).slice(0, 24);
78 + if (terms.length === 0) return `"${query.replaceAll('"', '""')}"`;
79 + return terms.map(term => `"${term.replaceAll('"', '""')}"`).join(" OR ");
82 +function snippetForQuery(text: string, query: string): string {
83 + if (byteLen(text) <= MAX_SNIPPET_BYTES) return text.trim();
83 84 const lower = text.toLowerCase();
84 85 let pos = -1;
85 86 for (const term of queryTerms(query)) {
86 87 const next = lower.indexOf(term);
87 88 if (next >= 0 && (pos < 0 || next < pos)) pos = next;
89 - if (pos < 0) return truncateBytes(text.trim(), max);
90 - let start = Math.max(0, pos - Math.floor(max / 3));
90 + if (pos < 0) return truncateBytes(text.trim());
91 + let start = Math.max(0, pos - Math.floor(MAX_SNIPPET_BYTES / 3));
91 92 const newline = text.lastIndexOf("\n", pos);
92 93 if (newline >= 0 && newline >= start - 500) start = newline + 1;
93 94 let snippet = text.slice(start);
94 95 if (start > 0) snippet = `…\n${snippet}`;
95 - return truncateBytes(snippet.trim(), max);
98 -function shortHash(text: string): string {
99 - return createHash("sha256").update(text).digest("hex").slice(0, 8);
102 -export function defaultDbPath(cwd = process.cwd(), ctx?: any): string {
103 - if (process.env.PI_CTX_DB_PATH) return process.env.PI_CTX_DB_PATH;
104 - const sessionFile = ctx?.sessionManager?.getSessionFile?.()
105 - ?? ctx?.sessionManager?.sessionFile
106 - ?? ctx?.sessionFile;
107 - const managerDir = ctx?.sessionManager?.getSessionDir?.()
108 - ?? ctx?.sessionManager?.sessionDir;
109 - const sessionDir = typeof sessionFile === "string" && sessionFile
110 - ? dirname(sessionFile)
111 - : typeof managerDir === "string" && managerDir
112 - ? managerDir
113 - : process.env.PI_CODING_AGENT_SESSION_DIR;
114 - if (typeof sessionDir !== "string" || !sessionDir) {
115 - throw new Error("ctx requires a Pi session dir; use PI_CTX_DB_PATH for tests/no-session runs");
116 - }
117 - const base = join(sessionDir, "ctx");
118 - mkdirSync(base, { recursive: true });
119 - const hash = createHash("sha256").update(resolve(cwd)).digest("hex").slice(0, 16);
120 - return join(base, `${hash}.db`);
123 -function jsonText(value: unknown): string {
124 - try { return JSON.stringify(value); } catch { return String(value); }
127 -function normalizeQueries(query: string | string[] | undefined): string[] {
128 - if (Array.isArray(query)) return query.map(q => q.trim()).filter(Boolean);
129 - if (typeof query === "string" && query.trim()) return [query.trim()];
130 - return [];
133 -function sourceFilter(source: string | undefined): string | undefined {
134 - const s = source?.trim();
135 - return s ? `%${s}%` : undefined;
96 + return truncateBytes(snippet.trim());
138 99 function chunkText(text: string): Array<{ title: string; text: string }> {
151 112 for (let i = 0; i < lines.length; i++) {
152 113 const line = lines[i] ?? "";
153 114 const n = byteLen(line) + 1;
154 - if (current.length > 0 && (current.length >= MAX_CHUNK_LINES || currentBytes + n > MAX_CHUNK_BYTES)) {
155 - flush(i);
156 - }
115 + if (current.length > 0 && (current.length >= MAX_CHUNK_LINES || currentBytes + n > MAX_CHUNK_BYTES)) flush(i);
157 116 current.push(line);
158 117 currentBytes += n;
161 120 return chunks.length ? chunks : [{ title: "output", text }];
123 +export function stateRoot(env: NodeJS.ProcessEnv = process.env, os = platform(), home = homedir()): string {
124 + if (os === "win32") return join(env.LOCALAPPDATA ?? join(home, "AppData", "Local"), "pi");
125 + if (os === "darwin") return join(home, "Library", "Application Support", "pi");
126 + return join(env.XDG_STATE_HOME ?? join(home, ".local", "state"), "pi");
129 +export function defaultDbPath(cwd = process.cwd(), ctx?: any): string {
130 + if (process.env.PI_CTX_DB_PATH) return process.env.PI_CTX_DB_PATH;
131 + const sessionFile = ctx?.sessionManager?.getSessionFile?.()
132 + ?? ctx?.sessionManager?.sessionFile
133 + ?? ctx?.sessionFile;
134 + if (typeof sessionFile !== "string" || !sessionFile) throw new Error("ctx requires a Pi session file; set PI_CTX_DB_PATH for no-session runs");
135 + const sessionId = basename(sessionFile).replace(/\.jsonl$/, "");
136 + const cwdHash = createHash("sha256").update(resolve(cwd)).digest("hex").slice(0, 16);
137 + const base = join(stateRoot(), "ctx");
138 + mkdirSync(base, { recursive: true });
139 + return join(base, `${sessionId}-${cwdHash}.db`);
164 142 export class CtxStore {
165 143 #db: any;
166 144 readonly dbPath: string;
195 173 indexText(input: { kind: string; label: string; text: string; metadata?: unknown }): { sourceId: number; chunkCount: number; bytes: number; label: string } {
196 174 const bytes = byteLen(input.text);
175 + const meta = input.metadata === undefined ? null : JSON.stringify(input.metadata);
197 176 const source = this.#db.prepare("INSERT INTO sources(kind, label, bytes, metadata) VALUES (?, ?, ?, ?)")
198 - .run(input.kind, input.label, bytes, input.metadata === undefined ? null : jsonText(input.metadata));
177 + .run(input.kind, input.label, bytes, meta);
199 178 const sourceId = Number(source.lastInsertRowid);
200 179 const insert = this.#db.prepare("INSERT INTO chunks_fts(source_id, title, text, label, kind) VALUES (?, ?, ?, ?, ?)");
201 180 const chunks = chunkText(input.text);
210 189 const clean = queries.map(q => q.trim()).filter(Boolean);
211 190 if (clean.length === 0) return "ctx: empty query";
212 191 const limit = Math.max(1, Math.min(opts.limit ?? 3, 10));
213 - const src = sourceFilter(opts.source);
192 + const source = opts.source?.trim();
214 193 const lines: string[] = [];
215 - for (const q of clean) {
216 - const rows = src
217 - ? this.#db.prepare("SELECT source_id, title, text, label, kind, bm25(chunks_fts) AS score FROM chunks_fts WHERE chunks_fts MATCH ? AND label LIKE ? ORDER BY score LIMIT ?").all(q, src, limit) as SearchRow[]
218 - : this.#db.prepare("SELECT source_id, title, text, label, kind, bm25(chunks_fts) AS score FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY score LIMIT ?").all(q, limit) as SearchRow[];
219 - lines.push(`## ${q}`);
194 + for (const query of clean) {
195 + const match = ftsQuery(query);
196 + const rows = source
197 + ? this.#db.prepare("SELECT source_id, title, text, label, kind, bm25(chunks_fts) AS score FROM chunks_fts WHERE chunks_fts MATCH ? AND label LIKE ? ORDER BY score LIMIT ?").all(match, `%${source}%`, limit) as SearchRow[]
198 + : this.#db.prepare("SELECT source_id, title, text, label, kind, bm25(chunks_fts) AS score FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY score LIMIT ?").all(match, limit) as SearchRow[];
199 + lines.push(`## ${query}`);
220 200 if (rows.length === 0) {
221 201 lines.push("no matches");
222 202 continue;
224 204 for (const row of rows) {
225 205 lines.push(`### ${row.label} · ${row.title}`);
226 - lines.push(snippetForQuery(row.text, q, 2500));
206 + lines.push(snippetForQuery(row.text, query));
229 209 return lines.join("\n");
232 - intentSearch(indexed: { sourceId: number; chunkCount: number; bytes: number; label: string }, query: string, maxResults = 5): string {
233 - const rows = this.#db.prepare("SELECT source_id, title, text, label, kind, bm25(chunks_fts) AS score FROM chunks_fts WHERE chunks_fts MATCH ? AND source_id = ? ORDER BY score LIMIT ?")
234 - .all(query, indexed.sourceId, maxResults) as SearchRow[];
235 - const kb = (indexed.bytes / 1024).toFixed(1);
236 - if (rows.length === 0) {
237 - return [
238 - `ctx:stored ${indexed.label} (${kb}KB, ${indexed.chunkCount} chunks).`,
239 - `No sections matched "${query}".`,
240 - `Use ctx search with source "${indexed.label}".`,
241 - ].join("\n");
242 - }
243 - const lines = [
244 - `ctx:stored ${indexed.label} (${kb}KB, ${indexed.chunkCount} chunks).`,
245 - `${rows.length} sections matched "${query}":`,
246 - "",
247 - ];
248 - for (const row of rows) {
249 - const preview = row.text.split(/\r?\n/).find(Boolean)?.slice(0, 160) ?? "";
250 - lines.push(`- ${row.title}: ${preview}`);
251 - }
252 - lines.push("");
253 - lines.push(`Use ctx search with source "${indexed.label}" for details.`);
254 - return lines.join("\n");
255 - }
257 212 pointer(indexed: { chunkCount: number; bytes: number; label: string }): string {
258 - const kb = (indexed.bytes / 1024).toFixed(1);
259 - return `ctx:stored ${indexed.label} (${kb}KB, ${indexed.chunkCount} chunks).\nUse ctx search with source "${indexed.label}".`;
260 - }
262 - stats(): { sources: number; bytes: number; recent: SourceRow[] } {
263 - const row = this.#db.prepare("SELECT COUNT(*) AS sources, COALESCE(SUM(bytes), 0) AS bytes FROM sources").get() as { sources: number; bytes: number };
264 - const recent = this.#db.prepare("SELECT id, kind, label, bytes, created_at FROM sources ORDER BY id DESC LIMIT 8").all() as SourceRow[];
265 - return { sources: Number(row.sources), bytes: Number(row.bytes), recent };
266 - }
268 - formatStats(): string {
269 - const stats = this.stats();
270 - const lines = [`ctx: ${stats.sources} sources, ${(stats.bytes / 1024).toFixed(1)}KB stored`];
271 - if (stats.recent.length) {
272 - lines.push("recent:");
273 - for (const s of stats.recent) lines.push(`- ${s.label} ${s.kind} ${(s.bytes / 1024).toFixed(1)}KB`);
274 - }
275 - lines.push("usage: /ctx <query> | /ctx --purge");
276 - return lines.join("\n");
277 - }
279 - purge(): void {
280 - this.#db.exec("DELETE FROM chunks_fts; DELETE FROM sources;");
213 + return `ctx:stored ${indexed.label} (${(indexed.bytes / 1024).toFixed(1)}KB, ${indexed.chunkCount} chunks).\nStored output is not in the workspace. Call the ctx tool with {"query":"...","source":"${indexed.label}"}.`;
283 216 close(): void {
288 -async function runProcess(command: string, args: string[], opts: { cwd: string; input?: string; env?: NodeJS.ProcessEnv; timeoutMs?: number; signal?: AbortSignal }): Promise<RunResult> {
289 - return await new Promise((resolve) => {
290 - let stdout = "";
291 - let stderr = "";
292 - let settled = false;
293 - const child = spawn(command, args, {
294 - cwd: opts.cwd,
295 - env: opts.env ?? process.env,
296 - stdio: [opts.input === undefined ? "ignore" : "pipe", "pipe", "pipe"],
297 - signal: opts.signal,
298 - });
299 - const finish = (result: RunResult) => {
300 - if (settled) return;
301 - settled = true;
302 - if (timer) clearTimeout(timer);
303 - resolve(result);
304 - };
305 - const timer = opts.timeoutMs && opts.timeoutMs > 0 ? setTimeout(() => {
306 - try { child.kill("SIGTERM"); } catch { /* noop */ }
307 - finish({ stdout, stderr, exitCode: null, timedOut: true });
308 - }, opts.timeoutMs) : undefined;
309 - child.stdout?.setEncoding("utf8");
310 - child.stderr?.setEncoding("utf8");
311 - child.stdout?.on("data", (chunk: string) => { stdout += chunk; });
312 - child.stderr?.on("data", (chunk: string) => { stderr += chunk; });
313 - child.on("error", (err) => finish({ stdout, stderr: stderr + String(err), exitCode: 1, timedOut: false }));
314 - child.on("close", (code) => finish({ stdout, stderr, exitCode: code, timedOut: false }));
315 - if (opts.input !== undefined && child.stdin) {
316 - child.stdin.end(opts.input);
317 - }
318 - });
221 +function makeStore(ctx?: any): CtxStore {
222 + return new CtxStore(defaultDbPath(ctx?.cwd ?? process.cwd(), ctx));
321 -async function withTempFile<T>(prefix: string, suffix: string, content: string, fn: (path: string) => Promise<T>): Promise<T> {
322 - const dir = await mkdtemp(join(tmpdir(), prefix));
323 - try {
324 - const file = join(dir, `script${suffix}`);
325 - await writeFile(file, content, "utf8");
326 - return await fn(file);
327 - } finally {
328 - await rm(dir, { recursive: true, force: true }).catch(() => {});
329 - }
225 +function normalizeQueries(query: string | string[]): string[] {
226 + return Array.isArray(query) ? query.map(q => q.trim()).filter(Boolean) : [query.trim()].filter(Boolean);
332 -function resultText(result: RunResult): { text: string; isError: boolean } {
333 - const isError = result.timedOut || (result.exitCode !== 0 && result.exitCode !== null);
334 - let text = result.stdout;
335 - if (result.stderr) text += (text ? "\n" : "") + result.stderr;
336 - if (result.timedOut) text += `\n(timed out)`;
337 - if (!text) text = "(no output)";
338 - return { text, isError };
229 +function normalizeSearchParams(raw: any): CtxSearchParams {
230 + if (!raw || typeof raw !== "object") throw new Error("ctx params object required");
231 + if (typeof raw.query === "string" || Array.isArray(raw.query)) return raw;
232 + throw new Error("ctx requires query");
341 -function finalizeOutput(store: CtxStore, source: { kind: string; label: string; metadata?: unknown }, text: string, isError: boolean, query?: string | string[]): ToolPatch {
342 - const bytes = byteLen(text);
343 - const queries = normalizeQueries(query);
344 - if (queries.length && bytes > INTENT_SEARCH_THRESHOLD) {
345 - const indexed = store.indexText({ ...source, text });
346 - return { content: [{ type: "text", text: store.intentSearch(indexed, queries.join(" ")) }], isError };
347 - }
348 - if (isError && bytes > INTENT_SEARCH_THRESHOLD) {
349 - const indexed = store.indexText({ ...source, text });
350 - return { content: [{ type: "text", text: store.intentSearch(indexed, ERROR_QUERY) }], isError };
351 - }
352 - if (bytes > LARGE_OUTPUT_THRESHOLD) {
353 - const indexed = store.indexText({ ...source, text });
354 - return { content: [{ type: "text", text: store.pointer(indexed) }], isError };
355 - }
356 - return { content: [{ type: "text", text: truncateBytes(text) }], isError };
359 -export async function executeCtxOperation(store: CtxStore, params: CtxToolParams, cwd: string, signal?: AbortSignal): Promise<ToolPatch> {
360 - const timeoutMs = "timeoutMs" in params ? params.timeoutMs : undefined;
361 - const query = "query" in params ? params.query : undefined;
362 - switch (params.op) {
363 - case "search": {
364 - return { content: [{ type: "text", text: store.search(normalizeQueries(params.query), { source: params.source, limit: params.limit }) }] };
365 - }
366 - case "run-sh": {
367 - const r = await runProcess("/bin/sh", ["-c", params.script], { cwd, timeoutMs, signal });
368 - const { text, isError } = resultText(r);
369 - return finalizeOutput(store, { kind: "run-sh", label: `run-sh:${shortHash(params.script)}`, metadata: { script: params.script } }, text, isError, query);
370 - }
371 - case "run-js": {
372 - return await withTempFile("pi-ctx-js-", ".js", params.code, async (file) => {
373 - const r = await runProcess("bun", [file], { cwd, timeoutMs, signal });
374 - const { text, isError } = resultText(r);
375 - return finalizeOutput(store, { kind: "run-js", label: `run-js:${shortHash(params.code)}`, metadata: { code: params.code } }, text, isError, query);
376 - });
377 - }
378 - case "run-py": {
379 - return await withTempFile("pi-ctx-py-", ".py", params.code, async (file) => {
380 - const r = await runProcess("python3", [file], { cwd, timeoutMs, signal });
381 - const { text, isError } = resultText(r);
382 - return finalizeOutput(store, { kind: "run-py", label: `run-py:${shortHash(params.code)}`, metadata: { code: params.code } }, text, isError, query);
383 - });
384 - }
385 - case "file-sh": {
386 - const path = isAbsolute(params.path) ? params.path : resolve(cwd, params.path);
387 - const r = await runProcess("/bin/sh", ["-c", params.script], { cwd, timeoutMs, signal, env: { ...process.env, CTX_FILE: path } });
388 - const { text, isError } = resultText(r);
389 - return finalizeOutput(store, { kind: "file-sh", label: `file-sh:${params.path}`, metadata: { path: params.path, script: params.script } }, text, isError, query);
390 - }
391 - case "file-js": {
392 - const path = isAbsolute(params.path) ? params.path : resolve(cwd, params.path);
393 - const wrapper = `const path = ${JSON.stringify(path)};\nconst text = await Bun.file(path).text();\n${params.code}\n`;
394 - return await withTempFile("pi-ctx-file-js-", ".js", wrapper, async (file) => {
395 - const r = await runProcess("bun", [file], { cwd, timeoutMs, signal });
396 - const { text, isError } = resultText(r);
397 - return finalizeOutput(store, { kind: "file-js", label: `file-js:${params.path}`, metadata: { path: params.path, code: params.code } }, text, isError, query);
398 - });
399 - }
400 - case "file-py": {
401 - const path = isAbsolute(params.path) ? params.path : resolve(cwd, params.path);
402 - const wrapper = `path = ${JSON.stringify(path)}\ntext = open(path, 'r', encoding='utf-8', errors='replace').read()\n${params.code}\n`;
403 - return await withTempFile("pi-ctx-file-py-", ".py", wrapper, async (file) => {
404 - const r = await runProcess("python3", [file], { cwd, timeoutMs, signal });
405 - const { text, isError } = resultText(r);
406 - return finalizeOutput(store, { kind: "file-py", label: `file-py:${params.path}`, metadata: { path: params.path, code: params.code } }, text, isError, query);
407 - });
408 - }
409 - }
235 +export function executeCtxSearch(store: CtxStore, params: CtxSearchParams): ToolPatch {
236 + return { content: [{ type: "text", text: store.search(normalizeQueries(params.query), { source: params.source, limit: params.limit }) }] };
412 239 export function handleBashToolResult(store: CtxStore, event: any): ToolPatch | undefined {
413 240 if (event?.toolName !== "bash") return undefined;
414 241 const visible = textOf(event.content);
415 - const fullPath = event.details?.fullOutputPath;
416 242 let full = visible;
417 - if (typeof fullPath === "string") {
418 - try {
419 - if (existsSync(fullPath)) full = readFileSync(fullPath, "utf8");
420 - } catch (err) {
421 - return {
422 - content: [{ type: "text", text: `ctx:suppressed bash output; failed to read fullOutputPath (${err instanceof Error ? err.message : String(err)})` }],
423 - details: { ...(event.details ?? {}), ctxSuppressed: true },
424 - isError: event.isError,
425 - };
426 - }
427 - }
243 + const fullPath = event.details?.fullOutputPath;
244 + if (typeof fullPath === "string" && existsSync(fullPath)) full = readFileSync(fullPath, "utf8");
428 245 const bytes = byteLen(full);
429 - if (!event.isError && bytes <= LARGE_OUTPUT_THRESHOLD) return undefined;
430 - if (event.isError && bytes <= INTENT_SEARCH_THRESHOLD) return undefined;
431 - try {
432 - const command = String(event.input?.command ?? "bash");
433 - const label = `${event.isError ? "bash-error" : "bash"}:${shortHash(command)}`;
434 - const indexed = store.indexText({ kind: event.isError ? "bash-error" : "bash", label, text: full, metadata: { command } });
435 - const text = event.isError ? store.intentSearch(indexed, ERROR_QUERY) : store.pointer(indexed);
436 - return {
437 - content: [{ type: "text", text }],
438 - details: { ...(event.details ?? {}), ctxStored: true, ctxSource: label },
439 - isError: event.isError,
440 - };
441 - } catch (err) {
442 - return {
443 - content: [{ type: "text", text: `ctx:suppressed ${bytes}B bash output; storage failed (${err instanceof Error ? err.message : String(err)})` }],
444 - details: { ...(event.details ?? {}), ctxSuppressed: true },
445 - isError: event.isError,
446 - };
447 - }
450 -function makeStore(ctx?: any): CtxStore {
451 - const cwd = ctx?.cwd ?? process.cwd();
452 - return new CtxStore(defaultDbPath(cwd, ctx));
246 + if (!event.isError && bytes <= largeOutputThreshold()) return undefined;
247 + if (event.isError && bytes <= errorOutputThreshold()) return undefined;
248 + const command = String(event.input?.command ?? "bash");
249 + const label = `${event.isError ? "bash-error" : "bash"}:${shortHash(command)}`;
250 + const indexed = store.indexText({ kind: event.isError ? "bash-error" : "bash", label, text: full, metadata: { command } });
251 + const text = event.isError
252 + ? `${store.pointer(indexed)}\n\n${store.search([ERROR_QUERY], { source: label, limit: 3 })}`
253 + : store.pointer(indexed);
254 + const { fullOutputPath: _fullOutputPath, ...safeDetails } = event.details ?? {};
255 + return {
256 + content: [{ type: "text", text }],
257 + details: { ...safeDetails, ctxStored: true, ctxSource: label },
258 + isError: event.isError,
259 + };
455 262 const Params = Type.Object({
456 - op: Type.Union([
457 - Type.Literal("run-sh"), Type.Literal("run-js"), Type.Literal("run-py"),
458 - Type.Literal("file-sh"), Type.Literal("file-js"), Type.Literal("file-py"), Type.Literal("search"),
459 - ], { description: "Operation" }),
460 - script: Type.Optional(Type.String({ description: "Shell script for run-sh/file-sh" })),
461 - code: Type.Optional(Type.String({ description: "JS/Python code for run-js/run-py/file-js/file-py" })),
462 - path: Type.Optional(Type.String({ description: "File path for file-* ops" })),
463 - query: Type.Optional(Type.Union([Type.String(), Type.Array(Type.String())], { description: "Search query / intent" })),
464 - source: Type.Optional(Type.String({ description: "Source label substring for search" })),
465 - limit: Type.Optional(Type.Number({ description: "Search results per query" })),
466 - timeoutMs: Type.Optional(Type.Number({ description: "Timeout in ms" })),
263 + query: Type.Union([Type.String(), Type.Array(Type.String())], { description: "Search query" }),
264 + source: Type.Optional(Type.String({ description: "Source label from ctx:stored" })),
265 + limit: Type.Optional(Type.Number({ description: "Max results per query" })),
469 -function normalizeParams(raw: any): CtxToolParams {
470 - if (!raw || typeof raw !== "object") throw new Error("ctx params object required");
471 - switch (raw.op) {
472 - case "run-sh": if (typeof raw.script === "string") return raw; break;
473 - case "run-js": if (typeof raw.code === "string") return raw; break;
474 - case "run-py": if (typeof raw.code === "string") return raw; break;
475 - case "file-sh": if (typeof raw.path === "string" && typeof raw.script === "string") return raw; break;
476 - case "file-js": if (typeof raw.path === "string" && typeof raw.code === "string") return raw; break;
477 - case "file-py": if (typeof raw.path === "string" && typeof raw.code === "string") return raw; break;
478 - case "search": if (typeof raw.query === "string" || Array.isArray(raw.query)) return raw; break;
479 - }
480 - throw new Error("invalid ctx params for op");
483 -export function registerCtxExtension(pi: Pick<ExtensionAPI, "registerTool" | "registerCommand" | "on">): void {
268 +export function registerCtxExtension(pi: Pick<ExtensionAPI, "registerTool" | "on">): void {
484 269 pi.on("tool_result", (event: any, ctx: any) => {
485 270 let store: CtxStore | undefined;
487 272 store = makeStore(ctx);
488 273 return handleBashToolResult(store, event);
489 274 } catch (err) {
490 - const text = textOf(event?.content);
491 - const maybeLarge = event?.toolName === "bash" && (event?.details?.fullOutputPath || byteLen(text) > LARGE_OUTPUT_THRESHOLD);
275 + const visible = textOf(event?.content);
276 + const maybeLarge = event?.toolName === "bash" && (event?.details?.fullOutputPath || byteLen(visible) > largeOutputThreshold());
492 277 if (!maybeLarge) return undefined;
278 + const { fullOutputPath: _fullOutputPath, ...safeDetails } = event.details ?? {};
493 279 return {
494 - content: [{ type: "text", text: `ctx:suppressed bash output; store unavailable (${err instanceof Error ? err.message : String(err)})` }],
495 - details: { ...(event.details ?? {}), ctxSuppressed: true },
280 + content: [{ type: "text", text: `ctx:suppressed bash output; storage failed (${err instanceof Error ? err.message : String(err)})` }],
281 + details: { ...safeDetails, ctxSuppressed: true },
496 282 isError: event.isError,
498 284 } finally {
503 289 pi.registerTool({
504 - name: "ctx",
505 - label: "ctx",
506 - description: "Run JS/Py/sh analysis off-context; search stored large bash output.",
507 - promptSnippet: "Run JS/Py/sh analysis off-context; search stored large bash output.",
508 - promptGuidelines: [
509 - "Use ctx run-* for large-output analysis; print only findings.",
510 - "Use ctx file-* for large file analysis without reading whole files into context.",
511 - "Use ctx search when bash output was stored as ctx:stored.",
512 - ],
290 + name: TOOL_NAME,
291 + label: "ctx search",
292 + description: "Search command output that ctx previously stored after it exceeded the safe context threshold.",
293 + promptSnippet: "Use only after a tool result says ctx:stored.",
294 + promptGuidelines: ["Call ctx with query and optional source from ctx:stored to inspect suppressed large bash output."],
513 295 parameters: Params,
514 - async execute(_toolCallId: string, raw: unknown, signal?: AbortSignal, _onUpdate?: unknown, ctx?: any) {
296 + async execute(_toolCallId: string, raw: unknown, _signal?: AbortSignal, _onUpdate?: unknown, ctx?: any) {
515 297 const store = makeStore(ctx);
517 - const params = normalizeParams(raw);
518 - return await executeCtxOperation(store, params, ctx?.cwd ?? process.cwd(), signal);
299 + return executeCtxSearch(store, normalizeSearchParams(raw));
519 300 } catch (err) {
520 301 return { content: [{ type: "text", text: `ctx error: ${err instanceof Error ? err.message : String(err)}` }], details: {}, isError: true };
521 302 } finally {
527 - pi.registerCommand("ctx", {
528 - description: "ctx stats/search/purge",
529 - handler: async (args: unknown, ctx: any) => {
530 - const q = String(args ?? "").trim();
531 - const store = makeStore(ctx);
532 - try {
533 - if (q === "--purge") {
534 - if (ctx?.hasUI) {
535 - const ok = await ctx.ui.confirm("ctx purge", "Delete ctx index for this project?");
536 - if (!ok) return { text: "ctx purge cancelled" };
537 - }
538 - store.purge();
539 - return { text: "ctx purged" };
540 - }
541 - if (!q) return { text: store.formatStats() };
542 - return { text: store.search([q], { limit: 5 }) };
543 - } finally {
544 - store.close();
545 - }
546 - },
547 - });
550 309 export default function (pi: ExtensionAPI): void {

pi/agent/extensions/ctx/package.json

Modified
15 15 "@marcfargas/pi-test-harness": "git+ssh://git@github.com/bugabinga/pi-test-harness.git#0cb5ce5f10fb18c9a80e904e2e3310cc98a28e29"
17 17 "license": "MIT",
18 - "description": "Off-context execution and large-output search for Pi.",
18 + "description": "Context blowup fuse and search for oversized Pi bash output.",
19 19 "keywords": [
20 20 "pi",
21 21 "pi-extension"