repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

refactor(ctx): compact large read results

8876c807a3e24f461e310368c74e45f1978b22a1

Oliver Jan Krylow <oliver@bugabinga.net> ·

unsigned

Browse filesCompare with HEAD

Changed files (7)

extensions/ctx/CHANGELOG.md

Modified
1 1 # Changelog
3 +## Unreleased
5 +- replace bash SQLite search with read-result compaction
6 +- remove ctx tool, DB, FTS, and bash interception
7 +- compact full reads to 8-line head plus 32-line tail
3 9 ## 0.1.0 - 2026-05-21
5 11 - a40a427 prepare extensions for npm release
6 12 - 44eb82c feat(ctx): add context indexing extension

extensions/ctx/README.md

Modified
1 1 # ctx
3 -Pi context blowup fuse for oversized `bash` output.
3 +Compacts oversized Pi `read` results before they hit context.
5 +## Install / load
7 +Loaded through the root `pi-ext` package. See [../../README.md](../../README.md).
9 +## Commands / tools / settings
11 +Pi commands: none.
13 +Pi tools: none.
15 +Hooks:
17 +- `tool_result`: watches builtin `read` results and may replace large full-file reads with a bounded head/tail view.
19 +Pi settings: none.
21 +Env vars:
23 +```text
24 +PI_CTX_COMPACT_THRESHOLD=20000
25 +PI_CTX_READ_OUTPUT_THRESHOLD=20000
26 +```
28 +`PI_CTX_COMPACT_THRESHOLD` wins if both are set.
30 +## Behavior
5 32 `ctx` does one thing:
7 -1. watches `bash` tool results
8 -2. if output is too large, stores full text off-context in SQLite FTS
9 -3. replaces visible output with a small `ctx:stored ...` pointer
10 -4. exposes `ctx` search for that stored output
34 +1. watches `read` tool results
35 +2. ignores errors and bounded reads with `limit`
36 +3. if full read output exceeds the threshold, returns:
37 + - first 8 lines
38 + - `[ctx compacted read: ...]` marker
39 + - grep/read-offset instruction
40 + - last 32 lines
41 +4. leaves `bash`, `grep`, `find`, `ls`, and all other tools untouched
12 -It does **not** run shell/JS/Python jobs. It does **not** inspect files. Use normal Pi tools for normal work.
43 +No SQLite. No search tool. No replacement tools. No workspace writes.
14 -## Tool
45 +Agents can recover hidden middle content with normal builtin tools:
16 -### `ctx`
47 +```text
48 +grep target path
49 +read path offset=120 limit=40
50 +```
18 -Search output previously stored by ctx:
52 +Patch metadata adds:
20 54 ```json
22 - "query": "TARGET_EVENT",
23 - "source": "bash:178810b1:42",
24 - "limit": 3
55 +{ "ctxCompactedRead": true }
28 -Range-read known lines, like `read`:
58 +## Demo
30 -```json
32 - "source": "bash:178810b1:42",
33 - "offset": 201,
34 - "limit": 200
36 -```
38 -Use the exact `source` label from `ctx:stored`.
40 -## Thresholds
42 -Defaults:
44 -```text
45 -PI_CTX_LARGE_OUTPUT_THRESHOLD=20000
46 -PI_CTX_ERROR_OUTPUT_THRESHOLD=5000
47 -```
49 -Meaning:
51 -- successful `bash` output > 20KB → stored
52 -- failed `bash` output > 5KB → stored + error-oriented preview
53 -- smaller output → unchanged
55 -## Storage
57 -Default DB path:
59 -```text
60 -${XDG_STATE_HOME:-~/.local/state}/pi/ctx/<session-file-basename>-<sha256(abs-cwd)[0..16]>.db
61 -```
63 -Platform fallbacks:
65 -```text
66 -Linux: ${XDG_STATE_HOME:-~/.local/state}/pi/ctx/
67 -macOS: ~/Library/Application Support/pi/ctx/
68 -Windows:%LOCALAPPDATA%\\pi\\ctx\\
69 -```
71 -`ctx` still requires the active Pi session file, but only to derive the session id. It does not store under `sessionManager.getSessionDir()`, because that can mix unrelated session runtime data.
73 -Override for tests/debug:
75 -```bash
76 -PI_CTX_DB_PATH=/tmp/ctx.db
77 -```
79 -In benchmark runs this is inside sandbox HOME:
81 -```text
82 -/home/bench/.local/state/pi/ctx/<session-id>-<cwd-hash>.db
83 -```
85 -The DB contains:
87 -- `sources` table: labels, byte counts, metadata
88 -- `chunks_fts` FTS5 table: chunked searchable output
90 -## Example flow
92 -```text
93 -bash ./dump.sh
94 -→ ctx:stored bash:178810b1:42 (2085.9KB, 135 chunks).
95 - Stored output is not in the workspace. Call the ctx tool with {"query":"...","source":"bash:178810b1:42"}.
97 -ctx {"query":"TARGET_EVENT","source":"bash:178810b1:42"}
98 -→ matching snippets only
100 -ctx {"source":"bash:178810b1:42","offset":201,"limit":200}
101 -→ exact line range
102 -```
60 +<!-- demo:tools_suite:start -->
61 +![Tools suite](assets/tools_suite.gif)
62 +<!-- demo:tools_suite:end -->

extensions/ctx/__tests__/ctx.test.ts

Modified
1 -import { afterEach, beforeEach, describe, expect, test } from "bun:test";
2 -import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
3 -import { tmpdir } from "node:os";
4 -import { join } from "node:path";
5 -import { CtxStore, defaultDbPath, executeCtxSearch, handleBashToolResult, registerCtxExtension, stateRoot } from "../index.ts";
1 +import { describe, expect, test } from "bun:test";
2 +import ctxExtension, { __test } from "../index.ts";
7 -let dir: string;
8 -let dbPath: string;
4 +const { compactRead, handleToolResult } = __test;
10 -beforeEach(() => {
11 - dir = mkdtempSync(join(tmpdir(), "pi-ctx-test-"));
12 - dbPath = join(dir, "ctx.db");
13 - process.env.XDG_STATE_HOME = join(dir, "state");
14 - process.env.PI_CTX_DB_PATH = dbPath;
15 -});
6 +describe("ctx read compactor", () => {
7 + test("registers no tools", () => {
8 + const tools: string[] = [];
9 + const handlers: string[] = [];
10 + ctxExtension({
11 + registerTool(tool: { name: string }) { tools.push(tool.name); },
12 + on(name: string) { handlers.push(name); },
13 + });
14 + expect(tools).toEqual([]);
15 + expect(handlers).toEqual(["tool_result"]);
16 + });
17 -afterEach(() => {
18 - delete process.env.PI_CTX_DB_PATH;
19 - delete process.env.XDG_STATE_HOME;
20 - rmSync(dir, { recursive: true, force: true });
21 -});
18 + test("keeps head and tail only", () => {
19 + const compacted = compactRead(Array.from({ length: 300 }, (_, i) => `line ${i + 1}`).join("\n"));
20 + expect(compacted).toContain("line 1");
21 + expect(compacted).toContain("line 300");
22 + expect(compacted).toContain("[ctx compacted read:");
23 + expect(compacted).not.toContain("line 100");
24 + });
23 -describe("ctx DB path", () => {
24 - test("defaults under current Pi session file", () => {
25 - delete process.env.PI_CTX_DB_PATH;
26 - const sessionFile = join(dir, "sessions", "session.jsonl");
27 - const path = defaultDbPath("/repo/project", {
28 - sessionManager: { getSessionFile: () => sessionFile },
29 - });
30 - expect(path).toStartWith(join(dir, "state", "pi", "ctx", "session-"));
31 - expect(path).toEndWith(".db");
32 - });
26 + test("skips explicit read window", () => {
27 + const text = "x".repeat(25_000);
28 + expect(handleToolResult({ toolName: "read", input: { path: "x", offset: 1, limit: 10 }, content: [{ type: "text", text }] })).toBeUndefined();
29 + });
34 - test("uses XDG/native state root", () => {
35 - expect(stateRoot({ XDG_STATE_HOME: "/state" } as NodeJS.ProcessEnv, "linux", "/home/me")).toBe("/state/pi");
36 - expect(stateRoot({} as NodeJS.ProcessEnv, "linux", "/home/me")).toBe("/home/me/.local/state/pi");
37 - expect(stateRoot({} as NodeJS.ProcessEnv, "darwin", "/Users/me")).toBe("/Users/me/Library/Application Support/pi");
38 - expect(stateRoot({ LOCALAPPDATA: "C:\\Users\\me\\AppData\\Local" } as NodeJS.ProcessEnv, "win32", "C:\\Users\\me")).toBe("C:\\Users\\me\\AppData\\Local/pi");
39 - });
31 + test("does not touch bash/grep/find/ls", () => {
32 + const text = "x".repeat(25_000);
33 + for (const toolName of ["bash", "grep", "find", "ls"]) {
34 + expect(handleToolResult({ toolName, content: [{ type: "text", text }] })).toBeUndefined();
35 + }
36 + });
41 - test("rejects sessionDir-only contexts because storage must be session-scoped", () => {
42 - delete process.env.PI_CTX_DB_PATH;
43 - expect(() => defaultDbPath("/repo/project", {
44 - sessionManager: { getSessionDir: () => join(dir, "session-dir") },
45 - })).toThrow("session file");
46 - });
47 -});
49 -describe("CtxStore", () => {
50 - test("indexes, searches, and range-reads chunks", () => {
51 - const store = new CtxStore(dbPath);
52 - const indexed = store.indexText({ kind: "test", label: "sample", text: "alpha beta\npanic stack trace\nomega" });
53 - expect(indexed.chunkCount).toBeGreaterThan(0);
54 - expect(indexed.label).toStartWith("sample:");
55 - expect(store.search(["panic trace"], { limit: 3 })).toContain("panic stack trace");
56 - expect(executeCtxSearch(store, { source: indexed.label }).content[0].text).toContain("alpha beta");
57 - expect(store.range({ source: indexed.label, offset: 2, limit: 1 })).toContain("panic stack trace");
58 - store.close();
59 - });
61 - test("deduplicates chunks across query lists", () => {
62 - const store = new CtxStore(dbPath);
63 - const indexed = store.indexText({ kind: "test", label: "dupe", text: "alpha beta gamma" });
64 - const out = store.search(["alpha", "beta", "gamma"], { source: indexed.label, limit: 3 });
65 - expect((out.match(/^### /gm) ?? []).length).toBe(1);
66 - store.close();
67 - });
68 -});
70 -describe("bash auto-collapse", () => {
71 - test("leaves small bash output unchanged", () => {
72 - const store = new CtxStore(dbPath);
73 - expect(handleBashToolResult(store, {
74 - toolName: "bash",
75 - input: { command: "echo small" },
76 - content: [{ type: "text", text: "small" }],
77 - details: {},
78 - isError: false,
79 - })).toBeUndefined();
80 - store.close();
81 - });
83 - test("uses fullOutputPath, stores raw, returns bounded replacement without leaking fullOutputPath", () => {
84 - const full = join(dir, "bash.log");
85 - const raw = "CTX_RAW_SENTINEL ".repeat(10_000);
86 - writeFileSync(full, raw, "utf8");
87 - const store = new CtxStore(dbPath);
88 - const patch = handleBashToolResult(store, {
89 - toolName: "bash",
90 - input: { command: "python noisy.py" },
91 - content: [{ type: "text", text: "[truncated]" }],
92 - details: { fullOutputPath: full, truncation: { truncated: true } },
93 - isError: false,
94 - });
95 - expect(patch).toBeDefined();
96 - const text = patch!.content[0].text;
97 - expect(text).toContain("ctx:stored");
98 - expect(text).toContain("Call the ctx tool");
99 - expect(text).not.toContain("CTX_RAW_SENTINEL");
100 - expect(JSON.stringify(patch)).not.toContain("fullOutputPath");
101 - expect(executeCtxSearch(store, { query: "CTX_RAW_SENTINEL", limit: 1 }).content[0].text).toContain("CTX_RAW_SENTINEL");
102 - store.close();
103 - });
105 - test("large error output is stored and searchable", () => {
106 - const full = join(dir, "bash-error.log");
107 - writeFileSync(full, `${"noise\n".repeat(1000)}panic: exploded\nstack trace here\n`, "utf8");
108 - const store = new CtxStore(dbPath);
109 - const patch = handleBashToolResult(store, {
110 - toolName: "bash",
111 - input: { command: "failing" },
112 - content: [{ type: "text", text: "[truncated]" }],
113 - details: { fullOutputPath: full },
114 - isError: true,
115 - });
116 - expect(patch?.isError).toBe(true);
117 - expect(patch?.content[0].text).toContain("ctx:stored");
118 - expect(executeCtxSearch(store, { query: "panic exploded", limit: 1 }).content[0].text).toContain("panic: exploded");
119 - store.close();
120 - });
121 -});
123 -describe("extension registration", () => {
124 - test("registers one search tool and a tool_result hook", () => {
125 - const tools: any[] = [];
126 - const handlers: Record<string, Function[]> = {};
127 - registerCtxExtension({
128 - registerTool: (tool: any) => tools.push(tool),
129 - on: (name: string, handler: Function) => { (handlers[name] ??= []).push(handler); },
130 - });
131 - expect(tools.map(t => t.name)).toEqual(["ctx"]);
132 - expect(Object.keys(handlers)).toEqual(["tool_result"]);
133 - });
38 + test("compacts large read results", () => {
39 + const text = Array.from({ length: 400 }, (_, i) => `line ${i + 1}`).join("\n") + "\n" + "x".repeat(25_000);
40 + const patch = handleToolResult({ toolName: "read", input: { path: "x" }, content: [{ type: "text", text }] });
41 + expect(patch?.content[0].text).toContain("[ctx compacted read:");
42 + expect(patch?.details?.ctxCompactedRead).toBe(true);
43 + });

extensions/ctx/__tests__/harness.test.ts

Modified
1 1 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2 -import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
3 -import { tmpdir } from "node:os";
4 -import path, { join } from "node:path";
5 2 import { createTestSession, type TestSession } from "@marcfargas/pi-test-harness";
3 +import path from "node:path";
8 -let root: string;
9 -let sessionDir: string;
10 5 let t: TestSession | undefined;
12 7 function ctxPath(): string {
13 - return path.resolve(import.meta.dir, "../index.ts");
16 -function ctxTool() {
17 - return t!.session.extensionRunner.getToolDefinition("ctx");
20 -function ctxExec(params: Record<string, unknown>) {
21 - return ctxTool().execute(
22 - `call-${crypto.randomUUID()}`,
23 - params,
24 - undefined,
25 - undefined,
26 - t!.session.extensionRunner.createContext(),
27 - );
8 + return path.resolve(import.meta.dir, "../index.ts");
30 11 function textOf(result: any): string {
31 - return result.content.map((part: { text?: string }) => part.text ?? "").join("\n");
12 + return result.content.map((part: { text?: string }) => part.text ?? "").join("\n");
34 -function dbFiles(): string[] {
35 - const dir = join(sessionDir, "ctx");
36 - if (!existsSync(dir)) return [];
37 - return readdirSync(dir).filter((name) => name.endsWith(".db"));
40 -async function emitBash(params: { command?: string; text: string; fullPath?: boolean; isError?: boolean }) {
41 - const details: Record<string, unknown> = {};
42 - let contentText = params.text;
43 - if (params.fullPath) {
44 - const full = join(root, `bash-${crypto.randomUUID()}.log`);
45 - writeFileSync(full, params.text, "utf8");
46 - details.fullOutputPath = full;
47 - details.truncation = { truncated: true };
48 - contentText = "[pi truncated visible output]";
49 - }
50 - return await t!.session.extensionRunner.emitToolResult({
51 - type: "tool_result",
52 - toolName: "bash",
53 - input: { command: params.command ?? "generate massive report" },
54 - content: [{ type: "text", text: contentText }],
55 - details,
56 - isError: params.isError ?? false,
57 - });
15 +async function emitRead(params: { text: string; input?: Record<string, unknown>; isError?: boolean }) {
16 + return await t!.session.extensionRunner.emitToolResult({
17 + type: "tool_result",
18 + toolName: "read",
19 + input: params.input ?? { path: "big.txt" },
20 + content: [{ type: "text", text: params.text }],
21 + details: {},
22 + isError: params.isError ?? false,
23 + });
60 26 beforeEach(async () => {
61 - root = mkdtempSync(join(tmpdir(), "pi-ctx-harness-"));
62 - sessionDir = join(root, "session");
63 - process.env.PI_CODING_AGENT_SESSION_DIR = sessionDir;
64 - process.env.PI_CTX_DB_PATH = join(sessionDir, "ctx", "harness.db");
65 - t = await createTestSession({ extensions: [ctxPath()] });
27 + t = await createTestSession({ extensions: [ctxPath()] });
68 30 afterEach(() => {
69 - t?.dispose();
70 - t = undefined;
71 - delete process.env.PI_CODING_AGENT_SESSION_DIR;
72 - delete process.env.PI_CTX_DB_PATH;
73 - rmSync(root, { recursive: true, force: true });
31 + t?.dispose();
32 + t = undefined;
76 35 describe("ctx harness integration", () => {
77 - it("loads upstream pi-test-harness from npm", async () => {
78 - const pkg = await import("@marcfargas/pi-test-harness");
79 - expect(typeof pkg.createTestSession).toBe("function");
80 - });
36 + it("loads upstream pi-test-harness from npm", async () => {
37 + const pkg = await import("@marcfargas/pi-test-harness");
38 + expect(typeof pkg.createTestSession).toBe("function");
39 + });
82 - it("registers exactly one ctx search tool and no command", () => {
83 - const [extension] = t!.session.extensionRunner.extensions;
84 - expect([...extension.tools.keys()]).toEqual(["ctx"]);
85 - expect([...extension.commands.keys()]).toEqual([]);
86 - });
41 + it("registers no tools and no commands", () => {
42 + const [extension] = t!.session.extensionRunner.extensions;
43 + expect([...extension.tools.keys()]).toEqual([]);
44 + expect([...extension.commands.keys()]).toEqual([]);
45 + });
88 - it("collapses huge bash tool_result using fullOutputPath before it reaches the transcript", async () => {
89 - const raw = [
90 - "header",
91 - ...Array.from({ length: 5 }, (_, i) => `bash noise ${i} ${"a".repeat(30)}`),
92 - "bashneedle24680 terminal fact hidden in full log",
93 - ...Array.from({ length: 7_500 }, (_, i) => `bash tail ${i} ${"a".repeat(30)}`),
94 - ].join("\n");
47 + it("compacts huge read output before it reaches the transcript", async () => {
48 + const raw = Array.from({ length: 600 }, (_, i) => `line ${i + 1} ${"x".repeat(80)}`).join("\n");
49 + const patched = await emitRead({ text: raw });
50 + const text = textOf(patched);
51 + expect(text).toContain("line 1");
52 + expect(text).toContain("line 600");
53 + expect(text).toContain("[ctx compacted read:");
54 + expect(text).toContain("Use grep or read offset/limit");
55 + expect(text).not.toContain("line 200");
56 + expect(patched.details.ctxCompactedRead).toBe(true);
57 + });
96 - const patched = await emitBash({ text: raw, fullPath: true });
97 - const text = textOf(patched);
98 - expect(text).toContain("ctx:stored");
99 - expect(text).toContain("Call the ctx tool");
100 - expect(text).not.toContain("bashneedle24680");
101 - expect(JSON.stringify(patched)).not.toContain("fullOutputPath");
102 - expect(patched.details.ctxStored).toBe(true);
103 - expect(dbFiles()).toHaveLength(1);
59 + it("does not compact bounded read windows", async () => {
60 + const raw = "x".repeat(25_000);
61 + const patched = await emitRead({ text: raw, input: { path: "big.txt", offset: 10, limit: 5 } });
62 + expect(patched).toBeUndefined();
63 + });
105 - const search = await ctxExec({ query: "bashneedle24680", limit: 1 });
106 - expect(textOf(search)).toContain("bashneedle24680 terminal fact");
107 - });
109 - it("extracts searchable failure context for huge failing bash output and keeps isError", async () => {
110 - const patched = await emitBash({
111 - text: `${"boring\n".repeat(5_000)}panic ctxboom999 exploded\nstack trace line\n`,
112 - fullPath: true,
113 - isError: true,
114 - command: "run failing suite",
115 - });
117 - expect(patched.isError).toBe(true);
118 - expect(textOf(patched)).toContain("ctx:stored");
119 - const search = await ctxExec({ query: "ctxboom999", limit: 1 });
120 - expect(textOf(search)).toContain("panic ctxboom999 exploded");
121 - });
123 - it("search returns the matching region when a hit is buried deep inside a chunk", async () => {
124 - const filler = Array.from({ length: 180 }, (_, i) => `preface ${i} ${"p".repeat(80)}`).join("\n");
125 - await emitBash({
126 - text: `${filler}\nburiedneedle424242 exact match near end of chunk\n${"tail ".repeat(5_000)}`,
127 - fullPath: false,
128 - });
130 - const search = await ctxExec({ query: "buriedneedle424242", limit: 1 });
131 - const out = textOf(search);
132 - expect(out).toContain("buriedneedle424242 exact match");
133 - expect(out.length).toBeLessThan(3_000);
134 - });
136 - it("collapses actual built-in bash execution output using Pi fullOutputPath metadata", async () => {
137 - const bash = (t!.session.agent as any).state.tools.find((tool: any) => tool.name === "bash");
138 - expect(bash).toBeDefined();
139 - const command = `python3 - <<'PY'\nfor i in range(7000): print(f'realbashneedle777 line {i}')\nPY`;
140 - const raw = await bash.execute("bash-real-1", { command }, undefined, undefined, t!.session.extensionRunner.createContext());
141 - expect(typeof raw.details?.fullOutputPath).toBe("string");
142 - expect(raw.details?.truncation?.truncated).toBe(true);
144 - try {
145 - const patched = await t!.session.extensionRunner.emitToolResult({
146 - type: "tool_result",
147 - toolCallId: "bash-real-1",
148 - toolName: "bash",
149 - input: { command },
150 - content: raw.content,
151 - details: raw.details,
152 - isError: raw.isError,
153 - });
155 - expect(textOf(patched)).toContain("ctx:stored");
156 - expect(textOf(patched)).not.toContain("realbashneedle777 line 6999");
157 - expect(JSON.stringify(patched)).not.toContain("fullOutputPath");
158 - const search = await ctxExec({ query: "realbashneedle777", limit: 1 });
159 - expect(textOf(search)).toContain("realbashneedle777 line");
160 - } finally {
161 - rmSync(raw.details?.fullOutputPath, { force: true });
162 - }
163 - });
165 - it("falls back to visible bash content when fullOutputPath is absent", async () => {
166 - const visible = `${"visible filler\n".repeat(9_000)}visibleonly333 final line`;
167 - const patched = await emitBash({ text: visible });
169 - expect(textOf(patched)).toContain("ctx:stored");
170 - expect(textOf(patched)).not.toContain("visibleonly333");
171 - const search = await ctxExec({ query: "visibleonly333", limit: 1 });
172 - expect(textOf(search)).toContain("visibleonly333 final line");
173 - });
175 - it("handles concurrent bash collapses without lock errors or lost sources", async () => {
176 - const count = 8;
177 - const results = await Promise.all(Array.from({ length: count }, (_, i) => emitBash({
178 - text: `concurrentneedle${i} ${"x".repeat(130_000)}`,
179 - command: `report ${i}`,
180 - })));
182 - for (const result of results) expect(textOf(result)).toContain("ctx:stored");
183 - for (let i = 0; i < count; i++) {
184 - const search = await ctxExec({ query: `concurrentneedle${i}`, limit: 1 });
185 - expect(textOf(search)).toContain(`concurrentneedle${i}`);
186 - }
187 - });
189 - it("keeps unicode output searchable", async () => {
190 - const marker = "unicodeNeedle555 測試 ✅";
191 - await emitBash({ text: `${"😀".repeat(40_000)} ${marker}` });
193 - const search = await ctxExec({ query: "unicodeNeedle555", limit: 1 });
194 - expect(textOf(search)).toContain(marker);
195 - expect(textOf(search)).not.toContain("�");
196 - });
198 - it("closes ctx resources so session directory can be removed after dispose", async () => {
199 - await emitBash({ text: `cleanupneedle ${"x".repeat(120_000)}` });
200 - expect(dbFiles()).toHaveLength(1);
201 - t!.dispose();
202 - t = undefined;
203 - rmSync(root, { recursive: true, force: true });
204 - expect(existsSync(root)).toBe(false);
205 - });
207 - it("does not touch small bash output", async () => {
208 - const unchanged = await emitBash({ text: "small output", command: "echo small" });
209 - expect(unchanged).toBeUndefined();
210 - const search = await ctxExec({ query: "small", limit: 1 });
211 - expect(textOf(search)).toContain("no matches");
212 - });
65 + it("does not compact read errors", async () => {
66 + const raw = "x".repeat(25_000);
67 + const patched = await emitRead({ text: raw, isError: true });
68 + expect(patched).toBeUndefined();
69 + });

extensions/ctx/bun.lock

Modified
7 7 "devDependencies": {
8 8 "@marcfargas/pi-test-harness": "^0.6.1",
10 - "peerDependencies": {
11 - "@earendil-works/pi-coding-agent": "*",
12 - "@earendil-works/pi-tui": "*",
13 - "typebox": "*",
14 - },
17 12 "packages": {

extensions/ctx/index.ts

Modified
1 -import { createHash } from "node:crypto";
2 -import { existsSync, mkdirSync, readFileSync } from "node:fs";
3 -import { basename, dirname, join, resolve } from "node:path";
4 -import { homedir, platform } from "node:os";
5 -import { Type } from "typebox";
6 -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
7 -import { Text } from "@earendil-works/pi-tui";
1 +const DEFAULT_THRESHOLD = 20_000;
9 -const TOOL_NAME = "ctx";
10 -const LARGE_OUTPUT_THRESHOLD = 20_000;
11 -const ERROR_OUTPUT_THRESHOLD = 5_000;
12 -const MAX_CHUNK_BYTES = 16_000;
13 -const MAX_CHUNK_LINES = 200;
14 -const MAX_SNIPPET_BYTES = 2_500;
15 -const ERROR_QUERY = "error failed failure exception panic traceback assertion";
17 -type TextContent = { type: "text"; text: string };
18 -type ToolPatch = { content: TextContent[]; details?: Record<string, unknown>; isError?: boolean };
19 -export type CtxSearchParams = { query?: string | string[]; source?: string; offset?: number; limit?: number };
21 -interface SearchRow {
22 - source_id: number;
23 - title: string;
24 - text: string;
25 - label: string;
26 - kind: string;
27 - score: number;
3 +function limit(): number {
4 + const n = Number(process.env.PI_CTX_COMPACT_THRESHOLD ?? process.env.PI_CTX_READ_OUTPUT_THRESHOLD);
5 + return Number.isFinite(n) && n >= 0 ? n : DEFAULT_THRESHOLD;
30 -function envThreshold(name: string, fallback: number): number {
31 - const raw = process.env[name];
32 - if (raw === undefined) return fallback;
33 - const n = Number(raw);
34 - return Number.isFinite(n) && n >= 0 ? n : fallback;
8 +function textOf(x: unknown): string {
9 + if (typeof x === "string") return x;
10 + if (!Array.isArray(x)) return String(x ?? "");
11 + return x.map((p: any) => p?.type === "text" && typeof p.text === "string" ? p.text : "").join("\n");
37 -function largeOutputThreshold(): number {
38 - return envThreshold("PI_CTX_LARGE_OUTPUT_THRESHOLD", LARGE_OUTPUT_THRESHOLD);
41 -function errorOutputThreshold(): number {
42 - return envThreshold("PI_CTX_ERROR_OUTPUT_THRESHOLD", ERROR_OUTPUT_THRESHOLD);
45 -function byteLen(text: string): number {
46 - return Buffer.byteLength(text, "utf8");
49 -function truncateBytes(text: string, max = MAX_SNIPPET_BYTES): string {
50 - if (byteLen(text) <= max) return text;
51 - let out = "";
52 - let used = 0;
53 - for (const ch of text) {
54 - const n = byteLen(ch);
55 - if (used + n > max) break;
56 - out += ch;
57 - used += n;
58 - }
59 - return `${out}\n… truncated to ${max}B`;
62 -function textOf(content: unknown): string {
63 - if (!Array.isArray(content)) return String(content ?? "");
64 - return content.map((c: any) => c?.type === "text" && typeof c.text === "string" ? c.text : "").join("\n");
67 -function shortHash(text: string): string {
68 - return createHash("sha256").update(text).digest("hex").slice(0, 8);
71 -function queryTerms(query: string): string[] {
72 - return [...query.matchAll(/[\p{L}\p{N}_]{2,}/gu)]
73 - .map(match => match[0].toLowerCase())
74 - .filter(term => !["and", "or", "not", "near"].includes(term));
77 -function ftsQuery(query: string): string {
78 - const terms = queryTerms(query).slice(0, 24);
79 - if (terms.length === 0) return `"${query.replaceAll('"', '""')}"`;
80 - return terms.map(term => `"${term.replaceAll('"', '""')}"`).join(" OR ");
83 -function snippetForQuery(text: string, query: string): string {
84 - if (byteLen(text) <= MAX_SNIPPET_BYTES) return text.trim();
85 - const lower = text.toLowerCase();
86 - let pos = -1;
87 - for (const term of queryTerms(query)) {
88 - const next = lower.indexOf(term);
89 - if (next >= 0 && (pos < 0 || next < pos)) pos = next;
90 - }
91 - if (pos < 0) return truncateBytes(text.trim());
92 - let start = Math.max(0, pos - Math.floor(MAX_SNIPPET_BYTES / 3));
93 - const newline = text.lastIndexOf("\n", pos);
94 - if (newline >= 0 && newline >= start - 500) start = newline + 1;
95 - let snippet = text.slice(start);
96 - if (start > 0) snippet = `…\n${snippet}`;
97 - return truncateBytes(snippet.trim());
100 -function chunkText(text: string): Array<{ title: string; text: string }> {
14 +export function compactRead(text: string): string {
101 15 const lines = text.split(/\r?\n/);
102 - const chunks: Array<{ title: string; text: string }> = [];
103 - let current: string[] = [];
104 - let currentBytes = 0;
105 - let start = 1;
106 - const flush = (end: number) => {
107 - if (current.length === 0) return;
108 - chunks.push({ title: `lines ${start}-${end}`, text: current.join("\n") });
109 - current = [];
110 - currentBytes = 0;
111 - start = end + 1;
112 - };
113 - for (let i = 0; i < lines.length; i++) {
114 - const line = lines[i] ?? "";
115 - const n = byteLen(line) + 1;
116 - if (current.length > 0 && (current.length >= MAX_CHUNK_LINES || currentBytes + n > MAX_CHUNK_BYTES)) flush(i);
117 - current.push(line);
118 - currentBytes += n;
119 - }
120 - flush(lines.length);
121 - return chunks.length ? chunks : [{ title: "output", text }];
16 + return [
17 + lines.slice(0, 8).join("\n"),
18 + `[ctx compacted read: ${Math.max(0, lines.length - 40)} lines omitted]`,
19 + "Use grep or read offset/limit for hidden target text before editing.",
20 + lines.slice(-32).join("\n"),
21 + ].join("\n");
124 -export function stateRoot(env: NodeJS.ProcessEnv = process.env, os = platform(), home = homedir()): string {
125 - if (os === "win32") return join(env.LOCALAPPDATA ?? join(home, "AppData", "Local"), "pi");
126 - if (os === "darwin") return join(home, "Library", "Application Support", "pi");
127 - return join(env.XDG_STATE_HOME ?? join(home, ".local", "state"), "pi");
24 +export function handleToolResult(event: any) {
25 + if (event?.toolName !== "read" || event.isError || event.input?.limit !== undefined || event.input?.l !== undefined) return;
26 + const text = textOf(event.content);
27 + if (Buffer.byteLength(text, "utf8") <= limit()) return;
28 + return { content: [{ type: "text", text: compactRead(text) }], details: { ...(event.details ?? {}), ctxCompactedRead: true }, isError: false };
130 -export function defaultDbPath(cwd = process.cwd(), ctx?: any): string {
131 - if (process.env.PI_CTX_DB_PATH) return process.env.PI_CTX_DB_PATH;
132 - const sessionFile = ctx?.sessionManager?.getSessionFile?.()
133 - ?? ctx?.sessionManager?.sessionFile
134 - ?? ctx?.sessionFile;
135 - if (typeof sessionFile !== "string" || !sessionFile) throw new Error("ctx requires a Pi session file; set PI_CTX_DB_PATH for no-session runs");
136 - const sessionId = basename(sessionFile).replace(/\.jsonl$/, "");
137 - const cwdHash = createHash("sha256").update(resolve(cwd)).digest("hex").slice(0, 16);
138 - const base = join(stateRoot(), "ctx");
139 - mkdirSync(base, { recursive: true });
140 - return join(base, `${sessionId}-${cwdHash}.db`);
143 -export class CtxStore {
144 - #db: any;
145 - readonly dbPath: string;
147 - constructor(dbPath = defaultDbPath()) {
148 - this.dbPath = dbPath;
149 - mkdirSync(dirname(dbPath), { recursive: true });
150 - const { Database } = require("bun:sqlite") as typeof import("bun:sqlite");
151 - this.#db = new Database(dbPath);
152 - this.#db.exec("PRAGMA journal_mode=WAL");
153 - this.#db.exec("PRAGMA busy_timeout=30000");
154 - this.#db.exec(`
155 - CREATE TABLE IF NOT EXISTS sources (
156 - id INTEGER PRIMARY KEY AUTOINCREMENT,
157 - kind TEXT NOT NULL,
158 - label TEXT NOT NULL,
159 - bytes INTEGER NOT NULL,
160 - metadata TEXT,
161 - created_at TEXT NOT NULL DEFAULT (datetime('now'))
162 - );
163 - CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
164 - source_id UNINDEXED,
165 - title,
166 - text,
167 - label UNINDEXED,
168 - kind UNINDEXED,
169 - tokenize='porter unicode61'
170 - );
171 - `);
172 - }
174 - indexText(input: { kind: string; label: string; text: string; metadata?: unknown }): { sourceId: number; chunkCount: number; bytes: number; label: string } {
175 - const bytes = byteLen(input.text);
176 - const meta = input.metadata === undefined ? null : JSON.stringify(input.metadata);
177 - const source = this.#db.prepare("INSERT INTO sources(kind, label, bytes, metadata) VALUES (?, ?, ?, ?)")
178 - .run(input.kind, input.label, bytes, meta);
179 - const sourceId = Number(source.lastInsertRowid);
180 - const label = `${input.label}:${sourceId}`;
181 - this.#db.prepare("UPDATE sources SET label = ? WHERE id = ?").run(label, sourceId);
182 - const insert = this.#db.prepare("INSERT INTO chunks_fts(source_id, title, text, label, kind) VALUES (?, ?, ?, ?, ?)");
183 - const chunks = chunkText(input.text);
184 - const tx = this.#db.transaction(() => {
185 - for (const chunk of chunks) insert.run(sourceId, chunk.title, chunk.text, label, input.kind);
186 - });
187 - tx();
188 - return { sourceId, chunkCount: chunks.length, bytes, label };
189 - }
191 - search(queries: string[], opts: { source?: string; limit?: number } = {}): string {
192 - const clean = queries.map(q => q.trim()).filter(Boolean).slice(0, 8);
193 - if (clean.length === 0) return "ctx: empty query";
194 - const limit = Math.max(1, Math.min(opts.limit ?? 3, 10));
195 - const source = opts.source?.trim();
196 - const seen = new Set<string>();
197 - const lines: string[] = [];
198 - for (const query of clean) {
199 - const match = ftsQuery(query);
200 - const rows = source
201 - ? this.#db.prepare("SELECT source_id, title, text, label, kind, bm25(chunks_fts) AS score FROM chunks_fts WHERE chunks_fts MATCH ? AND label = ? ORDER BY score LIMIT ?").all(match, source, limit) as SearchRow[]
202 - : this.#db.prepare("SELECT source_id, title, text, label, kind, bm25(chunks_fts) AS score FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY score LIMIT ?").all(match, limit) as SearchRow[];
203 - lines.push(`## ${query}`);
204 - let emitted = 0;
205 - for (const row of rows) {
206 - const key = `${row.source_id}:${row.title}`;
207 - if (seen.has(key)) continue;
208 - seen.add(key);
209 - emitted++;
210 - lines.push(`### ${row.label} · ${row.title}`);
211 - lines.push(snippetForQuery(row.text, query));
212 - }
213 - if (emitted === 0) lines.push(rows.length === 0 ? "no matches" : "no new matches");
214 - }
215 - return lines.join("\n");
216 - }
218 - range(opts: { source: string; offset?: number; limit?: number }): string {
219 - const source = opts.source.trim();
220 - if (!source) return "ctx: source required";
221 - const offset = Math.max(1, Math.floor(opts.offset ?? 1));
222 - const limit = Math.max(1, Math.min(Math.floor(opts.limit ?? MAX_CHUNK_LINES), MAX_CHUNK_LINES));
223 - const end = offset + limit - 1;
224 - const rows = this.#db.prepare("SELECT source_id, title, text, label, kind FROM chunks_fts WHERE label = ? ORDER BY rowid").all(source) as SearchRow[];
225 - if (rows.length === 0) return `ctx: source not found: ${source}`;
226 - const lines: string[] = [`## ${source} · lines ${offset}-${end}`];
227 - let emitted = 0;
228 - for (const row of rows) {
229 - const match = /^lines (\d+)-(\d+)$/.exec(row.title);
230 - if (!match) continue;
231 - const chunkStart = Number(match[1]);
232 - const chunkEnd = Number(match[2]);
233 - if (chunkEnd < offset || chunkStart > end) continue;
234 - const chunkLines = row.text.split(/\r?\n/);
235 - const sliceStart = Math.max(0, offset - chunkStart);
236 - const sliceEnd = Math.min(chunkLines.length, end - chunkStart + 1);
237 - lines.push(`### ${row.label} · lines ${chunkStart + sliceStart}-${chunkStart + sliceEnd - 1}`);
238 - lines.push(chunkLines.slice(sliceStart, sliceEnd).join("\n"));
239 - emitted += sliceEnd - sliceStart;
240 - }
241 - if (emitted === 0) lines.push("no lines in range");
242 - return lines.join("\n");
243 - }
245 - pointer(indexed: { chunkCount: number; bytes: number; label: string }): string {
246 - return `ctx:stored ${indexed.label} (${(indexed.bytes / 1024).toFixed(1)}KB, ${indexed.chunkCount} chunks).\nStored output is not in the workspace. Call the ctx tool with {"query":"...","source":"${indexed.label}"}.`;
247 - }
249 - close(): void {
250 - this.#db.close();
251 - }
254 -function makeStore(ctx?: any): CtxStore {
255 - return new CtxStore(defaultDbPath(ctx?.cwd ?? process.cwd(), ctx));
258 -function normalizeQueries(query: string | string[]): string[] {
259 - return Array.isArray(query) ? query.map(q => q.trim()).filter(Boolean) : [query.trim()].filter(Boolean);
262 -function normalizeSearchParams(raw: any): CtxSearchParams {
263 - if (!raw || typeof raw !== "object") throw new Error("ctx params object required");
264 - if (typeof raw.query === "string" || Array.isArray(raw.query) || typeof raw.source === "string") return raw;
265 - throw new Error("ctx requires query, or source for line ranges");
268 -function searchDetails(params: CtxSearchParams, text: string): Record<string, unknown> {
269 - const query = params.query;
270 - return {
271 - mode: query === undefined ? "range" : "search",
272 - source: params.source,
273 - queryCount: Array.isArray(query) ? query.length : query ? 1 : 0,
274 - offset: params.offset,
275 - limit: params.limit,
276 - bytes: byteLen(text),
277 - sections: (text.match(/^### /gm) ?? []).length,
278 - };
281 -export function executeCtxSearch(store: CtxStore, params: CtxSearchParams): ToolPatch {
282 - const text = params.query === undefined
283 - ? store.range({ source: params.source ?? "", offset: params.offset, limit: params.limit })
284 - : store.search(normalizeQueries(params.query), { source: params.source, limit: params.limit });
285 - return { content: [{ type: "text", text }], details: searchDetails(params, text) };
288 -export function handleBashToolResult(store: CtxStore, event: any): ToolPatch | undefined {
289 - if (event?.toolName !== "bash") return undefined;
290 - const visible = textOf(event.content);
291 - let full = visible;
292 - const fullPath = event.details?.fullOutputPath;
293 - if (typeof fullPath === "string" && existsSync(fullPath)) full = readFileSync(fullPath, "utf8");
294 - const bytes = byteLen(full);
295 - if (!event.isError && bytes <= largeOutputThreshold()) return undefined;
296 - if (event.isError && bytes <= errorOutputThreshold()) return undefined;
297 - const command = String(event.input?.command ?? "bash");
298 - const label = `${event.isError ? "bash-error" : "bash"}:${shortHash(command)}`;
299 - const indexed = store.indexText({ kind: event.isError ? "bash-error" : "bash", label, text: full, metadata: { command } });
300 - const text = event.isError
301 - ? `${store.pointer(indexed)}\n\n${store.search([ERROR_QUERY], { source: label, limit: 3 })}`
302 - : store.pointer(indexed);
303 - const { fullOutputPath: _fullOutputPath, ...safeDetails } = event.details ?? {};
304 - return {
305 - content: [{ type: "text", text }],
306 - details: { ...safeDetails, ctxStored: true, ctxSource: label },
307 - isError: event.isError,
308 - };
311 -const Params = Type.Object({
312 - query: Type.Optional(Type.Union([Type.String(), Type.Array(Type.String())], { description: "Search query. Omit for source line range reads." })),
313 - source: Type.Optional(Type.String({ description: "Exact source label from ctx:stored" })),
314 - offset: Type.Optional(Type.Number({ description: "1-indexed source line offset for range reads. Requires source and no query." })),
315 - limit: Type.Optional(Type.Number({ description: "Search: max results per query. Range: max lines to return." })),
316 -});
318 -export function registerCtxExtension(pi: Pick<ExtensionAPI, "registerTool" | "on">): void {
319 - pi.on("tool_result", (event: any, ctx: any) => {
320 - let store: CtxStore | undefined;
321 - try {
322 - store = makeStore(ctx);
323 - return handleBashToolResult(store, event);
324 - } catch (err) {
325 - const visible = textOf(event?.content);
326 - const maybeLarge = event?.toolName === "bash" && (event?.details?.fullOutputPath || byteLen(visible) > largeOutputThreshold());
327 - if (!maybeLarge) return undefined;
328 - const { fullOutputPath: _fullOutputPath, ...safeDetails } = event.details ?? {};
329 - return {
330 - content: [{ type: "text", text: `ctx:suppressed bash output; storage failed (${err instanceof Error ? err.message : String(err)})` }],
331 - details: { ...safeDetails, ctxSuppressed: true },
332 - isError: event.isError,
333 - };
334 - } finally {
335 - store?.close();
336 - }
337 - });
339 - pi.registerTool({
340 - name: TOOL_NAME,
341 - label: "ctx search",
342 - description: "Search or range-read command output that ctx previously stored after it exceeded the safe context threshold.",
343 - promptSnippet: "Use only after a tool result says ctx:stored. Search with query+source, or read known lines with source+offset+limit.",
344 - promptGuidelines: [
345 - "Use ctx only after a tool result says ctx:stored.",
346 - "Use ctx query+source for targeted searches in suppressed bash output.",
347 - "Use ctx source+offset+limit for known line ranges; avoid broad query lists that recreate the whole output.",
348 - ],
349 - parameters: Params,
350 - async execute(_toolCallId: string, raw: unknown, _signal?: AbortSignal, _onUpdate?: unknown, ctx?: any) {
351 - const store = makeStore(ctx);
352 - try {
353 - return executeCtxSearch(store, normalizeSearchParams(raw));
354 - } catch (err) {
355 - return { content: [{ type: "text", text: `ctx error: ${err instanceof Error ? err.message : String(err)}` }], details: {}, isError: true };
356 - } finally {
357 - store.close();
358 - }
359 - },
360 - renderCall(args: CtxSearchParams, theme: any) {
361 - const source = args.source ? theme.fg("accent", args.source) : theme.fg("dim", "all sources");
362 - let text = theme.fg("toolTitle", theme.bold("ctx ")) + source;
363 - if (args.query !== undefined) {
364 - const queries = Array.isArray(args.query) ? args.query : [args.query];
365 - const first = queries[0] ?? "";
366 - text += " " + theme.fg("muted", `search ${JSON.stringify(first)}`);
367 - if (queries.length > 1) text += " " + theme.fg("dim", `+${queries.length - 1}`);
368 - } else {
369 - const offset = Math.max(1, Math.floor(args.offset ?? 1));
370 - const limit = Math.max(1, Math.floor(args.limit ?? MAX_CHUNK_LINES));
371 - text += " " + theme.fg("muted", `lines ${offset}-${offset + limit - 1}`);
372 - }
373 - return new Text(text, 0, 0);
374 - },
375 - renderResult(result: ToolPatch, { expanded, isPartial }: { expanded: boolean; isPartial: boolean }, theme: any) {
376 - if (isPartial) return new Text(theme.fg("warning", "Searching ctx..."), 0, 0);
377 - const details = result.details as Record<string, any> | undefined;
378 - const content = textOf(result.content);
379 - const sections = details?.sections ?? (content.match(/^### /gm) ?? []).length;
380 - let text = theme.fg("success", `ctx ${details?.mode ?? "result"}`);
381 - text += theme.fg("muted", ` · ${sections} section${sections === 1 ? "" : "s"}`);
382 - if (typeof details?.bytes === "number") text += theme.fg("dim", ` · ${(details.bytes / 1024).toFixed(1)}KB`);
383 - if (expanded) text += `\n${theme.fg("dim", content)}`;
384 - return new Text(text, 0, 0);
385 - },
386 - });
389 -export default function (pi: ExtensionAPI): void {
390 - registerCtxExtension(pi);
31 +export const __test = { compactRead, handleToolResult };
32 +export default (pi: any) => pi.on("tool_result", handleToolResult);

extensions/ctx/package.json

Modified
6 6 "test": "bun test ./__tests__/*.test.ts",
7 7 "test:harness": "bun test ./__tests__/harness.test.ts"
9 - "peerDependencies": {
10 - "@earendil-works/pi-coding-agent": "*",
11 - "@earendil-works/pi-tui": "*",
12 - "typebox": "*"
13 - },
14 9 "devDependencies": {
15 10 "@marcfargas/pi-test-harness": "^0.6.1"
17 12 "license": "MIT",
18 - "description": "Context blowup fuse and search for oversized Pi bash output.",
13 + "description": "Compacts oversized Pi read results before they hit context.",
19 14 "keywords": [
20 15 "pi",
21 16 "pi-extension"