repositories / pi-ext

pi-ext

bugabingas pi extensions

owned by bugabinga

perf(cmprss): retain structured outliers

589854dc9304b6f3239abd3d74bc49fcf09954e8

Oliver Jan Krylow <oliver@bugabinga.net> ·

SSH signature · not checked

Browse filesCompare with HEAD

Changed files (3)

extensions/cmprss/__tests__/ab.test.ts

Modified
53 - test("requires retrieval for omitted facts while preserving structural signals", () => {
53 + test("preserves structured outliers and retrieves omitted unstructured facts", () => {
54 54 const compressed = Object.fromEntries(
55 55 Object.entries(benchmark.files).map(([path, content]) => [
56 56 path,
61 - expect(compressed["fixtures/data.json"]).not.toContain(
61 + expect(compressed["fixtures/data.json"]).toContain(
62 62 benchmark.expected.jsonCode,
64 - expect(compressed["fixtures/search.txt"]).not.toContain(
64 + expect(compressed["fixtures/search.txt"]).toContain(
65 65 benchmark.expected.searchNeedle,
67 67 expect(compressed["fixtures/narrative.txt"]).not.toContain(

extensions/cmprss/__tests__/cmprss.test.ts

Modified
350 350 expect(text).toContain("json array: 40 items");
353 + test("json array compression remains bounded for duplicate scalars", () => {
354 + const runtime = createRuntime({ ...DEFAULT_CONFIG, thresholdBytes: 80 });
355 + const result = compressToolResult(runtime, {
356 + toolName: "bash",
357 + input: {},
358 + content: [
359 + { type: "text", text: JSON.stringify(Array(1_000).fill("same")) },
360 + ],
361 + isError: false,
362 + });
364 + expect(Buffer.byteLength(result?.content?.[0]?.text ?? "")).toBeLessThan(
365 + 1_000,
366 + );
367 + });
369 + test("json object compression preserves structurally rare array items", () => {
370 + const runtime = createRuntime({ ...DEFAULT_CONFIG, thresholdBytes: 80 });
371 + const rows = Array.from({ length: 40 }, (_, i) => ({
372 + id: i + 1,
373 + status: "ready",
374 + ...(i === 20 ? { releaseCode: "RARE-21" } : {}),
375 + }));
376 + const result = compressToolResult(runtime, {
377 + toolName: "bash",
378 + input: {},
379 + content: [{ type: "text", text: JSON.stringify({ rows }) }],
380 + isError: false,
381 + });
383 + expect(result?.content?.[0]?.text).toContain('"releaseCode": "RARE-21"');
384 + });
353 386 test("search compression preserves paths and line numbers", () => {
354 387 const runtime = createRuntime({ ...DEFAULT_CONFIG, thresholdBytes: 80 });
355 388 const text = Array.from(
368 401 expect(result?.content?.[0]?.text).toContain("search: 80 matches");
404 + test("search compression preserves a short middle outlier", () => {
405 + const runtime = createRuntime({ ...DEFAULT_CONFIG, thresholdBytes: 80 });
406 + const text = Array.from({ length: 80 }, (_, i) =>
407 + i === 40
408 + ? "src/a.ts:41:needle=RARE-41"
409 + : `src/a.ts:${i + 1}:ordinary match ${"x".repeat(80)}`,
410 + ).join("\n");
411 + const result = compressToolResult(runtime, {
412 + toolName: "grep",
413 + input: {},
414 + content: [{ type: "text", text }],
415 + isError: false,
416 + });
418 + expect(result?.content?.[0]?.text).toContain("src/a.ts:41:needle=RARE-41");
419 + });
371 421 test("stacktrace compression wins before search-like frame paths", () => {
372 422 const runtime = createRuntime({ ...DEFAULT_CONFIG, thresholdBytes: 80 });
373 423 const text = [

extensions/cmprss/index.ts

Modified
151 151 return JSON.stringify(value, null, 2);
154 +function objectShape(value: unknown): string | undefined {
155 + return isRecord(value) ? Object.keys(value).sort().join("\0") : undefined;
158 +function summarizeJsonArray(items: unknown[]): {
159 + values: unknown[];
160 + kept: number;
161 +} {
162 + const shapes = new Map<string, number>();
163 + for (const item of items) {
164 + const shape = objectShape(item);
165 + if (shape) shapes.set(shape, (shapes.get(shape) ?? 0) + 1);
167 + const selected = new Set<number>();
168 + for (let index = 0; index < Math.min(5, items.length); index++)
169 + selected.add(index);
170 + for (let index = Math.max(0, items.length - 5); index < items.length; index++)
171 + selected.add(index);
172 + let errors = 0;
173 + let rareShapes = 0;
174 + for (const [index, item] of items.entries()) {
175 + if (errors < 10 && containsError(item)) {
176 + selected.add(index);
177 + errors++;
179 + const shape = objectShape(item);
180 + if (rareShapes < 10 && shape !== undefined && shapes.get(shape) === 1) {
181 + selected.add(index);
182 + rareShapes++;
185 + const values: unknown[] = [];
186 + let omitted = 0;
187 + for (const [index, item] of items.entries()) {
188 + if (!selected.has(index)) {
189 + omitted++;
190 + continue;
192 + if (omitted > 0) {
193 + values.push(`[cmprss omitted ${omitted} items]`);
194 + omitted = 0;
196 + values.push(item);
198 + if (omitted > 0) values.push(`[cmprss omitted ${omitted} items]`);
199 + return { values, kept: selected.size };
154 202 function compressJson(text: string): Compression | undefined {
155 203 let parsed: unknown;
159 207 return undefined;
161 209 if (Array.isArray(parsed)) {
162 - const first = parsed.slice(0, 5);
163 - const last = parsed.slice(-5);
164 - const errors = parsed.filter(containsError).slice(0, 10);
165 - const kept = uniq([...first, ...errors, ...last]);
210 + const summary = summarizeJsonArray(parsed);
166 211 return {
167 212 compressor: "json",
168 213 text: [
169 - `[cmprss json array: ${parsed.length} items, kept ${kept.length}]`,
170 - formatJson(kept),
214 + `[cmprss json array: ${parsed.length} items, kept ${summary.kept}]`,
215 + formatJson(summary.values),
171 216 ].join("\n"),
178 223 out[key] =
179 224 `${value.slice(0, 100)} …[cmprss ${value.length - 140} chars]… ${value.slice(-40)}`;
180 225 } else if (Array.isArray(value) && value.length > 12) {
181 - out[key] = [
182 - ...value.slice(0, 5),
183 - `[cmprss omitted ${value.length - 10} items]`,
184 - ...value.slice(-5),
185 - ];
226 + out[key] = summarizeJsonArray(value).values;
186 227 } else out[key] = value;
188 229 return {
223 264 const selected = uniq([
224 265 ...group.slice(0, 2),
225 266 ...group.filter((m) => ERROR_RE.test(m.body)).slice(0, 5),
267 + ...[...group].sort((a, b) => a.body.length - b.body.length).slice(0, 2),
226 268 ...group.slice(-2),
228 - for (const match of selected.slice(0, 5)) kept.push(match.raw);
270 + for (const match of selected.slice(0, 11)) kept.push(match.raw);
230 272 return {
231 273 compressor: "search",