pi-ext
bugabingas pi extensions
owned by bugabinga
perf(cmprss): retain structured outliers
589854dc9304b6f3239abd3d74bc49fcf09954e8
Oliver Jan Krylow <oliver@bugabinga.net> ·
SSH signature · not checked
Browse filesCompare with HEAD
Changed files (3)
extensions/cmprss/__tests__/ab.test.ts
Modified 53 - test(
"requires retrieval for omitted facts while preserving structural signals", ()
=> {
53 + test(
"preserves structured outliers and retrieves omitted unstructured facts", ()
=> {
54 54 const compressed
= Object.
fromEntries(
55 55 Object.
entries(benchmark.files).
map(([path, content])
=> [
61 - expect(compressed[
"fixtures/data.json"]).not.
toContain(
61 + expect(compressed[
"fixtures/data.json"]).
toContain(
62 62 benchmark.expected.jsonCode,
64 - expect(compressed[
"fixtures/search.txt"]).not.
toContain(
64 + expect(compressed[
"fixtures/search.txt"]).
toContain(
65 65 benchmark.expected.searchNeedle,
67 67 expect(compressed[
"fixtures/narrative.txt"]).not.
toContain(
extensions/cmprss/__tests__/cmprss.test.ts
Modified350 350 expect(text).
toContain(
"json array: 40 items");
353 + test(
"json array compression remains bounded for duplicate scalars", ()
=> {
354 + const runtime
= createRuntime({
...DEFAULT_CONFIG, thresholdBytes:
80 });
355 + const result
= compressToolResult(runtime, {
359 + { type:
"text", text:
JSON.
stringify(
Array(
1_000).
fill(
"same")) },
364 + expect(
Buffer.
byteLength(result?.
content?.[
0]?.
text ?? "")).
toBeLessThan(
369 + test(
"json object compression preserves structurally rare array items", ()
=> {
370 + const runtime
= createRuntime({
...DEFAULT_CONFIG, thresholdBytes:
80 });
371 + const rows
= Array.
from({ length:
40 }, (_, i)
=> ({
374 + ...(i
=== 20 ? { releaseCode:
"RARE-21" }
: {}),
376 + const result
= compressToolResult(runtime, {
379 + content: [{ type:
"text", text:
JSON.
stringify({ rows }) }],
383 + expect(result?.
content?.[
0]?.
text).
toContain(
'"releaseCode": "RARE-21"');
353 386 test(
"search compression preserves paths and line numbers", ()
=> {
354 387 const runtime
= createRuntime({
...DEFAULT_CONFIG, thresholdBytes:
80 });
355 388 const text
= Array.
from(
368 401 expect(result?.
content?.[
0]?.
text).
toContain(
"search: 80 matches");
404 + test(
"search compression preserves a short middle outlier", ()
=> {
405 + const runtime
= createRuntime({
...DEFAULT_CONFIG, thresholdBytes:
80 });
406 + const text
= Array.
from({ length:
80 }, (_, i)
=> 408 + ? "src/a.ts:41:needle=RARE-41" 409 + : `src/a.ts:${i + 1}:ordinary match ${"x".repeat(80)}`,
411 + const result
= compressToolResult(runtime, {
414 + content: [{ type:
"text", text }],
418 + expect(result?.
content?.[
0]?.
text).
toContain(
"src/a.ts:41:needle=RARE-41");
371 421 test(
"stacktrace compression wins before search-like frame paths", ()
=> {
372 422 const runtime
= createRuntime({
...DEFAULT_CONFIG, thresholdBytes:
80 });
extensions/cmprss/index.ts
Modified151 151 return JSON.
stringify(value,
null,
2);
154 +function objectShape(value
: unknown)
: string | undefined {
155 + return isRecord(value)
? Object.
keys(value).
sort().
join(
"\0")
: undefined;
158 +function summarizeJsonArray(items
: unknown[])
: {
162 + const shapes
= new Map<
string,
number>();
163 + for (
const item
of items) {
164 + const shape
= objectShape(item);
165 + if (shape) shapes.
set(shape, (shapes.
get(shape)
?? 0)
+ 1);
167 + const selected
= new Set<
number>();
168 + for (
let index
= 0; index
< Math.
min(
5, items.
length); index
++)
169 + selected.
add(index);
170 + for (
let index
= Math.
max(
0, items.
length - 5); index
< items.
length; index
++)
171 + selected.
add(index);
173 + let rareShapes
= 0;
174 + for (
const [index, item]
of items.
entries()) {
175 + if (errors
< 10 && containsError(item)) {
176 + selected.
add(index);
179 + const shape
= objectShape(item);
180 + if (rareShapes
< 10 && shape
!== undefined && shapes.
get(shape)
=== 1) {
181 + selected.
add(index);
185 + const values
: unknown[]
= [];
187 + for (
const [index, item]
of items.
entries()) {
188 + if (
!selected.
has(index)) {
193 + values.
push(
`[cmprss omitted ${omitted} items]`);
198 + if (omitted
> 0) values.
push(
`[cmprss omitted ${omitted} items]`);
199 + return { values, kept: selected.
size };
154 202 function compressJson(text
: string)
: Compression | undefined {
155 203 let parsed
: unknown;
159 207 return undefined;
161 209 if (
Array.
isArray(parsed)) {
162 - const first
= parsed.
slice(
0,
5);
163 - const last
= parsed.
slice(
-5);
164 - const errors
= parsed.
filter(containsError).
slice(
0,
10);
165 - const kept
= uniq([
...first,
...errors,
...last]);
210 + const summary
= summarizeJsonArray(parsed);
167 212 compressor:
"json",
169 - `[cmprss json array: ${parsed.length} items, kept ${kept.length}]`,
214 + `[cmprss json array: ${parsed.length} items, kept ${summary.kept}]`,
215 + formatJson(summary.values),
179 224 `${value.slice(0, 100)} …[cmprss ${value.length - 140} chars]… ${value.slice(-40)}`;
180 225 }
else if (
Array.
isArray(value)
&& value.
length > 12) {
182 - ...value.
slice(
0,
5),
183 - `[cmprss omitted ${value.length - 10} items]`,
184 - ...value.
slice(
-5),
226 + out[key]
= summarizeJsonArray(value).values;
186 227 }
else out[key]
= value;
223 264 const selected
= uniq([
224 265 ...group.
slice(
0,
2),
225 266 ...group.
filter((m)
=> ERROR_RE.
test(m.
body)).
slice(
0,
5),
267 + ...[
...group].
sort((a, b)
=> a.
body.
length - b.
body.
length).
slice(
0,
2),
226 268 ...group.
slice(
-2),
228 - for (
const match
of selected.
slice(
0,
5)) kept.
push(match.raw);
270 + for (
const match
of selected.
slice(
0,
11)) kept.
push(match.raw);
231 273 compressor:
"search",