Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-bench/budgets.toml

Raw
# Callgrind instruction budgets (Ir) per bench, gated within ±2 % by `cargo x perf`.
# Written by `cargo x perf --pin`; a re-pin is a deliberate commit that says why.
# Pinned with rustc 1.99.0 (b940084d7 2026-09-28) | glibc 2.41.

[budgets]
"instructions::agent::run_turn::history_1000_masked" = { ir = 1876260 }
"instructions::providers::sse::anthropic" = { ir = 116970 }
"instructions::providers::sse::gemini" = { ir = 51812 }
"instructions::providers::sse::openai" = { ir = 87904 }
"instructions::rpc::handle_line::ping_batch_128" = { ir = 961046 }
"instructions::session::canonical_json::nested" = { ir = 834862 }
"instructions::session::decode::linear_1000" = { ir = 1133717 }
"instructions::session::decode::tool_calls_1000" = { ir = 1120575 }
"instructions::session::encode_preallocated::linear_1000" = { ir = 814084 }
"instructions::session::encode_preallocated::tool_calls_1000" = { ir = 806364 }
"instructions::session::fork_at::linear_1000" = { ir = 167086 }
"instructions::session::from_frames::linear_1000" = { ir = 1394513 }
"startup::cli::smith::eval_mock" = { ir = 1101944 }
"startup::cli::smith::help" = { ir = 614764 }
"instructions::identity::proquint_decode::text" = { ir = 278 }
"instructions::identity::proquint_decode_floor::decode" = { ir = 51 }
"instructions::identity::proquint_encode::bits" = { ir = 615 }
"instructions::identity::proquint_encode_floor::encode" = { ir = 35 }
"instructions::providers::sse_floor::anthropic" = { ir = 161 }
"instructions::providers::sse_floor::gemini" = { ir = 89 }
"instructions::providers::sse_floor::openai" = { ir = 125 }
"instructions::session::canonical_json_floor::nested" = { ir = 54170 }
"instructions::session::decode_floor::linear_1000" = { ir = 146930 }
"instructions::session::decode_floor::tool_calls_1000" = { ir = 139678 }

[floors]
"instructions::providers::sse::anthropic" = "instructions::providers::sse_floor::anthropic"
"instructions::providers::sse::gemini" = "instructions::providers::sse_floor::gemini"
"instructions::providers::sse::openai" = "instructions::providers::sse_floor::openai"
"instructions::session::canonical_json::nested" = "instructions::session::canonical_json_floor::nested"
"instructions::session::decode::linear_1000" = "instructions::session::decode_floor::linear_1000"
"instructions::session::decode::tool_calls_1000" = "instructions::session::decode_floor::tool_calls_1000"
"instructions::identity::proquint_encode::bits" = "instructions::identity::proquint_encode_floor::encode"
"instructions::identity::proquint_decode::text" = "instructions::identity::proquint_decode_floor::decode"

[gaps]
"instructions::agent::run_turn::history_1000_masked" = "Gap unknown: no appropriate matched model for request decoding, masking, and recording task state."
"instructions::providers::sse::anthropic" = "Low confidence: byte-transfer reference excludes scan, parse, validation, and events; any large ratio is not justified by framing alone, and fixture-size scaling evidence is insufficient."
"instructions::providers::sse::gemini" = "Low confidence: byte-transfer reference excludes scan, parse, validation, and events; any large ratio is not justified by framing alone, and fixture-size scaling evidence is insufficient."
"instructions::providers::sse::openai" = "Low confidence: byte-transfer reference excludes scan, parse, validation, and events; any large ratio is not justified by framing alone, and fixture-size scaling evidence is insufficient."
"instructions::rpc::handle_line::ping_batch_128" = "Gap unknown: no appropriate matched model for per-line parsing, dispatch, response construction, and serialization."
"instructions::session::canonical_json::nested" = "Medium confidence: EntryFrame::new clones and sorts Value objects, then encodes CBOR; payload visit excludes those task costs and is a useful comparison, not a hardware limit."
"instructions::session::decode::linear_1000" = "Medium confidence: resident byte-transfer model excludes framing, CRC, parsing, validation, and record construction; ratio measures task work above movement, not an optimum."
"instructions::session::decode::tool_calls_1000" = "Medium confidence: resident byte-transfer model excludes framing, CRC, parsing, validation, and record construction; ratio measures task work above movement, not an optimum."
"instructions::session::encode_preallocated::linear_1000" = "Gap unknown: no appropriate matched model for encoding and checksum construction."
"instructions::session::encode_preallocated::tool_calls_1000" = "Gap unknown: no appropriate matched model for encoding and checksum construction."
"instructions::session::fork_at::linear_1000" = "Gap unknown: no appropriate matched model for branch selection and record ownership."
"instructions::session::from_frames::linear_1000" = "Gap unknown: no appropriate matched model for restore validation and record reconstruction."
"startup::cli::smith::eval_mock" = "Gap unknown: no appropriate matched model for process startup across OS and allocator boundaries."
"startup::cli::smith::help" = "Gap unknown: no appropriate matched model for process startup across OS and allocator boundaries."
"instructions::identity::proquint_encode::bits" = "Matched 16-byte input and 47-byte output traffic; reference repeats raw bytes, codec maps letters and allocates output; ratio alone cannot establish whether codec overhead is appropriate."
"instructions::identity::proquint_decode::text" = "Matched 47-byte input and 16-byte output traffic; reference includes XOR-reduction overhead, codec validates and maps letters; compare task costs, not distance from a literal optimum."