repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-bench/benches/instructions.rs
Raw//! Instruction budgets per hot path (`SMH-SPEC-SPEC0001`, Performance and
//! portability): callgrind counts through gungraun, pinned in
//! `smith-bench/budgets.toml` and gated by `cargo x perf` within ±2 %.
//!
//! Sizes follow the primitive targets: 1,000 entries for encoding and restore,
//! one provider stream per adapter, and 128 independent RPC control lines.
//! In-memory fixtures do not measure filesystem or paged I/O. Every bench takes its
//! input from setup and returns what it built, so neither construction nor drop
//! is counted. Process start-up lives in `startup.rs`.
#![expect(
missing_docs,
reason = "gungraun benchmark macros generate undocumented public modules and items"
)]
use gungraun::Callgrind;
use gungraun::prelude::*;
use smith_ai::{AnthropicDecoder, GeminiDecoder, OpenAiDecoder, StreamDecoder};
use smith_bench::{
anthropic_sse, byte_transfer, byte_transfer_fixture, encode_preallocated as encode_fixture,
frames_of, gemini_sse, linear_session, masked_agent, nested_json, openai_sse,
preallocated_encode_fixture, proquint_decode_floor as decode_model,
proquint_encode_floor as encode_model, tool_call_session, value_payload_fixture,
visit_value_payload,
};
use smith_core::agent::Agent;
use smith_core::frame::{DEFAULT_MAX_FRAME_BYTES, Frame, Recovery, read_frames};
use smith_core::session::{EntryContent, EntryFrame, Session};
use smith_rpc::RpcState;
use std::hint::black_box;
fn linear(len: usize) -> Session {
linear_session(len).expect("linear session")
}
fn tool_calls(pairs: usize) -> Session {
tool_call_session(pairs).expect("tool-call session")
}
/// An in-memory session and active-branch frames encoded back to back.
fn encoded(session: Session) -> (Session, Vec<u8>) {
encode_fixture(preallocated_encode_fixture(session))
}
#[library_benchmark]
#[bench::linear_1000(preallocated_encode_fixture(linear(1000)))]
#[bench::tool_calls_1000(preallocated_encode_fixture(tool_calls(500)))]
fn encode_preallocated(session: smith_bench::PreallocatedEncodeFixture) -> (Session, Vec<u8>) {
let (session, bytes) = encode_fixture(black_box(session));
(session, black_box(bytes))
}
#[library_benchmark]
#[bench::linear_1000(encoded(linear(1000)))]
#[bench::tool_calls_1000(encoded(tool_calls(500)))]
fn decode(input: (Session, Vec<u8>)) -> ((Session, Vec<u8>), Recovery) {
// In-memory frame decoding is a codec primitive, not a paged-I/O measure.
let recovery = read_frames(&mut black_box(input.1.as_slice()), DEFAULT_MAX_FRAME_BYTES)
.expect("in-memory frames read");
(input, black_box(recovery))
}
#[library_benchmark]
#[bench::linear_1000(frames_of(&linear(1000)))]
fn from_frames(frames: Vec<Frame>) -> (Vec<Frame>, Session) {
let session = Session::from_frames(black_box(&frames)).expect("frames restore");
(frames, black_box(session))
}
#[library_benchmark]
#[bench::linear_1000(linear(1000))]
fn fork_at(session: Session) -> (Session, Session) {
let leaf = session.selected_id().expect("non-empty session");
let fork = session.fork_at(black_box(leaf)).expect("leaf is known");
(session, black_box(fork))
}
/// Tool-call content with a nested, non-canonical input: recording it
/// canonicalizes the input before encoding (`canonical_json` is private;
/// `EntryFrame::new` is its public seam).
fn tool_call_with(input: serde_json::Value) -> EntryContent {
EntryContent::ToolCall {
name: "write".to_string(),
input: smith::ToolArguments::from_serializable(&input).expect("benchmark fixture encodes"),
call_id: smith::ToolCallId::new(),
}
}
#[library_benchmark]
#[bench::nested(tool_call_with(nested_json(4)))]
fn canonical_json(content: EntryContent) -> (EntryContent, EntryFrame) {
let frame = EntryFrame::new(None, black_box(&content)).expect("content encodes");
(content, black_box(frame))
}
// Frame decode model: every frame byte moves from resident input into a
// setup-allocated output. It intentionally excludes framing, checksums,
// parsing, validation, and session construction.
#[library_benchmark]
#[bench::linear_1000(byte_transfer_fixture(encoded(linear(1000)).1))]
#[bench::tool_calls_1000(byte_transfer_fixture(encoded(tool_calls(500)).1))]
fn decode_floor(input: (Vec<u8>, Vec<u8>)) -> (Vec<u8>, Vec<u8>) {
byte_transfer(input)
}
// Canonical JSON model: the actual `Value` tree is visited and its payload
// written to a setup-allocated buffer. This models tree residency and output
// writes, not canonical JSON's sorting, number formatting, or serialization.
#[library_benchmark]
#[bench::nested(value_payload_fixture(nested_json(4)))]
fn canonical_json_floor(input: (serde_json::Value, Vec<u8>)) -> (serde_json::Value, Vec<u8>) {
let (value, mut output) = input;
visit_value_payload(black_box(&value), &mut output);
(value, black_box(output))
}
library_benchmark_group!(
name = session,
benchmarks = [
encode_preallocated,
decode,
decode_floor,
from_frames,
fork_at,
canonical_json,
canonical_json_floor
]
);
fn masked(history: usize) -> Agent {
masked_agent(history).expect("agent history")
}
// One turn over a long history: the request build decodes and masks
// every message since the compaction boundary (`build_request` is
// private; `run_turn` is its smallest public seam and adds recording the
// user message and the one-delta reply).
#[library_benchmark]
#[bench::history_1000_masked(masked(1000))]
fn run_turn(mut agent: Agent) -> Agent {
let outcome = agent.run_turn(black_box("next step, please"));
black_box(outcome.expect("turn settles"));
agent
}
library_benchmark_group!(name = agent, benchmarks = [run_turn]);
fn stream_of(decoder: Box<dyn StreamDecoder>, bytes: Vec<u8>) -> (Box<dyn StreamDecoder>, Vec<u8>) {
(decoder, bytes)
}
// One fixture stream through a fresh adapter, in one chunk, to its stop.
#[library_benchmark]
#[bench::openai(stream_of(Box::new(OpenAiDecoder::new()), openai_sse()))]
#[bench::anthropic(stream_of(Box::new(AnthropicDecoder::new()), anthropic_sse()))]
#[bench::gemini(stream_of(Box::new(GeminiDecoder::new()), gemini_sse()))]
fn sse(input: (Box<dyn StreamDecoder>, Vec<u8>)) -> (Vec<u8>, Vec<smith::StreamEvent>) {
let (mut decoder, bytes) = input;
let mut events = decoder.push(black_box(&bytes)).expect("stream decodes");
events.extend(decoder.finish().expect("stream stops"));
(bytes, black_box(events))
}
// SSE model: vendor fixture bytes are bulk-copied into setup-allocated
// output. It excludes line scanning, JSON parsing, validation, and event
// construction, which are task costs of the corresponding decoder.
#[library_benchmark]
#[bench::openai(byte_transfer_fixture(openai_sse()))]
#[bench::anthropic(byte_transfer_fixture(anthropic_sse()))]
#[bench::gemini(byte_transfer_fixture(gemini_sse()))]
fn sse_floor(input: (Vec<u8>, Vec<u8>)) -> (Vec<u8>, Vec<u8>) {
byte_transfer(input)
}
library_benchmark_group!(name = providers, benchmarks = [sse, sse_floor]);
const PROQUINT_BITS: u128 = 0x0123_4567_89ab_cdef_0123_4567_89ab_cdef;
// Proquint encoding of one 128-bit identity into its fixed 47-byte text form.
#[library_benchmark]
#[bench::bits(PROQUINT_BITS)]
fn proquint_encode(bits: u128) -> String {
black_box(smith::id::proquint::encode(black_box(bits)))
}
// Proquint decoding of one canonical 47-byte identity string into 128 bits.
#[library_benchmark]
#[bench::text(smith::id::proquint::encode(PROQUINT_BITS))]
fn proquint_decode(text: String) -> (String, u128) {
let bits = smith::id::proquint::decode(black_box(&text)).expect("canonical proquint");
(text, black_box(bits))
}
// Proquint models match codec shapes: encode reads a 16-byte bit value and
// writes 47 text bytes; decode reads 47 text bytes and writes 16 bit-value bytes.
#[library_benchmark]
#[bench::encode((PROQUINT_BITS, vec![0; smith::id::proquint::encode(PROQUINT_BITS).len()]))]
fn proquint_encode_floor(input: (u128, Vec<u8>)) -> (u128, Vec<u8>) {
encode_model(input)
}
#[library_benchmark]
#[bench::decode((smith::id::proquint::encode(PROQUINT_BITS), vec![0; std::mem::size_of::<u128>()]))]
fn proquint_decode_floor(input: (String, Vec<u8>)) -> (String, Vec<u8>) {
decode_model(input)
}
library_benchmark_group!(
name = identity,
benchmarks = [
proquint_encode,
proquint_decode,
proquint_encode_floor,
proquint_decode_floor
]
);
const PING_BATCH: usize = 128;
const PING: &str = r#"{"id":1,"method":"ping"}"#;
fn rpc_state() -> RpcState {
RpcState::new(masked(0))
}
// Independent control lines through one reader. The agent worker is idle and
// not counted. A batch amortizes randomized serde_json map probe paths while
// retaining parsing, dispatch, response creation, and serialization per line.
#[library_benchmark]
#[bench::ping_batch_128(rpc_state())]
fn handle_line(mut state: RpcState) -> (RpcState, Vec<Vec<String>>) {
let mut responses = Vec::with_capacity(PING_BATCH);
for _ in 0..PING_BATCH {
responses.push(smith_rpc::handle_line(&mut state, black_box(PING)));
}
(state, black_box(responses))
}
library_benchmark_group!(name = rpc, benchmarks = [handle_line]);
main!(
config = LibraryBenchmarkConfig::default().tool(Callgrind::with_args(["--cache-sim=no"])),
library_benchmark_groups = [session, agent, providers, identity, rpc]
);