repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-bench/benches/instructions.rs
Raw//! Instruction budgets per hot path (`SMH-SPEC-SPEC0001`, Performance and
//! portability): callgrind counts through gungraun, pinned in
//! `smith-bench/budgets.toml` and gated by `cargo x perf` within ±2 %.
//!
//! Sizes follow the spec targets: 1,000 entries for encoding and restore,
//! one provider stream per adapter, one RPC line. Every bench takes its
//! input from a setup and returns what it built, so neither construction
//! nor drop is counted. Process start-up lives in `startup.rs`.
#![expect(
missing_docs,
reason = "gungraun benchmark macros generate undocumented public modules and items"
)]
use gungraun::Callgrind;
use gungraun::prelude::*;
use smith_ai::{AnthropicDecoder, GeminiDecoder, OpenAiDecoder, StreamDecoder};
use smith_bench::{
anthropic_sse, frames_of, gemini_sse, linear_session, masked_agent, nested_json, openai_sse,
tool_call_session,
};
use smith_core::agent::Agent;
use smith_core::frame::{DEFAULT_MAX_FRAME_BYTES, Frame, Recovery, encode_entry, read_frames};
use smith_core::session::{EntryContent, EntryFrame, Session};
use smith_rpc::RpcState;
use std::hint::black_box;
fn linear(len: usize) -> Session {
linear_session(len).expect("linear session")
}
fn tool_calls(pairs: usize) -> Session {
tool_call_session(pairs).expect("tool-call session")
}
/// A session and its entries encoded back to back, as a store file holds
/// them.
fn encoded(session: Session) -> (Session, Vec<u8>) {
let bytes = encode_all(&session);
(session, bytes)
}
fn encode_all(session: &Session) -> Vec<u8> {
let mut out = Vec::new();
for entry in session.active_branch().entries() {
let frame = encode_entry(entry, session.bytes(entry), DEFAULT_MAX_FRAME_BYTES)
.expect("entry encodes");
out.extend_from_slice(&frame);
}
out
}
#[library_benchmark]
#[bench::linear_1000(linear(1000))]
#[bench::tool_calls_1000(tool_calls(500))]
fn encode(session: Session) -> (Session, Vec<u8>) {
let bytes = encode_all(black_box(&session));
(session, black_box(bytes))
}
#[library_benchmark]
#[bench::linear_1000(encoded(linear(1000)))]
#[bench::tool_calls_1000(encoded(tool_calls(500)))]
fn decode(input: (Session, Vec<u8>)) -> ((Session, Vec<u8>), Recovery) {
let recovery = read_frames(&mut black_box(input.1.as_slice()), DEFAULT_MAX_FRAME_BYTES);
(input, black_box(recovery))
}
#[library_benchmark]
#[bench::linear_1000(frames_of(&linear(1000)))]
fn from_frames(frames: Vec<Frame>) -> (Vec<Frame>, Session) {
let session = Session::from_frames(black_box(&frames)).expect("frames restore");
(frames, black_box(session))
}
#[library_benchmark]
#[bench::linear_1000(linear(1000))]
fn fork_at(session: Session) -> (Session, Session) {
let leaf = session.selected_id().expect("non-empty session");
let fork = session.fork_at(black_box(leaf)).expect("leaf is known");
(session, black_box(fork))
}
/// Tool-call content with a nested, non-canonical input: recording it
/// canonicalizes the input before encoding (`canonical_json` is private;
/// `EntryFrame::new` is its public seam).
fn tool_call_with(input: serde_json::Value) -> EntryContent {
EntryContent::ToolCall {
name: "write".to_string(),
input,
call_id: smith::ToolCallId::new(),
}
}
#[library_benchmark]
#[bench::nested(tool_call_with(nested_json(4)))]
fn canonical_json(content: EntryContent) -> (EntryContent, EntryFrame) {
let frame = EntryFrame::new(None, black_box(&content)).expect("content encodes");
(content, black_box(frame))
}
library_benchmark_group!(
name = session,
benchmarks = [encode, decode, from_frames, fork_at, canonical_json]
);
fn masked(history: usize) -> Agent {
masked_agent(history).expect("agent history")
}
// One turn over a long history: the request build decodes and masks
// every message since the compaction boundary (`build_request` is
// private; `run_turn` is its smallest public seam and adds recording the
// user message and the one-delta reply).
#[library_benchmark]
#[bench::history_1000_masked(masked(1000))]
fn run_turn(mut agent: Agent) -> Agent {
let outcome = agent.run_turn(black_box("next step, please"));
black_box(outcome.expect("turn settles"));
agent
}
library_benchmark_group!(name = agent, benchmarks = [run_turn]);
fn stream_of(decoder: Box<dyn StreamDecoder>, bytes: Vec<u8>) -> (Box<dyn StreamDecoder>, Vec<u8>) {
(decoder, bytes)
}
// One fixture stream through a fresh adapter, in one chunk, to its stop.
#[library_benchmark]
#[bench::openai(stream_of(Box::new(OpenAiDecoder::new()), openai_sse()))]
#[bench::anthropic(stream_of(Box::new(AnthropicDecoder::new()), anthropic_sse()))]
#[bench::gemini(stream_of(Box::new(GeminiDecoder::new()), gemini_sse()))]
fn sse(input: (Box<dyn StreamDecoder>, Vec<u8>)) -> (Vec<u8>, Vec<smith::StreamEvent>) {
let (mut decoder, bytes) = input;
let mut events = decoder.push(black_box(&bytes)).expect("stream decodes");
events.extend(decoder.finish().expect("stream stops"));
(bytes, black_box(events))
}
library_benchmark_group!(name = providers, benchmarks = [sse]);
fn rpc_state() -> RpcState {
RpcState::new(masked(0))
}
// One control line through the reader; the agent worker thread is idle
// and not counted.
#[library_benchmark]
#[bench::ping(rpc_state())]
fn handle_line(mut state: RpcState) -> (RpcState, Vec<String>) {
let lines = smith_rpc::handle_line(&mut state, black_box(r#"{"id":1,"method":"ping"}"#));
(state, black_box(lines))
}
library_benchmark_group!(name = rpc, benchmarks = [handle_line]);
main!(
config = LibraryBenchmarkConfig::default().tool(Callgrind::with_args(["--cache-sim=no"])),
library_benchmark_groups = [session, agent, providers, rpc]
);