Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-bench/benches/instructions.rs

Raw
//! Instruction budgets per hot path (`SMH-SPEC-SPEC0001`, Performance and
//! portability): callgrind counts through gungraun, pinned in
//! `smith-bench/budgets.toml` and gated by `cargo x perf` within ±2 %.
//!
//! Sizes follow the primitive targets: 1,000 entries for encoding and restore,
//! one provider stream per adapter, and 128 independent RPC control lines.
//! In-memory fixtures do not measure filesystem or paged I/O. Every bench takes its
//! input from setup and returns what it built, so neither construction nor drop
//! is counted. Process start-up lives in `startup.rs`.

#![expect(
    missing_docs,
    reason = "gungraun benchmark macros generate undocumented public modules and items"
)]

use gungraun::Callgrind;
use gungraun::prelude::*;
use smith_ai::{AnthropicDecoder, GeminiDecoder, OpenAiDecoder, StreamDecoder};
use smith_bench::{
    anthropic_sse, byte_transfer, byte_transfer_fixture, encode_preallocated as encode_fixture,
    frames_of, gemini_sse, linear_session, masked_agent, nested_json, openai_sse,
    preallocated_encode_fixture, proquint_decode_floor as decode_model,
    proquint_encode_floor as encode_model, tool_call_session, value_payload_fixture,
    visit_value_payload,
};
use smith_core::agent::Agent;
use smith_core::frame::{DEFAULT_MAX_FRAME_BYTES, Frame, Recovery, read_frames};
use smith_core::session::{EntryContent, EntryFrame, Session};
use smith_rpc::RpcState;
use std::hint::black_box;

fn linear(len: usize) -> Session {
    linear_session(len).expect("linear session")
}

fn tool_calls(pairs: usize) -> Session {
    tool_call_session(pairs).expect("tool-call session")
}

/// An in-memory session and active-branch frames encoded back to back.
fn encoded(session: Session) -> (Session, Vec<u8>) {
    encode_fixture(preallocated_encode_fixture(session))
}

#[library_benchmark]
#[bench::linear_1000(preallocated_encode_fixture(linear(1000)))]
#[bench::tool_calls_1000(preallocated_encode_fixture(tool_calls(500)))]
fn encode_preallocated(session: smith_bench::PreallocatedEncodeFixture) -> (Session, Vec<u8>) {
    let (session, bytes) = encode_fixture(black_box(session));
    (session, black_box(bytes))
}

#[library_benchmark]
#[bench::linear_1000(encoded(linear(1000)))]
#[bench::tool_calls_1000(encoded(tool_calls(500)))]
fn decode(input: (Session, Vec<u8>)) -> ((Session, Vec<u8>), Recovery) {
    // In-memory frame decoding is a codec primitive, not a paged-I/O measure.
    let recovery = read_frames(&mut black_box(input.1.as_slice()), DEFAULT_MAX_FRAME_BYTES)
        .expect("in-memory frames read");
    (input, black_box(recovery))
}

#[library_benchmark]
#[bench::linear_1000(frames_of(&linear(1000)))]
fn from_frames(frames: Vec<Frame>) -> (Vec<Frame>, Session) {
    let session = Session::from_frames(black_box(&frames)).expect("frames restore");
    (frames, black_box(session))
}

#[library_benchmark]
#[bench::linear_1000(linear(1000))]
fn fork_at(session: Session) -> (Session, Session) {
    let leaf = session.selected_id().expect("non-empty session");
    let fork = session.fork_at(black_box(leaf)).expect("leaf is known");
    (session, black_box(fork))
}

/// Tool-call content with a nested, non-canonical input: recording it
/// canonicalizes the input before encoding (`canonical_json` is private;
/// `EntryFrame::new` is its public seam).
fn tool_call_with(input: serde_json::Value) -> EntryContent {
    EntryContent::ToolCall {
        name: "write".to_string(),
        input: smith::ToolArguments::from_serializable(&input).expect("benchmark fixture encodes"),
        call_id: smith::ToolCallId::new(),
    }
}

#[library_benchmark]
#[bench::nested(tool_call_with(nested_json(4)))]
fn canonical_json(content: EntryContent) -> (EntryContent, EntryFrame) {
    let frame = EntryFrame::new(None, black_box(&content)).expect("content encodes");
    (content, black_box(frame))
}

// Frame decode model: every frame byte moves from resident input into a
// setup-allocated output. It intentionally excludes framing, checksums,
// parsing, validation, and session construction.
#[library_benchmark]
#[bench::linear_1000(byte_transfer_fixture(encoded(linear(1000)).1))]
#[bench::tool_calls_1000(byte_transfer_fixture(encoded(tool_calls(500)).1))]
fn decode_floor(input: (Vec<u8>, Vec<u8>)) -> (Vec<u8>, Vec<u8>) {
    byte_transfer(input)
}

// Canonical JSON model: the actual `Value` tree is visited and its payload
// written to a setup-allocated buffer. This models tree residency and output
// writes, not canonical JSON's sorting, number formatting, or serialization.
#[library_benchmark]
#[bench::nested(value_payload_fixture(nested_json(4)))]
fn canonical_json_floor(input: (serde_json::Value, Vec<u8>)) -> (serde_json::Value, Vec<u8>) {
    let (value, mut output) = input;
    visit_value_payload(black_box(&value), &mut output);
    (value, black_box(output))
}

library_benchmark_group!(
    name = session,
    benchmarks = [
        encode_preallocated,
        decode,
        decode_floor,
        from_frames,
        fork_at,
        canonical_json,
        canonical_json_floor
    ]
);

fn masked(history: usize) -> Agent {
    masked_agent(history).expect("agent history")
}

// One turn over a long history: the request build decodes and masks
// every message since the compaction boundary (`build_request` is
// private; `run_turn` is its smallest public seam and adds recording the
// user message and the one-delta reply).
#[library_benchmark]
#[bench::history_1000_masked(masked(1000))]
fn run_turn(mut agent: Agent) -> Agent {
    let outcome = agent.run_turn(black_box("next step, please"));
    black_box(outcome.expect("turn settles"));
    agent
}

library_benchmark_group!(name = agent, benchmarks = [run_turn]);

fn stream_of(decoder: Box<dyn StreamDecoder>, bytes: Vec<u8>) -> (Box<dyn StreamDecoder>, Vec<u8>) {
    (decoder, bytes)
}

// One fixture stream through a fresh adapter, in one chunk, to its stop.
#[library_benchmark]
#[bench::openai(stream_of(Box::new(OpenAiDecoder::new()), openai_sse()))]
#[bench::anthropic(stream_of(Box::new(AnthropicDecoder::new()), anthropic_sse()))]
#[bench::gemini(stream_of(Box::new(GeminiDecoder::new()), gemini_sse()))]
fn sse(input: (Box<dyn StreamDecoder>, Vec<u8>)) -> (Vec<u8>, Vec<smith::StreamEvent>) {
    let (mut decoder, bytes) = input;
    let mut events = decoder.push(black_box(&bytes)).expect("stream decodes");
    events.extend(decoder.finish().expect("stream stops"));
    (bytes, black_box(events))
}

// SSE model: vendor fixture bytes are bulk-copied into setup-allocated
// output. It excludes line scanning, JSON parsing, validation, and event
// construction, which are task costs of the corresponding decoder.
#[library_benchmark]
#[bench::openai(byte_transfer_fixture(openai_sse()))]
#[bench::anthropic(byte_transfer_fixture(anthropic_sse()))]
#[bench::gemini(byte_transfer_fixture(gemini_sse()))]
fn sse_floor(input: (Vec<u8>, Vec<u8>)) -> (Vec<u8>, Vec<u8>) {
    byte_transfer(input)
}

library_benchmark_group!(name = providers, benchmarks = [sse, sse_floor]);

const PROQUINT_BITS: u128 = 0x0123_4567_89ab_cdef_0123_4567_89ab_cdef;

// Proquint encoding of one 128-bit identity into its fixed 47-byte text form.
#[library_benchmark]
#[bench::bits(PROQUINT_BITS)]
fn proquint_encode(bits: u128) -> String {
    black_box(smith::id::proquint::encode(black_box(bits)))
}

// Proquint decoding of one canonical 47-byte identity string into 128 bits.
#[library_benchmark]
#[bench::text(smith::id::proquint::encode(PROQUINT_BITS))]
fn proquint_decode(text: String) -> (String, u128) {
    let bits = smith::id::proquint::decode(black_box(&text)).expect("canonical proquint");
    (text, black_box(bits))
}

// Proquint models match codec shapes: encode reads a 16-byte bit value and
// writes 47 text bytes; decode reads 47 text bytes and writes 16 bit-value bytes.
#[library_benchmark]
#[bench::encode((PROQUINT_BITS, vec![0; smith::id::proquint::encode(PROQUINT_BITS).len()]))]
fn proquint_encode_floor(input: (u128, Vec<u8>)) -> (u128, Vec<u8>) {
    encode_model(input)
}

#[library_benchmark]
#[bench::decode((smith::id::proquint::encode(PROQUINT_BITS), vec![0; std::mem::size_of::<u128>()]))]
fn proquint_decode_floor(input: (String, Vec<u8>)) -> (String, Vec<u8>) {
    decode_model(input)
}

library_benchmark_group!(
    name = identity,
    benchmarks = [
        proquint_encode,
        proquint_decode,
        proquint_encode_floor,
        proquint_decode_floor
    ]
);

const PING_BATCH: usize = 128;
const PING: &str = r#"{"id":1,"method":"ping"}"#;

fn rpc_state() -> RpcState {
    RpcState::new(masked(0))
}

// Independent control lines through one reader. The agent worker is idle and
// not counted. A batch amortizes randomized serde_json map probe paths while
// retaining parsing, dispatch, response creation, and serialization per line.
#[library_benchmark]
#[bench::ping_batch_128(rpc_state())]
fn handle_line(mut state: RpcState) -> (RpcState, Vec<Vec<String>>) {
    let mut responses = Vec::with_capacity(PING_BATCH);
    for _ in 0..PING_BATCH {
        responses.push(smith_rpc::handle_line(&mut state, black_box(PING)));
    }
    (state, black_box(responses))
}

library_benchmark_group!(name = rpc, benchmarks = [handle_line]);

main!(
    config = LibraryBenchmarkConfig::default().tool(Callgrind::with_args(["--cache-sim=no"])),
    library_benchmark_groups = [session, agent, providers, identity, rpc]
);