repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-ai/src/conformance.rs
Raw//! Provider-agnostic stream conformance kit (`SMH-SPEC-SPEC0001`,
//! Providers: conformance).
//!
//! Every vendor adapter is an incremental byte decoder over one output
//! vocabulary. This module defines that decoder contract and the fixture
//! assertions all adapters must satisfy, so a new adapter proves itself
//! against the same fragmented, malformed, incomplete, and non-stream error
//! fixtures instead of its own hand-picked tests.
use smith::error::Result;
use smith::stream::{StopReason, StreamEvent, Usage};
/// An incremental vendor stream decoder.
///
/// Bytes in, normalized events out. Feeding is chunk-agnostic: any split of
/// the byte stream must produce the same events. A terminal outcome is
/// emitted exactly once, either a [`StreamEvent::Stop`] or a fault returned
/// as `Err`; a second terminal outcome is a protocol violation.
pub trait StreamDecoder {
/// Feed transport bytes; returns events decoded so far.
///
/// # Errors
///
/// Returns a provider fault when the stream violates protocol rules;
/// decoding cannot continue afterwards.
fn push(&mut self, bytes: &[u8]) -> Result<Vec<StreamEvent>>;
/// Signal end of transport bytes.
///
/// # Errors
///
/// Returns [`smith::error::ProviderFault::Incomplete`] when the stream
/// ended without a terminal outcome, or a protocol fault for a malformed
/// terminal sequence.
fn finish(&mut self) -> Result<Vec<StreamEvent>>;
}
/// One expected step of a decoded stream, compared by shape, not by identity.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Expected {
/// An assistant text delta.
Text(String),
/// A thinking delta.
Thinking(String),
/// A complete tool use.
Tool {
/// Tool name.
name: String,
/// Complete parsed input.
input: serde_json::Value,
},
/// A terminal stop.
Stop {
/// Normalized stop reason.
reason: StopReason,
/// Usage accounting, if the stream carried it.
usage: Option<Usage>,
},
/// A terminal fault with this error code.
Fault(String),
}
/// A conformance fixture: byte chunks fed to a fresh decoder, and the event
/// shapes the normalized stream must produce.
pub struct Fixture {
/// Fixture name, used in failure messages.
pub name: &'static str,
/// Byte chunks in arrival order.
pub chunks: Vec<Vec<u8>>,
/// Expected normalized shapes in order.
pub expected: Vec<Expected>,
}
/// Run fixtures against a decoder factory; returns one report line per
/// mismatching fixture, empty when all conform.
///
/// Intended for adapter test modules: assert the returned slice is empty.
pub fn conformance_report(
make: &dyn Fn() -> Box<dyn StreamDecoder>,
fixtures: &[Fixture],
) -> Vec<String> {
let mut failures = Vec::new();
for fixture in fixtures {
let mut decoder = make();
let mut actual: Vec<Expected> = Vec::new();
match drive(&mut *decoder, &fixture.chunks, &mut actual) {
Ok(()) => {}
Err(err) => actual.push(Expected::Fault(err.code().to_string())),
}
if actual != fixture.expected {
failures.push(format!(
"fixture {}: expected {:?}, got {:?}",
fixture.name, fixture.expected, actual
));
}
}
failures
}
fn drive(
decoder: &mut dyn StreamDecoder,
chunks: &[Vec<u8>],
actual: &mut Vec<Expected>,
) -> Result<()> {
for chunk in chunks {
for event in decoder.push(chunk)? {
push_shape(actual, event)?;
}
}
for event in decoder.finish()? {
push_shape(actual, event)?;
}
Ok(())
}
fn push_shape(actual: &mut Vec<Expected>, event: StreamEvent) -> Result<()> {
let shape = match event {
StreamEvent::TextDelta { delta, .. } => Expected::Text(delta),
StreamEvent::ThinkingDelta { delta, .. } => Expected::Thinking(delta),
StreamEvent::ToolUse { name, input, .. } => Expected::Tool { name, input },
StreamEvent::Stop { reason, usage } => Expected::Stop { reason, usage },
// Decoders surface failures as Err, never as Error events.
StreamEvent::Error { .. } | StreamEvent::ToolResult { .. } => {
return Err(smith::error::SmithError::Provider {
fault: smith::error::ProviderFault::Protocol {
message: format!("decoders must not emit {event:?}"),
},
});
}
};
actual.push(shape);
Ok(())
}
/// Chunk-boundary fixtures for SSE decoders that frame events by blank
/// lines: the data line `event` preceded by more blank lines than it is
/// long, then ended by a carriage-return padded terminator longer than it,
/// each split across chunks. `tail` completes the stream; both fixtures
/// expect `expected`.
#[cfg(test)]
pub(crate) fn sse_boundary_fixtures(
event: &str,
tail: &[u8],
expected: &[Expected],
) -> Vec<Fixture> {
let data_line = format!("{event}\n");
vec![
Fixture {
name: "leading blank lines skipped before an event split at its terminator",
chunks: vec![
format!("{}{data_line}", "\n".repeat(data_line.len() + 1)).into_bytes(),
b"\n".to_vec(),
tail.to_vec(),
],
expected: expected.to_vec(),
},
Fixture {
name: "carriage-return padded terminator split mid-padding",
chunks: vec![
data_line.clone().into_bytes(),
"\r".repeat(data_line.len() + 1).into_bytes(),
format!("{}\n", "\r".repeat(data_line.len())).into_bytes(),
tail.to_vec(),
],
expected: expected.to_vec(),
},
]
}
/// The fault each vendor error maps to, carrying the vendor message `m`,
/// for the adapters' error-mapping tables.
#[cfg(test)]
pub(crate) mod faults {
use smith::error::ProviderFault;
const M: &str = "m";
pub const fn rate_limit() -> ProviderFault {
ProviderFault::RateLimit {
retry_after_ms: None,
}
}
pub fn transient() -> ProviderFault {
ProviderFault::Transient {
message: M.to_string(),
}
}
pub fn invalid() -> ProviderFault {
ProviderFault::Invalid {
field: None,
message: M.to_string(),
}
}
pub fn authentication() -> ProviderFault {
ProviderFault::Authentication {
message: M.to_string(),
}
}
pub fn overloaded() -> ProviderFault {
ProviderFault::Overloaded {
message: M.to_string(),
}
}
pub fn protocol() -> ProviderFault {
ProviderFault::Protocol {
message: M.to_string(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Decodes nothing and ends cleanly.
struct Silent;
impl StreamDecoder for Silent {
fn push(&mut self, _bytes: &[u8]) -> Result<Vec<StreamEvent>> {
Ok(Vec::new())
}
fn finish(&mut self) -> Result<Vec<StreamEvent>> {
Ok(Vec::new())
}
}
#[test]
fn report_names_each_mismatching_fixture_and_skips_conforming_ones() {
let fixtures = [
Fixture {
name: "conforms",
chunks: vec![b"x".to_vec()],
expected: vec![],
},
Fixture {
name: "wants text",
chunks: vec![b"x".to_vec()],
expected: vec![Expected::Text("x".to_string())],
},
];
assert_eq!(
conformance_report(&|| Box::new(Silent), &fixtures),
vec![r#"fixture wants text: expected [Text("x")], got []"#.to_string()]
);
}
}