//! Provider-agnostic stream conformance kit (`SMH-SPEC-SPEC0001`, //! Providers: conformance). //! //! Every vendor adapter is an incremental byte decoder over one output //! vocabulary. This module defines that decoder contract and the fixture //! assertions all adapters must satisfy, so a new adapter proves itself //! against the same fragmented, malformed, incomplete, and non-stream error //! fixtures instead of its own hand-picked tests. use smith::error::Result; use smith::stream::{StopReason, StreamEvent, Usage}; /// An incremental vendor stream decoder. /// /// Bytes in, normalized events out. Feeding is chunk-agnostic: any split of /// the byte stream must produce the same events. A terminal outcome is /// emitted exactly once, either a [`StreamEvent::Stop`] or a fault returned /// as `Err`; a second terminal outcome is a protocol violation. pub trait StreamDecoder { /// Feed transport bytes; returns events decoded so far. /// /// # Errors /// /// Returns a provider fault when the stream violates protocol rules; /// decoding cannot continue afterwards. fn push(&mut self, bytes: &[u8]) -> Result>; /// Signal end of transport bytes. /// /// # Errors /// /// Returns [`smith::error::ProviderFault::Incomplete`] when the stream /// ended without a terminal outcome, or a protocol fault for a malformed /// terminal sequence. fn finish(&mut self) -> Result>; } /// One expected step of a decoded stream, compared by shape, not by identity. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Expected { /// An assistant text delta. Text(String), /// A thinking delta. Thinking(String), /// A complete tool use. Tool { /// Tool name. name: String, /// Complete parsed input. input: serde_json::Value, }, /// A terminal stop. Stop { /// Normalized stop reason. reason: StopReason, /// Usage accounting, if the stream carried it. usage: Option, }, /// A terminal fault with this error code. Fault(String), } /// A conformance fixture: byte chunks fed to a fresh decoder, and the event /// shapes the normalized stream must produce. pub struct Fixture { /// Fixture name, used in failure messages. pub name: &'static str, /// Byte chunks in arrival order. pub chunks: Vec>, /// Expected normalized shapes in order. pub expected: Vec, } /// Run fixtures against a decoder factory; returns one report line per /// mismatching fixture, empty when all conform. /// /// Intended for adapter test modules: assert the returned slice is empty. pub fn conformance_report( make: &dyn Fn() -> Box, fixtures: &[Fixture], ) -> Vec { let mut failures = Vec::new(); for fixture in fixtures { let mut decoder = make(); let mut actual: Vec = Vec::new(); match drive(&mut *decoder, &fixture.chunks, &mut actual) { Ok(()) => {} Err(err) => actual.push(Expected::Fault(err.code().to_string())), } if actual != fixture.expected { failures.push(format!( "fixture {}: expected {:?}, got {:?}", fixture.name, fixture.expected, actual )); } } failures } fn drive( decoder: &mut dyn StreamDecoder, chunks: &[Vec], actual: &mut Vec, ) -> Result<()> { for chunk in chunks { for event in decoder.push(chunk)? { push_shape(actual, event)?; } } for event in decoder.finish()? { push_shape(actual, event)?; } Ok(()) } fn push_shape(actual: &mut Vec, event: StreamEvent) -> Result<()> { let shape = match event { StreamEvent::TextDelta { delta, .. } => Expected::Text(delta), StreamEvent::ThinkingDelta { delta, .. } => Expected::Thinking(delta), StreamEvent::ToolUse { name, input, .. } => Expected::Tool { name, input }, StreamEvent::Stop { reason, usage } => Expected::Stop { reason, usage }, // Decoders surface failures as Err, never as Error events. StreamEvent::Error { .. } | StreamEvent::ToolResult { .. } => { return Err(smith::error::SmithError::Provider { fault: smith::error::ProviderFault::Protocol { message: format!("decoders must not emit {event:?}"), }, }); } }; actual.push(shape); Ok(()) } /// Chunk-boundary fixtures for SSE decoders that frame events by blank /// lines: the data line `event` preceded by more blank lines than it is /// long, then ended by a carriage-return padded terminator longer than it, /// each split across chunks. `tail` completes the stream; both fixtures /// expect `expected`. #[cfg(test)] pub(crate) fn sse_boundary_fixtures( event: &str, tail: &[u8], expected: &[Expected], ) -> Vec { let data_line = format!("{event}\n"); vec![ Fixture { name: "leading blank lines skipped before an event split at its terminator", chunks: vec![ format!("{}{data_line}", "\n".repeat(data_line.len() + 1)).into_bytes(), b"\n".to_vec(), tail.to_vec(), ], expected: expected.to_vec(), }, Fixture { name: "carriage-return padded terminator split mid-padding", chunks: vec![ data_line.clone().into_bytes(), "\r".repeat(data_line.len() + 1).into_bytes(), format!("{}\n", "\r".repeat(data_line.len())).into_bytes(), tail.to_vec(), ], expected: expected.to_vec(), }, ] } /// The fault each vendor error maps to, carrying the vendor message `m`, /// for the adapters' error-mapping tables. #[cfg(test)] pub(crate) mod faults { use smith::error::ProviderFault; const M: &str = "m"; pub const fn rate_limit() -> ProviderFault { ProviderFault::RateLimit { retry_after_ms: None, } } pub fn transient() -> ProviderFault { ProviderFault::Transient { message: M.to_string(), } } pub fn invalid() -> ProviderFault { ProviderFault::Invalid { field: None, message: M.to_string(), } } pub fn authentication() -> ProviderFault { ProviderFault::Authentication { message: M.to_string(), } } pub fn overloaded() -> ProviderFault { ProviderFault::Overloaded { message: M.to_string(), } } pub fn protocol() -> ProviderFault { ProviderFault::Protocol { message: M.to_string(), } } } #[cfg(test)] mod tests { use super::*; /// Decodes nothing and ends cleanly. struct Silent; impl StreamDecoder for Silent { fn push(&mut self, _bytes: &[u8]) -> Result> { Ok(Vec::new()) } fn finish(&mut self) -> Result> { Ok(Vec::new()) } } #[test] fn report_names_each_mismatching_fixture_and_skips_conforming_ones() { let fixtures = [ Fixture { name: "conforms", chunks: vec![b"x".to_vec()], expected: vec![], }, Fixture { name: "wants text", chunks: vec![b"x".to_vec()], expected: vec![Expected::Text("x".to_string())], }, ]; assert_eq!( conformance_report(&|| Box::new(Silent), &fixtures), vec![r#"fixture wants text: expected [Text("x")], got []"#.to_string()] ); } }