Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-ai/src/conformance.rs

Raw
//! Provider-agnostic stream conformance kit (`SMH-SPEC-SPEC0001`,
//! Providers: conformance).
//!
//! Every vendor adapter is an incremental byte decoder over one output
//! vocabulary. This module defines that decoder contract and the fixture
//! assertions all adapters must satisfy, so a new adapter proves itself
//! against the same fragmented, malformed, incomplete, and non-stream error
//! fixtures instead of its own hand-picked tests.

use smith::error::Result;
use smith::stream::{StopReason, StreamEvent, Usage};

/// An incremental vendor stream decoder.
///
/// Bytes in, normalized events out. Feeding is chunk-agnostic: any split of
/// the byte stream must produce the same events. A terminal outcome is
/// emitted exactly once, either a [`StreamEvent::Stop`] or a fault returned
/// as `Err`; a second terminal outcome is a protocol violation.
pub trait StreamDecoder {
    /// Feed transport bytes; returns events decoded so far.
    ///
    /// # Errors
    ///
    /// Returns a provider fault when the stream violates protocol rules;
    /// decoding cannot continue afterwards.
    fn push(&mut self, bytes: &[u8]) -> Result<Vec<StreamEvent>>;

    /// Signal end of transport bytes.
    ///
    /// # Errors
    ///
    /// Returns [`smith::error::ProviderFault::Incomplete`] when the stream
    /// ended without a terminal outcome, or a protocol fault for a malformed
    /// terminal sequence.
    fn finish(&mut self) -> Result<Vec<StreamEvent>>;
}

/// One expected step of a decoded stream, compared by shape, not by identity.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Expected {
    /// An assistant text delta.
    Text(String),
    /// A thinking delta.
    Thinking(String),
    /// A complete tool use.
    Tool {
        /// Tool name.
        name: String,
        /// Complete parsed input.
        input: serde_json::Value,
    },
    /// A terminal stop.
    Stop {
        /// Normalized stop reason.
        reason: StopReason,
        /// Usage accounting, if the stream carried it.
        usage: Option<Usage>,
    },
    /// A terminal fault with this error code.
    Fault(String),
}

/// A conformance fixture: byte chunks fed to a fresh decoder, and the event
/// shapes the normalized stream must produce.
pub struct Fixture {
    /// Fixture name, used in failure messages.
    pub name: &'static str,
    /// Byte chunks in arrival order.
    pub chunks: Vec<Vec<u8>>,
    /// Expected normalized shapes in order.
    pub expected: Vec<Expected>,
}

/// Run fixtures against a decoder factory; returns one report line per
/// mismatching fixture, empty when all conform.
///
/// Intended for adapter test modules: assert the returned slice is empty.
pub fn conformance_report(
    make: &dyn Fn() -> Box<dyn StreamDecoder>,
    fixtures: &[Fixture],
) -> Vec<String> {
    let mut failures = Vec::new();
    for fixture in fixtures {
        let mut decoder = make();
        let mut actual: Vec<Expected> = Vec::new();
        match drive(&mut *decoder, &fixture.chunks, &mut actual) {
            Ok(()) => {}
            Err(err) => actual.push(Expected::Fault(err.code().to_string())),
        }
        if actual != fixture.expected {
            failures.push(format!(
                "fixture {}: expected {:?}, got {:?}",
                fixture.name, fixture.expected, actual
            ));
        }
    }
    failures
}

fn drive(
    decoder: &mut dyn StreamDecoder,
    chunks: &[Vec<u8>],
    actual: &mut Vec<Expected>,
) -> Result<()> {
    for chunk in chunks {
        for event in decoder.push(chunk)? {
            push_shape(actual, event)?;
        }
    }
    for event in decoder.finish()? {
        push_shape(actual, event)?;
    }
    Ok(())
}

fn push_shape(actual: &mut Vec<Expected>, event: StreamEvent) -> Result<()> {
    let shape = match event {
        StreamEvent::TextDelta { delta, .. } => Expected::Text(delta),
        StreamEvent::ThinkingDelta { delta, .. } => Expected::Thinking(delta),
        StreamEvent::ToolUse { name, input, .. } => Expected::Tool { name, input },
        StreamEvent::Stop { reason, usage } => Expected::Stop { reason, usage },
        // Decoders surface failures as Err, never as Error events.
        StreamEvent::Error { .. } | StreamEvent::ToolResult { .. } => {
            return Err(smith::error::SmithError::Provider {
                fault: smith::error::ProviderFault::Protocol {
                    message: format!("decoders must not emit {event:?}"),
                },
            });
        }
    };
    actual.push(shape);
    Ok(())
}

/// Chunk-boundary fixtures for SSE decoders that frame events by blank
/// lines: the data line `event` preceded by more blank lines than it is
/// long, then ended by a carriage-return padded terminator longer than it,
/// each split across chunks. `tail` completes the stream; both fixtures
/// expect `expected`.
#[cfg(test)]
pub(crate) fn sse_boundary_fixtures(
    event: &str,
    tail: &[u8],
    expected: &[Expected],
) -> Vec<Fixture> {
    let data_line = format!("{event}\n");
    vec![
        Fixture {
            name: "leading blank lines skipped before an event split at its terminator",
            chunks: vec![
                format!("{}{data_line}", "\n".repeat(data_line.len() + 1)).into_bytes(),
                b"\n".to_vec(),
                tail.to_vec(),
            ],
            expected: expected.to_vec(),
        },
        Fixture {
            name: "carriage-return padded terminator split mid-padding",
            chunks: vec![
                data_line.clone().into_bytes(),
                "\r".repeat(data_line.len() + 1).into_bytes(),
                format!("{}\n", "\r".repeat(data_line.len())).into_bytes(),
                tail.to_vec(),
            ],
            expected: expected.to_vec(),
        },
    ]
}

/// The fault each vendor error maps to, carrying the vendor message `m`,
/// for the adapters' error-mapping tables.
#[cfg(test)]
pub(crate) mod faults {
    use smith::error::ProviderFault;

    const M: &str = "m";

    pub const fn rate_limit() -> ProviderFault {
        ProviderFault::RateLimit {
            retry_after_ms: None,
        }
    }

    pub fn transient() -> ProviderFault {
        ProviderFault::Transient {
            message: M.to_string(),
        }
    }

    pub fn invalid() -> ProviderFault {
        ProviderFault::Invalid {
            field: None,
            message: M.to_string(),
        }
    }

    pub fn authentication() -> ProviderFault {
        ProviderFault::Authentication {
            message: M.to_string(),
        }
    }

    pub fn overloaded() -> ProviderFault {
        ProviderFault::Overloaded {
            message: M.to_string(),
        }
    }

    pub fn protocol() -> ProviderFault {
        ProviderFault::Protocol {
            message: M.to_string(),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Decodes nothing and ends cleanly.
    struct Silent;

    impl StreamDecoder for Silent {
        fn push(&mut self, _bytes: &[u8]) -> Result<Vec<StreamEvent>> {
            Ok(Vec::new())
        }

        fn finish(&mut self) -> Result<Vec<StreamEvent>> {
            Ok(Vec::new())
        }
    }

    #[test]
    fn report_names_each_mismatching_fixture_and_skips_conforming_ones() {
        let fixtures = [
            Fixture {
                name: "conforms",
                chunks: vec![b"x".to_vec()],
                expected: vec![],
            },
            Fixture {
                name: "wants text",
                chunks: vec![b"x".to_vec()],
                expected: vec![Expected::Text("x".to_string())],
            },
        ];
        assert_eq!(
            conformance_report(&|| Box::new(Silent), &fixtures),
            vec![r#"fixture wants text: expected [Text("x")], got []"#.to_string()]
        );
    }
}