Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-harness/src/plugins/test.rs

Raw
//! Deterministic plugin fixtures through the sealed host (`SMH-SPEC-WASM0001`).

use super::{Capability, PluginPackage};
use crate::wasm::{MockEffectPlan, MockInvocation, PluginEngine, PluginGeneration};
use serde::Deserialize;
use smith::error::{ProviderFault, Result, SmithError};
use std::io::Read;
use std::path::Path;

const FIXTURE_FILE: &str = "plugin-test.json";
const FIXTURE_BYTES_MAX: usize = 64 * 1024;
const INVOCATIONS_MAX: usize = 128;
const EFFECTS_MAX: usize = 128;

/// Execute every declared fixture twice through fresh sealed-host generations.
///
/// # Errors
///
/// Returns a stage, cause, and user fix when fixture parsing, expected output,
/// effect ordering, grants, or repeated-call determinism fails.
pub fn run(package_root: &Path, grants: &[Capability]) -> Result<()> {
    let fixture = Fixture::read(package_root)?;
    let first = execute(package_root, grants, &fixture)?;
    let second = execute(package_root, grants, &fixture)?;
    if first != second {
        return Err(fixture_fault(
            "determinism",
            "same artifact and fixed host inputs produced different traces",
            "remove ambient state or make the guest output and effects deterministic",
        ));
    }
    Ok(())
}

#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Fixture {
    invocations: Vec<Invocation>,
}

#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Invocation {
    name: String,
    input_json: String,
    expected_output: String,
    effects: Vec<Effect>,
}

#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Effect {
    operation: String,
    request_json: String,
    response: String,
}

impl Fixture {
    fn read(package_root: &Path) -> Result<Self> {
        let path = package_root.join(FIXTURE_FILE);
        let metadata = std::fs::metadata(&path).map_err(|error| {
            fixture_fault(
                "fixture",
                &format!("missing {FIXTURE_FILE}: {error}"),
                &format!(
                    "add {FIXTURE_FILE} with invocation input, expected output, and effect plans"
                ),
            )
        })?;
        if metadata.len() > FIXTURE_BYTES_MAX as u64 {
            return Err(fixture_fault(
                "fixture",
                &format!("{FIXTURE_FILE} exceeds {FIXTURE_BYTES_MAX} bytes"),
                "reduce fixture data below the fixture byte limit",
            ));
        }
        let mut bytes = Vec::new();
        let file = std::fs::File::open(&path).map_err(|error| {
            fixture_fault("fixture", &error.to_string(), "make fixture readable")
        })?;
        #[expect(
            clippy::disallowed_methods,
            reason = "Take caps this fixture read at FIXTURE_BYTES_MAX + 1"
        )]
        file.take((FIXTURE_BYTES_MAX + 1) as u64)
            .read_to_end(&mut bytes)
            .map_err(|error| {
                fixture_fault("fixture", &error.to_string(), "make fixture readable")
            })?;
        let fixture: Self = serde_json::from_slice(&bytes).map_err(|error| {
            fixture_fault(
                "fixture",
                &format!("invalid {FIXTURE_FILE}: {error}"),
                "use bounded JSON with invocations, input_json, expected_output, and effects",
            )
        })?;
        if fixture.invocations.is_empty() || fixture.invocations.len() > INVOCATIONS_MAX {
            return Err(fixture_fault(
                "fixture",
                "invocation count must be 1 through 128",
                "provide at least one and no more than 128 invocations",
            ));
        }
        for invocation in &fixture.invocations {
            if invocation.name.is_empty() || invocation.effects.len() > EFFECTS_MAX {
                return Err(fixture_fault(
                    "fixture",
                    "invocation name is empty or effect count exceeds 128",
                    "name every invocation and bound its effect plan",
                ));
            }
            validate_json(&invocation.input_json, "input_json")?;
            for effect in &invocation.effects {
                if effect.operation.is_empty() {
                    return Err(fixture_fault(
                        "fixture",
                        "effect operation is empty",
                        "name each expected host operation",
                    ));
                }
                validate_json(&effect.request_json, "effect request_json")?;
            }
        }
        Ok(fixture)
    }
}

fn validate_json(text: &str, name: &str) -> Result<()> {
    serde_json::from_str::<serde_json::Value>(text).map_err(|error| {
        fixture_fault(
            "fixture",
            &format!("{name} is not JSON: {error}"),
            "encode fixture input and effect requests as JSON strings",
        )
    })?;
    Ok(())
}

fn execute(
    package_root: &Path,
    grants: &[Capability],
    fixture: &Fixture,
) -> Result<Vec<MockInvocation>> {
    let package = PluginPackage::discover(package_root, true).map_err(|error| {
        fixture_fault(
            "package",
            &error.to_string(),
            "make plugin.kdl and its artifact valid inside the package",
        )
    })?;
    let engine = PluginEngine::new().map_err(|error| {
        fixture_fault(
            "host",
            &error.to_string(),
            "repair the sealed plugin host configuration",
        )
    })?;
    let mut generation =
        PluginGeneration::load_package(&engine, package, grants).map_err(|error| {
            fixture_fault(
                "host",
                &error.to_string(),
                "grant every declared capability or repair the artifact",
            )
        })?;
    fixture
        .invocations
        .iter()
        .enumerate()
        .map(|(index, invocation)| {
            let plans = invocation
                .effects
                .iter()
                .map(|effect| MockEffectPlan {
                    operation: effect.operation.clone(),
                    request_json: effect.request_json.clone(),
                    response: effect.response.clone(),
                })
                .collect();
            let trace = generation
                .invoke_with_mock_effects(&invocation.name, &invocation.input_json, plans)
                .map_err(|error| {
                    fixture_fault(
                        "invoke",
                        &format!("invocation {index} {}: {error}", invocation.name),
                        "match declared tools, grants, input, and ordered mock effects",
                    )
                })?;
            if trace.output != invocation.expected_output {
                return Err(fixture_fault(
                    "output",
                    &format!(
                        "invocation {index} {} returned {:?}, expected {:?}",
                        invocation.name, trace.output, invocation.expected_output
                    ),
                    "update expected_output or fix the guest result",
                ));
            }
            Ok(trace)
        })
        .collect()
}

fn fixture_fault(stage: &str, cause: &str, fix: &str) -> SmithError {
    SmithError::Provider {
        fault: ProviderFault::Plugin {
            message: format!("test {stage}: {cause}; fix: {fix}"),
        },
    }
}