repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-harness/src/plugins/test.rs
Raw//! Deterministic plugin fixtures through the sealed host (`SMH-SPEC-WASM0001`).
use super::{Capability, PluginPackage};
use crate::wasm::{MockEffectPlan, MockInvocation, PluginEngine, PluginGeneration};
use serde::Deserialize;
use smith::error::{ProviderFault, Result, SmithError};
use std::io::Read;
use std::path::Path;
const FIXTURE_FILE: &str = "plugin-test.json";
const FIXTURE_BYTES_MAX: usize = 64 * 1024;
const INVOCATIONS_MAX: usize = 128;
const EFFECTS_MAX: usize = 128;
/// Execute every declared fixture twice through fresh sealed-host generations.
///
/// # Errors
///
/// Returns a stage, cause, and user fix when fixture parsing, expected output,
/// effect ordering, grants, or repeated-call determinism fails.
pub fn run(package_root: &Path, grants: &[Capability]) -> Result<()> {
let fixture = Fixture::read(package_root)?;
let first = execute(package_root, grants, &fixture)?;
let second = execute(package_root, grants, &fixture)?;
if first != second {
return Err(fixture_fault(
"determinism",
"same artifact and fixed host inputs produced different traces",
"remove ambient state or make the guest output and effects deterministic",
));
}
Ok(())
}
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Fixture {
invocations: Vec<Invocation>,
}
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Invocation {
name: String,
input_json: String,
expected_output: String,
effects: Vec<Effect>,
}
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Effect {
operation: String,
request_json: String,
response: String,
}
impl Fixture {
fn read(package_root: &Path) -> Result<Self> {
let path = package_root.join(FIXTURE_FILE);
let metadata = std::fs::metadata(&path).map_err(|error| {
fixture_fault(
"fixture",
&format!("missing {FIXTURE_FILE}: {error}"),
&format!(
"add {FIXTURE_FILE} with invocation input, expected output, and effect plans"
),
)
})?;
if metadata.len() > FIXTURE_BYTES_MAX as u64 {
return Err(fixture_fault(
"fixture",
&format!("{FIXTURE_FILE} exceeds {FIXTURE_BYTES_MAX} bytes"),
"reduce fixture data below the fixture byte limit",
));
}
let mut bytes = Vec::new();
let file = std::fs::File::open(&path).map_err(|error| {
fixture_fault("fixture", &error.to_string(), "make fixture readable")
})?;
#[expect(
clippy::disallowed_methods,
reason = "Take caps this fixture read at FIXTURE_BYTES_MAX + 1"
)]
file.take((FIXTURE_BYTES_MAX + 1) as u64)
.read_to_end(&mut bytes)
.map_err(|error| {
fixture_fault("fixture", &error.to_string(), "make fixture readable")
})?;
let fixture: Self = serde_json::from_slice(&bytes).map_err(|error| {
fixture_fault(
"fixture",
&format!("invalid {FIXTURE_FILE}: {error}"),
"use bounded JSON with invocations, input_json, expected_output, and effects",
)
})?;
if fixture.invocations.is_empty() || fixture.invocations.len() > INVOCATIONS_MAX {
return Err(fixture_fault(
"fixture",
"invocation count must be 1 through 128",
"provide at least one and no more than 128 invocations",
));
}
for invocation in &fixture.invocations {
if invocation.name.is_empty() || invocation.effects.len() > EFFECTS_MAX {
return Err(fixture_fault(
"fixture",
"invocation name is empty or effect count exceeds 128",
"name every invocation and bound its effect plan",
));
}
validate_json(&invocation.input_json, "input_json")?;
for effect in &invocation.effects {
if effect.operation.is_empty() {
return Err(fixture_fault(
"fixture",
"effect operation is empty",
"name each expected host operation",
));
}
validate_json(&effect.request_json, "effect request_json")?;
}
}
Ok(fixture)
}
}
fn validate_json(text: &str, name: &str) -> Result<()> {
serde_json::from_str::<serde_json::Value>(text).map_err(|error| {
fixture_fault(
"fixture",
&format!("{name} is not JSON: {error}"),
"encode fixture input and effect requests as JSON strings",
)
})?;
Ok(())
}
fn execute(
package_root: &Path,
grants: &[Capability],
fixture: &Fixture,
) -> Result<Vec<MockInvocation>> {
let package = PluginPackage::discover(package_root, true).map_err(|error| {
fixture_fault(
"package",
&error.to_string(),
"make plugin.kdl and its artifact valid inside the package",
)
})?;
let engine = PluginEngine::new().map_err(|error| {
fixture_fault(
"host",
&error.to_string(),
"repair the sealed plugin host configuration",
)
})?;
let mut generation =
PluginGeneration::load_package(&engine, package, grants).map_err(|error| {
fixture_fault(
"host",
&error.to_string(),
"grant every declared capability or repair the artifact",
)
})?;
fixture
.invocations
.iter()
.enumerate()
.map(|(index, invocation)| {
let plans = invocation
.effects
.iter()
.map(|effect| MockEffectPlan {
operation: effect.operation.clone(),
request_json: effect.request_json.clone(),
response: effect.response.clone(),
})
.collect();
let trace = generation
.invoke_with_mock_effects(&invocation.name, &invocation.input_json, plans)
.map_err(|error| {
fixture_fault(
"invoke",
&format!("invocation {index} {}: {error}", invocation.name),
"match declared tools, grants, input, and ordered mock effects",
)
})?;
if trace.output != invocation.expected_output {
return Err(fixture_fault(
"output",
&format!(
"invocation {index} {} returned {:?}, expected {:?}",
invocation.name, trace.output, invocation.expected_output
),
"update expected_output or fix the guest result",
));
}
Ok(trace)
})
.collect()
}
fn fixture_fault(stage: &str, cause: &str, fix: &str) -> SmithError {
SmithError::Provider {
fault: ProviderFault::Plugin {
message: format!("test {stage}: {cause}; fix: {fix}"),
},
}
}