Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-harness/tests/openai_responses.rs

Raw
//! Responses component contract (`SMH-SPEC-PLUG0001`, `SMH-SPEC-SPEC0001`).

use smith_harness::plugins::PluginManifest;
use smith_harness::wasm::bindings::exports::smith::plugin::provider::{
    CatalogModel, CatalogSnapshot, CatalogSource, MessageBlock, MessageRole, ModelCapabilities,
    ProviderMessage, ProviderRequest, ProviderSettings, ProviderTool, SettingsProvenance,
    StopReason, StreamEvent, ToolResult, ToolUse,
};
use smith_harness::wasm::{PluginEngine, PluginGeneration};

fn generation(engine: &PluginEngine) -> PluginGeneration {
    let directory = std::env::var_os("SMITH_COMPONENT_ARTIFACT_DIR").map_or_else(
        || {
            let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
            let target = std::env::var_os("CARGO_TARGET_DIR").unwrap_or_else(|| "target".into());
            workspace.join(target).join("wasm32-wasip2/release")
        },
        std::path::PathBuf::from,
    );
    let artifact = directory.join("smith_plugin_protocol_openai_responses.wasm");
    let digest = smith::sha256::digest_hex(&std::fs::read(&artifact).unwrap());
    assert_eq!(
        digest,
        include_str!("../../plugins/protocol-openai-responses/component.sha256").trim()
    );
    let manifest = PluginManifest::parse(
        "id \"smith:protocol-openai-responses\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"smith_plugin_protocol_openai_responses.wasm\"", 
    ).unwrap();
    PluginGeneration::load(engine, manifest, &artifact, &[]).unwrap()
}

fn request() -> ProviderRequest {
    ProviderRequest {
        code_assist: None,
        messages: vec![ProviderMessage {
            role: MessageRole::User,
            blocks: vec![MessageBlock::Text("Hello".into())],
        }],
        tools: Vec::new(),
        output_tokens_max: None,
    }
}

fn catalog() -> CatalogSnapshot {
    CatalogSnapshot {
        generation: 1,
        source: CatalogSource::Bundled,
        models: vec![CatalogModel {
            id: "fixture-model".into(),
            capabilities: ModelCapabilities {
                output_tokens_max: 1024,
                tool_use: false,
                thinking: false,
            },
        }],
    }
}

fn settings() -> ProviderSettings {
    ProviderSettings {
        include_usage: true,
        subscription: false,
        provenance: SettingsProvenance::BuiltIn,
    }
}

#[test]
fn responses_request_is_nonpersistent_streaming_complete_history() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    let template = generation
        .build_provider_request("fixture-model", &request(), settings(), &catalog())
        .unwrap();
    let body: serde_json::Value = serde_json::from_str(&template.body).unwrap();
    assert_eq!(body["store"], false);
    assert_eq!(body["stream"], true);
    assert_eq!(body["input"][0]["content"], "Hello");
    assert!(body.get("previous_response_id").is_none());
    assert!(body.get("messages").is_none());
    assert_eq!(template.url, "https://protocol.invalid/responses");
    assert_eq!(template.authorization, "Bearer {{secret:connection-oauth}}");
}

#[test]
fn responses_refuses_reasoning_models_before_advertising_unsupported_events() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    let mut catalog = catalog();
    catalog.models[0].capabilities.thinking = true;
    assert!(
        generation
            .build_provider_request("fixture-model", &request(), settings(), &catalog)
            .is_err()
    );
}

const COMPLETED: &str = "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":7,\"output_tokens\":3,\"input_tokens_details\":{\"cached_tokens\":2}}}}\n\n";

#[test]
fn responses_complete_history_preserves_developer_messages_calls_and_outputs() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    let mut request = request();
    request.messages.insert(
        0,
        ProviderMessage {
            role: MessageRole::System,
            blocks: vec![MessageBlock::Text("Follow instructions".into())],
        },
    );
    request.messages.push(ProviderMessage {
        role: MessageRole::Assistant,
        blocks: vec![MessageBlock::ToolUse(ToolUse {
            name: "read".into(),
            call_id: "call-1".into(),
            input_json: r#"{"path":"a"}"#.into(),
        })],
    });
    request.messages.push(ProviderMessage {
        role: MessageRole::Tool,
        blocks: vec![MessageBlock::ToolResult(ToolResult {
            name: "read".into(),
            call_id: "call-1".into(),
            output: "file bytes".into(),
        })],
    });
    let template = generation
        .build_provider_request("fixture-model", &request, settings(), &catalog())
        .unwrap();
    let body: serde_json::Value = serde_json::from_str(&template.body).unwrap();
    assert_eq!(body["input"][0]["role"], "developer");
    assert_eq!(body["input"].as_array().unwrap().len(), 4);
    assert_eq!(body["input"][2]["type"], "function_call");
    assert_eq!(body["input"][2]["namespace"], "smith");
    assert_eq!(body["input"][3]["type"], "function_call_output");
    assert_eq!(body["input"][3]["call_id"], "call-1");
    assert_eq!(body["input"][3]["output"], "file bytes");
}

#[test]
fn responses_rejects_unsupported_output_limit_before_transport() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    let mut request = request();
    request.output_tokens_max = Some(1);
    let error = generation
        .build_provider_request("fixture-model", &request, settings(), &catalog())
        .unwrap_err();
    assert!(
        error
            .to_string()
            .contains("does not support max_output_tokens")
    );
}

#[test]
fn responses_tools_require_account_capability_and_use_namespaces() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    let mut request = request();
    request.tools.push(ProviderTool {
        name: "read".into(),
        description: "Read a local file".into(),
        input_schema_json: r#"{"type":"object","properties":{"path":{"type":"string"}}}"#.into(),
    });
    assert!(
        generation
            .build_provider_request("fixture-model", &request, settings(), &catalog())
            .is_err()
    );
    let mut catalog = catalog();
    catalog.models[0].capabilities.tool_use = true;
    let template = generation
        .build_provider_request("fixture-model", &request, settings(), &catalog)
        .unwrap();
    let body: serde_json::Value = serde_json::from_str(&template.body).unwrap();
    assert_eq!(body["tools"][0]["type"], "namespace");
    assert_eq!(body["tools"][0]["name"], "smith");
    assert_eq!(body["tools"][0]["tools"][0]["name"], "read");
    assert_eq!(body["tools"][0]["tools"][0]["type"], "function");
    assert!(
        generation
            .build_provider_request("unavailable-model", &request, settings(), &catalog)
            .is_err()
    );
}

#[test]
fn responses_component_preserves_utf8_and_completion_at_every_byte_split() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    let stream = format!(
        "data: {{\"type\":\"response.output_text.delta\",\"delta\":\"☃\"}}\r\n\r\n{COMPLETED}"
    );
    for split in 0..=stream.len() {
        let first = generation
            .decode_provider_chunk(&[], &stream.as_bytes()[..split], false)
            .unwrap();
        assert!(
            !first
                .events
                .iter()
                .any(|event| matches!(event, StreamEvent::Stop(_)))
                || split == stream.len()
        );
        let second = generation
            .decode_provider_chunk(&first.state, &stream.as_bytes()[split..], true)
            .unwrap();
        let events: Vec<_> = first.events.into_iter().chain(second.events).collect();
        assert_eq!(
            events
                .iter()
                .filter(|event| matches!(event, StreamEvent::Text(text) if text == "☃"))
                .count(),
            1,
            "split {split}"
        );
        assert_eq!(events.iter().filter(|event| matches!(event, StreamEvent::Stop((StopReason::EndTurn, Some(usage))) if usage.input_tokens == 7 && usage.output_tokens == 3 && usage.input_cache_hit_tokens == 2)).count(), 1, "split {split}");
    }
}

#[test]
fn responses_component_requires_completed_not_chat_completions_done() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    for invalid in [
        "data: [DONE]\n\n",
        "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
        "data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\"}\n\n",
        "data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"code\":\"subscription_sharing_usage_limit_exceeded\"}}}\n\n",
        "data: {\"type\":\"response.incomplete\"}\n\n",
        "data: {\"type\":\"error\"}\n\n",
        "data: {bad-json}\n\n",
    ] {
        assert!(
            generation
                .decode_provider_chunk(&[], invalid.as_bytes(), true)
                .is_err(),
            "{invalid}"
        );
    }
    let truncated = COMPLETED.trim_end();
    assert!(
        generation
            .decode_provider_chunk(&[], truncated.as_bytes(), true)
            .is_err()
    );
    let duplicated = format!("{COMPLETED}{COMPLETED}");
    assert!(
        generation
            .decode_provider_chunk(&[], duplicated.as_bytes(), true)
            .is_err()
    );
}

#[test]
fn responses_component_emits_function_only_from_complete_output_item() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    let prefix = b"data: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"path\\\"\"}\n\n";
    let step = generation
        .decode_provider_chunk(&[], prefix, false)
        .unwrap();
    assert!(step.events.is_empty());
    let call = "data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"status\":\"completed\",\"namespace\":\"smith\",\"name\":\"read\",\"call_id\":\"call-1\",\"arguments\":\"{\\\"path\\\":\\\"a\\\"}\"}}\n\n";
    let stream = format!("{call}{COMPLETED}");
    let step = generation
        .decode_provider_chunk(&step.state, stream.as_bytes(), true)
        .unwrap();
    assert!(
        matches!(&step.events[0], StreamEvent::ToolUse(tool) if tool.name == "read" && tool.call_id == "call-1" && tool.input_json == r#"{"path":"a"}"#)
    );
    assert!(matches!(
        &step.events[1],
        StreamEvent::Stop((StopReason::ToolUse, Some(_)))
    ));
    let duplicate = format!("{call}{call}{COMPLETED}");
    assert!(
        generation
            .decode_provider_chunk(&[], duplicate.as_bytes(), true)
            .is_err()
    );
    for namespace in [
        "",
        "\"namespace\":null,",
        "\"namespace\":\"\",",
        "\"namespace\":\"foreign\",",
    ] {
        let invalid = stream.replace("\"namespace\":\"smith\",", namespace);
        assert!(
            generation
                .decode_provider_chunk(&[], invalid.as_bytes(), true)
                .is_err(),
            "unadvertised namespace accepted: {namespace:?}"
        );
    }
}

#[test]
fn responses_component_bounds_frame_state_and_request_bytes() {
    let engine = PluginEngine::new().unwrap();
    let mut generation = generation(&engine);
    assert!(
        generation
            .decode_provider_chunk(&[], &vec![b'x'; 64 * 1024 + 1], false)
            .is_err()
    );
    assert!(
        generation
            .decode_provider_chunk(&[1, 0], &[], false)
            .is_err()
    );
    let mut request = request();
    request.messages[0].blocks = vec![MessageBlock::Text("x".repeat(64 * 1024 + 1))];
    assert!(
        generation
            .build_provider_request("fixture-model", &request, settings(), &catalog())
            .is_err()
    );
}

#[test]
fn responses_component_does_not_advertise_unbound_subscription_provider() {
    let engine = PluginEngine::new().unwrap();
    let generation = generation(&engine);
    assert!(generation.registrations().providers.is_empty());
    assert!(generation.registrations().credential_sources.is_empty());
}