repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-harness/tests/openai_responses.rs
Raw//! Responses component contract (`SMH-SPEC-PLUG0001`, `SMH-SPEC-SPEC0001`).
use smith_harness::plugins::PluginManifest;
use smith_harness::wasm::bindings::exports::smith::plugin::provider::{
CatalogModel, CatalogSnapshot, CatalogSource, MessageBlock, MessageRole, ModelCapabilities,
ProviderMessage, ProviderRequest, ProviderSettings, ProviderTool, SettingsProvenance,
StopReason, StreamEvent, ToolResult, ToolUse,
};
use smith_harness::wasm::{PluginEngine, PluginGeneration};
fn generation(engine: &PluginEngine) -> PluginGeneration {
let directory = std::env::var_os("SMITH_COMPONENT_ARTIFACT_DIR").map_or_else(
|| {
let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
let target = std::env::var_os("CARGO_TARGET_DIR").unwrap_or_else(|| "target".into());
workspace.join(target).join("wasm32-wasip2/release")
},
std::path::PathBuf::from,
);
let artifact = directory.join("smith_plugin_protocol_openai_responses.wasm");
let digest = smith::sha256::digest_hex(&std::fs::read(&artifact).unwrap());
assert_eq!(
digest,
include_str!("../../plugins/protocol-openai-responses/component.sha256").trim()
);
let manifest = PluginManifest::parse(
"id \"smith:protocol-openai-responses\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"smith_plugin_protocol_openai_responses.wasm\"",
).unwrap();
PluginGeneration::load(engine, manifest, &artifact, &[]).unwrap()
}
fn request() -> ProviderRequest {
ProviderRequest {
code_assist: None,
messages: vec![ProviderMessage {
role: MessageRole::User,
blocks: vec![MessageBlock::Text("Hello".into())],
}],
tools: Vec::new(),
output_tokens_max: None,
}
}
fn catalog() -> CatalogSnapshot {
CatalogSnapshot {
generation: 1,
source: CatalogSource::Bundled,
models: vec![CatalogModel {
id: "fixture-model".into(),
capabilities: ModelCapabilities {
output_tokens_max: 1024,
tool_use: false,
thinking: false,
},
}],
}
}
fn settings() -> ProviderSettings {
ProviderSettings {
include_usage: true,
subscription: false,
provenance: SettingsProvenance::BuiltIn,
}
}
#[test]
fn responses_request_is_nonpersistent_streaming_complete_history() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
let template = generation
.build_provider_request("fixture-model", &request(), settings(), &catalog())
.unwrap();
let body: serde_json::Value = serde_json::from_str(&template.body).unwrap();
assert_eq!(body["store"], false);
assert_eq!(body["stream"], true);
assert_eq!(body["input"][0]["content"], "Hello");
assert!(body.get("previous_response_id").is_none());
assert!(body.get("messages").is_none());
assert_eq!(template.url, "https://protocol.invalid/responses");
assert_eq!(template.authorization, "Bearer {{secret:connection-oauth}}");
}
#[test]
fn responses_refuses_reasoning_models_before_advertising_unsupported_events() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
let mut catalog = catalog();
catalog.models[0].capabilities.thinking = true;
assert!(
generation
.build_provider_request("fixture-model", &request(), settings(), &catalog)
.is_err()
);
}
const COMPLETED: &str = "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":7,\"output_tokens\":3,\"input_tokens_details\":{\"cached_tokens\":2}}}}\n\n";
#[test]
fn responses_complete_history_preserves_developer_messages_calls_and_outputs() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
let mut request = request();
request.messages.insert(
0,
ProviderMessage {
role: MessageRole::System,
blocks: vec![MessageBlock::Text("Follow instructions".into())],
},
);
request.messages.push(ProviderMessage {
role: MessageRole::Assistant,
blocks: vec![MessageBlock::ToolUse(ToolUse {
name: "read".into(),
call_id: "call-1".into(),
input_json: r#"{"path":"a"}"#.into(),
})],
});
request.messages.push(ProviderMessage {
role: MessageRole::Tool,
blocks: vec![MessageBlock::ToolResult(ToolResult {
name: "read".into(),
call_id: "call-1".into(),
output: "file bytes".into(),
})],
});
let template = generation
.build_provider_request("fixture-model", &request, settings(), &catalog())
.unwrap();
let body: serde_json::Value = serde_json::from_str(&template.body).unwrap();
assert_eq!(body["input"][0]["role"], "developer");
assert_eq!(body["input"].as_array().unwrap().len(), 4);
assert_eq!(body["input"][2]["type"], "function_call");
assert_eq!(body["input"][2]["namespace"], "smith");
assert_eq!(body["input"][3]["type"], "function_call_output");
assert_eq!(body["input"][3]["call_id"], "call-1");
assert_eq!(body["input"][3]["output"], "file bytes");
}
#[test]
fn responses_rejects_unsupported_output_limit_before_transport() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
let mut request = request();
request.output_tokens_max = Some(1);
let error = generation
.build_provider_request("fixture-model", &request, settings(), &catalog())
.unwrap_err();
assert!(
error
.to_string()
.contains("does not support max_output_tokens")
);
}
#[test]
fn responses_tools_require_account_capability_and_use_namespaces() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
let mut request = request();
request.tools.push(ProviderTool {
name: "read".into(),
description: "Read a local file".into(),
input_schema_json: r#"{"type":"object","properties":{"path":{"type":"string"}}}"#.into(),
});
assert!(
generation
.build_provider_request("fixture-model", &request, settings(), &catalog())
.is_err()
);
let mut catalog = catalog();
catalog.models[0].capabilities.tool_use = true;
let template = generation
.build_provider_request("fixture-model", &request, settings(), &catalog)
.unwrap();
let body: serde_json::Value = serde_json::from_str(&template.body).unwrap();
assert_eq!(body["tools"][0]["type"], "namespace");
assert_eq!(body["tools"][0]["name"], "smith");
assert_eq!(body["tools"][0]["tools"][0]["name"], "read");
assert_eq!(body["tools"][0]["tools"][0]["type"], "function");
assert!(
generation
.build_provider_request("unavailable-model", &request, settings(), &catalog)
.is_err()
);
}
#[test]
fn responses_component_preserves_utf8_and_completion_at_every_byte_split() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
let stream = format!(
"data: {{\"type\":\"response.output_text.delta\",\"delta\":\"☃\"}}\r\n\r\n{COMPLETED}"
);
for split in 0..=stream.len() {
let first = generation
.decode_provider_chunk(&[], &stream.as_bytes()[..split], false)
.unwrap();
assert!(
!first
.events
.iter()
.any(|event| matches!(event, StreamEvent::Stop(_)))
|| split == stream.len()
);
let second = generation
.decode_provider_chunk(&first.state, &stream.as_bytes()[split..], true)
.unwrap();
let events: Vec<_> = first.events.into_iter().chain(second.events).collect();
assert_eq!(
events
.iter()
.filter(|event| matches!(event, StreamEvent::Text(text) if text == "☃"))
.count(),
1,
"split {split}"
);
assert_eq!(events.iter().filter(|event| matches!(event, StreamEvent::Stop((StopReason::EndTurn, Some(usage))) if usage.input_tokens == 7 && usage.output_tokens == 3 && usage.input_cache_hit_tokens == 2)).count(), 1, "split {split}");
}
}
#[test]
fn responses_component_requires_completed_not_chat_completions_done() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
for invalid in [
"data: [DONE]\n\n",
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n",
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\"}\n\n",
"data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"code\":\"subscription_sharing_usage_limit_exceeded\"}}}\n\n",
"data: {\"type\":\"response.incomplete\"}\n\n",
"data: {\"type\":\"error\"}\n\n",
"data: {bad-json}\n\n",
] {
assert!(
generation
.decode_provider_chunk(&[], invalid.as_bytes(), true)
.is_err(),
"{invalid}"
);
}
let truncated = COMPLETED.trim_end();
assert!(
generation
.decode_provider_chunk(&[], truncated.as_bytes(), true)
.is_err()
);
let duplicated = format!("{COMPLETED}{COMPLETED}");
assert!(
generation
.decode_provider_chunk(&[], duplicated.as_bytes(), true)
.is_err()
);
}
#[test]
fn responses_component_emits_function_only_from_complete_output_item() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
let prefix = b"data: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"path\\\"\"}\n\n";
let step = generation
.decode_provider_chunk(&[], prefix, false)
.unwrap();
assert!(step.events.is_empty());
let call = "data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"status\":\"completed\",\"namespace\":\"smith\",\"name\":\"read\",\"call_id\":\"call-1\",\"arguments\":\"{\\\"path\\\":\\\"a\\\"}\"}}\n\n";
let stream = format!("{call}{COMPLETED}");
let step = generation
.decode_provider_chunk(&step.state, stream.as_bytes(), true)
.unwrap();
assert!(
matches!(&step.events[0], StreamEvent::ToolUse(tool) if tool.name == "read" && tool.call_id == "call-1" && tool.input_json == r#"{"path":"a"}"#)
);
assert!(matches!(
&step.events[1],
StreamEvent::Stop((StopReason::ToolUse, Some(_)))
));
let duplicate = format!("{call}{call}{COMPLETED}");
assert!(
generation
.decode_provider_chunk(&[], duplicate.as_bytes(), true)
.is_err()
);
for namespace in [
"",
"\"namespace\":null,",
"\"namespace\":\"\",",
"\"namespace\":\"foreign\",",
] {
let invalid = stream.replace("\"namespace\":\"smith\",", namespace);
assert!(
generation
.decode_provider_chunk(&[], invalid.as_bytes(), true)
.is_err(),
"unadvertised namespace accepted: {namespace:?}"
);
}
}
#[test]
fn responses_component_bounds_frame_state_and_request_bytes() {
let engine = PluginEngine::new().unwrap();
let mut generation = generation(&engine);
assert!(
generation
.decode_provider_chunk(&[], &vec![b'x'; 64 * 1024 + 1], false)
.is_err()
);
assert!(
generation
.decode_provider_chunk(&[1, 0], &[], false)
.is_err()
);
let mut request = request();
request.messages[0].blocks = vec![MessageBlock::Text("x".repeat(64 * 1024 + 1))];
assert!(
generation
.build_provider_request("fixture-model", &request, settings(), &catalog())
.is_err()
);
}
#[test]
fn responses_component_does_not_advertise_unbound_subscription_provider() {
let engine = PluginEngine::new().unwrap();
let generation = generation(&engine);
assert!(generation.registrations().providers.is_empty());
assert!(generation.registrations().credential_sources.is_empty());
}