//! Plugin package checks through the built binary (`SMH-SPEC-WASM0001`, Tooling; //! `SMH-SPEC-SPEC0001`, Providers: authentication; `SMH-SPEC-PLUG0001`, Tool visibility and authority). use assert_cmd::Command; use smith_core::frame::Frame; use smith_core::session::{EntryContent, PROVIDER_DISPATCH_PROVENANCE_KIND}; use smith_core::store::SessionReader; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; struct TestDir(PathBuf); impl TestDir { fn new(prefix: &str) -> Self { let root = Self::vacant(prefix); std::fs::create_dir(&root.0) .unwrap_or_else(|error| panic!("create fixture root {}: {error}", root.display())); root } fn vacant(prefix: &str) -> Self { static NEXT: AtomicUsize = AtomicUsize::new(0); loop { let path = std::env::temp_dir().join(format!( "{prefix}-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed) )); if !path.exists() { return Self(path); } } } } impl Deref for TestDir { type Target = Path; fn deref(&self) -> &Self::Target { &self.0 } } impl AsRef for TestDir { fn as_ref(&self) -> &Path { self } } impl AsRef for TestDir { fn as_ref(&self) -> &std::ffi::OsStr { self.0.as_os_str() } } impl Drop for TestDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); } } struct TestChild(std::process::Child); impl TestChild { fn wait_with_output(mut self) -> std::process::Output { let status = self.0.wait().unwrap(); self.output(status) } fn kill_with_output(&mut self) -> std::process::Output { let _ = self.0.kill(); let status = self.0.wait().unwrap(); self.output(status) } fn output(&mut self, status: std::process::ExitStatus) -> std::process::Output { let mut stdout = Vec::new(); let mut stderr = Vec::new(); std::io::Read::read_to_end(self.0.stdout.as_mut().unwrap(), &mut stdout).unwrap(); std::io::Read::read_to_end(self.0.stderr.as_mut().unwrap(), &mut stderr).unwrap(); std::process::Output { status, stdout, stderr, } } } impl Drop for TestChild { fn drop(&mut self) { if self.0.try_wait().ok().flatten().is_none() { let _ = self.0.kill(); } let _ = self.0.wait(); } } fn package(name: &str, manifest: &str, artifact: &Path) -> TestDir { let root = TestDir::new(&format!("smith-plugin-check-{name}")); std::fs::write(root.join("plugin.kdl"), manifest).unwrap(); std::fs::copy(artifact, root.join("fixture.wasm")).unwrap(); root } fn fixture_package(name: &str, manifest: &str) -> TestDir { package( name, manifest, &Path::new(env!("CARGO_MANIFEST_DIR")) .join("../smith-harness/tests/fixtures/plugin-package/fixture.wasm"), ) } fn echo_artifact() -> PathBuf { let status = std::process::Command::new("cargo") .args([ "build", "-p", "smith-plugin-echo", "--target", "wasm32-wasip2", "--locked", ]) .status() .unwrap(); assert!(status.success(), "echo plugin build failed"); Path::new(env!("CARGO_MANIFEST_DIR")) .join("../target/wasm32-wasip2/debug/smith_plugin_echo.wasm") } fn initialized_package(name: &str) -> TestDir { let root = TestDir::vacant(&format!("smith-plugin-test-{name}")); Command::cargo_bin("smith") .unwrap() .env("HOME", root.join("home")) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .args(["plugin", "init"]) .arg(&root) .args(["--name", &format!("example:{name}")]) .assert() .success(); root } fn plugin_test(package: &Path, grants: &[&str]) -> std::process::Output { let mut command = Command::cargo_bin("smith").unwrap(); command.args(["plugin", "test"]).arg(package); for grant in grants { command.args(["--grant", grant]); } command.output().unwrap() } fn smith(package: &Path, grants: &[&str]) -> std::process::Output { let home = package.join("xdg"); let mut command = Command::cargo_bin("smith").unwrap(); command .env("XDG_CONFIG_HOME", home.join("config")) .env("XDG_DATA_HOME", home.join("data")) .env("XDG_STATE_HOME", home.join("state")) .env("XDG_CACHE_HOME", home.join("cache")) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args(["plugin", "check", "--trust"]) .arg(package); for grant in grants { command.args(["--grant", grant]); } command.output().unwrap() } #[test] fn typescript_plugin_builds_tests_and_loads_with_the_sealed_host() { let package = Path::new(env!("CARGO_MANIFEST_DIR")).join("../plugins/example-typescript"); let output = Command::cargo_bin("smith") .unwrap() .args(["plugin", "test"]) .arg(&package) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!( output.stdout, format!("{}: test passed\n", package.display()).as_bytes() ); } #[test] fn plugin_init_typescript_scaffolds_the_jco_adapter() { let root = TestDir::new("smith-plugin-init-typescript"); let package = root.join("demo"); let output = Command::cargo_bin("smith") .unwrap() .args(["plugin", "init"]) .arg(&package) .args(["--name", "example:demo", "--language", "typescript"]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert!(package.join("package.json").is_file()); assert!(package.join("tsconfig.json").is_file()); assert!(package.join("src/plugin.ts").is_file()); assert!(package.join("wit/smith-plugin.wit").is_file()); assert!(!package.join("Cargo.toml").exists()); } #[test] fn plugin_init_build_and_test_scaffold_a_pinned_raw_wit_component() { let root = TestDir::new("smith-plugin-init"); let package = root.join("demo"); let init = Command::cargo_bin("smith") .unwrap() .args(["plugin", "init"]) .arg(&package) .args(["--name", "example:demo"]) .output() .unwrap(); assert!( init.status.success(), "{}", String::from_utf8_lossy(&init.stderr) ); assert!(package.join("wit/smith-plugin.wit").is_file()); assert!(package.join("build.rs").is_file()); assert!(package.join("Cargo.lock").is_file()); let build = Command::cargo_bin("smith") .unwrap() .args(["plugin", "build"]) .arg(&package) .output() .unwrap(); assert!( build.status.success(), "{}", String::from_utf8_lossy(&build.stderr) ); let test = Command::cargo_bin("smith") .unwrap() .args(["plugin", "test"]) .arg(&package) .output() .unwrap(); assert!( test.status.success(), "{}", String::from_utf8_lossy(&test.stderr) ); assert_eq!( test.stdout, format!("{}: test passed\n", package.display()).as_bytes() ); std::fs::remove_dir_all(root).unwrap(); } #[test] fn plugin_test_runs_echo_fixtures_and_reports_fixture_failures() { let package = initialized_package("fixture-echo"); std::fs::write( package.join("plugin-test.json"), r#"{"invocations":[{"name":"fixture","input_json":"{\"text\":\"hello\"}","expected_output":"{\"text\":\"hello\"}","effects":[]}]}"#, ) .unwrap(); let success = plugin_test(&package, &[]); assert!( success.status.success(), "{}", String::from_utf8_lossy(&success.stderr) ); assert_eq!( success.stdout, format!("{}: test passed\n", package.display()).as_bytes() ); std::fs::write( package.join("plugin-test.json"), r#"{"invocations":[{"name":"fixture","input_json":"{}","expected_output":"wrong","effects":[]}]}"#, ) .unwrap(); let mismatch = plugin_test(&package, &[]); assert!(!mismatch.status.success()); insta::assert_snapshot!( "plugin_test_output_mismatch", String::from_utf8(mismatch.stderr).unwrap() ); std::fs::write( package.join("plugin.kdl"), "id \"example:fixture-echo\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_fixture_echo.wasm\"\ncapability \"workspace\"\n", ) .unwrap(); let ungranted = plugin_test(&package, &[]); assert!(!ungranted.status.success()); insta::assert_snapshot!( "plugin_test_ungranted", String::from_utf8(ungranted.stderr).unwrap() ); std::fs::remove_dir_all(package).unwrap(); plugin_test_mocks_coding_read_and_rejects_unexpected_effects(); plugin_test_mocks_http_secret_and_log_effects_deterministically(); } fn plugin_test_mocks_http_secret_and_log_effects_deterministically() { use std::net::TcpListener; let package = initialized_package("fixture-http-secret"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let destination = format!("http://{}", listener.local_addr().unwrap()); std::fs::write( package.join("src/lib.rs"), r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")] #![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")] wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" }); use exports::smith::plugin::plugin::{Guest, ToolDeclaration}; use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance}; use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations}; use smith::plugin::log; use smith::plugin::net_scope::{self, Header, HttpRequest, RequestOptions}; use smith::plugin::secret_proxy::{self, CredentialExchange, SecretRef}; struct Plugin; impl RegistrationGuest for Plugin { fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } } } impl ProviderGuest for Plugin { fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } } fn validate_settings(_: ProviderSettings) -> Result { Err("no codec".to_string()) } fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result { Err("no codec".to_string()) } fn decode_response(_: Vec, _: Vec, _: bool) -> Result { Err("no codec".to_string()) } } impl Guest for Plugin { fn declare_tools() -> Vec { vec![ToolDeclaration { name: "round".to_string(), description: "Mocked HTTP, secret, and log.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] } fn invoke(_: String, _: String) -> Result { let destination = "http://127.0.0.1:9".to_string(); let secret = secret_proxy::exchange(&CredentialExchange { source: SecretRef { id: "fixture-key".to_string() }, audience: destination.clone(), scopes: vec!["http-request".to_string()] })?; log::write("info", "fixture secret")?; let response = net_scope::request(&HttpRequest { destination, method: "POST".to_string(), headers: vec![Header { name: "Authorization".to_string(), value: format!("{{{{secret:{}}}}}", secret.id) }], body: Vec::new(), options: RequestOptions { timeout_ms: 1000, response_bytes_max: 64 } })?; Ok(format!(r#"{{"status":{}}}"#, response.status)) } } export!(Plugin); "##, ) .unwrap(); let source = std::fs::read_to_string(package.join("src/lib.rs")).unwrap(); std::fs::write( package.join("src/lib.rs"), source.replace("http://127.0.0.1:9", &destination), ) .unwrap(); std::fs::write( package.join("plugin.kdl"), "id \"example:fixture-http-secret\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_fixture_http_secret.wasm\"\ncapability \"net-scope\"\ncapability \"secret-proxy\"\ncapability \"log\"\n", ) .unwrap(); let request_secret = serde_json::json!({"source":{"id":"fixture-key"},"audience":destination.clone(),"scopes":["http-request"]}).to_string(); let request_log = r#"{"level":"info","message":"fixture secret"}"#; let request_http = serde_json::json!({"destination":destination,"method":"POST","headers":[{"name":"Authorization","value":"{{secret:mock-token}}"}],"body":[],"options":{"timeout_ms":1000,"response_bytes_max":64}}).to_string(); let fixture = format!( r#"{{"invocations":[{{"name":"round","input_json":"{{}}","expected_output":"{{\"status\":200}}","effects":[{{"operation":"credential-exchange","request_json":{},"response":"mock-token"}},{{"operation":"log-write","request_json":{},"response":""}},{{"operation":"http-request","request_json":{},"response":"{{\"status\":200,\"headers\":[],\"body\":[]}}"}}]}}]}}"#, serde_json::to_string(&request_secret).unwrap(), serde_json::to_string(request_log).unwrap(), serde_json::to_string(&request_http).unwrap(), ); std::fs::write(package.join("plugin-test.json"), &fixture).unwrap(); let xdg = package.join("isolated-xdg"); let run = || { let mut command = Command::cargo_bin("smith").unwrap(); command .env("XDG_CONFIG_HOME", xdg.join("config")) .env("XDG_DATA_HOME", xdg.join("data")) .env("XDG_STATE_HOME", xdg.join("state")) .env("XDG_CACHE_HOME", xdg.join("cache")) .env_remove("OPENAI_API_KEY") .env_remove("SMITH_PROVIDER") .args(["plugin", "test"]) .arg(&package) .args([ "--grant", "net-scope", "--grant", "secret-proxy", "--grant", "log", ]) .output() .unwrap() }; let mocked = run(); assert!( mocked.status.success(), "{}", String::from_utf8_lossy(&mocked.stderr) ); assert!( !xdg.join("data/smith/auth").exists(), "fixture must not open real auth store" ); listener.set_nonblocking(true).unwrap(); assert_eq!( listener.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock, "mocked HTTP must not reach network" ); let unexpected_fixture = fixture.replace( "\"operation\":\"http-request\"", "\"operation\":\"http-wrong\"", ); std::fs::write(package.join("plugin-test.json"), unexpected_fixture).unwrap(); let unexpected = run(); assert!(!unexpected.status.success()); assert!( String::from_utf8_lossy(&unexpected.stderr).contains("fixture effect"), "{}", String::from_utf8_lossy(&unexpected.stderr) ); assert!(!xdg.join("data/smith/auth").exists()); assert_eq!( listener.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock, "denied HTTP fixture must not reach network" ); std::fs::remove_dir_all(package).unwrap(); } fn plugin_test_mocks_coding_read_and_rejects_unexpected_effects() { let package = initialized_package("fixture-read"); std::fs::write( package.join("src/lib.rs"), r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")] #![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")] wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" }); use exports::smith::plugin::plugin::{Guest, ToolDeclaration}; use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance}; use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations}; use smith::plugin::tooling::{self, FileRead}; struct Plugin; impl RegistrationGuest for Plugin { fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } } } impl ProviderGuest for Plugin { fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } } fn validate_settings(_: ProviderSettings) -> Result { Err("no codec".to_string()) } fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result { Err("no codec".to_string()) } fn decode_response(_: Vec, _: Vec, _: bool) -> Result { Err("no codec".to_string()) } } impl Guest for Plugin { fn declare_tools() -> Vec { vec![ToolDeclaration { name: "read".to_string(), description: "Fixture read.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] } fn invoke(_: String, _: String) -> Result { tooling::read(&FileRead { path: "never-read.txt".to_string(), offset: 0, limit: 1, max_bytes: 64 }).map(|content| content.text) } } export!(Plugin); "##, ) .unwrap(); std::fs::write( package.join("plugin.kdl"), "id \"example:fixture-read\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_fixture_read.wasm\"\ncapability \"workspace\"\n", ) .unwrap(); std::fs::write( package.join("plugin-test.json"), r#"{"invocations":[{"name":"read","input_json":"{}","expected_output":"mocked","effects":[{"operation":"read","request_json":"{\"path\":\"never-read.txt\",\"offset\":0,\"limit\":1,\"max_bytes\":64}","response":"mocked"}]}]}"#, ) .unwrap(); let mocked = plugin_test(&package, &["workspace"]); assert!( mocked.status.success(), "{}", String::from_utf8_lossy(&mocked.stderr) ); assert!(!package.join("never-read.txt").exists()); std::fs::write( package.join("plugin-test.json"), r#"{"invocations":[{"name":"read","input_json":"{}","expected_output":"mocked","effects":[]}]}"#, ) .unwrap(); let unexpected = plugin_test(&package, &["workspace"]); assert!(!unexpected.status.success()); insta::assert_snapshot!( "plugin_test_unexpected_effect", String::from_utf8(unexpected.stderr).unwrap() ); std::fs::remove_dir_all(package).unwrap(); } #[test] fn plugin_install_list_reload_and_remove_keep_a_hash_pinned_registry() { let artifact = echo_artifact(); let package = package( "installed-registry", "id \"example:registry\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n", &artifact, ); let config = package.join("xdg/config"); let run = |arguments: &[&str]| { Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", &config) .env("XDG_DATA_HOME", package.join("xdg/data")) .env("XDG_STATE_HOME", package.join("xdg/state")) .env("XDG_CACHE_HOME", package.join("xdg/cache")) .args(arguments) .output() .unwrap() }; let install = run(&["plugin", "install", package.to_str().unwrap()]); assert!( install.status.success(), "{}", String::from_utf8_lossy(&install.stderr) ); assert_eq!(install.stdout, b"example:registry: installed\n"); let listed = run(&["plugin", "list"]); assert!(listed.status.success()); let listed = String::from_utf8(listed.stdout).unwrap(); assert!(listed.starts_with("installed example:registry ")); assert_eq!(listed.split_ascii_whitespace().count(), 5); let reloaded = run(&["plugin", "reload"]); assert!(!reloaded.status.success()); assert!(String::from_utf8_lossy(&reloaded.stderr).contains("runtime is unavailable")); std::fs::write(package.join("fixture.wasm"), b"changed").unwrap(); let rejected = run(&["plugin", "reload"]); assert!(!rejected.status.success()); assert!(String::from_utf8_lossy(&rejected.stderr).contains("changed since install")); let retained = run(&["plugin", "list"]); assert!(retained.status.success()); assert!( String::from_utf8(retained.stdout) .unwrap() .starts_with("installed example:registry ") ); let removed = run(&["plugin", "remove", "example:registry"]); assert!(removed.status.success()); assert_eq!(removed.stdout, b"example:registry: removed\n"); let empty = run(&["plugin", "list"]); assert!(empty.status.success()); assert!(empty.stdout.is_empty()); std::fs::remove_dir_all(package).unwrap(); } #[test] fn plugin_link_precedes_installed_package_for_reload() { let artifact = echo_artifact(); let installed = package( "link-precedence-installed", "id \"example:precedence\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n", &artifact, ); let linked = package( "link-precedence-linked", "id \"example:precedence\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n", &artifact, ); let config = installed.join("xdg/config"); let run = |arguments: &[&str]| { Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", &config) .env("XDG_DATA_HOME", installed.join("xdg/data")) .env("XDG_STATE_HOME", installed.join("xdg/state")) .env("XDG_CACHE_HOME", installed.join("xdg/cache")) .args(arguments) .output() .unwrap() }; assert!( run(&["plugin", "install", installed.to_str().unwrap()]) .status .success() ); assert!( run(&["plugin", "link", linked.to_str().unwrap()]) .status .success() ); std::fs::write(installed.join("fixture.wasm"), b"installed package changed").unwrap(); let reload = run(&["plugin", "reload"]); assert!(!reload.status.success()); assert!(String::from_utf8_lossy(&reload.stderr).contains("runtime is unavailable")); let listed = String::from_utf8(run(&["plugin", "list"]).stdout).unwrap(); assert!(listed.contains("installed example:precedence")); assert!(listed.contains("linked example:precedence")); std::fs::remove_dir_all(installed).unwrap(); std::fs::remove_dir_all(linked).unwrap(); } #[test] fn eval_loads_an_installed_plugin_into_provider_tool_rounds() { use std::net::TcpListener; let artifact = echo_artifact(); let package = package( "runtime-registry", "id \"example:runtime\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n", &artifact, ); let root = package.join("xdg"); let install = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .args(["plugin", "install"]) .arg(&package) .output() .unwrap(); assert!( install.status.success(), "{}", String::from_utf8_lossy(&install.stderr) ); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let changed_artifact = package.join("fixture.wasm"); let server = std::thread::spawn(move || { let (mut first, _) = listener.accept().unwrap(); let request = read_http_json(&mut first); assert!( request["tools"] .as_array() .unwrap() .iter() .any(|tool| tool["function"]["name"] == "example:runtime/echo") ); send_sse( &mut first, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"runtime-echo\",\"function\":{\"name\":\"example:runtime/echo\",\"arguments\":\"{\\\"value\\\":\\\"hello\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", ); std::fs::write(changed_artifact, b"installed artifact changed").unwrap(); let (mut second, _) = listener.accept().unwrap(); let request = read_http_json(&mut second); assert!( request["tools"] .as_array() .unwrap() .iter() .any(|tool| tool["function"]["name"] == "example:runtime/echo") ); let tool = request["messages"] .as_array() .unwrap() .iter() .find(|message| message["role"] == "tool") .unwrap(); assert_eq!(tool["tool_call_id"], "runtime-echo"); assert_eq!(tool["content"], "{\"value\":\"hello\"}"); send_sse( &mut second, "data: {\"choices\":[{\"delta\":{\"content\":\"installed plugin complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); }); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("OPENAI_API_KEY", "runtime-plugin-secret") .env_remove("ANTHROPIC_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("GOOGLE_API_KEY") .args([ "eval", "use installed tool", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{address}"), ]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"installed plugin complete\n"); server.join().unwrap(); std::fs::remove_dir_all(package).unwrap(); } #[test] fn eval_plugin_net_scope_records_redacted_intent_before_local_server_handles_request() { plugin_net_scope_records_redacted_intent(false); } #[test] fn admitted_plugin_http_secret_and_log_commit_nested_progress_before_outer_completion() { plugin_net_scope_records_redacted_intent(true); } fn plugin_net_scope_records_redacted_intent(explicit_submission: bool) { use std::io::{Read, Write}; use std::net::TcpListener; let package = initialized_package("http-round"); std::fs::write( package.join("plugin.kdl"), "id \"example:http\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_http_round.wasm\"\ncapability \"net-scope\"\ncapability \"secret-proxy\"\ncapability \"log\"\ncapability \"workspace\"\n", ) .unwrap(); std::fs::write( package.join("src/lib.rs"), r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")] #![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")] wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" }); use exports::smith::plugin::plugin::{Guest, ToolDeclaration}; use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance}; use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations}; use smith::plugin::log; use smith::plugin::net_scope::{self, Header, HttpRequest, RequestOptions}; use smith::plugin::secret_proxy::{self, CredentialExchange, SecretRef}; struct Plugin; impl RegistrationGuest for Plugin { fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } } } impl ProviderGuest for Plugin { fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } } fn validate_settings(_: ProviderSettings) -> Result { Err("no codec".to_string()) } fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result { Err("no codec".to_string()) } fn decode_response(_: Vec, _: Vec, _: bool) -> Result { Err("no codec".to_string()) } } impl Guest for Plugin { fn declare_tools() -> Vec { vec![ToolDeclaration { name: "round".to_string(), description: "HTTP round trip.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] } fn invoke(_: String, _: String) -> Result { let destination = std::env::var("UNAVAILABLE").unwrap_or_else(|_| "http://127.0.0.1:1".to_string()); let secret = secret_proxy::exchange(&CredentialExchange { source: SecretRef { id: "connection-api-key".to_string() }, audience: destination.clone(), scopes: vec!["http-request".to_string()] })?; log::write("info", "authorization: guest-http-secret")?; let response = net_scope::request(&HttpRequest { destination, method: "POST".to_string(), headers: vec![Header { name: "Authorization".to_string(), value: format!("{{{{secret:{}}}}}", secret.id) }], body: b"guest-body".to_vec(), options: RequestOptions { timeout_ms: 1_000, response_bytes_max: 64 } })?; Ok(format!(r#"{{"status":{}}}"#, response.status)) } } export!(Plugin); "##, ) .unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let source = std::fs::read_to_string(package.join("src/lib.rs")).unwrap(); let source = if explicit_submission { source.replace("log::write", r#"smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite { path: "nested-first.txt".into(), content: "guest-http-secret".into() })?; smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite { path: "nested-second.txt".into(), content: "guest-http-secret".into() })?; log::write"#).replace("body: b\"guest-body\".to_vec()", "body: b\"guest-http-secret\".repeat(5000)") .replace("response_bytes_max: 64", "response_bytes_max: 1048576") .replace("timeout_ms: 1_000", "timeout_ms: 30_000") } else { source }; std::fs::write( package.join("src/lib.rs"), source.replace("http://127.0.0.1:1", &format!("http://{address}")), ) .unwrap(); let root = package.join("xdg"); let run = |arguments: &[&str]| { Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .args(arguments) .output() .unwrap() }; assert!( run(&["plugin", "build", package.to_str().unwrap()]) .status .success() ); let install = run(&[ "plugin", "install", package.to_str().unwrap(), "--grant", "net-scope", "--net-origin", &format!("http://{address}"), "--grant", "secret-proxy", "--grant", "log", "--grant", "workspace", "--secret-source", "connection-api-key", "--secret-provider", "openai", "--secret-origin", &format!("http://{address}"), "--secret-scope", "http-request", ]); assert!( install.status.success(), "{}", String::from_utf8_lossy(&install.stderr) ); let data = root.join("data"); let auth = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .args(["auth", "add", "--provider", "openai"]) .write_stdin("guest-http-secret\n") .output() .unwrap(); assert!( auth.status.success(), "{}", String::from_utf8_lossy(&auth.stderr) ); let submission = if explicit_submission { assert!(run(&["eval", "--mock", "seed"]).status.success()); let listed: serde_json::Value = serde_json::from_slice(&run(&["sessions", "list", "--json"]).stdout).unwrap(); let session = listed["session"].as_str().unwrap().to_string(); let submitted = run(&["sessions", "submit", &session, "use http"]); assert!( submitted.status.success(), "{}", String::from_utf8_lossy(&submitted.stderr) ); let submitted: serde_json::Value = serde_json::from_slice(&submitted.stdout).unwrap(); Some(( session, submitted["submission"].as_str().unwrap().to_string(), )) } else { None }; let server_data = data.clone(); let server = std::thread::spawn(move || { let (mut provider, _) = listener.accept().unwrap(); let request = read_http_json(&mut provider); assert!( request["tools"] .as_array() .unwrap() .iter() .any(|tool| tool["function"]["name"] == "example:http/round") ); send_sse( &mut provider, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"http-round\",\"function\":{\"name\":\"example:http/round\",\"arguments\":\"{}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", ); let (mut guest, _) = listener.accept().unwrap(); guest .set_read_timeout(Some(std::time::Duration::from_secs(120))) .unwrap(); guest .set_write_timeout(Some(std::time::Duration::from_secs(120))) .unwrap(); let mut header = Vec::new(); while !header.ends_with(b"\r\n\r\n") { let mut byte = [0]; guest.read_exact(&mut byte).unwrap(); header.push(byte[0]); assert!(header.len() <= 16 * 1024); } let expected_body = if explicit_submission { b"guest-http-secret".repeat(5000) } else { b"guest-body".to_vec() }; let mut body = vec![0; expected_body.len()]; guest.read_exact(&mut body).unwrap(); assert_eq!(body, expected_body); let request = std::str::from_utf8(&header).unwrap(); assert!(request.contains("POST / HTTP/1.1")); assert!( request .to_ascii_lowercase() .contains("authorization: guest-http-secret"), "{request}" ); let session = std::fs::read_dir(server_data.join("smith/sessions/run")) .unwrap() .find_map(|entry| { let path = entry.unwrap().path(); (path.extension().is_some_and(|extension| extension == "smh")).then_some(path) }) .unwrap(); let mut intent = Vec::new(); let journal = SessionReader::open(&session).unwrap(); journal .scan( smith::config::Config::default_valid().frame_bytes_max, |frame| { if let Frame::Known { entry } = frame && let EntryContent::Meta { kind, detail } = EntryContent::decode(&entry.content)? && kind == smith_core::metadata::NESTED_INTENT_KIND { let detail = serde_json::to_value(detail).unwrap(); if detail["operation"] == "http" { intent.push( serde_json::from_str::( &journal .nested_payload( entry.id, smith::config::Config::default_valid().frame_bytes_max, ) .unwrap(), ) .unwrap(), ); } } Ok(()) }, ) .unwrap(); assert_eq!( intent.len(), 1, "durable HTTP intent must precede server handling" ); assert_eq!(intent[0]["headers"][0]["value"], ""); if explicit_submission { write!( guest, "HTTP/1.1 201 Created\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", expected_body.len() ) .unwrap(); guest.write_all(&expected_body).unwrap(); } else { guest .write_all( b"HTTP/1.1 201 Created\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", ) .unwrap(); } let (mut provider, _) = listener.accept().unwrap(); let request = read_http_json(&mut provider); assert!( request["messages"].as_array().unwrap().iter().any( |message| message["role"] == "tool" && message["content"] == "{\"status\":201}" ) ); send_sse( &mut provider, "data: {\"choices\":[{\"delta\":{\"content\":\"http complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); }); let mut arguments = if let Some((session, submission)) = submission { vec![ "sessions".to_string(), "execute".to_string(), session, submission, ] } else { vec!["eval".to_string(), "use http".to_string()] }; arguments.extend([ "--provider".to_string(), "openai".to_string(), "--model".to_string(), "gpt-4.1".to_string(), "--base-url".to_string(), format!("http://{address}"), ]); let output = Command::cargo_bin("smith") .unwrap() .timeout(std::time::Duration::from_secs(180)) .current_dir(&package) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("OPENAI_API_KEY", "provider-secret") .args(arguments) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); server.join().unwrap(); if explicit_submission { let receipt: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); assert_eq!(receipt["state"], "terminal"); assert_eq!(receipt["outcome"], "completed"); } else { assert_eq!(output.stdout, b"http complete\n"); } assert!(!String::from_utf8_lossy(&output.stdout).contains("guest-http-secret")); assert!(!String::from_utf8_lossy(&output.stderr).contains("guest-http-secret")); let mut nested_intents = Vec::new(); let mut nested_completions = Vec::new(); let mut outer_commits = 0; for entry in std::fs::read_dir(data.join("smith/sessions/run")).unwrap() { let path = entry.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } let bytes = std::fs::read(&path).unwrap(); assert!(!String::from_utf8_lossy(&bytes).contains("guest-http-secret")); let journal = SessionReader::open(&path).unwrap(); journal .scan( smith::config::Config::default_valid().frame_bytes_max, |frame| { if let Frame::Commit { ref entries, .. } = frame { outer_commits += 1; let EntryContent::Meta { detail, .. } = EntryContent::decode(&entries[1].content)? else { panic!("outer mutation evidence"); }; let mutation: smith_core::metadata::ToolMutationMetadata = detail.deserialize_as().unwrap(); assert_eq!( mutation.mutation_state, smith::error::MutationState::Applied ); } for entry in frame.entries() { if let EntryContent::Meta { kind, detail } = EntryContent::decode(&entry.content)? { let mut detail = serde_json::to_value(detail).unwrap(); if kind == smith_core::metadata::NESTED_INTENT_KIND || kind == smith_core::metadata::NESTED_COMPLETION_KIND { let field = if kind == smith_core::metadata::NESTED_INTENT_KIND { "input" } else { "output" }; detail[field] = serde_json::Value::String(journal.nested_payload( entry.id, smith::config::Config::default_valid().frame_bytes_max, )?); } if kind == smith_core::metadata::NESTED_INTENT_KIND { nested_intents.push(detail); } else if kind == smith_core::metadata::NESTED_COMPLETION_KIND { assert!(matches!(frame, Frame::NestedCommit { .. })); nested_completions.push(detail); } } } Ok(()) }, ) .unwrap(); } assert_eq!(outer_commits, 1); assert_eq!( nested_intents.len(), if explicit_submission { 5 } else { 3 } ); assert_eq!(nested_completions.len(), nested_intents.len()); for (index, (intent, completion)) in nested_intents.iter().zip(&nested_completions).enumerate() { assert_eq!(intent["index"], index); assert_eq!(completion["index"], index); assert_eq!(intent["call_id"], completion["call_id"]); assert_eq!(intent["submission_id"], completion["submission_id"]); assert_eq!(intent["component"]["source_id"], "example:http"); assert_eq!(completion["ok"], true); } if explicit_submission { assert_eq!( std::fs::read_to_string(package.join("nested-first.txt")).unwrap(), "guest-http-secret" ); assert_eq!( std::fs::read_to_string(package.join("nested-second.txt")).unwrap(), "guest-http-secret" ); assert_eq!( nested_intents[0]["component"]["component_sha256"] .as_str() .unwrap() .len(), 64 ); assert_eq!(nested_intents[0]["operation"], "secret-exchange"); assert_eq!( nested_completions[0]["output"], "secret-ref:connection-api-key:0" ); assert_eq!(nested_intents[1]["operation"], "write"); assert_eq!(nested_intents[2]["operation"], "write"); assert_eq!(nested_intents[3]["operation"], "log"); assert_eq!(nested_intents[4]["operation"], "http"); let request: serde_json::Value = serde_json::from_str(nested_intents[4]["input"].as_str().unwrap()).unwrap(); assert_eq!(request["body_hex"], "3c72656461637465643e".repeat(5000)); assert_eq!(request["response_bytes_max"], 1048576); let response: serde_json::Value = serde_json::from_str(nested_completions[4]["output"].as_str().unwrap()).unwrap(); assert_eq!(response["status"], 201); assert_eq!(response["body_hex"], "3c72656461637465643e".repeat(5000)); assert_eq!(response["error"], ""); return; } assert_eq!(nested_intents[0]["operation"], "secret-exchange"); assert_eq!(nested_intents[1]["operation"], "log"); assert_eq!(nested_intents[2]["operation"], "http"); assert_eq!(nested_completions[2]["ok"], true); let request: serde_json::Value = serde_json::from_str(nested_intents[2]["input"].as_str().unwrap()).unwrap(); assert_eq!(request["headers"][0]["value"], ""); assert_eq!(request["body_hex"], "67756573742d626f6479"); assert_eq!(request["response_bytes_max"], 64); let response: serde_json::Value = serde_json::from_str(nested_completions[2]["output"].as_str().unwrap()).unwrap(); assert_eq!(response["status"], 201); assert_eq!(response["body_hex"], "6f6b"); std::fs::remove_dir_all(package).unwrap(); } #[test] fn eval_plugin_secret_source_and_origin_denials_never_reach_guest_http_server() { use std::net::TcpListener; let package = TestDir::vacant("smith-plugin-test-http-secret-denial"); Command::cargo_bin("smith") .unwrap() .args(["plugin", "init"]) .arg(&package) .args(["--name", "example:http-secret-denial"]) .assert() .success(); let origin = TcpListener::bind("127.0.0.1:0").unwrap(); let destination = format!("http://{}", origin.local_addr().unwrap()); std::fs::write( package.join("plugin.kdl"), "id \"example:http-denial\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_http_secret_denial.wasm\"\ncapability \"net-scope\"\ncapability \"secret-proxy\"\ncapability \"log\"\n", ) .unwrap(); std::fs::write( package.join("src/lib.rs"), r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")] #![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")] wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" }); use exports::smith::plugin::plugin::{Guest, ToolDeclaration}; use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance}; use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations}; use smith::plugin::net_scope::{self, Header, HttpRequest, RequestOptions}; use smith::plugin::secret_proxy::{self, CredentialExchange, SecretRef}; struct Plugin; impl RegistrationGuest for Plugin { fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } } } impl ProviderGuest for Plugin { fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } } fn validate_settings(_: ProviderSettings) -> Result { Err("no codec".to_string()) } fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result { Err("no codec".to_string()) } fn decode_response(_: Vec, _: Vec, _: bool) -> Result { Err("no codec".to_string()) } } impl Guest for Plugin { fn declare_tools() -> Vec { vec![ToolDeclaration { name: "round".to_string(), description: "Secret-gated HTTP.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] } fn invoke(_: String, _: String) -> Result { let destination = "__DESTINATION__".to_string(); let secret = secret_proxy::exchange(&CredentialExchange { source: SecretRef { id: "connection-api-key".to_string() }, audience: destination.clone(), scopes: vec!["http-request".to_string()] })?; let response = net_scope::request(&HttpRequest { destination, method: "POST".to_string(), headers: vec![Header { name: "Authorization".to_string(), value: format!("{{{{secret:{}}}}}", secret.id) }], body: b"guest-body".to_vec(), options: RequestOptions { timeout_ms: 1000, response_bytes_max: 64 } })?; Ok(format!(r#"{{"status":{}}}"#, response.status)) } } export!(Plugin); "## .replace("__DESTINATION__", &destination), ) .unwrap(); let root = package.join("build-xdg"); let workspace_target = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .unwrap() .join("target"); let build = Command::cargo_bin("smith") .unwrap() .env("CARGO_TARGET_DIR", &workspace_target) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .args(["plugin", "build"]) .arg(&package) .output() .unwrap(); assert!( build.status.success(), "{}", String::from_utf8_lossy(&build.stderr) ); let artifact = workspace_target.join("wasm32-wasip2/debug/smith_plugin_example_http_secret_denial.wasm"); let package_artifact = package.join("target/wasm32-wasip2/debug/smith_plugin_example_http_secret_denial.wasm"); std::fs::create_dir_all(package_artifact.parent().unwrap()).unwrap(); std::fs::copy(artifact, package_artifact).unwrap(); for (case, source_grant) in [("missing-source", false), ("wrong-origin", true)] { let provider = TcpListener::bind("127.0.0.1:0").unwrap(); let provider_address = provider.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut initial, _) = provider.accept().unwrap(); initial .set_read_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); let request = read_http_json(&mut initial); assert!( request["tools"] .as_array() .unwrap() .iter() .any(|tool| tool["function"]["name"] == "example:http-denial/round") ); send_sse( &mut initial, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"denied-call\",\"function\":{\"name\":\"example:http-denial/round\",\"arguments\":\"{}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", ); let (mut followup, _) = provider.accept().unwrap(); followup .set_read_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); let request = read_http_json(&mut followup); assert!( request["messages"] .as_array() .unwrap() .iter() .any(|message| message["role"] == "tool") ); send_sse( &mut followup, "data: {\"choices\":[{\"delta\":{\"content\":\"denied safely\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); }); let xdg = package.join(format!("xdg-{case}")); let mut install = Command::cargo_bin("smith").unwrap(); install .env("XDG_CONFIG_HOME", xdg.join("config")) .env("XDG_DATA_HOME", xdg.join("data")) .env("XDG_STATE_HOME", xdg.join("state")) .env("XDG_CACHE_HOME", xdg.join("cache")) .args(["plugin", "install"]) .arg(&package) .args([ "--grant", "net-scope", "--net-origin", &destination, "--grant", "secret-proxy", "--grant", "log", ]); if source_grant { install.args([ "--secret-source", "connection-api-key", "--secret-provider", "openai", "--secret-origin", "http://127.0.0.1:1", "--secret-scope", "http-request", "--net-origin", "http://127.0.0.1:1", ]); } let install = install.output().unwrap(); assert!( install.status.success(), "{}", String::from_utf8_lossy(&install.stderr) ); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", xdg.join("config")) .env("XDG_DATA_HOME", xdg.join("data")) .env("XDG_STATE_HOME", xdg.join("state")) .env("XDG_CACHE_HOME", xdg.join("cache")) .env("OPENAI_API_KEY", "provider-secret") .args([ "eval", "use http", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{provider_address}"), ]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"denied safely\n"); server.join().unwrap(); origin.set_nonblocking(true).unwrap(); assert_eq!( origin.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock, "{case}: denied secret exchange must prevent origin request" ); } std::fs::remove_dir_all(package).unwrap(); } #[test] fn plugin_reload_pins_active_tool_batch_then_publishes_the_next_generation() { use std::io::Write; use std::net::TcpListener; use std::sync::mpsc; use std::time::Duration; let package = initialized_package("reload-live"); let source = package.join("src/lib.rs"); replace_fixture_output(&source, "one"); let root = package.join("xdg"); let runtime = std::path::PathBuf::from("/var/tmp/smith-session-e2e/final-gate-fixes") .join(format!("reload-runtime-{}", std::process::id())); let _ = std::fs::remove_dir_all(&runtime); std::fs::create_dir_all(&runtime).unwrap(); #[cfg(unix)] std::fs::set_permissions( &runtime, std::os::unix::fs::PermissionsExt::from_mode(0o700), ) .unwrap(); let run = |arguments: &[&str]| { Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("XDG_RUNTIME_DIR", &runtime) .args(arguments) .output() .unwrap() }; assert!( run(&["plugin", "build", package.to_str().unwrap()]) .status .success() ); assert!( run(&["plugin", "link", package.to_str().unwrap()]) .status .success() ); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let (first_request_sent, first_request_received) = mpsc::sync_channel(1); let (first_release_sent, first_release_received) = mpsc::sync_channel(1); let (server_stage_sent, server_stage_received) = mpsc::sync_channel(2); let server = std::thread::spawn(move || { let (mut first, _) = listener.accept().unwrap(); let request = read_http_json(&mut first); assert!( request["tools"] .as_array() .unwrap() .iter() .any(|tool| tool["function"]["name"] == "example:reload-live/fixture") ); let initial = "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"one\",\"function\":{\"name\":\"example:reload-live/fixture\",\"arguments\":\"{}\"}}]}}]}\n\n"; let finished = "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n"; write!( first, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{initial}", initial.len() + finished.len() ) .unwrap(); first.flush().unwrap(); first_request_sent.send(()).unwrap(); first_release_received .recv_timeout(Duration::from_secs(30)) .unwrap(); first.write_all(finished.as_bytes()).unwrap(); let (mut second, _) = listener.accept().unwrap(); let request = read_http_json(&mut second); server_stage_sent.send("second").unwrap(); assert!( request["messages"] .as_array() .unwrap() .iter() .any(|message| message["role"] == "tool" && message["content"] == "one") ); send_sse( &mut second, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"two\",\"function\":{\"name\":\"example:reload-live/fixture\",\"arguments\":\"{}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", ); let (mut third, _) = listener.accept().unwrap(); let request = read_http_json(&mut third); server_stage_sent.send("third").unwrap(); assert!( request["messages"] .as_array() .unwrap() .iter() .any(|message| message["role"] == "tool" && message["content"] == "two") ); send_sse( &mut third, "data: {\"choices\":[{\"delta\":{\"content\":\"reloaded\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); }); let mut child = TestChild( std::process::Command::new(env!("CARGO_BIN_EXE_smith")) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("XDG_RUNTIME_DIR", &runtime) .env("OPENAI_API_KEY", "runtime-plugin-secret") .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .args([ "eval", "reload", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{address}"), ]) .spawn() .unwrap(), ); if let Err(error) = first_request_received.recv_timeout(Duration::from_secs(30)) { let output = child.kill_with_output(); panic!( "waiting for first provider request: {error}; status={}; stderr={}", output.status, String::from_utf8_lossy(&output.stderr) ); } let artifact = package.join("target/wasm32-wasip2/debug/smith_plugin_example_reload_live.wasm"); std::fs::write(&artifact, b"broken").unwrap(); let failed = run(&["plugin", "reload"]); assert!(!failed.status.success()); replace_fixture_output(&source, "two"); assert!( run(&["plugin", "build", package.to_str().unwrap()]) .status .success() ); let published = run(&["plugin", "reload"]); assert!( published.status.success(), "{}", String::from_utf8_lossy(&published.stderr) ); first_release_sent.send(()).unwrap(); assert_eq!( server_stage_received .recv_timeout(Duration::from_secs(30)) .unwrap(), "second" ); assert_eq!( server_stage_received .recv_timeout(Duration::from_secs(30)) .unwrap(), "third" ); let output = child.wait_with_output(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"reloaded\n"); server.join().unwrap(); std::fs::remove_dir_all(&runtime).unwrap(); std::fs::remove_dir_all(package).unwrap(); } fn replace_fixture_output(source: &Path, output: &str) { let original = std::fs::read_to_string(source).unwrap(); let result = format!("Ok(\"{output}\".to_string())"); let fixture = if original.contains("Ok(input_json)") { original.replacen("Ok(input_json)", &result, 1) } else { original.replacen("Ok(\"one\".to_string())", &result, 1) }; assert_ne!(fixture, original, "generated fixture shape changed"); std::fs::write(source, fixture).unwrap(); } #[test] fn plugin_check_rejects_process_capability_when_only_filesystem_is_granted() { let package = fixture_package( "process-without-grant", "id \"example:process\"\nversion \"1\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"process\"", ); let output = smith(&package, &["workspace"]); assert!(!output.status.success()); let stderr = String::from_utf8(output.stderr).unwrap(); assert!(stderr.contains("missing grant process"), "{stderr}"); std::fs::remove_dir_all(package).unwrap(); } #[test] fn plugin_check_discloses_active_process_authority() { let package = package( "process-with-grant", "id \"example:process\"\nversion \"1\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"process\"", &echo_artifact(), ); let output = smith(&package, &["process"]); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!( output.stdout, b"example:process: check passed\nprocess grant active: commands inherit host-user filesystem and network access\n" ); std::fs::remove_dir_all(package).unwrap(); } const CATALOG_LISTING_FIXTURE: &str = r#"{ "openai": {"models": {"shared": {"id": "shared", "attachment": false, "reasoning": false, "tool_call": true, "structured_output": false, "limit": {"context": 8, "output": 4}}}}, "zai": {"models": {"shared": {"id": "shared", "attachment": false, "reasoning": false, "tool_call": true, "structured_output": false, "limit": {"context": 16, "output": 8}}}}, "unsupported": {"models": {"hidden": {"id": "hidden", "attachment": false, "reasoning": false, "tool_call": true, "structured_output": false, "limit": {"context": 8, "output": 4}}}} }"#; fn catalog_cache(source: &str) -> String { serde_json::json!({"revision": 7, "source": source}).to_string() } #[test] fn catalog_show_lists_connection_models_without_colliding_provider_ids() { let root = TestDir::new("smith-catalog-listing-e2e"); let cache = root.join("catalog.json"); std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap(); let output = Command::cargo_bin("smith") .unwrap() .args(["catalog", "show", "--cache"]) .arg(&cache) .output() .unwrap(); assert!(output.status.success(), "{output:?}"); insta::assert_snapshot!(String::from_utf8(output.stdout).unwrap()); } #[test] fn catalog_show_json_filters_zai_coding_plan_connection() { let root = TestDir::new("smith-catalog-json-listing-e2e"); let cache = root.join("catalog.json"); std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap(); let output = Command::cargo_bin("smith") .unwrap() .args([ "catalog", "show", "--cache", cache.to_str().unwrap(), "--provider", "zai-coding-plan", "--json", ]) .output() .unwrap(); assert!(output.status.success(), "{output:?}"); let listing: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); insta::assert_snapshot!(serde_json::to_string_pretty(&listing).unwrap()); } #[test] fn catalog_json_provider_names_select_the_reported_connection() { let root = TestDir::new("smith-catalog-selectable-provider-e2e"); let cache = root.join("catalog.json"); std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap(); let output = Command::cargo_bin("smith") .unwrap() .args(["catalog", "show", "--cache"]) .arg(&cache) .arg("--json") .output() .unwrap(); assert!(output.status.success(), "{output:?}"); let listing: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); for provider in listing["providers"].as_array().unwrap() { let name = provider["provider"].as_str().unwrap(); let connection = provider["connection"].as_str().unwrap(); let selected = Command::cargo_bin("smith") .unwrap() .args(["catalog", "show", "--cache"]) .arg(&cache) .args(["--provider", name, "--json"]) .output() .unwrap(); assert!(selected.status.success(), "{name}: {selected:?}"); let selected: serde_json::Value = serde_json::from_slice(&selected.stdout).unwrap(); assert_eq!(selected["providers"].as_array().unwrap().len(), 1, "{name}"); assert_eq!(selected["providers"][0]["provider"], name, "{name}"); assert_eq!(selected["providers"][0]["connection"], connection, "{name}"); assert_eq!( selected["providers"][0]["models"], provider["models"], "{name}" ); } } #[test] fn catalog_show_json_is_byte_identical_for_repeated_offline_listings() { let root = TestDir::new("smith-catalog-deterministic-json-e2e"); let cache = root.join("catalog.json"); std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap(); let run = || { Command::cargo_bin("smith") .unwrap() .args(["catalog", "show", "--cache"]) .arg(&cache) .arg("--json") .output() .unwrap() }; let first = run(); let second = run(); assert!(first.status.success(), "{first:?}"); assert!(second.status.success(), "{second:?}"); assert_eq!(first.stdout, second.stdout); } #[test] fn catalog_show_escapes_model_control_characters_for_humans() { let root = TestDir::new("smith-catalog-control-e2e"); let cache = root.join("catalog.json"); let source = r#"{ "zai": {"models": {"unsafe": { "id": "\u001b[31munsafe", "limit": {"context": 8, "output": 4} }}} }"#; std::fs::write(&cache, catalog_cache(source)).unwrap(); let output = Command::cargo_bin("smith") .unwrap() .args(["catalog", "show", "--cache"]) .arg(&cache) .args(["--provider", "zai"]) .output() .unwrap(); assert!(output.status.success()); let stdout = String::from_utf8(output.stdout).unwrap(); assert!(stdout.contains(r#"\u{1b}[31munsafe"#)); assert!(!stdout.contains('\u{1b}')); } #[test] fn catalog_show_distinguishes_an_empty_valid_connection_from_an_invalid_filter() { let root = TestDir::new("smith-catalog-empty-filter-e2e"); let cache = root.join("catalog.json"); std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap(); let empty = Command::cargo_bin("smith") .unwrap() .args(["catalog", "show", "--cache"]) .arg(&cache) .args(["--provider", "anthropic", "--json"]) .output() .unwrap(); assert!(empty.status.success()); let listing: serde_json::Value = serde_json::from_slice(&empty.stdout).unwrap(); assert_eq!(listing["providers"][0]["connection"], "anthropic-api"); assert_eq!(listing["providers"][0]["models"], serde_json::json!([])); let invalid = Command::cargo_bin("smith") .unwrap() .args(["catalog", "show", "--provider", "not-a-provider"]) .output() .unwrap(); assert_eq!(invalid.status.code(), Some(2)); assert!( String::from_utf8(invalid.stderr) .unwrap() .contains("possible values"), ); } #[test] fn catalog_status_cli_cache_overrides_environment_cache() { let root = TestDir::new("smith-catalog-cache-precedence-e2e"); let environment_cache = root.join("environment.json"); let cli_cache = root.join("cli.json"); std::fs::write(&environment_cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap(); std::fs::write( &cli_cache, catalog_cache(CATALOG_LISTING_FIXTURE).replace("\"revision\":7", "\"revision\":8"), ) .unwrap(); let output = Command::cargo_bin("smith") .unwrap() .env("SMITH_CATALOG_CACHE", &environment_cache) .args(["catalog", "status", "--cache"]) .arg(&cli_cache) .output() .unwrap(); assert!(output.status.success()); assert!( String::from_utf8(output.stdout) .unwrap() .contains("generation=8") ); } #[test] fn catalog_show_and_status_do_not_resolve_credentials_or_proxy_offline() { use std::net::TcpListener; let root = TestDir::new("smith-catalog-offline-e2e"); let data = root.join("data"); std::fs::create_dir_all(data.join("smith")).unwrap(); std::fs::write(data.join("smith/auth"), "poisoned auth path").unwrap(); let proxy = TcpListener::bind("127.0.0.1:0").unwrap(); proxy.set_nonblocking(true).unwrap(); let proxy_url = format!("http://{}", proxy.local_addr().unwrap()); let run = |arguments: &[&str]| { Command::cargo_bin("smith") .unwrap() .env("HOME", root.join("home")) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("HTTP_PROXY", &proxy_url) .env("HTTPS_PROXY", &proxy_url) .env_remove("SMITH_CATALOG_CACHE") .env_remove("OPENAI_API_KEY") .env_remove("ANTHROPIC_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("ZAI_API_KEY") .args(arguments) .output() .unwrap() }; let show = run(&["catalog", "show", "--provider", "zai-coding-plan", "--json"]); let status = run(&["catalog", "status"]); assert!(show.status.success(), "{show:?}"); assert!(status.status.success(), "{status:?}"); assert!( String::from_utf8(show.stdout) .unwrap() .contains("zai-coding-plan"), "show must use only offline metadata" ); assert!( String::from_utf8(status.stdout) .unwrap() .starts_with("catalog: bundled; generation=0;"), "status must use the bundled offline catalog" ); assert_eq!( proxy.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock, "catalog inspection must not make proxy requests" ); } #[test] fn catalog_show_uses_bundled_snapshot_offline() { let status = std::process::Command::new(env!("CARGO")) .args(["build", "--locked", "-p", "smith-cli", "--bin", "smith"]) .status() .unwrap(); assert!(status.success(), "smith binary build failed"); let root = TestDir::new("smith-catalog-e2e"); let binary = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .unwrap() .join("target/debug") .join(if cfg!(windows) { "smith.exe" } else { "smith" }); assert!( binary.is_file(), "missing built smith binary: {}", binary.display() ); let binary = binary.canonicalize().unwrap(); let mut command = Command::new(binary); command .env("XDG_CACHE_HOME", root.join("cache")) .env_remove("SMITH_CATALOG_CACHE") .args(["catalog", "status"]); let output = command.output().unwrap(); assert!(output.status.success()); let stdout = String::from_utf8(output.stdout).unwrap(); assert!(stdout.starts_with("catalog: bundled; generation=0; models=")); let _ = std::fs::remove_dir_all(root); } #[test] fn catalog_status_reports_bundled_for_an_invalid_cache_file() { let root = TestDir::new("smith-catalog-invalid-e2e"); let cache = root.join("catalog.json"); std::fs::create_dir_all(&root).unwrap(); std::fs::write(&cache, "not a catalog").unwrap(); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CACHE_HOME", root.join("cache")) .args(["catalog", "status", "--cache"]) .arg(&cache) .output() .unwrap(); assert!(output.status.success()); assert!( String::from_utf8(output.stdout) .unwrap() .starts_with("catalog: bundled; generation=0;") ); std::fs::remove_dir_all(root).unwrap(); } #[test] fn auth_zai_coding_plan_stores_and_checks_its_own_authority() { let root = TestDir::new("smith-zai-auth"); let home = root.join("data"); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &home) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env_remove("ZAI_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .args(["auth", "add", "--provider", "zai-coding-plan"]) .write_stdin("plan-secret\n") .output() .unwrap(); assert!(output.status.success()); assert_eq!(output.stdout, b"zai-coding-plan: stored\n"); assert!(home.join("smith/auth/zai-coding-plan").is_file()); assert!(!home.join("smith/auth/plugin").exists()); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &home) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env_remove("ZAI_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .args(["auth", "check", "--provider", "zai-coding-plan"]) .output() .unwrap(); assert!(output.status.success()); assert_eq!(output.stdout, b"zai-coding-plan: store\n"); std::fs::remove_dir_all(root).unwrap(); } #[test] fn eval_zai_component_uses_official_environment_credential_redacts_it_and_records_effects() { use std::io::{Read, Write}; use std::net::TcpListener; let root = TestDir::new("smith-zai-e2e"); let data = root.join("data"); let secret = "zai-e2e-secret"; std::fs::create_dir_all(&data).unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let mut request = [0_u8; 4096]; let bytes = stream.read(&mut request).unwrap(); let request = std::str::from_utf8(&request[..bytes]).unwrap(); assert!( request .to_ascii_lowercase() .contains("authorization: bearer zai-e2e-secret") ); assert!(request.contains("POST /api/paas/v4/chat/completions")); let body = b"data: {\"choices\":[{\"delta\":{\"content\":\"guest accepted\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"; stream .write_all( format!( "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len() ) .as_bytes(), ) .unwrap(); stream.write_all(body).unwrap(); }); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("ZAI_API_KEY", secret) .env_remove("ANTHROPIC_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("OPENAI_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .env_remove("SMITH_ZAI_PLUGIN") .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args([ "eval", "hello", "--provider", "zai", "--model", "glm-5.3", "--base-url", &format!("http://{address}"), ]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); server.join().unwrap(); assert_eq!(output.stdout, b"guest accepted\n"); assert!(!String::from_utf8_lossy(&output.stderr).contains(secret)); let sessions = std::fs::read_dir(data.join("smith/sessions/run")).unwrap(); let mut provenances = Vec::new(); for entry in sessions { let path = entry.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } let bytes = std::fs::read(&path).unwrap(); assert!(!String::from_utf8_lossy(&bytes).contains(secret)); assert!(String::from_utf8_lossy(&bytes).contains("")); SessionReader::open(&path) .unwrap() .scan( smith::config::Config::default_valid().frame_bytes_max, |frame| { if let Frame::Known { entry } = frame && let EntryContent::Meta { kind, detail } = EntryContent::decode(&entry.content)? && kind == PROVIDER_DISPATCH_PROVENANCE_KIND { provenances.push(serde_json::to_value(detail).unwrap()); } Ok(()) }, ) .unwrap(); } assert_eq!( provenances, vec![serde_json::json!({ "connection_id": "zai-api", "protocol_manifest_id": "smith:protocol-openai-compatible", "protocol_component_sha256": smith_harness::protocol::embedded_openai_compatible_component_sha256(), "catalog_generation": 0, "catalog_source": "bundled", "settings": {"include_usage": true, "provenance": "built_in"}, })] ); std::fs::remove_dir_all(root).unwrap(); } #[test] fn eval_vendor_protocol_components_use_loopback_credentials_and_redact_durable_effects() { use std::io::{Read, Write}; use std::net::TcpListener; let cases = [ ( "anthropic", "claude-sonnet-4-5", "ANTHROPIC_API_KEY", "anthropic-e2e-secret", "/v1/messages", "x-api-key: anthropic-e2e-secret", b"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"anthropic guest\"}}\n\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\ndata: {\"type\":\"message_stop\"}\n\n".as_slice(), "anthropic guest\n", ), ( "google", "gemini-flash-latest", "GOOGLE_API_KEY", "google-e2e-secret", "/v1beta/models/gemini-flash-latest:streamgeneratecontent?alt=sse", "x-goog-api-key: google-e2e-secret", b"data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"google guest\"}]},\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":1,\"candidatesTokenCount\":1}}\n\n".as_slice(), "google guest\n", ), ]; for ( provider, model, credential_name, secret, request_path, credential_header, body, expected, ) in cases { let root = TestDir::new(&format!("smith-{provider}-component-e2e")); let data = root.join("data"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let mut request = [0_u8; 4096]; let bytes = stream.read(&mut request).unwrap(); let request = std::str::from_utf8(&request[..bytes]) .unwrap() .to_ascii_lowercase(); assert!(request.contains(&format!("post {request_path}"))); assert!(request.contains(credential_header)); stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes()).unwrap(); stream.write_all(body).unwrap(); }); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env(credential_name, secret) .env_remove("ANTHROPIC_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("OPENAI_API_KEY") .args([ "eval", "hello", "--provider", provider, "--model", model, "--base-url", &format!("http://{address}"), ]) .env(credential_name, secret) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); server.join().unwrap(); assert_eq!(output.stdout, expected.as_bytes()); let stderr = String::from_utf8_lossy(&output.stderr); assert!(!stderr.contains(secret)); let mut effect_recorded = false; let mut provenances = Vec::new(); for entry in std::fs::read_dir(data.join("smith/sessions/run")).unwrap() { let path = entry.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } let bytes = std::fs::read(&path).unwrap(); let session = String::from_utf8_lossy(&bytes); assert!(!session.contains(secret)); effect_recorded |= session.contains(""); SessionReader::open(&path) .unwrap() .scan( smith::config::Config::default_valid().frame_bytes_max, |frame| { if let Frame::Known { entry } = frame && let EntryContent::Meta { kind, detail } = EntryContent::decode(&entry.content)? && kind == PROVIDER_DISPATCH_PROVENANCE_KIND { provenances.push(serde_json::to_value(detail).unwrap()); } Ok(()) }, ) .unwrap(); } assert!( effect_recorded, "{provider} intent must be durably redacted" ); let (connection_id, protocol_manifest_id, protocol_component_sha256) = match provider { "anthropic" => ( "anthropic-api", "smith:protocol-anthropic", smith_harness::protocol::embedded_anthropic_component_sha256(), ), "google" => ( "google-api", "smith:protocol-google", smith_harness::protocol::embedded_google_component_sha256(), ), _ => unreachable!("unknown test provider {provider}"), }; assert_eq!( provenances, vec![serde_json::json!({ "connection_id": connection_id, "protocol_manifest_id": protocol_manifest_id, "protocol_component_sha256": protocol_component_sha256, "catalog_generation": 0, "catalog_source": "bundled", "settings": {"include_usage": true, "provenance": "built_in"}, })], "{provider} must retain the selected protocol and pinned catalog" ); std::fs::remove_dir_all(root).unwrap(); } } const E2E_SOCKET_DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); const E2E_REQUEST_BYTES_MAX: usize = 1024 * 1024; fn read_http_json(stream: &mut std::net::TcpStream) -> serde_json::Value { use std::io::Read; stream.set_read_timeout(Some(E2E_SOCKET_DEADLINE)).unwrap(); let mut request = Vec::new(); let mut buffer = [0_u8; 4096]; let header_end = loop { let count = stream.read(&mut buffer).unwrap(); assert!(count > 0 && request.len() + count <= E2E_REQUEST_BYTES_MAX); request.extend_from_slice(&buffer[..count]); if let Some(position) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") { break position + 4; } }; let headers = std::str::from_utf8(&request[..header_end]).unwrap(); let content_length = headers .lines() .find_map(|line| { line.to_ascii_lowercase() .strip_prefix("content-length: ") .and_then(|value| value.trim().parse::().ok()) }) .unwrap(); assert!(header_end + content_length <= E2E_REQUEST_BYTES_MAX); while request.len() < header_end + content_length { let count = stream.read(&mut buffer).unwrap(); assert!(count > 0 && request.len() + count <= E2E_REQUEST_BYTES_MAX); request.extend_from_slice(&buffer[..count]); } serde_json::from_slice(&request[header_end..header_end + content_length]).unwrap() } fn send_sse(stream: &mut std::net::TcpStream, body: &str) { use std::io::Write; write!( stream, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len() ) .unwrap(); } fn eval_vendor_tool_round(provider: &'static str, model: &'static str, credential: &'static str) { use std::net::TcpListener; let root = TestDir::new(&format!("smith-{provider}-tool-round-e2e")); let data = root.join("data"); let workdir = root.join("workspace"); std::fs::create_dir_all(&workdir).unwrap(); std::fs::write(workdir.join("read-fixture.txt"), "fixture contents\n").unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut first, _) = listener.accept().unwrap(); let first_request = read_http_json(&mut first); let tools = match provider { "openai" => &first_request["tools"], "anthropic" => &first_request["tools"], "google" => &first_request["tools"][0]["functionDeclarations"], _ => unreachable!(), }; assert!( tools .as_array() .unwrap() .iter() .any(|tool| tool["name"] == "read" || tool["function"]["name"] == "read") ); let first_response = match provider { "openai" => { "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"openai-call-id\",\"function\":{\"name\":\"read\",\"arguments\":\"{\\\"path\\\":\\\"read-fixture.txt\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n" } "anthropic" => { "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"anthropic-call-id\",\"name\":\"read\"}}\n\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\":\\\"read-fixture.txt\\\"}\"}}\n\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n" } "google" => { "data: {\"candidates\":[{\"content\":{\"parts\":[{\"functionCall\":{\"id\":\"google-call-id\",\"name\":\"read\",\"args\":{\"path\":\"read-fixture.txt\"}}}]},\"finishReason\":\"STOP\"}]}\n\n" } _ => unreachable!(), }; send_sse(&mut first, first_response); let (mut second, _) = listener.accept().unwrap(); let second_request = read_http_json(&mut second); match provider { "openai" => { let messages = second_request["messages"].as_array().unwrap(); let assistant = messages .iter() .find(|message| message["tool_calls"].is_array()) .unwrap(); let tool = messages .iter() .find(|message| message["role"] == "tool") .unwrap(); assert_eq!(assistant["tool_calls"][0]["id"], "openai-call-id"); assert_eq!(tool["tool_call_id"], "openai-call-id"); } "anthropic" => { let messages = second_request["messages"].as_array().unwrap(); let assistant = messages .iter() .find(|message| message["role"] == "assistant") .unwrap(); let tool = messages .iter() .find(|message| { message["role"] == "user" && message["content"][0]["tool_use_id"].is_string() }) .unwrap(); assert_eq!(assistant["content"][0]["id"], "anthropic-call-id"); assert_eq!(tool["content"][0]["tool_use_id"], "anthropic-call-id"); } "google" => { let contents = second_request["contents"].as_array().unwrap(); let tool = contents .iter() .find(|content| content["parts"][0]["functionResponse"].is_object()) .unwrap(); assert_eq!(tool["parts"][0]["functionResponse"]["name"], "read"); } _ => unreachable!(), } let final_response = match provider { "openai" => { "data: {\"choices\":[{\"delta\":{\"content\":\"openai tool complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n" } "anthropic" => { "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"anthropic tool complete\"}}\n\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n" } "google" => { "data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"google tool complete\"}]},\"finishReason\":\"STOP\"}]}\n\n" } _ => unreachable!(), }; send_sse(&mut second, final_response); }); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .current_dir(&workdir) .env(credential, format!("{provider}-tool-round-secret")) .env_remove("ANTHROPIC_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("OPENAI_API_KEY") .env(credential, format!("{provider}-tool-round-secret")) .args([ "eval", "read fixture", "--provider", provider, "--model", model, "--base-url", &format!("http://{address}"), ]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); server.join().unwrap(); assert_eq!( output.stdout, format!("{provider} tool complete\n").as_bytes() ); let secret = format!("{provider}-tool-round-secret"); for entry in std::fs::read_dir(data.join("smith/sessions/run")).unwrap() { let path = entry.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } let bytes = std::fs::read(path).unwrap(); let record = String::from_utf8_lossy(&bytes); assert!(!record.contains(&secret)); assert!(record.contains("")); } std::fs::remove_dir_all(root).unwrap(); } #[test] fn eval_openai_component_preserves_vendor_tool_call_id_across_read_round() { eval_vendor_tool_round("openai", "gpt-4.1", "OPENAI_API_KEY"); } #[test] fn eval_anthropic_component_preserves_vendor_tool_call_id_across_read_round() { eval_vendor_tool_round("anthropic", "claude-sonnet-4-5", "ANTHROPIC_API_KEY"); } #[test] fn eval_google_component_matches_function_response_name_across_read_round() { eval_vendor_tool_round("google", "gemini-flash-latest", "GOOGLE_API_KEY"); } #[test] fn eval_openai_resume_preserves_two_durable_vendor_tool_call_ids() { use std::net::TcpListener; let root = TestDir::new("smith-openai-resume-tool-ids-e2e"); let data = root.join("data"); let workdir = root.join("workspace"); std::fs::create_dir_all(&workdir).unwrap(); std::fs::write(workdir.join("fixture.txt"), "fixture\n").unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let tool_call = |id: &str| { format!( "data: {{\"choices\":[{{\"delta\":{{\"tool_calls\":[{{\"index\":0,\"id\":\"{id}\",\"function\":{{\"name\":\"read\",\"arguments\":\"{{\\\"path\\\":\\\"fixture.txt\\\"}}\"}}}}]}}}}]}}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n" ) }; let final_response = "data: {\"choices\":[{\"delta\":{\"content\":\"done\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"; for id in ["vendor-first", "vendor-second"] { let (mut socket, _) = listener.accept().unwrap(); let _request = read_http_json(&mut socket); send_sse(&mut socket, &tool_call(id)); } for expected in ["vendor-first", "vendor-second"] { let (mut socket, _) = listener.accept().unwrap(); let request = read_http_json(&mut socket); let messages = request["messages"].as_array().unwrap(); for id in ["vendor-first", "vendor-second"] { assert!( messages .iter() .any(|message| message["tool_calls"][0]["id"] == id), "missing assistant {id}: {messages:?}" ); assert!( messages.iter().any(|message| message["tool_call_id"] == id), "missing result {id}: {messages:?}" ); } send_sse(&mut socket, final_response); if expected == "vendor-first" { continue; } } }); let run = |session: Option<&str>, prompt: &str| { let mut command = Command::cargo_bin("smith").unwrap(); command .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .current_dir(&workdir) .env("OPENAI_API_KEY", "openai-resume-secret") .env_remove("ANTHROPIC_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("GOOGLE_API_KEY") .args([ "eval", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{address}"), ]); if let Some(session) = session { command.args(["--session", session]); } command.arg(prompt).output().unwrap() }; let first = run(None, "read twice"); assert!( first.status.success(), "{}", String::from_utf8_lossy(&first.stderr) ); assert_eq!(first.stdout, b"done\n"); let session = std::fs::read_dir(data.join("smith/sessions/run")) .unwrap() .find_map(|entry| { let path = entry.unwrap().path(); (path.extension().is_some_and(|extension| extension == "smh")) .then(|| path.file_stem().unwrap().to_string_lossy().into_owned()) }) .unwrap(); let resumed = run(Some(&session), "continue"); assert!( resumed.status.success(), "{}", String::from_utf8_lossy(&resumed.stderr) ); assert_eq!(resumed.stdout, b"done\n"); server.join().unwrap(); std::fs::remove_dir_all(root).unwrap(); } #[test] fn eval_zai_rejects_another_vendor_environment_credential() { let root = TestDir::new("smith-zai-wrong-env-e2e"); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("OPENAI_API_KEY", "wrong-vendor-secret") .env_remove("ANTHROPIC_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .env_remove("SMITH_ZAI_PLUGIN") .env_remove("ZAI_API_KEY") .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args(["eval", "hello", "--provider", "zai", "--model", "glm-5.3"]) .output() .unwrap(); let stderr = String::from_utf8_lossy(&output.stderr); assert!(!output.status.success()); assert!(stderr.contains("no credential for zai")); assert!(stderr.contains("env ZAI_API_KEY")); assert!(!stderr.contains("SMITH_PLUGIN_API_KEY")); assert!(!stderr.contains("no credential for plugin")); assert!(!stderr.contains("wrong-vendor-secret")); std::fs::remove_dir_all(root).unwrap(); } #[test] fn eval_zai_coding_plan_uses_its_selected_route_with_official_credential() { use std::io::{Read, Write}; use std::net::TcpListener; let root = TestDir::new("smith-zai-coding-plan"); let secret = "zai-coding-plan-secret"; let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let mut request = [0_u8; 4096]; let bytes = stream.read(&mut request).unwrap(); let request = std::str::from_utf8(&request[..bytes]).unwrap(); assert!(request.contains("POST /api/coding/paas/v4/chat/completions")); assert!( request .to_ascii_lowercase() .contains("authorization: bearer zai-coding-plan-secret") ); let body = b"data: {\"choices\":[{\"delta\":{\"content\":\"coding plan\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"; stream .write_all( format!( "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len() ) .as_bytes(), ) .unwrap(); stream.write_all(body).unwrap(); }); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("ZAI_API_KEY", secret) .env_remove("SMITH_PLUGIN_API_KEY") .env_remove("SMITH_ZAI_PLUGIN") .args([ "eval", "hello", "--provider", "zai-coding-plan", "--model", "glm-5.3", "--base-url", &format!("http://{address}"), ]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); server.join().unwrap(); assert_eq!(output.stdout, b"coding plan\n"); assert!(!String::from_utf8_lossy(&output.stderr).contains(secret)); std::fs::remove_dir_all(root).unwrap(); } #[test] fn eval_zai_bad_explicit_component_override_fails_without_embedded_fallback() { let root = TestDir::new("smith-zai-override-e2e"); let data = root.join("data"); std::fs::create_dir_all(data.join("smith/auth")).unwrap(); std::fs::write(data.join("smith/auth/zai"), "test-credential").unwrap(); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", &data) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("SMITH_ZAI_PLUGIN", root.join("missing.wasm")) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args(["eval", "hello", "--provider", "zai", "--model", "glm-5.3"]) .output() .unwrap(); assert!(!output.status.success()); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("validation failed"), "{stderr}"); std::fs::remove_dir_all(root).unwrap(); } #[test] fn plugin_check_reports_trust_grant_world_and_artifact_without_invoking_tools() { let manifest = "id \"example:fixture\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n"; let untrusted = fixture_package("untrusted", manifest); let output = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", untrusted.join("xdg/config")) .args(["plugin", "check"]) .arg(&untrusted) .output() .unwrap(); assert_eq!(output.status.code(), Some(1)); assert!( String::from_utf8_lossy(&output.stderr).contains("trust the project"), "{}", String::from_utf8_lossy(&output.stderr) ); let grant = fixture_package( "grant", "id \"example:fixture\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"log\"\n", ); let output = smith(&grant, &[]); assert_eq!(output.status.code(), Some(1)); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("missing grant log"), "{stderr}"); let world = fixture_package( "world", "id \"example:fixture\"\nversion \"0.1.0\"\nworld \"9.0.0\"\nartifact \"fixture.wasm\"\n", ); let output = smith(&world, &[]); assert_eq!(output.status.code(), Some(1)); assert!(String::from_utf8_lossy(&output.stderr).contains("unsupported world version 9.0.0")); let artifact = fixture_package("artifact", manifest); let output = smith(&artifact, &["log"]); assert_eq!(output.status.code(), Some(1)); assert!(String::from_utf8_lossy(&output.stderr).contains("validation failed")); for package in [untrusted, grant, world, artifact] { std::fs::remove_dir_all(package).unwrap(); } } #[test] fn plugin_trust_persists_grants_for_a_later_check() { let artifact = echo_artifact(); let package = package( "persisted-trust", "id \"example:echo\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"log\"\ncapability \"net-scope\"\n", &artifact, ); let config = package.join("xdg/config"); let trust = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", &config) .args(["plugin", "trust"]) .arg(&package) .args([ "--grant", "log", "--grant", "net-scope", "--net-origin", "http://127.0.0.1:17777", ]) .output() .unwrap(); assert!( trust.status.success(), "{}", String::from_utf8_lossy(&trust.stderr) ); let check = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", &config) .args(["plugin", "check"]) .arg(&package) .output() .unwrap(); assert!( check.status.success(), "{}", String::from_utf8_lossy(&check.stderr) ); assert_eq!(check.stdout, b"example:echo: check passed\n"); let trust = std::fs::read_to_string(config.join("smith/plugins/project-trust.kdl")).unwrap(); assert!(trust.contains("net-origin \"http://127.0.0.1:17777\"")); let invalid = Command::cargo_bin("smith") .unwrap() .env("XDG_CONFIG_HOME", &config) .args(["plugin", "trust"]) .arg(&package) .args([ "--grant", "net-scope", "--net-origin", "https://example.test/path", ]) .output() .unwrap(); assert!(!invalid.status.success()); assert!(String::from_utf8_lossy(&invalid.stderr).contains("without path")); std::fs::remove_dir_all(package).unwrap(); } #[test] fn plugin_check_loads_a_valid_component_without_invoking_a_declared_tool() { let artifact = echo_artifact(); let package = package( "valid", "id \"example:echo\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n", &artifact, ); let output = smith(&package, &[]); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"example:echo: check passed\n"); std::fs::remove_dir_all(package).unwrap(); } #[test] fn eval_openai_uses_trusted_protocol_dev_link_and_rejects_broken_replacement() { let package = initialized_package("protocol-dev-link"); let root = TestDir::new("smith-protocol-dev-link"); let run = |args: &[&str]| { Command::cargo_bin("smith") .unwrap() .timeout(std::time::Duration::from_secs(180)) .current_dir(&root) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("OPENAI_API_KEY", "dev-link-secret") .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args(args) .output() .unwrap() }; let source = std::fs::read_to_string(package.join("src/lib.rs")).unwrap(); let start = source.find("impl ProviderGuest for Plugin {").unwrap(); let end = source.find("impl Guest for Plugin {").unwrap(); let provider = r#"impl ProviderGuest for Plugin { fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: true, subscription: false, provenance: SettingsProvenance::BuiltIn } } fn validate_settings(settings: ProviderSettings) -> Result { Ok(settings) } fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result { Ok(RequestTemplate { url: "https://protocol.invalid/chat/completions".into(), authorization: "Bearer {{secret:connection-api-key}}".into(), body: "{}".into() }) } fn decode_response(_: Vec, _: Vec, finished: bool) -> Result { use exports::smith::plugin::provider::{StreamEvent, StopReason}; Ok(DecodeStep { state: Vec::new(), events: if finished { vec![StreamEvent::Text("linked codec".into()), StreamEvent::Stop((StopReason::EndTurn, None))] } else { Vec::new() } }) } } "#; let source = format!("{}{}{}", &source[..start], provider, &source[end..]) .replace("providers: Vec::new()", "providers: vec![exports::smith::plugin::registration::ProviderContribution { id: \"openai-compatible\".into(), protocol: exports::smith::plugin::registration::ProtocolBinding::OpenaiCompatible, credential_source_id: \"connection-api-key\".into() }]") .replace("credential_sources: Vec::new()", "credential_sources: vec![exports::smith::plugin::registration::CredentialSource { id: \"connection-api-key\".into() }]") .replace("vec![ToolDeclaration { name: \"fixture\".to_string(), description: \"Fixture echo tool.\".to_string(), input_schema_json: r#\"{\"type\":\"object\"}\"#.to_string() }]", "Vec::new()"); std::fs::write(package.join("src/lib.rs"), source).unwrap(); let manifest_path = package.join("plugin.kdl"); let manifest = std::fs::read_to_string(&manifest_path).unwrap(); std::fs::write( &manifest_path, format!("{manifest}\ncapability \"net-scope\"\ncapability \"secret-proxy\"\n"), ) .unwrap(); let built = run(&["plugin", "build", package.to_str().unwrap()]); assert!( built.status.success(), "{}", String::from_utf8_lossy(&built.stderr) ); let linked = run(&[ "plugin", "link", package.to_str().unwrap(), "--grant", "net-scope", "--grant", "secret-proxy", ]); assert!( linked.status.success(), "{}", String::from_utf8_lossy(&linked.stderr) ); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut socket, _) = listener.accept().unwrap(); let _ = read_http_json(&mut socket); send_sse(&mut socket, "data: [DONE]\n\n"); }); let endpoint = format!("http://{address}"); let args = [ "eval", "hello", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &endpoint, ]; let output = run(&args); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"linked codec\n"); server.join().unwrap(); let artifact = std::fs::read_dir(package.join("target/wasm32-wasip2/debug")) .unwrap() .map(|entry| entry.unwrap().path()) .find(|path| path.extension().is_some_and(|ext| ext == "wasm")) .unwrap(); std::fs::write(artifact, b"invalid component").unwrap(); let failed = run(&args); assert!(!failed.status.success()); assert!( String::from_utf8_lossy(&failed.stderr).contains("validation failed"), "{}", String::from_utf8_lossy(&failed.stderr) ); assert!(!String::from_utf8_lossy(&failed.stderr).contains("dev-link-secret")); } #[test] fn eval_explicit_connection_setting_replaces_user_value_and_records_source() { let root = TestDir::new("smith-provider-settings-e2e"); std::fs::create_dir_all(root.join("config/smith")).unwrap(); std::fs::write( root.join("config/smith/provider-settings.kdl"), "include-usage #true", ) .unwrap(); std::fs::create_dir(root.join(".smith")).unwrap(); // Untrusted project config must not even be parsed. std::fs::write(root.join(".smith/provider-settings.kdl"), "not valid KDL").unwrap(); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut socket, _) = listener.accept().unwrap(); let request = read_http_json(&mut socket); assert_eq!(request["stream_options"]["include_usage"], false); send_sse( &mut socket, "data: {\"choices\":[{\"delta\":{\"content\":\"configured\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); }); let output = Command::cargo_bin("smith") .unwrap() .timeout(std::time::Duration::from_secs(30)) .current_dir(&root) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("OPENAI_API_KEY", "settings-secret") .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args([ "eval", "hello", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{address}"), "--include-usage", "false", ]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"configured\n"); assert!( String::from_utf8_lossy(&output.stderr) .contains("untrusted project provider settings ignored") ); server.join().unwrap(); assert_eq!( provider_settings_records(&root), vec![serde_json::json!({"include_usage":false, "provenance":"explicit"})] ); } #[test] fn eval_trusted_project_setting_replaces_user_value_without_accepting_host_authority() { let root = TestDir::new("smith-trusted-provider-settings"); std::fs::create_dir_all(root.join("config/smith")).unwrap(); std::fs::create_dir(root.join(".smith")).unwrap(); std::fs::write( root.join("config/smith/provider-settings.kdl"), "include-usage #false", ) .unwrap(); let project_file = root.join(".smith/provider-settings.kdl"); std::fs::write(&project_file, "include-usage #true").unwrap(); let run = |args: &[&str]| { Command::cargo_bin("smith") .unwrap() .timeout(std::time::Duration::from_secs(30)) .current_dir(&root) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("OPENAI_API_KEY", "trusted-settings-secret") .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args(args) .output() .unwrap() }; let trusted = run(&["plugin", "trust", root.to_str().unwrap()]); assert!( trusted.status.success(), "{}", String::from_utf8_lossy(&trusted.stderr) ); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut socket, _) = listener.accept().unwrap(); let request = read_http_json(&mut socket); assert_eq!(request["stream_options"]["include_usage"], true); send_sse( &mut socket, "data: {\"choices\":[{\"delta\":{\"content\":\"project settings\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); }); let endpoint = format!("http://{address}"); let args = [ "eval", "hello", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &endpoint, ]; let output = run(&args); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"project settings\n"); assert!(!String::from_utf8_lossy(&output.stderr).contains("untrusted project")); server.join().unwrap(); assert_eq!( provider_settings_records(&root), vec![serde_json::json!({"include_usage":true,"provenance":"trusted_project"})] ); for field in ["endpoint", "subscription", "grants", "unknown_setting"] { std::fs::write(&project_file, field).unwrap(); let rejected = run(&args); assert!(!rejected.status.success()); let diagnostic = String::from_utf8_lossy(&rejected.stderr); assert!( diagnostic.contains("unknown provider setting") && diagnostic.contains(field), "{diagnostic}" ); assert!(!diagnostic.contains("trusted-settings-secret")); } std::fs::write(project_file, "").unwrap(); let unsupported = run(&[ "eval", "hello", "--provider", "google", "--model", "gemini-flash-latest", "--base-url", &endpoint, "--include-usage", "false", ]); assert!(!unsupported.status.success()); assert!( String::from_utf8_lossy(&unsupported.stderr).contains("does not support disabling usage") ); } fn provider_settings_records(root: &Path) -> Vec { let mut settings = Vec::new(); for entry in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() { let path = entry.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } SessionReader::open(&path) .unwrap() .scan( smith::config::Config::default_valid().frame_bytes_max, |frame| { if let Frame::Known { entry } = frame && let EntryContent::Meta { kind, detail } = EntryContent::decode(&entry.content)? && kind == PROVIDER_DISPATCH_PROVENANCE_KIND { settings.push(serde_json::to_value(detail).unwrap()["settings"].clone()); } Ok(()) }, ) .unwrap(); } settings } #[test] fn eval_component_tool_authority_precedes_guest_and_effects() { use std::net::TcpListener; let package = initialized_package("authority"); let mut source = std::fs::read_to_string( Path::new(env!("CARGO_MANIFEST_DIR")).join("../plugins/echo/src/lib.rs"), ) .unwrap(); source = source.replace("../../wit/smith-plugin.wit", "wit/smith-plugin.wit") .replace("\"echo\"", "\"write\"") .replace(r#"{"type":"object","properties":{},"required":[]}"#, r#"{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}"#) .replace("Ok(input_json)", r#"smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite { path: "guest-entered.txt".into(), content: input_json.clone() })?; Ok(input_json)"#); std::fs::write(package.join("src/lib.rs"), source).unwrap(); let manifest = std::fs::read_to_string(package.join("plugin.kdl")).unwrap(); std::fs::write( package.join("plugin.kdl"), format!("{manifest}\ncapability \"workspace\"\n"), ) .unwrap(); let build = Command::cargo_bin("smith") .unwrap() .env("HOME", package.join("home")) .env("XDG_CONFIG_HOME", package.join("config")) .env("XDG_DATA_HOME", package.join("data")) .env("XDG_STATE_HOME", package.join("state")) .env("XDG_CACHE_HOME", package.join("cache")) .args(["plugin", "build"]) .arg(&package) .output() .unwrap(); assert!(build.status.success(), "{build:?}"); for (state, name, arguments, code) in [ ( "advertised", "example:authority/write", r#"{"value":"accepted"}"#, None, ), ( "hidden", "example:authority/write", r#"{"value":"denied"}"#, Some("TOOL_NOT_ADVERTISED"), ), ( "disabled", "example:authority/write", r#"{"value":"denied"}"#, Some("TOOL_DISABLED"), ), ( "advertised", "example:authority/write", r#"{"value":17}"#, Some("VALIDATION"), ), ( "hidden", "write", r#"{"path":"bundled-component.txt","content":"component-owned"}"#, None, ), ( "advertised", "missing:authority/write", r#"{"path":"native-fallback.txt","content":"denied"}"#, Some("TOOL_NOT_REGISTERED"), ), ] { let root = TestDir::new("smith-tool-authority-case"); let command = || { let mut command = Command::cargo_bin("smith").unwrap(); command .timeout(std::time::Duration::from_secs(30)) .current_dir(&root) .env("HOME", root.join("home")) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .env_remove("SMITH_CATALOG_CACHE") .env_remove("ANTHROPIC_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("ZAI_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .env("OPENAI_API_KEY", "authority-fixture-secret"); command }; let install = command() .args(["plugin", "install"]) .arg(&package) .args(["--grant", "workspace"]) .output() .unwrap(); assert!(install.status.success(), "{install:?}"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let request = read_http_json(&mut stream); assert_eq!( request["tools"] .as_array() .unwrap() .iter() .any(|tool| tool["function"]["name"] == "example:authority/write"), state == "advertised" ); let delta = serde_json::json!({"choices":[{"delta":{"tool_calls":[{"index":0,"id":"authority-call","function":{"name":name,"arguments":arguments}}]}}]}); send_sse( &mut stream, &format!( "data: {delta}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n" ), ); if code.is_none() { let (mut stream, _) = listener.accept().unwrap(); let request = read_http_json(&mut stream); let result = request["messages"] .as_array() .unwrap() .iter() .find(|message| message["role"] == "tool") .unwrap(); if name == "example:authority/write" { assert_eq!(result["content"], arguments); } else { assert!(!result["content"].as_str().unwrap().is_empty()); } send_sse( &mut stream, "data: {\"choices\":[{\"delta\":{\"content\":\"authority complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); } }); let output = command() .args([ "eval", "invoke fixture", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{address}"), "--tool-policy", &format!("example:authority/write={state}"), ]) .output() .unwrap(); let stderr = String::from_utf8_lossy(&output.stderr); assert!(!stderr.contains("authority-fixture-secret")); assert!(!String::from_utf8_lossy(&output.stdout).contains("authority-fixture-secret")); if let Some(code) = code { assert!(!output.status.success(), "{state}: {output:?}"); assert!( stderr.contains(code) && stderr.contains(name), "{state}: {stderr}" ); assert!(!root.join("guest-entered.txt").exists()); assert!(!root.join("native-fallback.txt").exists()); } else { assert!(output.status.success(), "{output:?}"); assert_eq!(output.stdout, b"authority complete\n"); if name == "example:authority/write" { assert_eq!( std::fs::read_to_string(root.join("guest-entered.txt")).unwrap(), arguments ); } else { assert!(!root.join("guest-entered.txt").exists()); assert_eq!( std::fs::read_to_string(root.join("bundled-component.txt")).unwrap(), "component-owned" ); } } server.join().unwrap(); let mut calls = 0; let mut nested = 0; for file in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() { let path = file.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } let bytes = std::fs::read(&path).unwrap(); assert!(!String::from_utf8_lossy(&bytes).contains("authority-fixture-secret")); SessionReader::open(&path) .unwrap() .scan( smith::config::Config::default_valid().frame_bytes_max, |frame| { if let Frame::Known { entry } = frame { match EntryContent::decode(&entry.content)? { EntryContent::ToolCall { .. } => calls += 1, EntryContent::Meta { kind, detail } if kind == smith_core::metadata::NESTED_INTENT_KIND => { nested += 1; let detail = serde_json::to_value(detail).unwrap(); assert_eq!( detail["component"]["source_id"], if name == "write" { "smith:coding-tools" } else { "example:authority" } ); } _ => {} } } Ok(()) }, ) .unwrap(); } assert_eq!(calls, usize::from(code.is_none())); assert_eq!(nested, usize::from(code.is_none())); } } #[test] fn eval_guest_trap_retains_completed_write_and_stops_remaining_tool_batch() { use std::net::{TcpListener, TcpStream}; use std::time::Duration; let package = initialized_package("terminal"); let source = std::fs::read_to_string( Path::new(env!("CARGO_MANIFEST_DIR")).join("../plugins/echo/src/lib.rs"), ) .unwrap() .replace("../../wit/smith-plugin.wit", "wit/smith-plugin.wit") .replace( "Ok(input_json)", r#"smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite { path: "completed.txt".into(), content: "retained".into() })?; std::process::abort()"#, ); std::fs::write(package.join("src/lib.rs"), source).unwrap(); let manifest = std::fs::read_to_string(package.join("plugin.kdl")).unwrap(); std::fs::write( package.join("plugin.kdl"), format!("{manifest}\ncapability \"workspace\"\n"), ) .unwrap(); let root = TestDir::new("smith-terminal-guest"); let command = || { let mut command = Command::cargo_bin("smith").unwrap(); command .timeout(Duration::from_secs(60)) .current_dir(&root) .env("HOME", root.join("home")) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .env_remove("SMITH_CATALOG_CACHE") .env_remove("ANTHROPIC_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("ZAI_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .env("OPENAI_API_KEY", "terminal-fixture-secret"); command }; command() .args(["plugin", "build"]) .arg(&package) .assert() .success(); command() .args(["plugin", "install"]) .arg(&package) .args(["--grant", "workspace"]) .assert() .success(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); stream .set_read_timeout(Some(Duration::from_secs(60))) .unwrap(); stream .set_write_timeout(Some(Duration::from_secs(60))) .unwrap(); let request = read_http_json(&mut stream); assert!( request["tools"] .as_array() .unwrap() .iter() .any(|tool| tool["function"]["name"] == "example:terminal/echo") ); let delta = serde_json::json!({"choices":[{"delta":{"tool_calls":[ {"index":0,"id":"trap-call","function":{"name":"example:terminal/echo","arguments":"{}"}}, {"index":1,"id":"never-call","function":{"name":"write","arguments":"{\"path\":\"never.txt\",\"content\":\"denied\"}"}} ]}}]}); send_sse( &mut stream, &format!( "data: {delta}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n" ), ); }); let output = command() .args([ "eval", "trap after write", "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{address}"), ]) .output() .unwrap(); // Unblock accept if CLI failed before provider admission; the stream deadline // then bounds teardown. No polling or waiting for wall time. let _unblock = TcpStream::connect_timeout(&address, Duration::from_secs(1)); server.join().unwrap(); assert!(!output.status.success(), "{output:?}"); let stderr = String::from_utf8_lossy(&output.stderr); assert!( stderr.contains("terminal host invocation stop: Sandbox"), "{stderr}" ); assert!(!stderr.contains("terminal-fixture-secret")); assert_eq!( std::fs::read_to_string(root.join("completed.txt")).unwrap(), "retained" ); assert!(!root.join("never.txt").exists()); let mut calls = 0; let mut failed_results = 0; let mut applied = 0; for file in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() { let path = file.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } SessionReader::open(&path) .unwrap() .scan( smith::config::Config::default_valid().frame_bytes_max, |frame| { let mut inspect = |entry: smith_core::session::EntryFrame| -> smith::Result<()> { match EntryContent::decode(&entry.content)? { EntryContent::ToolCall { .. } => calls += 1, EntryContent::ToolResult { ok: false, .. } => failed_results += 1, EntryContent::Meta { kind, detail } if kind == smith_core::metadata::NESTED_COMPLETION_KIND => { let detail = serde_json::to_value(detail).unwrap(); if detail["mutation_state"] == serde_json::to_value( smith::error::MutationState::Applied, ) .unwrap() { applied += 1; } } _ => {} } Ok(()) }; match frame { Frame::Known { entry } | Frame::NestedCommit { entry } => inspect(entry)?, Frame::Commit { entries, .. } => { for entry in *entries { inspect(entry)?; } } _ => {} } Ok(()) }, ) .unwrap(); } assert_eq!((calls, failed_results, applied), (1, 1, 1)); } #[test] fn eval_and_rpc_compose_hidden_target_with_durable_child_evidence() { use std::io::{BufRead, Write}; use std::process::Stdio; use std::time::Duration; let source = initialized_package("composition-source"); std::fs::write( source.join("src/lib.rs"), include_str!("../../smith-harness/tests/fixtures/composition.rs") .replace("WIT_PATH", "wit/smith-plugin.wit"), ) .unwrap(); let manifest = std::fs::read_to_string(source.join("plugin.kdl")).unwrap(); std::fs::write( source.join("plugin.kdl"), format!("{manifest}\ncapability \"workspace\"\n"), ) .unwrap(); let build = Command::cargo_bin("smith") .unwrap() .env("HOME", source.join("home")) .env("XDG_CONFIG_HOME", source.join("config")) .env("XDG_DATA_HOME", source.join("data")) .env("XDG_STATE_HOME", source.join("state")) .env("XDG_CACHE_HOME", source.join("cache")) .args(["plugin", "build"]) .arg(&source) .output() .unwrap(); assert!(build.status.success(), "{build:?}"); let manifest = smith_harness::plugins::PluginManifest::parse( &std::fs::read_to_string(source.join("plugin.kdl")).unwrap(), ) .unwrap(); let artifact = source.join(manifest.artifact()); let packages = ["a", "b", "target"].map(|id| package(id, &format!("id \"fixture:{id}\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"workspace\"\n"), &artifact)); for (surface, caller, mode, grant, disabled, success, written) in [ ("eval", "a", "caller", true, false, true, true), ("rpc", "b", "caller", true, false, true, true), ("eval", "a", "catch", false, false, false, false), ("eval", "a", "escape-catch", true, false, false, false), ("eval", "a", "catch", true, true, false, false), ("eval", "a", "invalid-catch", true, false, true, false), ("eval", "a", "parent-fail", true, false, false, true), ] { let root = TestDir::new("smith-composition-binary"); std::fs::create_dir(root.join("A")).unwrap(); std::fs::create_dir(root.join("B")).unwrap(); let command = || { let mut command = std::process::Command::new(assert_cmd::cargo::cargo_bin("smith")); command .current_dir(&root) .env("HOME", root.join("home")) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("XDG_RUNTIME_DIR", root.join("runtime")) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .env_remove("SMITH_CATALOG_CACHE") .env_remove("ANTHROPIC_API_KEY") .env_remove("GOOGLE_API_KEY") .env_remove("GEMINI_API_KEY") .env_remove("ZAI_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .env("OPENAI_API_KEY", "composition-fixture-secret"); command }; for package in &packages { let output = command() .args(["plugin", "install"]) .arg(package) .args(["--grant", "workspace"]) .output() .unwrap(); assert!(output.status.success(), "{output:?}"); } let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let server = std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let request = read_http_json(&mut stream); let tools = request["tools"].as_array().unwrap(); assert!( !tools .iter() .any(|tool| tool["function"]["name"] == "fixture:target/fixture") ); assert!( tools .iter() .any(|tool| tool["function"]["name"] == "fixture:a/fixture") ); assert!( tools .iter() .any(|tool| tool["function"]["name"] == "fixture:b/fixture") ); let delta = serde_json::json!({"choices":[{"delta":{"tool_calls":[{"index":0,"id":"compose-call","function":{"name":format!("fixture:{caller}/fixture"),"arguments":format!(r#"{{"mode":"{mode}"}}"#)}}]}}]}); send_sse( &mut stream, &format!( "data: {delta}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n" ), ); if success { let (mut stream, _) = listener.accept().unwrap(); let request = read_http_json(&mut stream); let output = request["messages"] .as_array() .unwrap() .iter() .find(|message| message["role"] == "tool") .unwrap(); assert_eq!( output["content"], if written { "child complete" } else { "caught ordinary guest error" } ); send_sse( &mut stream, "data: {\"choices\":[{\"delta\":{\"content\":\"composition complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ); } }); let mut invocation = command(); invocation.arg(surface); if surface == "eval" { invocation.arg("compose"); } invocation.args([ "--provider", "openai", "--model", "gpt-4.1", "--base-url", &format!("http://{address}"), "--tool-policy", &format!( "fixture:target/fixture={}", if disabled { "disabled" } else { "hidden" } ), ]); if grant { invocation.args([ "--tool-delegation", &format!("fixture:{caller}/fixture=fixture:target/fixture#A"), ]); } if surface == "rpc" { let mut child = TestChild( invocation .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .unwrap(), ); let mut stdin = child.0.stdin.take().unwrap(); let stdout = child.0.stdout.take().unwrap(); let (sender, received) = std::sync::mpsc::sync_channel(32); let reader = std::thread::spawn(move || { for line in std::io::BufReader::new(stdout).lines() { sender.send(line.unwrap()).unwrap(); } }); writeln!(stdin, "{{\"id\":1,\"method\":\"prompt\",\"params\":{{\"input\":\"compose\"}}}}\n{{\"id\":2,\"method\":\"wait\"}}").unwrap(); stdin.flush().unwrap(); let mut ended = false; for _ in 0..32 { let line = received.recv_timeout(Duration::from_secs(30)).unwrap(); assert!(!line.contains("composition-fixture-secret")); let value: serde_json::Value = serde_json::from_str(&line).unwrap(); if value["event"] == "end" { assert_eq!(value["text"], "composition complete"); ended = true; } if value["id"] == 2 { assert_eq!(value["settled"], true); break; } } assert!(ended); drop(stdin); assert!(received.recv_timeout(Duration::from_secs(30)).is_err()); reader.join().unwrap(); assert!(child.0.wait().unwrap().success()); } else { let output = Command::from(invocation) .timeout(Duration::from_secs(30)) .output() .unwrap(); assert_eq!(output.status.success(), success, "{mode}: {output:?}"); assert!( !String::from_utf8_lossy(&output.stderr).contains("composition-fixture-secret") ); if success { assert_eq!(output.stdout, b"composition complete\n"); } } server.join().unwrap(); assert_eq!(root.join("A/written.txt").exists(), written, "{mode}"); assert!(!root.join("B/denied.txt").exists()); assert!(!root.join("written.txt").exists()); if written { let mut child_intents = 0; let mut writes = 0; for entry in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() { let path = entry.unwrap().path(); if path.extension().is_none_or(|extension| extension != "smh") { continue; } let reader = SessionReader::open(&path).unwrap(); let limit = smith::config::Config::default_valid().frame_bytes_max; reader.validate_work(limit).unwrap(); reader .scan(limit, |frame| { if let Frame::Known { entry } | Frame::NestedCommit { entry } = frame && let EntryContent::Meta { kind, detail } = EntryContent::decode(&entry.content)? { let detail = serde_json::to_value(detail).unwrap(); if kind == smith_core::metadata::NESTED_INTENT_KIND && detail["causal"]["invocation"] == true { child_intents += 1; assert_eq!(detail["component"]["source_id"], "fixture:target"); } if kind == smith_core::metadata::NESTED_COMPLETION_KIND && detail["causal"]["invocation"] == false { assert_eq!( detail["mutation_state"], serde_json::to_value(smith::error::MutationState::Applied) .unwrap() ); writes += 1; } } Ok(()) }) .unwrap(); } assert_eq!(child_intents, 1, "{mode}"); assert_eq!(writes, 1, "{mode}"); } } }