repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-cli/tests/e2e_plugin.rs
Raw//! Plugin package checks through the built binary (`SMH-SPEC-WASM0001`, Tooling;
//! `SMH-SPEC-SPEC0001`, Providers: authentication; `SMH-SPEC-PLUG0001`, Tool visibility and authority).
use assert_cmd::Command;
use smith_core::frame::Frame;
use smith_core::session::{EntryContent, PROVIDER_DISPATCH_PROVENANCE_KIND};
use smith_core::store::SessionReader;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
struct TestDir(PathBuf);
impl TestDir {
fn new(prefix: &str) -> Self {
let root = Self::vacant(prefix);
std::fs::create_dir(&root.0)
.unwrap_or_else(|error| panic!("create fixture root {}: {error}", root.display()));
root
}
fn vacant(prefix: &str) -> Self {
static NEXT: AtomicUsize = AtomicUsize::new(0);
loop {
let path = std::env::temp_dir().join(format!(
"{prefix}-{}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
if !path.exists() {
return Self(path);
}
}
}
}
impl Deref for TestDir {
type Target = Path;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl AsRef<Path> for TestDir {
fn as_ref(&self) -> &Path {
self
}
}
impl AsRef<std::ffi::OsStr> for TestDir {
fn as_ref(&self) -> &std::ffi::OsStr {
self.0.as_os_str()
}
}
impl Drop for TestDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
struct TestChild(std::process::Child);
impl TestChild {
fn wait_with_output(mut self) -> std::process::Output {
let status = self.0.wait().unwrap();
self.output(status)
}
fn kill_with_output(&mut self) -> std::process::Output {
let _ = self.0.kill();
let status = self.0.wait().unwrap();
self.output(status)
}
fn output(&mut self, status: std::process::ExitStatus) -> std::process::Output {
let mut stdout = Vec::new();
let mut stderr = Vec::new();
std::io::Read::read_to_end(self.0.stdout.as_mut().unwrap(), &mut stdout).unwrap();
std::io::Read::read_to_end(self.0.stderr.as_mut().unwrap(), &mut stderr).unwrap();
std::process::Output {
status,
stdout,
stderr,
}
}
}
impl Drop for TestChild {
fn drop(&mut self) {
if self.0.try_wait().ok().flatten().is_none() {
let _ = self.0.kill();
}
let _ = self.0.wait();
}
}
fn package(name: &str, manifest: &str, artifact: &Path) -> TestDir {
let root = TestDir::new(&format!("smith-plugin-check-{name}"));
std::fs::write(root.join("plugin.kdl"), manifest).unwrap();
std::fs::copy(artifact, root.join("fixture.wasm")).unwrap();
root
}
fn fixture_package(name: &str, manifest: &str) -> TestDir {
package(
name,
manifest,
&Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../smith-harness/tests/fixtures/plugin-package/fixture.wasm"),
)
}
fn echo_artifact() -> PathBuf {
let status = std::process::Command::new("cargo")
.args([
"build",
"-p",
"smith-plugin-echo",
"--target",
"wasm32-wasip2",
"--locked",
])
.status()
.unwrap();
assert!(status.success(), "echo plugin build failed");
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../target/wasm32-wasip2/debug/smith_plugin_echo.wasm")
}
fn initialized_package(name: &str) -> TestDir {
let root = TestDir::vacant(&format!("smith-plugin-test-{name}"));
Command::cargo_bin("smith")
.unwrap()
.env("HOME", root.join("home"))
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.args(["plugin", "init"])
.arg(&root)
.args(["--name", &format!("example:{name}")])
.assert()
.success();
root
}
fn plugin_test(package: &Path, grants: &[&str]) -> std::process::Output {
let mut command = Command::cargo_bin("smith").unwrap();
command.args(["plugin", "test"]).arg(package);
for grant in grants {
command.args(["--grant", grant]);
}
command.output().unwrap()
}
fn smith(package: &Path, grants: &[&str]) -> std::process::Output {
let home = package.join("xdg");
let mut command = Command::cargo_bin("smith").unwrap();
command
.env("XDG_CONFIG_HOME", home.join("config"))
.env("XDG_DATA_HOME", home.join("data"))
.env("XDG_STATE_HOME", home.join("state"))
.env("XDG_CACHE_HOME", home.join("cache"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args(["plugin", "check", "--trust"])
.arg(package);
for grant in grants {
command.args(["--grant", grant]);
}
command.output().unwrap()
}
#[test]
fn typescript_plugin_builds_tests_and_loads_with_the_sealed_host() {
let package = Path::new(env!("CARGO_MANIFEST_DIR")).join("../plugins/example-typescript");
let output = Command::cargo_bin("smith")
.unwrap()
.args(["plugin", "test"])
.arg(&package)
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(
output.stdout,
format!("{}: test passed\n", package.display()).as_bytes()
);
}
#[test]
fn plugin_init_typescript_scaffolds_the_jco_adapter() {
let root = TestDir::new("smith-plugin-init-typescript");
let package = root.join("demo");
let output = Command::cargo_bin("smith")
.unwrap()
.args(["plugin", "init"])
.arg(&package)
.args(["--name", "example:demo", "--language", "typescript"])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert!(package.join("package.json").is_file());
assert!(package.join("tsconfig.json").is_file());
assert!(package.join("src/plugin.ts").is_file());
assert!(package.join("wit/smith-plugin.wit").is_file());
assert!(!package.join("Cargo.toml").exists());
}
#[test]
fn plugin_init_build_and_test_scaffold_a_pinned_raw_wit_component() {
let root = TestDir::new("smith-plugin-init");
let package = root.join("demo");
let init = Command::cargo_bin("smith")
.unwrap()
.args(["plugin", "init"])
.arg(&package)
.args(["--name", "example:demo"])
.output()
.unwrap();
assert!(
init.status.success(),
"{}",
String::from_utf8_lossy(&init.stderr)
);
assert!(package.join("wit/smith-plugin.wit").is_file());
assert!(package.join("build.rs").is_file());
assert!(package.join("Cargo.lock").is_file());
let build = Command::cargo_bin("smith")
.unwrap()
.args(["plugin", "build"])
.arg(&package)
.output()
.unwrap();
assert!(
build.status.success(),
"{}",
String::from_utf8_lossy(&build.stderr)
);
let test = Command::cargo_bin("smith")
.unwrap()
.args(["plugin", "test"])
.arg(&package)
.output()
.unwrap();
assert!(
test.status.success(),
"{}",
String::from_utf8_lossy(&test.stderr)
);
assert_eq!(
test.stdout,
format!("{}: test passed\n", package.display()).as_bytes()
);
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn plugin_test_runs_echo_fixtures_and_reports_fixture_failures() {
let package = initialized_package("fixture-echo");
std::fs::write(
package.join("plugin-test.json"),
r#"{"invocations":[{"name":"fixture","input_json":"{\"text\":\"hello\"}","expected_output":"{\"text\":\"hello\"}","effects":[]}]}"#,
)
.unwrap();
let success = plugin_test(&package, &[]);
assert!(
success.status.success(),
"{}",
String::from_utf8_lossy(&success.stderr)
);
assert_eq!(
success.stdout,
format!("{}: test passed\n", package.display()).as_bytes()
);
std::fs::write(
package.join("plugin-test.json"),
r#"{"invocations":[{"name":"fixture","input_json":"{}","expected_output":"wrong","effects":[]}]}"#,
)
.unwrap();
let mismatch = plugin_test(&package, &[]);
assert!(!mismatch.status.success());
insta::assert_snapshot!(
"plugin_test_output_mismatch",
String::from_utf8(mismatch.stderr).unwrap()
);
std::fs::write(
package.join("plugin.kdl"),
"id \"example:fixture-echo\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_fixture_echo.wasm\"\ncapability \"workspace\"\n",
)
.unwrap();
let ungranted = plugin_test(&package, &[]);
assert!(!ungranted.status.success());
insta::assert_snapshot!(
"plugin_test_ungranted",
String::from_utf8(ungranted.stderr).unwrap()
);
std::fs::remove_dir_all(package).unwrap();
plugin_test_mocks_coding_read_and_rejects_unexpected_effects();
plugin_test_mocks_http_secret_and_log_effects_deterministically();
}
fn plugin_test_mocks_http_secret_and_log_effects_deterministically() {
use std::net::TcpListener;
let package = initialized_package("fixture-http-secret");
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let destination = format!("http://{}", listener.local_addr().unwrap());
std::fs::write(
package.join("src/lib.rs"),
r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")]
#![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")]
wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" });
use exports::smith::plugin::plugin::{Guest, ToolDeclaration};
use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance};
use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations};
use smith::plugin::log;
use smith::plugin::net_scope::{self, Header, HttpRequest, RequestOptions};
use smith::plugin::secret_proxy::{self, CredentialExchange, SecretRef};
struct Plugin;
impl RegistrationGuest for Plugin { fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } } }
impl ProviderGuest for Plugin {
fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } }
fn validate_settings(_: ProviderSettings) -> Result<ProviderSettings, String> { Err("no codec".to_string()) }
fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result<RequestTemplate, String> { Err("no codec".to_string()) }
fn decode_response(_: Vec<u8>, _: Vec<u8>, _: bool) -> Result<DecodeStep, String> { Err("no codec".to_string()) }
}
impl Guest for Plugin {
fn declare_tools() -> Vec<ToolDeclaration> { vec![ToolDeclaration { name: "round".to_string(), description: "Mocked HTTP, secret, and log.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] }
fn invoke(_: String, _: String) -> Result<String, String> {
let destination = "http://127.0.0.1:9".to_string();
let secret = secret_proxy::exchange(&CredentialExchange { source: SecretRef { id: "fixture-key".to_string() }, audience: destination.clone(), scopes: vec!["http-request".to_string()] })?;
log::write("info", "fixture secret")?;
let response = net_scope::request(&HttpRequest { destination, method: "POST".to_string(), headers: vec![Header { name: "Authorization".to_string(), value: format!("{{{{secret:{}}}}}", secret.id) }], body: Vec::new(), options: RequestOptions { timeout_ms: 1000, response_bytes_max: 64 } })?;
Ok(format!(r#"{{"status":{}}}"#, response.status))
}
}
export!(Plugin);
"##,
)
.unwrap();
let source = std::fs::read_to_string(package.join("src/lib.rs")).unwrap();
std::fs::write(
package.join("src/lib.rs"),
source.replace("http://127.0.0.1:9", &destination),
)
.unwrap();
std::fs::write(
package.join("plugin.kdl"),
"id \"example:fixture-http-secret\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_fixture_http_secret.wasm\"\ncapability \"net-scope\"\ncapability \"secret-proxy\"\ncapability \"log\"\n",
)
.unwrap();
let request_secret = serde_json::json!({"source":{"id":"fixture-key"},"audience":destination.clone(),"scopes":["http-request"]}).to_string();
let request_log = r#"{"level":"info","message":"fixture secret"}"#;
let request_http = serde_json::json!({"destination":destination,"method":"POST","headers":[{"name":"Authorization","value":"{{secret:mock-token}}"}],"body":[],"options":{"timeout_ms":1000,"response_bytes_max":64}}).to_string();
let fixture = format!(
r#"{{"invocations":[{{"name":"round","input_json":"{{}}","expected_output":"{{\"status\":200}}","effects":[{{"operation":"credential-exchange","request_json":{},"response":"mock-token"}},{{"operation":"log-write","request_json":{},"response":""}},{{"operation":"http-request","request_json":{},"response":"{{\"status\":200,\"headers\":[],\"body\":[]}}"}}]}}]}}"#,
serde_json::to_string(&request_secret).unwrap(),
serde_json::to_string(request_log).unwrap(),
serde_json::to_string(&request_http).unwrap(),
);
std::fs::write(package.join("plugin-test.json"), &fixture).unwrap();
let xdg = package.join("isolated-xdg");
let run = || {
let mut command = Command::cargo_bin("smith").unwrap();
command
.env("XDG_CONFIG_HOME", xdg.join("config"))
.env("XDG_DATA_HOME", xdg.join("data"))
.env("XDG_STATE_HOME", xdg.join("state"))
.env("XDG_CACHE_HOME", xdg.join("cache"))
.env_remove("OPENAI_API_KEY")
.env_remove("SMITH_PROVIDER")
.args(["plugin", "test"])
.arg(&package)
.args([
"--grant",
"net-scope",
"--grant",
"secret-proxy",
"--grant",
"log",
])
.output()
.unwrap()
};
let mocked = run();
assert!(
mocked.status.success(),
"{}",
String::from_utf8_lossy(&mocked.stderr)
);
assert!(
!xdg.join("data/smith/auth").exists(),
"fixture must not open real auth store"
);
listener.set_nonblocking(true).unwrap();
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock,
"mocked HTTP must not reach network"
);
let unexpected_fixture = fixture.replace(
"\"operation\":\"http-request\"",
"\"operation\":\"http-wrong\"",
);
std::fs::write(package.join("plugin-test.json"), unexpected_fixture).unwrap();
let unexpected = run();
assert!(!unexpected.status.success());
assert!(
String::from_utf8_lossy(&unexpected.stderr).contains("fixture effect"),
"{}",
String::from_utf8_lossy(&unexpected.stderr)
);
assert!(!xdg.join("data/smith/auth").exists());
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock,
"denied HTTP fixture must not reach network"
);
std::fs::remove_dir_all(package).unwrap();
}
fn plugin_test_mocks_coding_read_and_rejects_unexpected_effects() {
let package = initialized_package("fixture-read");
std::fs::write(
package.join("src/lib.rs"),
r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")]
#![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")]
wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" });
use exports::smith::plugin::plugin::{Guest, ToolDeclaration};
use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance};
use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations};
use smith::plugin::tooling::{self, FileRead};
struct Plugin;
impl RegistrationGuest for Plugin {
fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } }
}
impl ProviderGuest for Plugin {
fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } }
fn validate_settings(_: ProviderSettings) -> Result<ProviderSettings, String> { Err("no codec".to_string()) }
fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result<RequestTemplate, String> { Err("no codec".to_string()) }
fn decode_response(_: Vec<u8>, _: Vec<u8>, _: bool) -> Result<DecodeStep, String> { Err("no codec".to_string()) }
}
impl Guest for Plugin {
fn declare_tools() -> Vec<ToolDeclaration> { vec![ToolDeclaration { name: "read".to_string(), description: "Fixture read.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] }
fn invoke(_: String, _: String) -> Result<String, String> { tooling::read(&FileRead { path: "never-read.txt".to_string(), offset: 0, limit: 1, max_bytes: 64 }).map(|content| content.text) }
}
export!(Plugin);
"##,
)
.unwrap();
std::fs::write(
package.join("plugin.kdl"),
"id \"example:fixture-read\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_fixture_read.wasm\"\ncapability \"workspace\"\n",
)
.unwrap();
std::fs::write(
package.join("plugin-test.json"),
r#"{"invocations":[{"name":"read","input_json":"{}","expected_output":"mocked","effects":[{"operation":"read","request_json":"{\"path\":\"never-read.txt\",\"offset\":0,\"limit\":1,\"max_bytes\":64}","response":"mocked"}]}]}"#,
)
.unwrap();
let mocked = plugin_test(&package, &["workspace"]);
assert!(
mocked.status.success(),
"{}",
String::from_utf8_lossy(&mocked.stderr)
);
assert!(!package.join("never-read.txt").exists());
std::fs::write(
package.join("plugin-test.json"),
r#"{"invocations":[{"name":"read","input_json":"{}","expected_output":"mocked","effects":[]}]}"#,
)
.unwrap();
let unexpected = plugin_test(&package, &["workspace"]);
assert!(!unexpected.status.success());
insta::assert_snapshot!(
"plugin_test_unexpected_effect",
String::from_utf8(unexpected.stderr).unwrap()
);
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn plugin_install_list_reload_and_remove_keep_a_hash_pinned_registry() {
let artifact = echo_artifact();
let package = package(
"installed-registry",
"id \"example:registry\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n",
&artifact,
);
let config = package.join("xdg/config");
let run = |arguments: &[&str]| {
Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", &config)
.env("XDG_DATA_HOME", package.join("xdg/data"))
.env("XDG_STATE_HOME", package.join("xdg/state"))
.env("XDG_CACHE_HOME", package.join("xdg/cache"))
.args(arguments)
.output()
.unwrap()
};
let install = run(&["plugin", "install", package.to_str().unwrap()]);
assert!(
install.status.success(),
"{}",
String::from_utf8_lossy(&install.stderr)
);
assert_eq!(install.stdout, b"example:registry: installed\n");
let listed = run(&["plugin", "list"]);
assert!(listed.status.success());
let listed = String::from_utf8(listed.stdout).unwrap();
assert!(listed.starts_with("installed example:registry "));
assert_eq!(listed.split_ascii_whitespace().count(), 5);
let reloaded = run(&["plugin", "reload"]);
assert!(!reloaded.status.success());
assert!(String::from_utf8_lossy(&reloaded.stderr).contains("runtime is unavailable"));
std::fs::write(package.join("fixture.wasm"), b"changed").unwrap();
let rejected = run(&["plugin", "reload"]);
assert!(!rejected.status.success());
assert!(String::from_utf8_lossy(&rejected.stderr).contains("changed since install"));
let retained = run(&["plugin", "list"]);
assert!(retained.status.success());
assert!(
String::from_utf8(retained.stdout)
.unwrap()
.starts_with("installed example:registry ")
);
let removed = run(&["plugin", "remove", "example:registry"]);
assert!(removed.status.success());
assert_eq!(removed.stdout, b"example:registry: removed\n");
let empty = run(&["plugin", "list"]);
assert!(empty.status.success());
assert!(empty.stdout.is_empty());
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn plugin_link_precedes_installed_package_for_reload() {
let artifact = echo_artifact();
let installed = package(
"link-precedence-installed",
"id \"example:precedence\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n",
&artifact,
);
let linked = package(
"link-precedence-linked",
"id \"example:precedence\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n",
&artifact,
);
let config = installed.join("xdg/config");
let run = |arguments: &[&str]| {
Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", &config)
.env("XDG_DATA_HOME", installed.join("xdg/data"))
.env("XDG_STATE_HOME", installed.join("xdg/state"))
.env("XDG_CACHE_HOME", installed.join("xdg/cache"))
.args(arguments)
.output()
.unwrap()
};
assert!(
run(&["plugin", "install", installed.to_str().unwrap()])
.status
.success()
);
assert!(
run(&["plugin", "link", linked.to_str().unwrap()])
.status
.success()
);
std::fs::write(installed.join("fixture.wasm"), b"installed package changed").unwrap();
let reload = run(&["plugin", "reload"]);
assert!(!reload.status.success());
assert!(String::from_utf8_lossy(&reload.stderr).contains("runtime is unavailable"));
let listed = String::from_utf8(run(&["plugin", "list"]).stdout).unwrap();
assert!(listed.contains("installed example:precedence"));
assert!(listed.contains("linked example:precedence"));
std::fs::remove_dir_all(installed).unwrap();
std::fs::remove_dir_all(linked).unwrap();
}
#[test]
fn eval_loads_an_installed_plugin_into_provider_tool_rounds() {
use std::net::TcpListener;
let artifact = echo_artifact();
let package = package(
"runtime-registry",
"id \"example:runtime\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n",
&artifact,
);
let root = package.join("xdg");
let install = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.args(["plugin", "install"])
.arg(&package)
.output()
.unwrap();
assert!(
install.status.success(),
"{}",
String::from_utf8_lossy(&install.stderr)
);
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let changed_artifact = package.join("fixture.wasm");
let server = std::thread::spawn(move || {
let (mut first, _) = listener.accept().unwrap();
let request = read_http_json(&mut first);
assert!(
request["tools"]
.as_array()
.unwrap()
.iter()
.any(|tool| tool["function"]["name"] == "example:runtime/echo")
);
send_sse(
&mut first,
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"runtime-echo\",\"function\":{\"name\":\"example:runtime/echo\",\"arguments\":\"{\\\"value\\\":\\\"hello\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
);
std::fs::write(changed_artifact, b"installed artifact changed").unwrap();
let (mut second, _) = listener.accept().unwrap();
let request = read_http_json(&mut second);
assert!(
request["tools"]
.as_array()
.unwrap()
.iter()
.any(|tool| tool["function"]["name"] == "example:runtime/echo")
);
let tool = request["messages"]
.as_array()
.unwrap()
.iter()
.find(|message| message["role"] == "tool")
.unwrap();
assert_eq!(tool["tool_call_id"], "runtime-echo");
assert_eq!(tool["content"], "{\"value\":\"hello\"}");
send_sse(
&mut second,
"data: {\"choices\":[{\"delta\":{\"content\":\"installed plugin complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
});
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("OPENAI_API_KEY", "runtime-plugin-secret")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("GOOGLE_API_KEY")
.args([
"eval",
"use installed tool",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{address}"),
])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"installed plugin complete\n");
server.join().unwrap();
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn eval_plugin_net_scope_records_redacted_intent_before_local_server_handles_request() {
plugin_net_scope_records_redacted_intent(false);
}
#[test]
fn admitted_plugin_http_secret_and_log_commit_nested_progress_before_outer_completion() {
plugin_net_scope_records_redacted_intent(true);
}
fn plugin_net_scope_records_redacted_intent(explicit_submission: bool) {
use std::io::{Read, Write};
use std::net::TcpListener;
let package = initialized_package("http-round");
std::fs::write(
package.join("plugin.kdl"),
"id \"example:http\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_http_round.wasm\"\ncapability \"net-scope\"\ncapability \"secret-proxy\"\ncapability \"log\"\ncapability \"workspace\"\n",
)
.unwrap();
std::fs::write(
package.join("src/lib.rs"),
r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")]
#![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")]
wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" });
use exports::smith::plugin::plugin::{Guest, ToolDeclaration};
use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance};
use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations};
use smith::plugin::log;
use smith::plugin::net_scope::{self, Header, HttpRequest, RequestOptions};
use smith::plugin::secret_proxy::{self, CredentialExchange, SecretRef};
struct Plugin;
impl RegistrationGuest for Plugin { fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } } }
impl ProviderGuest for Plugin {
fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } }
fn validate_settings(_: ProviderSettings) -> Result<ProviderSettings, String> { Err("no codec".to_string()) }
fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result<RequestTemplate, String> { Err("no codec".to_string()) }
fn decode_response(_: Vec<u8>, _: Vec<u8>, _: bool) -> Result<DecodeStep, String> { Err("no codec".to_string()) }
}
impl Guest for Plugin {
fn declare_tools() -> Vec<ToolDeclaration> { vec![ToolDeclaration { name: "round".to_string(), description: "HTTP round trip.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] }
fn invoke(_: String, _: String) -> Result<String, String> {
let destination = std::env::var("UNAVAILABLE").unwrap_or_else(|_| "http://127.0.0.1:1".to_string());
let secret = secret_proxy::exchange(&CredentialExchange { source: SecretRef { id: "connection-api-key".to_string() }, audience: destination.clone(), scopes: vec!["http-request".to_string()] })?;
log::write("info", "authorization: guest-http-secret")?;
let response = net_scope::request(&HttpRequest { destination, method: "POST".to_string(), headers: vec![Header { name: "Authorization".to_string(), value: format!("{{{{secret:{}}}}}", secret.id) }], body: b"guest-body".to_vec(), options: RequestOptions { timeout_ms: 1_000, response_bytes_max: 64 } })?;
Ok(format!(r#"{{"status":{}}}"#, response.status))
}
}
export!(Plugin);
"##,
)
.unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let source = std::fs::read_to_string(package.join("src/lib.rs")).unwrap();
let source = if explicit_submission {
source.replace("log::write", r#"smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite { path: "nested-first.txt".into(), content: "guest-http-secret".into() })?;
smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite { path: "nested-second.txt".into(), content: "guest-http-secret".into() })?;
log::write"#).replace("body: b\"guest-body\".to_vec()", "body: b\"guest-http-secret\".repeat(5000)")
.replace("response_bytes_max: 64", "response_bytes_max: 1048576")
.replace("timeout_ms: 1_000", "timeout_ms: 30_000")
} else {
source
};
std::fs::write(
package.join("src/lib.rs"),
source.replace("http://127.0.0.1:1", &format!("http://{address}")),
)
.unwrap();
let root = package.join("xdg");
let run = |arguments: &[&str]| {
Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.args(arguments)
.output()
.unwrap()
};
assert!(
run(&["plugin", "build", package.to_str().unwrap()])
.status
.success()
);
let install = run(&[
"plugin",
"install",
package.to_str().unwrap(),
"--grant",
"net-scope",
"--net-origin",
&format!("http://{address}"),
"--grant",
"secret-proxy",
"--grant",
"log",
"--grant",
"workspace",
"--secret-source",
"connection-api-key",
"--secret-provider",
"openai",
"--secret-origin",
&format!("http://{address}"),
"--secret-scope",
"http-request",
]);
assert!(
install.status.success(),
"{}",
String::from_utf8_lossy(&install.stderr)
);
let data = root.join("data");
let auth = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.args(["auth", "add", "--provider", "openai"])
.write_stdin("guest-http-secret\n")
.output()
.unwrap();
assert!(
auth.status.success(),
"{}",
String::from_utf8_lossy(&auth.stderr)
);
let submission = if explicit_submission {
assert!(run(&["eval", "--mock", "seed"]).status.success());
let listed: serde_json::Value =
serde_json::from_slice(&run(&["sessions", "list", "--json"]).stdout).unwrap();
let session = listed["session"].as_str().unwrap().to_string();
let submitted = run(&["sessions", "submit", &session, "use http"]);
assert!(
submitted.status.success(),
"{}",
String::from_utf8_lossy(&submitted.stderr)
);
let submitted: serde_json::Value = serde_json::from_slice(&submitted.stdout).unwrap();
Some((
session,
submitted["submission"].as_str().unwrap().to_string(),
))
} else {
None
};
let server_data = data.clone();
let server = std::thread::spawn(move || {
let (mut provider, _) = listener.accept().unwrap();
let request = read_http_json(&mut provider);
assert!(
request["tools"]
.as_array()
.unwrap()
.iter()
.any(|tool| tool["function"]["name"] == "example:http/round")
);
send_sse(
&mut provider,
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"http-round\",\"function\":{\"name\":\"example:http/round\",\"arguments\":\"{}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
);
let (mut guest, _) = listener.accept().unwrap();
guest
.set_read_timeout(Some(std::time::Duration::from_secs(120)))
.unwrap();
guest
.set_write_timeout(Some(std::time::Duration::from_secs(120)))
.unwrap();
let mut header = Vec::new();
while !header.ends_with(b"\r\n\r\n") {
let mut byte = [0];
guest.read_exact(&mut byte).unwrap();
header.push(byte[0]);
assert!(header.len() <= 16 * 1024);
}
let expected_body = if explicit_submission {
b"guest-http-secret".repeat(5000)
} else {
b"guest-body".to_vec()
};
let mut body = vec![0; expected_body.len()];
guest.read_exact(&mut body).unwrap();
assert_eq!(body, expected_body);
let request = std::str::from_utf8(&header).unwrap();
assert!(request.contains("POST / HTTP/1.1"));
assert!(
request
.to_ascii_lowercase()
.contains("authorization: guest-http-secret"),
"{request}"
);
let session = std::fs::read_dir(server_data.join("smith/sessions/run"))
.unwrap()
.find_map(|entry| {
let path = entry.unwrap().path();
(path.extension().is_some_and(|extension| extension == "smh")).then_some(path)
})
.unwrap();
let mut intent = Vec::new();
let journal = SessionReader::open(&session).unwrap();
journal
.scan(
smith::config::Config::default_valid().frame_bytes_max,
|frame| {
if let Frame::Known { entry } = frame
&& let EntryContent::Meta { kind, detail } =
EntryContent::decode(&entry.content)?
&& kind == smith_core::metadata::NESTED_INTENT_KIND
{
let detail = serde_json::to_value(detail).unwrap();
if detail["operation"] == "http" {
intent.push(
serde_json::from_str::<serde_json::Value>(
&journal
.nested_payload(
entry.id,
smith::config::Config::default_valid().frame_bytes_max,
)
.unwrap(),
)
.unwrap(),
);
}
}
Ok(())
},
)
.unwrap();
assert_eq!(
intent.len(),
1,
"durable HTTP intent must precede server handling"
);
assert_eq!(intent[0]["headers"][0]["value"], "<redacted>");
if explicit_submission {
write!(
guest,
"HTTP/1.1 201 Created\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
expected_body.len()
)
.unwrap();
guest.write_all(&expected_body).unwrap();
} else {
guest
.write_all(
b"HTTP/1.1 201 Created\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
)
.unwrap();
}
let (mut provider, _) = listener.accept().unwrap();
let request = read_http_json(&mut provider);
assert!(
request["messages"].as_array().unwrap().iter().any(
|message| message["role"] == "tool" && message["content"] == "{\"status\":201}"
)
);
send_sse(
&mut provider,
"data: {\"choices\":[{\"delta\":{\"content\":\"http complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
});
let mut arguments = if let Some((session, submission)) = submission {
vec![
"sessions".to_string(),
"execute".to_string(),
session,
submission,
]
} else {
vec!["eval".to_string(), "use http".to_string()]
};
arguments.extend([
"--provider".to_string(),
"openai".to_string(),
"--model".to_string(),
"gpt-4.1".to_string(),
"--base-url".to_string(),
format!("http://{address}"),
]);
let output = Command::cargo_bin("smith")
.unwrap()
.timeout(std::time::Duration::from_secs(180))
.current_dir(&package)
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("OPENAI_API_KEY", "provider-secret")
.args(arguments)
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
server.join().unwrap();
if explicit_submission {
let receipt: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(receipt["state"], "terminal");
assert_eq!(receipt["outcome"], "completed");
} else {
assert_eq!(output.stdout, b"http complete\n");
}
assert!(!String::from_utf8_lossy(&output.stdout).contains("guest-http-secret"));
assert!(!String::from_utf8_lossy(&output.stderr).contains("guest-http-secret"));
let mut nested_intents = Vec::new();
let mut nested_completions = Vec::new();
let mut outer_commits = 0;
for entry in std::fs::read_dir(data.join("smith/sessions/run")).unwrap() {
let path = entry.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
let bytes = std::fs::read(&path).unwrap();
assert!(!String::from_utf8_lossy(&bytes).contains("guest-http-secret"));
let journal = SessionReader::open(&path).unwrap();
journal
.scan(
smith::config::Config::default_valid().frame_bytes_max,
|frame| {
if let Frame::Commit { ref entries, .. } = frame {
outer_commits += 1;
let EntryContent::Meta { detail, .. } =
EntryContent::decode(&entries[1].content)?
else {
panic!("outer mutation evidence");
};
let mutation: smith_core::metadata::ToolMutationMetadata =
detail.deserialize_as().unwrap();
assert_eq!(
mutation.mutation_state,
smith::error::MutationState::Applied
);
}
for entry in frame.entries() {
if let EntryContent::Meta { kind, detail } =
EntryContent::decode(&entry.content)?
{
let mut detail = serde_json::to_value(detail).unwrap();
if kind == smith_core::metadata::NESTED_INTENT_KIND
|| kind == smith_core::metadata::NESTED_COMPLETION_KIND
{
let field = if kind == smith_core::metadata::NESTED_INTENT_KIND {
"input"
} else {
"output"
};
detail[field] = serde_json::Value::String(journal.nested_payload(
entry.id,
smith::config::Config::default_valid().frame_bytes_max,
)?);
}
if kind == smith_core::metadata::NESTED_INTENT_KIND {
nested_intents.push(detail);
} else if kind == smith_core::metadata::NESTED_COMPLETION_KIND {
assert!(matches!(frame, Frame::NestedCommit { .. }));
nested_completions.push(detail);
}
}
}
Ok(())
},
)
.unwrap();
}
assert_eq!(outer_commits, 1);
assert_eq!(
nested_intents.len(),
if explicit_submission { 5 } else { 3 }
);
assert_eq!(nested_completions.len(), nested_intents.len());
for (index, (intent, completion)) in nested_intents.iter().zip(&nested_completions).enumerate()
{
assert_eq!(intent["index"], index);
assert_eq!(completion["index"], index);
assert_eq!(intent["call_id"], completion["call_id"]);
assert_eq!(intent["submission_id"], completion["submission_id"]);
assert_eq!(intent["component"]["source_id"], "example:http");
assert_eq!(completion["ok"], true);
}
if explicit_submission {
assert_eq!(
std::fs::read_to_string(package.join("nested-first.txt")).unwrap(),
"guest-http-secret"
);
assert_eq!(
std::fs::read_to_string(package.join("nested-second.txt")).unwrap(),
"guest-http-secret"
);
assert_eq!(
nested_intents[0]["component"]["component_sha256"]
.as_str()
.unwrap()
.len(),
64
);
assert_eq!(nested_intents[0]["operation"], "secret-exchange");
assert_eq!(
nested_completions[0]["output"],
"secret-ref:connection-api-key:0"
);
assert_eq!(nested_intents[1]["operation"], "write");
assert_eq!(nested_intents[2]["operation"], "write");
assert_eq!(nested_intents[3]["operation"], "log");
assert_eq!(nested_intents[4]["operation"], "http");
let request: serde_json::Value =
serde_json::from_str(nested_intents[4]["input"].as_str().unwrap()).unwrap();
assert_eq!(request["body_hex"], "3c72656461637465643e".repeat(5000));
assert_eq!(request["response_bytes_max"], 1048576);
let response: serde_json::Value =
serde_json::from_str(nested_completions[4]["output"].as_str().unwrap()).unwrap();
assert_eq!(response["status"], 201);
assert_eq!(response["body_hex"], "3c72656461637465643e".repeat(5000));
assert_eq!(response["error"], "");
return;
}
assert_eq!(nested_intents[0]["operation"], "secret-exchange");
assert_eq!(nested_intents[1]["operation"], "log");
assert_eq!(nested_intents[2]["operation"], "http");
assert_eq!(nested_completions[2]["ok"], true);
let request: serde_json::Value =
serde_json::from_str(nested_intents[2]["input"].as_str().unwrap()).unwrap();
assert_eq!(request["headers"][0]["value"], "<redacted>");
assert_eq!(request["body_hex"], "67756573742d626f6479");
assert_eq!(request["response_bytes_max"], 64);
let response: serde_json::Value =
serde_json::from_str(nested_completions[2]["output"].as_str().unwrap()).unwrap();
assert_eq!(response["status"], 201);
assert_eq!(response["body_hex"], "6f6b");
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn eval_plugin_secret_source_and_origin_denials_never_reach_guest_http_server() {
use std::net::TcpListener;
let package = TestDir::vacant("smith-plugin-test-http-secret-denial");
Command::cargo_bin("smith")
.unwrap()
.args(["plugin", "init"])
.arg(&package)
.args(["--name", "example:http-secret-denial"])
.assert()
.success();
let origin = TcpListener::bind("127.0.0.1:0").unwrap();
let destination = format!("http://{}", origin.local_addr().unwrap());
std::fs::write(
package.join("plugin.kdl"),
"id \"example:http-denial\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"target/wasm32-wasip2/debug/smith_plugin_example_http_secret_denial.wasm\"\ncapability \"net-scope\"\ncapability \"secret-proxy\"\ncapability \"log\"\n",
)
.unwrap();
std::fs::write(
package.join("src/lib.rs"),
r##"#![expect(missing_docs, reason = "wit-bindgen generates the world surface")]
#![expect(clippy::pedantic, reason = "wit-bindgen generates the world surface")]
wit_bindgen::generate!({ path: "wit/smith-plugin.wit", world: "plugin-world" });
use exports::smith::plugin::plugin::{Guest, ToolDeclaration};
use exports::smith::plugin::provider::{CatalogSnapshot, DecodeStep, Guest as ProviderGuest, ProviderRequest, ProviderSettings, RequestTemplate, SettingsProvenance};
use exports::smith::plugin::registration::{Guest as RegistrationGuest, Registrations};
use smith::plugin::net_scope::{self, Header, HttpRequest, RequestOptions};
use smith::plugin::secret_proxy::{self, CredentialExchange, SecretRef};
struct Plugin;
impl RegistrationGuest for Plugin { fn declare() -> Registrations { Registrations { credential_sources: Vec::new(), providers: Vec::new(), models: Vec::new() } } }
impl ProviderGuest for Plugin {
fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: false, subscription: false, provenance: SettingsProvenance::BuiltIn } }
fn validate_settings(_: ProviderSettings) -> Result<ProviderSettings, String> { Err("no codec".to_string()) }
fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result<RequestTemplate, String> { Err("no codec".to_string()) }
fn decode_response(_: Vec<u8>, _: Vec<u8>, _: bool) -> Result<DecodeStep, String> { Err("no codec".to_string()) }
}
impl Guest for Plugin {
fn declare_tools() -> Vec<ToolDeclaration> { vec![ToolDeclaration { name: "round".to_string(), description: "Secret-gated HTTP.".to_string(), input_schema_json: r#"{"type":"object"}"#.to_string() }] }
fn invoke(_: String, _: String) -> Result<String, String> {
let destination = "__DESTINATION__".to_string();
let secret = secret_proxy::exchange(&CredentialExchange { source: SecretRef { id: "connection-api-key".to_string() }, audience: destination.clone(), scopes: vec!["http-request".to_string()] })?;
let response = net_scope::request(&HttpRequest { destination, method: "POST".to_string(), headers: vec![Header { name: "Authorization".to_string(), value: format!("{{{{secret:{}}}}}", secret.id) }], body: b"guest-body".to_vec(), options: RequestOptions { timeout_ms: 1000, response_bytes_max: 64 } })?;
Ok(format!(r#"{{"status":{}}}"#, response.status))
}
}
export!(Plugin);
"##
.replace("__DESTINATION__", &destination),
)
.unwrap();
let root = package.join("build-xdg");
let workspace_target = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.join("target");
let build = Command::cargo_bin("smith")
.unwrap()
.env("CARGO_TARGET_DIR", &workspace_target)
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.args(["plugin", "build"])
.arg(&package)
.output()
.unwrap();
assert!(
build.status.success(),
"{}",
String::from_utf8_lossy(&build.stderr)
);
let artifact =
workspace_target.join("wasm32-wasip2/debug/smith_plugin_example_http_secret_denial.wasm");
let package_artifact =
package.join("target/wasm32-wasip2/debug/smith_plugin_example_http_secret_denial.wasm");
std::fs::create_dir_all(package_artifact.parent().unwrap()).unwrap();
std::fs::copy(artifact, package_artifact).unwrap();
for (case, source_grant) in [("missing-source", false), ("wrong-origin", true)] {
let provider = TcpListener::bind("127.0.0.1:0").unwrap();
let provider_address = provider.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut initial, _) = provider.accept().unwrap();
initial
.set_read_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
let request = read_http_json(&mut initial);
assert!(
request["tools"]
.as_array()
.unwrap()
.iter()
.any(|tool| tool["function"]["name"] == "example:http-denial/round")
);
send_sse(
&mut initial,
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"denied-call\",\"function\":{\"name\":\"example:http-denial/round\",\"arguments\":\"{}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
);
let (mut followup, _) = provider.accept().unwrap();
followup
.set_read_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
let request = read_http_json(&mut followup);
assert!(
request["messages"]
.as_array()
.unwrap()
.iter()
.any(|message| message["role"] == "tool")
);
send_sse(
&mut followup,
"data: {\"choices\":[{\"delta\":{\"content\":\"denied safely\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
});
let xdg = package.join(format!("xdg-{case}"));
let mut install = Command::cargo_bin("smith").unwrap();
install
.env("XDG_CONFIG_HOME", xdg.join("config"))
.env("XDG_DATA_HOME", xdg.join("data"))
.env("XDG_STATE_HOME", xdg.join("state"))
.env("XDG_CACHE_HOME", xdg.join("cache"))
.args(["plugin", "install"])
.arg(&package)
.args([
"--grant",
"net-scope",
"--net-origin",
&destination,
"--grant",
"secret-proxy",
"--grant",
"log",
]);
if source_grant {
install.args([
"--secret-source",
"connection-api-key",
"--secret-provider",
"openai",
"--secret-origin",
"http://127.0.0.1:1",
"--secret-scope",
"http-request",
"--net-origin",
"http://127.0.0.1:1",
]);
}
let install = install.output().unwrap();
assert!(
install.status.success(),
"{}",
String::from_utf8_lossy(&install.stderr)
);
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", xdg.join("config"))
.env("XDG_DATA_HOME", xdg.join("data"))
.env("XDG_STATE_HOME", xdg.join("state"))
.env("XDG_CACHE_HOME", xdg.join("cache"))
.env("OPENAI_API_KEY", "provider-secret")
.args([
"eval",
"use http",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{provider_address}"),
])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"denied safely\n");
server.join().unwrap();
origin.set_nonblocking(true).unwrap();
assert_eq!(
origin.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock,
"{case}: denied secret exchange must prevent origin request"
);
}
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn plugin_reload_pins_active_tool_batch_then_publishes_the_next_generation() {
use std::io::Write;
use std::net::TcpListener;
use std::sync::mpsc;
use std::time::Duration;
let package = initialized_package("reload-live");
let source = package.join("src/lib.rs");
replace_fixture_output(&source, "one");
let root = package.join("xdg");
let runtime = std::path::PathBuf::from("/var/tmp/smith-session-e2e/final-gate-fixes")
.join(format!("reload-runtime-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&runtime);
std::fs::create_dir_all(&runtime).unwrap();
#[cfg(unix)]
std::fs::set_permissions(
&runtime,
std::os::unix::fs::PermissionsExt::from_mode(0o700),
)
.unwrap();
let run = |arguments: &[&str]| {
Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("XDG_RUNTIME_DIR", &runtime)
.args(arguments)
.output()
.unwrap()
};
assert!(
run(&["plugin", "build", package.to_str().unwrap()])
.status
.success()
);
assert!(
run(&["plugin", "link", package.to_str().unwrap()])
.status
.success()
);
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let (first_request_sent, first_request_received) = mpsc::sync_channel(1);
let (first_release_sent, first_release_received) = mpsc::sync_channel(1);
let (server_stage_sent, server_stage_received) = mpsc::sync_channel(2);
let server = std::thread::spawn(move || {
let (mut first, _) = listener.accept().unwrap();
let request = read_http_json(&mut first);
assert!(
request["tools"]
.as_array()
.unwrap()
.iter()
.any(|tool| tool["function"]["name"] == "example:reload-live/fixture")
);
let initial = "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"one\",\"function\":{\"name\":\"example:reload-live/fixture\",\"arguments\":\"{}\"}}]}}]}\n\n";
let finished = "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n";
write!(
first,
"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{initial}",
initial.len() + finished.len()
)
.unwrap();
first.flush().unwrap();
first_request_sent.send(()).unwrap();
first_release_received
.recv_timeout(Duration::from_secs(30))
.unwrap();
first.write_all(finished.as_bytes()).unwrap();
let (mut second, _) = listener.accept().unwrap();
let request = read_http_json(&mut second);
server_stage_sent.send("second").unwrap();
assert!(
request["messages"]
.as_array()
.unwrap()
.iter()
.any(|message| message["role"] == "tool" && message["content"] == "one")
);
send_sse(
&mut second,
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"two\",\"function\":{\"name\":\"example:reload-live/fixture\",\"arguments\":\"{}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
);
let (mut third, _) = listener.accept().unwrap();
let request = read_http_json(&mut third);
server_stage_sent.send("third").unwrap();
assert!(
request["messages"]
.as_array()
.unwrap()
.iter()
.any(|message| message["role"] == "tool" && message["content"] == "two")
);
send_sse(
&mut third,
"data: {\"choices\":[{\"delta\":{\"content\":\"reloaded\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
});
let mut child = TestChild(
std::process::Command::new(env!("CARGO_BIN_EXE_smith"))
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("XDG_RUNTIME_DIR", &runtime)
.env("OPENAI_API_KEY", "runtime-plugin-secret")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.args([
"eval",
"reload",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{address}"),
])
.spawn()
.unwrap(),
);
if let Err(error) = first_request_received.recv_timeout(Duration::from_secs(30)) {
let output = child.kill_with_output();
panic!(
"waiting for first provider request: {error}; status={}; stderr={}",
output.status,
String::from_utf8_lossy(&output.stderr)
);
}
let artifact = package.join("target/wasm32-wasip2/debug/smith_plugin_example_reload_live.wasm");
std::fs::write(&artifact, b"broken").unwrap();
let failed = run(&["plugin", "reload"]);
assert!(!failed.status.success());
replace_fixture_output(&source, "two");
assert!(
run(&["plugin", "build", package.to_str().unwrap()])
.status
.success()
);
let published = run(&["plugin", "reload"]);
assert!(
published.status.success(),
"{}",
String::from_utf8_lossy(&published.stderr)
);
first_release_sent.send(()).unwrap();
assert_eq!(
server_stage_received
.recv_timeout(Duration::from_secs(30))
.unwrap(),
"second"
);
assert_eq!(
server_stage_received
.recv_timeout(Duration::from_secs(30))
.unwrap(),
"third"
);
let output = child.wait_with_output();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"reloaded\n");
server.join().unwrap();
std::fs::remove_dir_all(&runtime).unwrap();
std::fs::remove_dir_all(package).unwrap();
}
fn replace_fixture_output(source: &Path, output: &str) {
let original = std::fs::read_to_string(source).unwrap();
let result = format!("Ok(\"{output}\".to_string())");
let fixture = if original.contains("Ok(input_json)") {
original.replacen("Ok(input_json)", &result, 1)
} else {
original.replacen("Ok(\"one\".to_string())", &result, 1)
};
assert_ne!(fixture, original, "generated fixture shape changed");
std::fs::write(source, fixture).unwrap();
}
#[test]
fn plugin_check_rejects_process_capability_when_only_filesystem_is_granted() {
let package = fixture_package(
"process-without-grant",
"id \"example:process\"\nversion \"1\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"process\"",
);
let output = smith(&package, &["workspace"]);
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(stderr.contains("missing grant process"), "{stderr}");
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn plugin_check_discloses_active_process_authority() {
let package = package(
"process-with-grant",
"id \"example:process\"\nversion \"1\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"process\"",
&echo_artifact(),
);
let output = smith(&package, &["process"]);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(
output.stdout,
b"example:process: check passed\nprocess grant active: commands inherit host-user filesystem and network access\n"
);
std::fs::remove_dir_all(package).unwrap();
}
const CATALOG_LISTING_FIXTURE: &str = r#"{
"openai": {"models": {"shared": {"id": "shared", "attachment": false, "reasoning": false, "tool_call": true, "structured_output": false, "limit": {"context": 8, "output": 4}}}},
"zai": {"models": {"shared": {"id": "shared", "attachment": false, "reasoning": false, "tool_call": true, "structured_output": false, "limit": {"context": 16, "output": 8}}}},
"unsupported": {"models": {"hidden": {"id": "hidden", "attachment": false, "reasoning": false, "tool_call": true, "structured_output": false, "limit": {"context": 8, "output": 4}}}}
}"#;
fn catalog_cache(source: &str) -> String {
serde_json::json!({"revision": 7, "source": source}).to_string()
}
#[test]
fn catalog_show_lists_connection_models_without_colliding_provider_ids() {
let root = TestDir::new("smith-catalog-listing-e2e");
let cache = root.join("catalog.json");
std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap();
let output = Command::cargo_bin("smith")
.unwrap()
.args(["catalog", "show", "--cache"])
.arg(&cache)
.output()
.unwrap();
assert!(output.status.success(), "{output:?}");
insta::assert_snapshot!(String::from_utf8(output.stdout).unwrap());
}
#[test]
fn catalog_show_json_filters_zai_coding_plan_connection() {
let root = TestDir::new("smith-catalog-json-listing-e2e");
let cache = root.join("catalog.json");
std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap();
let output = Command::cargo_bin("smith")
.unwrap()
.args([
"catalog",
"show",
"--cache",
cache.to_str().unwrap(),
"--provider",
"zai-coding-plan",
"--json",
])
.output()
.unwrap();
assert!(output.status.success(), "{output:?}");
let listing: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
insta::assert_snapshot!(serde_json::to_string_pretty(&listing).unwrap());
}
#[test]
fn catalog_json_provider_names_select_the_reported_connection() {
let root = TestDir::new("smith-catalog-selectable-provider-e2e");
let cache = root.join("catalog.json");
std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap();
let output = Command::cargo_bin("smith")
.unwrap()
.args(["catalog", "show", "--cache"])
.arg(&cache)
.arg("--json")
.output()
.unwrap();
assert!(output.status.success(), "{output:?}");
let listing: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
for provider in listing["providers"].as_array().unwrap() {
let name = provider["provider"].as_str().unwrap();
let connection = provider["connection"].as_str().unwrap();
let selected = Command::cargo_bin("smith")
.unwrap()
.args(["catalog", "show", "--cache"])
.arg(&cache)
.args(["--provider", name, "--json"])
.output()
.unwrap();
assert!(selected.status.success(), "{name}: {selected:?}");
let selected: serde_json::Value = serde_json::from_slice(&selected.stdout).unwrap();
assert_eq!(selected["providers"].as_array().unwrap().len(), 1, "{name}");
assert_eq!(selected["providers"][0]["provider"], name, "{name}");
assert_eq!(selected["providers"][0]["connection"], connection, "{name}");
assert_eq!(
selected["providers"][0]["models"], provider["models"],
"{name}"
);
}
}
#[test]
fn catalog_show_json_is_byte_identical_for_repeated_offline_listings() {
let root = TestDir::new("smith-catalog-deterministic-json-e2e");
let cache = root.join("catalog.json");
std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap();
let run = || {
Command::cargo_bin("smith")
.unwrap()
.args(["catalog", "show", "--cache"])
.arg(&cache)
.arg("--json")
.output()
.unwrap()
};
let first = run();
let second = run();
assert!(first.status.success(), "{first:?}");
assert!(second.status.success(), "{second:?}");
assert_eq!(first.stdout, second.stdout);
}
#[test]
fn catalog_show_escapes_model_control_characters_for_humans() {
let root = TestDir::new("smith-catalog-control-e2e");
let cache = root.join("catalog.json");
let source = r#"{
"zai": {"models": {"unsafe": {
"id": "\u001b[31munsafe", "limit": {"context": 8, "output": 4}
}}}
}"#;
std::fs::write(&cache, catalog_cache(source)).unwrap();
let output = Command::cargo_bin("smith")
.unwrap()
.args(["catalog", "show", "--cache"])
.arg(&cache)
.args(["--provider", "zai"])
.output()
.unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(stdout.contains(r#"\u{1b}[31munsafe"#));
assert!(!stdout.contains('\u{1b}'));
}
#[test]
fn catalog_show_distinguishes_an_empty_valid_connection_from_an_invalid_filter() {
let root = TestDir::new("smith-catalog-empty-filter-e2e");
let cache = root.join("catalog.json");
std::fs::write(&cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap();
let empty = Command::cargo_bin("smith")
.unwrap()
.args(["catalog", "show", "--cache"])
.arg(&cache)
.args(["--provider", "anthropic", "--json"])
.output()
.unwrap();
assert!(empty.status.success());
let listing: serde_json::Value = serde_json::from_slice(&empty.stdout).unwrap();
assert_eq!(listing["providers"][0]["connection"], "anthropic-api");
assert_eq!(listing["providers"][0]["models"], serde_json::json!([]));
let invalid = Command::cargo_bin("smith")
.unwrap()
.args(["catalog", "show", "--provider", "not-a-provider"])
.output()
.unwrap();
assert_eq!(invalid.status.code(), Some(2));
assert!(
String::from_utf8(invalid.stderr)
.unwrap()
.contains("possible values"),
);
}
#[test]
fn catalog_status_cli_cache_overrides_environment_cache() {
let root = TestDir::new("smith-catalog-cache-precedence-e2e");
let environment_cache = root.join("environment.json");
let cli_cache = root.join("cli.json");
std::fs::write(&environment_cache, catalog_cache(CATALOG_LISTING_FIXTURE)).unwrap();
std::fs::write(
&cli_cache,
catalog_cache(CATALOG_LISTING_FIXTURE).replace("\"revision\":7", "\"revision\":8"),
)
.unwrap();
let output = Command::cargo_bin("smith")
.unwrap()
.env("SMITH_CATALOG_CACHE", &environment_cache)
.args(["catalog", "status", "--cache"])
.arg(&cli_cache)
.output()
.unwrap();
assert!(output.status.success());
assert!(
String::from_utf8(output.stdout)
.unwrap()
.contains("generation=8")
);
}
#[test]
fn catalog_show_and_status_do_not_resolve_credentials_or_proxy_offline() {
use std::net::TcpListener;
let root = TestDir::new("smith-catalog-offline-e2e");
let data = root.join("data");
std::fs::create_dir_all(data.join("smith")).unwrap();
std::fs::write(data.join("smith/auth"), "poisoned auth path").unwrap();
let proxy = TcpListener::bind("127.0.0.1:0").unwrap();
proxy.set_nonblocking(true).unwrap();
let proxy_url = format!("http://{}", proxy.local_addr().unwrap());
let run = |arguments: &[&str]| {
Command::cargo_bin("smith")
.unwrap()
.env("HOME", root.join("home"))
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("HTTP_PROXY", &proxy_url)
.env("HTTPS_PROXY", &proxy_url)
.env_remove("SMITH_CATALOG_CACHE")
.env_remove("OPENAI_API_KEY")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("ZAI_API_KEY")
.args(arguments)
.output()
.unwrap()
};
let show = run(&["catalog", "show", "--provider", "zai-coding-plan", "--json"]);
let status = run(&["catalog", "status"]);
assert!(show.status.success(), "{show:?}");
assert!(status.status.success(), "{status:?}");
assert!(
String::from_utf8(show.stdout)
.unwrap()
.contains("zai-coding-plan"),
"show must use only offline metadata"
);
assert!(
String::from_utf8(status.stdout)
.unwrap()
.starts_with("catalog: bundled; generation=0;"),
"status must use the bundled offline catalog"
);
assert_eq!(
proxy.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock,
"catalog inspection must not make proxy requests"
);
}
#[test]
fn catalog_show_uses_bundled_snapshot_offline() {
let status = std::process::Command::new(env!("CARGO"))
.args(["build", "--locked", "-p", "smith-cli", "--bin", "smith"])
.status()
.unwrap();
assert!(status.success(), "smith binary build failed");
let root = TestDir::new("smith-catalog-e2e");
let binary = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.join("target/debug")
.join(if cfg!(windows) { "smith.exe" } else { "smith" });
assert!(
binary.is_file(),
"missing built smith binary: {}",
binary.display()
);
let binary = binary.canonicalize().unwrap();
let mut command = Command::new(binary);
command
.env("XDG_CACHE_HOME", root.join("cache"))
.env_remove("SMITH_CATALOG_CACHE")
.args(["catalog", "status"]);
let output = command.output().unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
assert!(stdout.starts_with("catalog: bundled; generation=0; models="));
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn catalog_status_reports_bundled_for_an_invalid_cache_file() {
let root = TestDir::new("smith-catalog-invalid-e2e");
let cache = root.join("catalog.json");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(&cache, "not a catalog").unwrap();
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CACHE_HOME", root.join("cache"))
.args(["catalog", "status", "--cache"])
.arg(&cache)
.output()
.unwrap();
assert!(output.status.success());
assert!(
String::from_utf8(output.stdout)
.unwrap()
.starts_with("catalog: bundled; generation=0;")
);
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn auth_zai_coding_plan_stores_and_checks_its_own_authority() {
let root = TestDir::new("smith-zai-auth");
let home = root.join("data");
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &home)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.args(["auth", "add", "--provider", "zai-coding-plan"])
.write_stdin("plan-secret\n")
.output()
.unwrap();
assert!(output.status.success());
assert_eq!(output.stdout, b"zai-coding-plan: stored\n");
assert!(home.join("smith/auth/zai-coding-plan").is_file());
assert!(!home.join("smith/auth/plugin").exists());
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &home)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.args(["auth", "check", "--provider", "zai-coding-plan"])
.output()
.unwrap();
assert!(output.status.success());
assert_eq!(output.stdout, b"zai-coding-plan: store\n");
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn eval_zai_component_uses_official_environment_credential_redacts_it_and_records_effects() {
use std::io::{Read, Write};
use std::net::TcpListener;
let root = TestDir::new("smith-zai-e2e");
let data = root.join("data");
let secret = "zai-e2e-secret";
std::fs::create_dir_all(&data).unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut request = [0_u8; 4096];
let bytes = stream.read(&mut request).unwrap();
let request = std::str::from_utf8(&request[..bytes]).unwrap();
assert!(
request
.to_ascii_lowercase()
.contains("authorization: bearer zai-e2e-secret")
);
assert!(request.contains("POST /api/paas/v4/chat/completions"));
let body = b"data: {\"choices\":[{\"delta\":{\"content\":\"guest accepted\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
stream
.write_all(
format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.as_bytes(),
)
.unwrap();
stream.write_all(body).unwrap();
});
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("ZAI_API_KEY", secret)
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("OPENAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env_remove("SMITH_ZAI_PLUGIN")
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args([
"eval",
"hello",
"--provider",
"zai",
"--model",
"glm-5.3",
"--base-url",
&format!("http://{address}"),
])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
server.join().unwrap();
assert_eq!(output.stdout, b"guest accepted\n");
assert!(!String::from_utf8_lossy(&output.stderr).contains(secret));
let sessions = std::fs::read_dir(data.join("smith/sessions/run")).unwrap();
let mut provenances = Vec::new();
for entry in sessions {
let path = entry.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
let bytes = std::fs::read(&path).unwrap();
assert!(!String::from_utf8_lossy(&bytes).contains(secret));
assert!(String::from_utf8_lossy(&bytes).contains("<redacted>"));
SessionReader::open(&path)
.unwrap()
.scan(
smith::config::Config::default_valid().frame_bytes_max,
|frame| {
if let Frame::Known { entry } = frame
&& let EntryContent::Meta { kind, detail } =
EntryContent::decode(&entry.content)?
&& kind == PROVIDER_DISPATCH_PROVENANCE_KIND
{
provenances.push(serde_json::to_value(detail).unwrap());
}
Ok(())
},
)
.unwrap();
}
assert_eq!(
provenances,
vec![serde_json::json!({
"connection_id": "zai-api",
"protocol_manifest_id": "smith:protocol-openai-compatible",
"protocol_component_sha256": smith_harness::protocol::embedded_openai_compatible_component_sha256(),
"catalog_generation": 0,
"catalog_source": "bundled",
"settings": {"include_usage": true, "provenance": "built_in"},
})]
);
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn eval_vendor_protocol_components_use_loopback_credentials_and_redact_durable_effects() {
use std::io::{Read, Write};
use std::net::TcpListener;
let cases = [
(
"anthropic",
"claude-sonnet-4-5",
"ANTHROPIC_API_KEY",
"anthropic-e2e-secret",
"/v1/messages",
"x-api-key: anthropic-e2e-secret",
b"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"anthropic guest\"}}\n\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\ndata: {\"type\":\"message_stop\"}\n\n".as_slice(),
"anthropic guest\n",
),
(
"google",
"gemini-flash-latest",
"GOOGLE_API_KEY",
"google-e2e-secret",
"/v1beta/models/gemini-flash-latest:streamgeneratecontent?alt=sse",
"x-goog-api-key: google-e2e-secret",
b"data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"google guest\"}]},\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":1,\"candidatesTokenCount\":1}}\n\n".as_slice(),
"google guest\n",
),
];
for (
provider,
model,
credential_name,
secret,
request_path,
credential_header,
body,
expected,
) in cases
{
let root = TestDir::new(&format!("smith-{provider}-component-e2e"));
let data = root.join("data");
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut request = [0_u8; 4096];
let bytes = stream.read(&mut request).unwrap();
let request = std::str::from_utf8(&request[..bytes])
.unwrap()
.to_ascii_lowercase();
assert!(request.contains(&format!("post {request_path}")));
assert!(request.contains(credential_header));
stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes()).unwrap();
stream.write_all(body).unwrap();
});
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env(credential_name, secret)
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("OPENAI_API_KEY")
.args([
"eval",
"hello",
"--provider",
provider,
"--model",
model,
"--base-url",
&format!("http://{address}"),
])
.env(credential_name, secret)
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
server.join().unwrap();
assert_eq!(output.stdout, expected.as_bytes());
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!stderr.contains(secret));
let mut effect_recorded = false;
let mut provenances = Vec::new();
for entry in std::fs::read_dir(data.join("smith/sessions/run")).unwrap() {
let path = entry.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
let bytes = std::fs::read(&path).unwrap();
let session = String::from_utf8_lossy(&bytes);
assert!(!session.contains(secret));
effect_recorded |= session.contains("<redacted>");
SessionReader::open(&path)
.unwrap()
.scan(
smith::config::Config::default_valid().frame_bytes_max,
|frame| {
if let Frame::Known { entry } = frame
&& let EntryContent::Meta { kind, detail } =
EntryContent::decode(&entry.content)?
&& kind == PROVIDER_DISPATCH_PROVENANCE_KIND
{
provenances.push(serde_json::to_value(detail).unwrap());
}
Ok(())
},
)
.unwrap();
}
assert!(
effect_recorded,
"{provider} intent must be durably redacted"
);
let (connection_id, protocol_manifest_id, protocol_component_sha256) = match provider {
"anthropic" => (
"anthropic-api",
"smith:protocol-anthropic",
smith_harness::protocol::embedded_anthropic_component_sha256(),
),
"google" => (
"google-api",
"smith:protocol-google",
smith_harness::protocol::embedded_google_component_sha256(),
),
_ => unreachable!("unknown test provider {provider}"),
};
assert_eq!(
provenances,
vec![serde_json::json!({
"connection_id": connection_id,
"protocol_manifest_id": protocol_manifest_id,
"protocol_component_sha256": protocol_component_sha256,
"catalog_generation": 0,
"catalog_source": "bundled",
"settings": {"include_usage": true, "provenance": "built_in"},
})],
"{provider} must retain the selected protocol and pinned catalog"
);
std::fs::remove_dir_all(root).unwrap();
}
}
const E2E_SOCKET_DEADLINE: std::time::Duration = std::time::Duration::from_secs(10);
const E2E_REQUEST_BYTES_MAX: usize = 1024 * 1024;
fn read_http_json(stream: &mut std::net::TcpStream) -> serde_json::Value {
use std::io::Read;
stream.set_read_timeout(Some(E2E_SOCKET_DEADLINE)).unwrap();
let mut request = Vec::new();
let mut buffer = [0_u8; 4096];
let header_end = loop {
let count = stream.read(&mut buffer).unwrap();
assert!(count > 0 && request.len() + count <= E2E_REQUEST_BYTES_MAX);
request.extend_from_slice(&buffer[..count]);
if let Some(position) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
break position + 4;
}
};
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
let content_length = headers
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length: ")
.and_then(|value| value.trim().parse::<usize>().ok())
})
.unwrap();
assert!(header_end + content_length <= E2E_REQUEST_BYTES_MAX);
while request.len() < header_end + content_length {
let count = stream.read(&mut buffer).unwrap();
assert!(count > 0 && request.len() + count <= E2E_REQUEST_BYTES_MAX);
request.extend_from_slice(&buffer[..count]);
}
serde_json::from_slice(&request[header_end..header_end + content_length]).unwrap()
}
fn send_sse(stream: &mut std::net::TcpStream, body: &str) {
use std::io::Write;
write!(
stream,
"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
)
.unwrap();
}
fn eval_vendor_tool_round(provider: &'static str, model: &'static str, credential: &'static str) {
use std::net::TcpListener;
let root = TestDir::new(&format!("smith-{provider}-tool-round-e2e"));
let data = root.join("data");
let workdir = root.join("workspace");
std::fs::create_dir_all(&workdir).unwrap();
std::fs::write(workdir.join("read-fixture.txt"), "fixture contents\n").unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut first, _) = listener.accept().unwrap();
let first_request = read_http_json(&mut first);
let tools = match provider {
"openai" => &first_request["tools"],
"anthropic" => &first_request["tools"],
"google" => &first_request["tools"][0]["functionDeclarations"],
_ => unreachable!(),
};
assert!(
tools
.as_array()
.unwrap()
.iter()
.any(|tool| tool["name"] == "read" || tool["function"]["name"] == "read")
);
let first_response = match provider {
"openai" => {
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"openai-call-id\",\"function\":{\"name\":\"read\",\"arguments\":\"{\\\"path\\\":\\\"read-fixture.txt\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n"
}
"anthropic" => {
"data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"anthropic-call-id\",\"name\":\"read\"}}\n\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\":\\\"read-fixture.txt\\\"}\"}}\n\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n"
}
"google" => {
"data: {\"candidates\":[{\"content\":{\"parts\":[{\"functionCall\":{\"id\":\"google-call-id\",\"name\":\"read\",\"args\":{\"path\":\"read-fixture.txt\"}}}]},\"finishReason\":\"STOP\"}]}\n\n"
}
_ => unreachable!(),
};
send_sse(&mut first, first_response);
let (mut second, _) = listener.accept().unwrap();
let second_request = read_http_json(&mut second);
match provider {
"openai" => {
let messages = second_request["messages"].as_array().unwrap();
let assistant = messages
.iter()
.find(|message| message["tool_calls"].is_array())
.unwrap();
let tool = messages
.iter()
.find(|message| message["role"] == "tool")
.unwrap();
assert_eq!(assistant["tool_calls"][0]["id"], "openai-call-id");
assert_eq!(tool["tool_call_id"], "openai-call-id");
}
"anthropic" => {
let messages = second_request["messages"].as_array().unwrap();
let assistant = messages
.iter()
.find(|message| message["role"] == "assistant")
.unwrap();
let tool = messages
.iter()
.find(|message| {
message["role"] == "user"
&& message["content"][0]["tool_use_id"].is_string()
})
.unwrap();
assert_eq!(assistant["content"][0]["id"], "anthropic-call-id");
assert_eq!(tool["content"][0]["tool_use_id"], "anthropic-call-id");
}
"google" => {
let contents = second_request["contents"].as_array().unwrap();
let tool = contents
.iter()
.find(|content| content["parts"][0]["functionResponse"].is_object())
.unwrap();
assert_eq!(tool["parts"][0]["functionResponse"]["name"], "read");
}
_ => unreachable!(),
}
let final_response = match provider {
"openai" => {
"data: {\"choices\":[{\"delta\":{\"content\":\"openai tool complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"
}
"anthropic" => {
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"anthropic tool complete\"}}\n\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\ndata: {\"type\":\"message_stop\"}\n\n"
}
"google" => {
"data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"google tool complete\"}]},\"finishReason\":\"STOP\"}]}\n\n"
}
_ => unreachable!(),
};
send_sse(&mut second, final_response);
});
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.current_dir(&workdir)
.env(credential, format!("{provider}-tool-round-secret"))
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("OPENAI_API_KEY")
.env(credential, format!("{provider}-tool-round-secret"))
.args([
"eval",
"read fixture",
"--provider",
provider,
"--model",
model,
"--base-url",
&format!("http://{address}"),
])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
server.join().unwrap();
assert_eq!(
output.stdout,
format!("{provider} tool complete\n").as_bytes()
);
let secret = format!("{provider}-tool-round-secret");
for entry in std::fs::read_dir(data.join("smith/sessions/run")).unwrap() {
let path = entry.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
let bytes = std::fs::read(path).unwrap();
let record = String::from_utf8_lossy(&bytes);
assert!(!record.contains(&secret));
assert!(record.contains("<redacted>"));
}
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn eval_openai_component_preserves_vendor_tool_call_id_across_read_round() {
eval_vendor_tool_round("openai", "gpt-4.1", "OPENAI_API_KEY");
}
#[test]
fn eval_anthropic_component_preserves_vendor_tool_call_id_across_read_round() {
eval_vendor_tool_round("anthropic", "claude-sonnet-4-5", "ANTHROPIC_API_KEY");
}
#[test]
fn eval_google_component_matches_function_response_name_across_read_round() {
eval_vendor_tool_round("google", "gemini-flash-latest", "GOOGLE_API_KEY");
}
#[test]
fn eval_openai_resume_preserves_two_durable_vendor_tool_call_ids() {
use std::net::TcpListener;
let root = TestDir::new("smith-openai-resume-tool-ids-e2e");
let data = root.join("data");
let workdir = root.join("workspace");
std::fs::create_dir_all(&workdir).unwrap();
std::fs::write(workdir.join("fixture.txt"), "fixture\n").unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let tool_call = |id: &str| {
format!(
"data: {{\"choices\":[{{\"delta\":{{\"tool_calls\":[{{\"index\":0,\"id\":\"{id}\",\"function\":{{\"name\":\"read\",\"arguments\":\"{{\\\"path\\\":\\\"fixture.txt\\\"}}\"}}}}]}}}}]}}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n"
)
};
let final_response = "data: {\"choices\":[{\"delta\":{\"content\":\"done\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
for id in ["vendor-first", "vendor-second"] {
let (mut socket, _) = listener.accept().unwrap();
let _request = read_http_json(&mut socket);
send_sse(&mut socket, &tool_call(id));
}
for expected in ["vendor-first", "vendor-second"] {
let (mut socket, _) = listener.accept().unwrap();
let request = read_http_json(&mut socket);
let messages = request["messages"].as_array().unwrap();
for id in ["vendor-first", "vendor-second"] {
assert!(
messages
.iter()
.any(|message| message["tool_calls"][0]["id"] == id),
"missing assistant {id}: {messages:?}"
);
assert!(
messages.iter().any(|message| message["tool_call_id"] == id),
"missing result {id}: {messages:?}"
);
}
send_sse(&mut socket, final_response);
if expected == "vendor-first" {
continue;
}
}
});
let run = |session: Option<&str>, prompt: &str| {
let mut command = Command::cargo_bin("smith").unwrap();
command
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.current_dir(&workdir)
.env("OPENAI_API_KEY", "openai-resume-secret")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("GOOGLE_API_KEY")
.args([
"eval",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{address}"),
]);
if let Some(session) = session {
command.args(["--session", session]);
}
command.arg(prompt).output().unwrap()
};
let first = run(None, "read twice");
assert!(
first.status.success(),
"{}",
String::from_utf8_lossy(&first.stderr)
);
assert_eq!(first.stdout, b"done\n");
let session = std::fs::read_dir(data.join("smith/sessions/run"))
.unwrap()
.find_map(|entry| {
let path = entry.unwrap().path();
(path.extension().is_some_and(|extension| extension == "smh"))
.then(|| path.file_stem().unwrap().to_string_lossy().into_owned())
})
.unwrap();
let resumed = run(Some(&session), "continue");
assert!(
resumed.status.success(),
"{}",
String::from_utf8_lossy(&resumed.stderr)
);
assert_eq!(resumed.stdout, b"done\n");
server.join().unwrap();
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn eval_zai_rejects_another_vendor_environment_credential() {
let root = TestDir::new("smith-zai-wrong-env-e2e");
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("OPENAI_API_KEY", "wrong-vendor-secret")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env_remove("SMITH_ZAI_PLUGIN")
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args(["eval", "hello", "--provider", "zai", "--model", "glm-5.3"])
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!output.status.success());
assert!(stderr.contains("no credential for zai"));
assert!(stderr.contains("env ZAI_API_KEY"));
assert!(!stderr.contains("SMITH_PLUGIN_API_KEY"));
assert!(!stderr.contains("no credential for plugin"));
assert!(!stderr.contains("wrong-vendor-secret"));
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn eval_zai_coding_plan_uses_its_selected_route_with_official_credential() {
use std::io::{Read, Write};
use std::net::TcpListener;
let root = TestDir::new("smith-zai-coding-plan");
let secret = "zai-coding-plan-secret";
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut request = [0_u8; 4096];
let bytes = stream.read(&mut request).unwrap();
let request = std::str::from_utf8(&request[..bytes]).unwrap();
assert!(request.contains("POST /api/coding/paas/v4/chat/completions"));
assert!(
request
.to_ascii_lowercase()
.contains("authorization: bearer zai-coding-plan-secret")
);
let body = b"data: {\"choices\":[{\"delta\":{\"content\":\"coding plan\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
stream
.write_all(
format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.as_bytes(),
)
.unwrap();
stream.write_all(body).unwrap();
});
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("ZAI_API_KEY", secret)
.env_remove("SMITH_PLUGIN_API_KEY")
.env_remove("SMITH_ZAI_PLUGIN")
.args([
"eval",
"hello",
"--provider",
"zai-coding-plan",
"--model",
"glm-5.3",
"--base-url",
&format!("http://{address}"),
])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
server.join().unwrap();
assert_eq!(output.stdout, b"coding plan\n");
assert!(!String::from_utf8_lossy(&output.stderr).contains(secret));
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn eval_zai_bad_explicit_component_override_fails_without_embedded_fallback() {
let root = TestDir::new("smith-zai-override-e2e");
let data = root.join("data");
std::fs::create_dir_all(data.join("smith/auth")).unwrap();
std::fs::write(data.join("smith/auth/zai"), "test-credential").unwrap();
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", &data)
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("SMITH_ZAI_PLUGIN", root.join("missing.wasm"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args(["eval", "hello", "--provider", "zai", "--model", "glm-5.3"])
.output()
.unwrap();
assert!(!output.status.success());
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("validation failed"), "{stderr}");
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn plugin_check_reports_trust_grant_world_and_artifact_without_invoking_tools() {
let manifest =
"id \"example:fixture\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n";
let untrusted = fixture_package("untrusted", manifest);
let output = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", untrusted.join("xdg/config"))
.args(["plugin", "check"])
.arg(&untrusted)
.output()
.unwrap();
assert_eq!(output.status.code(), Some(1));
assert!(
String::from_utf8_lossy(&output.stderr).contains("trust the project"),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let grant = fixture_package(
"grant",
"id \"example:fixture\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"log\"\n",
);
let output = smith(&grant, &[]);
assert_eq!(output.status.code(), Some(1));
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("missing grant log"), "{stderr}");
let world = fixture_package(
"world",
"id \"example:fixture\"\nversion \"0.1.0\"\nworld \"9.0.0\"\nartifact \"fixture.wasm\"\n",
);
let output = smith(&world, &[]);
assert_eq!(output.status.code(), Some(1));
assert!(String::from_utf8_lossy(&output.stderr).contains("unsupported world version 9.0.0"));
let artifact = fixture_package("artifact", manifest);
let output = smith(&artifact, &["log"]);
assert_eq!(output.status.code(), Some(1));
assert!(String::from_utf8_lossy(&output.stderr).contains("validation failed"));
for package in [untrusted, grant, world, artifact] {
std::fs::remove_dir_all(package).unwrap();
}
}
#[test]
fn plugin_trust_persists_grants_for_a_later_check() {
let artifact = echo_artifact();
let package = package(
"persisted-trust",
"id \"example:echo\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"log\"\ncapability \"net-scope\"\n",
&artifact,
);
let config = package.join("xdg/config");
let trust = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", &config)
.args(["plugin", "trust"])
.arg(&package)
.args([
"--grant",
"log",
"--grant",
"net-scope",
"--net-origin",
"http://127.0.0.1:17777",
])
.output()
.unwrap();
assert!(
trust.status.success(),
"{}",
String::from_utf8_lossy(&trust.stderr)
);
let check = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", &config)
.args(["plugin", "check"])
.arg(&package)
.output()
.unwrap();
assert!(
check.status.success(),
"{}",
String::from_utf8_lossy(&check.stderr)
);
assert_eq!(check.stdout, b"example:echo: check passed\n");
let trust = std::fs::read_to_string(config.join("smith/plugins/project-trust.kdl")).unwrap();
assert!(trust.contains("net-origin \"http://127.0.0.1:17777\""));
let invalid = Command::cargo_bin("smith")
.unwrap()
.env("XDG_CONFIG_HOME", &config)
.args(["plugin", "trust"])
.arg(&package)
.args([
"--grant",
"net-scope",
"--net-origin",
"https://example.test/path",
])
.output()
.unwrap();
assert!(!invalid.status.success());
assert!(String::from_utf8_lossy(&invalid.stderr).contains("without path"));
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn plugin_check_loads_a_valid_component_without_invoking_a_declared_tool() {
let artifact = echo_artifact();
let package = package(
"valid",
"id \"example:echo\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\n",
&artifact,
);
let output = smith(&package, &[]);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"example:echo: check passed\n");
std::fs::remove_dir_all(package).unwrap();
}
#[test]
fn eval_openai_uses_trusted_protocol_dev_link_and_rejects_broken_replacement() {
let package = initialized_package("protocol-dev-link");
let root = TestDir::new("smith-protocol-dev-link");
let run = |args: &[&str]| {
Command::cargo_bin("smith")
.unwrap()
.timeout(std::time::Duration::from_secs(180))
.current_dir(&root)
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("OPENAI_API_KEY", "dev-link-secret")
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args(args)
.output()
.unwrap()
};
let source = std::fs::read_to_string(package.join("src/lib.rs")).unwrap();
let start = source.find("impl ProviderGuest for Plugin {").unwrap();
let end = source.find("impl Guest for Plugin {").unwrap();
let provider = r#"impl ProviderGuest for Plugin {
fn declare_settings() -> ProviderSettings { ProviderSettings { include_usage: true, subscription: false, provenance: SettingsProvenance::BuiltIn } }
fn validate_settings(settings: ProviderSettings) -> Result<ProviderSettings, String> { Ok(settings) }
fn build_request(_: String, _: ProviderRequest, _: ProviderSettings, _: CatalogSnapshot) -> Result<RequestTemplate, String> {
Ok(RequestTemplate { url: "https://protocol.invalid/chat/completions".into(), authorization: "Bearer {{secret:connection-api-key}}".into(), body: "{}".into() })
}
fn decode_response(_: Vec<u8>, _: Vec<u8>, finished: bool) -> Result<DecodeStep, String> {
use exports::smith::plugin::provider::{StreamEvent, StopReason};
Ok(DecodeStep { state: Vec::new(), events: if finished { vec![StreamEvent::Text("linked codec".into()), StreamEvent::Stop((StopReason::EndTurn, None))] } else { Vec::new() } })
}
}
"#;
let source = format!("{}{}{}", &source[..start], provider, &source[end..])
.replace("providers: Vec::new()", "providers: vec![exports::smith::plugin::registration::ProviderContribution { id: \"openai-compatible\".into(), protocol: exports::smith::plugin::registration::ProtocolBinding::OpenaiCompatible, credential_source_id: \"connection-api-key\".into() }]")
.replace("credential_sources: Vec::new()", "credential_sources: vec![exports::smith::plugin::registration::CredentialSource { id: \"connection-api-key\".into() }]")
.replace("vec![ToolDeclaration { name: \"fixture\".to_string(), description: \"Fixture echo tool.\".to_string(), input_schema_json: r#\"{\"type\":\"object\"}\"#.to_string() }]", "Vec::new()");
std::fs::write(package.join("src/lib.rs"), source).unwrap();
let manifest_path = package.join("plugin.kdl");
let manifest = std::fs::read_to_string(&manifest_path).unwrap();
std::fs::write(
&manifest_path,
format!("{manifest}\ncapability \"net-scope\"\ncapability \"secret-proxy\"\n"),
)
.unwrap();
let built = run(&["plugin", "build", package.to_str().unwrap()]);
assert!(
built.status.success(),
"{}",
String::from_utf8_lossy(&built.stderr)
);
let linked = run(&[
"plugin",
"link",
package.to_str().unwrap(),
"--grant",
"net-scope",
"--grant",
"secret-proxy",
]);
assert!(
linked.status.success(),
"{}",
String::from_utf8_lossy(&linked.stderr)
);
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut socket, _) = listener.accept().unwrap();
let _ = read_http_json(&mut socket);
send_sse(&mut socket, "data: [DONE]\n\n");
});
let endpoint = format!("http://{address}");
let args = [
"eval",
"hello",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&endpoint,
];
let output = run(&args);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"linked codec\n");
server.join().unwrap();
let artifact = std::fs::read_dir(package.join("target/wasm32-wasip2/debug"))
.unwrap()
.map(|entry| entry.unwrap().path())
.find(|path| path.extension().is_some_and(|ext| ext == "wasm"))
.unwrap();
std::fs::write(artifact, b"invalid component").unwrap();
let failed = run(&args);
assert!(!failed.status.success());
assert!(
String::from_utf8_lossy(&failed.stderr).contains("validation failed"),
"{}",
String::from_utf8_lossy(&failed.stderr)
);
assert!(!String::from_utf8_lossy(&failed.stderr).contains("dev-link-secret"));
}
#[test]
fn eval_explicit_connection_setting_replaces_user_value_and_records_source() {
let root = TestDir::new("smith-provider-settings-e2e");
std::fs::create_dir_all(root.join("config/smith")).unwrap();
std::fs::write(
root.join("config/smith/provider-settings.kdl"),
"include-usage #true",
)
.unwrap();
std::fs::create_dir(root.join(".smith")).unwrap();
// Untrusted project config must not even be parsed.
std::fs::write(root.join(".smith/provider-settings.kdl"), "not valid KDL").unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut socket, _) = listener.accept().unwrap();
let request = read_http_json(&mut socket);
assert_eq!(request["stream_options"]["include_usage"], false);
send_sse(
&mut socket,
"data: {\"choices\":[{\"delta\":{\"content\":\"configured\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
});
let output = Command::cargo_bin("smith")
.unwrap()
.timeout(std::time::Duration::from_secs(30))
.current_dir(&root)
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("OPENAI_API_KEY", "settings-secret")
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args([
"eval",
"hello",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{address}"),
"--include-usage",
"false",
])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"configured\n");
assert!(
String::from_utf8_lossy(&output.stderr)
.contains("untrusted project provider settings ignored")
);
server.join().unwrap();
assert_eq!(
provider_settings_records(&root),
vec![serde_json::json!({"include_usage":false, "provenance":"explicit"})]
);
}
#[test]
fn eval_trusted_project_setting_replaces_user_value_without_accepting_host_authority() {
let root = TestDir::new("smith-trusted-provider-settings");
std::fs::create_dir_all(root.join("config/smith")).unwrap();
std::fs::create_dir(root.join(".smith")).unwrap();
std::fs::write(
root.join("config/smith/provider-settings.kdl"),
"include-usage #false",
)
.unwrap();
let project_file = root.join(".smith/provider-settings.kdl");
std::fs::write(&project_file, "include-usage #true").unwrap();
let run = |args: &[&str]| {
Command::cargo_bin("smith")
.unwrap()
.timeout(std::time::Duration::from_secs(30))
.current_dir(&root)
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("OPENAI_API_KEY", "trusted-settings-secret")
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args(args)
.output()
.unwrap()
};
let trusted = run(&["plugin", "trust", root.to_str().unwrap()]);
assert!(
trusted.status.success(),
"{}",
String::from_utf8_lossy(&trusted.stderr)
);
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut socket, _) = listener.accept().unwrap();
let request = read_http_json(&mut socket);
assert_eq!(request["stream_options"]["include_usage"], true);
send_sse(
&mut socket,
"data: {\"choices\":[{\"delta\":{\"content\":\"project settings\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
});
let endpoint = format!("http://{address}");
let args = [
"eval",
"hello",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&endpoint,
];
let output = run(&args);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"project settings\n");
assert!(!String::from_utf8_lossy(&output.stderr).contains("untrusted project"));
server.join().unwrap();
assert_eq!(
provider_settings_records(&root),
vec![serde_json::json!({"include_usage":true,"provenance":"trusted_project"})]
);
for field in ["endpoint", "subscription", "grants", "unknown_setting"] {
std::fs::write(&project_file, field).unwrap();
let rejected = run(&args);
assert!(!rejected.status.success());
let diagnostic = String::from_utf8_lossy(&rejected.stderr);
assert!(
diagnostic.contains("unknown provider setting") && diagnostic.contains(field),
"{diagnostic}"
);
assert!(!diagnostic.contains("trusted-settings-secret"));
}
std::fs::write(project_file, "").unwrap();
let unsupported = run(&[
"eval",
"hello",
"--provider",
"google",
"--model",
"gemini-flash-latest",
"--base-url",
&endpoint,
"--include-usage",
"false",
]);
assert!(!unsupported.status.success());
assert!(
String::from_utf8_lossy(&unsupported.stderr).contains("does not support disabling usage")
);
}
fn provider_settings_records(root: &Path) -> Vec<serde_json::Value> {
let mut settings = Vec::new();
for entry in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() {
let path = entry.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
SessionReader::open(&path)
.unwrap()
.scan(
smith::config::Config::default_valid().frame_bytes_max,
|frame| {
if let Frame::Known { entry } = frame
&& let EntryContent::Meta { kind, detail } =
EntryContent::decode(&entry.content)?
&& kind == PROVIDER_DISPATCH_PROVENANCE_KIND
{
settings.push(serde_json::to_value(detail).unwrap()["settings"].clone());
}
Ok(())
},
)
.unwrap();
}
settings
}
#[test]
fn eval_component_tool_authority_precedes_guest_and_effects() {
use std::net::TcpListener;
let package = initialized_package("authority");
let mut source = std::fs::read_to_string(
Path::new(env!("CARGO_MANIFEST_DIR")).join("../plugins/echo/src/lib.rs"),
)
.unwrap();
source = source.replace("../../wit/smith-plugin.wit", "wit/smith-plugin.wit")
.replace("\"echo\"", "\"write\"")
.replace(r#"{"type":"object","properties":{},"required":[]}"#, r#"{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}"#)
.replace("Ok(input_json)", r#"smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite { path: "guest-entered.txt".into(), content: input_json.clone() })?; Ok(input_json)"#);
std::fs::write(package.join("src/lib.rs"), source).unwrap();
let manifest = std::fs::read_to_string(package.join("plugin.kdl")).unwrap();
std::fs::write(
package.join("plugin.kdl"),
format!("{manifest}\ncapability \"workspace\"\n"),
)
.unwrap();
let build = Command::cargo_bin("smith")
.unwrap()
.env("HOME", package.join("home"))
.env("XDG_CONFIG_HOME", package.join("config"))
.env("XDG_DATA_HOME", package.join("data"))
.env("XDG_STATE_HOME", package.join("state"))
.env("XDG_CACHE_HOME", package.join("cache"))
.args(["plugin", "build"])
.arg(&package)
.output()
.unwrap();
assert!(build.status.success(), "{build:?}");
for (state, name, arguments, code) in [
(
"advertised",
"example:authority/write",
r#"{"value":"accepted"}"#,
None,
),
(
"hidden",
"example:authority/write",
r#"{"value":"denied"}"#,
Some("TOOL_NOT_ADVERTISED"),
),
(
"disabled",
"example:authority/write",
r#"{"value":"denied"}"#,
Some("TOOL_DISABLED"),
),
(
"advertised",
"example:authority/write",
r#"{"value":17}"#,
Some("VALIDATION"),
),
(
"hidden",
"write",
r#"{"path":"bundled-component.txt","content":"component-owned"}"#,
None,
),
(
"advertised",
"missing:authority/write",
r#"{"path":"native-fallback.txt","content":"denied"}"#,
Some("TOOL_NOT_REGISTERED"),
),
] {
let root = TestDir::new("smith-tool-authority-case");
let command = || {
let mut command = Command::cargo_bin("smith").unwrap();
command
.timeout(std::time::Duration::from_secs(30))
.current_dir(&root)
.env("HOME", root.join("home"))
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("SMITH_CATALOG_CACHE")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env("OPENAI_API_KEY", "authority-fixture-secret");
command
};
let install = command()
.args(["plugin", "install"])
.arg(&package)
.args(["--grant", "workspace"])
.output()
.unwrap();
assert!(install.status.success(), "{install:?}");
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let request = read_http_json(&mut stream);
assert_eq!(
request["tools"]
.as_array()
.unwrap()
.iter()
.any(|tool| tool["function"]["name"] == "example:authority/write"),
state == "advertised"
);
let delta = serde_json::json!({"choices":[{"delta":{"tool_calls":[{"index":0,"id":"authority-call","function":{"name":name,"arguments":arguments}}]}}]});
send_sse(
&mut stream,
&format!(
"data: {delta}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n"
),
);
if code.is_none() {
let (mut stream, _) = listener.accept().unwrap();
let request = read_http_json(&mut stream);
let result = request["messages"]
.as_array()
.unwrap()
.iter()
.find(|message| message["role"] == "tool")
.unwrap();
if name == "example:authority/write" {
assert_eq!(result["content"], arguments);
} else {
assert!(!result["content"].as_str().unwrap().is_empty());
}
send_sse(
&mut stream,
"data: {\"choices\":[{\"delta\":{\"content\":\"authority complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
}
});
let output = command()
.args([
"eval",
"invoke fixture",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{address}"),
"--tool-policy",
&format!("example:authority/write={state}"),
])
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!stderr.contains("authority-fixture-secret"));
assert!(!String::from_utf8_lossy(&output.stdout).contains("authority-fixture-secret"));
if let Some(code) = code {
assert!(!output.status.success(), "{state}: {output:?}");
assert!(
stderr.contains(code) && stderr.contains(name),
"{state}: {stderr}"
);
assert!(!root.join("guest-entered.txt").exists());
assert!(!root.join("native-fallback.txt").exists());
} else {
assert!(output.status.success(), "{output:?}");
assert_eq!(output.stdout, b"authority complete\n");
if name == "example:authority/write" {
assert_eq!(
std::fs::read_to_string(root.join("guest-entered.txt")).unwrap(),
arguments
);
} else {
assert!(!root.join("guest-entered.txt").exists());
assert_eq!(
std::fs::read_to_string(root.join("bundled-component.txt")).unwrap(),
"component-owned"
);
}
}
server.join().unwrap();
let mut calls = 0;
let mut nested = 0;
for file in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() {
let path = file.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
let bytes = std::fs::read(&path).unwrap();
assert!(!String::from_utf8_lossy(&bytes).contains("authority-fixture-secret"));
SessionReader::open(&path)
.unwrap()
.scan(
smith::config::Config::default_valid().frame_bytes_max,
|frame| {
if let Frame::Known { entry } = frame {
match EntryContent::decode(&entry.content)? {
EntryContent::ToolCall { .. } => calls += 1,
EntryContent::Meta { kind, detail }
if kind == smith_core::metadata::NESTED_INTENT_KIND =>
{
nested += 1;
let detail = serde_json::to_value(detail).unwrap();
assert_eq!(
detail["component"]["source_id"],
if name == "write" {
"smith:coding-tools"
} else {
"example:authority"
}
);
}
_ => {}
}
}
Ok(())
},
)
.unwrap();
}
assert_eq!(calls, usize::from(code.is_none()));
assert_eq!(nested, usize::from(code.is_none()));
}
}
#[test]
fn eval_guest_trap_retains_completed_write_and_stops_remaining_tool_batch() {
use std::net::{TcpListener, TcpStream};
use std::time::Duration;
let package = initialized_package("terminal");
let source = std::fs::read_to_string(
Path::new(env!("CARGO_MANIFEST_DIR")).join("../plugins/echo/src/lib.rs"),
)
.unwrap()
.replace("../../wit/smith-plugin.wit", "wit/smith-plugin.wit")
.replace(
"Ok(input_json)",
r#"smith::plugin::tooling::write(&smith::plugin::tooling::FileWrite {
path: "completed.txt".into(), content: "retained".into()
})?; std::process::abort()"#,
);
std::fs::write(package.join("src/lib.rs"), source).unwrap();
let manifest = std::fs::read_to_string(package.join("plugin.kdl")).unwrap();
std::fs::write(
package.join("plugin.kdl"),
format!("{manifest}\ncapability \"workspace\"\n"),
)
.unwrap();
let root = TestDir::new("smith-terminal-guest");
let command = || {
let mut command = Command::cargo_bin("smith").unwrap();
command
.timeout(Duration::from_secs(60))
.current_dir(&root)
.env("HOME", root.join("home"))
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("SMITH_CATALOG_CACHE")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env("OPENAI_API_KEY", "terminal-fixture-secret");
command
};
command()
.args(["plugin", "build"])
.arg(&package)
.assert()
.success();
command()
.args(["plugin", "install"])
.arg(&package)
.args(["--grant", "workspace"])
.assert()
.success();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(60)))
.unwrap();
stream
.set_write_timeout(Some(Duration::from_secs(60)))
.unwrap();
let request = read_http_json(&mut stream);
assert!(
request["tools"]
.as_array()
.unwrap()
.iter()
.any(|tool| tool["function"]["name"] == "example:terminal/echo")
);
let delta = serde_json::json!({"choices":[{"delta":{"tool_calls":[
{"index":0,"id":"trap-call","function":{"name":"example:terminal/echo","arguments":"{}"}},
{"index":1,"id":"never-call","function":{"name":"write","arguments":"{\"path\":\"never.txt\",\"content\":\"denied\"}"}}
]}}]});
send_sse(
&mut stream,
&format!(
"data: {delta}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n"
),
);
});
let output = command()
.args([
"eval",
"trap after write",
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{address}"),
])
.output()
.unwrap();
// Unblock accept if CLI failed before provider admission; the stream deadline
// then bounds teardown. No polling or waiting for wall time.
let _unblock = TcpStream::connect_timeout(&address, Duration::from_secs(1));
server.join().unwrap();
assert!(!output.status.success(), "{output:?}");
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
stderr.contains("terminal host invocation stop: Sandbox"),
"{stderr}"
);
assert!(!stderr.contains("terminal-fixture-secret"));
assert_eq!(
std::fs::read_to_string(root.join("completed.txt")).unwrap(),
"retained"
);
assert!(!root.join("never.txt").exists());
let mut calls = 0;
let mut failed_results = 0;
let mut applied = 0;
for file in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() {
let path = file.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
SessionReader::open(&path)
.unwrap()
.scan(
smith::config::Config::default_valid().frame_bytes_max,
|frame| {
let mut inspect =
|entry: smith_core::session::EntryFrame| -> smith::Result<()> {
match EntryContent::decode(&entry.content)? {
EntryContent::ToolCall { .. } => calls += 1,
EntryContent::ToolResult { ok: false, .. } => failed_results += 1,
EntryContent::Meta { kind, detail }
if kind == smith_core::metadata::NESTED_COMPLETION_KIND =>
{
let detail = serde_json::to_value(detail).unwrap();
if detail["mutation_state"]
== serde_json::to_value(
smith::error::MutationState::Applied,
)
.unwrap()
{
applied += 1;
}
}
_ => {}
}
Ok(())
};
match frame {
Frame::Known { entry } | Frame::NestedCommit { entry } => inspect(entry)?,
Frame::Commit { entries, .. } => {
for entry in *entries {
inspect(entry)?;
}
}
_ => {}
}
Ok(())
},
)
.unwrap();
}
assert_eq!((calls, failed_results, applied), (1, 1, 1));
}
#[test]
fn eval_and_rpc_compose_hidden_target_with_durable_child_evidence() {
use std::io::{BufRead, Write};
use std::process::Stdio;
use std::time::Duration;
let source = initialized_package("composition-source");
std::fs::write(
source.join("src/lib.rs"),
include_str!("../../smith-harness/tests/fixtures/composition.rs")
.replace("WIT_PATH", "wit/smith-plugin.wit"),
)
.unwrap();
let manifest = std::fs::read_to_string(source.join("plugin.kdl")).unwrap();
std::fs::write(
source.join("plugin.kdl"),
format!("{manifest}\ncapability \"workspace\"\n"),
)
.unwrap();
let build = Command::cargo_bin("smith")
.unwrap()
.env("HOME", source.join("home"))
.env("XDG_CONFIG_HOME", source.join("config"))
.env("XDG_DATA_HOME", source.join("data"))
.env("XDG_STATE_HOME", source.join("state"))
.env("XDG_CACHE_HOME", source.join("cache"))
.args(["plugin", "build"])
.arg(&source)
.output()
.unwrap();
assert!(build.status.success(), "{build:?}");
let manifest = smith_harness::plugins::PluginManifest::parse(
&std::fs::read_to_string(source.join("plugin.kdl")).unwrap(),
)
.unwrap();
let artifact = source.join(manifest.artifact());
let packages = ["a", "b", "target"].map(|id| package(id, &format!("id \"fixture:{id}\"\nversion \"0.1.0\"\nworld \"0.1.0\"\nartifact \"fixture.wasm\"\ncapability \"workspace\"\n"), &artifact));
for (surface, caller, mode, grant, disabled, success, written) in [
("eval", "a", "caller", true, false, true, true),
("rpc", "b", "caller", true, false, true, true),
("eval", "a", "catch", false, false, false, false),
("eval", "a", "escape-catch", true, false, false, false),
("eval", "a", "catch", true, true, false, false),
("eval", "a", "invalid-catch", true, false, true, false),
("eval", "a", "parent-fail", true, false, false, true),
] {
let root = TestDir::new("smith-composition-binary");
std::fs::create_dir(root.join("A")).unwrap();
std::fs::create_dir(root.join("B")).unwrap();
let command = || {
let mut command = std::process::Command::new(assert_cmd::cargo::cargo_bin("smith"));
command
.current_dir(&root)
.env("HOME", root.join("home"))
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("XDG_RUNTIME_DIR", root.join("runtime"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("SMITH_CATALOG_CACHE")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env("OPENAI_API_KEY", "composition-fixture-secret");
command
};
for package in &packages {
let output = command()
.args(["plugin", "install"])
.arg(package)
.args(["--grant", "workspace"])
.output()
.unwrap();
assert!(output.status.success(), "{output:?}");
}
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let request = read_http_json(&mut stream);
let tools = request["tools"].as_array().unwrap();
assert!(
!tools
.iter()
.any(|tool| tool["function"]["name"] == "fixture:target/fixture")
);
assert!(
tools
.iter()
.any(|tool| tool["function"]["name"] == "fixture:a/fixture")
);
assert!(
tools
.iter()
.any(|tool| tool["function"]["name"] == "fixture:b/fixture")
);
let delta = serde_json::json!({"choices":[{"delta":{"tool_calls":[{"index":0,"id":"compose-call","function":{"name":format!("fixture:{caller}/fixture"),"arguments":format!(r#"{{"mode":"{mode}"}}"#)}}]}}]});
send_sse(
&mut stream,
&format!(
"data: {delta}\n\ndata: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"tool_calls\"}}]}}\n\ndata: [DONE]\n\n"
),
);
if success {
let (mut stream, _) = listener.accept().unwrap();
let request = read_http_json(&mut stream);
let output = request["messages"]
.as_array()
.unwrap()
.iter()
.find(|message| message["role"] == "tool")
.unwrap();
assert_eq!(
output["content"],
if written {
"child complete"
} else {
"caught ordinary guest error"
}
);
send_sse(
&mut stream,
"data: {\"choices\":[{\"delta\":{\"content\":\"composition complete\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
);
}
});
let mut invocation = command();
invocation.arg(surface);
if surface == "eval" {
invocation.arg("compose");
}
invocation.args([
"--provider",
"openai",
"--model",
"gpt-4.1",
"--base-url",
&format!("http://{address}"),
"--tool-policy",
&format!(
"fixture:target/fixture={}",
if disabled { "disabled" } else { "hidden" }
),
]);
if grant {
invocation.args([
"--tool-delegation",
&format!("fixture:{caller}/fixture=fixture:target/fixture#A"),
]);
}
if surface == "rpc" {
let mut child = TestChild(
invocation
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap(),
);
let mut stdin = child.0.stdin.take().unwrap();
let stdout = child.0.stdout.take().unwrap();
let (sender, received) = std::sync::mpsc::sync_channel(32);
let reader = std::thread::spawn(move || {
for line in std::io::BufReader::new(stdout).lines() {
sender.send(line.unwrap()).unwrap();
}
});
writeln!(stdin, "{{\"id\":1,\"method\":\"prompt\",\"params\":{{\"input\":\"compose\"}}}}\n{{\"id\":2,\"method\":\"wait\"}}").unwrap();
stdin.flush().unwrap();
let mut ended = false;
for _ in 0..32 {
let line = received.recv_timeout(Duration::from_secs(30)).unwrap();
assert!(!line.contains("composition-fixture-secret"));
let value: serde_json::Value = serde_json::from_str(&line).unwrap();
if value["event"] == "end" {
assert_eq!(value["text"], "composition complete");
ended = true;
}
if value["id"] == 2 {
assert_eq!(value["settled"], true);
break;
}
}
assert!(ended);
drop(stdin);
assert!(received.recv_timeout(Duration::from_secs(30)).is_err());
reader.join().unwrap();
assert!(child.0.wait().unwrap().success());
} else {
let output = Command::from(invocation)
.timeout(Duration::from_secs(30))
.output()
.unwrap();
assert_eq!(output.status.success(), success, "{mode}: {output:?}");
assert!(
!String::from_utf8_lossy(&output.stderr).contains("composition-fixture-secret")
);
if success {
assert_eq!(output.stdout, b"composition complete\n");
}
}
server.join().unwrap();
assert_eq!(root.join("A/written.txt").exists(), written, "{mode}");
assert!(!root.join("B/denied.txt").exists());
assert!(!root.join("written.txt").exists());
if written {
let mut child_intents = 0;
let mut writes = 0;
for entry in std::fs::read_dir(root.join("data/smith/sessions/run")).unwrap() {
let path = entry.unwrap().path();
if path.extension().is_none_or(|extension| extension != "smh") {
continue;
}
let reader = SessionReader::open(&path).unwrap();
let limit = smith::config::Config::default_valid().frame_bytes_max;
reader.validate_work(limit).unwrap();
reader
.scan(limit, |frame| {
if let Frame::Known { entry } | Frame::NestedCommit { entry } = frame
&& let EntryContent::Meta { kind, detail } =
EntryContent::decode(&entry.content)?
{
let detail = serde_json::to_value(detail).unwrap();
if kind == smith_core::metadata::NESTED_INTENT_KIND
&& detail["causal"]["invocation"] == true
{
child_intents += 1;
assert_eq!(detail["component"]["source_id"], "fixture:target");
}
if kind == smith_core::metadata::NESTED_COMPLETION_KIND
&& detail["causal"]["invocation"] == false
{
assert_eq!(
detail["mutation_state"],
serde_json::to_value(smith::error::MutationState::Applied)
.unwrap()
);
writes += 1;
}
}
Ok(())
})
.unwrap();
}
assert_eq!(child_intents, 1, "{mode}");
assert_eq!(writes, 1, "{mode}");
}
}
}