repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-cli/tests/e2e_atomic_tool_kill.rs
Raw//! SMH-SPEC-SPEC0001: missing effect completion recovers as unknown, never retried.
#![cfg(unix)]
use assert_cmd::Command;
use serde_json::{Value, json};
use std::io::{Read, Write};
use std::path::Path;
use std::process::{Child, Stdio};
use std::time::Duration;
#[path = "common/mod.rs"]
mod common;
#[path = "common/home.rs"]
mod home;
use common::{COMPONENT_INVOCATION_DEADLINE, COMPONENT_STARTUP_DEADLINE};
use home::TestHome;
fn smith(home: &Path) -> Command {
let mut command = Command::cargo_bin("smith").unwrap();
command
.current_dir(home)
.env("XDG_DATA_HOME", home.join("data"))
.env("XDG_CONFIG_HOME", home.join("config"))
.env("XDG_CACHE_HOME", home.join("cache"))
.env("XDG_STATE_HOME", home.join("state"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env_remove("OPENAI_API_KEY")
.timeout(COMPONENT_INVOCATION_DEADLINE);
command
}
fn request(socket: &mut std::net::TcpStream) -> Value {
socket
.set_read_timeout(Some(Duration::from_secs(15)))
.unwrap();
let mut bytes = Vec::new();
let mut buffer = [0; 4096];
let end = loop {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && bytes.len() + n <= 1 << 20);
bytes.extend_from_slice(&buffer[..n]);
if let Some(at) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
break at + 4;
}
};
let length: usize = std::str::from_utf8(&bytes[..end])
.unwrap()
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length: ")
.and_then(|n| n.parse().ok())
})
.unwrap();
while bytes.len() < end + length {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && bytes.len() + n <= 1 << 20);
bytes.extend_from_slice(&buffer[..n]);
}
serde_json::from_slice(&bytes[end..end + length]).unwrap()
}
fn respond(socket: &mut std::net::TcpStream, delta: Value, reason: &str) {
let body = format!(
"data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
json!({"choices":[{"delta":delta}]}),
json!({"choices":[{"delta":{},"finish_reason":reason}]})
);
write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap();
}
struct KillOnDrop(Child);
impl KillOnDrop {
fn kill_and_reap(&mut self) {
self.0.kill().unwrap();
self.0.wait().unwrap();
}
fn kill_capture_stderr(&mut self) -> String {
let _ = self.0.kill();
let mut stderr = String::new();
if let Some(mut pipe) = self.0.stderr.take() {
let _ = pipe.read_to_string(&mut stderr);
}
let _ = self.0.wait();
stderr
}
}
impl Drop for KillOnDrop {
fn drop(&mut self) {
if self.0.try_wait().unwrap().is_none() {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
}
#[test]
fn killing_during_bash_effect_preserves_unknown_intent_without_retry() {
let home = TestHome::new("smith-kill-effect");
smith(&home)
.args(["eval", "--mock", "seed"])
.assert()
.success();
let listed = smith(&home)
.args(["sessions", "list", "--json"])
.output()
.unwrap();
let list: Value = serde_json::from_slice(&listed.stdout).unwrap();
let session = list["session"].as_str().unwrap();
let admitted = smith(&home)
.args([
"sessions",
"submit",
session,
"run the admitted bash command",
])
.output()
.unwrap();
assert!(admitted.status.success());
let admitted: Value = serde_json::from_slice(&admitted.stdout).unwrap();
let submission = admitted["submission"].as_str().unwrap();
let provider = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let provider_addr = provider.local_addr().unwrap();
let endpoint = format!("http://{provider_addr}/v1");
let effect_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let effect_addr = effect_listener.local_addr().unwrap();
let effect_path = home.join("effect.txt");
let effect_path = effect_path.to_str().unwrap();
let command_text = format!(
"printf 'effect-applied\\n' > '{effect_path}'; exec 3<>/dev/tcp/{}/{}; printf 'READY\\n' >&3; IFS= read -r _ <&3",
effect_addr.ip(),
effect_addr.port()
);
let (effect_tx, effect_rx) = std::sync::mpsc::sync_channel(1);
let effect_fixture = std::thread::spawn(move || {
let (mut socket, _) = effect_listener.accept().unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(15)))
.unwrap();
let mut ready = [0; 6];
socket.read_exact(&mut ready).unwrap();
effect_tx.send((socket, ready)).unwrap();
});
let (provider_request_tx, provider_request_rx) = std::sync::mpsc::sync_channel(1);
let provider_fixture = std::thread::spawn(move || {
let (mut socket, _) = provider.accept().unwrap();
let first = request(&mut socket);
provider_request_tx.send(first).unwrap();
respond(
&mut socket,
json!({"tool_calls":[{"index":0,"id":"call_bash_effect","function":{"name":"bash","arguments":json!({"command":command_text}).to_string()}}]}),
"tool_calls",
);
drop(socket);
provider
});
let mut child = KillOnDrop(
std::process::Command::new(assert_cmd::cargo::cargo_bin("smith"))
.current_dir(&home)
.env("XDG_DATA_HOME", home.join("data"))
.env("XDG_CONFIG_HOME", home.join("config"))
.env("XDG_CACHE_HOME", home.join("cache"))
.env("XDG_STATE_HOME", home.join("state"))
.env("OPENAI_API_KEY", "fixture-key")
.args([
"sessions",
"execute",
session,
submission,
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&endpoint,
])
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.unwrap(),
);
let first = match provider_request_rx.recv_timeout(COMPONENT_STARTUP_DEADLINE) {
Ok(request) => request,
Err(error) => {
let _ = std::net::TcpStream::connect(provider_addr);
let stderr = child.kill_capture_stderr();
let _ = provider_fixture.join();
panic!("provider request was not received: {error}; stderr: {stderr}");
}
};
assert!(
first["tools"]
.as_array()
.is_some_and(|tools| { tools.iter().any(|tool| tool["function"]["name"] == "bash") }),
"bundled bash tool absent, Process grant not admitted: {first}"
);
let (effect_socket, ready) = match effect_rx.recv_timeout(COMPONENT_STARTUP_DEADLINE) {
Ok(accepted) => accepted,
Err(error) => {
let _ = std::net::TcpStream::connect(effect_addr);
let stderr = child.kill_capture_stderr();
effect_fixture.join().unwrap();
panic!("bash effect did not signal READY: {error}; stderr: {stderr}");
}
};
assert_eq!(&ready, b"READY\n");
effect_fixture.join().unwrap();
assert_eq!(
std::fs::read_to_string(effect_path).unwrap(),
"effect-applied\n"
);
child.kill_and_reap();
drop(effect_socket);
let provider = provider_fixture.join().unwrap();
provider.set_nonblocking(true).unwrap();
let queried = smith(&home)
.args(["sessions", "receipt", session, submission])
.output()
.unwrap();
assert!(queried.status.success());
let receipt: Value = serde_json::from_slice(&queried.stdout).unwrap();
assert_eq!(receipt["state"], "running");
assert_eq!(receipt["recovery_required"], true);
let retried = smith(&home)
.env("OPENAI_API_KEY", "fixture-key")
.args([
"sessions",
"execute",
session,
submission,
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&endpoint,
])
.output()
.unwrap();
assert!(!retried.status.success());
assert_eq!(
provider.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
assert_eq!(
std::fs::read_to_string(effect_path).unwrap(),
"effect-applied\n"
);
let history = smith(&home)
.args(["sessions", "get", session, "--json"])
.output()
.unwrap();
assert!(history.status.success());
let records: Vec<Value> = String::from_utf8(history.stdout)
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
let provenance_index = records
.iter()
.position(|record| {
record["content"]["kind"] == "Meta"
&& record["content"]["data"]["kind"] == "tool-loadout-provenance"
})
.expect("tool loadout provenance missing");
let components = records[provenance_index]["content"]["data"]["detail"]["components"]
.as_array()
.expect("provenance components missing");
let coding_tools = components
.iter()
.find(|component| component["source_id"] == "smith:coding-tools")
.expect("bundled wasm component provenance missing");
for hash in ["manifest_sha256", "component_sha256"] {
assert_eq!(coding_tools[hash].as_str().unwrap().len(), 64);
assert!(
coding_tools[hash]
.as_str()
.unwrap()
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
);
}
let tool_call_index = records
.iter()
.position(|record| {
record["content"]["kind"] == "ToolCall"
&& record["content"]["data"]["name"] == "bash"
&& record["content"]["data"]["input"]["command"]
.as_str()
.is_some_and(|command| command.contains("effect-applied"))
})
.expect("raw bash intent missing");
assert!(
provenance_index < tool_call_index,
"loadout provenance must be recorded before tool dispatch: {records:?}"
);
assert!(
!records.iter().any(|record| {
record["content"]["kind"] == "ToolResult"
|| record.to_string().contains("ToolCompletion")
}),
"completion unexpectedly recorded: {records:?}"
);
}