repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-cli/tests/e2e_atomic_tool_completion.rs
Raw//! SMH-SPEC-SPEC0001: admitted component writes, paired followup, durable receipt.
use assert_cmd::Command;
use serde_json::{Value, json};
use std::io::{Read, Write};
use std::path::Path;
use std::time::Duration;
#[path = "common/mod.rs"]
mod common;
#[path = "common/home.rs"]
mod home;
use common::{COMPONENT_INVOCATION_DEADLINE, COMPONENT_STARTUP_DEADLINE};
use home::TestHome;
fn smith(home: &Path) -> Command {
let mut command = Command::cargo_bin("smith").unwrap();
command
.current_dir(home)
.env("XDG_DATA_HOME", home.join("data"))
.env("XDG_CONFIG_HOME", home.join("config"))
.env("XDG_CACHE_HOME", home.join("cache"))
.env("XDG_STATE_HOME", home.join("state"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GOOGLE_API_KEY")
.env_remove("GEMINI_API_KEY")
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env_remove("OPENAI_API_KEY")
.timeout(COMPONENT_INVOCATION_DEADLINE);
command
}
fn request(socket: &mut std::net::TcpStream) -> Value {
socket
.set_read_timeout(Some(Duration::from_secs(10)))
.unwrap();
socket
.set_write_timeout(Some(Duration::from_secs(10)))
.unwrap();
let mut bytes = Vec::new();
let mut buffer = [0; 4096];
let end = loop {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && bytes.len() + n <= 1 << 20);
bytes.extend_from_slice(&buffer[..n]);
if let Some(at) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
break at + 4;
}
};
let length: usize = std::str::from_utf8(&bytes[..end])
.unwrap()
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length: ")
.and_then(|n| n.parse().ok())
})
.unwrap();
while bytes.len() < end + length {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && bytes.len() + n <= 1 << 20);
bytes.extend_from_slice(&buffer[..n]);
}
serde_json::from_slice(&bytes[end..end + length]).unwrap()
}
fn respond(socket: &mut std::net::TcpStream, delta: Value, reason: &str) {
let body = format!(
"data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
json!({"choices":[{"delta":delta}]}),
json!({"choices":[{"delta":{},"finish_reason":reason}]})
);
write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap();
}
#[test]
fn admitted_component_writes_preserve_ordered_paired_followup_before_terminal_receipt() {
let home = TestHome::new("smith-atomic-tool-e2e");
smith(&home)
.args(["eval", "--mock", "seed"])
.assert()
.success();
let listed = smith(&home)
.args(["sessions", "list", "--json"])
.output()
.unwrap();
let list: Value = serde_json::from_slice(&listed.stdout).unwrap();
let session = list["session"].as_str().unwrap();
let admitted = smith(&home)
.args(["sessions", "submit", session, "write a file"])
.output()
.unwrap();
assert!(admitted.status.success());
let admitted: Value = serde_json::from_slice(&admitted.stdout).unwrap();
let submission = admitted["submission"].as_str().unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}/v1", listener.local_addr().unwrap());
let (tx, rx) = std::sync::mpsc::sync_channel(2);
let workdir = home.as_ref().to_path_buf();
let fixture = std::thread::spawn(move || {
// The process deadline and bounded request signals keep fixture failure
// from turning a missing connection into an unbounded join.
let (mut socket, _) = listener.accept().unwrap();
let first = request(&mut socket);
tx.send(first).unwrap();
respond(
&mut socket,
json!({"tool_calls":[
{"index":0,"id":"call_write_first","function":{
"name":"write", "arguments":json!({"path":"first.txt","content":"once"}).to_string()
}},
{"index":1,"id":"call_write_second","function":{
"name":"write", "arguments":json!({"path":"second.txt","content":"twice"}).to_string()
}}
]}),
"tool_calls",
);
drop(socket);
let (mut socket, _) = listener.accept().unwrap();
let second = request(&mut socket);
assert_eq!(
std::fs::read_to_string(workdir.join("first.txt")).unwrap(),
"once"
);
assert_eq!(
std::fs::read_to_string(workdir.join("second.txt")).unwrap(),
"twice"
);
let messages = second["messages"].as_array().unwrap();
let results: Vec<_> = messages
.iter()
.filter(|message| message["role"] == "tool")
.collect();
assert_eq!(results.len(), 2, "{second}");
assert_eq!(results[0]["tool_call_id"], "call_write_first");
assert_eq!(results[1]["tool_call_id"], "call_write_second");
assert_eq!(
messages
.iter()
.filter(|message| message.get("tool_calls").is_some())
.count(),
1
);
tx.send(second).unwrap();
respond(&mut socket, json!({"content":"done"}), "stop");
listener
});
let mut command = smith(&home);
command.env("OPENAI_API_KEY", "fixture-key").args([
"sessions",
"execute",
session,
submission,
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&endpoint,
]);
let output = command.output().unwrap();
let first = rx.recv_timeout(COMPONENT_STARTUP_DEADLINE).unwrap();
assert!(
first["tools"]
.as_array()
.is_some_and(|tools| tools.iter().any(|tool| tool["function"]["name"] == "write")),
"{first}"
);
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let _followup = rx.recv_timeout(COMPONENT_STARTUP_DEADLINE).unwrap();
let listener = fixture.join().unwrap();
let queried = smith(&home)
.args(["sessions", "receipt", session, submission])
.output()
.unwrap();
let receipt: Value = serde_json::from_slice(&queried.stdout).unwrap();
assert_eq!(receipt["state"], "terminal");
assert_eq!(receipt["outcome"], "completed");
let history = smith(&home)
.args(["sessions", "get", session, "--json"])
.output()
.unwrap();
assert!(history.status.success());
let history = String::from_utf8(history.stdout).unwrap();
assert!(!history.contains("fixture-key"));
let records: Vec<Value> = history
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(
records
.iter()
.filter(|record| record["content"]["kind"] == "ToolResult")
.count(),
2
);
assert_eq!(
records
.iter()
.filter(|record| record["content"]["kind"] == "Message"
&& record["content"]["data"]["role"] == "Tool")
.count(),
2
);
smith(&home)
.env("OPENAI_API_KEY", "fixture-key")
.args([
"sessions",
"execute",
session,
submission,
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&endpoint,
])
.assert()
.failure();
listener.set_nonblocking(true).unwrap();
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
assert_eq!(
std::fs::read_to_string(home.join("first.txt")).unwrap(),
"once"
);
assert_eq!(
std::fs::read_to_string(home.join("second.txt")).unwrap(),
"twice"
);
}