repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-cli/tests/e2e_eval.rs
Raw//! End-to-end `smith eval` workflows through the built binary
//! (`SMH-SPEC-SPEC0001`, Interface modes: Eval).
use assert_cmd::Command;
use smith_core::TraceRecord;
use smith_core::store::SessionReader;
use smith_core::trace::trace;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
const MOCK_REPLY: &str = "mock provider: no model configured";
static SCRATCH_INDEX: AtomicUsize = AtomicUsize::new(0);
/// A fresh data home per test; stale runs never leak into assertions.
fn scratch(_name: &str) -> PathBuf {
let index = SCRATCH_INDEX.fetch_add(1, Ordering::Relaxed);
let dir = PathBuf::from("/var/tmp/smith-session-e2e")
.join(format!("e-{}-{index}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
dir
}
fn smith(data_home: &Path) -> Command {
let mut command = Command::cargo_bin("smith").unwrap();
command
.env("XDG_CONFIG_HOME", data_home.join("config"))
.env("XDG_DATA_HOME", data_home)
.env("XDG_STATE_HOME", data_home.join("state"))
.env("XDG_CACHE_HOME", data_home.join("cache"))
.env("XDG_RUNTIME_DIR", data_home.join("runtime"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("OPENAI_API_KEY")
.env_remove("GEMINI_API_KEY");
command
}
fn sessions(data_home: &Path) -> Vec<PathBuf> {
let dir = data_home.join("smith/sessions/run");
let mut paths: Vec<_> = std::fs::read_dir(dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension().is_some_and(|ext| ext == "smh"))
.collect();
paths.sort();
paths
}
/// Decode the session file the binary left behind into its trace.
fn trace_of(path: &Path) -> Vec<TraceRecord> {
let max = smith::config::Config::default_valid().frame_bytes_max;
let reader = SessionReader::open(path).unwrap();
let recovery = reader.scan(max, |_| Ok(())).unwrap();
assert!(recovery.damage.is_empty(), "{:?}", recovery.damage);
assert_eq!(recovery.torn_tail_at, None);
let mut records = Vec::new();
trace(&reader, max, |record| {
records.push(record);
Ok(())
})
.unwrap();
records
}
#[test]
fn eval_session_directory_failure_reports_storage_path() {
let home = scratch("blocked-session-dir");
assert!(std::fs::write(&home, "block data directory").is_ok());
let attempted = home.join("smith/sessions/run");
let output = smith(&home)
.args(["eval", "--mock", "hello"])
.output()
.unwrap();
assert!(!output.status.success());
assert!(output.stdout.is_empty());
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(stderr.starts_with("smith: storage [RUNTIME_DIR] create_dir_all "));
assert!(stderr.contains(&attempted.display().to_string()));
assert!(!stderr.contains("block data directory"));
assert!(std::fs::remove_file(home).is_ok());
}
#[test]
fn eval_leaves_a_terminal_durable_receipt_after_reopen() {
let home = scratch("durable-receipt");
smith(&home)
.args(["eval", "--mock", "hello"])
.assert()
.success()
.stdout(format!("{MOCK_REPLY}\n"));
let paths = sessions(&home);
assert_eq!(paths.len(), 1);
let reader = SessionReader::open(&paths[0]).unwrap();
let limit = smith::config::Config::default_valid().frame_bytes_max;
let mut admissions = Vec::new();
reader
.scan(limit, |frame| {
if let smith_core::frame::Frame::Admission { entry, .. } = frame {
admissions.push(entry.id);
}
Ok(())
})
.unwrap();
assert_eq!(admissions.len(), 1, "eval must use durable admission");
let receipt = reader.receipt(admissions[0], limit).unwrap().unwrap();
assert_eq!(receipt.session_id, reader.header(limit).unwrap().session_id);
assert_eq!(
reader.work_status(admissions[0], limit).unwrap(),
Some(smith_core::store::WorkStatus::Terminal(Some(
smith::stream::StopReason::EndTurn
)))
);
assert!(reader.work_output(admissions[0], limit).unwrap().is_some());
reader.verify_full(limit).unwrap();
}
#[test]
fn eval_mock_turn_prints_the_reply() {
let home = scratch("text");
smith(&home)
.args(["eval", "--mock", "hello"])
.assert()
.success()
.stdout(format!("{MOCK_REPLY}\n"))
.stderr("");
}
#[test]
fn eval_tool_round_persists_binary_content_for_get_and_grep() {
let home = scratch("tool-inspection");
let workspace = home.join("workspace");
std::fs::create_dir_all(&workspace).unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let server = std::thread::spawn(move || {
let responses = [
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"write\",\"arguments\":\"{\\\"path\\\":\\\"tool-fixture.txt\\\",\\\"content\\\":\\\"written\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
"data: {\"choices\":[{\"delta\":{\"content\":\"Done.\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
];
for response in responses {
let (mut socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
let mut request = Vec::new();
let mut buffer = [0u8; 4096];
let header_end = loop {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && request.len() + n <= 1 << 20);
request.extend_from_slice(&buffer[..n]);
if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
break at + 4;
}
};
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
let len: usize = headers
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length: ")
.and_then(|value| value.trim().parse().ok())
})
.unwrap();
while request.len() < header_end + len {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && request.len() + n <= 1 << 20);
request.extend_from_slice(&buffer[..n]);
}
write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
}
});
let output = smith(&home)
.current_dir(&workspace)
.env("OPENAI_API_KEY", "sk-fixture")
.args([
"eval",
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&format!("http://127.0.0.1:{port}/v1"),
"write fixture",
])
.output()
.unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"Done.\n");
server.join().unwrap();
assert_eq!(
std::fs::read(workspace.join("tool-fixture.txt")).unwrap(),
b"written"
);
assert!(!home.join("smith/sessions/run/tool-fixture.txt").exists());
let stored = sessions(&home);
assert_eq!(stored.len(), 1);
let id = stored[0].file_stem().unwrap().to_str().unwrap();
let got = smith(&home)
.args(["sessions", "get", id, "--json"])
.output()
.unwrap();
assert!(
got.status.success(),
"{}",
String::from_utf8_lossy(&got.stderr)
);
let records: Vec<serde_json::Value> = String::from_utf8(got.stdout)
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert!(records.iter().any(|r| r["content"]["kind"] == "ToolCall"
&& r["content"]["data"]["input"]["content"] == "written"));
assert!(
records
.iter()
.any(|r| r["content"]["kind"] == "ToolResult" && r["content"]["data"]["ok"] == true)
);
let grep = smith(&home)
.args(["sessions", "grep", "tool-fixture.txt", "--json"])
.output()
.unwrap();
assert!(
grep.status.success(),
"{}",
String::from_utf8_lossy(&grep.stderr)
);
assert!(String::from_utf8(grep.stdout).unwrap().contains("ToolCall"));
let _ = std::fs::remove_dir_all(home);
}
#[test]
fn eval_tool_round_guest_accepts_advertised_read_options() {
let home = scratch("wasm-tool-dispatch");
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let server = std::thread::spawn(move || {
let responses = [
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_wasm\",\"function\":{\"name\":\"read\",\"arguments\":\"{\\\"path\\\":\\\"missing.txt\\\",\\\"max_bytes\\\":7}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
"data: {\"choices\":[{\"delta\":{\"content\":\"Done.\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
];
let mut requests = Vec::new();
for response in responses {
let (mut socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
let mut request = Vec::new();
let mut buffer = [0_u8; 4096];
while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
let read = socket.read(&mut buffer).unwrap();
assert!(read > 0);
request.extend_from_slice(&buffer[..read]);
}
let header_end = request
.windows(4)
.position(|bytes| bytes == b"\r\n\r\n")
.unwrap()
+ 4;
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
let length: usize = headers
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length: ")
.and_then(|value| value.parse().ok())
})
.unwrap();
while request.len() < header_end + length {
let read = socket.read(&mut buffer).unwrap();
assert!(read > 0);
request.extend_from_slice(&buffer[..read]);
}
requests.push(String::from_utf8(request).unwrap());
write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
}
requests
});
smith(&home)
.env("OPENAI_API_KEY", "sk-fixture")
.args([
"eval",
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&format!("http://127.0.0.1:{port}/v1"),
"accept read options",
])
.assert()
.success()
.stdout("Done.\n");
let requests = server.join().unwrap();
assert!(requests[1].contains("READ_ERROR"));
assert!(!requests[1].contains("invalid tool input"));
let _ = std::fs::remove_dir_all(home);
}
#[test]
fn eval_json_reports_text_reason_and_cost() {
let home = scratch("json");
let output = smith(&home)
.args(["eval", "--mock", "--json", "hello"])
.output()
.unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
assert_eq!(stdout.lines().count(), 1, "{stdout}");
// The mock reply and its zero cost are fixed; the line is pinned byte for byte.
insta::assert_snapshot!("eval_mock_json", stdout);
}
#[test]
fn eval_creates_distinct_sessions_and_explicit_resume_keeps_history() {
let home = scratch("resume");
smith(&home)
.args(["eval", "--mock", "first"])
.assert()
.success();
let first = sessions(&home);
assert_eq!(first.len(), 1);
let original = trace_of(&first[0]);
assert!(original.len() > 1);
smith(&home)
.args(["eval", "--mock", "second"])
.assert()
.success();
assert_eq!(sessions(&home).len(), 2);
assert_eq!(trace_of(&first[0]), original);
let id = first[0].file_stem().unwrap().to_str().unwrap();
smith(&home)
.args(["eval", "--mock", "--session", id, "third"])
.assert()
.success();
assert_eq!(sessions(&home).len(), 2);
let resumed = trace_of(&first[0]);
assert!(resumed.len() > original.len());
assert_eq!(resumed[..original.len()], original);
}
#[test]
fn eval_split_credential_in_provider_error_never_reaches_outputs_or_session_bytes() {
provider_error_never_reaches_outputs_or_session_bytes(false);
}
#[test]
fn eval_json_escaped_credential_in_provider_error_never_reaches_outputs_or_session_bytes() {
provider_error_never_reaches_outputs_or_session_bytes(true);
}
fn provider_error_never_reaches_outputs_or_session_bytes(escaped: bool) {
let home = scratch("error-redaction");
let credential = "sk-e2e-split-fixture";
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let response = format!(
"data: {}\n\n",
serde_json::json!({"error":{"message":credential}})
);
let (response, split) = if escaped {
let spelling = r"\u0073k-e2e-\u0073plit-fixture";
let response = response.replace(credential, spelling);
let split = response.find(spelling).unwrap() + 4;
(response, split)
} else {
let split = response.find(credential).unwrap() + "sk-e2e-split-".len();
(response, split)
};
let server = std::thread::spawn(move || {
let (mut socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
let mut request = Vec::new();
let mut buffer = [0_u8; 4096];
while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
let read = socket.read(&mut buffer).unwrap();
assert!(read > 0);
request.extend_from_slice(&buffer[..read]);
}
write!(
socket,
"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n"
)
.unwrap();
socket.write_all(&response.as_bytes()[..split]).unwrap();
socket.flush().unwrap();
socket.write_all(&response.as_bytes()[split..]).unwrap();
});
let output = smith(&home)
.env("OPENAI_API_KEY", credential)
.args([
"eval",
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&format!("http://127.0.0.1:{port}/v1"),
"error",
])
.output()
.unwrap();
server.join().unwrap();
assert!(!output.status.success());
let encoded_credential = serde_json::to_string(credential).unwrap();
for bytes in [&output.stdout, &output.stderr] {
assert!(
!bytes
.windows(credential.len())
.any(|bytes| bytes == credential.as_bytes())
);
assert!(
!bytes
.windows(encoded_credential.len())
.any(|bytes| bytes == encoded_credential.as_bytes())
);
}
let session = sessions(&home).pop().unwrap();
let bytes = std::fs::read(&session).unwrap();
assert!(
!bytes
.windows(credential.len())
.any(|bytes| bytes == credential.as_bytes())
);
assert!(
!bytes
.windows(encoded_credential.len())
.any(|bytes| bytes == encoded_credential.as_bytes())
);
let id = session.file_stem().unwrap().to_str().unwrap();
let inspection = smith(&home)
.args(["sessions", "get", id, "--json"])
.output()
.unwrap();
assert!(inspection.status.success());
let inspection_text = String::from_utf8_lossy(&inspection.stdout);
assert!(inspection_text.contains("\"status\":200"));
assert!(inspection_text.contains("\"response_bytes\":"));
assert!(inspection_text.contains("\"outcome\":\"failed\""));
assert!(
!inspection
.stdout
.windows(credential.len())
.any(|bytes| bytes == credential.as_bytes())
);
assert!(
!inspection
.stdout
.windows(encoded_credential.len())
.any(|bytes| bytes == encoded_credential.as_bytes())
);
let _ = std::fs::remove_dir_all(home);
}