//! End-to-end `smith eval` workflows through the built binary //! (`SMH-SPEC-SPEC0001`, Interface modes: Eval). use assert_cmd::Command; use smith_core::TraceRecord; use smith_core::store::SessionReader; use smith_core::trace::trace; use std::io::{Read, Write}; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; const MOCK_REPLY: &str = "mock provider: no model configured"; static SCRATCH_INDEX: AtomicUsize = AtomicUsize::new(0); /// A fresh data home per test; stale runs never leak into assertions. fn scratch(_name: &str) -> PathBuf { let index = SCRATCH_INDEX.fetch_add(1, Ordering::Relaxed); let dir = PathBuf::from("/var/tmp/smith-session-e2e") .join(format!("e-{}-{index}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); dir } fn smith(data_home: &Path) -> Command { let mut command = Command::cargo_bin("smith").unwrap(); command .env("XDG_CONFIG_HOME", data_home.join("config")) .env("XDG_DATA_HOME", data_home) .env("XDG_STATE_HOME", data_home.join("state")) .env("XDG_CACHE_HOME", data_home.join("cache")) .env("XDG_RUNTIME_DIR", data_home.join("runtime")) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .env_remove("ANTHROPIC_API_KEY") .env_remove("OPENAI_API_KEY") .env_remove("GEMINI_API_KEY"); command } fn sessions(data_home: &Path) -> Vec { let dir = data_home.join("smith/sessions/run"); let mut paths: Vec<_> = std::fs::read_dir(dir) .unwrap() .map(|entry| entry.unwrap().path()) .filter(|path| path.extension().is_some_and(|ext| ext == "smh")) .collect(); paths.sort(); paths } /// Decode the session file the binary left behind into its trace. fn trace_of(path: &Path) -> Vec { let max = smith::config::Config::default_valid().frame_bytes_max; let reader = SessionReader::open(path).unwrap(); let recovery = reader.scan(max, |_| Ok(())).unwrap(); assert!(recovery.damage.is_empty(), "{:?}", recovery.damage); assert_eq!(recovery.torn_tail_at, None); let mut records = Vec::new(); trace(&reader, max, |record| { records.push(record); Ok(()) }) .unwrap(); records } #[test] fn eval_session_directory_failure_reports_storage_path() { let home = scratch("blocked-session-dir"); assert!(std::fs::write(&home, "block data directory").is_ok()); let attempted = home.join("smith/sessions/run"); let output = smith(&home) .args(["eval", "--mock", "hello"]) .output() .unwrap(); assert!(!output.status.success()); assert!(output.stdout.is_empty()); let stderr = String::from_utf8(output.stderr).unwrap(); assert!(stderr.starts_with("smith: storage [RUNTIME_DIR] create_dir_all ")); assert!(stderr.contains(&attempted.display().to_string())); assert!(!stderr.contains("block data directory")); assert!(std::fs::remove_file(home).is_ok()); } #[test] fn eval_leaves_a_terminal_durable_receipt_after_reopen() { let home = scratch("durable-receipt"); smith(&home) .args(["eval", "--mock", "hello"]) .assert() .success() .stdout(format!("{MOCK_REPLY}\n")); let paths = sessions(&home); assert_eq!(paths.len(), 1); let reader = SessionReader::open(&paths[0]).unwrap(); let limit = smith::config::Config::default_valid().frame_bytes_max; let mut admissions = Vec::new(); reader .scan(limit, |frame| { if let smith_core::frame::Frame::Admission { entry, .. } = frame { admissions.push(entry.id); } Ok(()) }) .unwrap(); assert_eq!(admissions.len(), 1, "eval must use durable admission"); let receipt = reader.receipt(admissions[0], limit).unwrap().unwrap(); assert_eq!(receipt.session_id, reader.header(limit).unwrap().session_id); assert_eq!( reader.work_status(admissions[0], limit).unwrap(), Some(smith_core::store::WorkStatus::Terminal(Some( smith::stream::StopReason::EndTurn ))) ); assert!(reader.work_output(admissions[0], limit).unwrap().is_some()); reader.verify_full(limit).unwrap(); } #[test] fn eval_mock_turn_prints_the_reply() { let home = scratch("text"); smith(&home) .args(["eval", "--mock", "hello"]) .assert() .success() .stdout(format!("{MOCK_REPLY}\n")) .stderr(""); } #[test] fn eval_tool_round_persists_binary_content_for_get_and_grep() { let home = scratch("tool-inspection"); let workspace = home.join("workspace"); std::fs::create_dir_all(&workspace).unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); let server = std::thread::spawn(move || { let responses = [ "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"write\",\"arguments\":\"{\\\"path\\\":\\\"tool-fixture.txt\\\",\\\"content\\\":\\\"written\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "data: {\"choices\":[{\"delta\":{\"content\":\"Done.\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ]; for response in responses { let (mut socket, _) = listener.accept().unwrap(); socket .set_read_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); let mut request = Vec::new(); let mut buffer = [0u8; 4096]; let header_end = loop { let n = socket.read(&mut buffer).unwrap(); assert!(n > 0 && request.len() + n <= 1 << 20); request.extend_from_slice(&buffer[..n]); if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") { break at + 4; } }; let headers = std::str::from_utf8(&request[..header_end]).unwrap(); let len: usize = headers .lines() .find_map(|line| { line.to_ascii_lowercase() .strip_prefix("content-length: ") .and_then(|value| value.trim().parse().ok()) }) .unwrap(); while request.len() < header_end + len { let n = socket.read(&mut buffer).unwrap(); assert!(n > 0 && request.len() + n <= 1 << 20); request.extend_from_slice(&buffer[..n]); } write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap(); } }); let output = smith(&home) .current_dir(&workspace) .env("OPENAI_API_KEY", "sk-fixture") .args([ "eval", "--provider", "openai", "--model", "test-model", "--base-url", &format!("http://127.0.0.1:{port}/v1"), "write fixture", ]) .output() .unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); assert_eq!(output.stdout, b"Done.\n"); server.join().unwrap(); assert_eq!( std::fs::read(workspace.join("tool-fixture.txt")).unwrap(), b"written" ); assert!(!home.join("smith/sessions/run/tool-fixture.txt").exists()); let stored = sessions(&home); assert_eq!(stored.len(), 1); let id = stored[0].file_stem().unwrap().to_str().unwrap(); let got = smith(&home) .args(["sessions", "get", id, "--json"]) .output() .unwrap(); assert!( got.status.success(), "{}", String::from_utf8_lossy(&got.stderr) ); let records: Vec = String::from_utf8(got.stdout) .unwrap() .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert!(records.iter().any(|r| r["content"]["kind"] == "ToolCall" && r["content"]["data"]["input"]["content"] == "written")); assert!( records .iter() .any(|r| r["content"]["kind"] == "ToolResult" && r["content"]["data"]["ok"] == true) ); let grep = smith(&home) .args(["sessions", "grep", "tool-fixture.txt", "--json"]) .output() .unwrap(); assert!( grep.status.success(), "{}", String::from_utf8_lossy(&grep.stderr) ); assert!(String::from_utf8(grep.stdout).unwrap().contains("ToolCall")); let _ = std::fs::remove_dir_all(home); } #[test] fn eval_tool_round_guest_accepts_advertised_read_options() { let home = scratch("wasm-tool-dispatch"); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); let server = std::thread::spawn(move || { let responses = [ "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_wasm\",\"function\":{\"name\":\"read\",\"arguments\":\"{\\\"path\\\":\\\"missing.txt\\\",\\\"max_bytes\\\":7}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "data: {\"choices\":[{\"delta\":{\"content\":\"Done.\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", ]; let mut requests = Vec::new(); for response in responses { let (mut socket, _) = listener.accept().unwrap(); socket .set_read_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); let mut request = Vec::new(); let mut buffer = [0_u8; 4096]; while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { let read = socket.read(&mut buffer).unwrap(); assert!(read > 0); request.extend_from_slice(&buffer[..read]); } let header_end = request .windows(4) .position(|bytes| bytes == b"\r\n\r\n") .unwrap() + 4; let headers = std::str::from_utf8(&request[..header_end]).unwrap(); let length: usize = headers .lines() .find_map(|line| { line.to_ascii_lowercase() .strip_prefix("content-length: ") .and_then(|value| value.parse().ok()) }) .unwrap(); while request.len() < header_end + length { let read = socket.read(&mut buffer).unwrap(); assert!(read > 0); request.extend_from_slice(&buffer[..read]); } requests.push(String::from_utf8(request).unwrap()); write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap(); } requests }); smith(&home) .env("OPENAI_API_KEY", "sk-fixture") .args([ "eval", "--provider", "openai", "--model", "test-model", "--base-url", &format!("http://127.0.0.1:{port}/v1"), "accept read options", ]) .assert() .success() .stdout("Done.\n"); let requests = server.join().unwrap(); assert!(requests[1].contains("READ_ERROR")); assert!(!requests[1].contains("invalid tool input")); let _ = std::fs::remove_dir_all(home); } #[test] fn eval_json_reports_text_reason_and_cost() { let home = scratch("json"); let output = smith(&home) .args(["eval", "--mock", "--json", "hello"]) .output() .unwrap(); assert!(output.status.success()); let stdout = String::from_utf8(output.stdout).unwrap(); assert_eq!(stdout.lines().count(), 1, "{stdout}"); // The mock reply and its zero cost are fixed; the line is pinned byte for byte. insta::assert_snapshot!("eval_mock_json", stdout); } #[test] fn eval_creates_distinct_sessions_and_explicit_resume_keeps_history() { let home = scratch("resume"); smith(&home) .args(["eval", "--mock", "first"]) .assert() .success(); let first = sessions(&home); assert_eq!(first.len(), 1); let original = trace_of(&first[0]); assert!(original.len() > 1); smith(&home) .args(["eval", "--mock", "second"]) .assert() .success(); assert_eq!(sessions(&home).len(), 2); assert_eq!(trace_of(&first[0]), original); let id = first[0].file_stem().unwrap().to_str().unwrap(); smith(&home) .args(["eval", "--mock", "--session", id, "third"]) .assert() .success(); assert_eq!(sessions(&home).len(), 2); let resumed = trace_of(&first[0]); assert!(resumed.len() > original.len()); assert_eq!(resumed[..original.len()], original); } #[test] fn eval_split_credential_in_provider_error_never_reaches_outputs_or_session_bytes() { provider_error_never_reaches_outputs_or_session_bytes(false); } #[test] fn eval_json_escaped_credential_in_provider_error_never_reaches_outputs_or_session_bytes() { provider_error_never_reaches_outputs_or_session_bytes(true); } fn provider_error_never_reaches_outputs_or_session_bytes(escaped: bool) { let home = scratch("error-redaction"); let credential = "sk-e2e-split-fixture"; let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); let response = format!( "data: {}\n\n", serde_json::json!({"error":{"message":credential}}) ); let (response, split) = if escaped { let spelling = r"\u0073k-e2e-\u0073plit-fixture"; let response = response.replace(credential, spelling); let split = response.find(spelling).unwrap() + 4; (response, split) } else { let split = response.find(credential).unwrap() + "sk-e2e-split-".len(); (response, split) }; let server = std::thread::spawn(move || { let (mut socket, _) = listener.accept().unwrap(); socket .set_read_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); let mut request = Vec::new(); let mut buffer = [0_u8; 4096]; while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { let read = socket.read(&mut buffer).unwrap(); assert!(read > 0); request.extend_from_slice(&buffer[..read]); } write!( socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n" ) .unwrap(); socket.write_all(&response.as_bytes()[..split]).unwrap(); socket.flush().unwrap(); socket.write_all(&response.as_bytes()[split..]).unwrap(); }); let output = smith(&home) .env("OPENAI_API_KEY", credential) .args([ "eval", "--provider", "openai", "--model", "e2e-model", "--base-url", &format!("http://127.0.0.1:{port}/v1"), "error", ]) .output() .unwrap(); server.join().unwrap(); assert!(!output.status.success()); let encoded_credential = serde_json::to_string(credential).unwrap(); for bytes in [&output.stdout, &output.stderr] { assert!( !bytes .windows(credential.len()) .any(|bytes| bytes == credential.as_bytes()) ); assert!( !bytes .windows(encoded_credential.len()) .any(|bytes| bytes == encoded_credential.as_bytes()) ); } let session = sessions(&home).pop().unwrap(); let bytes = std::fs::read(&session).unwrap(); assert!( !bytes .windows(credential.len()) .any(|bytes| bytes == credential.as_bytes()) ); assert!( !bytes .windows(encoded_credential.len()) .any(|bytes| bytes == encoded_credential.as_bytes()) ); let id = session.file_stem().unwrap().to_str().unwrap(); let inspection = smith(&home) .args(["sessions", "get", id, "--json"]) .output() .unwrap(); assert!(inspection.status.success()); let inspection_text = String::from_utf8_lossy(&inspection.stdout); assert!(inspection_text.contains("\"status\":200")); assert!(inspection_text.contains("\"response_bytes\":")); assert!(inspection_text.contains("\"outcome\":\"failed\"")); assert!( !inspection .stdout .windows(credential.len()) .any(|bytes| bytes == credential.as_bytes()) ); assert!( !inspection .stdout .windows(encoded_credential.len()) .any(|bytes| bytes == encoded_credential.as_bytes()) ); let _ = std::fs::remove_dir_all(home); }