//! Read-only inspection through the built binary (`SMH-SPEC-SPEC0001`, Session inspection). use assert_cmd::Command; use serde_json::Value; use std::path::Path; use std::process::{Child, Output}; mod common; use common::{COMPONENT_INVOCATION_DEADLINE, COMPONENT_STARTUP_DEADLINE}; struct ChildGuard(Option); impl ChildGuard { fn child_mut(&mut self) -> &mut Child { self.0.as_mut().unwrap() } fn wait_with_output(mut self) -> Output { self.0.take().unwrap().wait_with_output().unwrap() } } impl Drop for ChildGuard { fn drop(&mut self) { if let Some(mut child) = self.0.take() { let _ = child.kill(); let _ = child.wait(); } } } fn smith(home: &Path, args: &[&str]) -> std::process::Output { Command::cargo_bin("smith") .unwrap() .env("XDG_DATA_HOME", home) .env("XDG_CONFIG_HOME", home.join("config")) .env("XDG_STATE_HOME", home.join("state")) .env("XDG_CACHE_HOME", home.join("cache")) .env_remove("GOOGLE_API_KEY") .env_remove("ZAI_API_KEY") .env_remove("SMITH_PLUGIN_API_KEY") .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .env_remove("OPENAI_API_KEY") .env_remove("ANTHROPIC_API_KEY") .env_remove("GEMINI_API_KEY") .args(args) .timeout(COMPONENT_INVOCATION_DEADLINE) .output() .unwrap() } #[test] fn admitted_text_executes_once_and_receipt_survives_reopen() { let home = std::env::temp_dir().join(format!("smith-execute-e2e-{}", std::process::id())); let seed = smith(&home, &["eval", "--mock", "seed"]); assert!( seed.status.success(), "{}", String::from_utf8_lossy(&seed.stderr) ); let listed = smith(&home, &["sessions", "list", "--json"]); let list: Value = serde_json::from_slice(&listed.stdout).unwrap(); let id = list["session"].as_str().unwrap(); let submitted = smith(&home, &["sessions", "submit", id, "admitted input"]); let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap(); let submission = receipt["submission"].as_str().unwrap(); let preparation = smith( &home, &[ "sessions", "execute", id, submission, "--provider", "openai", "--model", "test-model", "--base-url", "invalid-endpoint", ], ); assert!(!preparation.status.success()); let queued = smith(&home, &["sessions", "receipt", id, submission]); let queued: Value = serde_json::from_slice(&queued.stdout).unwrap(); assert_eq!(queued["state"], "queued"); let executed = smith(&home, &["sessions", "execute", id, submission, "--mock"]); assert!( executed.status.success(), "{}", String::from_utf8_lossy(&executed.stderr) ); let queried = smith(&home, &["sessions", "receipt", id, submission]); let terminal: Value = serde_json::from_slice(&queried.stdout).unwrap(); assert_eq!(terminal["state"], "terminal"); assert!(terminal["output"].as_str().is_some()); assert_eq!(terminal["stop_reason"], "EndTurn"); assert!( !smith(&home, &["sessions", "execute", id, submission, "--mock"]) .status .success() ); let entries = smith(&home, &["sessions", "get", id, "--json"]); assert!( entries.status.success(), "{}", String::from_utf8_lossy(&entries.stderr) ); let entries = String::from_utf8(entries.stdout).unwrap(); assert_eq!(entries.matches("admitted input").count(), 1); assert_eq!( entries .matches("mock provider: no model configured") .count(), 2 ); std::fs::remove_dir_all(home).unwrap(); } #[test] fn admitted_real_provider_executes_once_with_advertised_tools() { admitted_loopback(false, false, false, false); } #[test] fn admitted_unadvertised_tool_use_fails_without_filesystem_effect() { admitted_loopback(true, false, false, false); } #[test] fn admitted_provider_response_loss_is_terminal_and_never_retried() { admitted_loopback(false, true, false, false); } #[test] fn admitted_execution_isolates_next_queued_input() { admitted_loopback(false, false, true, false); } #[test] fn admitted_receipt_survives_caller_response_loss() { admitted_loopback(false, false, false, true); } fn admitted_loopback(tool_use: bool, drop_response: bool, queue_next: bool, drop_stdout: bool) { use std::io::{Read, Write}; #[derive(Debug, PartialEq, Eq)] enum FixtureSignal { RequestComplete, ResponseDropped, } let home = std::env::temp_dir().join(format!( "smith-admitted-real-{}-{tool_use}-{drop_response}-{queue_next}-{drop_stdout}", std::process::id() )); std::fs::create_dir_all(&home).unwrap(); assert!(smith(&home, &["eval", "--mock", "seed"]).status.success()); let listed = smith(&home, &["sessions", "list", "--json"]); let listed: Value = serde_json::from_slice(&listed.stdout).unwrap(); let id = listed["session"].as_str().unwrap(); let submitted = smith(&home, &["sessions", "submit", id, "real admitted input"]); let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap(); let submission = receipt["submission"].as_str().unwrap(); let next_submission = if queue_next { let second = smith(&home, &["sessions", "submit", id, "queued beta"]); let second: Value = serde_json::from_slice(&second.stdout).unwrap(); Some(second["submission"].as_str().unwrap().to_owned()) } else { None }; let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); let (tx, rx) = std::sync::mpsc::sync_channel(0); let server = std::thread::spawn(move || { let (mut socket, _) = listener.accept().unwrap(); socket .set_read_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); socket .set_write_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); let mut request = Vec::new(); let mut buffer = [0; 4096]; let header_end = loop { let n = socket.read(&mut buffer).unwrap(); assert!(n > 0 && request.len() + n <= 1 << 20); request.extend_from_slice(&buffer[..n]); if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") { break at + 4; } }; let headers = std::str::from_utf8(&request[..header_end]).unwrap(); let length: usize = headers .lines() .find_map(|line| { line.to_ascii_lowercase() .strip_prefix("content-length: ") .and_then(|value| value.trim().parse().ok()) }) .unwrap(); while request.len() < header_end + length { let n = socket.read(&mut buffer).unwrap(); assert!(n > 0 && request.len() + n <= 1 << 20); request.extend_from_slice(&buffer[..n]); } let body: Value = serde_json::from_slice(&request[header_end..header_end + length]).unwrap(); assert!( body["tools"] .as_array() .is_some_and(|tools| tools.iter().any(|tool| tool["function"]["name"] == "write")) ); let messages = body["messages"].as_array().unwrap(); assert_eq!( messages .iter() .filter(|message| message["content"] == "real admitted input") .count(), 1 ); assert!(!body.to_string().contains("queued beta")); if drop_response { socket.shutdown(std::net::Shutdown::Both).unwrap(); tx.send(FixtureSignal::ResponseDropped).unwrap(); return listener; } tx.send(FixtureSignal::RequestComplete).unwrap(); let response = if tool_use { "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"unregistered-tool\",\"arguments\":\"{\\\"path\\\":\\\"denied.txt\\\",\\\"content\\\":\\\"bad\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n" } else { "data: {\"choices\":[{\"delta\":{\"content\":\"real answer\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n" }; write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap(); listener }); let endpoint = format!("http://{address}/v1"); let execute = |wait_for_fixture: bool| { let mut command = std::process::Command::new(assert_cmd::cargo::cargo_bin("smith")); command .current_dir(&home) .env("XDG_DATA_HOME", &home) .env("XDG_CONFIG_HOME", home.join("config")) .env("OPENAI_API_KEY", "fixture-key") .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .args([ "sessions", "execute", id, submission, "--provider", "openai", "--model", "test-model", "--base-url", &endpoint, ]); let mut child = ChildGuard(Some( command .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn() .unwrap(), )); if drop_stdout { drop(child.child_mut().stdout.take()); } if wait_for_fixture { let signal = rx .recv_timeout(COMPONENT_STARTUP_DEADLINE) .unwrap_or_else(|error| { panic!("provider fixture readiness deadline elapsed: {error}") }); assert_eq!( signal, if drop_response { FixtureSignal::ResponseDropped } else { FixtureSignal::RequestComplete } ); } child.wait_with_output() }; let output = execute(true); assert_eq!( output.status.success(), !tool_use && !drop_response && !drop_stdout, "{}", String::from_utf8_lossy(&output.stderr) ); let listener = server.join().unwrap(); assert!(!execute(false).status.success()); listener.set_nonblocking(true).unwrap(); assert_eq!( listener.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock ); assert!(!home.join("denied.txt").exists()); let queried = smith(&home, &["sessions", "receipt", id, submission]); let receipt: Value = serde_json::from_slice(&queried.stdout).unwrap(); assert_eq!(receipt["state"], "terminal"); assert_eq!( receipt["outcome"], if tool_use || drop_response { "failed" } else { "completed" } ); if drop_response { assert_eq!(receipt["output"], Value::Null); assert_eq!(receipt["stop_reason"], Value::Null); } if let Some(next_submission) = next_submission { let queued = smith(&home, &["sessions", "receipt", id, &next_submission]); let queued: Value = serde_json::from_slice(&queued.stdout).unwrap(); assert_eq!(queued["state"], "queued"); } let entries = smith(&home, &["sessions", "get", id, "--json"]); assert!( entries.status.success(), "{}", String::from_utf8_lossy(&entries.stderr) ); assert_eq!( String::from_utf8_lossy(&entries.stdout).contains("real answer"), !tool_use && !drop_response ); std::fs::remove_dir_all(home).unwrap(); } #[test] fn interrupted_claim_reopens_recovery_required_without_dispatch() { use std::io::Read; let home = std::env::temp_dir().join(format!("smith-interrupted-{}", std::process::id())); assert!(smith(&home, &["eval", "--mock", "seed"]).status.success()); let listed = smith(&home, &["sessions", "list", "--json"]); let listed: Value = serde_json::from_slice(&listed.stdout).unwrap(); let id = listed["session"].as_str().unwrap(); let submitted = smith(&home, &["sessions", "submit", id, "interrupt input"]); let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap(); let submission = receipt["submission"].as_str().unwrap(); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let endpoint = format!("http://{}/v1", listener.local_addr().unwrap()); let (started_tx, started_rx) = std::sync::mpsc::sync_channel(1); let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); let server = std::thread::spawn(move || { let (mut socket, _) = listener.accept().unwrap(); socket .set_read_timeout(Some(std::time::Duration::from_secs(10))) .unwrap(); let mut byte = [0; 1]; socket.read_exact(&mut byte).unwrap(); started_tx.send(()).unwrap(); release_rx .recv_timeout(std::time::Duration::from_secs(20)) .unwrap(); listener }); let mut child = std::process::Command::new(assert_cmd::cargo::cargo_bin("smith")) .env("XDG_DATA_HOME", &home) .env("XDG_CONFIG_HOME", home.join("config")) .env("OPENAI_API_KEY", "fixture-key") .args([ "sessions", "execute", id, submission, "--provider", "openai", "--model", "test-model", "--base-url", &endpoint, ]) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .spawn() .unwrap(); started_rx .recv_timeout(std::time::Duration::from_secs(15)) .unwrap(); child.kill().unwrap(); child.wait().unwrap(); release_tx.send(()).unwrap(); let listener = server.join().unwrap(); let observed = smith(&home, &["sessions", "receipt", id, submission]); assert!( observed.status.success(), "{}", String::from_utf8_lossy(&observed.stderr) ); let observed: Value = serde_json::from_slice(&observed.stdout).unwrap(); assert_eq!(observed["state"], "running"); assert_eq!(observed["recovery_required"], true); assert!( !smith(&home, &["sessions", "execute", id, submission, "--mock"]) .status .success() ); listener.set_nonblocking(true).unwrap(); assert_eq!( listener.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock ); std::fs::remove_dir_all(home).unwrap(); } #[test] fn session_list_limit_reports_capacity_not_corruption() { // Prepare complete, valid session files directly; the real CLI remains the listing SUT. use smith::config::Config; use smith_core::session::Session; use smith_core::store::create_session; let home = std::env::temp_dir().join(format!("smith-list-limit-e2e-{}", std::process::id())); let session_dir = home.join("smith/sessions/run"); std::fs::create_dir_all(&session_dir).unwrap(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); for _ in 0..1025 { assert!( std::time::Instant::now() < deadline, "session fixture deadline" ); let session = Session::new(); let path = session_dir.join(format!("{}.smh", session.id())); drop(create_session(&path, &session, Config::default_valid()).unwrap()); } assert!( std::time::Instant::now() < deadline, "session fixture deadline" ); let listed = smith(&home, &["sessions", "list", "--json"]); assert!(!listed.status.success()); let diagnostic = String::from_utf8(listed.stderr).unwrap(); assert!(diagnostic.contains("SESSION_LIST_LIMIT")); assert!(diagnostic.contains("inspect sessions individually")); assert!(!diagnostic.contains("SESSION_CORRUPT")); let records = String::from_utf8(listed.stdout).unwrap(); assert_eq!(records.lines().count(), 1024); assert!( records .lines() .all(|line| serde_json::from_str::(line).is_ok()) ); std::fs::remove_dir_all(home).unwrap(); } #[test] fn sessions_list_get_grep_jsonl_and_no_match() { let home = std::env::temp_dir().join(format!("smith-sessions-e2e-{}", std::process::id())); let _ = std::fs::remove_dir_all(&home); assert!( smith(&home, &["eval", "--mock", "needle \u{1b}[31m in haystack"]) .status .success() ); assert!( smith(&home, &["eval", "--mock", "other haystack"]) .status .success() ); let listed = smith(&home, &["sessions", "list", "--json"]); assert!( listed.status.success(), "{}", String::from_utf8_lossy(&listed.stderr) ); let listed = String::from_utf8(listed.stdout).unwrap(); let lines: Vec = listed .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!(lines.len(), 2); let id = lines[0]["session"].as_str().unwrap(); let human_list = smith(&home, &["sessions", "list"]); assert!(human_list.status.success()); let (mut json_bytes, mut human_bytes) = (String::new(), String::new()); for line in &lines { let (session, bytes) = (line["session"].as_str().unwrap(), &line["bytes"]); json_bytes += &format!("{{\"session\":\"{session}\",\"bytes\":{bytes}}}\n"); human_bytes += &format!("{session} {bytes} bytes\n"); } assert_eq!(listed, json_bytes, "JSON list lines keep their exact bytes"); assert_eq!( String::from_utf8(human_list.stdout).unwrap(), human_bytes, "human list lines keep their exact bytes" ); let retrieved = smith(&home, &["sessions", "get", id, "--json"]); assert!( retrieved.status.success(), "{}", String::from_utf8_lossy(&retrieved.stderr) ); let records: Vec = String::from_utf8(retrieved.stdout) .unwrap() .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!(records.len(), 3); assert_eq!(records[0]["content"]["kind"], "Message"); assert_eq!(records[1]["content"]["kind"], "Meta"); assert_eq!( records[1]["content"]["data"]["kind"], "tool-loadout-provenance" ); assert_eq!(records[2]["content"]["kind"], "Message"); for record in [&records[0], &records[2]] { assert_eq!(record["content"]["data"]["blocks"][0]["type"], "Text"); assert!(record["content"]["data"]["blocks"][0]["data"].is_string()); } let human_get = smith(&home, &["sessions", "get", id]); assert!(human_get.status.success()); assert!(!human_get.stdout.contains(&0x1b)); let matches = smith(&home, &["sessions", "grep", "needle", "--json"]); assert!( matches.status.success(), "{}", String::from_utf8_lossy(&matches.stderr) ); let found: Vec = String::from_utf8(matches.stdout) .unwrap() .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!(found.len(), 1); assert!(found[0]["entry"].is_string()); assert_eq!( found[0]["content"]["data"]["blocks"][0]["data"], "needle \u{1b}[31m in haystack" ); let human_grep = smith(&home, &["sessions", "grep", "needle"]); assert!(human_grep.status.success()); assert!( String::from_utf8(human_grep.stdout) .unwrap() .contains("needle") ); let none = smith(&home, &["sessions", "grep", "not-in-any-session"]); assert_eq!(none.status.code(), Some(1)); assert!(none.stdout.is_empty()); assert!(none.stderr.is_empty()); let _ = std::fs::remove_dir_all(home); } #[test] fn sessions_submit_reopens_queued_receipt_without_duplicate_input() { let home = std::env::temp_dir().join(format!("smith-admission-e2e-{}", std::process::id())); let _ = std::fs::remove_dir_all(&home); let created = smith(&home, &["eval", "--mock", "before"]); assert!( created.status.success(), "{}", String::from_utf8_lossy(&created.stderr) ); let sessions = smith(&home, &["sessions", "list", "--json"]); let id = serde_json::from_str::(std::str::from_utf8(&sessions.stdout).unwrap()).unwrap() ["session"] .as_str() .unwrap() .to_owned(); let submitted = smith( &home, &["sessions", "submit", &id, "queued", "--key", "retry"], ); assert!( submitted.status.success(), "{}", String::from_utf8_lossy(&submitted.stderr) ); let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap(); assert_eq!(receipt["session"], id); assert_eq!(receipt["state"], "queued"); assert_eq!(receipt["durability"], "acknowledged"); let submission = receipt["submission"].as_str().unwrap().to_owned(); let retried = smith( &home, &["sessions", "submit", &id, "queued", "--key", "retry"], ); assert!( retried.status.success(), "{}", String::from_utf8_lossy(&retried.stderr) ); assert_eq!(retried.stdout, submitted.stdout); let inspected = smith(&home, &["sessions", "receipt", &id, &submission]); assert!( inspected.status.success(), "{}", String::from_utf8_lossy(&inspected.stderr) ); let inspected: Value = serde_json::from_slice(&inspected.stdout).unwrap(); assert_eq!(inspected["session"], id); assert_eq!(inspected["submission"], submission); assert_eq!(inspected["state"], "queued"); assert_eq!(inspected["visibility"], "visible"); assert!(inspected.get("durability").is_none()); let stored = smith(&home, &["sessions", "get", &id, "--json"]); assert!(stored.status.success()); let inputs = String::from_utf8(stored.stdout) .unwrap() .lines() .filter(|line| line.contains("queued")) .count(); assert_eq!(inputs, 1); let _ = std::fs::remove_dir_all(home); } #[test] fn eval_session_rejects_pending_submission_without_running_provider() { let home = std::env::temp_dir().join(format!( "smith-admission-runtime-e2e-{}", std::process::id() )); let _ = std::fs::remove_dir_all(&home); assert!(smith(&home, &["eval", "--mock", "before"]).status.success()); let listed = smith(&home, &["sessions", "list", "--json"]); let id = serde_json::from_slice::(&listed.stdout).unwrap()["session"] .as_str() .unwrap() .to_owned(); assert!( smith(&home, &["sessions", "submit", &id, "must not execute"]) .status .success() ); let rejected = smith(&home, &["eval", "--session", &id, "--mock", "next"]); assert_eq!(rejected.status.code(), Some(1)); assert!(String::from_utf8_lossy(&rejected.stderr).contains("pending submission")); assert!(rejected.stdout.is_empty()); let _ = std::fs::remove_dir_all(home); } #[test] fn sessions_submit_rejects_busy_writer_before_mutation() { let home = std::env::temp_dir().join(format!("smith-admission-busy-e2e-{}", std::process::id())); let _ = std::fs::remove_dir_all(&home); assert!(smith(&home, &["eval", "--mock", "before"]).status.success()); let listed = smith(&home, &["sessions", "list", "--json"]); let id = serde_json::from_slice::(&listed.stdout).unwrap()["session"] .as_str() .unwrap() .to_owned(); let path = home.join("smith/sessions/run").join(format!("{id}.smh")); let writer = smith_core::store::SessionWriter::open(&path, smith::config::Config::default_valid()) .unwrap(); let bytes = std::fs::read(&path).unwrap(); let denied = smith(&home, &["sessions", "submit", &id, "must not persist"]); assert_eq!(denied.status.code(), Some(2)); assert!(String::from_utf8_lossy(&denied.stderr).contains("STORE_BUSY")); assert_eq!(std::fs::read(&path).unwrap(), bytes); drop(writer); let _ = std::fs::remove_dir_all(home); } #[test] fn corrupt_session_is_reported_without_repairing_source() { // SMH-SPEC-SPEC0001: Session inspection never repairs or hides corruption. let home = std::env::temp_dir().join(format!("smith-corrupt-e2e-{}", std::process::id())); let _ = std::fs::remove_dir_all(&home); assert!(smith(&home, &["eval", "--mock", "before"]).status.success()); let dir = home.join("smith/sessions/run"); let path = std::fs::read_dir(&dir) .unwrap() .map(|entry| entry.unwrap().path()) .find(|path| path.extension().is_some_and(|ext| ext == "smh")) .unwrap(); let mut bytes = std::fs::read(&path).unwrap(); let last = bytes.len() - 1; bytes[last] ^= 0xff; std::fs::write(&path, &bytes).unwrap(); let id = path.file_stem().unwrap().to_str().unwrap(); let output = smith(&home, &["sessions", "get", id]); assert_eq!(output.status.code(), Some(2)); assert!(String::from_utf8_lossy(&output.stderr).contains("SESSION_CORRUPT")); assert_eq!(std::fs::read(path).unwrap(), bytes); let _ = std::fs::remove_dir_all(home); }