repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-cli/tests/e2e_sessions.rs
Raw//! Read-only inspection through the built binary (`SMH-SPEC-SPEC0001`, Session inspection).
use assert_cmd::Command;
use serde_json::Value;
use std::path::Path;
use std::process::{Child, Output};
mod common;
use common::{COMPONENT_INVOCATION_DEADLINE, COMPONENT_STARTUP_DEADLINE};
struct ChildGuard(Option<Child>);
impl ChildGuard {
fn child_mut(&mut self) -> &mut Child {
self.0.as_mut().unwrap()
}
fn wait_with_output(mut self) -> Output {
self.0.take().unwrap().wait_with_output().unwrap()
}
}
impl Drop for ChildGuard {
fn drop(&mut self) {
if let Some(mut child) = self.0.take() {
let _ = child.kill();
let _ = child.wait();
}
}
}
fn smith(home: &Path, args: &[&str]) -> std::process::Output {
Command::cargo_bin("smith")
.unwrap()
.env("XDG_DATA_HOME", home)
.env("XDG_CONFIG_HOME", home.join("config"))
.env("XDG_STATE_HOME", home.join("state"))
.env("XDG_CACHE_HOME", home.join("cache"))
.env_remove("GOOGLE_API_KEY")
.env_remove("ZAI_API_KEY")
.env_remove("SMITH_PLUGIN_API_KEY")
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("OPENAI_API_KEY")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("GEMINI_API_KEY")
.args(args)
.timeout(COMPONENT_INVOCATION_DEADLINE)
.output()
.unwrap()
}
#[test]
fn admitted_text_executes_once_and_receipt_survives_reopen() {
let home = std::env::temp_dir().join(format!("smith-execute-e2e-{}", std::process::id()));
let seed = smith(&home, &["eval", "--mock", "seed"]);
assert!(
seed.status.success(),
"{}",
String::from_utf8_lossy(&seed.stderr)
);
let listed = smith(&home, &["sessions", "list", "--json"]);
let list: Value = serde_json::from_slice(&listed.stdout).unwrap();
let id = list["session"].as_str().unwrap();
let submitted = smith(&home, &["sessions", "submit", id, "admitted input"]);
let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap();
let submission = receipt["submission"].as_str().unwrap();
let preparation = smith(
&home,
&[
"sessions",
"execute",
id,
submission,
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
"invalid-endpoint",
],
);
assert!(!preparation.status.success());
let queued = smith(&home, &["sessions", "receipt", id, submission]);
let queued: Value = serde_json::from_slice(&queued.stdout).unwrap();
assert_eq!(queued["state"], "queued");
let executed = smith(&home, &["sessions", "execute", id, submission, "--mock"]);
assert!(
executed.status.success(),
"{}",
String::from_utf8_lossy(&executed.stderr)
);
let queried = smith(&home, &["sessions", "receipt", id, submission]);
let terminal: Value = serde_json::from_slice(&queried.stdout).unwrap();
assert_eq!(terminal["state"], "terminal");
assert!(terminal["output"].as_str().is_some());
assert_eq!(terminal["stop_reason"], "EndTurn");
assert!(
!smith(&home, &["sessions", "execute", id, submission, "--mock"])
.status
.success()
);
let entries = smith(&home, &["sessions", "get", id, "--json"]);
assert!(
entries.status.success(),
"{}",
String::from_utf8_lossy(&entries.stderr)
);
let entries = String::from_utf8(entries.stdout).unwrap();
assert_eq!(entries.matches("admitted input").count(), 1);
assert_eq!(
entries
.matches("mock provider: no model configured")
.count(),
2
);
std::fs::remove_dir_all(home).unwrap();
}
#[test]
fn admitted_real_provider_executes_once_with_advertised_tools() {
admitted_loopback(false, false, false, false);
}
#[test]
fn admitted_unadvertised_tool_use_fails_without_filesystem_effect() {
admitted_loopback(true, false, false, false);
}
#[test]
fn admitted_provider_response_loss_is_terminal_and_never_retried() {
admitted_loopback(false, true, false, false);
}
#[test]
fn admitted_execution_isolates_next_queued_input() {
admitted_loopback(false, false, true, false);
}
#[test]
fn admitted_receipt_survives_caller_response_loss() {
admitted_loopback(false, false, false, true);
}
fn admitted_loopback(tool_use: bool, drop_response: bool, queue_next: bool, drop_stdout: bool) {
use std::io::{Read, Write};
#[derive(Debug, PartialEq, Eq)]
enum FixtureSignal {
RequestComplete,
ResponseDropped,
}
let home = std::env::temp_dir().join(format!(
"smith-admitted-real-{}-{tool_use}-{drop_response}-{queue_next}-{drop_stdout}",
std::process::id()
));
std::fs::create_dir_all(&home).unwrap();
assert!(smith(&home, &["eval", "--mock", "seed"]).status.success());
let listed = smith(&home, &["sessions", "list", "--json"]);
let listed: Value = serde_json::from_slice(&listed.stdout).unwrap();
let id = listed["session"].as_str().unwrap();
let submitted = smith(&home, &["sessions", "submit", id, "real admitted input"]);
let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap();
let submission = receipt["submission"].as_str().unwrap();
let next_submission = if queue_next {
let second = smith(&home, &["sessions", "submit", id, "queued beta"]);
let second: Value = serde_json::from_slice(&second.stdout).unwrap();
Some(second["submission"].as_str().unwrap().to_owned())
} else {
None
};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let (tx, rx) = std::sync::mpsc::sync_channel(0);
let server = std::thread::spawn(move || {
let (mut socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
socket
.set_write_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
let mut request = Vec::new();
let mut buffer = [0; 4096];
let header_end = loop {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && request.len() + n <= 1 << 20);
request.extend_from_slice(&buffer[..n]);
if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
break at + 4;
}
};
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
let length: usize = headers
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length: ")
.and_then(|value| value.trim().parse().ok())
})
.unwrap();
while request.len() < header_end + length {
let n = socket.read(&mut buffer).unwrap();
assert!(n > 0 && request.len() + n <= 1 << 20);
request.extend_from_slice(&buffer[..n]);
}
let body: Value =
serde_json::from_slice(&request[header_end..header_end + length]).unwrap();
assert!(
body["tools"]
.as_array()
.is_some_and(|tools| tools.iter().any(|tool| tool["function"]["name"] == "write"))
);
let messages = body["messages"].as_array().unwrap();
assert_eq!(
messages
.iter()
.filter(|message| message["content"] == "real admitted input")
.count(),
1
);
assert!(!body.to_string().contains("queued beta"));
if drop_response {
socket.shutdown(std::net::Shutdown::Both).unwrap();
tx.send(FixtureSignal::ResponseDropped).unwrap();
return listener;
}
tx.send(FixtureSignal::RequestComplete).unwrap();
let response = if tool_use {
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"unregistered-tool\",\"arguments\":\"{\\\"path\\\":\\\"denied.txt\\\",\\\"content\\\":\\\"bad\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n"
} else {
"data: {\"choices\":[{\"delta\":{\"content\":\"real answer\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n"
};
write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
listener
});
let endpoint = format!("http://{address}/v1");
let execute = |wait_for_fixture: bool| {
let mut command = std::process::Command::new(assert_cmd::cargo::cargo_bin("smith"));
command
.current_dir(&home)
.env("XDG_DATA_HOME", &home)
.env("XDG_CONFIG_HOME", home.join("config"))
.env("OPENAI_API_KEY", "fixture-key")
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args([
"sessions",
"execute",
id,
submission,
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&endpoint,
]);
let mut child = ChildGuard(Some(
command
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.unwrap(),
));
if drop_stdout {
drop(child.child_mut().stdout.take());
}
if wait_for_fixture {
let signal = rx
.recv_timeout(COMPONENT_STARTUP_DEADLINE)
.unwrap_or_else(|error| {
panic!("provider fixture readiness deadline elapsed: {error}")
});
assert_eq!(
signal,
if drop_response {
FixtureSignal::ResponseDropped
} else {
FixtureSignal::RequestComplete
}
);
}
child.wait_with_output()
};
let output = execute(true);
assert_eq!(
output.status.success(),
!tool_use && !drop_response && !drop_stdout,
"{}",
String::from_utf8_lossy(&output.stderr)
);
let listener = server.join().unwrap();
assert!(!execute(false).status.success());
listener.set_nonblocking(true).unwrap();
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
assert!(!home.join("denied.txt").exists());
let queried = smith(&home, &["sessions", "receipt", id, submission]);
let receipt: Value = serde_json::from_slice(&queried.stdout).unwrap();
assert_eq!(receipt["state"], "terminal");
assert_eq!(
receipt["outcome"],
if tool_use || drop_response {
"failed"
} else {
"completed"
}
);
if drop_response {
assert_eq!(receipt["output"], Value::Null);
assert_eq!(receipt["stop_reason"], Value::Null);
}
if let Some(next_submission) = next_submission {
let queued = smith(&home, &["sessions", "receipt", id, &next_submission]);
let queued: Value = serde_json::from_slice(&queued.stdout).unwrap();
assert_eq!(queued["state"], "queued");
}
let entries = smith(&home, &["sessions", "get", id, "--json"]);
assert!(
entries.status.success(),
"{}",
String::from_utf8_lossy(&entries.stderr)
);
assert_eq!(
String::from_utf8_lossy(&entries.stdout).contains("real answer"),
!tool_use && !drop_response
);
std::fs::remove_dir_all(home).unwrap();
}
#[test]
fn interrupted_claim_reopens_recovery_required_without_dispatch() {
use std::io::Read;
let home = std::env::temp_dir().join(format!("smith-interrupted-{}", std::process::id()));
assert!(smith(&home, &["eval", "--mock", "seed"]).status.success());
let listed = smith(&home, &["sessions", "list", "--json"]);
let listed: Value = serde_json::from_slice(&listed.stdout).unwrap();
let id = listed["session"].as_str().unwrap();
let submitted = smith(&home, &["sessions", "submit", id, "interrupt input"]);
let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap();
let submission = receipt["submission"].as_str().unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}/v1", listener.local_addr().unwrap());
let (started_tx, started_rx) = std::sync::mpsc::sync_channel(1);
let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1);
let server = std::thread::spawn(move || {
let (mut socket, _) = listener.accept().unwrap();
socket
.set_read_timeout(Some(std::time::Duration::from_secs(10)))
.unwrap();
let mut byte = [0; 1];
socket.read_exact(&mut byte).unwrap();
started_tx.send(()).unwrap();
release_rx
.recv_timeout(std::time::Duration::from_secs(20))
.unwrap();
listener
});
let mut child = std::process::Command::new(assert_cmd::cargo::cargo_bin("smith"))
.env("XDG_DATA_HOME", &home)
.env("XDG_CONFIG_HOME", home.join("config"))
.env("OPENAI_API_KEY", "fixture-key")
.args([
"sessions",
"execute",
id,
submission,
"--provider",
"openai",
"--model",
"test-model",
"--base-url",
&endpoint,
])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.unwrap();
started_rx
.recv_timeout(std::time::Duration::from_secs(15))
.unwrap();
child.kill().unwrap();
child.wait().unwrap();
release_tx.send(()).unwrap();
let listener = server.join().unwrap();
let observed = smith(&home, &["sessions", "receipt", id, submission]);
assert!(
observed.status.success(),
"{}",
String::from_utf8_lossy(&observed.stderr)
);
let observed: Value = serde_json::from_slice(&observed.stdout).unwrap();
assert_eq!(observed["state"], "running");
assert_eq!(observed["recovery_required"], true);
assert!(
!smith(&home, &["sessions", "execute", id, submission, "--mock"])
.status
.success()
);
listener.set_nonblocking(true).unwrap();
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
std::fs::remove_dir_all(home).unwrap();
}
#[test]
fn session_list_limit_reports_capacity_not_corruption() {
// Prepare complete, valid session files directly; the real CLI remains the listing SUT.
use smith::config::Config;
use smith_core::session::Session;
use smith_core::store::create_session;
let home = std::env::temp_dir().join(format!("smith-list-limit-e2e-{}", std::process::id()));
let session_dir = home.join("smith/sessions/run");
std::fs::create_dir_all(&session_dir).unwrap();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120);
for _ in 0..1025 {
assert!(
std::time::Instant::now() < deadline,
"session fixture deadline"
);
let session = Session::new();
let path = session_dir.join(format!("{}.smh", session.id()));
drop(create_session(&path, &session, Config::default_valid()).unwrap());
}
assert!(
std::time::Instant::now() < deadline,
"session fixture deadline"
);
let listed = smith(&home, &["sessions", "list", "--json"]);
assert!(!listed.status.success());
let diagnostic = String::from_utf8(listed.stderr).unwrap();
assert!(diagnostic.contains("SESSION_LIST_LIMIT"));
assert!(diagnostic.contains("inspect sessions individually"));
assert!(!diagnostic.contains("SESSION_CORRUPT"));
let records = String::from_utf8(listed.stdout).unwrap();
assert_eq!(records.lines().count(), 1024);
assert!(
records
.lines()
.all(|line| serde_json::from_str::<Value>(line).is_ok())
);
std::fs::remove_dir_all(home).unwrap();
}
#[test]
fn sessions_list_get_grep_jsonl_and_no_match() {
let home = std::env::temp_dir().join(format!("smith-sessions-e2e-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
assert!(
smith(&home, &["eval", "--mock", "needle \u{1b}[31m in haystack"])
.status
.success()
);
assert!(
smith(&home, &["eval", "--mock", "other haystack"])
.status
.success()
);
let listed = smith(&home, &["sessions", "list", "--json"]);
assert!(
listed.status.success(),
"{}",
String::from_utf8_lossy(&listed.stderr)
);
let listed = String::from_utf8(listed.stdout).unwrap();
let lines: Vec<Value> = listed
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(lines.len(), 2);
let id = lines[0]["session"].as_str().unwrap();
let human_list = smith(&home, &["sessions", "list"]);
assert!(human_list.status.success());
let (mut json_bytes, mut human_bytes) = (String::new(), String::new());
for line in &lines {
let (session, bytes) = (line["session"].as_str().unwrap(), &line["bytes"]);
json_bytes += &format!("{{\"session\":\"{session}\",\"bytes\":{bytes}}}\n");
human_bytes += &format!("{session} {bytes} bytes\n");
}
assert_eq!(listed, json_bytes, "JSON list lines keep their exact bytes");
assert_eq!(
String::from_utf8(human_list.stdout).unwrap(),
human_bytes,
"human list lines keep their exact bytes"
);
let retrieved = smith(&home, &["sessions", "get", id, "--json"]);
assert!(
retrieved.status.success(),
"{}",
String::from_utf8_lossy(&retrieved.stderr)
);
let records: Vec<Value> = String::from_utf8(retrieved.stdout)
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(records.len(), 3);
assert_eq!(records[0]["content"]["kind"], "Message");
assert_eq!(records[1]["content"]["kind"], "Meta");
assert_eq!(
records[1]["content"]["data"]["kind"],
"tool-loadout-provenance"
);
assert_eq!(records[2]["content"]["kind"], "Message");
for record in [&records[0], &records[2]] {
assert_eq!(record["content"]["data"]["blocks"][0]["type"], "Text");
assert!(record["content"]["data"]["blocks"][0]["data"].is_string());
}
let human_get = smith(&home, &["sessions", "get", id]);
assert!(human_get.status.success());
assert!(!human_get.stdout.contains(&0x1b));
let matches = smith(&home, &["sessions", "grep", "needle", "--json"]);
assert!(
matches.status.success(),
"{}",
String::from_utf8_lossy(&matches.stderr)
);
let found: Vec<Value> = String::from_utf8(matches.stdout)
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(found.len(), 1);
assert!(found[0]["entry"].is_string());
assert_eq!(
found[0]["content"]["data"]["blocks"][0]["data"],
"needle \u{1b}[31m in haystack"
);
let human_grep = smith(&home, &["sessions", "grep", "needle"]);
assert!(human_grep.status.success());
assert!(
String::from_utf8(human_grep.stdout)
.unwrap()
.contains("needle")
);
let none = smith(&home, &["sessions", "grep", "not-in-any-session"]);
assert_eq!(none.status.code(), Some(1));
assert!(none.stdout.is_empty());
assert!(none.stderr.is_empty());
let _ = std::fs::remove_dir_all(home);
}
#[test]
fn sessions_submit_reopens_queued_receipt_without_duplicate_input() {
let home = std::env::temp_dir().join(format!("smith-admission-e2e-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
let created = smith(&home, &["eval", "--mock", "before"]);
assert!(
created.status.success(),
"{}",
String::from_utf8_lossy(&created.stderr)
);
let sessions = smith(&home, &["sessions", "list", "--json"]);
let id = serde_json::from_str::<Value>(std::str::from_utf8(&sessions.stdout).unwrap()).unwrap()
["session"]
.as_str()
.unwrap()
.to_owned();
let submitted = smith(
&home,
&["sessions", "submit", &id, "queued", "--key", "retry"],
);
assert!(
submitted.status.success(),
"{}",
String::from_utf8_lossy(&submitted.stderr)
);
let receipt: Value = serde_json::from_slice(&submitted.stdout).unwrap();
assert_eq!(receipt["session"], id);
assert_eq!(receipt["state"], "queued");
assert_eq!(receipt["durability"], "acknowledged");
let submission = receipt["submission"].as_str().unwrap().to_owned();
let retried = smith(
&home,
&["sessions", "submit", &id, "queued", "--key", "retry"],
);
assert!(
retried.status.success(),
"{}",
String::from_utf8_lossy(&retried.stderr)
);
assert_eq!(retried.stdout, submitted.stdout);
let inspected = smith(&home, &["sessions", "receipt", &id, &submission]);
assert!(
inspected.status.success(),
"{}",
String::from_utf8_lossy(&inspected.stderr)
);
let inspected: Value = serde_json::from_slice(&inspected.stdout).unwrap();
assert_eq!(inspected["session"], id);
assert_eq!(inspected["submission"], submission);
assert_eq!(inspected["state"], "queued");
assert_eq!(inspected["visibility"], "visible");
assert!(inspected.get("durability").is_none());
let stored = smith(&home, &["sessions", "get", &id, "--json"]);
assert!(stored.status.success());
let inputs = String::from_utf8(stored.stdout)
.unwrap()
.lines()
.filter(|line| line.contains("queued"))
.count();
assert_eq!(inputs, 1);
let _ = std::fs::remove_dir_all(home);
}
#[test]
fn eval_session_rejects_pending_submission_without_running_provider() {
let home = std::env::temp_dir().join(format!(
"smith-admission-runtime-e2e-{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&home);
assert!(smith(&home, &["eval", "--mock", "before"]).status.success());
let listed = smith(&home, &["sessions", "list", "--json"]);
let id = serde_json::from_slice::<Value>(&listed.stdout).unwrap()["session"]
.as_str()
.unwrap()
.to_owned();
assert!(
smith(&home, &["sessions", "submit", &id, "must not execute"])
.status
.success()
);
let rejected = smith(&home, &["eval", "--session", &id, "--mock", "next"]);
assert_eq!(rejected.status.code(), Some(1));
assert!(String::from_utf8_lossy(&rejected.stderr).contains("pending submission"));
assert!(rejected.stdout.is_empty());
let _ = std::fs::remove_dir_all(home);
}
#[test]
fn sessions_submit_rejects_busy_writer_before_mutation() {
let home =
std::env::temp_dir().join(format!("smith-admission-busy-e2e-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
assert!(smith(&home, &["eval", "--mock", "before"]).status.success());
let listed = smith(&home, &["sessions", "list", "--json"]);
let id = serde_json::from_slice::<Value>(&listed.stdout).unwrap()["session"]
.as_str()
.unwrap()
.to_owned();
let path = home.join("smith/sessions/run").join(format!("{id}.smh"));
let writer =
smith_core::store::SessionWriter::open(&path, smith::config::Config::default_valid())
.unwrap();
let bytes = std::fs::read(&path).unwrap();
let denied = smith(&home, &["sessions", "submit", &id, "must not persist"]);
assert_eq!(denied.status.code(), Some(2));
assert!(String::from_utf8_lossy(&denied.stderr).contains("STORE_BUSY"));
assert_eq!(std::fs::read(&path).unwrap(), bytes);
drop(writer);
let _ = std::fs::remove_dir_all(home);
}
#[test]
fn corrupt_session_is_reported_without_repairing_source() {
// SMH-SPEC-SPEC0001: Session inspection never repairs or hides corruption.
let home = std::env::temp_dir().join(format!("smith-corrupt-e2e-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
assert!(smith(&home, &["eval", "--mock", "before"]).status.success());
let dir = home.join("smith/sessions/run");
let path = std::fs::read_dir(&dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.find(|path| path.extension().is_some_and(|ext| ext == "smh"))
.unwrap();
let mut bytes = std::fs::read(&path).unwrap();
let last = bytes.len() - 1;
bytes[last] ^= 0xff;
std::fs::write(&path, &bytes).unwrap();
let id = path.file_stem().unwrap().to_str().unwrap();
let output = smith(&home, &["sessions", "get", id]);
assert_eq!(output.status.code(), Some(2));
assert!(String::from_utf8_lossy(&output.stderr).contains("SESSION_CORRUPT"));
assert_eq!(std::fs::read(path).unwrap(), bytes);
let _ = std::fs::remove_dir_all(home);
}