repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-cli/tests/e2e_rpc.rs
Raw//! End-to-end `smith rpc` workflows over piped standard input and output
//! (`SMH-SPEC-SPEC0001`, Interface modes: JSON-RPC).
//!
//! Every read waits on a line or end of output with a deadline; nothing
//! sleeps or polls.
use serde_json::{Value, json};
use std::io::{BufRead, BufReader, Read, Write};
use std::net::TcpListener;
use std::path::PathBuf;
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{
Arc,
mpsc::{self, Receiver},
};
use std::time::Duration;
const DEADLINE: Duration = Duration::from_secs(10);
const STARTUP_DEADLINE: Duration = Duration::from_secs(10);
const EXIT_DEADLINE: Duration = Duration::from_secs(10);
const DIAGNOSTIC_QUEUE_LINES: usize = 32;
const DIAGNOSTIC_LINE_BYTES: usize = 4 * 1024;
const MOCK_REPLY: &str = "rpc mock provider ready";
static RPC_HOME_INDEX: AtomicUsize = AtomicUsize::new(0);
/// One running `smith rpc` process; killed on drop if a test fails early.
struct Rpc {
child: Child,
home: PathBuf,
stdin: Option<ChildStdin>,
/// Raw stdout lines; `None` marks end of output.
lines: Receiver<Option<String>>,
/// Child diagnostics emitted before a fixture deadline expires.
stderr: Receiver<String>,
/// Every line sent (`> `) and received (`< `), in order.
transcript: Vec<String>,
}
impl Rpc {
fn spawn(_name: &str, args: &[&str], envs: &[(&str, &str)]) -> Self {
let index = RPC_HOME_INDEX.fetch_add(1, Ordering::Relaxed);
let home = PathBuf::from("/var/tmp/smith-session-e2e")
.join(format!("r-{}-{index}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
std::fs::create_dir_all(&home).unwrap();
let mut child = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.arg("rpc")
.args(args)
.current_dir(&home)
.env("HOME", &home)
.env("XDG_CACHE_HOME", home.join("cache"))
.env("XDG_CONFIG_HOME", home.join("config"))
.env("XDG_DATA_HOME", &home)
.env("XDG_RUNTIME_DIR", home.join("runtime"))
.env("XDG_STATE_HOME", home.join("state"))
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.env_remove("ANTHROPIC_API_KEY")
.env_remove("OPENAI_API_KEY")
.env_remove("GEMINI_API_KEY")
.envs(envs.iter().copied())
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let stdout = child.stdout.take().unwrap();
let stderr = child.stderr.take().unwrap();
let (sender, lines) = mpsc::channel();
std::thread::spawn(move || {
for line in BufReader::new(stdout).lines() {
if sender.send(Some(line.unwrap())).is_err() {
return;
}
}
let _ = sender.send(None);
});
let (stderr_sender, stderr_lines) = mpsc::channel();
std::thread::spawn(move || {
for line in BufReader::new(stderr).lines() {
if stderr_sender.send(line.unwrap()).is_err() {
return;
}
}
});
let stdin = child.stdin.take();
Self {
child,
home,
stdin,
lines,
stderr: stderr_lines,
transcript: Vec::new(),
}
}
fn mock(name: &str) -> Self {
Self::spawn(name, &[], &[])
}
fn send(&mut self, line: &str) {
self.transcript.push(format!("> {line}"));
let stdin = self.stdin.as_mut().unwrap();
stdin.write_all(line.as_bytes()).unwrap();
stdin.write_all(b"\n").unwrap();
stdin.flush().unwrap();
}
fn next(&mut self) -> Value {
match self.lines.recv_timeout(STARTUP_DEADLINE) {
Ok(Some(line)) => {
let value = serde_json::from_str(&line)
.unwrap_or_else(|error| panic!("stdout line is not JSON ({error}): {line}"));
self.transcript.push(format!("< {line}"));
value
}
Ok(None) => panic!("rpc output ended early"),
Err(error) => panic!("no rpc line within {STARTUP_DEADLINE:?}: {error}"),
}
}
fn request_diagnostics(&mut self, session: &str) -> String {
let queued_stdout = self
.lines
.try_iter()
.take(DIAGNOSTIC_QUEUE_LINES)
.map(|line| line.map(diagnostic_line))
.collect::<Vec<_>>();
let stderr = self
.stderr
.try_iter()
.take(DIAGNOSTIC_QUEUE_LINES)
.map(diagnostic_line)
.collect::<Vec<_>>()
.join("\\n");
let journal = assert_cmd::Command::new(assert_cmd::cargo::cargo_bin("smith"))
.timeout(DEADLINE)
.args(["sessions", "get", session, "--json"])
.env("XDG_DATA_HOME", &self.home)
.output()
.map_or_else(
|error| format!("inspection spawn failed: {error}"),
|output| {
format!(
"status={}; stdout={}; stderr={}",
output.status,
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
)
},
);
format!(
"rpc status={:?}; transcript={:?}; queued_stdout={queued_stdout:?}; stderr={stderr:?}; committed.txt={:?}; journal={journal}",
self.child.try_wait(),
self.transcript,
std::fs::read_to_string(self.home.join("committed.txt")),
)
}
/// Close stdin, then expect end of output and the exit status.
fn close(self) -> std::process::ExitStatus {
self.close_with_stderr().0
}
fn close_with_stderr(mut self) -> (std::process::ExitStatus, String) {
drop(self.stdin.take());
match self.lines.recv_timeout(EXIT_DEADLINE) {
Ok(None) => {}
Ok(Some(line)) => panic!("unexpected rpc line after close: {line}"),
Err(error) => panic!("rpc did not end output within {EXIT_DEADLINE:?}: {error}"),
}
let status = self.child.wait().unwrap();
(status, self.stderr.iter().collect::<Vec<_>>().join("\n"))
}
}
fn diagnostic_line(line: String) -> String {
if line.len() <= DIAGNOSTIC_LINE_BYTES {
return line;
}
let end = line
.char_indices()
.take_while(|(index, _)| *index <= DIAGNOSTIC_LINE_BYTES)
.map(|(index, _)| index)
.last()
.unwrap_or(0);
format!("{}… [truncated]", &line[..end])
}
fn transcript_for_snapshot(lines: &[String]) -> String {
lines
.iter()
.map(|line| {
let Some(json) = line.strip_prefix("< ") else {
return line.clone();
};
let Ok(mut value) = serde_json::from_str::<Value>(json) else {
return line.clone();
};
if let Some(receipt) = value.get_mut("receipt").and_then(Value::as_object_mut) {
receipt.insert("session_id".into(), json!("<session>"));
receipt.insert("submission_id".into(), json!("<submission>"));
}
if value.get("submission_id").is_some() {
value["submission_id"] = json!("<submission>");
}
format!("< {value}")
})
.collect::<Vec<_>>()
.join("\n")
}
impl Drop for Rpc {
fn drop(&mut self) {
if matches!(self.child.try_wait(), Ok(None)) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
}
#[test]
fn rpc_event_and_shutdown_deadlines_stay_at_ten_seconds() {
assert_eq!(STARTUP_DEADLINE, DEADLINE);
assert_eq!(EXIT_DEADLINE, DEADLINE);
}
#[test]
fn rpc_rejects_oversized_stdin_line_without_protocol_output() {
let mut command = assert_cmd::Command::cargo_bin("smith").unwrap();
command
.arg("rpc")
.env(
"XDG_DATA_HOME",
std::env::temp_dir().join(format!("smith-rpc-test-oversized-{}", std::process::id())),
)
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.write_stdin("x".repeat(1024 * 1024 + 1))
.assert()
.failure()
.stdout("")
.stderr(predicates::str::contains(
"JSON-RPC request exceeds 1048576 bytes",
));
}
#[test]
fn rpc_ping_correlates_the_request_id() {
let mut rpc = Rpc::mock("ping");
rpc.send(r#"{"id": 7, "method": "ping"}"#);
assert_eq!(rpc.next(), json!({"id": 7, "pong": true}));
rpc.send(r#"{"id": "seven", "method": "ping"}"#);
assert_eq!(rpc.next(), json!({"id": "seven", "pong": true}));
assert!(rpc.close().success());
}
#[test]
fn rpc_prompt_streams_start_delta_end_then_wait_and_status_report_settled() {
let mut rpc = Rpc::mock("prompt");
rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "hi"}}"#);
rpc.send(r#"{"id": 2, "method": "wait"}"#);
let start = rpc.next();
assert_eq!(start["id"], 1);
assert_eq!(start["event"], "start");
assert_eq!(start["receipt"]["status"], "Queued");
assert_eq!(start["receipt"]["cancel_requested"], false);
assert!(start["receipt"]["submission_id"].is_string());
assert_eq!(
rpc.next(),
json!({"id": 1, "event": "delta", "text": MOCK_REPLY})
);
assert_eq!(
rpc.next(),
json!({
"id": 1,
"event": "end",
"text": MOCK_REPLY,
"reason": "EndTurn",
"cost": null,
"cost_complete": false,
})
);
// The worker answers `wait` only after the prompt has settled.
assert_eq!(rpc.next(), json!({"id": 2, "settled": true}));
// `wait` answers once the aborted turn has settled; `status` then agrees.
rpc.send(r#"{"id": 3, "method": "wait"}"#);
assert_eq!(rpc.next(), json!({"id": 3, "settled": true}));
rpc.send(r#"{"id": 4, "method": "status"}"#);
assert_eq!(
rpc.next(),
json!({"id": 4, "settled": true, "access_mode": "writer", "active_submission_id": null})
);
assert!(rpc.close().success());
}
#[test]
fn rpc_read_only_rejects_prompt_and_abort_without_mutating_the_session() {
let home = std::env::temp_dir().join(format!("smith-rpc-read-only-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
let seed = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["eval", "--mock", "seed"])
.env("XDG_DATA_HOME", &home)
.env_remove("SMITH_PROVIDER")
.output()
.unwrap();
assert!(
seed.status.success(),
"{}",
String::from_utf8_lossy(&seed.stderr)
);
let listed = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "list", "--json"])
.env("XDG_DATA_HOME", &home)
.output()
.unwrap();
let session = serde_json::from_slice::<Value>(&listed.stdout).unwrap()["session"]
.as_str()
.unwrap()
.to_owned();
let path = home
.join("smith/sessions/run")
.join(format!("{session}.smh"));
let before = std::fs::read(&path).unwrap();
let output = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args([
"rpc",
"--session",
&session,
"--read-only",
"--provider",
"openai",
"--model",
"must-not-initialize",
"--base-url",
"http://127.0.0.1:1",
])
.env("XDG_DATA_HOME", &home)
.env_remove("OPENAI_API_KEY")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.spawn()
.unwrap();
let mut child = output;
let mut stdin = child.stdin.take().unwrap();
stdin
.write_all(
b"{\"id\":1,\"method\":\"status\"}\n{\"id\":2,\"method\":\"prompt\",\"params\":{\"input\":\"denied\"}}\n{\"id\":3,\"method\":\"abort\"}\n{\"id\":4,\"method\":\"inspect\",\"params\":{\"id\":\"babab-babab-babab-babab-babab-babab-babab-babab\"}}\n",
)
.unwrap();
drop(stdin);
let output = child.wait_with_output().unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let records: Vec<Value> = String::from_utf8(output.stdout)
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(
records[0],
json!({"id": 1, "settled": true, "access_mode": "read_only", "active_submission_id": null})
);
assert_eq!(records[1]["error"]["code"], "STATE");
assert_eq!(records[2]["error"]["code"], "STATE");
assert_eq!(records[3], json!({"id": 4, "receipt": null}));
assert_eq!(std::fs::read(&path).unwrap(), before);
std::fs::remove_dir_all(home).unwrap();
}
#[test]
fn rpc_malformed_line_reports_parse_error_with_null_id() {
let mut rpc = Rpc::mock("malformed");
rpc.send("not json");
let record = rpc.next();
assert_eq!(record["id"], Value::Null);
assert_eq!(record["error"]["code"], "PARSE");
// The server keeps serving after a bad line.
rpc.send(r#"{"id": 1, "method": "ping"}"#);
assert_eq!(rpc.next(), json!({"id": 1, "pong": true}));
assert!(rpc.close().success());
}
#[test]
fn rpc_unknown_method_reports_method_unknown() {
let mut rpc = Rpc::mock("unknown");
rpc.send(r#"{"id": "x", "method": "explode"}"#);
assert_eq!(
rpc.next(),
json!({
"id": "x",
"error": {"code": "METHOD_UNKNOWN", "message": "unknown method explode"},
})
);
assert!(rpc.close().success());
}
#[test]
fn rpc_closing_stdin_after_settle_exits_zero() {
let mut rpc = Rpc::mock("shutdown");
rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "hi"}}"#);
rpc.send(r#"{"id": 2, "method": "wait"}"#);
let mut records = Vec::new();
loop {
let record = rpc.next();
let done = record["id"] == 2;
records.push(record);
if done {
break;
}
}
assert!(records.iter().any(|r| r["id"] == 1 && r["event"] == "end"));
let status = rpc.close();
assert_eq!(status.code(), Some(0));
}
/// A loopback provider endpoint that holds its response: it reports each
/// received request, then answers only when released.
struct StalledProvider {
url: String,
received: Receiver<()>,
release: mpsc::Sender<()>,
requests: Arc<AtomicUsize>,
}
impl StalledProvider {
fn start(drop_response: bool, complete_response: bool) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let (received_tx, received) = mpsc::channel();
let (release, release_rx) = mpsc::channel::<()>();
let requests = Arc::new(AtomicUsize::new(0));
let request_count = Arc::clone(&requests);
std::thread::spawn(move || {
let Ok((stream, _)) = listener.accept() else {
return;
};
let mut reader = BufReader::new(stream);
let mut length = 0;
loop {
let mut header = String::new();
if reader.read_line(&mut header).unwrap_or(0) == 0 {
return;
}
let header = header.trim_end();
if header.is_empty() {
break;
}
if let Some((name, value)) = header.split_once(':')
&& name.eq_ignore_ascii_case("content-length")
{
length = value.trim().parse().unwrap();
}
}
let mut body = vec![0; length];
reader.read_exact(&mut body).unwrap();
request_count.fetch_add(1, Ordering::SeqCst);
let _ = received_tx.send(());
if release_rx.recv().is_err() || drop_response {
return;
}
let body = if complete_response {
format!(
"data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
json!({"choices": [{"index": 0, "delta": {"content": "late"}}]}),
json!({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]})
)
} else {
format!(
"data: {}\n\n",
json!({"choices": [{"index": 0, "delta": {"content": "late"}}]})
)
};
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{body}",
body.len()
);
let _ = reader.get_mut().write_all(response.as_bytes());
});
Self {
url,
received,
release,
requests,
}
}
}
fn spawn_unread_stdout(
name: &str,
provider: &StalledProvider,
) -> (Child, ChildStdin, ChildStdout, PathBuf) {
let home = std::env::temp_dir().join(format!("smith-rpc-test-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
let mut child = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args([
"rpc",
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&provider.url,
])
.env("XDG_DATA_HOME", &home)
.env("OPENAI_API_KEY", "sk-e2e-fixture")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.unwrap();
let stdin = child.stdin.take().unwrap();
let stdout = child.stdout.take().unwrap();
(child, stdin, stdout, home)
}
fn acknowledge_and_return_stdout(stdout: ChildStdout) -> (Value, ChildStdout) {
let (sender, receiver) = mpsc::sync_channel(1);
let reader = std::thread::spawn(move || {
let mut reader = BufReader::new(stdout);
let mut line = String::new();
assert_ne!(
reader.read_line(&mut line).unwrap(),
0,
"RPC ended before ACK"
);
sender
.send((serde_json::from_str(&line).unwrap(), reader.into_inner()))
.unwrap();
});
let acknowledgement = receiver
.recv_timeout(STARTUP_DEADLINE)
.expect("RPC ACK within startup deadline");
reader.join().unwrap();
acknowledgement
}
fn wait_for_exit(mut child: Child) -> std::process::ExitStatus {
let (sender, receiver) = mpsc::sync_channel(1);
let waiter = std::thread::spawn(move || sender.send(child.wait().unwrap()).unwrap());
let status = receiver
.recv_timeout(EXIT_DEADLINE)
.expect("RPC child exit within exit deadline");
waiter.join().unwrap();
status
}
#[test]
fn rpc_provider_control_plane_remains_available_until_explicit_release() {
let provider = StalledProvider::start(false, true);
let mut rpc = Rpc::spawn(
"provider-control-plane",
&[
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&provider.url,
],
&[("OPENAI_API_KEY", "sk-e2e-fixture")],
);
rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"block","key":"once"}}"#);
let start = rpc.next();
assert_eq!(start["id"], 1);
assert_eq!(start["event"], "start");
let session = start["receipt"]["session_id"].as_str().unwrap().to_owned();
let submission = start["receipt"]["submission_id"]
.as_str()
.unwrap()
.to_owned();
provider
.received
.recv_timeout(STARTUP_DEADLINE)
.expect("provider request within startup deadline");
rpc.send(&format!(
r#"{{"id":2,"method":"inspect","params":{{"id":"{submission}"}}}}"#
));
let inspect = rpc.next();
assert_eq!(inspect["id"], 2);
assert_eq!(inspect["receipt"]["submission_id"], submission);
assert_eq!(inspect["receipt"]["status"], "Running");
rpc.send(r#"{"id":3,"method":"prompt","params":{"input":"other"}}"#);
assert_eq!(
rpc.next(),
json!({"id": 3, "error": {"code": "BUSY", "message": "a prompt is already running"}})
);
rpc.send(r#"{"id":4,"method":"prompt","params":{"input":"block","key":"once"}}"#);
let duplicate = rpc.next();
assert_eq!(duplicate["id"], 4);
assert_eq!(duplicate["event"], "start");
assert_eq!(duplicate["duplicate"], true);
assert_eq!(duplicate["receipt"]["submission_id"], submission);
rpc.send(r#"{"id":5,"method":"wait"}"#);
provider.release.send(()).unwrap();
assert_eq!(rpc.next()["id"], 1);
let end = rpc.next();
assert_eq!(end["id"], 1);
assert_eq!(end["event"], "end");
assert_eq!(end["reason"], "EndTurn");
assert_eq!(rpc.next(), json!({"id": 5, "settled": true}));
assert_eq!(provider.requests.load(Ordering::SeqCst), 1);
let home = rpc.home.clone();
assert!(rpc.close().success());
let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "receipt", &session, &submission])
.env("XDG_DATA_HOME", &home)
.output()
.unwrap();
assert!(
receipt.status.success(),
"{}",
String::from_utf8_lossy(&receipt.stderr)
);
let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
assert_eq!(receipt["state"], "terminal");
assert_eq!(receipt["outcome"], "completed");
}
#[test]
fn rpc_response_loss_after_ack_never_fabricates_abort_or_replays() {
let provider = StalledProvider::start(false, true);
let (child, mut stdin, stdout, home) =
spawn_unread_stdout("response-loss-after-ack", &provider);
stdin
.write_all(
b"{\"id\":1,\"method\":\"prompt\",\"params\":{\"input\":\"drop\",\"key\":\"once\"}}\n",
)
.unwrap();
stdin.flush().unwrap();
let (acknowledgement, stdout) = acknowledge_and_return_stdout(stdout);
assert_eq!(acknowledgement["id"], 1);
assert_eq!(acknowledgement["event"], "start");
let session = acknowledgement["receipt"]["session_id"]
.as_str()
.unwrap()
.to_owned();
let submission = acknowledgement["receipt"]["submission_id"]
.as_str()
.unwrap()
.to_owned();
provider
.received
.recv_timeout(STARTUP_DEADLINE)
.expect("provider request within startup deadline");
drop(stdout);
provider.release.send(()).unwrap();
drop(stdin);
let exit = wait_for_exit(child);
assert!(
!exit.success(),
"closed stdout must reach the child as a write failure"
);
assert_eq!(provider.requests.load(Ordering::SeqCst), 1);
let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "receipt", &session, &submission])
.env("XDG_DATA_HOME", &home)
.output()
.unwrap();
assert!(
receipt.status.success(),
"{}",
String::from_utf8_lossy(&receipt.stderr)
);
let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
assert_ne!(receipt["outcome"], "aborted");
match receipt["state"].as_str() {
Some("terminal") => assert_eq!(receipt["outcome"], "completed"),
Some("running") => assert!(receipt["outcome"].is_null()),
state => panic!("unexpected post-broken-pipe receipt state: {state:?}"),
}
}
#[test]
fn rpc_response_loss_is_terminal_after_reopen_inspection_without_reschedule() {
let provider = StalledProvider::start(true, false);
let mut rpc = Rpc::spawn(
"response-loss",
&[
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&provider.url,
],
&[("OPENAI_API_KEY", "sk-e2e-fixture")],
);
rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"drop"}}"#);
let start = rpc.next();
let session = start["receipt"]["session_id"].as_str().unwrap().to_owned();
let submission = start["receipt"]["submission_id"]
.as_str()
.unwrap()
.to_owned();
provider
.received
.recv_timeout(DEADLINE)
.expect("provider request within the deadline");
provider.release.send(()).unwrap();
assert_eq!(rpc.next()["error"]["code"], "TURN_FAILED");
let home = rpc.home.clone();
assert!(rpc.close().success());
let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "receipt", &session, &submission])
.env("XDG_DATA_HOME", &home)
.output()
.unwrap();
assert!(
receipt.status.success(),
"{}",
String::from_utf8_lossy(&receipt.stderr)
);
let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
assert_eq!(receipt["state"], "terminal");
assert_eq!(receipt["outcome"], "failed");
std::fs::remove_dir_all(home).unwrap();
}
fn provider_request(stream: std::net::TcpStream) -> (std::net::TcpStream, Value) {
stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap();
stream.set_write_timeout(Some(STARTUP_DEADLINE)).unwrap();
let mut reader = BufReader::new(stream);
let mut length = 0;
loop {
let mut header = String::new();
assert_ne!(reader.read_line(&mut header).unwrap(), 0);
if header.trim().is_empty() {
break;
}
if let Some((name, value)) = header.split_once(':')
&& name.eq_ignore_ascii_case("content-length")
{
length = value.trim().parse().unwrap();
}
}
let mut body = vec![0; length];
reader.read_exact(&mut body).unwrap();
(reader.into_inner(), serde_json::from_slice(&body).unwrap())
}
#[test]
fn rpc_partial_write_abort_preserves_text_usage_receipt_and_keyed_retry() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let (second, received) = mpsc::sync_channel(1);
let (release, released) = mpsc::sync_channel(1);
let server = std::thread::spawn(move || {
let (mut stream, _) = provider_request(listener.accept().unwrap().0);
let body = format!(
"data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
json!({"choices":[{"index":0,"delta":{"content":"committed prefix","tool_calls":[{"index":0,"id":"write-call","function":{"name":"write","arguments":"{\"path\":\"committed.txt\",\"content\":\"committed effect\\n\"}"}}]}}]}),
json!({"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":23,"completion_tokens":29}})
);
write!(stream, "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{body}", body.len()).unwrap();
drop(stream);
let (mut stream, request) = provider_request(listener.accept().unwrap().0);
stream
.write_all(
b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n",
)
.unwrap();
stream.flush().unwrap();
second.send(request).unwrap();
released.recv_timeout(STARTUP_DEADLINE).unwrap();
drop(stream);
});
let mut rpc = Rpc::spawn(
"partial-write-abort",
&[
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&url,
],
&[("OPENAI_API_KEY", "sk-e2e-fixture")],
);
rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#);
let start = rpc.next();
assert_eq!(start["event"], "start");
assert_eq!(
rpc.next(),
json!({"id":1,"event":"delta","text":"committed prefix"})
);
let request = match received.recv_timeout(STARTUP_DEADLINE) {
Ok(request) => request,
Err(error) => panic!(
"second provider request was not received: {error}; {}",
rpc.request_diagnostics(start["receipt"]["session_id"].as_str().unwrap())
),
};
assert!(
request["messages"]
.as_array()
.unwrap()
.iter()
.any(|message| message["role"] == "tool")
);
assert!(request.to_string().contains("committed prefix"));
assert_eq!(
std::fs::read_to_string(rpc.home.join("committed.txt")).unwrap(),
"committed effect\n"
);
rpc.send(r#"{"id":2,"method":"abort"}"#);
assert_eq!(rpc.next()["aborting"], true);
release.send(()).unwrap();
server.join().unwrap();
assert_eq!(
rpc.next(),
json!({"id":1,"event":"end","text":"committed prefix","reason":"Abort","cost":{"input_tokens":23,"output_tokens":29},"cost_complete":false})
);
let submission = start["receipt"]["submission_id"].as_str().unwrap();
rpc.send(&json!({"id":3,"method":"inspect","params":{"id":submission}}).to_string());
let receipt = rpc.next();
assert_eq!(receipt["receipt"]["status"], "Terminal(Some(Abort))");
assert_eq!(receipt["receipt"]["cancel_requested"], true);
rpc.send(r#"{"id":4,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#);
let duplicate = rpc.next();
assert_eq!(duplicate["duplicate"], true);
assert_eq!(duplicate["receipt"]["submission_id"], submission);
let home = rpc.home.clone();
assert!(rpc.close().success());
let session = start["receipt"]["session_id"].as_str().unwrap();
let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "get", session, "--json"])
.env("XDG_DATA_HOME", &home)
.output()
.unwrap();
assert!(
inspection.status.success(),
"{}",
String::from_utf8_lossy(&inspection.stderr)
);
let entries = String::from_utf8(inspection.stdout).unwrap();
assert!(entries.contains("committed prefix"));
assert!(!entries.contains("sk-e2e-fixture"));
// Reopening the keyed receipt never dispatches to the now-closed endpoint.
let mut reopened = Rpc::spawn(
"partial-write-reopened",
&["--session", session],
&[("XDG_DATA_HOME", home.to_str().unwrap())],
);
// Reopen by durable session path, independent of the new isolated HOME.
reopened
.send(r#"{"id":5,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#);
let retry = reopened.next();
assert_eq!(retry["duplicate"], true);
assert_eq!(retry["receipt"]["submission_id"], submission);
assert!(reopened.close().success());
assert_eq!(
std::fs::read_to_string(home.join("committed.txt")).unwrap(),
"committed effect\n"
);
}
#[test]
fn rpc_live_credential_echo_is_redacted_before_terminal_and_session() {
assert_live_prefix(
"live-prefix",
"sk-e2e-fixture",
"sk-e2e-fixture",
"{{secret:connection-api-key}}",
);
}
#[test]
fn rpc_large_credential_echo_is_redacted_live() {
let credential = "a".repeat(4096);
assert_live_prefix(
"large-live-secret",
&credential,
&credential,
"{{secret:connection-api-key}}",
);
}
#[test]
fn rpc_large_auth_credential_preserves_live_public_progress() {
let credential = "a".repeat(4096);
let public = "public.".repeat(128);
assert_live_prefix("large-auth-public", &credential, &public, &public);
}
fn assert_live_prefix(name: &str, credential: &str, text: &str, expected: &str) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let (release, released) = mpsc::sync_channel(1);
let server_text = text.to_owned();
let server = std::thread::spawn(move || {
let (stream, _) = listener.accept().unwrap();
stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap();
stream.set_write_timeout(Some(STARTUP_DEADLINE)).unwrap();
let mut reader = BufReader::new(stream);
let mut length = 0;
loop {
let mut header = String::new();
reader.read_line(&mut header).unwrap();
if header.trim().is_empty() {
break;
}
if let Some((name, value)) = header.split_once(':')
&& name.eq_ignore_ascii_case("content-length")
{
length = value.trim().parse().unwrap();
}
}
reader.read_exact(&mut vec![0; length]).unwrap();
let stream = reader.get_mut();
stream
.write_all(
b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n",
)
.unwrap();
let delta = format!(
"data: {}\n\n",
json!({"choices":[{"index":0,"delta":{"content":server_text}}]})
);
let split = delta.find(server_text.as_str()).unwrap() + server_text.len() / 2;
stream.write_all(&delta.as_bytes()[..split]).unwrap();
stream.flush().unwrap();
stream.write_all(&delta.as_bytes()[split..]).unwrap();
stream.flush().unwrap();
if released.recv_timeout(STARTUP_DEADLINE).is_ok() {
write!(stream, "data: {}\n\ndata: [DONE]\n\n", json!({"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":23,"completion_tokens":29}})).unwrap();
}
});
let mut rpc = Rpc::spawn(
name,
&[
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&url,
],
&[("OPENAI_API_KEY", credential)],
);
rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"stream"}}"#);
let start = rpc.next();
assert_eq!(start["event"], "start");
let prefix = rpc.lines.recv_timeout(DEADLINE);
// Release and reap even when the regression assertion fails.
release.send(()).unwrap();
server.join().unwrap();
let prefix: Value =
serde_json::from_str(&prefix.expect("live delta before provider release").unwrap())
.unwrap();
assert_eq!(prefix, json!({"id":1,"event":"delta","text":expected}));
assert!(!prefix.to_string().contains(credential));
let end = rpc.next();
assert_eq!(end["event"], "end");
assert_eq!(end["text"], expected);
assert!(!end.to_string().contains(credential));
let session = start["receipt"]["session_id"].as_str().unwrap().to_string();
let home = rpc.home.clone();
assert!(rpc.close().success());
let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "get", &session, "--json"])
.env("XDG_DATA_HOME", &home)
.output()
.unwrap();
assert!(inspection.status.success());
assert!(
!String::from_utf8(inspection.stdout)
.unwrap()
.contains(credential)
);
}
#[test]
fn rpc_unsplit_credential_in_provider_error_never_reaches_rpc_or_session_bytes() {
provider_error_never_reaches_rpc_or_session_bytes(false, 200);
}
#[test]
fn rpc_json_escaped_credential_in_provider_error_never_reaches_rpc_or_session_bytes() {
provider_error_never_reaches_rpc_or_session_bytes(true, 200);
}
#[test]
fn rpc_non_success_provider_error_reports_sanitized_vendor_diagnostic() {
provider_error_never_reaches_rpc_or_session_bytes(false, 429);
}
#[test]
fn rpc_non_success_json_escaped_credential_never_reaches_rpc_or_session_bytes() {
provider_error_never_reaches_rpc_or_session_bytes(true, 429);
}
fn provider_error_never_reaches_rpc_or_session_bytes(escaped: bool, status: u16) {
let credential = "sk-e2e-fixture";
let escaped_credential = r"\u0073k-e2e-\u0066ixture";
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let server = std::thread::spawn(move || {
let (stream, _) = listener.accept().unwrap();
stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap();
let mut reader = BufReader::new(stream);
let mut content_length = 0;
loop {
let mut header = String::new();
reader.read_line(&mut header).unwrap();
if header.trim().is_empty() {
break;
}
if let Some((name, value)) = header.split_once(':')
&& name.eq_ignore_ascii_case("content-length")
{
content_length = value.trim().parse::<usize>().unwrap();
}
}
assert!(
content_length <= 1024 * 1024,
"provider request exceeded fixture bound"
);
let mut request_body = vec![0; content_length];
reader.read_exact(&mut request_body).unwrap();
assert!(
request_body
.windows(b"error".len())
.any(|window| window == b"error")
);
let error = if status == 200 {
format!("data: {}\n\n", json!({"error":{"message":credential}}))
} else {
json!({"error":{"code":"rate_limited","message":credential}}).to_string()
};
let error = if escaped {
error.replace(credential, escaped_credential)
} else {
error
};
let stream = reader.get_mut();
write!(stream, "HTTP/1.1 {status} fixture\r\ncontent-type: application/json\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{error}", error.len()).unwrap();
});
let mut rpc = Rpc::spawn(
"error-redaction",
&[
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&url,
],
&[("OPENAI_API_KEY", credential)],
);
rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"error"}}"#);
let start = rpc.next();
assert_eq!(start["event"], "start");
let end = rpc.next();
server.join().unwrap();
assert_eq!(end["event"], "end");
assert_eq!(end["error"]["code"], "TURN_FAILED");
assert!(!end.to_string().contains(credential));
assert!(!end.to_string().contains(escaped_credential));
if status != 200 {
assert!(
end.to_string()
.contains("provider returned HTTP 429: rate_limited"),
"unexpected sanitized terminal outcome: code={}, error={}",
end["error"]["code"],
end["error"]["message"]
);
}
let session = start["receipt"]["session_id"].as_str().unwrap();
let session_path = rpc
.home
.join("smith/sessions/run")
.join(format!("{session}.smh"));
let session_bytes = std::fs::read(session_path).unwrap();
for secret in [credential, escaped_credential] {
assert!(
!session_bytes
.windows(secret.len())
.any(|bytes| bytes == secret.as_bytes())
);
}
let home = rpc.home.clone();
let (exit, stderr) = rpc.close_with_stderr();
assert!(exit.success(), "{stderr}");
for secret in [credential, escaped_credential] {
assert!(!stderr.contains(secret));
}
let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "get", session, "--json"])
.env("XDG_DATA_HOME", home)
.output()
.unwrap();
assert!(inspection.status.success());
let inspection = String::from_utf8(inspection.stdout).unwrap();
for secret in [credential, escaped_credential] {
assert!(!inspection.contains(secret));
}
if status != 200 {
assert!(inspection.contains("\"status\":429"));
assert!(inspection.contains("\"response_bytes\":"));
assert!(inspection.contains("\"outcome\":\"failed\""));
}
}
#[test]
fn rpc_cancellation_receipt_does_not_hide_response_head_failure() {
let provider = StalledProvider::start(true, false);
let mut rpc = Rpc::spawn(
"cancel-head-failure",
&[
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&provider.url,
],
&[("OPENAI_API_KEY", "sk-e2e-fixture")],
);
rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"block"}}"#);
let start = rpc.next();
assert_eq!(start["event"], "start");
if let Err(error) = provider.received.recv_timeout(DEADLINE) {
panic!(
"provider did not receive the first request within {DEADLINE:?}: {error}; {}",
rpc.request_diagnostics(start["receipt"]["session_id"].as_str().unwrap())
);
}
rpc.send(r#"{"id":2,"method":"abort"}"#);
assert_eq!(rpc.next()["aborting"], true);
provider.release.send(()).unwrap();
let end = rpc.next();
assert_eq!(end["event"], "end");
assert_eq!(end["error"]["code"], "TURN_FAILED");
assert!(end["reason"].is_null());
rpc.send(
&json!({"id":3,"method":"inspect","params":{"id":start["receipt"]["submission_id"]}})
.to_string(),
);
let receipt = rpc.next();
assert_eq!(receipt["receipt"]["cancel_requested"], true);
assert_eq!(receipt["receipt"]["status"], "Terminal(Some(Abort))");
assert!(rpc.close().success());
}
#[test]
fn rpc_abort_reports_durable_abort_outcome() {
let provider = StalledProvider::start(false, false);
let mut rpc = Rpc::spawn(
"abort",
&[
"--provider",
"openai",
"--model",
"e2e-model",
"--base-url",
&provider.url,
],
&[("OPENAI_API_KEY", "sk-e2e-fixture")],
);
// Each step waits for its reply before the next; the provider holds its
// response until the abort is acknowledged, so the order is fixed.
rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "block"}}"#);
let start = rpc.next();
assert_eq!(start["id"], 1);
assert_eq!(start["event"], "start");
provider
.received
.recv_timeout(DEADLINE)
.expect("provider request within the deadline");
rpc.send(r#"{"id": 2, "method": "abort"}"#);
let abort = rpc.next();
assert_eq!(abort["id"], 2);
assert_eq!(abort["aborting"], true);
provider.release.send(()).unwrap();
assert_eq!(
rpc.next(),
json!({
"id": 1,
"event": "end",
"text": "",
"reason": "Abort",
"cost": null,
"cost_complete": false,
})
);
// `wait` answers once the aborted turn has settled; `status` then agrees.
rpc.send(r#"{"id": 3, "method": "wait"}"#);
assert_eq!(rpc.next(), json!({"id": 3, "settled": true}));
rpc.send(r#"{"id": 4, "method": "status"}"#);
assert_eq!(
rpc.next(),
json!({"id": 4, "settled": true, "access_mode": "writer", "active_submission_id": null})
);
insta::assert_snapshot!(
"rpc_prompt_then_abort_then_wait",
transcript_for_snapshot(&rpc.transcript)
);
let session = start["receipt"]["session_id"].as_str().unwrap().to_owned();
let submission = start["receipt"]["submission_id"]
.as_str()
.unwrap()
.to_owned();
let home = rpc.home.clone();
assert!(rpc.close().success());
let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
.args(["sessions", "receipt", &session, &submission])
.env("XDG_DATA_HOME", &home)
.output()
.unwrap();
assert!(receipt.status.success());
let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
assert_eq!(receipt["state"], "terminal");
assert_eq!(receipt["outcome"], "cancelled");
let mut entries = std::fs::read_dir(&home)
.unwrap()
.map(|entry| entry.unwrap().file_name())
.collect::<Vec<_>>();
entries.sort();
assert_eq!(entries, ["runtime", "smith"]);
}