//! End-to-end `smith rpc` workflows over piped standard input and output //! (`SMH-SPEC-SPEC0001`, Interface modes: JSON-RPC). //! //! Every read waits on a line or end of output with a deadline; nothing //! sleeps or polls. use serde_json::{Value, json}; use std::io::{BufRead, BufReader, Read, Write}; use std::net::TcpListener; use std::path::PathBuf; use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{ Arc, mpsc::{self, Receiver}, }; use std::time::Duration; const DEADLINE: Duration = Duration::from_secs(10); const STARTUP_DEADLINE: Duration = Duration::from_secs(10); const EXIT_DEADLINE: Duration = Duration::from_secs(10); const DIAGNOSTIC_QUEUE_LINES: usize = 32; const DIAGNOSTIC_LINE_BYTES: usize = 4 * 1024; const MOCK_REPLY: &str = "rpc mock provider ready"; static RPC_HOME_INDEX: AtomicUsize = AtomicUsize::new(0); /// One running `smith rpc` process; killed on drop if a test fails early. struct Rpc { child: Child, home: PathBuf, stdin: Option, /// Raw stdout lines; `None` marks end of output. lines: Receiver>, /// Child diagnostics emitted before a fixture deadline expires. stderr: Receiver, /// Every line sent (`> `) and received (`< `), in order. transcript: Vec, } impl Rpc { fn spawn(_name: &str, args: &[&str], envs: &[(&str, &str)]) -> Self { let index = RPC_HOME_INDEX.fetch_add(1, Ordering::Relaxed); let home = PathBuf::from("/var/tmp/smith-session-e2e") .join(format!("r-{}-{index}", std::process::id())); let _ = std::fs::remove_dir_all(&home); std::fs::create_dir_all(&home).unwrap(); let mut child = Command::new(assert_cmd::cargo::cargo_bin("smith")) .arg("rpc") .args(args) .current_dir(&home) .env("HOME", &home) .env("XDG_CACHE_HOME", home.join("cache")) .env("XDG_CONFIG_HOME", home.join("config")) .env("XDG_DATA_HOME", &home) .env("XDG_RUNTIME_DIR", home.join("runtime")) .env("XDG_STATE_HOME", home.join("state")) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .env_remove("ANTHROPIC_API_KEY") .env_remove("OPENAI_API_KEY") .env_remove("GEMINI_API_KEY") .envs(envs.iter().copied()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .unwrap(); let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); let (sender, lines) = mpsc::channel(); std::thread::spawn(move || { for line in BufReader::new(stdout).lines() { if sender.send(Some(line.unwrap())).is_err() { return; } } let _ = sender.send(None); }); let (stderr_sender, stderr_lines) = mpsc::channel(); std::thread::spawn(move || { for line in BufReader::new(stderr).lines() { if stderr_sender.send(line.unwrap()).is_err() { return; } } }); let stdin = child.stdin.take(); Self { child, home, stdin, lines, stderr: stderr_lines, transcript: Vec::new(), } } fn mock(name: &str) -> Self { Self::spawn(name, &[], &[]) } fn send(&mut self, line: &str) { self.transcript.push(format!("> {line}")); let stdin = self.stdin.as_mut().unwrap(); stdin.write_all(line.as_bytes()).unwrap(); stdin.write_all(b"\n").unwrap(); stdin.flush().unwrap(); } fn next(&mut self) -> Value { match self.lines.recv_timeout(STARTUP_DEADLINE) { Ok(Some(line)) => { let value = serde_json::from_str(&line) .unwrap_or_else(|error| panic!("stdout line is not JSON ({error}): {line}")); self.transcript.push(format!("< {line}")); value } Ok(None) => panic!("rpc output ended early"), Err(error) => panic!("no rpc line within {STARTUP_DEADLINE:?}: {error}"), } } fn request_diagnostics(&mut self, session: &str) -> String { let queued_stdout = self .lines .try_iter() .take(DIAGNOSTIC_QUEUE_LINES) .map(|line| line.map(diagnostic_line)) .collect::>(); let stderr = self .stderr .try_iter() .take(DIAGNOSTIC_QUEUE_LINES) .map(diagnostic_line) .collect::>() .join("\\n"); let journal = assert_cmd::Command::new(assert_cmd::cargo::cargo_bin("smith")) .timeout(DEADLINE) .args(["sessions", "get", session, "--json"]) .env("XDG_DATA_HOME", &self.home) .output() .map_or_else( |error| format!("inspection spawn failed: {error}"), |output| { format!( "status={}; stdout={}; stderr={}", output.status, String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ) }, ); format!( "rpc status={:?}; transcript={:?}; queued_stdout={queued_stdout:?}; stderr={stderr:?}; committed.txt={:?}; journal={journal}", self.child.try_wait(), self.transcript, std::fs::read_to_string(self.home.join("committed.txt")), ) } /// Close stdin, then expect end of output and the exit status. fn close(self) -> std::process::ExitStatus { self.close_with_stderr().0 } fn close_with_stderr(mut self) -> (std::process::ExitStatus, String) { drop(self.stdin.take()); match self.lines.recv_timeout(EXIT_DEADLINE) { Ok(None) => {} Ok(Some(line)) => panic!("unexpected rpc line after close: {line}"), Err(error) => panic!("rpc did not end output within {EXIT_DEADLINE:?}: {error}"), } let status = self.child.wait().unwrap(); (status, self.stderr.iter().collect::>().join("\n")) } } fn diagnostic_line(line: String) -> String { if line.len() <= DIAGNOSTIC_LINE_BYTES { return line; } let end = line .char_indices() .take_while(|(index, _)| *index <= DIAGNOSTIC_LINE_BYTES) .map(|(index, _)| index) .last() .unwrap_or(0); format!("{}… [truncated]", &line[..end]) } fn transcript_for_snapshot(lines: &[String]) -> String { lines .iter() .map(|line| { let Some(json) = line.strip_prefix("< ") else { return line.clone(); }; let Ok(mut value) = serde_json::from_str::(json) else { return line.clone(); }; if let Some(receipt) = value.get_mut("receipt").and_then(Value::as_object_mut) { receipt.insert("session_id".into(), json!("")); receipt.insert("submission_id".into(), json!("")); } if value.get("submission_id").is_some() { value["submission_id"] = json!(""); } format!("< {value}") }) .collect::>() .join("\n") } impl Drop for Rpc { fn drop(&mut self) { if matches!(self.child.try_wait(), Ok(None)) { let _ = self.child.kill(); let _ = self.child.wait(); } } } #[test] fn rpc_event_and_shutdown_deadlines_stay_at_ten_seconds() { assert_eq!(STARTUP_DEADLINE, DEADLINE); assert_eq!(EXIT_DEADLINE, DEADLINE); } #[test] fn rpc_rejects_oversized_stdin_line_without_protocol_output() { let mut command = assert_cmd::Command::cargo_bin("smith").unwrap(); command .arg("rpc") .env( "XDG_DATA_HOME", std::env::temp_dir().join(format!("smith-rpc-test-oversized-{}", std::process::id())), ) .env_remove("SMITH_PROVIDER") .env_remove("SMITH_MODEL") .env_remove("SMITH_BASE_URL") .write_stdin("x".repeat(1024 * 1024 + 1)) .assert() .failure() .stdout("") .stderr(predicates::str::contains( "JSON-RPC request exceeds 1048576 bytes", )); } #[test] fn rpc_ping_correlates_the_request_id() { let mut rpc = Rpc::mock("ping"); rpc.send(r#"{"id": 7, "method": "ping"}"#); assert_eq!(rpc.next(), json!({"id": 7, "pong": true})); rpc.send(r#"{"id": "seven", "method": "ping"}"#); assert_eq!(rpc.next(), json!({"id": "seven", "pong": true})); assert!(rpc.close().success()); } #[test] fn rpc_prompt_streams_start_delta_end_then_wait_and_status_report_settled() { let mut rpc = Rpc::mock("prompt"); rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "hi"}}"#); rpc.send(r#"{"id": 2, "method": "wait"}"#); let start = rpc.next(); assert_eq!(start["id"], 1); assert_eq!(start["event"], "start"); assert_eq!(start["receipt"]["status"], "Queued"); assert_eq!(start["receipt"]["cancel_requested"], false); assert!(start["receipt"]["submission_id"].is_string()); assert_eq!( rpc.next(), json!({"id": 1, "event": "delta", "text": MOCK_REPLY}) ); assert_eq!( rpc.next(), json!({ "id": 1, "event": "end", "text": MOCK_REPLY, "reason": "EndTurn", "cost": null, "cost_complete": false, }) ); // The worker answers `wait` only after the prompt has settled. assert_eq!(rpc.next(), json!({"id": 2, "settled": true})); // `wait` answers once the aborted turn has settled; `status` then agrees. rpc.send(r#"{"id": 3, "method": "wait"}"#); assert_eq!(rpc.next(), json!({"id": 3, "settled": true})); rpc.send(r#"{"id": 4, "method": "status"}"#); assert_eq!( rpc.next(), json!({"id": 4, "settled": true, "access_mode": "writer", "active_submission_id": null}) ); assert!(rpc.close().success()); } #[test] fn rpc_read_only_rejects_prompt_and_abort_without_mutating_the_session() { let home = std::env::temp_dir().join(format!("smith-rpc-read-only-{}", std::process::id())); let _ = std::fs::remove_dir_all(&home); let seed = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["eval", "--mock", "seed"]) .env("XDG_DATA_HOME", &home) .env_remove("SMITH_PROVIDER") .output() .unwrap(); assert!( seed.status.success(), "{}", String::from_utf8_lossy(&seed.stderr) ); let listed = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "list", "--json"]) .env("XDG_DATA_HOME", &home) .output() .unwrap(); let session = serde_json::from_slice::(&listed.stdout).unwrap()["session"] .as_str() .unwrap() .to_owned(); let path = home .join("smith/sessions/run") .join(format!("{session}.smh")); let before = std::fs::read(&path).unwrap(); let output = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args([ "rpc", "--session", &session, "--read-only", "--provider", "openai", "--model", "must-not-initialize", "--base-url", "http://127.0.0.1:1", ]) .env("XDG_DATA_HOME", &home) .env_remove("OPENAI_API_KEY") .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() .unwrap(); let mut child = output; let mut stdin = child.stdin.take().unwrap(); stdin .write_all( b"{\"id\":1,\"method\":\"status\"}\n{\"id\":2,\"method\":\"prompt\",\"params\":{\"input\":\"denied\"}}\n{\"id\":3,\"method\":\"abort\"}\n{\"id\":4,\"method\":\"inspect\",\"params\":{\"id\":\"babab-babab-babab-babab-babab-babab-babab-babab\"}}\n", ) .unwrap(); drop(stdin); let output = child.wait_with_output().unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); let records: Vec = String::from_utf8(output.stdout) .unwrap() .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!( records[0], json!({"id": 1, "settled": true, "access_mode": "read_only", "active_submission_id": null}) ); assert_eq!(records[1]["error"]["code"], "STATE"); assert_eq!(records[2]["error"]["code"], "STATE"); assert_eq!(records[3], json!({"id": 4, "receipt": null})); assert_eq!(std::fs::read(&path).unwrap(), before); std::fs::remove_dir_all(home).unwrap(); } #[test] fn rpc_malformed_line_reports_parse_error_with_null_id() { let mut rpc = Rpc::mock("malformed"); rpc.send("not json"); let record = rpc.next(); assert_eq!(record["id"], Value::Null); assert_eq!(record["error"]["code"], "PARSE"); // The server keeps serving after a bad line. rpc.send(r#"{"id": 1, "method": "ping"}"#); assert_eq!(rpc.next(), json!({"id": 1, "pong": true})); assert!(rpc.close().success()); } #[test] fn rpc_unknown_method_reports_method_unknown() { let mut rpc = Rpc::mock("unknown"); rpc.send(r#"{"id": "x", "method": "explode"}"#); assert_eq!( rpc.next(), json!({ "id": "x", "error": {"code": "METHOD_UNKNOWN", "message": "unknown method explode"}, }) ); assert!(rpc.close().success()); } #[test] fn rpc_closing_stdin_after_settle_exits_zero() { let mut rpc = Rpc::mock("shutdown"); rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "hi"}}"#); rpc.send(r#"{"id": 2, "method": "wait"}"#); let mut records = Vec::new(); loop { let record = rpc.next(); let done = record["id"] == 2; records.push(record); if done { break; } } assert!(records.iter().any(|r| r["id"] == 1 && r["event"] == "end")); let status = rpc.close(); assert_eq!(status.code(), Some(0)); } /// A loopback provider endpoint that holds its response: it reports each /// received request, then answers only when released. struct StalledProvider { url: String, received: Receiver<()>, release: mpsc::Sender<()>, requests: Arc, } impl StalledProvider { fn start(drop_response: bool, complete_response: bool) -> Self { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let (received_tx, received) = mpsc::channel(); let (release, release_rx) = mpsc::channel::<()>(); let requests = Arc::new(AtomicUsize::new(0)); let request_count = Arc::clone(&requests); std::thread::spawn(move || { let Ok((stream, _)) = listener.accept() else { return; }; let mut reader = BufReader::new(stream); let mut length = 0; loop { let mut header = String::new(); if reader.read_line(&mut header).unwrap_or(0) == 0 { return; } let header = header.trim_end(); if header.is_empty() { break; } if let Some((name, value)) = header.split_once(':') && name.eq_ignore_ascii_case("content-length") { length = value.trim().parse().unwrap(); } } let mut body = vec![0; length]; reader.read_exact(&mut body).unwrap(); request_count.fetch_add(1, Ordering::SeqCst); let _ = received_tx.send(()); if release_rx.recv().is_err() || drop_response { return; } let body = if complete_response { format!( "data: {}\n\ndata: {}\n\ndata: [DONE]\n\n", json!({"choices": [{"index": 0, "delta": {"content": "late"}}]}), json!({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}) ) } else { format!( "data: {}\n\n", json!({"choices": [{"index": 0, "delta": {"content": "late"}}]}) ) }; let response = format!( "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{body}", body.len() ); let _ = reader.get_mut().write_all(response.as_bytes()); }); Self { url, received, release, requests, } } } fn spawn_unread_stdout( name: &str, provider: &StalledProvider, ) -> (Child, ChildStdin, ChildStdout, PathBuf) { let home = std::env::temp_dir().join(format!("smith-rpc-test-{name}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&home); let mut child = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args([ "rpc", "--provider", "openai", "--model", "e2e-model", "--base-url", &provider.url, ]) .env("XDG_DATA_HOME", &home) .env("OPENAI_API_KEY", "sk-e2e-fixture") .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() .unwrap(); let stdin = child.stdin.take().unwrap(); let stdout = child.stdout.take().unwrap(); (child, stdin, stdout, home) } fn acknowledge_and_return_stdout(stdout: ChildStdout) -> (Value, ChildStdout) { let (sender, receiver) = mpsc::sync_channel(1); let reader = std::thread::spawn(move || { let mut reader = BufReader::new(stdout); let mut line = String::new(); assert_ne!( reader.read_line(&mut line).unwrap(), 0, "RPC ended before ACK" ); sender .send((serde_json::from_str(&line).unwrap(), reader.into_inner())) .unwrap(); }); let acknowledgement = receiver .recv_timeout(STARTUP_DEADLINE) .expect("RPC ACK within startup deadline"); reader.join().unwrap(); acknowledgement } fn wait_for_exit(mut child: Child) -> std::process::ExitStatus { let (sender, receiver) = mpsc::sync_channel(1); let waiter = std::thread::spawn(move || sender.send(child.wait().unwrap()).unwrap()); let status = receiver .recv_timeout(EXIT_DEADLINE) .expect("RPC child exit within exit deadline"); waiter.join().unwrap(); status } #[test] fn rpc_provider_control_plane_remains_available_until_explicit_release() { let provider = StalledProvider::start(false, true); let mut rpc = Rpc::spawn( "provider-control-plane", &[ "--provider", "openai", "--model", "e2e-model", "--base-url", &provider.url, ], &[("OPENAI_API_KEY", "sk-e2e-fixture")], ); rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"block","key":"once"}}"#); let start = rpc.next(); assert_eq!(start["id"], 1); assert_eq!(start["event"], "start"); let session = start["receipt"]["session_id"].as_str().unwrap().to_owned(); let submission = start["receipt"]["submission_id"] .as_str() .unwrap() .to_owned(); provider .received .recv_timeout(STARTUP_DEADLINE) .expect("provider request within startup deadline"); rpc.send(&format!( r#"{{"id":2,"method":"inspect","params":{{"id":"{submission}"}}}}"# )); let inspect = rpc.next(); assert_eq!(inspect["id"], 2); assert_eq!(inspect["receipt"]["submission_id"], submission); assert_eq!(inspect["receipt"]["status"], "Running"); rpc.send(r#"{"id":3,"method":"prompt","params":{"input":"other"}}"#); assert_eq!( rpc.next(), json!({"id": 3, "error": {"code": "BUSY", "message": "a prompt is already running"}}) ); rpc.send(r#"{"id":4,"method":"prompt","params":{"input":"block","key":"once"}}"#); let duplicate = rpc.next(); assert_eq!(duplicate["id"], 4); assert_eq!(duplicate["event"], "start"); assert_eq!(duplicate["duplicate"], true); assert_eq!(duplicate["receipt"]["submission_id"], submission); rpc.send(r#"{"id":5,"method":"wait"}"#); provider.release.send(()).unwrap(); assert_eq!(rpc.next()["id"], 1); let end = rpc.next(); assert_eq!(end["id"], 1); assert_eq!(end["event"], "end"); assert_eq!(end["reason"], "EndTurn"); assert_eq!(rpc.next(), json!({"id": 5, "settled": true})); assert_eq!(provider.requests.load(Ordering::SeqCst), 1); let home = rpc.home.clone(); assert!(rpc.close().success()); let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "receipt", &session, &submission]) .env("XDG_DATA_HOME", &home) .output() .unwrap(); assert!( receipt.status.success(), "{}", String::from_utf8_lossy(&receipt.stderr) ); let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap(); assert_eq!(receipt["state"], "terminal"); assert_eq!(receipt["outcome"], "completed"); } #[test] fn rpc_response_loss_after_ack_never_fabricates_abort_or_replays() { let provider = StalledProvider::start(false, true); let (child, mut stdin, stdout, home) = spawn_unread_stdout("response-loss-after-ack", &provider); stdin .write_all( b"{\"id\":1,\"method\":\"prompt\",\"params\":{\"input\":\"drop\",\"key\":\"once\"}}\n", ) .unwrap(); stdin.flush().unwrap(); let (acknowledgement, stdout) = acknowledge_and_return_stdout(stdout); assert_eq!(acknowledgement["id"], 1); assert_eq!(acknowledgement["event"], "start"); let session = acknowledgement["receipt"]["session_id"] .as_str() .unwrap() .to_owned(); let submission = acknowledgement["receipt"]["submission_id"] .as_str() .unwrap() .to_owned(); provider .received .recv_timeout(STARTUP_DEADLINE) .expect("provider request within startup deadline"); drop(stdout); provider.release.send(()).unwrap(); drop(stdin); let exit = wait_for_exit(child); assert!( !exit.success(), "closed stdout must reach the child as a write failure" ); assert_eq!(provider.requests.load(Ordering::SeqCst), 1); let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "receipt", &session, &submission]) .env("XDG_DATA_HOME", &home) .output() .unwrap(); assert!( receipt.status.success(), "{}", String::from_utf8_lossy(&receipt.stderr) ); let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap(); assert_ne!(receipt["outcome"], "aborted"); match receipt["state"].as_str() { Some("terminal") => assert_eq!(receipt["outcome"], "completed"), Some("running") => assert!(receipt["outcome"].is_null()), state => panic!("unexpected post-broken-pipe receipt state: {state:?}"), } } #[test] fn rpc_response_loss_is_terminal_after_reopen_inspection_without_reschedule() { let provider = StalledProvider::start(true, false); let mut rpc = Rpc::spawn( "response-loss", &[ "--provider", "openai", "--model", "e2e-model", "--base-url", &provider.url, ], &[("OPENAI_API_KEY", "sk-e2e-fixture")], ); rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"drop"}}"#); let start = rpc.next(); let session = start["receipt"]["session_id"].as_str().unwrap().to_owned(); let submission = start["receipt"]["submission_id"] .as_str() .unwrap() .to_owned(); provider .received .recv_timeout(DEADLINE) .expect("provider request within the deadline"); provider.release.send(()).unwrap(); assert_eq!(rpc.next()["error"]["code"], "TURN_FAILED"); let home = rpc.home.clone(); assert!(rpc.close().success()); let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "receipt", &session, &submission]) .env("XDG_DATA_HOME", &home) .output() .unwrap(); assert!( receipt.status.success(), "{}", String::from_utf8_lossy(&receipt.stderr) ); let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap(); assert_eq!(receipt["state"], "terminal"); assert_eq!(receipt["outcome"], "failed"); std::fs::remove_dir_all(home).unwrap(); } fn provider_request(stream: std::net::TcpStream) -> (std::net::TcpStream, Value) { stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap(); stream.set_write_timeout(Some(STARTUP_DEADLINE)).unwrap(); let mut reader = BufReader::new(stream); let mut length = 0; loop { let mut header = String::new(); assert_ne!(reader.read_line(&mut header).unwrap(), 0); if header.trim().is_empty() { break; } if let Some((name, value)) = header.split_once(':') && name.eq_ignore_ascii_case("content-length") { length = value.trim().parse().unwrap(); } } let mut body = vec![0; length]; reader.read_exact(&mut body).unwrap(); (reader.into_inner(), serde_json::from_slice(&body).unwrap()) } #[test] fn rpc_partial_write_abort_preserves_text_usage_receipt_and_keyed_retry() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let (second, received) = mpsc::sync_channel(1); let (release, released) = mpsc::sync_channel(1); let server = std::thread::spawn(move || { let (mut stream, _) = provider_request(listener.accept().unwrap().0); let body = format!( "data: {}\n\ndata: {}\n\ndata: [DONE]\n\n", json!({"choices":[{"index":0,"delta":{"content":"committed prefix","tool_calls":[{"index":0,"id":"write-call","function":{"name":"write","arguments":"{\"path\":\"committed.txt\",\"content\":\"committed effect\\n\"}"}}]}}]}), json!({"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":23,"completion_tokens":29}}) ); write!(stream, "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{body}", body.len()).unwrap(); drop(stream); let (mut stream, request) = provider_request(listener.accept().unwrap().0); stream .write_all( b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n", ) .unwrap(); stream.flush().unwrap(); second.send(request).unwrap(); released.recv_timeout(STARTUP_DEADLINE).unwrap(); drop(stream); }); let mut rpc = Rpc::spawn( "partial-write-abort", &[ "--provider", "openai", "--model", "e2e-model", "--base-url", &url, ], &[("OPENAI_API_KEY", "sk-e2e-fixture")], ); rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#); let start = rpc.next(); assert_eq!(start["event"], "start"); assert_eq!( rpc.next(), json!({"id":1,"event":"delta","text":"committed prefix"}) ); let request = match received.recv_timeout(STARTUP_DEADLINE) { Ok(request) => request, Err(error) => panic!( "second provider request was not received: {error}; {}", rpc.request_diagnostics(start["receipt"]["session_id"].as_str().unwrap()) ), }; assert!( request["messages"] .as_array() .unwrap() .iter() .any(|message| message["role"] == "tool") ); assert!(request.to_string().contains("committed prefix")); assert_eq!( std::fs::read_to_string(rpc.home.join("committed.txt")).unwrap(), "committed effect\n" ); rpc.send(r#"{"id":2,"method":"abort"}"#); assert_eq!(rpc.next()["aborting"], true); release.send(()).unwrap(); server.join().unwrap(); assert_eq!( rpc.next(), json!({"id":1,"event":"end","text":"committed prefix","reason":"Abort","cost":{"input_tokens":23,"output_tokens":29},"cost_complete":false}) ); let submission = start["receipt"]["submission_id"].as_str().unwrap(); rpc.send(&json!({"id":3,"method":"inspect","params":{"id":submission}}).to_string()); let receipt = rpc.next(); assert_eq!(receipt["receipt"]["status"], "Terminal(Some(Abort))"); assert_eq!(receipt["receipt"]["cancel_requested"], true); rpc.send(r#"{"id":4,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#); let duplicate = rpc.next(); assert_eq!(duplicate["duplicate"], true); assert_eq!(duplicate["receipt"]["submission_id"], submission); let home = rpc.home.clone(); assert!(rpc.close().success()); let session = start["receipt"]["session_id"].as_str().unwrap(); let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "get", session, "--json"]) .env("XDG_DATA_HOME", &home) .output() .unwrap(); assert!( inspection.status.success(), "{}", String::from_utf8_lossy(&inspection.stderr) ); let entries = String::from_utf8(inspection.stdout).unwrap(); assert!(entries.contains("committed prefix")); assert!(!entries.contains("sk-e2e-fixture")); // Reopening the keyed receipt never dispatches to the now-closed endpoint. let mut reopened = Rpc::spawn( "partial-write-reopened", &["--session", session], &[("XDG_DATA_HOME", home.to_str().unwrap())], ); // Reopen by durable session path, independent of the new isolated HOME. reopened .send(r#"{"id":5,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#); let retry = reopened.next(); assert_eq!(retry["duplicate"], true); assert_eq!(retry["receipt"]["submission_id"], submission); assert!(reopened.close().success()); assert_eq!( std::fs::read_to_string(home.join("committed.txt")).unwrap(), "committed effect\n" ); } #[test] fn rpc_live_credential_echo_is_redacted_before_terminal_and_session() { assert_live_prefix( "live-prefix", "sk-e2e-fixture", "sk-e2e-fixture", "{{secret:connection-api-key}}", ); } #[test] fn rpc_large_credential_echo_is_redacted_live() { let credential = "a".repeat(4096); assert_live_prefix( "large-live-secret", &credential, &credential, "{{secret:connection-api-key}}", ); } #[test] fn rpc_large_auth_credential_preserves_live_public_progress() { let credential = "a".repeat(4096); let public = "public.".repeat(128); assert_live_prefix("large-auth-public", &credential, &public, &public); } fn assert_live_prefix(name: &str, credential: &str, text: &str, expected: &str) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let (release, released) = mpsc::sync_channel(1); let server_text = text.to_owned(); let server = std::thread::spawn(move || { let (stream, _) = listener.accept().unwrap(); stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap(); stream.set_write_timeout(Some(STARTUP_DEADLINE)).unwrap(); let mut reader = BufReader::new(stream); let mut length = 0; loop { let mut header = String::new(); reader.read_line(&mut header).unwrap(); if header.trim().is_empty() { break; } if let Some((name, value)) = header.split_once(':') && name.eq_ignore_ascii_case("content-length") { length = value.trim().parse().unwrap(); } } reader.read_exact(&mut vec![0; length]).unwrap(); let stream = reader.get_mut(); stream .write_all( b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n", ) .unwrap(); let delta = format!( "data: {}\n\n", json!({"choices":[{"index":0,"delta":{"content":server_text}}]}) ); let split = delta.find(server_text.as_str()).unwrap() + server_text.len() / 2; stream.write_all(&delta.as_bytes()[..split]).unwrap(); stream.flush().unwrap(); stream.write_all(&delta.as_bytes()[split..]).unwrap(); stream.flush().unwrap(); if released.recv_timeout(STARTUP_DEADLINE).is_ok() { write!(stream, "data: {}\n\ndata: [DONE]\n\n", json!({"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":23,"completion_tokens":29}})).unwrap(); } }); let mut rpc = Rpc::spawn( name, &[ "--provider", "openai", "--model", "e2e-model", "--base-url", &url, ], &[("OPENAI_API_KEY", credential)], ); rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"stream"}}"#); let start = rpc.next(); assert_eq!(start["event"], "start"); let prefix = rpc.lines.recv_timeout(DEADLINE); // Release and reap even when the regression assertion fails. release.send(()).unwrap(); server.join().unwrap(); let prefix: Value = serde_json::from_str(&prefix.expect("live delta before provider release").unwrap()) .unwrap(); assert_eq!(prefix, json!({"id":1,"event":"delta","text":expected})); assert!(!prefix.to_string().contains(credential)); let end = rpc.next(); assert_eq!(end["event"], "end"); assert_eq!(end["text"], expected); assert!(!end.to_string().contains(credential)); let session = start["receipt"]["session_id"].as_str().unwrap().to_string(); let home = rpc.home.clone(); assert!(rpc.close().success()); let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "get", &session, "--json"]) .env("XDG_DATA_HOME", &home) .output() .unwrap(); assert!(inspection.status.success()); assert!( !String::from_utf8(inspection.stdout) .unwrap() .contains(credential) ); } #[test] fn rpc_unsplit_credential_in_provider_error_never_reaches_rpc_or_session_bytes() { provider_error_never_reaches_rpc_or_session_bytes(false, 200); } #[test] fn rpc_json_escaped_credential_in_provider_error_never_reaches_rpc_or_session_bytes() { provider_error_never_reaches_rpc_or_session_bytes(true, 200); } #[test] fn rpc_non_success_provider_error_reports_sanitized_vendor_diagnostic() { provider_error_never_reaches_rpc_or_session_bytes(false, 429); } #[test] fn rpc_non_success_json_escaped_credential_never_reaches_rpc_or_session_bytes() { provider_error_never_reaches_rpc_or_session_bytes(true, 429); } fn provider_error_never_reaches_rpc_or_session_bytes(escaped: bool, status: u16) { let credential = "sk-e2e-fixture"; let escaped_credential = r"\u0073k-e2e-\u0066ixture"; let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let server = std::thread::spawn(move || { let (stream, _) = listener.accept().unwrap(); stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap(); let mut reader = BufReader::new(stream); let mut content_length = 0; loop { let mut header = String::new(); reader.read_line(&mut header).unwrap(); if header.trim().is_empty() { break; } if let Some((name, value)) = header.split_once(':') && name.eq_ignore_ascii_case("content-length") { content_length = value.trim().parse::().unwrap(); } } assert!( content_length <= 1024 * 1024, "provider request exceeded fixture bound" ); let mut request_body = vec![0; content_length]; reader.read_exact(&mut request_body).unwrap(); assert!( request_body .windows(b"error".len()) .any(|window| window == b"error") ); let error = if status == 200 { format!("data: {}\n\n", json!({"error":{"message":credential}})) } else { json!({"error":{"code":"rate_limited","message":credential}}).to_string() }; let error = if escaped { error.replace(credential, escaped_credential) } else { error }; let stream = reader.get_mut(); write!(stream, "HTTP/1.1 {status} fixture\r\ncontent-type: application/json\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{error}", error.len()).unwrap(); }); let mut rpc = Rpc::spawn( "error-redaction", &[ "--provider", "openai", "--model", "e2e-model", "--base-url", &url, ], &[("OPENAI_API_KEY", credential)], ); rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"error"}}"#); let start = rpc.next(); assert_eq!(start["event"], "start"); let end = rpc.next(); server.join().unwrap(); assert_eq!(end["event"], "end"); assert_eq!(end["error"]["code"], "TURN_FAILED"); assert!(!end.to_string().contains(credential)); assert!(!end.to_string().contains(escaped_credential)); if status != 200 { assert!( end.to_string() .contains("provider returned HTTP 429: rate_limited"), "unexpected sanitized terminal outcome: code={}, error={}", end["error"]["code"], end["error"]["message"] ); } let session = start["receipt"]["session_id"].as_str().unwrap(); let session_path = rpc .home .join("smith/sessions/run") .join(format!("{session}.smh")); let session_bytes = std::fs::read(session_path).unwrap(); for secret in [credential, escaped_credential] { assert!( !session_bytes .windows(secret.len()) .any(|bytes| bytes == secret.as_bytes()) ); } let home = rpc.home.clone(); let (exit, stderr) = rpc.close_with_stderr(); assert!(exit.success(), "{stderr}"); for secret in [credential, escaped_credential] { assert!(!stderr.contains(secret)); } let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "get", session, "--json"]) .env("XDG_DATA_HOME", home) .output() .unwrap(); assert!(inspection.status.success()); let inspection = String::from_utf8(inspection.stdout).unwrap(); for secret in [credential, escaped_credential] { assert!(!inspection.contains(secret)); } if status != 200 { assert!(inspection.contains("\"status\":429")); assert!(inspection.contains("\"response_bytes\":")); assert!(inspection.contains("\"outcome\":\"failed\"")); } } #[test] fn rpc_cancellation_receipt_does_not_hide_response_head_failure() { let provider = StalledProvider::start(true, false); let mut rpc = Rpc::spawn( "cancel-head-failure", &[ "--provider", "openai", "--model", "e2e-model", "--base-url", &provider.url, ], &[("OPENAI_API_KEY", "sk-e2e-fixture")], ); rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"block"}}"#); let start = rpc.next(); assert_eq!(start["event"], "start"); if let Err(error) = provider.received.recv_timeout(DEADLINE) { panic!( "provider did not receive the first request within {DEADLINE:?}: {error}; {}", rpc.request_diagnostics(start["receipt"]["session_id"].as_str().unwrap()) ); } rpc.send(r#"{"id":2,"method":"abort"}"#); assert_eq!(rpc.next()["aborting"], true); provider.release.send(()).unwrap(); let end = rpc.next(); assert_eq!(end["event"], "end"); assert_eq!(end["error"]["code"], "TURN_FAILED"); assert!(end["reason"].is_null()); rpc.send( &json!({"id":3,"method":"inspect","params":{"id":start["receipt"]["submission_id"]}}) .to_string(), ); let receipt = rpc.next(); assert_eq!(receipt["receipt"]["cancel_requested"], true); assert_eq!(receipt["receipt"]["status"], "Terminal(Some(Abort))"); assert!(rpc.close().success()); } #[test] fn rpc_abort_reports_durable_abort_outcome() { let provider = StalledProvider::start(false, false); let mut rpc = Rpc::spawn( "abort", &[ "--provider", "openai", "--model", "e2e-model", "--base-url", &provider.url, ], &[("OPENAI_API_KEY", "sk-e2e-fixture")], ); // Each step waits for its reply before the next; the provider holds its // response until the abort is acknowledged, so the order is fixed. rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "block"}}"#); let start = rpc.next(); assert_eq!(start["id"], 1); assert_eq!(start["event"], "start"); provider .received .recv_timeout(DEADLINE) .expect("provider request within the deadline"); rpc.send(r#"{"id": 2, "method": "abort"}"#); let abort = rpc.next(); assert_eq!(abort["id"], 2); assert_eq!(abort["aborting"], true); provider.release.send(()).unwrap(); assert_eq!( rpc.next(), json!({ "id": 1, "event": "end", "text": "", "reason": "Abort", "cost": null, "cost_complete": false, }) ); // `wait` answers once the aborted turn has settled; `status` then agrees. rpc.send(r#"{"id": 3, "method": "wait"}"#); assert_eq!(rpc.next(), json!({"id": 3, "settled": true})); rpc.send(r#"{"id": 4, "method": "status"}"#); assert_eq!( rpc.next(), json!({"id": 4, "settled": true, "access_mode": "writer", "active_submission_id": null}) ); insta::assert_snapshot!( "rpc_prompt_then_abort_then_wait", transcript_for_snapshot(&rpc.transcript) ); let session = start["receipt"]["session_id"].as_str().unwrap().to_owned(); let submission = start["receipt"]["submission_id"] .as_str() .unwrap() .to_owned(); let home = rpc.home.clone(); assert!(rpc.close().success()); let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith")) .args(["sessions", "receipt", &session, &submission]) .env("XDG_DATA_HOME", &home) .output() .unwrap(); assert!(receipt.status.success()); let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap(); assert_eq!(receipt["state"], "terminal"); assert_eq!(receipt["outcome"], "cancelled"); let mut entries = std::fs::read_dir(&home) .unwrap() .map(|entry| entry.unwrap().file_name()) .collect::>(); entries.sort(); assert_eq!(entries, ["runtime", "smith"]); }