Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-cli/tests/e2e_rpc.rs

Raw
//! End-to-end `smith rpc` workflows over piped standard input and output
//! (`SMH-SPEC-SPEC0001`, Interface modes: JSON-RPC).
//!
//! Every read waits on a line or end of output with a deadline; nothing
//! sleeps or polls.

use serde_json::{Value, json};
use std::io::{BufRead, BufReader, Read, Write};
use std::net::TcpListener;
use std::path::PathBuf;
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{
    Arc,
    mpsc::{self, Receiver},
};
use std::time::Duration;

const DEADLINE: Duration = Duration::from_secs(10);
const STARTUP_DEADLINE: Duration = Duration::from_secs(10);
const EXIT_DEADLINE: Duration = Duration::from_secs(10);
const DIAGNOSTIC_QUEUE_LINES: usize = 32;
const DIAGNOSTIC_LINE_BYTES: usize = 4 * 1024;
const MOCK_REPLY: &str = "rpc mock provider ready";
static RPC_HOME_INDEX: AtomicUsize = AtomicUsize::new(0);

/// One running `smith rpc` process; killed on drop if a test fails early.
struct Rpc {
    child: Child,
    home: PathBuf,
    stdin: Option<ChildStdin>,
    /// Raw stdout lines; `None` marks end of output.
    lines: Receiver<Option<String>>,
    /// Child diagnostics emitted before a fixture deadline expires.
    stderr: Receiver<String>,
    /// Every line sent (`> `) and received (`< `), in order.
    transcript: Vec<String>,
}

impl Rpc {
    fn spawn(_name: &str, args: &[&str], envs: &[(&str, &str)]) -> Self {
        let index = RPC_HOME_INDEX.fetch_add(1, Ordering::Relaxed);
        let home = PathBuf::from("/var/tmp/smith-session-e2e")
            .join(format!("r-{}-{index}", std::process::id()));
        let _ = std::fs::remove_dir_all(&home);
        std::fs::create_dir_all(&home).unwrap();
        let mut child = Command::new(assert_cmd::cargo::cargo_bin("smith"))
            .arg("rpc")
            .args(args)
            .current_dir(&home)
            .env("HOME", &home)
            .env("XDG_CACHE_HOME", home.join("cache"))
            .env("XDG_CONFIG_HOME", home.join("config"))
            .env("XDG_DATA_HOME", &home)
            .env("XDG_RUNTIME_DIR", home.join("runtime"))
            .env("XDG_STATE_HOME", home.join("state"))
            .env_remove("SMITH_PROVIDER")
            .env_remove("SMITH_MODEL")
            .env_remove("SMITH_BASE_URL")
            .env_remove("ANTHROPIC_API_KEY")
            .env_remove("OPENAI_API_KEY")
            .env_remove("GEMINI_API_KEY")
            .envs(envs.iter().copied())
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()
            .unwrap();
        let stdout = child.stdout.take().unwrap();
        let stderr = child.stderr.take().unwrap();
        let (sender, lines) = mpsc::channel();
        std::thread::spawn(move || {
            for line in BufReader::new(stdout).lines() {
                if sender.send(Some(line.unwrap())).is_err() {
                    return;
                }
            }
            let _ = sender.send(None);
        });
        let (stderr_sender, stderr_lines) = mpsc::channel();
        std::thread::spawn(move || {
            for line in BufReader::new(stderr).lines() {
                if stderr_sender.send(line.unwrap()).is_err() {
                    return;
                }
            }
        });
        let stdin = child.stdin.take();
        Self {
            child,
            home,
            stdin,
            lines,
            stderr: stderr_lines,
            transcript: Vec::new(),
        }
    }

    fn mock(name: &str) -> Self {
        Self::spawn(name, &[], &[])
    }

    fn send(&mut self, line: &str) {
        self.transcript.push(format!("> {line}"));
        let stdin = self.stdin.as_mut().unwrap();
        stdin.write_all(line.as_bytes()).unwrap();
        stdin.write_all(b"\n").unwrap();
        stdin.flush().unwrap();
    }

    fn next(&mut self) -> Value {
        match self.lines.recv_timeout(STARTUP_DEADLINE) {
            Ok(Some(line)) => {
                let value = serde_json::from_str(&line)
                    .unwrap_or_else(|error| panic!("stdout line is not JSON ({error}): {line}"));
                self.transcript.push(format!("< {line}"));
                value
            }
            Ok(None) => panic!("rpc output ended early"),
            Err(error) => panic!("no rpc line within {STARTUP_DEADLINE:?}: {error}"),
        }
    }

    fn request_diagnostics(&mut self, session: &str) -> String {
        let queued_stdout = self
            .lines
            .try_iter()
            .take(DIAGNOSTIC_QUEUE_LINES)
            .map(|line| line.map(diagnostic_line))
            .collect::<Vec<_>>();
        let stderr = self
            .stderr
            .try_iter()
            .take(DIAGNOSTIC_QUEUE_LINES)
            .map(diagnostic_line)
            .collect::<Vec<_>>()
            .join("\\n");
        let journal = assert_cmd::Command::new(assert_cmd::cargo::cargo_bin("smith"))
            .timeout(DEADLINE)
            .args(["sessions", "get", session, "--json"])
            .env("XDG_DATA_HOME", &self.home)
            .output()
            .map_or_else(
                |error| format!("inspection spawn failed: {error}"),
                |output| {
                    format!(
                        "status={}; stdout={}; stderr={}",
                        output.status,
                        String::from_utf8_lossy(&output.stdout),
                        String::from_utf8_lossy(&output.stderr)
                    )
                },
            );
        format!(
            "rpc status={:?}; transcript={:?}; queued_stdout={queued_stdout:?}; stderr={stderr:?}; committed.txt={:?}; journal={journal}",
            self.child.try_wait(),
            self.transcript,
            std::fs::read_to_string(self.home.join("committed.txt")),
        )
    }

    /// Close stdin, then expect end of output and the exit status.
    fn close(self) -> std::process::ExitStatus {
        self.close_with_stderr().0
    }

    fn close_with_stderr(mut self) -> (std::process::ExitStatus, String) {
        drop(self.stdin.take());
        match self.lines.recv_timeout(EXIT_DEADLINE) {
            Ok(None) => {}
            Ok(Some(line)) => panic!("unexpected rpc line after close: {line}"),
            Err(error) => panic!("rpc did not end output within {EXIT_DEADLINE:?}: {error}"),
        }
        let status = self.child.wait().unwrap();
        (status, self.stderr.iter().collect::<Vec<_>>().join("\n"))
    }
}

fn diagnostic_line(line: String) -> String {
    if line.len() <= DIAGNOSTIC_LINE_BYTES {
        return line;
    }
    let end = line
        .char_indices()
        .take_while(|(index, _)| *index <= DIAGNOSTIC_LINE_BYTES)
        .map(|(index, _)| index)
        .last()
        .unwrap_or(0);
    format!("{}… [truncated]", &line[..end])
}

fn transcript_for_snapshot(lines: &[String]) -> String {
    lines
        .iter()
        .map(|line| {
            let Some(json) = line.strip_prefix("< ") else {
                return line.clone();
            };
            let Ok(mut value) = serde_json::from_str::<Value>(json) else {
                return line.clone();
            };
            if let Some(receipt) = value.get_mut("receipt").and_then(Value::as_object_mut) {
                receipt.insert("session_id".into(), json!("<session>"));
                receipt.insert("submission_id".into(), json!("<submission>"));
            }
            if value.get("submission_id").is_some() {
                value["submission_id"] = json!("<submission>");
            }
            format!("< {value}")
        })
        .collect::<Vec<_>>()
        .join("\n")
}

impl Drop for Rpc {
    fn drop(&mut self) {
        if matches!(self.child.try_wait(), Ok(None)) {
            let _ = self.child.kill();
            let _ = self.child.wait();
        }
    }
}

#[test]
fn rpc_event_and_shutdown_deadlines_stay_at_ten_seconds() {
    assert_eq!(STARTUP_DEADLINE, DEADLINE);
    assert_eq!(EXIT_DEADLINE, DEADLINE);
}

#[test]
fn rpc_rejects_oversized_stdin_line_without_protocol_output() {
    let mut command = assert_cmd::Command::cargo_bin("smith").unwrap();
    command
        .arg("rpc")
        .env(
            "XDG_DATA_HOME",
            std::env::temp_dir().join(format!("smith-rpc-test-oversized-{}", std::process::id())),
        )
        .env_remove("SMITH_PROVIDER")
        .env_remove("SMITH_MODEL")
        .env_remove("SMITH_BASE_URL")
        .write_stdin("x".repeat(1024 * 1024 + 1))
        .assert()
        .failure()
        .stdout("")
        .stderr(predicates::str::contains(
            "JSON-RPC request exceeds 1048576 bytes",
        ));
}

#[test]
fn rpc_ping_correlates_the_request_id() {
    let mut rpc = Rpc::mock("ping");
    rpc.send(r#"{"id": 7, "method": "ping"}"#);
    assert_eq!(rpc.next(), json!({"id": 7, "pong": true}));
    rpc.send(r#"{"id": "seven", "method": "ping"}"#);
    assert_eq!(rpc.next(), json!({"id": "seven", "pong": true}));
    assert!(rpc.close().success());
}

#[test]
fn rpc_prompt_streams_start_delta_end_then_wait_and_status_report_settled() {
    let mut rpc = Rpc::mock("prompt");
    rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "hi"}}"#);
    rpc.send(r#"{"id": 2, "method": "wait"}"#);
    let start = rpc.next();
    assert_eq!(start["id"], 1);
    assert_eq!(start["event"], "start");
    assert_eq!(start["receipt"]["status"], "Queued");
    assert_eq!(start["receipt"]["cancel_requested"], false);
    assert!(start["receipt"]["submission_id"].is_string());
    assert_eq!(
        rpc.next(),
        json!({"id": 1, "event": "delta", "text": MOCK_REPLY})
    );
    assert_eq!(
        rpc.next(),
        json!({
            "id": 1,
            "event": "end",
            "text": MOCK_REPLY,
            "reason": "EndTurn",
            "cost": null,
            "cost_complete": false,
        })
    );
    // The worker answers `wait` only after the prompt has settled.
    assert_eq!(rpc.next(), json!({"id": 2, "settled": true}));
    // `wait` answers once the aborted turn has settled; `status` then agrees.
    rpc.send(r#"{"id": 3, "method": "wait"}"#);
    assert_eq!(rpc.next(), json!({"id": 3, "settled": true}));
    rpc.send(r#"{"id": 4, "method": "status"}"#);
    assert_eq!(
        rpc.next(),
        json!({"id": 4, "settled": true, "access_mode": "writer", "active_submission_id": null})
    );
    assert!(rpc.close().success());
}

#[test]
fn rpc_read_only_rejects_prompt_and_abort_without_mutating_the_session() {
    let home = std::env::temp_dir().join(format!("smith-rpc-read-only-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&home);
    let seed = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["eval", "--mock", "seed"])
        .env("XDG_DATA_HOME", &home)
        .env_remove("SMITH_PROVIDER")
        .output()
        .unwrap();
    assert!(
        seed.status.success(),
        "{}",
        String::from_utf8_lossy(&seed.stderr)
    );
    let listed = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "list", "--json"])
        .env("XDG_DATA_HOME", &home)
        .output()
        .unwrap();
    let session = serde_json::from_slice::<Value>(&listed.stdout).unwrap()["session"]
        .as_str()
        .unwrap()
        .to_owned();
    let path = home
        .join("smith/sessions/run")
        .join(format!("{session}.smh"));
    let before = std::fs::read(&path).unwrap();
    let output = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args([
            "rpc",
            "--session",
            &session,
            "--read-only",
            "--provider",
            "openai",
            "--model",
            "must-not-initialize",
            "--base-url",
            "http://127.0.0.1:1",
        ])
        .env("XDG_DATA_HOME", &home)
        .env_remove("OPENAI_API_KEY")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .spawn()
        .unwrap();
    let mut child = output;
    let mut stdin = child.stdin.take().unwrap();
    stdin
        .write_all(
            b"{\"id\":1,\"method\":\"status\"}\n{\"id\":2,\"method\":\"prompt\",\"params\":{\"input\":\"denied\"}}\n{\"id\":3,\"method\":\"abort\"}\n{\"id\":4,\"method\":\"inspect\",\"params\":{\"id\":\"babab-babab-babab-babab-babab-babab-babab-babab\"}}\n",
        )
        .unwrap();
    drop(stdin);
    let output = child.wait_with_output().unwrap();
    assert!(
        output.status.success(),
        "{}",
        String::from_utf8_lossy(&output.stderr)
    );
    let records: Vec<Value> = String::from_utf8(output.stdout)
        .unwrap()
        .lines()
        .map(|line| serde_json::from_str(line).unwrap())
        .collect();
    assert_eq!(
        records[0],
        json!({"id": 1, "settled": true, "access_mode": "read_only", "active_submission_id": null})
    );
    assert_eq!(records[1]["error"]["code"], "STATE");
    assert_eq!(records[2]["error"]["code"], "STATE");
    assert_eq!(records[3], json!({"id": 4, "receipt": null}));
    assert_eq!(std::fs::read(&path).unwrap(), before);
    std::fs::remove_dir_all(home).unwrap();
}

#[test]
fn rpc_malformed_line_reports_parse_error_with_null_id() {
    let mut rpc = Rpc::mock("malformed");
    rpc.send("not json");
    let record = rpc.next();
    assert_eq!(record["id"], Value::Null);
    assert_eq!(record["error"]["code"], "PARSE");
    // The server keeps serving after a bad line.
    rpc.send(r#"{"id": 1, "method": "ping"}"#);
    assert_eq!(rpc.next(), json!({"id": 1, "pong": true}));
    assert!(rpc.close().success());
}

#[test]
fn rpc_unknown_method_reports_method_unknown() {
    let mut rpc = Rpc::mock("unknown");
    rpc.send(r#"{"id": "x", "method": "explode"}"#);
    assert_eq!(
        rpc.next(),
        json!({
            "id": "x",
            "error": {"code": "METHOD_UNKNOWN", "message": "unknown method explode"},
        })
    );
    assert!(rpc.close().success());
}

#[test]
fn rpc_closing_stdin_after_settle_exits_zero() {
    let mut rpc = Rpc::mock("shutdown");
    rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "hi"}}"#);
    rpc.send(r#"{"id": 2, "method": "wait"}"#);
    let mut records = Vec::new();
    loop {
        let record = rpc.next();
        let done = record["id"] == 2;
        records.push(record);
        if done {
            break;
        }
    }
    assert!(records.iter().any(|r| r["id"] == 1 && r["event"] == "end"));
    let status = rpc.close();
    assert_eq!(status.code(), Some(0));
}

/// A loopback provider endpoint that holds its response: it reports each
/// received request, then answers only when released.
struct StalledProvider {
    url: String,
    received: Receiver<()>,
    release: mpsc::Sender<()>,
    requests: Arc<AtomicUsize>,
}

impl StalledProvider {
    fn start(drop_response: bool, complete_response: bool) -> Self {
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let url = format!("http://{}", listener.local_addr().unwrap());
        let (received_tx, received) = mpsc::channel();
        let (release, release_rx) = mpsc::channel::<()>();
        let requests = Arc::new(AtomicUsize::new(0));
        let request_count = Arc::clone(&requests);
        std::thread::spawn(move || {
            let Ok((stream, _)) = listener.accept() else {
                return;
            };
            let mut reader = BufReader::new(stream);
            let mut length = 0;
            loop {
                let mut header = String::new();
                if reader.read_line(&mut header).unwrap_or(0) == 0 {
                    return;
                }
                let header = header.trim_end();
                if header.is_empty() {
                    break;
                }
                if let Some((name, value)) = header.split_once(':')
                    && name.eq_ignore_ascii_case("content-length")
                {
                    length = value.trim().parse().unwrap();
                }
            }
            let mut body = vec![0; length];
            reader.read_exact(&mut body).unwrap();
            request_count.fetch_add(1, Ordering::SeqCst);
            let _ = received_tx.send(());
            if release_rx.recv().is_err() || drop_response {
                return;
            }
            let body = if complete_response {
                format!(
                    "data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
                    json!({"choices": [{"index": 0, "delta": {"content": "late"}}]}),
                    json!({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]})
                )
            } else {
                format!(
                    "data: {}\n\n",
                    json!({"choices": [{"index": 0, "delta": {"content": "late"}}]})
                )
            };
            let response = format!(
                "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{body}",
                body.len()
            );
            let _ = reader.get_mut().write_all(response.as_bytes());
        });
        Self {
            url,
            received,
            release,
            requests,
        }
    }
}

fn spawn_unread_stdout(
    name: &str,
    provider: &StalledProvider,
) -> (Child, ChildStdin, ChildStdout, PathBuf) {
    let home = std::env::temp_dir().join(format!("smith-rpc-test-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&home);
    let mut child = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args([
            "rpc",
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &provider.url,
        ])
        .env("XDG_DATA_HOME", &home)
        .env("OPENAI_API_KEY", "sk-e2e-fixture")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::null())
        .spawn()
        .unwrap();
    let stdin = child.stdin.take().unwrap();
    let stdout = child.stdout.take().unwrap();
    (child, stdin, stdout, home)
}

fn acknowledge_and_return_stdout(stdout: ChildStdout) -> (Value, ChildStdout) {
    let (sender, receiver) = mpsc::sync_channel(1);
    let reader = std::thread::spawn(move || {
        let mut reader = BufReader::new(stdout);
        let mut line = String::new();
        assert_ne!(
            reader.read_line(&mut line).unwrap(),
            0,
            "RPC ended before ACK"
        );
        sender
            .send((serde_json::from_str(&line).unwrap(), reader.into_inner()))
            .unwrap();
    });
    let acknowledgement = receiver
        .recv_timeout(STARTUP_DEADLINE)
        .expect("RPC ACK within startup deadline");
    reader.join().unwrap();
    acknowledgement
}

fn wait_for_exit(mut child: Child) -> std::process::ExitStatus {
    let (sender, receiver) = mpsc::sync_channel(1);
    let waiter = std::thread::spawn(move || sender.send(child.wait().unwrap()).unwrap());
    let status = receiver
        .recv_timeout(EXIT_DEADLINE)
        .expect("RPC child exit within exit deadline");
    waiter.join().unwrap();
    status
}

#[test]
fn rpc_provider_control_plane_remains_available_until_explicit_release() {
    let provider = StalledProvider::start(false, true);
    let mut rpc = Rpc::spawn(
        "provider-control-plane",
        &[
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &provider.url,
        ],
        &[("OPENAI_API_KEY", "sk-e2e-fixture")],
    );
    rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"block","key":"once"}}"#);
    let start = rpc.next();
    assert_eq!(start["id"], 1);
    assert_eq!(start["event"], "start");
    let session = start["receipt"]["session_id"].as_str().unwrap().to_owned();
    let submission = start["receipt"]["submission_id"]
        .as_str()
        .unwrap()
        .to_owned();
    provider
        .received
        .recv_timeout(STARTUP_DEADLINE)
        .expect("provider request within startup deadline");

    rpc.send(&format!(
        r#"{{"id":2,"method":"inspect","params":{{"id":"{submission}"}}}}"#
    ));
    let inspect = rpc.next();
    assert_eq!(inspect["id"], 2);
    assert_eq!(inspect["receipt"]["submission_id"], submission);
    assert_eq!(inspect["receipt"]["status"], "Running");
    rpc.send(r#"{"id":3,"method":"prompt","params":{"input":"other"}}"#);
    assert_eq!(
        rpc.next(),
        json!({"id": 3, "error": {"code": "BUSY", "message": "a prompt is already running"}})
    );
    rpc.send(r#"{"id":4,"method":"prompt","params":{"input":"block","key":"once"}}"#);
    let duplicate = rpc.next();
    assert_eq!(duplicate["id"], 4);
    assert_eq!(duplicate["event"], "start");
    assert_eq!(duplicate["duplicate"], true);
    assert_eq!(duplicate["receipt"]["submission_id"], submission);

    rpc.send(r#"{"id":5,"method":"wait"}"#);
    provider.release.send(()).unwrap();
    assert_eq!(rpc.next()["id"], 1);
    let end = rpc.next();
    assert_eq!(end["id"], 1);
    assert_eq!(end["event"], "end");
    assert_eq!(end["reason"], "EndTurn");
    assert_eq!(rpc.next(), json!({"id": 5, "settled": true}));
    assert_eq!(provider.requests.load(Ordering::SeqCst), 1);
    let home = rpc.home.clone();
    assert!(rpc.close().success());

    let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "receipt", &session, &submission])
        .env("XDG_DATA_HOME", &home)
        .output()
        .unwrap();
    assert!(
        receipt.status.success(),
        "{}",
        String::from_utf8_lossy(&receipt.stderr)
    );
    let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
    assert_eq!(receipt["state"], "terminal");
    assert_eq!(receipt["outcome"], "completed");
}

#[test]
fn rpc_response_loss_after_ack_never_fabricates_abort_or_replays() {
    let provider = StalledProvider::start(false, true);
    let (child, mut stdin, stdout, home) =
        spawn_unread_stdout("response-loss-after-ack", &provider);
    stdin
        .write_all(
            b"{\"id\":1,\"method\":\"prompt\",\"params\":{\"input\":\"drop\",\"key\":\"once\"}}\n",
        )
        .unwrap();
    stdin.flush().unwrap();
    let (acknowledgement, stdout) = acknowledge_and_return_stdout(stdout);
    assert_eq!(acknowledgement["id"], 1);
    assert_eq!(acknowledgement["event"], "start");
    let session = acknowledgement["receipt"]["session_id"]
        .as_str()
        .unwrap()
        .to_owned();
    let submission = acknowledgement["receipt"]["submission_id"]
        .as_str()
        .unwrap()
        .to_owned();
    provider
        .received
        .recv_timeout(STARTUP_DEADLINE)
        .expect("provider request within startup deadline");
    drop(stdout);
    provider.release.send(()).unwrap();
    drop(stdin);
    let exit = wait_for_exit(child);
    assert!(
        !exit.success(),
        "closed stdout must reach the child as a write failure"
    );
    assert_eq!(provider.requests.load(Ordering::SeqCst), 1);

    let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "receipt", &session, &submission])
        .env("XDG_DATA_HOME", &home)
        .output()
        .unwrap();
    assert!(
        receipt.status.success(),
        "{}",
        String::from_utf8_lossy(&receipt.stderr)
    );
    let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
    assert_ne!(receipt["outcome"], "aborted");
    match receipt["state"].as_str() {
        Some("terminal") => assert_eq!(receipt["outcome"], "completed"),
        Some("running") => assert!(receipt["outcome"].is_null()),
        state => panic!("unexpected post-broken-pipe receipt state: {state:?}"),
    }
}

#[test]
fn rpc_response_loss_is_terminal_after_reopen_inspection_without_reschedule() {
    let provider = StalledProvider::start(true, false);
    let mut rpc = Rpc::spawn(
        "response-loss",
        &[
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &provider.url,
        ],
        &[("OPENAI_API_KEY", "sk-e2e-fixture")],
    );
    rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"drop"}}"#);
    let start = rpc.next();
    let session = start["receipt"]["session_id"].as_str().unwrap().to_owned();
    let submission = start["receipt"]["submission_id"]
        .as_str()
        .unwrap()
        .to_owned();
    provider
        .received
        .recv_timeout(DEADLINE)
        .expect("provider request within the deadline");
    provider.release.send(()).unwrap();
    assert_eq!(rpc.next()["error"]["code"], "TURN_FAILED");
    let home = rpc.home.clone();
    assert!(rpc.close().success());
    let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "receipt", &session, &submission])
        .env("XDG_DATA_HOME", &home)
        .output()
        .unwrap();
    assert!(
        receipt.status.success(),
        "{}",
        String::from_utf8_lossy(&receipt.stderr)
    );
    let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
    assert_eq!(receipt["state"], "terminal");
    assert_eq!(receipt["outcome"], "failed");
    std::fs::remove_dir_all(home).unwrap();
}

fn provider_request(stream: std::net::TcpStream) -> (std::net::TcpStream, Value) {
    stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap();
    stream.set_write_timeout(Some(STARTUP_DEADLINE)).unwrap();
    let mut reader = BufReader::new(stream);
    let mut length = 0;
    loop {
        let mut header = String::new();
        assert_ne!(reader.read_line(&mut header).unwrap(), 0);
        if header.trim().is_empty() {
            break;
        }
        if let Some((name, value)) = header.split_once(':')
            && name.eq_ignore_ascii_case("content-length")
        {
            length = value.trim().parse().unwrap();
        }
    }
    let mut body = vec![0; length];
    reader.read_exact(&mut body).unwrap();
    (reader.into_inner(), serde_json::from_slice(&body).unwrap())
}

#[test]
fn rpc_partial_write_abort_preserves_text_usage_receipt_and_keyed_retry() {
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let url = format!("http://{}", listener.local_addr().unwrap());
    let (second, received) = mpsc::sync_channel(1);
    let (release, released) = mpsc::sync_channel(1);
    let server = std::thread::spawn(move || {
        let (mut stream, _) = provider_request(listener.accept().unwrap().0);
        let body = format!(
            "data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
            json!({"choices":[{"index":0,"delta":{"content":"committed prefix","tool_calls":[{"index":0,"id":"write-call","function":{"name":"write","arguments":"{\"path\":\"committed.txt\",\"content\":\"committed effect\\n\"}"}}]}}]}),
            json!({"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":23,"completion_tokens":29}})
        );
        write!(stream, "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{body}", body.len()).unwrap();
        drop(stream);
        let (mut stream, request) = provider_request(listener.accept().unwrap().0);
        stream
            .write_all(
                b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n",
            )
            .unwrap();
        stream.flush().unwrap();
        second.send(request).unwrap();
        released.recv_timeout(STARTUP_DEADLINE).unwrap();
        drop(stream);
    });
    let mut rpc = Rpc::spawn(
        "partial-write-abort",
        &[
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &url,
        ],
        &[("OPENAI_API_KEY", "sk-e2e-fixture")],
    );
    rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#);
    let start = rpc.next();
    assert_eq!(start["event"], "start");
    assert_eq!(
        rpc.next(),
        json!({"id":1,"event":"delta","text":"committed prefix"})
    );
    let request = match received.recv_timeout(STARTUP_DEADLINE) {
        Ok(request) => request,
        Err(error) => panic!(
            "second provider request was not received: {error}; {}",
            rpc.request_diagnostics(start["receipt"]["session_id"].as_str().unwrap())
        ),
    };
    assert!(
        request["messages"]
            .as_array()
            .unwrap()
            .iter()
            .any(|message| message["role"] == "tool")
    );
    assert!(request.to_string().contains("committed prefix"));
    assert_eq!(
        std::fs::read_to_string(rpc.home.join("committed.txt")).unwrap(),
        "committed effect\n"
    );
    rpc.send(r#"{"id":2,"method":"abort"}"#);
    assert_eq!(rpc.next()["aborting"], true);
    release.send(()).unwrap();
    server.join().unwrap();
    assert_eq!(
        rpc.next(),
        json!({"id":1,"event":"end","text":"committed prefix","reason":"Abort","cost":{"input_tokens":23,"output_tokens":29},"cost_complete":false})
    );
    let submission = start["receipt"]["submission_id"].as_str().unwrap();
    rpc.send(&json!({"id":3,"method":"inspect","params":{"id":submission}}).to_string());
    let receipt = rpc.next();
    assert_eq!(receipt["receipt"]["status"], "Terminal(Some(Abort))");
    assert_eq!(receipt["receipt"]["cancel_requested"], true);
    rpc.send(r#"{"id":4,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#);
    let duplicate = rpc.next();
    assert_eq!(duplicate["duplicate"], true);
    assert_eq!(duplicate["receipt"]["submission_id"], submission);
    let home = rpc.home.clone();
    assert!(rpc.close().success());
    let session = start["receipt"]["session_id"].as_str().unwrap();
    let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "get", session, "--json"])
        .env("XDG_DATA_HOME", &home)
        .output()
        .unwrap();
    assert!(
        inspection.status.success(),
        "{}",
        String::from_utf8_lossy(&inspection.stderr)
    );
    let entries = String::from_utf8(inspection.stdout).unwrap();
    assert!(entries.contains("committed prefix"));
    assert!(!entries.contains("sk-e2e-fixture"));
    // Reopening the keyed receipt never dispatches to the now-closed endpoint.
    let mut reopened = Rpc::spawn(
        "partial-write-reopened",
        &["--session", session],
        &[("XDG_DATA_HOME", home.to_str().unwrap())],
    );
    // Reopen by durable session path, independent of the new isolated HOME.
    reopened
        .send(r#"{"id":5,"method":"prompt","params":{"input":"write then abort","key":"once"}}"#);
    let retry = reopened.next();
    assert_eq!(retry["duplicate"], true);
    assert_eq!(retry["receipt"]["submission_id"], submission);
    assert!(reopened.close().success());
    assert_eq!(
        std::fs::read_to_string(home.join("committed.txt")).unwrap(),
        "committed effect\n"
    );
}

#[test]
fn rpc_live_credential_echo_is_redacted_before_terminal_and_session() {
    assert_live_prefix(
        "live-prefix",
        "sk-e2e-fixture",
        "sk-e2e-fixture",
        "{{secret:connection-api-key}}",
    );
}

#[test]
fn rpc_large_credential_echo_is_redacted_live() {
    let credential = "a".repeat(4096);
    assert_live_prefix(
        "large-live-secret",
        &credential,
        &credential,
        "{{secret:connection-api-key}}",
    );
}

#[test]
fn rpc_large_auth_credential_preserves_live_public_progress() {
    let credential = "a".repeat(4096);
    let public = "public.".repeat(128);
    assert_live_prefix("large-auth-public", &credential, &public, &public);
}

fn assert_live_prefix(name: &str, credential: &str, text: &str, expected: &str) {
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let url = format!("http://{}", listener.local_addr().unwrap());
    let (release, released) = mpsc::sync_channel(1);
    let server_text = text.to_owned();
    let server = std::thread::spawn(move || {
        let (stream, _) = listener.accept().unwrap();
        stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap();
        stream.set_write_timeout(Some(STARTUP_DEADLINE)).unwrap();
        let mut reader = BufReader::new(stream);
        let mut length = 0;
        loop {
            let mut header = String::new();
            reader.read_line(&mut header).unwrap();
            if header.trim().is_empty() {
                break;
            }
            if let Some((name, value)) = header.split_once(':')
                && name.eq_ignore_ascii_case("content-length")
            {
                length = value.trim().parse().unwrap();
            }
        }
        reader.read_exact(&mut vec![0; length]).unwrap();
        let stream = reader.get_mut();
        stream
            .write_all(
                b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n",
            )
            .unwrap();
        let delta = format!(
            "data: {}\n\n",
            json!({"choices":[{"index":0,"delta":{"content":server_text}}]})
        );
        let split = delta.find(server_text.as_str()).unwrap() + server_text.len() / 2;
        stream.write_all(&delta.as_bytes()[..split]).unwrap();
        stream.flush().unwrap();
        stream.write_all(&delta.as_bytes()[split..]).unwrap();
        stream.flush().unwrap();
        if released.recv_timeout(STARTUP_DEADLINE).is_ok() {
            write!(stream, "data: {}\n\ndata: [DONE]\n\n", json!({"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":23,"completion_tokens":29}})).unwrap();
        }
    });
    let mut rpc = Rpc::spawn(
        name,
        &[
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &url,
        ],
        &[("OPENAI_API_KEY", credential)],
    );
    rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"stream"}}"#);
    let start = rpc.next();
    assert_eq!(start["event"], "start");
    let prefix = rpc.lines.recv_timeout(DEADLINE);
    // Release and reap even when the regression assertion fails.
    release.send(()).unwrap();
    server.join().unwrap();
    let prefix: Value =
        serde_json::from_str(&prefix.expect("live delta before provider release").unwrap())
            .unwrap();
    assert_eq!(prefix, json!({"id":1,"event":"delta","text":expected}));
    assert!(!prefix.to_string().contains(credential));
    let end = rpc.next();
    assert_eq!(end["event"], "end");
    assert_eq!(end["text"], expected);
    assert!(!end.to_string().contains(credential));
    let session = start["receipt"]["session_id"].as_str().unwrap().to_string();
    let home = rpc.home.clone();
    assert!(rpc.close().success());
    let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "get", &session, "--json"])
        .env("XDG_DATA_HOME", &home)
        .output()
        .unwrap();
    assert!(inspection.status.success());
    assert!(
        !String::from_utf8(inspection.stdout)
            .unwrap()
            .contains(credential)
    );
}

#[test]
fn rpc_unsplit_credential_in_provider_error_never_reaches_rpc_or_session_bytes() {
    provider_error_never_reaches_rpc_or_session_bytes(false, 200);
}

#[test]
fn rpc_json_escaped_credential_in_provider_error_never_reaches_rpc_or_session_bytes() {
    provider_error_never_reaches_rpc_or_session_bytes(true, 200);
}

#[test]
fn rpc_non_success_provider_error_reports_sanitized_vendor_diagnostic() {
    provider_error_never_reaches_rpc_or_session_bytes(false, 429);
}

#[test]
fn rpc_non_success_json_escaped_credential_never_reaches_rpc_or_session_bytes() {
    provider_error_never_reaches_rpc_or_session_bytes(true, 429);
}

fn provider_error_never_reaches_rpc_or_session_bytes(escaped: bool, status: u16) {
    let credential = "sk-e2e-fixture";
    let escaped_credential = r"\u0073k-e2e-\u0066ixture";
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let url = format!("http://{}", listener.local_addr().unwrap());
    let server = std::thread::spawn(move || {
        let (stream, _) = listener.accept().unwrap();
        stream.set_read_timeout(Some(STARTUP_DEADLINE)).unwrap();
        let mut reader = BufReader::new(stream);
        let mut content_length = 0;
        loop {
            let mut header = String::new();
            reader.read_line(&mut header).unwrap();
            if header.trim().is_empty() {
                break;
            }
            if let Some((name, value)) = header.split_once(':')
                && name.eq_ignore_ascii_case("content-length")
            {
                content_length = value.trim().parse::<usize>().unwrap();
            }
        }
        assert!(
            content_length <= 1024 * 1024,
            "provider request exceeded fixture bound"
        );
        let mut request_body = vec![0; content_length];
        reader.read_exact(&mut request_body).unwrap();
        assert!(
            request_body
                .windows(b"error".len())
                .any(|window| window == b"error")
        );
        let error = if status == 200 {
            format!("data: {}\n\n", json!({"error":{"message":credential}}))
        } else {
            json!({"error":{"code":"rate_limited","message":credential}}).to_string()
        };
        let error = if escaped {
            error.replace(credential, escaped_credential)
        } else {
            error
        };
        let stream = reader.get_mut();
        write!(stream, "HTTP/1.1 {status} fixture\r\ncontent-type: application/json\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{error}", error.len()).unwrap();
    });
    let mut rpc = Rpc::spawn(
        "error-redaction",
        &[
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &url,
        ],
        &[("OPENAI_API_KEY", credential)],
    );
    rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"error"}}"#);
    let start = rpc.next();
    assert_eq!(start["event"], "start");
    let end = rpc.next();
    server.join().unwrap();
    assert_eq!(end["event"], "end");
    assert_eq!(end["error"]["code"], "TURN_FAILED");
    assert!(!end.to_string().contains(credential));
    assert!(!end.to_string().contains(escaped_credential));
    if status != 200 {
        assert!(
            end.to_string()
                .contains("provider returned HTTP 429: rate_limited"),
            "unexpected sanitized terminal outcome: code={}, error={}",
            end["error"]["code"],
            end["error"]["message"]
        );
    }
    let session = start["receipt"]["session_id"].as_str().unwrap();
    let session_path = rpc
        .home
        .join("smith/sessions/run")
        .join(format!("{session}.smh"));
    let session_bytes = std::fs::read(session_path).unwrap();
    for secret in [credential, escaped_credential] {
        assert!(
            !session_bytes
                .windows(secret.len())
                .any(|bytes| bytes == secret.as_bytes())
        );
    }
    let home = rpc.home.clone();
    let (exit, stderr) = rpc.close_with_stderr();
    assert!(exit.success(), "{stderr}");
    for secret in [credential, escaped_credential] {
        assert!(!stderr.contains(secret));
    }
    let inspection = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "get", session, "--json"])
        .env("XDG_DATA_HOME", home)
        .output()
        .unwrap();
    assert!(inspection.status.success());
    let inspection = String::from_utf8(inspection.stdout).unwrap();
    for secret in [credential, escaped_credential] {
        assert!(!inspection.contains(secret));
    }
    if status != 200 {
        assert!(inspection.contains("\"status\":429"));
        assert!(inspection.contains("\"response_bytes\":"));
        assert!(inspection.contains("\"outcome\":\"failed\""));
    }
}

#[test]
fn rpc_cancellation_receipt_does_not_hide_response_head_failure() {
    let provider = StalledProvider::start(true, false);
    let mut rpc = Rpc::spawn(
        "cancel-head-failure",
        &[
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &provider.url,
        ],
        &[("OPENAI_API_KEY", "sk-e2e-fixture")],
    );
    rpc.send(r#"{"id":1,"method":"prompt","params":{"input":"block"}}"#);
    let start = rpc.next();
    assert_eq!(start["event"], "start");
    if let Err(error) = provider.received.recv_timeout(DEADLINE) {
        panic!(
            "provider did not receive the first request within {DEADLINE:?}: {error}; {}",
            rpc.request_diagnostics(start["receipt"]["session_id"].as_str().unwrap())
        );
    }
    rpc.send(r#"{"id":2,"method":"abort"}"#);
    assert_eq!(rpc.next()["aborting"], true);
    provider.release.send(()).unwrap();
    let end = rpc.next();
    assert_eq!(end["event"], "end");
    assert_eq!(end["error"]["code"], "TURN_FAILED");
    assert!(end["reason"].is_null());
    rpc.send(
        &json!({"id":3,"method":"inspect","params":{"id":start["receipt"]["submission_id"]}})
            .to_string(),
    );
    let receipt = rpc.next();
    assert_eq!(receipt["receipt"]["cancel_requested"], true);
    assert_eq!(receipt["receipt"]["status"], "Terminal(Some(Abort))");
    assert!(rpc.close().success());
}

#[test]
fn rpc_abort_reports_durable_abort_outcome() {
    let provider = StalledProvider::start(false, false);
    let mut rpc = Rpc::spawn(
        "abort",
        &[
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &provider.url,
        ],
        &[("OPENAI_API_KEY", "sk-e2e-fixture")],
    );
    // Each step waits for its reply before the next; the provider holds its
    // response until the abort is acknowledged, so the order is fixed.
    rpc.send(r#"{"id": 1, "method": "prompt", "params": {"input": "block"}}"#);
    let start = rpc.next();
    assert_eq!(start["id"], 1);
    assert_eq!(start["event"], "start");
    provider
        .received
        .recv_timeout(DEADLINE)
        .expect("provider request within the deadline");
    rpc.send(r#"{"id": 2, "method": "abort"}"#);
    let abort = rpc.next();
    assert_eq!(abort["id"], 2);
    assert_eq!(abort["aborting"], true);
    provider.release.send(()).unwrap();
    assert_eq!(
        rpc.next(),
        json!({
            "id": 1,
            "event": "end",
            "text": "",
            "reason": "Abort",
            "cost": null,
            "cost_complete": false,
        })
    );
    // `wait` answers once the aborted turn has settled; `status` then agrees.
    rpc.send(r#"{"id": 3, "method": "wait"}"#);
    assert_eq!(rpc.next(), json!({"id": 3, "settled": true}));
    rpc.send(r#"{"id": 4, "method": "status"}"#);
    assert_eq!(
        rpc.next(),
        json!({"id": 4, "settled": true, "access_mode": "writer", "active_submission_id": null})
    );
    insta::assert_snapshot!(
        "rpc_prompt_then_abort_then_wait",
        transcript_for_snapshot(&rpc.transcript)
    );
    let session = start["receipt"]["session_id"].as_str().unwrap().to_owned();
    let submission = start["receipt"]["submission_id"]
        .as_str()
        .unwrap()
        .to_owned();
    let home = rpc.home.clone();
    assert!(rpc.close().success());
    let receipt = Command::new(assert_cmd::cargo::cargo_bin("smith"))
        .args(["sessions", "receipt", &session, &submission])
        .env("XDG_DATA_HOME", &home)
        .output()
        .unwrap();
    assert!(receipt.status.success());
    let receipt: Value = serde_json::from_slice(&receipt.stdout).unwrap();
    assert_eq!(receipt["state"], "terminal");
    assert_eq!(receipt["outcome"], "cancelled");
    let mut entries = std::fs::read_dir(&home)
        .unwrap()
        .map(|entry| entry.unwrap().file_name())
        .collect::<Vec<_>>();
    entries.sort();
    assert_eq!(entries, ["runtime", "smith"]);
}