Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-cli/tests/e2e_eval.rs

Raw
//! End-to-end `smith eval` workflows through the built binary
//! (`SMH-SPEC-SPEC0001`, Interface modes: Eval).

use assert_cmd::Command;
use smith_core::TraceRecord;
use smith_core::store::SessionReader;
use smith_core::trace::trace;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};

const MOCK_REPLY: &str = "mock provider: no model configured";
static SCRATCH_INDEX: AtomicUsize = AtomicUsize::new(0);

/// A fresh data home per test; stale runs never leak into assertions.
fn scratch(_name: &str) -> PathBuf {
    let index = SCRATCH_INDEX.fetch_add(1, Ordering::Relaxed);
    let dir = PathBuf::from("/var/tmp/smith-session-e2e")
        .join(format!("e-{}-{index}", std::process::id()));
    let _ = std::fs::remove_dir_all(&dir);
    dir
}

fn smith(data_home: &Path) -> Command {
    let mut command = Command::cargo_bin("smith").unwrap();
    command
        .env("XDG_CONFIG_HOME", data_home.join("config"))
        .env("XDG_DATA_HOME", data_home)
        .env("XDG_STATE_HOME", data_home.join("state"))
        .env("XDG_CACHE_HOME", data_home.join("cache"))
        .env("XDG_RUNTIME_DIR", data_home.join("runtime"))
        .env_remove("SMITH_PROVIDER")
        .env_remove("SMITH_MODEL")
        .env_remove("SMITH_BASE_URL")
        .env_remove("ANTHROPIC_API_KEY")
        .env_remove("OPENAI_API_KEY")
        .env_remove("GEMINI_API_KEY");
    command
}

fn sessions(data_home: &Path) -> Vec<PathBuf> {
    let dir = data_home.join("smith/sessions/run");
    let mut paths: Vec<_> = std::fs::read_dir(dir)
        .unwrap()
        .map(|entry| entry.unwrap().path())
        .filter(|path| path.extension().is_some_and(|ext| ext == "smh"))
        .collect();
    paths.sort();
    paths
}

/// Decode the session file the binary left behind into its trace.
fn trace_of(path: &Path) -> Vec<TraceRecord> {
    let max = smith::config::Config::default_valid().frame_bytes_max;
    let reader = SessionReader::open(path).unwrap();
    let recovery = reader.scan(max, |_| Ok(())).unwrap();
    assert!(recovery.damage.is_empty(), "{:?}", recovery.damage);
    assert_eq!(recovery.torn_tail_at, None);
    let mut records = Vec::new();
    trace(&reader, max, |record| {
        records.push(record);
        Ok(())
    })
    .unwrap();
    records
}

#[test]
fn eval_session_directory_failure_reports_storage_path() {
    let home = scratch("blocked-session-dir");
    assert!(std::fs::write(&home, "block data directory").is_ok());
    let attempted = home.join("smith/sessions/run");
    let output = smith(&home)
        .args(["eval", "--mock", "hello"])
        .output()
        .unwrap();
    assert!(!output.status.success());
    assert!(output.stdout.is_empty());
    let stderr = String::from_utf8(output.stderr).unwrap();
    assert!(stderr.starts_with("smith: storage [RUNTIME_DIR] create_dir_all "));
    assert!(stderr.contains(&attempted.display().to_string()));
    assert!(!stderr.contains("block data directory"));
    assert!(std::fs::remove_file(home).is_ok());
}

#[test]
fn eval_leaves_a_terminal_durable_receipt_after_reopen() {
    let home = scratch("durable-receipt");
    smith(&home)
        .args(["eval", "--mock", "hello"])
        .assert()
        .success()
        .stdout(format!("{MOCK_REPLY}\n"));
    let paths = sessions(&home);
    assert_eq!(paths.len(), 1);
    let reader = SessionReader::open(&paths[0]).unwrap();
    let limit = smith::config::Config::default_valid().frame_bytes_max;
    let mut admissions = Vec::new();
    reader
        .scan(limit, |frame| {
            if let smith_core::frame::Frame::Admission { entry, .. } = frame {
                admissions.push(entry.id);
            }
            Ok(())
        })
        .unwrap();
    assert_eq!(admissions.len(), 1, "eval must use durable admission");
    let receipt = reader.receipt(admissions[0], limit).unwrap().unwrap();
    assert_eq!(receipt.session_id, reader.header(limit).unwrap().session_id);
    assert_eq!(
        reader.work_status(admissions[0], limit).unwrap(),
        Some(smith_core::store::WorkStatus::Terminal(Some(
            smith::stream::StopReason::EndTurn
        )))
    );
    assert!(reader.work_output(admissions[0], limit).unwrap().is_some());
    reader.verify_full(limit).unwrap();
}

#[test]
fn eval_mock_turn_prints_the_reply() {
    let home = scratch("text");
    smith(&home)
        .args(["eval", "--mock", "hello"])
        .assert()
        .success()
        .stdout(format!("{MOCK_REPLY}\n"))
        .stderr("");
}

#[test]
fn eval_tool_round_persists_binary_content_for_get_and_grep() {
    let home = scratch("tool-inspection");
    let workspace = home.join("workspace");
    std::fs::create_dir_all(&workspace).unwrap();
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let port = listener.local_addr().unwrap().port();
    let server = std::thread::spawn(move || {
        let responses = [
            "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"write\",\"arguments\":\"{\\\"path\\\":\\\"tool-fixture.txt\\\",\\\"content\\\":\\\"written\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
            "data: {\"choices\":[{\"delta\":{\"content\":\"Done.\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
        ];
        for response in responses {
            let (mut socket, _) = listener.accept().unwrap();
            socket
                .set_read_timeout(Some(std::time::Duration::from_secs(10)))
                .unwrap();
            let mut request = Vec::new();
            let mut buffer = [0u8; 4096];
            let header_end = loop {
                let n = socket.read(&mut buffer).unwrap();
                assert!(n > 0 && request.len() + n <= 1 << 20);
                request.extend_from_slice(&buffer[..n]);
                if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
                    break at + 4;
                }
            };
            let headers = std::str::from_utf8(&request[..header_end]).unwrap();
            let len: usize = headers
                .lines()
                .find_map(|line| {
                    line.to_ascii_lowercase()
                        .strip_prefix("content-length: ")
                        .and_then(|value| value.trim().parse().ok())
                })
                .unwrap();
            while request.len() < header_end + len {
                let n = socket.read(&mut buffer).unwrap();
                assert!(n > 0 && request.len() + n <= 1 << 20);
                request.extend_from_slice(&buffer[..n]);
            }
            write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
        }
    });
    let output = smith(&home)
        .current_dir(&workspace)
        .env("OPENAI_API_KEY", "sk-fixture")
        .args([
            "eval",
            "--provider",
            "openai",
            "--model",
            "test-model",
            "--base-url",
            &format!("http://127.0.0.1:{port}/v1"),
            "write fixture",
        ])
        .output()
        .unwrap();
    assert!(
        output.status.success(),
        "{}",
        String::from_utf8_lossy(&output.stderr)
    );
    assert_eq!(output.stdout, b"Done.\n");
    server.join().unwrap();
    assert_eq!(
        std::fs::read(workspace.join("tool-fixture.txt")).unwrap(),
        b"written"
    );
    assert!(!home.join("smith/sessions/run/tool-fixture.txt").exists());
    let stored = sessions(&home);
    assert_eq!(stored.len(), 1);
    let id = stored[0].file_stem().unwrap().to_str().unwrap();
    let got = smith(&home)
        .args(["sessions", "get", id, "--json"])
        .output()
        .unwrap();
    assert!(
        got.status.success(),
        "{}",
        String::from_utf8_lossy(&got.stderr)
    );
    let records: Vec<serde_json::Value> = String::from_utf8(got.stdout)
        .unwrap()
        .lines()
        .map(|line| serde_json::from_str(line).unwrap())
        .collect();
    assert!(records.iter().any(|r| r["content"]["kind"] == "ToolCall"
        && r["content"]["data"]["input"]["content"] == "written"));
    assert!(
        records
            .iter()
            .any(|r| r["content"]["kind"] == "ToolResult" && r["content"]["data"]["ok"] == true)
    );
    let grep = smith(&home)
        .args(["sessions", "grep", "tool-fixture.txt", "--json"])
        .output()
        .unwrap();
    assert!(
        grep.status.success(),
        "{}",
        String::from_utf8_lossy(&grep.stderr)
    );
    assert!(String::from_utf8(grep.stdout).unwrap().contains("ToolCall"));
    let _ = std::fs::remove_dir_all(home);
}

#[test]
fn eval_tool_round_guest_accepts_advertised_read_options() {
    let home = scratch("wasm-tool-dispatch");
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let port = listener.local_addr().unwrap().port();
    let server = std::thread::spawn(move || {
        let responses = [
            "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_wasm\",\"function\":{\"name\":\"read\",\"arguments\":\"{\\\"path\\\":\\\"missing.txt\\\",\\\"max_bytes\\\":7}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
            "data: {\"choices\":[{\"delta\":{\"content\":\"Done.\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
        ];
        let mut requests = Vec::new();
        for response in responses {
            let (mut socket, _) = listener.accept().unwrap();
            socket
                .set_read_timeout(Some(std::time::Duration::from_secs(10)))
                .unwrap();
            let mut request = Vec::new();
            let mut buffer = [0_u8; 4096];
            while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
                let read = socket.read(&mut buffer).unwrap();
                assert!(read > 0);
                request.extend_from_slice(&buffer[..read]);
            }
            let header_end = request
                .windows(4)
                .position(|bytes| bytes == b"\r\n\r\n")
                .unwrap()
                + 4;
            let headers = std::str::from_utf8(&request[..header_end]).unwrap();
            let length: usize = headers
                .lines()
                .find_map(|line| {
                    line.to_ascii_lowercase()
                        .strip_prefix("content-length: ")
                        .and_then(|value| value.parse().ok())
                })
                .unwrap();
            while request.len() < header_end + length {
                let read = socket.read(&mut buffer).unwrap();
                assert!(read > 0);
                request.extend_from_slice(&buffer[..read]);
            }
            requests.push(String::from_utf8(request).unwrap());
            write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
        }
        requests
    });
    smith(&home)
        .env("OPENAI_API_KEY", "sk-fixture")
        .args([
            "eval",
            "--provider",
            "openai",
            "--model",
            "test-model",
            "--base-url",
            &format!("http://127.0.0.1:{port}/v1"),
            "accept read options",
        ])
        .assert()
        .success()
        .stdout("Done.\n");
    let requests = server.join().unwrap();
    assert!(requests[1].contains("READ_ERROR"));
    assert!(!requests[1].contains("invalid tool input"));
    let _ = std::fs::remove_dir_all(home);
}

#[test]
fn eval_json_reports_text_reason_and_cost() {
    let home = scratch("json");
    let output = smith(&home)
        .args(["eval", "--mock", "--json", "hello"])
        .output()
        .unwrap();
    assert!(output.status.success());
    let stdout = String::from_utf8(output.stdout).unwrap();
    assert_eq!(stdout.lines().count(), 1, "{stdout}");
    // The mock reply and its zero cost are fixed; the line is pinned byte for byte.
    insta::assert_snapshot!("eval_mock_json", stdout);
}

#[test]
fn eval_creates_distinct_sessions_and_explicit_resume_keeps_history() {
    let home = scratch("resume");
    smith(&home)
        .args(["eval", "--mock", "first"])
        .assert()
        .success();
    let first = sessions(&home);
    assert_eq!(first.len(), 1);
    let original = trace_of(&first[0]);
    assert!(original.len() > 1);
    smith(&home)
        .args(["eval", "--mock", "second"])
        .assert()
        .success();
    assert_eq!(sessions(&home).len(), 2);
    assert_eq!(trace_of(&first[0]), original);

    let id = first[0].file_stem().unwrap().to_str().unwrap();
    smith(&home)
        .args(["eval", "--mock", "--session", id, "third"])
        .assert()
        .success();
    assert_eq!(sessions(&home).len(), 2);
    let resumed = trace_of(&first[0]);
    assert!(resumed.len() > original.len());
    assert_eq!(resumed[..original.len()], original);
}

#[test]
fn eval_split_credential_in_provider_error_never_reaches_outputs_or_session_bytes() {
    provider_error_never_reaches_outputs_or_session_bytes(false);
}

#[test]
fn eval_json_escaped_credential_in_provider_error_never_reaches_outputs_or_session_bytes() {
    provider_error_never_reaches_outputs_or_session_bytes(true);
}

fn provider_error_never_reaches_outputs_or_session_bytes(escaped: bool) {
    let home = scratch("error-redaction");
    let credential = "sk-e2e-split-fixture";
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let port = listener.local_addr().unwrap().port();
    let response = format!(
        "data: {}\n\n",
        serde_json::json!({"error":{"message":credential}})
    );
    let (response, split) = if escaped {
        let spelling = r"\u0073k-e2e-\u0073plit-fixture";
        let response = response.replace(credential, spelling);
        let split = response.find(spelling).unwrap() + 4;
        (response, split)
    } else {
        let split = response.find(credential).unwrap() + "sk-e2e-split-".len();
        (response, split)
    };
    let server = std::thread::spawn(move || {
        let (mut socket, _) = listener.accept().unwrap();
        socket
            .set_read_timeout(Some(std::time::Duration::from_secs(10)))
            .unwrap();
        let mut request = Vec::new();
        let mut buffer = [0_u8; 4096];
        while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
            let read = socket.read(&mut buffer).unwrap();
            assert!(read > 0);
            request.extend_from_slice(&buffer[..read]);
        }
        write!(
            socket,
            "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n"
        )
        .unwrap();
        socket.write_all(&response.as_bytes()[..split]).unwrap();
        socket.flush().unwrap();
        socket.write_all(&response.as_bytes()[split..]).unwrap();
    });
    let output = smith(&home)
        .env("OPENAI_API_KEY", credential)
        .args([
            "eval",
            "--provider",
            "openai",
            "--model",
            "e2e-model",
            "--base-url",
            &format!("http://127.0.0.1:{port}/v1"),
            "error",
        ])
        .output()
        .unwrap();
    server.join().unwrap();
    assert!(!output.status.success());
    let encoded_credential = serde_json::to_string(credential).unwrap();
    for bytes in [&output.stdout, &output.stderr] {
        assert!(
            !bytes
                .windows(credential.len())
                .any(|bytes| bytes == credential.as_bytes())
        );
        assert!(
            !bytes
                .windows(encoded_credential.len())
                .any(|bytes| bytes == encoded_credential.as_bytes())
        );
    }
    let session = sessions(&home).pop().unwrap();
    let bytes = std::fs::read(&session).unwrap();
    assert!(
        !bytes
            .windows(credential.len())
            .any(|bytes| bytes == credential.as_bytes())
    );
    assert!(
        !bytes
            .windows(encoded_credential.len())
            .any(|bytes| bytes == encoded_credential.as_bytes())
    );
    let id = session.file_stem().unwrap().to_str().unwrap();
    let inspection = smith(&home)
        .args(["sessions", "get", id, "--json"])
        .output()
        .unwrap();
    assert!(inspection.status.success());
    let inspection_text = String::from_utf8_lossy(&inspection.stdout);
    assert!(inspection_text.contains("\"status\":200"));
    assert!(inspection_text.contains("\"response_bytes\":"));
    assert!(inspection_text.contains("\"outcome\":\"failed\""));
    assert!(
        !inspection
            .stdout
            .windows(credential.len())
            .any(|bytes| bytes == credential.as_bytes())
    );
    assert!(
        !inspection
            .stdout
            .windows(encoded_credential.len())
            .any(|bytes| bytes == encoded_credential.as_bytes())
    );
    let _ = std::fs::remove_dir_all(home);
}