smith

//! End-to-end `smith eval` workflows through the built binary
//! (`SMH-SPEC-SPEC0001`, Interface modes: Eval).

use assert_cmd::Command;
use smith_core::TraceRecord;
use smith_core::store::SessionReader;
use smith_core::trace::trace;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::{Path, PathBuf};

const MOCK_REPLY: &str = "mock provider: no model configured";

/// A fresh data home per test; stale runs never leak into assertions.
fn scratch(name: &str) -> PathBuf {
    let dir = std::env::temp_dir().join(format!("smith-eval-test-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&dir);
    dir
}

fn smith(data_home: &Path) -> Command {
    let mut command = Command::cargo_bin("smith").unwrap();
    command
        .env("XDG_DATA_HOME", data_home)
        .env_remove("SMITH_PROVIDER")
        .env_remove("SMITH_MODEL")
        .env_remove("SMITH_BASE_URL")
        .env_remove("ANTHROPIC_API_KEY")
        .env_remove("OPENAI_API_KEY")
        .env_remove("GEMINI_API_KEY");
    command
}

fn sessions(data_home: &Path) -> Vec<PathBuf> {
    let dir = data_home.join("smith/sessions/run");
    let mut paths: Vec<_> = std::fs::read_dir(dir)
        .unwrap()
        .map(|entry| entry.unwrap().path())
        .filter(|path| path.extension().is_some_and(|ext| ext == "smh"))
        .collect();
    paths.sort();
    paths
}

/// Decode the session file the binary left behind into its trace.
fn trace_of(path: &Path) -> Vec<TraceRecord> {
    let max = smith::config::Config::default_valid().frame_bytes_max;
    let reader = SessionReader::open(path).unwrap();
    let recovery = reader.scan(max, |_| Ok(())).unwrap();
    assert!(recovery.damage.is_empty(), "{:?}", recovery.damage);
    assert_eq!(recovery.torn_tail_at, None);
    let mut records = Vec::new();
    trace(&reader, max, |record| {
        records.push(record);
        Ok(())
    })
    .unwrap();
    records
}

#[test]
fn eval_session_directory_failure_reports_storage_path() {
    let home = scratch("blocked-session-dir");
    assert!(std::fs::write(&home, "block data directory").is_ok());
    let attempted = home.join("smith/sessions/run");
    let output = smith(&home)
        .args(["eval", "--mock", "hello"])
        .output()
        .unwrap();
    assert!(!output.status.success());
    assert!(output.stdout.is_empty());
    let stderr = String::from_utf8(output.stderr).unwrap();
    assert!(stderr.starts_with("smith: storage [RUNTIME_DIR] create_dir_all "));
    assert!(stderr.contains(&attempted.display().to_string()));
    assert!(!stderr.contains("block data directory"));
    assert!(std::fs::remove_file(home).is_ok());
}

#[test]
fn eval_mock_turn_prints_the_reply() {
    let home = scratch("text");
    smith(&home)
        .args(["eval", "--mock", "hello"])
        .assert()
        .success()
        .stdout(format!("{MOCK_REPLY}\n"))
        .stderr("");
}

#[test]
fn eval_tool_round_persists_binary_content_for_get_and_grep() {
    let home = scratch("tool-inspection");
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let port = listener.local_addr().unwrap().port();
    let server = std::thread::spawn(move || {
        let responses = [
            "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"write\",\"arguments\":\"{\\\"path\\\":\\\"tool-fixture.txt\\\",\\\"content\\\":\\\"written\\\"}\"}}]}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n",
            "data: {\"choices\":[{\"delta\":{\"content\":\"Done.\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n",
        ];
        for response in responses {
            let (mut socket, _) = listener.accept().unwrap();
            socket
                .set_read_timeout(Some(std::time::Duration::from_secs(10)))
                .unwrap();
            let mut request = Vec::new();
            let mut buffer = [0u8; 4096];
            let header_end = loop {
                let n = socket.read(&mut buffer).unwrap();
                assert!(n > 0 && request.len() + n <= 1 << 20);
                request.extend_from_slice(&buffer[..n]);
                if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
                    break at + 4;
                }
            };
            let headers = std::str::from_utf8(&request[..header_end]).unwrap();
            let len: usize = headers
                .lines()
                .find_map(|line| {
                    line.to_ascii_lowercase()
                        .strip_prefix("content-length: ")
                        .and_then(|value| value.trim().parse().ok())
                })
                .unwrap();
            while request.len() < header_end + len {
                let n = socket.read(&mut buffer).unwrap();
                assert!(n > 0 && request.len() + n <= 1 << 20);
                request.extend_from_slice(&buffer[..n]);
            }
            write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
        }
    });
    let output = smith(&home)
        .env("OPENAI_API_KEY", "sk-fixture")
        .args([
            "eval",
            "--provider",
            "openai",
            "--model",
            "test-model",
            "--base-url",
            &format!("http://127.0.0.1:{port}/v1"),
            "write fixture",
        ])
        .output()
        .unwrap();
    assert!(
        output.status.success(),
        "{}",
        String::from_utf8_lossy(&output.stderr)
    );
    assert_eq!(output.stdout, b"Done.\n");
    server.join().unwrap();
    assert_eq!(
        std::fs::read(home.join("smith/sessions/run/tool-fixture.txt")).unwrap(),
        b"written"
    );
    let stored = sessions(&home);
    assert_eq!(stored.len(), 1);
    let id = stored[0].file_stem().unwrap().to_str().unwrap();
    let got = smith(&home)
        .args(["sessions", "get", id, "--json"])
        .output()
        .unwrap();
    assert!(
        got.status.success(),
        "{}",
        String::from_utf8_lossy(&got.stderr)
    );
    let records: Vec<serde_json::Value> = String::from_utf8(got.stdout)
        .unwrap()
        .lines()
        .map(|line| serde_json::from_str(line).unwrap())
        .collect();
    assert!(records.iter().any(|r| r["content"]["kind"] == "ToolCall"
        && r["content"]["data"]["input"]["content"] == "written"));
    assert!(
        records
            .iter()
            .any(|r| r["content"]["kind"] == "ToolResult" && r["content"]["data"]["ok"] == true)
    );
    let grep = smith(&home)
        .args(["sessions", "grep", "tool-fixture.txt", "--json"])
        .output()
        .unwrap();
    assert!(
        grep.status.success(),
        "{}",
        String::from_utf8_lossy(&grep.stderr)
    );
    assert!(String::from_utf8(grep.stdout).unwrap().contains("ToolCall"));
    let _ = std::fs::remove_dir_all(home);
}

#[test]
fn eval_json_reports_text_reason_and_cost() {
    let home = scratch("json");
    let output = smith(&home)
        .args(["eval", "--mock", "--json", "hello"])
        .output()
        .unwrap();
    assert!(output.status.success());
    let stdout = String::from_utf8(output.stdout).unwrap();
    assert_eq!(stdout.lines().count(), 1, "{stdout}");
    // The mock reply and its zero cost are fixed; the line is pinned byte for byte.
    insta::assert_snapshot!("eval_mock_json", stdout);
}

#[test]
fn eval_creates_distinct_sessions_and_explicit_resume_keeps_history() {
    let home = scratch("resume");
    smith(&home)
        .args(["eval", "--mock", "first"])
        .assert()
        .success();
    let first = sessions(&home);
    assert_eq!(first.len(), 1);
    let original = trace_of(&first[0]);
    assert!(original.len() > 1);
    smith(&home)
        .args(["eval", "--mock", "second"])
        .assert()
        .success();
    assert_eq!(sessions(&home).len(), 2);
    assert_eq!(trace_of(&first[0]), original);

    let id = first[0].file_stem().unwrap().to_str().unwrap();
    smith(&home)
        .args(["eval", "--mock", "--session", id, "third"])
        .assert()
        .success();
    assert_eq!(sessions(&home).len(), 2);
    let resumed = trace_of(&first[0]);
    assert!(resumed.len() > original.len());
    assert_eq!(resumed[..original.len()], original);
}