Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-cli/tests/e2e_atomic_tool_completion.rs

Raw
//! SMH-SPEC-SPEC0001: admitted component writes, paired followup, durable receipt.
use assert_cmd::Command;
use serde_json::{Value, json};
use std::io::{Read, Write};
use std::path::Path;
use std::time::Duration;

#[path = "common/mod.rs"]
mod common;
#[path = "common/home.rs"]
mod home;
use common::{COMPONENT_INVOCATION_DEADLINE, COMPONENT_STARTUP_DEADLINE};
use home::TestHome;

fn smith(home: &Path) -> Command {
    let mut command = Command::cargo_bin("smith").unwrap();
    command
        .current_dir(home)
        .env("XDG_DATA_HOME", home.join("data"))
        .env("XDG_CONFIG_HOME", home.join("config"))
        .env("XDG_CACHE_HOME", home.join("cache"))
        .env("XDG_STATE_HOME", home.join("state"))
        .env_remove("SMITH_PROVIDER")
        .env_remove("SMITH_MODEL")
        .env_remove("SMITH_BASE_URL")
        .env_remove("ANTHROPIC_API_KEY")
        .env_remove("GOOGLE_API_KEY")
        .env_remove("GEMINI_API_KEY")
        .env_remove("ZAI_API_KEY")
        .env_remove("SMITH_PLUGIN_API_KEY")
        .env_remove("OPENAI_API_KEY")
        .timeout(COMPONENT_INVOCATION_DEADLINE);
    command
}

fn request(socket: &mut std::net::TcpStream) -> Value {
    socket
        .set_read_timeout(Some(Duration::from_secs(10)))
        .unwrap();
    socket
        .set_write_timeout(Some(Duration::from_secs(10)))
        .unwrap();
    let mut bytes = Vec::new();
    let mut buffer = [0; 4096];
    let end = loop {
        let n = socket.read(&mut buffer).unwrap();
        assert!(n > 0 && bytes.len() + n <= 1 << 20);
        bytes.extend_from_slice(&buffer[..n]);
        if let Some(at) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
            break at + 4;
        }
    };
    let length: usize = std::str::from_utf8(&bytes[..end])
        .unwrap()
        .lines()
        .find_map(|line| {
            line.to_ascii_lowercase()
                .strip_prefix("content-length: ")
                .and_then(|n| n.parse().ok())
        })
        .unwrap();
    while bytes.len() < end + length {
        let n = socket.read(&mut buffer).unwrap();
        assert!(n > 0 && bytes.len() + n <= 1 << 20);
        bytes.extend_from_slice(&buffer[..n]);
    }
    serde_json::from_slice(&bytes[end..end + length]).unwrap()
}

fn respond(socket: &mut std::net::TcpStream, delta: Value, reason: &str) {
    let body = format!(
        "data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
        json!({"choices":[{"delta":delta}]}),
        json!({"choices":[{"delta":{},"finish_reason":reason}]})
    );
    write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap();
}

#[test]
fn admitted_component_writes_preserve_ordered_paired_followup_before_terminal_receipt() {
    let home = TestHome::new("smith-atomic-tool-e2e");
    smith(&home)
        .args(["eval", "--mock", "seed"])
        .assert()
        .success();
    let listed = smith(&home)
        .args(["sessions", "list", "--json"])
        .output()
        .unwrap();
    let list: Value = serde_json::from_slice(&listed.stdout).unwrap();
    let session = list["session"].as_str().unwrap();
    let admitted = smith(&home)
        .args(["sessions", "submit", session, "write a file"])
        .output()
        .unwrap();
    assert!(admitted.status.success());
    let admitted: Value = serde_json::from_slice(&admitted.stdout).unwrap();
    let submission = admitted["submission"].as_str().unwrap();
    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let endpoint = format!("http://{}/v1", listener.local_addr().unwrap());
    let (tx, rx) = std::sync::mpsc::sync_channel(2);
    let workdir = home.as_ref().to_path_buf();
    let fixture = std::thread::spawn(move || {
        // The process deadline and bounded request signals keep fixture failure
        // from turning a missing connection into an unbounded join.
        let (mut socket, _) = listener.accept().unwrap();
        let first = request(&mut socket);
        tx.send(first).unwrap();
        respond(
            &mut socket,
            json!({"tool_calls":[
                {"index":0,"id":"call_write_first","function":{
                    "name":"write", "arguments":json!({"path":"first.txt","content":"once"}).to_string()
                }},
                {"index":1,"id":"call_write_second","function":{
                    "name":"write", "arguments":json!({"path":"second.txt","content":"twice"}).to_string()
                }}
            ]}),
            "tool_calls",
        );
        drop(socket);
        let (mut socket, _) = listener.accept().unwrap();
        let second = request(&mut socket);
        assert_eq!(
            std::fs::read_to_string(workdir.join("first.txt")).unwrap(),
            "once"
        );
        assert_eq!(
            std::fs::read_to_string(workdir.join("second.txt")).unwrap(),
            "twice"
        );
        let messages = second["messages"].as_array().unwrap();
        let results: Vec<_> = messages
            .iter()
            .filter(|message| message["role"] == "tool")
            .collect();
        assert_eq!(results.len(), 2, "{second}");
        assert_eq!(results[0]["tool_call_id"], "call_write_first");
        assert_eq!(results[1]["tool_call_id"], "call_write_second");
        assert_eq!(
            messages
                .iter()
                .filter(|message| message.get("tool_calls").is_some())
                .count(),
            1
        );
        tx.send(second).unwrap();
        respond(&mut socket, json!({"content":"done"}), "stop");
        listener
    });
    let mut command = smith(&home);
    command.env("OPENAI_API_KEY", "fixture-key").args([
        "sessions",
        "execute",
        session,
        submission,
        "--provider",
        "openai",
        "--model",
        "test-model",
        "--base-url",
        &endpoint,
    ]);
    let output = command.output().unwrap();
    let first = rx.recv_timeout(COMPONENT_STARTUP_DEADLINE).unwrap();
    assert!(
        first["tools"]
            .as_array()
            .is_some_and(|tools| tools.iter().any(|tool| tool["function"]["name"] == "write")),
        "{first}"
    );
    assert!(
        output.status.success(),
        "{}",
        String::from_utf8_lossy(&output.stderr)
    );
    let _followup = rx.recv_timeout(COMPONENT_STARTUP_DEADLINE).unwrap();
    let listener = fixture.join().unwrap();
    let queried = smith(&home)
        .args(["sessions", "receipt", session, submission])
        .output()
        .unwrap();
    let receipt: Value = serde_json::from_slice(&queried.stdout).unwrap();
    assert_eq!(receipt["state"], "terminal");
    assert_eq!(receipt["outcome"], "completed");
    let history = smith(&home)
        .args(["sessions", "get", session, "--json"])
        .output()
        .unwrap();
    assert!(history.status.success());
    let history = String::from_utf8(history.stdout).unwrap();
    assert!(!history.contains("fixture-key"));
    let records: Vec<Value> = history
        .lines()
        .map(|line| serde_json::from_str(line).unwrap())
        .collect();
    assert_eq!(
        records
            .iter()
            .filter(|record| record["content"]["kind"] == "ToolResult")
            .count(),
        2
    );
    assert_eq!(
        records
            .iter()
            .filter(|record| record["content"]["kind"] == "Message"
                && record["content"]["data"]["role"] == "Tool")
            .count(),
        2
    );
    smith(&home)
        .env("OPENAI_API_KEY", "fixture-key")
        .args([
            "sessions",
            "execute",
            session,
            submission,
            "--provider",
            "openai",
            "--model",
            "test-model",
            "--base-url",
            &endpoint,
        ])
        .assert()
        .failure();
    listener.set_nonblocking(true).unwrap();
    assert_eq!(
        listener.accept().unwrap_err().kind(),
        std::io::ErrorKind::WouldBlock
    );
    assert_eq!(
        std::fs::read_to_string(home.join("first.txt")).unwrap(),
        "once"
    );
    assert_eq!(
        std::fs::read_to_string(home.join("second.txt")).unwrap(),
        "twice"
    );
}