Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-cli/tests/e2e_atomic_tool_kill.rs

Raw
//! SMH-SPEC-SPEC0001: missing effect completion recovers as unknown, never retried.

#![cfg(unix)]

use assert_cmd::Command;
use serde_json::{Value, json};
use std::io::{Read, Write};
use std::path::Path;
use std::process::{Child, Stdio};
use std::time::Duration;

#[path = "common/mod.rs"]
mod common;
#[path = "common/home.rs"]
mod home;
use common::{COMPONENT_INVOCATION_DEADLINE, COMPONENT_STARTUP_DEADLINE};
use home::TestHome;

fn smith(home: &Path) -> Command {
    let mut command = Command::cargo_bin("smith").unwrap();
    command
        .current_dir(home)
        .env("XDG_DATA_HOME", home.join("data"))
        .env("XDG_CONFIG_HOME", home.join("config"))
        .env("XDG_CACHE_HOME", home.join("cache"))
        .env("XDG_STATE_HOME", home.join("state"))
        .env_remove("SMITH_PROVIDER")
        .env_remove("SMITH_MODEL")
        .env_remove("SMITH_BASE_URL")
        .env_remove("ANTHROPIC_API_KEY")
        .env_remove("GOOGLE_API_KEY")
        .env_remove("GEMINI_API_KEY")
        .env_remove("ZAI_API_KEY")
        .env_remove("SMITH_PLUGIN_API_KEY")
        .env_remove("OPENAI_API_KEY")
        .timeout(COMPONENT_INVOCATION_DEADLINE);
    command
}

fn request(socket: &mut std::net::TcpStream) -> Value {
    socket
        .set_read_timeout(Some(Duration::from_secs(15)))
        .unwrap();
    let mut bytes = Vec::new();
    let mut buffer = [0; 4096];
    let end = loop {
        let n = socket.read(&mut buffer).unwrap();
        assert!(n > 0 && bytes.len() + n <= 1 << 20);
        bytes.extend_from_slice(&buffer[..n]);
        if let Some(at) = bytes.windows(4).position(|part| part == b"\r\n\r\n") {
            break at + 4;
        }
    };
    let length: usize = std::str::from_utf8(&bytes[..end])
        .unwrap()
        .lines()
        .find_map(|line| {
            line.to_ascii_lowercase()
                .strip_prefix("content-length: ")
                .and_then(|n| n.parse().ok())
        })
        .unwrap();
    while bytes.len() < end + length {
        let n = socket.read(&mut buffer).unwrap();
        assert!(n > 0 && bytes.len() + n <= 1 << 20);
        bytes.extend_from_slice(&buffer[..n]);
    }
    serde_json::from_slice(&bytes[end..end + length]).unwrap()
}

fn respond(socket: &mut std::net::TcpStream, delta: Value, reason: &str) {
    let body = format!(
        "data: {}\n\ndata: {}\n\ndata: [DONE]\n\n",
        json!({"choices":[{"delta":delta}]}),
        json!({"choices":[{"delta":{},"finish_reason":reason}]})
    );
    write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap();
}

struct KillOnDrop(Child);

impl KillOnDrop {
    fn kill_and_reap(&mut self) {
        self.0.kill().unwrap();
        self.0.wait().unwrap();
    }

    fn kill_capture_stderr(&mut self) -> String {
        let _ = self.0.kill();
        let mut stderr = String::new();
        if let Some(mut pipe) = self.0.stderr.take() {
            let _ = pipe.read_to_string(&mut stderr);
        }
        let _ = self.0.wait();
        stderr
    }
}

impl Drop for KillOnDrop {
    fn drop(&mut self) {
        if self.0.try_wait().unwrap().is_none() {
            let _ = self.0.kill();
            let _ = self.0.wait();
        }
    }
}

#[test]
fn killing_during_bash_effect_preserves_unknown_intent_without_retry() {
    let home = TestHome::new("smith-kill-effect");
    smith(&home)
        .args(["eval", "--mock", "seed"])
        .assert()
        .success();
    let listed = smith(&home)
        .args(["sessions", "list", "--json"])
        .output()
        .unwrap();
    let list: Value = serde_json::from_slice(&listed.stdout).unwrap();
    let session = list["session"].as_str().unwrap();
    let admitted = smith(&home)
        .args([
            "sessions",
            "submit",
            session,
            "run the admitted bash command",
        ])
        .output()
        .unwrap();
    assert!(admitted.status.success());
    let admitted: Value = serde_json::from_slice(&admitted.stdout).unwrap();
    let submission = admitted["submission"].as_str().unwrap();

    let provider = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let provider_addr = provider.local_addr().unwrap();
    let endpoint = format!("http://{provider_addr}/v1");
    let effect_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let effect_addr = effect_listener.local_addr().unwrap();
    let effect_path = home.join("effect.txt");
    let effect_path = effect_path.to_str().unwrap();
    let command_text = format!(
        "printf 'effect-applied\\n' > '{effect_path}'; exec 3<>/dev/tcp/{}/{}; printf 'READY\\n' >&3; IFS= read -r _ <&3",
        effect_addr.ip(),
        effect_addr.port()
    );
    let (effect_tx, effect_rx) = std::sync::mpsc::sync_channel(1);
    let effect_fixture = std::thread::spawn(move || {
        let (mut socket, _) = effect_listener.accept().unwrap();
        socket
            .set_read_timeout(Some(Duration::from_secs(15)))
            .unwrap();
        let mut ready = [0; 6];
        socket.read_exact(&mut ready).unwrap();
        effect_tx.send((socket, ready)).unwrap();
    });
    let (provider_request_tx, provider_request_rx) = std::sync::mpsc::sync_channel(1);
    let provider_fixture = std::thread::spawn(move || {
        let (mut socket, _) = provider.accept().unwrap();
        let first = request(&mut socket);
        provider_request_tx.send(first).unwrap();
        respond(
            &mut socket,
            json!({"tool_calls":[{"index":0,"id":"call_bash_effect","function":{"name":"bash","arguments":json!({"command":command_text}).to_string()}}]}),
            "tool_calls",
        );
        drop(socket);
        provider
    });

    let mut child = KillOnDrop(
        std::process::Command::new(assert_cmd::cargo::cargo_bin("smith"))
            .current_dir(&home)
            .env("XDG_DATA_HOME", home.join("data"))
            .env("XDG_CONFIG_HOME", home.join("config"))
            .env("XDG_CACHE_HOME", home.join("cache"))
            .env("XDG_STATE_HOME", home.join("state"))
            .env("OPENAI_API_KEY", "fixture-key")
            .args([
                "sessions",
                "execute",
                session,
                submission,
                "--provider",
                "openai",
                "--model",
                "test-model",
                "--base-url",
                &endpoint,
            ])
            .stdout(Stdio::null())
            .stderr(Stdio::piped())
            .spawn()
            .unwrap(),
    );
    let first = match provider_request_rx.recv_timeout(COMPONENT_STARTUP_DEADLINE) {
        Ok(request) => request,
        Err(error) => {
            let _ = std::net::TcpStream::connect(provider_addr);
            let stderr = child.kill_capture_stderr();
            let _ = provider_fixture.join();
            panic!("provider request was not received: {error}; stderr: {stderr}");
        }
    };
    assert!(
        first["tools"]
            .as_array()
            .is_some_and(|tools| { tools.iter().any(|tool| tool["function"]["name"] == "bash") }),
        "bundled bash tool absent, Process grant not admitted: {first}"
    );

    let (effect_socket, ready) = match effect_rx.recv_timeout(COMPONENT_STARTUP_DEADLINE) {
        Ok(accepted) => accepted,
        Err(error) => {
            let _ = std::net::TcpStream::connect(effect_addr);
            let stderr = child.kill_capture_stderr();
            effect_fixture.join().unwrap();
            panic!("bash effect did not signal READY: {error}; stderr: {stderr}");
        }
    };
    assert_eq!(&ready, b"READY\n");
    effect_fixture.join().unwrap();
    assert_eq!(
        std::fs::read_to_string(effect_path).unwrap(),
        "effect-applied\n"
    );

    child.kill_and_reap();
    drop(effect_socket);
    let provider = provider_fixture.join().unwrap();
    provider.set_nonblocking(true).unwrap();

    let queried = smith(&home)
        .args(["sessions", "receipt", session, submission])
        .output()
        .unwrap();
    assert!(queried.status.success());
    let receipt: Value = serde_json::from_slice(&queried.stdout).unwrap();
    assert_eq!(receipt["state"], "running");
    assert_eq!(receipt["recovery_required"], true);

    let retried = smith(&home)
        .env("OPENAI_API_KEY", "fixture-key")
        .args([
            "sessions",
            "execute",
            session,
            submission,
            "--provider",
            "openai",
            "--model",
            "test-model",
            "--base-url",
            &endpoint,
        ])
        .output()
        .unwrap();
    assert!(!retried.status.success());
    assert_eq!(
        provider.accept().unwrap_err().kind(),
        std::io::ErrorKind::WouldBlock
    );
    assert_eq!(
        std::fs::read_to_string(effect_path).unwrap(),
        "effect-applied\n"
    );

    let history = smith(&home)
        .args(["sessions", "get", session, "--json"])
        .output()
        .unwrap();
    assert!(history.status.success());
    let records: Vec<Value> = String::from_utf8(history.stdout)
        .unwrap()
        .lines()
        .map(|line| serde_json::from_str(line).unwrap())
        .collect();
    let provenance_index = records
        .iter()
        .position(|record| {
            record["content"]["kind"] == "Meta"
                && record["content"]["data"]["kind"] == "tool-loadout-provenance"
        })
        .expect("tool loadout provenance missing");
    let components = records[provenance_index]["content"]["data"]["detail"]["components"]
        .as_array()
        .expect("provenance components missing");
    let coding_tools = components
        .iter()
        .find(|component| component["source_id"] == "smith:coding-tools")
        .expect("bundled wasm component provenance missing");
    for hash in ["manifest_sha256", "component_sha256"] {
        assert_eq!(coding_tools[hash].as_str().unwrap().len(), 64);
        assert!(
            coding_tools[hash]
                .as_str()
                .unwrap()
                .bytes()
                .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
        );
    }
    let tool_call_index = records
        .iter()
        .position(|record| {
            record["content"]["kind"] == "ToolCall"
                && record["content"]["data"]["name"] == "bash"
                && record["content"]["data"]["input"]["command"]
                    .as_str()
                    .is_some_and(|command| command.contains("effect-applied"))
        })
        .expect("raw bash intent missing");
    assert!(
        provenance_index < tool_call_index,
        "loadout provenance must be recorded before tool dispatch: {records:?}"
    );
    assert!(
        !records.iter().any(|record| {
            record["content"]["kind"] == "ToolResult"
                || record.to_string().contains("ToolCompletion")
        }),
        "completion unexpectedly recorded: {records:?}"
    );
}