Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-cli/tests/e2e_broken_pipe.rs

Raw
//! Closed-pipe CLI output regressions through the built binary (`SMH-SPEC-SPEC0001`).

use assert_cmd::Command as AssertCommand;
use smith_core::frame::Frame;
use smith_core::store::{SessionReader, WorkStatus};
use std::io;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;

static SCRATCH_INDEX: AtomicUsize = AtomicUsize::new(0);
const CHILD_DEADLINE: Duration = Duration::from_secs(30);

struct Scratch(PathBuf);

impl Scratch {
    fn new(name: &str) -> Self {
        let parent = std::env::temp_dir();
        loop {
            let path = parent.join(format!(
                "s-{name}-{}-{}",
                std::process::id(),
                SCRATCH_INDEX.fetch_add(1, Ordering::Relaxed)
            ));
            match std::fs::create_dir(&path) {
                Ok(()) => return Self(path),
                Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
                Err(error) => panic!("creating scratch directory {}: {error}", path.display()),
            }
        }
    }
}

impl Deref for Scratch {
    type Target = Path;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl Drop for Scratch {
    fn drop(&mut self) {
        std::fs::remove_dir_all(&self.0).unwrap();
    }
}

fn smith(root: &Path, arguments: &[&str]) -> Command {
    let mut command = Command::new(env!("CARGO_BIN_EXE_smith"));
    command
        .args(arguments)
        .current_dir(root)
        .env_clear()
        .env("PATH", std::env::var_os("PATH").unwrap_or_default())
        .env(
            "CARGO_HOME",
            std::env::var_os("CARGO_HOME").unwrap_or_else(|| {
                std::env::var_os("HOME")
                    .map(PathBuf::from)
                    .unwrap_or_default()
                    .join(".cargo")
                    .into_os_string()
            }),
        )
        .env(
            "RUSTUP_HOME",
            std::env::var_os("RUSTUP_HOME").unwrap_or_else(|| {
                std::env::var_os("HOME")
                    .map(PathBuf::from)
                    .unwrap_or_default()
                    .join(".rustup")
                    .into_os_string()
            }),
        )
        .env("RUSTUP_TOOLCHAIN", "stable")
        .env("HOME", root.join("home"))
        .env("XDG_CONFIG_HOME", root.join("config"))
        .env("XDG_DATA_HOME", root.join("data"))
        .env("XDG_STATE_HOME", root.join("state"))
        .env("XDG_CACHE_HOME", root.join("cache"))
        .env("XDG_RUNTIME_DIR", root.join("r"));
    command
}

fn closed_stdout(root: &Path, arguments: &[&str]) -> (i32, String) {
    let (reader, writer) = std::io::pipe().unwrap();
    drop(reader);
    let mut command = smith(root, arguments);
    command.stdout(Stdio::from(writer)).stderr(Stdio::piped());
    let mut command = AssertCommand::from_std(command);
    command.timeout(CHILD_DEADLINE);
    command.write_stdin("broken-pipe-secret\n");
    let output = command.output().unwrap();
    (
        output.status.code().unwrap_or(-1),
        String::from_utf8_lossy(&output.stderr).into_owned(),
    )
}

fn closed_stderr(root: &Path, arguments: &[&str]) -> i32 {
    let (reader, writer) = std::io::pipe().unwrap();
    drop(reader);
    let mut command = smith(root, arguments);
    command.stdout(Stdio::piped()).stderr(Stdio::from(writer));
    let mut command = AssertCommand::from_std(command);
    command.timeout(CHILD_DEADLINE);
    let output = command.output().unwrap();
    output.status.code().unwrap_or(-1)
}

fn verify_completed_eval_session(root: &Path) {
    let session_dir = root.join("data/smith/sessions/run");
    let paths: Vec<_> = std::fs::read_dir(session_dir)
        .unwrap()
        .map(|entry| entry.unwrap().path())
        .filter(|path| path.extension().is_some_and(|extension| extension == "smh"))
        .collect();
    assert_eq!(paths.len(), 1);

    let reader = SessionReader::open(&paths[0]).unwrap();
    let frame_bytes_max = smith::config::Config::default_valid().frame_bytes_max;
    let mut admissions = Vec::new();
    reader
        .scan(frame_bytes_max, |frame| {
            if let Frame::Admission { entry, .. } = frame {
                admissions.push(entry.id);
            }
            Ok(())
        })
        .unwrap();
    assert_eq!(
        admissions.len(),
        1,
        "eval must durably admit one submission"
    );
    assert_eq!(
        reader.work_status(admissions[0], frame_bytes_max).unwrap(),
        Some(WorkStatus::Terminal(Some(
            smith::stream::StopReason::EndTurn
        )))
    );
    assert!(
        reader
            .work_output(admissions[0], frame_bytes_max)
            .unwrap()
            .is_some()
    );
    reader.verify_full(frame_bytes_max).unwrap();
}

#[test]
fn closed_stdout_cancels_catalog_eval_auth_and_plugin_output_after_durable_work() {
    let root = Scratch::new("leaf-commands");
    let package = root.join("plugin");
    for arguments in [
        vec!["catalog", "show"],
        vec!["eval", "--mock", "--json", "hello"],
        vec!["auth", "add", "--provider", "openai"],
        vec![
            "plugin",
            "init",
            package.to_str().unwrap(),
            "--name",
            "pipe",
        ],
    ] {
        let (exit, stderr) = closed_stdout(&root, &arguments);
        assert_eq!(exit, 0, "{arguments:?}: {stderr}");
        assert!(!stderr.contains("panicked"), "{arguments:?}: {stderr}");
    }
    verify_completed_eval_session(&root);
    assert!(root.join("data/smith/auth/openai").is_file());
    assert!(package.join("plugin.kdl").is_file());
}

#[test]
fn closed_stdout_auth_list_cancels_cleanly() {
    let root = Scratch::new("auth-list");
    let auth = root.join("data/smith/auth");
    std::fs::create_dir_all(&auth).unwrap();
    std::fs::write(auth.join("google"), "credential").unwrap();

    let (reader, writer) = std::io::pipe().unwrap();
    drop(reader);
    let mut command = smith(&root, &["auth", "list"]);
    command.stdout(Stdio::from(writer)).stderr(Stdio::piped());
    let mut command = AssertCommand::from_std(command);
    command.timeout(CHILD_DEADLINE);
    let output = command.output().unwrap();
    assert_eq!(output.status.code(), Some(0));
    assert!(!String::from_utf8_lossy(&output.stderr).contains("panicked"));
}

#[test]
fn closed_stderr_preserves_session_inspection_exit_status() {
    let root = Scratch::new("stderr-inspection");
    let arguments = [
        "sessions",
        "get",
        "babab-babab-babab-babab-babab-babab-babab-babab",
    ];
    assert_eq!(closed_stderr(&root, &arguments), 2);

    let mut command = AssertCommand::from_std(smith(&root, &arguments));
    command.timeout(CHILD_DEADLINE);
    let output = command.output().unwrap();
    assert_eq!(output.status.code(), Some(2));
    assert!(String::from_utf8_lossy(&output.stderr).starts_with("smith: "));
}

#[test]
fn closed_stderr_preserves_non_inspection_failure_exit_status() {
    let root = Scratch::new("stderr-execution");
    assert_eq!(closed_stderr(&root, &["eval", "hello"]), 1);
}