repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
smith-cli/tests/e2e_broken_pipe.rs
Raw//! Closed-pipe CLI output regressions through the built binary (`SMH-SPEC-SPEC0001`).
use assert_cmd::Command as AssertCommand;
use smith_core::frame::Frame;
use smith_core::store::{SessionReader, WorkStatus};
use std::io;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
static SCRATCH_INDEX: AtomicUsize = AtomicUsize::new(0);
const CHILD_DEADLINE: Duration = Duration::from_secs(30);
struct Scratch(PathBuf);
impl Scratch {
fn new(name: &str) -> Self {
let parent = std::env::temp_dir();
loop {
let path = parent.join(format!(
"s-{name}-{}-{}",
std::process::id(),
SCRATCH_INDEX.fetch_add(1, Ordering::Relaxed)
));
match std::fs::create_dir(&path) {
Ok(()) => return Self(path),
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
Err(error) => panic!("creating scratch directory {}: {error}", path.display()),
}
}
}
}
impl Deref for Scratch {
type Target = Path;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Drop for Scratch {
fn drop(&mut self) {
std::fs::remove_dir_all(&self.0).unwrap();
}
}
fn smith(root: &Path, arguments: &[&str]) -> Command {
let mut command = Command::new(env!("CARGO_BIN_EXE_smith"));
command
.args(arguments)
.current_dir(root)
.env_clear()
.env("PATH", std::env::var_os("PATH").unwrap_or_default())
.env(
"CARGO_HOME",
std::env::var_os("CARGO_HOME").unwrap_or_else(|| {
std::env::var_os("HOME")
.map(PathBuf::from)
.unwrap_or_default()
.join(".cargo")
.into_os_string()
}),
)
.env(
"RUSTUP_HOME",
std::env::var_os("RUSTUP_HOME").unwrap_or_else(|| {
std::env::var_os("HOME")
.map(PathBuf::from)
.unwrap_or_default()
.join(".rustup")
.into_os_string()
}),
)
.env("RUSTUP_TOOLCHAIN", "stable")
.env("HOME", root.join("home"))
.env("XDG_CONFIG_HOME", root.join("config"))
.env("XDG_DATA_HOME", root.join("data"))
.env("XDG_STATE_HOME", root.join("state"))
.env("XDG_CACHE_HOME", root.join("cache"))
.env("XDG_RUNTIME_DIR", root.join("r"));
command
}
fn closed_stdout(root: &Path, arguments: &[&str]) -> (i32, String) {
let (reader, writer) = std::io::pipe().unwrap();
drop(reader);
let mut command = smith(root, arguments);
command.stdout(Stdio::from(writer)).stderr(Stdio::piped());
let mut command = AssertCommand::from_std(command);
command.timeout(CHILD_DEADLINE);
command.write_stdin("broken-pipe-secret\n");
let output = command.output().unwrap();
(
output.status.code().unwrap_or(-1),
String::from_utf8_lossy(&output.stderr).into_owned(),
)
}
fn closed_stderr(root: &Path, arguments: &[&str]) -> i32 {
let (reader, writer) = std::io::pipe().unwrap();
drop(reader);
let mut command = smith(root, arguments);
command.stdout(Stdio::piped()).stderr(Stdio::from(writer));
let mut command = AssertCommand::from_std(command);
command.timeout(CHILD_DEADLINE);
let output = command.output().unwrap();
output.status.code().unwrap_or(-1)
}
fn verify_completed_eval_session(root: &Path) {
let session_dir = root.join("data/smith/sessions/run");
let paths: Vec<_> = std::fs::read_dir(session_dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension().is_some_and(|extension| extension == "smh"))
.collect();
assert_eq!(paths.len(), 1);
let reader = SessionReader::open(&paths[0]).unwrap();
let frame_bytes_max = smith::config::Config::default_valid().frame_bytes_max;
let mut admissions = Vec::new();
reader
.scan(frame_bytes_max, |frame| {
if let Frame::Admission { entry, .. } = frame {
admissions.push(entry.id);
}
Ok(())
})
.unwrap();
assert_eq!(
admissions.len(),
1,
"eval must durably admit one submission"
);
assert_eq!(
reader.work_status(admissions[0], frame_bytes_max).unwrap(),
Some(WorkStatus::Terminal(Some(
smith::stream::StopReason::EndTurn
)))
);
assert!(
reader
.work_output(admissions[0], frame_bytes_max)
.unwrap()
.is_some()
);
reader.verify_full(frame_bytes_max).unwrap();
}
#[test]
fn closed_stdout_cancels_catalog_eval_auth_and_plugin_output_after_durable_work() {
let root = Scratch::new("leaf-commands");
let package = root.join("plugin");
for arguments in [
vec!["catalog", "show"],
vec!["eval", "--mock", "--json", "hello"],
vec!["auth", "add", "--provider", "openai"],
vec![
"plugin",
"init",
package.to_str().unwrap(),
"--name",
"pipe",
],
] {
let (exit, stderr) = closed_stdout(&root, &arguments);
assert_eq!(exit, 0, "{arguments:?}: {stderr}");
assert!(!stderr.contains("panicked"), "{arguments:?}: {stderr}");
}
verify_completed_eval_session(&root);
assert!(root.join("data/smith/auth/openai").is_file());
assert!(package.join("plugin.kdl").is_file());
}
#[test]
fn closed_stdout_auth_list_cancels_cleanly() {
let root = Scratch::new("auth-list");
let auth = root.join("data/smith/auth");
std::fs::create_dir_all(&auth).unwrap();
std::fs::write(auth.join("google"), "credential").unwrap();
let (reader, writer) = std::io::pipe().unwrap();
drop(reader);
let mut command = smith(&root, &["auth", "list"]);
command.stdout(Stdio::from(writer)).stderr(Stdio::piped());
let mut command = AssertCommand::from_std(command);
command.timeout(CHILD_DEADLINE);
let output = command.output().unwrap();
assert_eq!(output.status.code(), Some(0));
assert!(!String::from_utf8_lossy(&output.stderr).contains("panicked"));
}
#[test]
fn closed_stderr_preserves_session_inspection_exit_status() {
let root = Scratch::new("stderr-inspection");
let arguments = [
"sessions",
"get",
"babab-babab-babab-babab-babab-babab-babab-babab",
];
assert_eq!(closed_stderr(&root, &arguments), 2);
let mut command = AssertCommand::from_std(smith(&root, &arguments));
command.timeout(CHILD_DEADLINE);
let output = command.output().unwrap();
assert_eq!(output.status.code(), Some(2));
assert!(String::from_utf8_lossy(&output.stderr).starts_with("smith: "));
}
#[test]
fn closed_stderr_preserves_non_inspection_failure_exit_status() {
let root = Scratch::new("stderr-execution");
assert_eq!(closed_stderr(&root, &["eval", "hello"]), 1);
}