//! Closed-pipe CLI output regressions through the built binary (`SMH-SPEC-SPEC0001`). use assert_cmd::Command as AssertCommand; use smith_core::frame::Frame; use smith_core::store::{SessionReader, WorkStatus}; use std::io; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; static SCRATCH_INDEX: AtomicUsize = AtomicUsize::new(0); const CHILD_DEADLINE: Duration = Duration::from_secs(30); struct Scratch(PathBuf); impl Scratch { fn new(name: &str) -> Self { let parent = std::env::temp_dir(); loop { let path = parent.join(format!( "s-{name}-{}-{}", std::process::id(), SCRATCH_INDEX.fetch_add(1, Ordering::Relaxed) )); match std::fs::create_dir(&path) { Ok(()) => return Self(path), Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, Err(error) => panic!("creating scratch directory {}: {error}", path.display()), } } } } impl Deref for Scratch { type Target = Path; fn deref(&self) -> &Self::Target { &self.0 } } impl Drop for Scratch { fn drop(&mut self) { std::fs::remove_dir_all(&self.0).unwrap(); } } fn smith(root: &Path, arguments: &[&str]) -> Command { let mut command = Command::new(env!("CARGO_BIN_EXE_smith")); command .args(arguments) .current_dir(root) .env_clear() .env("PATH", std::env::var_os("PATH").unwrap_or_default()) .env( "CARGO_HOME", std::env::var_os("CARGO_HOME").unwrap_or_else(|| { std::env::var_os("HOME") .map(PathBuf::from) .unwrap_or_default() .join(".cargo") .into_os_string() }), ) .env( "RUSTUP_HOME", std::env::var_os("RUSTUP_HOME").unwrap_or_else(|| { std::env::var_os("HOME") .map(PathBuf::from) .unwrap_or_default() .join(".rustup") .into_os_string() }), ) .env("RUSTUP_TOOLCHAIN", "stable") .env("HOME", root.join("home")) .env("XDG_CONFIG_HOME", root.join("config")) .env("XDG_DATA_HOME", root.join("data")) .env("XDG_STATE_HOME", root.join("state")) .env("XDG_CACHE_HOME", root.join("cache")) .env("XDG_RUNTIME_DIR", root.join("r")); command } fn closed_stdout(root: &Path, arguments: &[&str]) -> (i32, String) { let (reader, writer) = std::io::pipe().unwrap(); drop(reader); let mut command = smith(root, arguments); command.stdout(Stdio::from(writer)).stderr(Stdio::piped()); let mut command = AssertCommand::from_std(command); command.timeout(CHILD_DEADLINE); command.write_stdin("broken-pipe-secret\n"); let output = command.output().unwrap(); ( output.status.code().unwrap_or(-1), String::from_utf8_lossy(&output.stderr).into_owned(), ) } fn closed_stderr(root: &Path, arguments: &[&str]) -> i32 { let (reader, writer) = std::io::pipe().unwrap(); drop(reader); let mut command = smith(root, arguments); command.stdout(Stdio::piped()).stderr(Stdio::from(writer)); let mut command = AssertCommand::from_std(command); command.timeout(CHILD_DEADLINE); let output = command.output().unwrap(); output.status.code().unwrap_or(-1) } fn verify_completed_eval_session(root: &Path) { let session_dir = root.join("data/smith/sessions/run"); let paths: Vec<_> = std::fs::read_dir(session_dir) .unwrap() .map(|entry| entry.unwrap().path()) .filter(|path| path.extension().is_some_and(|extension| extension == "smh")) .collect(); assert_eq!(paths.len(), 1); let reader = SessionReader::open(&paths[0]).unwrap(); let frame_bytes_max = smith::config::Config::default_valid().frame_bytes_max; let mut admissions = Vec::new(); reader .scan(frame_bytes_max, |frame| { if let Frame::Admission { entry, .. } = frame { admissions.push(entry.id); } Ok(()) }) .unwrap(); assert_eq!( admissions.len(), 1, "eval must durably admit one submission" ); assert_eq!( reader.work_status(admissions[0], frame_bytes_max).unwrap(), Some(WorkStatus::Terminal(Some( smith::stream::StopReason::EndTurn ))) ); assert!( reader .work_output(admissions[0], frame_bytes_max) .unwrap() .is_some() ); reader.verify_full(frame_bytes_max).unwrap(); } #[test] fn closed_stdout_cancels_catalog_eval_auth_and_plugin_output_after_durable_work() { let root = Scratch::new("leaf-commands"); let package = root.join("plugin"); for arguments in [ vec!["catalog", "show"], vec!["eval", "--mock", "--json", "hello"], vec!["auth", "add", "--provider", "openai"], vec![ "plugin", "init", package.to_str().unwrap(), "--name", "pipe", ], ] { let (exit, stderr) = closed_stdout(&root, &arguments); assert_eq!(exit, 0, "{arguments:?}: {stderr}"); assert!(!stderr.contains("panicked"), "{arguments:?}: {stderr}"); } verify_completed_eval_session(&root); assert!(root.join("data/smith/auth/openai").is_file()); assert!(package.join("plugin.kdl").is_file()); } #[test] fn closed_stdout_auth_list_cancels_cleanly() { let root = Scratch::new("auth-list"); let auth = root.join("data/smith/auth"); std::fs::create_dir_all(&auth).unwrap(); std::fs::write(auth.join("google"), "credential").unwrap(); let (reader, writer) = std::io::pipe().unwrap(); drop(reader); let mut command = smith(&root, &["auth", "list"]); command.stdout(Stdio::from(writer)).stderr(Stdio::piped()); let mut command = AssertCommand::from_std(command); command.timeout(CHILD_DEADLINE); let output = command.output().unwrap(); assert_eq!(output.status.code(), Some(0)); assert!(!String::from_utf8_lossy(&output.stderr).contains("panicked")); } #[test] fn closed_stderr_preserves_session_inspection_exit_status() { let root = Scratch::new("stderr-inspection"); let arguments = [ "sessions", "get", "babab-babab-babab-babab-babab-babab-babab-babab", ]; assert_eq!(closed_stderr(&root, &arguments), 2); let mut command = AssertCommand::from_std(smith(&root, &arguments)); command.timeout(CHILD_DEADLINE); let output = command.output().unwrap(); assert_eq!(output.status.code(), Some(2)); assert!(String::from_utf8_lossy(&output.stderr).starts_with("smith: ")); } #[test] fn closed_stderr_preserves_non_inspection_failure_exit_status() { let root = Scratch::new("stderr-execution"); assert_eq!(closed_stderr(&root, &["eval", "hello"]), 1); }