repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
xtask/tests/e2e_dev.rs
Raw#![cfg(unix)]
//! Automatic development links dispatch through real binaries (`SMH-SPEC-WASM0001`).
use std::{
env, fs,
io::{Read, Write},
os::unix::process::CommandExt,
os::unix::{
ffi::OsStrExt,
fs::{PermissionsExt, symlink},
},
path::{Path, PathBuf},
process::{Command, Output, Stdio},
sync::{
atomic::{AtomicUsize, Ordering},
mpsc,
},
thread,
time::Duration,
};
const DEADLINE: Duration = Duration::from_secs(30);
const SECRET: &str = "dev-loop-sentinel-key";
static SCRATCH_COUNTER: AtomicUsize = AtomicUsize::new(0);
#[test]
fn e2e_dev_eval_links_protocol_and_records_dev_provenance() {
let scratch = Scratch::new();
let workspace_target = workspace_target();
let source_smith = workspace_target.join("debug").join("smith");
assert!(
source_smith.is_file(),
"missing built smith: {}",
source_smith.display()
);
let target = scratch.path().join("target");
let release = target.join("wasm32-wasip2/release");
fs::create_dir_all(&release).unwrap();
for source in protocol_components(&workspace_target) {
fs::copy(&source, release.join(source.file_name().unwrap())).unwrap();
}
fs::create_dir_all(target.join("debug")).unwrap();
symlink(&source_smith, target.join("debug/smith")).unwrap();
let cargo = scratch.path().join("bin/cargo");
fs::create_dir_all(cargo.parent().unwrap()).unwrap();
fs::write(&cargo, "#!/bin/sh\nexit 0\n").unwrap();
fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let (request_sent, request_received) = mpsc::sync_channel(0);
let server = thread::spawn(move || serve_once(listener, request_sent));
let profile = scratch.path().join("real-profile-like-home");
let profile_config = profile.join(".config/smith/sentinel");
fs::create_dir_all(&profile_config).unwrap();
fs::write(profile_config.join("unchanged"), b"real profile sentinel").unwrap();
let profile_before = fs::read(profile.join(".config/smith/sentinel/unchanged")).unwrap();
let base_url = format!("http://127.0.0.1:{port}/v1");
let mut command = Command::new(env!("CARGO_BIN_EXE_xtask"));
command
.current_dir(workspace_root())
.env("PATH", cargo.parent().unwrap())
.env("CARGO_TARGET_DIR", &target)
.env("HOME", &profile)
.env("OPENAI_API_KEY", SECRET)
.env_remove("SMITH_PROVIDER")
.env_remove("SMITH_MODEL")
.env_remove("SMITH_BASE_URL")
.args([
"dev",
"eval",
"--provider",
"openai",
"--base-url",
&base_url,
"--model",
"gpt-4o-mini",
"hello",
]);
let output = output_before_deadline(command);
assert!(
output.status.success(),
"xtask dev eval failed:\n{}",
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(output.stdout, b"dev-linked\n");
request_received.recv_timeout(DEADLINE).unwrap();
server.join().unwrap();
let dev_home = target.join("dev-home");
let sessions = dev_home.join("data/smith/sessions/run");
let entries: Vec<PathBuf> = fs::read_dir(&sessions)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension().is_some_and(|extension| extension == "smh"))
.collect();
assert_eq!(entries.len(), 1, "session files: {entries:?}");
let bytes = fs::read(&entries[0]).unwrap();
assert!(
bytes
.windows(b"provider-dispatch-provenance".len())
.any(|part| part == b"provider-dispatch-provenance")
);
assert!(
bytes
.windows(b"dev:protocol-openai-compatible".len())
.any(|part| part == b"dev:protocol-openai-compatible")
);
assert!(
!bytes
.windows(SECRET.len())
.any(|part| part == SECRET.as_bytes())
);
let registry = fs::read_to_string(dev_home.join("config/smith/plugins/registry.kdl")).unwrap();
let package = dev_home.join("packages/protocol-openai-compatible");
assert!(registry.contains("dev:protocol-openai-compatible"));
assert!(
registry.contains(&native_path_identity(&package)),
"registry: {registry}"
);
assert!(package.is_absolute());
assert_eq!(
fs::read(profile.join(".config/smith/sentinel/unchanged")).unwrap(),
profile_before
);
assert!(
!profile.join(".config/smith/plugins/registry.kdl").exists(),
"dev registry escaped isolated dev-home"
);
}
fn workspace_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}
fn workspace_target() -> PathBuf {
let executable = env::current_exe().unwrap();
executable
.parent()
.and_then(Path::parent)
.and_then(Path::parent)
.map(Path::to_path_buf)
.filter(|target| target.join("debug/smith").is_file())
.unwrap_or_else(|| {
panic!(
"cannot derive workspace target from {}",
executable.display()
)
})
}
fn native_path_identity(path: &Path) -> String {
format!(
"unix:{}",
path.as_os_str()
.as_bytes()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>()
)
}
fn protocol_components(target: &Path) -> Vec<PathBuf> {
let release = target.join("wasm32-wasip2/release");
let mut components: Vec<PathBuf> = fs::read_dir(&release)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| {
path.file_name().is_some_and(|name| {
name.to_string_lossy().starts_with("smith_plugin_protocol_")
&& name.to_string_lossy().ends_with(".wasm")
})
})
.collect();
components.sort();
assert_eq!(
components.len(),
4,
"protocol artifacts in {}: {components:?}",
release.display()
);
components
}
fn serve_once(listener: std::net::TcpListener, ready: mpsc::SyncSender<()>) {
let (mut socket, _) = listener.accept().unwrap();
socket.set_read_timeout(Some(DEADLINE)).unwrap();
socket.set_write_timeout(Some(DEADLINE)).unwrap();
let mut request = Vec::new();
let mut buffer = [0_u8; 4096];
let header_end = loop {
let read = socket.read(&mut buffer).unwrap();
assert!(read > 0 && request.len() + read <= 1 << 20);
request.extend_from_slice(&buffer[..read]);
if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
break at + 4;
}
};
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
assert!(
headers
.to_ascii_lowercase()
.contains(&format!("authorization: bearer {SECRET}"))
);
let response = "data: {\"choices\":[{\"delta\":{\"content\":\"dev-linked\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
ready.send(()).unwrap();
}
fn output_before_deadline(mut command: Command) -> Output {
command
.process_group(0)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child = command.spawn().unwrap();
let process_group = child.id();
let (sent, received) = mpsc::sync_channel(1);
thread::spawn(move || {
let _ = sent.send(child.wait_with_output());
});
match received.recv_timeout(DEADLINE) {
Ok(output) => output.unwrap(),
Err(error) => {
let group = format!("-{process_group}");
let status = Command::new("/bin/kill")
.args(["-KILL", "--", &group])
.status()
.unwrap();
assert!(
status.success(),
"failed to kill xtask process group {process_group}"
);
panic!("xtask dev eval deadline elapsed: {error}");
}
}
}
struct Scratch(PathBuf);
impl Scratch {
fn new() -> Self {
let path = env::temp_dir().join(format!(
"smith-e2e-dev-{}-{}",
std::process::id(),
SCRATCH_COUNTER.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir_all(&path).unwrap();
Self(path)
}
fn path(&self) -> &Path {
&self.0
}
}
impl Drop for Scratch {
fn drop(&mut self) {
fs::remove_dir_all(&self.0).unwrap();
}
}