Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

xtask/tests/e2e_dev.rs

Raw
#![cfg(unix)]

//! Automatic development links dispatch through real binaries (`SMH-SPEC-WASM0001`).

use std::{
    env, fs,
    io::{Read, Write},
    os::unix::process::CommandExt,
    os::unix::{
        ffi::OsStrExt,
        fs::{PermissionsExt, symlink},
    },
    path::{Path, PathBuf},
    process::{Command, Output, Stdio},
    sync::{
        atomic::{AtomicUsize, Ordering},
        mpsc,
    },
    thread,
    time::Duration,
};

const DEADLINE: Duration = Duration::from_secs(30);
const SECRET: &str = "dev-loop-sentinel-key";
static SCRATCH_COUNTER: AtomicUsize = AtomicUsize::new(0);

#[test]
fn e2e_dev_eval_links_protocol_and_records_dev_provenance() {
    let scratch = Scratch::new();
    let workspace_target = workspace_target();
    let source_smith = workspace_target.join("debug").join("smith");
    assert!(
        source_smith.is_file(),
        "missing built smith: {}",
        source_smith.display()
    );

    let target = scratch.path().join("target");
    let release = target.join("wasm32-wasip2/release");
    fs::create_dir_all(&release).unwrap();
    for source in protocol_components(&workspace_target) {
        fs::copy(&source, release.join(source.file_name().unwrap())).unwrap();
    }
    fs::create_dir_all(target.join("debug")).unwrap();
    symlink(&source_smith, target.join("debug/smith")).unwrap();
    let cargo = scratch.path().join("bin/cargo");
    fs::create_dir_all(cargo.parent().unwrap()).unwrap();
    fs::write(&cargo, "#!/bin/sh\nexit 0\n").unwrap();
    fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap();

    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let port = listener.local_addr().unwrap().port();
    let (request_sent, request_received) = mpsc::sync_channel(0);
    let server = thread::spawn(move || serve_once(listener, request_sent));

    let profile = scratch.path().join("real-profile-like-home");
    let profile_config = profile.join(".config/smith/sentinel");
    fs::create_dir_all(&profile_config).unwrap();
    fs::write(profile_config.join("unchanged"), b"real profile sentinel").unwrap();
    let profile_before = fs::read(profile.join(".config/smith/sentinel/unchanged")).unwrap();
    let base_url = format!("http://127.0.0.1:{port}/v1");
    let mut command = Command::new(env!("CARGO_BIN_EXE_xtask"));
    command
        .current_dir(workspace_root())
        .env("PATH", cargo.parent().unwrap())
        .env("CARGO_TARGET_DIR", &target)
        .env("HOME", &profile)
        .env("OPENAI_API_KEY", SECRET)
        .env_remove("SMITH_PROVIDER")
        .env_remove("SMITH_MODEL")
        .env_remove("SMITH_BASE_URL")
        .args([
            "dev",
            "eval",
            "--provider",
            "openai",
            "--base-url",
            &base_url,
            "--model",
            "gpt-4o-mini",
            "hello",
        ]);
    let output = output_before_deadline(command);
    assert!(
        output.status.success(),
        "xtask dev eval failed:\n{}",
        String::from_utf8_lossy(&output.stderr)
    );
    assert_eq!(output.stdout, b"dev-linked\n");
    request_received.recv_timeout(DEADLINE).unwrap();
    server.join().unwrap();

    let dev_home = target.join("dev-home");
    let sessions = dev_home.join("data/smith/sessions/run");
    let entries: Vec<PathBuf> = fs::read_dir(&sessions)
        .unwrap()
        .map(|entry| entry.unwrap().path())
        .filter(|path| path.extension().is_some_and(|extension| extension == "smh"))
        .collect();
    assert_eq!(entries.len(), 1, "session files: {entries:?}");
    let bytes = fs::read(&entries[0]).unwrap();
    assert!(
        bytes
            .windows(b"provider-dispatch-provenance".len())
            .any(|part| part == b"provider-dispatch-provenance")
    );
    assert!(
        bytes
            .windows(b"dev:protocol-openai-compatible".len())
            .any(|part| part == b"dev:protocol-openai-compatible")
    );
    assert!(
        !bytes
            .windows(SECRET.len())
            .any(|part| part == SECRET.as_bytes())
    );
    let registry = fs::read_to_string(dev_home.join("config/smith/plugins/registry.kdl")).unwrap();
    let package = dev_home.join("packages/protocol-openai-compatible");
    assert!(registry.contains("dev:protocol-openai-compatible"));
    assert!(
        registry.contains(&native_path_identity(&package)),
        "registry: {registry}"
    );
    assert!(package.is_absolute());
    assert_eq!(
        fs::read(profile.join(".config/smith/sentinel/unchanged")).unwrap(),
        profile_before
    );
    assert!(
        !profile.join(".config/smith/plugins/registry.kdl").exists(),
        "dev registry escaped isolated dev-home"
    );
}

fn workspace_root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}

fn workspace_target() -> PathBuf {
    let executable = env::current_exe().unwrap();
    executable
        .parent()
        .and_then(Path::parent)
        .and_then(Path::parent)
        .map(Path::to_path_buf)
        .filter(|target| target.join("debug/smith").is_file())
        .unwrap_or_else(|| {
            panic!(
                "cannot derive workspace target from {}",
                executable.display()
            )
        })
}

fn native_path_identity(path: &Path) -> String {
    format!(
        "unix:{}",
        path.as_os_str()
            .as_bytes()
            .iter()
            .map(|byte| format!("{byte:02x}"))
            .collect::<String>()
    )
}

fn protocol_components(target: &Path) -> Vec<PathBuf> {
    let release = target.join("wasm32-wasip2/release");
    let mut components: Vec<PathBuf> = fs::read_dir(&release)
        .unwrap()
        .map(|entry| entry.unwrap().path())
        .filter(|path| {
            path.file_name().is_some_and(|name| {
                name.to_string_lossy().starts_with("smith_plugin_protocol_")
                    && name.to_string_lossy().ends_with(".wasm")
            })
        })
        .collect();
    components.sort();
    assert_eq!(
        components.len(),
        4,
        "protocol artifacts in {}: {components:?}",
        release.display()
    );
    components
}

fn serve_once(listener: std::net::TcpListener, ready: mpsc::SyncSender<()>) {
    let (mut socket, _) = listener.accept().unwrap();
    socket.set_read_timeout(Some(DEADLINE)).unwrap();
    socket.set_write_timeout(Some(DEADLINE)).unwrap();
    let mut request = Vec::new();
    let mut buffer = [0_u8; 4096];
    let header_end = loop {
        let read = socket.read(&mut buffer).unwrap();
        assert!(read > 0 && request.len() + read <= 1 << 20);
        request.extend_from_slice(&buffer[..read]);
        if let Some(at) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
            break at + 4;
        }
    };
    let headers = std::str::from_utf8(&request[..header_end]).unwrap();
    assert!(
        headers
            .to_ascii_lowercase()
            .contains(&format!("authorization: bearer {SECRET}"))
    );
    let response = "data: {\"choices\":[{\"delta\":{\"content\":\"dev-linked\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
    write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{response}", response.len()).unwrap();
    ready.send(()).unwrap();
}

fn output_before_deadline(mut command: Command) -> Output {
    command
        .process_group(0)
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let child = command.spawn().unwrap();
    let process_group = child.id();
    let (sent, received) = mpsc::sync_channel(1);
    thread::spawn(move || {
        let _ = sent.send(child.wait_with_output());
    });
    match received.recv_timeout(DEADLINE) {
        Ok(output) => output.unwrap(),
        Err(error) => {
            let group = format!("-{process_group}");
            let status = Command::new("/bin/kill")
                .args(["-KILL", "--", &group])
                .status()
                .unwrap();
            assert!(
                status.success(),
                "failed to kill xtask process group {process_group}"
            );
            panic!("xtask dev eval deadline elapsed: {error}");
        }
    }
}

struct Scratch(PathBuf);

impl Scratch {
    fn new() -> Self {
        let path = env::temp_dir().join(format!(
            "smith-e2e-dev-{}-{}",
            std::process::id(),
            SCRATCH_COUNTER.fetch_add(1, Ordering::Relaxed)
        ));
        fs::create_dir_all(&path).unwrap();
        Self(path)
    }

    fn path(&self) -> &Path {
        &self.0
    }
}

impl Drop for Scratch {
    fn drop(&mut self) {
        fs::remove_dir_all(&self.0).unwrap();
    }
}