Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

xtask/tests/e2e_guest_build.rs

Raw
//! Guest build respects target-generic CPU defaults (`SMH-SPEC-WASM0001`, `SMH-SPEC-SPEC0001`).

use std::{
    fs,
    os::unix::fs::{PermissionsExt, symlink},
    path::{Path, PathBuf},
    sync::atomic::{AtomicUsize, Ordering},
    time::Duration,
};

static SCRATCH_COUNTER: AtomicUsize = AtomicUsize::new(0);

#[test]
fn e2e_guest_build_ignores_inherited_native_cpu_flags_without_changing_lockfile() {
    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
    let lockfile = root.join("Cargo.lock");
    let lock_before = fs::read(&lockfile).unwrap();
    let scratch = Scratch::new("dependency-guest-build");

    let output = assert_cmd::Command::new("cargo")
        .current_dir(&root)
        .env_remove("RUSTFLAGS")
        .env_remove("CARGO_ENCODED_RUSTFLAGS")
        .args([
            "build",
            "--locked",
            "--offline",
            "--target",
            "wasm32-wasip2",
            "-p",
            "smith-plugin-echo",
            "--config",
            "build.rustflags=[\"-C\",\"target-cpu=native\"]",
            "--target-dir",
        ])
        .arg(scratch.path().join("target"))
        .timeout(Duration::from_secs(120))
        .output()
        .unwrap();
    let stderr = String::from_utf8_lossy(&output.stderr);
    assert!(output.status.success(), "cargo build failed:\n{stderr}");
    assert!(
        !stderr.contains("not a recognized processor")
            && !stderr.contains("not a recognized feature"),
        "guest build inherited native CPU flags:\n{stderr}"
    );
    assert_eq!(fs::read(lockfile).unwrap(), lock_before);
}

#[test]
fn e2e_guest_component_bytes_ignore_workspace_and_cargo_home_paths() {
    let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
    let scratch = Scratch::new("guest-remap");
    let copied = scratch.path().join("workspace with ünicode");
    copy_workspace(&source, &copied);
    let source_component = build_openai_component(&source, scratch.path().join("source"), "fat");
    let copied_component = build_openai_component(&copied, scratch.path().join("copied"), "thin");
    let source = fs::read(source_component).unwrap();
    let copied = fs::read(copied_component).unwrap();
    let difference = first_difference(&source, &copied);
    assert!(
        difference.is_none(),
        "canonical guest recipe retained workspace or Cargo-home paths; source bytes={}, copied bytes={}, first mismatch at {difference:?}, source window={:?}, copied window={:?}",
        source.len(),
        copied.len(),
        difference.map(|at| difference_window(&source, at)),
        difference.map(|at| difference_window(&copied, at)),
    );
}

#[test]
fn e2e_bench_builds_each_guest_before_the_host_and_stops_on_guest_failure() {
    let test = FakeCargo::new();

    test.xtask("bench", &["--", "--sample-size", "1"])
        .assert()
        .success();
    assert_eq!(
        test.log(),
        [
            "build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-openai-compatible --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-openai-responses --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-anthropic --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-google --target wasm32-wasip2 --release",
            "bench --locked -p smith-bench --bench wall_clock -- --sample-size 1",
        ]
    );

    test.clear_log();
    let output = test
        .xtask("bench", &["--", "--sample-size", "1"])
        .env("SMITH_FAIL_GUEST", "smith-plugin-coding-tools")
        .output()
        .unwrap();
    assert!(!output.status.success());
    assert_eq!(
        test.log(),
        [
            "build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
        ]
    );
}

#[test]
fn e2e_dev_builds_guests_before_host_without_running_plugin_tests() {
    let test = FakeCargo::new();

    test.xtask("dev", &["eval", "--provider", "zai", "hi"])
        .assert()
        .success();

    assert_eq!(
        test.log(),
        [
            "build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-openai-compatible --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-openai-responses --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-anthropic --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-google --target wasm32-wasip2 --release",
            "build --locked -p smith-cli --bin smith",
        ]
    );
}

#[test]
fn e2e_perf_prepares_every_guest_before_the_release_host_build() {
    let test = FakeCargo::new();
    let output = test.xtask("perf", &[]).output().unwrap();

    assert!(!output.status.success());
    let log = test.log();
    assert!(
        log.first()
            .is_some_and(|line| line.starts_with("binstall --no-confirm gungraun-runner@")),
        "perf did not mock cargo binstall: {log:?}"
    );
    assert_eq!(
        &log[1..],
        [
            "build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-openai-compatible --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-openai-responses --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-anthropic --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-protocol-google --target wasm32-wasip2 --release",
            "build --locked --release -p smith-cli --bin smith",
        ]
    );

    test.clear_log();
    let output = test
        .xtask("perf", &[])
        .env("SMITH_FAIL_GUEST", "smith-plugin-coding-tools")
        .output()
        .unwrap();
    assert!(!output.status.success());
    let log = test.log();
    assert_eq!(
        &log[1..],
        [
            "build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
            "build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
        ]
    );
}

fn build_openai_component(
    workspace: &std::path::Path,
    root: PathBuf,
    ambient_lto: &str,
) -> PathBuf {
    let bin = root.join("bin");
    let cargo_home = root.join("cargo home ünicode");
    let target = root.join("target");
    fs::create_dir_all(&bin).unwrap();
    fs::create_dir_all(&cargo_home).unwrap();
    let real_home = real_cargo_home();
    symlink(real_home.join("registry"), cargo_home.join("registry")).unwrap();
    if real_home.join("git").exists() {
        symlink(real_home.join("git"), cargo_home.join("git")).unwrap();
    }
    let cargo = bin.join("cargo");
    fs::write(
        &cargo,
        "#!/bin/sh\ncase \" $* \" in *\" -p smith-plugin-protocol-openai-compatible \"*) exec \"$SMITH_REAL_CARGO\" \"$@\" --offline;; esac\nexit 0\n",
    )
    .unwrap();
    fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap();
    for (program, variable, executable) in [
        ("rustc", "SMITH_REAL_RUSTC", host_rustc()),
        ("cc", "SMITH_REAL_CC", host_cc()),
    ] {
        let shim = bin.join(program);
        fs::write(&shim, format!("#!/bin/sh\nexec \"${variable}\" \"$@\"\n")).unwrap();
        fs::set_permissions(&shim, fs::Permissions::from_mode(0o755)).unwrap();
        assert!(
            executable.is_file(),
            "missing host {program}: {}",
            executable.display()
        );
    }
    let status = assert_cmd::Command::new(env!("CARGO_BIN_EXE_xtask"))
        .current_dir(workspace)
        .env("PATH", &bin)
        .env_remove("RUSTC_WRAPPER")
        .env_remove("SCCACHE")
        .env(
            "SMITH_REAL_CARGO",
            std::env::var_os("CARGO").expect("Cargo supplies CARGO to integration tests"),
        )
        .env("SMITH_REAL_RUSTC", host_rustc())
        .env("SMITH_REAL_CC", host_cc())
        .env("CARGO_HOME", &cargo_home)
        .env("CARGO_TARGET_DIR", &target)
        .env("CARGO_PROFILE_RELEASE_LTO", ambient_lto)
        .env("HOME", root.join("home"))
        .env(
            "RUSTUP_HOME",
            std::env::var_os("RUSTUP_HOME").unwrap_or_else(|| {
                PathBuf::from(std::env::var_os("HOME").expect("HOME is set for integration tests"))
                    .join(".rustup")
                    .into_os_string()
            }),
        )
        .args(["bench", "--", "--help"])
        .timeout(Duration::from_secs(120))
        .output()
        .unwrap();
    assert!(
        status.status.success(),
        "guest preflight failed:\n{}",
        String::from_utf8_lossy(&status.stderr)
    );
    target.join("wasm32-wasip2/release/smith_plugin_protocol_openai_compatible.wasm")
}

fn real_cargo_home() -> PathBuf {
    let cargo_home = std::env::var_os("CARGO_HOME")
        .map(PathBuf::from)
        .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".cargo")))
        .unwrap_or_else(|| PathBuf::from(".cargo"));
    if cargo_home.is_absolute() {
        cargo_home
    } else {
        std::env::current_dir().unwrap().join(cargo_home)
    }
}

fn host_cc() -> PathBuf {
    let cc = PathBuf::from("/usr/bin/cc");
    assert!(cc.is_file(), "missing host C compiler: {}", cc.display());
    cc
}

fn host_rustc() -> PathBuf {
    let cargo = PathBuf::from(
        std::env::var_os("CARGO").expect("Cargo supplies CARGO to integration tests"),
    );
    let bin = cargo
        .parent()
        .expect("Cargo executable has a parent directory");
    let rustc = bin.join("rustc");
    if rustc.is_file() {
        return rustc;
    }
    let rustup = bin.join("rustup");
    let output = std::process::Command::new(&rustup)
        .args(["which", "rustc"])
        .output()
        .unwrap();
    assert!(
        output.status.success(),
        "{} could not locate rustc",
        rustup.display()
    );
    PathBuf::from(String::from_utf8(output.stdout).unwrap().trim())
}

const COPY_PATHS: &[&str] = &[
    ".cargo",
    "Cargo.lock",
    "Cargo.toml",
    "smith/Cargo.toml",
    "smith/src/lib.rs",
    "smith-alloc/Cargo.toml",
    "smith-alloc/src/lib.rs",
    "smith-ai/Cargo.toml",
    "smith-ai/src/lib.rs",
    "smith-bench/Cargo.toml",
    "smith-bench/src/lib.rs",
    "smith-bench/benches",
    "smith-cli/Cargo.toml",
    "smith-cli/src/main.rs",
    "smith-core/Cargo.toml",
    "smith-core/src/lib.rs",
    "smith-harness/Cargo.toml",
    "smith-harness/src/lib.rs",
    "smith-rpc/Cargo.toml",
    "smith-rpc/src/lib.rs",
    "smith-tui/Cargo.toml",
    "smith-tui/src/lib.rs",
    "smith-ui/Cargo.toml",
    "smith-ui/src/lib.rs",
    "xtask/Cargo.toml",
    "xtask/src/main.rs",
    "plugins/oauth/Cargo.toml",
    "plugins/echo/Cargo.toml",
    "plugins/coding-tools/Cargo.toml",
    "plugins/protocol-anthropic/Cargo.toml",
    "plugins/protocol-google/Cargo.toml",
    "plugins/protocol-openai-compatible",
    "plugins/protocol-openai-responses",
    "rust-toolchain.toml",
    "wit",
];
const COPY_MAX_DEPTH: usize = 8;
const COPY_MAX_FILES: usize = 128;
const COPY_MAX_BYTES: u64 = 1024 * 1024;

fn copy_workspace(source: &Path, destination: &Path) {
    fs::create_dir_all(destination).unwrap();
    let mut copied = CopyBounds::default();
    for path in COPY_PATHS {
        copy_source(
            &source.join(path),
            &destination.join(path),
            COPY_MAX_DEPTH,
            &mut copied,
        )
        .unwrap();
    }
}

#[derive(Default)]
struct CopyBounds {
    files: usize,
    bytes: u64,
}

fn copy_source(
    source: &Path,
    destination: &Path,
    remaining_depth: usize,
    copied: &mut CopyBounds,
) -> Result<(), String> {
    let metadata = fs::symlink_metadata(source)
        .map_err(|error| format!("cannot inspect {}: {error}", source.display()))?;
    if metadata.file_type().is_symlink() {
        return Err(format!("refusing symlink {}", source.display()));
    }
    if metadata.is_file() {
        copied.files = copied
            .files
            .checked_add(1)
            .ok_or_else(|| "copied file count overflow".to_string())?;
        if copied.files > COPY_MAX_FILES {
            return Err(format!("copied file limit {COPY_MAX_FILES} exceeded"));
        }
        copied.bytes = copied
            .bytes
            .checked_add(metadata.len())
            .ok_or_else(|| "copied byte count overflow".to_string())?;
        if copied.bytes > COPY_MAX_BYTES {
            return Err(format!("copied byte limit {COPY_MAX_BYTES} exceeded"));
        }
        let parent = destination
            .parent()
            .ok_or_else(|| format!("copy destination has no parent: {}", destination.display()))?;
        fs::create_dir_all(parent)
            .map_err(|error| format!("cannot create {}: {error}", parent.display()))?;
        fs::copy(source, destination)
            .map_err(|error| format!("cannot copy {}: {error}", source.display()))?;
        return Ok(());
    }
    if !metadata.is_dir() {
        return Err(format!("refusing non-file source {}", source.display()));
    }
    if remaining_depth == 0 {
        return Err(format!("copy depth limit {COPY_MAX_DEPTH} exceeded"));
    }
    fs::create_dir_all(destination)
        .map_err(|error| format!("cannot create {}: {error}", destination.display()))?;
    for entry in fs::read_dir(source)
        .map_err(|error| format!("cannot read {}: {error}", source.display()))?
    {
        let entry = entry.map_err(|error| format!("cannot read copy entry: {error}"))?;
        copy_source(
            &entry.path(),
            &destination.join(entry.file_name()),
            remaining_depth - 1,
            copied,
        )?;
    }
    Ok(())
}

fn first_difference(left: &[u8], right: &[u8]) -> Option<usize> {
    left.iter()
        .zip(right)
        .position(|(left, right)| left != right)
        .or_else(|| (left.len() != right.len()).then_some(left.len().min(right.len())))
}

fn difference_window(bytes: &[u8], at: usize) -> &[u8] {
    const WINDOW_BYTES: usize = 32;
    let start = at.saturating_sub(WINDOW_BYTES / 2);
    &bytes[start..bytes.len().min(start.saturating_add(WINDOW_BYTES))]
}

struct FakeCargo {
    scratch: Scratch,
    bin: PathBuf,
    log: PathBuf,
}

impl FakeCargo {
    fn new() -> Self {
        let scratch = Scratch::new("xtask-guests");
        let bin = scratch.path().join("bin");
        let log = scratch.path().join("cargo.log");
        fs::create_dir_all(&bin).unwrap();
        fs::create_dir_all(scratch.path().join("cargo-home")).unwrap();
        let release = scratch.path().join("target/wasm32-wasip2/release");
        fs::create_dir_all(&release).unwrap();
        for role in [
            "openai_compatible",
            "openai_responses",
            "anthropic",
            "google",
        ] {
            fs::write(
                release.join(format!("smith_plugin_protocol_{role}.wasm")),
                b"fake build output",
            )
            .unwrap();
        }

        let tools = scratch.path().join("target/tools");
        let jco = tools.join("jco/node_modules/.bin/jco");
        fs::create_dir_all(jco.parent().unwrap()).unwrap();
        fs::write(&jco, "#!/bin/sh\nprintf '1.32.1\\n'\n").unwrap();
        fs::set_permissions(&jco, fs::Permissions::from_mode(0o755)).unwrap();
        let wasm_tools = tools.join("bin/wasm-tools");
        fs::create_dir_all(wasm_tools.parent().unwrap()).unwrap();
        fs::write(&wasm_tools, "#!/bin/sh\nprintf '1.258.0\\n'\n").unwrap();
        fs::set_permissions(&wasm_tools, fs::Permissions::from_mode(0o755)).unwrap();
        let node = bin.join("node");
        fs::write(&node, "#!/bin/sh\nprintf '1.32.1\\n'\n").unwrap();
        fs::set_permissions(&node, fs::Permissions::from_mode(0o755)).unwrap();
        let cargo = bin.join("cargo");
        fs::write(
            &cargo,
            "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$SMITH_XTASK_LOG\"\n[ -n \"${SMITH_FAIL_GUEST:-}\" ] && [ \"$SMITH_FAIL_GUEST\" = \"$4\" ] && exit 17\ncase \" $* \" in *\" build --locked --release -p smith-cli --bin smith \"*) exit 19;; *\" build --locked -p smith-cli --bin smith \"*) /usr/bin/mkdir -p \"$CARGO_TARGET_DIR/debug\"; printf '#!/bin/sh\\nexit 0\\n' > \"$CARGO_TARGET_DIR/debug/smith\"; /usr/bin/chmod +x \"$CARGO_TARGET_DIR/debug/smith\";; esac\nexit 0\n",
        )
        .unwrap();
        fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap();
        Self { scratch, bin, log }
    }

    fn xtask(&self, command: &str, arguments: &[&str]) -> assert_cmd::Command {
        let mut process = assert_cmd::Command::new(env!("CARGO_BIN_EXE_xtask"));
        process
            .current_dir(self.scratch.path())
            .env("PATH", &self.bin)
            .env("HOME", self.scratch.path().join("home"))
            .env("XDG_CONFIG_HOME", self.scratch.path().join("config"))
            .env("XDG_CACHE_HOME", self.scratch.path().join("cache"))
            .env("XDG_DATA_HOME", self.scratch.path().join("data"))
            .env("XDG_STATE_HOME", self.scratch.path().join("state"))
            .env("CARGO_HOME", self.scratch.path().join("cargo-home"))
            .env("CARGO_TARGET_DIR", self.scratch.path().join("target"))
            .env("SMITH_XTASK_LOG", &self.log)
            .args([command])
            .args(arguments)
            .timeout(Duration::from_secs(10));
        process
    }

    fn clear_log(&self) {
        fs::write(&self.log, "").unwrap();
    }

    fn log(&self) -> Vec<String> {
        fs::read_to_string(&self.log)
            .unwrap()
            .lines()
            .map(str::to_owned)
            .collect()
    }
}

struct Scratch(PathBuf);

impl Scratch {
    fn new(name: &str) -> Self {
        let path = std::env::temp_dir().join(format!(
            "smith-{name}-{}-{}",
            std::process::id(),
            SCRATCH_COUNTER.fetch_add(1, Ordering::Relaxed),
        ));
        fs::create_dir_all(&path).unwrap();
        Self(path)
    }

    fn path(&self) -> &std::path::Path {
        &self.0
    }
}

impl Drop for Scratch {
    fn drop(&mut self) {
        fs::remove_dir_all(&self.0).unwrap();
    }
}