repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
xtask/tests/e2e_guest_build.rs
Raw//! Guest build respects target-generic CPU defaults (`SMH-SPEC-WASM0001`, `SMH-SPEC-SPEC0001`).
use std::{
fs,
os::unix::fs::{PermissionsExt, symlink},
path::{Path, PathBuf},
sync::atomic::{AtomicUsize, Ordering},
time::Duration,
};
static SCRATCH_COUNTER: AtomicUsize = AtomicUsize::new(0);
#[test]
fn e2e_guest_build_ignores_inherited_native_cpu_flags_without_changing_lockfile() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
let lockfile = root.join("Cargo.lock");
let lock_before = fs::read(&lockfile).unwrap();
let scratch = Scratch::new("dependency-guest-build");
let output = assert_cmd::Command::new("cargo")
.current_dir(&root)
.env_remove("RUSTFLAGS")
.env_remove("CARGO_ENCODED_RUSTFLAGS")
.args([
"build",
"--locked",
"--offline",
"--target",
"wasm32-wasip2",
"-p",
"smith-plugin-echo",
"--config",
"build.rustflags=[\"-C\",\"target-cpu=native\"]",
"--target-dir",
])
.arg(scratch.path().join("target"))
.timeout(Duration::from_secs(120))
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(output.status.success(), "cargo build failed:\n{stderr}");
assert!(
!stderr.contains("not a recognized processor")
&& !stderr.contains("not a recognized feature"),
"guest build inherited native CPU flags:\n{stderr}"
);
assert_eq!(fs::read(lockfile).unwrap(), lock_before);
}
#[test]
fn e2e_guest_component_bytes_ignore_workspace_and_cargo_home_paths() {
let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
let scratch = Scratch::new("guest-remap");
let copied = scratch.path().join("workspace with ünicode");
copy_workspace(&source, &copied);
let source_component = build_openai_component(&source, scratch.path().join("source"), "fat");
let copied_component = build_openai_component(&copied, scratch.path().join("copied"), "thin");
let source = fs::read(source_component).unwrap();
let copied = fs::read(copied_component).unwrap();
let difference = first_difference(&source, &copied);
assert!(
difference.is_none(),
"canonical guest recipe retained workspace or Cargo-home paths; source bytes={}, copied bytes={}, first mismatch at {difference:?}, source window={:?}, copied window={:?}",
source.len(),
copied.len(),
difference.map(|at| difference_window(&source, at)),
difference.map(|at| difference_window(&copied, at)),
);
}
#[test]
fn e2e_bench_builds_each_guest_before_the_host_and_stops_on_guest_failure() {
let test = FakeCargo::new();
test.xtask("bench", &["--", "--sample-size", "1"])
.assert()
.success();
assert_eq!(
test.log(),
[
"build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-openai-compatible --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-openai-responses --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-anthropic --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-google --target wasm32-wasip2 --release",
"bench --locked -p smith-bench --bench wall_clock -- --sample-size 1",
]
);
test.clear_log();
let output = test
.xtask("bench", &["--", "--sample-size", "1"])
.env("SMITH_FAIL_GUEST", "smith-plugin-coding-tools")
.output()
.unwrap();
assert!(!output.status.success());
assert_eq!(
test.log(),
[
"build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
]
);
}
#[test]
fn e2e_dev_builds_guests_before_host_without_running_plugin_tests() {
let test = FakeCargo::new();
test.xtask("dev", &["eval", "--provider", "zai", "hi"])
.assert()
.success();
assert_eq!(
test.log(),
[
"build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-openai-compatible --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-openai-responses --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-anthropic --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-google --target wasm32-wasip2 --release",
"build --locked -p smith-cli --bin smith",
]
);
}
#[test]
fn e2e_perf_prepares_every_guest_before_the_release_host_build() {
let test = FakeCargo::new();
let output = test.xtask("perf", &[]).output().unwrap();
assert!(!output.status.success());
let log = test.log();
assert!(
log.first()
.is_some_and(|line| line.starts_with("binstall --no-confirm gungraun-runner@")),
"perf did not mock cargo binstall: {log:?}"
);
assert_eq!(
&log[1..],
[
"build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-openai-compatible --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-openai-responses --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-anthropic --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-protocol-google --target wasm32-wasip2 --release",
"build --locked --release -p smith-cli --bin smith",
]
);
test.clear_log();
let output = test
.xtask("perf", &[])
.env("SMITH_FAIL_GUEST", "smith-plugin-coding-tools")
.output()
.unwrap();
assert!(!output.status.success());
let log = test.log();
assert_eq!(
&log[1..],
[
"build --locked -p smith-plugin-oauth --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-echo --target wasm32-wasip2 --release",
"build --locked -p smith-plugin-coding-tools --target wasm32-wasip2 --release",
]
);
}
fn build_openai_component(
workspace: &std::path::Path,
root: PathBuf,
ambient_lto: &str,
) -> PathBuf {
let bin = root.join("bin");
let cargo_home = root.join("cargo home ünicode");
let target = root.join("target");
fs::create_dir_all(&bin).unwrap();
fs::create_dir_all(&cargo_home).unwrap();
let real_home = real_cargo_home();
symlink(real_home.join("registry"), cargo_home.join("registry")).unwrap();
if real_home.join("git").exists() {
symlink(real_home.join("git"), cargo_home.join("git")).unwrap();
}
let cargo = bin.join("cargo");
fs::write(
&cargo,
"#!/bin/sh\ncase \" $* \" in *\" -p smith-plugin-protocol-openai-compatible \"*) exec \"$SMITH_REAL_CARGO\" \"$@\" --offline;; esac\nexit 0\n",
)
.unwrap();
fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap();
for (program, variable, executable) in [
("rustc", "SMITH_REAL_RUSTC", host_rustc()),
("cc", "SMITH_REAL_CC", host_cc()),
] {
let shim = bin.join(program);
fs::write(&shim, format!("#!/bin/sh\nexec \"${variable}\" \"$@\"\n")).unwrap();
fs::set_permissions(&shim, fs::Permissions::from_mode(0o755)).unwrap();
assert!(
executable.is_file(),
"missing host {program}: {}",
executable.display()
);
}
let status = assert_cmd::Command::new(env!("CARGO_BIN_EXE_xtask"))
.current_dir(workspace)
.env("PATH", &bin)
.env_remove("RUSTC_WRAPPER")
.env_remove("SCCACHE")
.env(
"SMITH_REAL_CARGO",
std::env::var_os("CARGO").expect("Cargo supplies CARGO to integration tests"),
)
.env("SMITH_REAL_RUSTC", host_rustc())
.env("SMITH_REAL_CC", host_cc())
.env("CARGO_HOME", &cargo_home)
.env("CARGO_TARGET_DIR", &target)
.env("CARGO_PROFILE_RELEASE_LTO", ambient_lto)
.env("HOME", root.join("home"))
.env(
"RUSTUP_HOME",
std::env::var_os("RUSTUP_HOME").unwrap_or_else(|| {
PathBuf::from(std::env::var_os("HOME").expect("HOME is set for integration tests"))
.join(".rustup")
.into_os_string()
}),
)
.args(["bench", "--", "--help"])
.timeout(Duration::from_secs(120))
.output()
.unwrap();
assert!(
status.status.success(),
"guest preflight failed:\n{}",
String::from_utf8_lossy(&status.stderr)
);
target.join("wasm32-wasip2/release/smith_plugin_protocol_openai_compatible.wasm")
}
fn real_cargo_home() -> PathBuf {
let cargo_home = std::env::var_os("CARGO_HOME")
.map(PathBuf::from)
.or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".cargo")))
.unwrap_or_else(|| PathBuf::from(".cargo"));
if cargo_home.is_absolute() {
cargo_home
} else {
std::env::current_dir().unwrap().join(cargo_home)
}
}
fn host_cc() -> PathBuf {
let cc = PathBuf::from("/usr/bin/cc");
assert!(cc.is_file(), "missing host C compiler: {}", cc.display());
cc
}
fn host_rustc() -> PathBuf {
let cargo = PathBuf::from(
std::env::var_os("CARGO").expect("Cargo supplies CARGO to integration tests"),
);
let bin = cargo
.parent()
.expect("Cargo executable has a parent directory");
let rustc = bin.join("rustc");
if rustc.is_file() {
return rustc;
}
let rustup = bin.join("rustup");
let output = std::process::Command::new(&rustup)
.args(["which", "rustc"])
.output()
.unwrap();
assert!(
output.status.success(),
"{} could not locate rustc",
rustup.display()
);
PathBuf::from(String::from_utf8(output.stdout).unwrap().trim())
}
const COPY_PATHS: &[&str] = &[
".cargo",
"Cargo.lock",
"Cargo.toml",
"smith/Cargo.toml",
"smith/src/lib.rs",
"smith-alloc/Cargo.toml",
"smith-alloc/src/lib.rs",
"smith-ai/Cargo.toml",
"smith-ai/src/lib.rs",
"smith-bench/Cargo.toml",
"smith-bench/src/lib.rs",
"smith-bench/benches",
"smith-cli/Cargo.toml",
"smith-cli/src/main.rs",
"smith-core/Cargo.toml",
"smith-core/src/lib.rs",
"smith-harness/Cargo.toml",
"smith-harness/src/lib.rs",
"smith-rpc/Cargo.toml",
"smith-rpc/src/lib.rs",
"smith-tui/Cargo.toml",
"smith-tui/src/lib.rs",
"smith-ui/Cargo.toml",
"smith-ui/src/lib.rs",
"xtask/Cargo.toml",
"xtask/src/main.rs",
"plugins/oauth/Cargo.toml",
"plugins/echo/Cargo.toml",
"plugins/coding-tools/Cargo.toml",
"plugins/protocol-anthropic/Cargo.toml",
"plugins/protocol-google/Cargo.toml",
"plugins/protocol-openai-compatible",
"plugins/protocol-openai-responses",
"rust-toolchain.toml",
"wit",
];
const COPY_MAX_DEPTH: usize = 8;
const COPY_MAX_FILES: usize = 128;
const COPY_MAX_BYTES: u64 = 1024 * 1024;
fn copy_workspace(source: &Path, destination: &Path) {
fs::create_dir_all(destination).unwrap();
let mut copied = CopyBounds::default();
for path in COPY_PATHS {
copy_source(
&source.join(path),
&destination.join(path),
COPY_MAX_DEPTH,
&mut copied,
)
.unwrap();
}
}
#[derive(Default)]
struct CopyBounds {
files: usize,
bytes: u64,
}
fn copy_source(
source: &Path,
destination: &Path,
remaining_depth: usize,
copied: &mut CopyBounds,
) -> Result<(), String> {
let metadata = fs::symlink_metadata(source)
.map_err(|error| format!("cannot inspect {}: {error}", source.display()))?;
if metadata.file_type().is_symlink() {
return Err(format!("refusing symlink {}", source.display()));
}
if metadata.is_file() {
copied.files = copied
.files
.checked_add(1)
.ok_or_else(|| "copied file count overflow".to_string())?;
if copied.files > COPY_MAX_FILES {
return Err(format!("copied file limit {COPY_MAX_FILES} exceeded"));
}
copied.bytes = copied
.bytes
.checked_add(metadata.len())
.ok_or_else(|| "copied byte count overflow".to_string())?;
if copied.bytes > COPY_MAX_BYTES {
return Err(format!("copied byte limit {COPY_MAX_BYTES} exceeded"));
}
let parent = destination
.parent()
.ok_or_else(|| format!("copy destination has no parent: {}", destination.display()))?;
fs::create_dir_all(parent)
.map_err(|error| format!("cannot create {}: {error}", parent.display()))?;
fs::copy(source, destination)
.map_err(|error| format!("cannot copy {}: {error}", source.display()))?;
return Ok(());
}
if !metadata.is_dir() {
return Err(format!("refusing non-file source {}", source.display()));
}
if remaining_depth == 0 {
return Err(format!("copy depth limit {COPY_MAX_DEPTH} exceeded"));
}
fs::create_dir_all(destination)
.map_err(|error| format!("cannot create {}: {error}", destination.display()))?;
for entry in fs::read_dir(source)
.map_err(|error| format!("cannot read {}: {error}", source.display()))?
{
let entry = entry.map_err(|error| format!("cannot read copy entry: {error}"))?;
copy_source(
&entry.path(),
&destination.join(entry.file_name()),
remaining_depth - 1,
copied,
)?;
}
Ok(())
}
fn first_difference(left: &[u8], right: &[u8]) -> Option<usize> {
left.iter()
.zip(right)
.position(|(left, right)| left != right)
.or_else(|| (left.len() != right.len()).then_some(left.len().min(right.len())))
}
fn difference_window(bytes: &[u8], at: usize) -> &[u8] {
const WINDOW_BYTES: usize = 32;
let start = at.saturating_sub(WINDOW_BYTES / 2);
&bytes[start..bytes.len().min(start.saturating_add(WINDOW_BYTES))]
}
struct FakeCargo {
scratch: Scratch,
bin: PathBuf,
log: PathBuf,
}
impl FakeCargo {
fn new() -> Self {
let scratch = Scratch::new("xtask-guests");
let bin = scratch.path().join("bin");
let log = scratch.path().join("cargo.log");
fs::create_dir_all(&bin).unwrap();
fs::create_dir_all(scratch.path().join("cargo-home")).unwrap();
let release = scratch.path().join("target/wasm32-wasip2/release");
fs::create_dir_all(&release).unwrap();
for role in [
"openai_compatible",
"openai_responses",
"anthropic",
"google",
] {
fs::write(
release.join(format!("smith_plugin_protocol_{role}.wasm")),
b"fake build output",
)
.unwrap();
}
let tools = scratch.path().join("target/tools");
let jco = tools.join("jco/node_modules/.bin/jco");
fs::create_dir_all(jco.parent().unwrap()).unwrap();
fs::write(&jco, "#!/bin/sh\nprintf '1.32.1\\n'\n").unwrap();
fs::set_permissions(&jco, fs::Permissions::from_mode(0o755)).unwrap();
let wasm_tools = tools.join("bin/wasm-tools");
fs::create_dir_all(wasm_tools.parent().unwrap()).unwrap();
fs::write(&wasm_tools, "#!/bin/sh\nprintf '1.258.0\\n'\n").unwrap();
fs::set_permissions(&wasm_tools, fs::Permissions::from_mode(0o755)).unwrap();
let node = bin.join("node");
fs::write(&node, "#!/bin/sh\nprintf '1.32.1\\n'\n").unwrap();
fs::set_permissions(&node, fs::Permissions::from_mode(0o755)).unwrap();
let cargo = bin.join("cargo");
fs::write(
&cargo,
"#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$SMITH_XTASK_LOG\"\n[ -n \"${SMITH_FAIL_GUEST:-}\" ] && [ \"$SMITH_FAIL_GUEST\" = \"$4\" ] && exit 17\ncase \" $* \" in *\" build --locked --release -p smith-cli --bin smith \"*) exit 19;; *\" build --locked -p smith-cli --bin smith \"*) /usr/bin/mkdir -p \"$CARGO_TARGET_DIR/debug\"; printf '#!/bin/sh\\nexit 0\\n' > \"$CARGO_TARGET_DIR/debug/smith\"; /usr/bin/chmod +x \"$CARGO_TARGET_DIR/debug/smith\";; esac\nexit 0\n",
)
.unwrap();
fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap();
Self { scratch, bin, log }
}
fn xtask(&self, command: &str, arguments: &[&str]) -> assert_cmd::Command {
let mut process = assert_cmd::Command::new(env!("CARGO_BIN_EXE_xtask"));
process
.current_dir(self.scratch.path())
.env("PATH", &self.bin)
.env("HOME", self.scratch.path().join("home"))
.env("XDG_CONFIG_HOME", self.scratch.path().join("config"))
.env("XDG_CACHE_HOME", self.scratch.path().join("cache"))
.env("XDG_DATA_HOME", self.scratch.path().join("data"))
.env("XDG_STATE_HOME", self.scratch.path().join("state"))
.env("CARGO_HOME", self.scratch.path().join("cargo-home"))
.env("CARGO_TARGET_DIR", self.scratch.path().join("target"))
.env("SMITH_XTASK_LOG", &self.log)
.args([command])
.args(arguments)
.timeout(Duration::from_secs(10));
process
}
fn clear_log(&self) {
fs::write(&self.log, "").unwrap();
}
fn log(&self) -> Vec<String> {
fs::read_to_string(&self.log)
.unwrap()
.lines()
.map(str::to_owned)
.collect()
}
}
struct Scratch(PathBuf);
impl Scratch {
fn new(name: &str) -> Self {
let path = std::env::temp_dir().join(format!(
"smith-{name}-{}-{}",
std::process::id(),
SCRATCH_COUNTER.fetch_add(1, Ordering::Relaxed),
));
fs::create_dir_all(&path).unwrap();
Self(path)
}
fn path(&self) -> &std::path::Path {
&self.0
}
}
impl Drop for Scratch {
fn drop(&mut self) {
fs::remove_dir_all(&self.0).unwrap();
}
}