#![forbid(unsafe_code)] #![warn(missing_docs)] //! Cargo-only Smith project automation (`SMH-SPEC-SPEC0001`, acceptance: //! quality, architecture, documentation gates). //! //! Gate tools follow one rule: install immutable artifacts or pinned //! versions with `--locked`, never live dependency resolution, so upstream //! releases cannot break a gate without a change in this repository. use std::{ env, ffi::OsString, fs, io::{BufRead, Write}, path::{Path, PathBuf}, process::Command, process::ExitCode, }; use std::collections::BTreeMap; const HELP: &str = concat!( "usage: cargo x \n", " cargo x perf [--pin] [args...] instruction budgets (callgrind), gated against smith-bench/budgets.toml\n", " cargo x bench [args...] wall-clock criterion benches; never gate\n", " cargo x dev [args...] build and launch the dev smith binary\n", ); /// Version of `cargo-nextest` used by the test gate. const NEXTEST_VERSION: &str = "0.9.146"; /// Version of `cargo-binstall` used to install `cargo-nextest`. const BINSTALL_VERSION: &str = "1.23.0"; const MACHETE_VERSION: &str = "0.9.2"; const DENY_VERSION: &str = "0.20.2"; const AUDIT_VERSION: &str = "0.22.2"; const MUTANTS_VERSION: &str = "27.1.0"; const LLVM_COV_VERSION: &str = "0.9.1"; /// Version of `gungraun-runner`; must match the `gungraun` crate in the lockfile. const GUNGRAUN_VERSION: &str = "0.19.4"; /// gungraun bench targets of `smith-bench`, gated by `cargo x perf`. const INSTRUCTION_BENCHES: &[&str] = &["instructions", "startup"]; /// criterion bench target of `smith-bench`, run by `cargo x bench`. const WALL_CLOCK_BENCH: &str = "wall_clock"; /// Instruction pins, one per gungraun bench. const BUDGETS: &str = "smith-bench/budgets.toml"; /// Allowed deviation of a measured `Ir` from its pin, either direction. const BAND_PERCENT: u64 = 2; /// Provider selection and credential variables that never reach a /// measured `smith` run. const PROVIDER_VARS: &[&str] = &[ "SMITH_PROVIDER", "SMITH_MODEL", "SMITH_BASE_URL", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", ]; const CRATES: &[&str] = &[ "smith", "smith-core", "smith-ai", "smith-ui", "smith-tui", "smith-rpc", "smith-harness", "smith-cli", "smith-alloc", "smith-bench", "xtask", ]; fn main() -> ExitCode { let mut args = env::args_os(); let _program = args.next(); let Some(command) = args.next() else { return fail(HELP); }; let rest: Vec = args.collect(); if command.to_str() == Some("dev") { return dev(&rest); } match command.to_str() { Some("check") => check(&rest), Some("lint") => lint(&rest), Some("fmt") => cargo_fmt_step(&rest), Some("test") => cargo_nextest(&rest), Some("e2e") => cargo_e2e(&rest), Some("coverage") => cargo_coverage(&rest), Some("report") => report(&rest), Some("doc") => cargo_doc(&rest), Some("arch") => arch(&rest), Some("mutants") => cargo_mutants(&rest), Some("perf") => perf(&rest), Some("bench") => bench(&rest), Some("audit") => audit(&rest), Some("ci") => ci(&rest), Some("release") => release(&rest), Some("bootstrap") => bootstrap(&rest), _ => fail(HELP), } } fn check(extra: &[OsString]) -> ExitCode { sequence( &[ cargo_fmt_step, cargo_clippy, arch, cargo_nextest, cargo_doc, cargo_machete, ], extra, ) } type Step = fn(&[OsString]) -> ExitCode; fn sequence(steps: &[Step], extra: &[OsString]) -> ExitCode { for step in steps { let code = step(extra); if code != ExitCode::SUCCESS { return code; } } ExitCode::SUCCESS } /// Append passthrough args; package scoping overrides workspace defaults, /// so `-p crate` drops the `--workspace` flag rather than conflicting. fn scoped(base: &[&str], extra: &[OsString]) -> Vec { let mut args: Vec = base.iter().map(OsString::from).collect(); let narrows = extra.iter().any(|arg| { arg == "-p" || arg == "--package" || arg.to_string_lossy().starts_with("--package=") }); if narrows { args.retain(|arg| arg != "--workspace"); } args.extend_from_slice(extra); args } fn lint(extra: &[OsString]) -> ExitCode { sequence(&[cargo_fmt_step, cargo_clippy], extra) } /// Audit gates read the lockfile; scoping does not apply. fn audit(extra: &[OsString]) -> ExitCode { if !extra.is_empty() { return fail("audit is lockfile-wide and admits no scoping\n"); } sequence( &[install_deny, install_audit, cargo_deny, cargo_audit], extra, ) } /// The merge-gate pipeline: every gate, machine context, fixed order. /// Passthrough is rejected so workflows cannot drift from the definition. fn ci(extra: &[OsString]) -> ExitCode { if !extra.is_empty() { return fail("ci is a fixed pipeline and admits no scoping\n"); } sequence(&[bootstrap, check, audit], extra) } /// Release gates are defined by the release-readiness plan; the command /// exists so workflows reference a stable name today. fn release(extra: &[OsString]) -> ExitCode { let _ = extra; fail("release gates are not defined yet; see SMH-PLAN-RELS0001\n") } /// Mutation testing runs on demand, never in the blocking check chain. /// /// Split of concerns, kept CI-sensible: /// - `.cargo/mutants.toml` owns tree properties (scope, tool, profiles) /// so bare `cargo mutants` behaves identically anywhere. /// - this step owns run context: disk-backed scratch (system tmpfs is /// 16G RAM and tree copies crawl the machine), a modest default job /// count overridable through `CARGO_MUTANTS_JOBS`, and a skipped /// baseline because `cargo x check` owns test verification. fn cargo_mutants(extra: &[OsString]) -> ExitCode { let installed = install_mutants(&[]); if installed != ExitCode::SUCCESS { return installed; } let scratch = target_dir().join("mutants-tmp"); if std::fs::create_dir_all(&scratch).is_err() { return fail("cannot create target/mutants-tmp\n"); } // `cargo x mutants -- -f x` and `cargo x mutants -f x` are equivalent, // so cargo-run habits cannot turn scope flags into stray nextest test // filters. let extra = without_separator(extra); let all = extra.iter().any(|arg| arg == "--all"); if !all && !mutants_scoped(extra) { return fail(MUTANTS_SCOPE_HELP); } let passthrough: Vec<&OsString> = extra.iter().filter(|arg| *arg != "--all").collect(); // Local runs are pinned to one job and a third of the cores for both // the build and the test phase: four jobs and a full fan-out crashed // this machine. Only `--all` (CI) reads `CARGO_MUTANTS_JOBS`. if !all && passthrough.iter().any(|arg| jobs_flag(arg)) { return fail("cargo x mutants runs one job locally; drop -j/--jobs\n"); } let jobs = if all { std::env::var("CARGO_MUTANTS_JOBS").unwrap_or_else(|_| "1".to_string()) } else { "1".to_string() }; let cores = std::thread::available_parallelism().map_or(2, |cores| (cores.get() / 3).max(2)); let mut command = Command::new("nice"); command .args(["-n", "19", "cargo-mutants"]) .env("TMPDIR", &scratch) .args([ "mutants", "--output", "target", "--jobs", &jobs, "--baseline", "skip", "--timeout", "120", // A mutant once hung rustc itself; builds are bounded too. "--build-timeout", "300", "--no-shuffle", ]); if !all { command .env("NEXTEST_TEST_THREADS", cores.to_string()) .env("CARGO_BUILD_JOBS", cores.to_string()); } // Passthrough args come last and override the defaults above. command.args(passthrough); match command.status() { Ok(status) if status.success() => ExitCode::SUCCESS, Ok(status) if status.code() == Some(2) => fail( "missed mutants: tests did not catch everything (see target/mutants.out/missed.txt)\n", ), Ok(status) => fail(&format!("cargo mutants exited with {status}\n")), Err(error) => fail(&format!("failed to run cargo mutants: {error}\n")), } } const MUTANTS_SCOPE_HELP: &str = "\ cargo x mutants needs a scope: a workspace-wide run copies the tree per job and \ has crashed this machine before. -f one file or directory, e.g. -f smith-core/src/session.rs -D mutants on changed lines only (jj diff --git > /tmp/d.patch) -p one crate --shard k/n one of n equal shards, e.g. --shard 0/8 -F/-E mutant-name filters --list inventory only, no build --all the whole workspace; CI only "; fn jobs_flag(arg: &OsString) -> bool { let arg = arg.to_string_lossy(); arg == "-j" || arg == "--jobs" || arg.starts_with("-j") && arg[2..].bytes().all(|b| b.is_ascii_digit()) || arg.starts_with("--jobs=") } /// Whether the passthrough arguments narrow a mutants run. fn mutants_scoped(extra: &[OsString]) -> bool { extra.iter().any(|arg| { let arg = arg.to_string_lossy(); matches!( arg.as_ref(), "-f" | "--file" | "-D" | "--in-diff" | "-p" | "--package" | "--shard" | "-F" | "--re" | "-E" | "--exclude-re" | "--list" ) || arg.starts_with("--file=") || arg.starts_with("--in-diff=") || arg.starts_with("--package=") || arg.starts_with("--shard=") || arg.starts_with("--re=") || arg.starts_with("--exclude-re=") }) } /// Instruction budgets: every gungraun bench's callgrind `Ir` against its /// pin in `smith-bench/budgets.toml`, within ±2 % in either direction, so /// a regression fails and an improvement fails until re-pinned. /// /// `--pin` rewrites the pins from this run instead of gating. Other /// arguments go to the gungraun runner; a filter narrows the run, and pins /// of benches it skipped are kept rather than reported stale. Every run /// writes `target/report/perf.md`. fn perf(extra: &[OsString]) -> ExitCode { let repin = extra.iter().any(|arg| arg == "--pin"); let passthrough: Vec = without_separator(extra) .iter() .filter(|arg| *arg != "--pin") .cloned() .collect(); let installed = install_gungraun_runner(&[]); if installed != ExitCode::SUCCESS { return installed; } let smith = match release_smith() { Ok(path) => path, Err(code) => return code, }; let measured = match instruction_counts(&smith, &passthrough) { Ok(measured) => measured, Err(message) => return fail(&message), }; let pins = match fs::read_to_string(BUDGETS) { Ok(text) => parse_budgets(&text), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(BTreeMap::new()), Err(error) => Err(format!("cannot read {BUDGETS}: {error}")), }; let pins = match pins { Ok(pins) => pins, Err(message) => return fail(&format!("{message}\n")), }; let complete = passthrough.is_empty(); let platform = platform(); let written = write_perf_report(&measured, &pins, complete, &platform); if written != ExitCode::SUCCESS { return written; } if repin { let mut pinned = if complete { BTreeMap::new() } else { pins }; pinned.extend(measured); if fs::write(BUDGETS, render_budgets(&pinned, &platform)).is_err() { return fail(&format!("cannot write {BUDGETS}\n")); } note(&format!( "perf: pinned {} benches in {BUDGETS}\n", pinned.len() )); return ExitCode::SUCCESS; } // Instruction counts belong to one toolchain and libc: the gate runs // where the pins were taken (the CI image); elsewhere the run reports. let pinned_on = fs::read_to_string(BUDGETS) .ok() .and_then(|text| pinned_platform(&text)); if pinned_on.as_deref() != Some(platform.as_str()) { note(&format!( "perf: pins belong to `{}`, this run is `{platform}`; counts reported in target/report/perf.md, gate skipped\n", pinned_on.as_deref().unwrap_or("no platform") )); return ExitCode::SUCCESS; } let violations = budget_violations(&measured, &pins, complete); if violations.is_empty() { note(&format!( "perf: {} benches within ±{BAND_PERCENT} % of their pins\n", measured.len() )); ExitCode::SUCCESS } else { fail(&format!( "instruction budgets violated (re-pin deliberately with `cargo x perf --pin`):\n{}", violations.concat() )) } } /// Wall clock: the criterion benches, for A/B comparisons of two builds /// on one machine in one session. Numbers are never committed and never /// gate. Arguments go to criterion, e.g. `--save-baseline `. fn bench(extra: &[OsString]) -> ExitCode { let mut args: Vec = [ "bench", "--locked", "-p", "smith-bench", "--bench", WALL_CLOCK_BENCH, "--", ] .map(OsString::from) .to_vec(); args.extend_from_slice(without_separator(extra)); run_os("cargo", &args) } /// Build the release `smith` binary and return its absolute path. fn release_smith() -> Result { let built = cargo(&[ "build", "--locked", "--release", "-p", "smith-cli", "--bin", "smith", ]); if built != ExitCode::SUCCESS { return Err(built); } let binary = target_dir().join("release").join(binary_name()); std::path::absolute(&binary) .map_err(|error| fail(&format!("cannot resolve {}: {error}\n", binary.display()))) } /// Run the gungraun benches against `smith` and collect every bench's /// `Ir` from the JSON summary lines on standard output. fn instruction_counts( smith: &Path, passthrough: &[OsString], ) -> Result, String> { let mut command = Command::new("cargo"); command.args(["bench", "--locked", "-p", "smith-bench"]); for bench in INSTRUCTION_BENCHES { command.args(["--bench", bench]); } command // ASLR stays on: containers refuse the personality switch, and // cache simulation is off, so instruction counts are unaffected. .args([ "--", "--save-summary=json", "--output-format=json", "--allow-aslr=true", ]) .args(passthrough) .env("SMITH_BIN", smith) .stderr(std::process::Stdio::inherit()); let output = command .output() .map_err(|error| format!("failed to run cargo bench: {error}\n"))?; if !output.status.success() { // Exit code 3 is gungraun's own regression verdict from limits // passed through; anything else is a build or runner failure. return Err(format!("cargo bench exited with {}\n", output.status)); } let stdout = String::from_utf8_lossy(&output.stdout); let mut measured = BTreeMap::new(); for line in stdout.lines().filter(|line| line.starts_with('{')) { let Some((bench, ir)) = bench_ir(line) else { return Err(format!("unrecognized gungraun summary line: {line}\n")); }; measured.insert(bench, ir); } if measured.is_empty() { return Err("gungraun reported no benches\n".to_string()); } Ok(measured) } /// `(::, Ir)` of one gungraun `--output-format=json` /// line (summary schema v6). `module_path` and `id` are top-level strings; /// the total `Ir` is the first integer under `total` → `Ir` → `metrics`, /// which reads `{"Left":{"Int":n}}` on a first run and /// `{"Both":[{"Int":n},{"Int":previous}]}` once a previous run exists. fn bench_ir(line: &str) -> Option<(String, u64)> { let module = json_string(line, "module_path")?; let id = json_string(line, "id")?; let after = |text: &'_ str, anchor: &str| -> Option { text.find(anchor).map(|at| at + anchor.len()) }; let total = &line[after(line, "\"total\":")?..]; let ir = &total[after(total, "\"Ir\":")?..]; let metrics = &ir[after(ir, "\"metrics\":")?..]; let int = &metrics[after(metrics, "{\"Int\":")?..]; let digits = &int[..int.find(|c: char| !c.is_ascii_digit())?]; Some((format!("{module}::{id}"), digits.parse().ok()?)) } /// The first string value of `key`; bench names carry no escapes. fn json_string<'a>(line: &'a str, key: &str) -> Option<&'a str> { let anchor = format!("\"{key}\":\""); let rest = &line[line.find(&anchor)? + anchor.len()..]; rest.find('"').map(|end| &rest[..end]) } /// Pins of `budgets.toml`: a `[budgets]` table of /// `"::::::" = { ir = }` lines, the shape /// `--pin` writes; any other line is rejected rather than guessed at. fn parse_budgets(text: &str) -> Result, String> { let mut pins = BTreeMap::new(); let mut in_table = false; for (number, line) in text.lines().enumerate() { let line = line.trim(); if line.is_empty() || line.starts_with('#') { continue; } if line == "[budgets]" { in_table = true; continue; } let Some((bench, ir)) = in_table.then(|| budget_line(line)).flatten() else { return Err(format!( "{BUDGETS}:{}: expected `\"\" = {{ ir = }}` under [budgets]", number + 1 )); }; if pins.insert(bench.to_string(), ir).is_some() { return Err(format!("{BUDGETS}:{}: {bench} pinned twice", number + 1)); } } Ok(pins) } fn budget_line(line: &str) -> Option<(&str, u64)> { let (bench, rest) = line.strip_prefix('"')?.split_once('"')?; let table = rest.trim_start().strip_prefix('=')?.trim(); let fields = table.strip_prefix('{')?.strip_suffix('}')?.trim(); let ir = fields .strip_prefix("ir")? .trim_start() .strip_prefix('=')? .trim(); Some((bench, ir.parse().ok()?)) } /// The platform named in a budgets file header, if any. fn pinned_platform(text: &str) -> Option { text.lines() .find_map(|line| line.trim().strip_prefix("# Pinned with ")) .map(|rest| rest.trim_end_matches('.').to_string()) } fn render_budgets(pins: &BTreeMap, platform: &str) -> String { use std::fmt::Write as _; let mut out = format!( "# Callgrind instruction budgets (Ir) per bench, gated within ±{BAND_PERCENT} % by `cargo x perf`.\n\ # Written by `cargo x perf --pin`; a re-pin is a deliberate commit that says why.\n\ # Pinned with {platform}.\n\n[budgets]\n" ); for (bench, ir) in pins { let _ = writeln!(out, "\"{bench}\" = {{ ir = {ir} }}"); } out } /// One line per bench outside its band, without a pin, or, when every /// bench ran, pinned but no longer measured. fn budget_violations( measured: &BTreeMap, pins: &BTreeMap, complete: bool, ) -> Vec { let mut violations = Vec::new(); for (bench, ir) in measured { match pins.get(bench) { None => violations.push(format!(" {bench}: measured {ir}, no pin\n")), Some(pin) if !within_band(*pin, *ir) => violations.push(format!( " {bench}: pin {pin}, measured {ir}, delta {} (band ±{BAND_PERCENT} %)\n", delta(*pin, *ir) )), Some(_) => {} } } if complete { for (bench, pin) in pins { if !measured.contains_key(bench) { violations.push(format!(" {bench}: pin {pin}, not measured (stale pin)\n")); } } } violations } fn within_band(pin: u64, ir: u64) -> bool { u128::from(ir.abs_diff(pin)) * 100 <= u128::from(pin) * u128::from(BAND_PERCENT) } /// Signed change of `ir` against `pin` in percent, two decimals. fn delta(pin: u64, ir: u64) -> String { if pin == 0 { return "n/a".to_string(); } let basis_points = (i128::from(ir) - i128::from(pin)) * 10_000 / i128::from(pin); let sign = if basis_points < 0 { '-' } else { '+' }; let magnitude = basis_points.unsigned_abs(); format!("{sign}{}.{:02} %", magnitude / 100, magnitude % 100) } /// `target/report/perf.md`: every measured bench against its pin, with the /// toolchain the counts belong to. fn write_perf_report( measured: &BTreeMap, pins: &BTreeMap, complete: bool, platform: &str, ) -> ExitCode { use std::fmt::Write as _; let mut table = format!( "Callgrind instruction counts (`Ir`) against `{BUDGETS}`, band ±{BAND_PERCENT} %.\n\n\ Platform: `{platform}`.\n\n| bench | ir | pin | delta |\n|---|---|---|---|\n" ); for (bench, ir) in measured { let (pin, change) = pins.get(bench).map_or_else( || ("none".to_string(), "n/a".to_string()), |pin| (pin.to_string(), delta(*pin, *ir)), ); let _ = writeln!(table, "| `{bench}` | {ir} | {pin} | {change} |"); } if complete { for (bench, pin) in pins .iter() .filter(|(bench, _)| !measured.contains_key(*bench)) { let _ = writeln!(table, "| `{bench}` | not measured | {pin} | n/a |"); } } let out = target_dir().join("report"); if fs::create_dir_all(&out).is_err() || fs::write(out.join("perf.md"), table).is_err() { return fail("cannot write target/report/perf.md\n"); } ExitCode::SUCCESS } /// What instruction counts depend on: the toolchain building the benches /// and the libc they link, e.g. `rustc 1.98.1 (…) | glibc 2.41`. fn platform() -> String { format!("{} | {}", tool_first_line("rustc"), libc_version()) } fn tool_first_line(program: &str) -> String { Command::new(program) .arg("--version") .output() .ok() .filter(|output| output.status.success()) .and_then(|output| { String::from_utf8_lossy(&output.stdout) .lines() .next() .map(str::to_string) }) .unwrap_or_else(|| format!("{program} version unavailable")) } /// `glibc ` from `ldd --version`, whose first line ends with the /// version on every glibc distribution; other libcs report as unknown. fn libc_version() -> String { let line = tool_first_line("ldd"); line.rsplit(' ') .next() .filter(|last| last.starts_with(|c: char| c.is_ascii_digit())) .map_or_else( || "libc unknown".to_string(), |version| format!("glibc {version}"), ) } /// `extra` without one leading `--`: `cargo x cmd -- args` and /// `cargo x cmd args` pass the same arguments on. fn without_separator(extra: &[OsString]) -> &[OsString] { match extra.split_first() { Some((first, rest)) if first == "--" => rest, _ => extra, } } fn cargo_fmt_step(extra: &[OsString]) -> ExitCode { let args = scoped(&["fmt", "--check"], extra); run_os("cargo", &args) } fn cargo_clippy(extra: &[OsString]) -> ExitCode { sequence(&[clippy_prod, clippy_tests, clippy_guests], extra) } /// Production targets carry the full strict gate. fn clippy_prod(extra: &[OsString]) -> ExitCode { clippy_phase(&prod_targets(extra), &[], extra) } /// `--lib` and `--bins` as the scoped package actually has them; cargo /// rejects a target flag the package cannot satisfy. fn prod_targets(extra: &[OsString]) -> Vec<&'static str> { let package = extra .iter() .position(|arg| arg == "-p" || arg == "--package") .and_then(|at| extra.get(at + 1)) .map(|name| name.to_string_lossy().into_owned()); let Some(package) = package else { return vec!["--lib", "--bins"]; }; let src = Path::new(&package).join("src"); let mut targets = Vec::new(); if src.join("lib.rs").is_file() { targets.push("--lib"); } if src.join("main.rs").is_file() || Path::new(&package).join("src/bin").is_dir() { targets.push("--bins"); } if targets.is_empty() { targets = vec!["--lib", "--bins"]; } targets } /// Test and bench targets may assert, panic, and print: outcomes are the /// point. Pedantic relaxes too; correctness lints stay on so test quality /// is still policed. fn clippy_tests(extra: &[OsString]) -> ExitCode { clippy_phase( &["--tests", "--benches"], &[ "-A", "clippy::unwrap_used", "-A", "clippy::expect_used", "-A", "clippy::panic", "-A", "clippy::todo", "-A", "clippy::unimplemented", "-A", "clippy::print_stdout", "-A", "clippy::print_stderr", "-A", "clippy::dbg_macro", "-A", "clippy::pedantic", ], extra, ) } /// Guest crates stay strict, cross-checked on their wasm target; the /// crate-level expectations on generated surfaces carry the exceptions. fn clippy_guests(extra: &[OsString]) -> ExitCode { if extra.iter().any(|arg| arg == "-p" || arg == "--package") { return ExitCode::SUCCESS; } clippy_phase( &[ "-p", "smith-plugin-echo", "--target", "wasm32-wasip2", "--lib", ], &[], extra, ) } fn clippy_phase(targets: &[&str], lint_args: &[&str], extra: &[OsString]) -> ExitCode { let mut base = vec!["clippy", "--locked"]; base.extend_from_slice(targets); base.push("--all-features"); let args = scoped(&base, extra); let mut command = Command::new("cargo"); command.env("CLIPPY_CONF_DIR", ".config").args(&args); if !lint_args.is_empty() { command.arg("--"); command.args(lint_args); } match command.status() { Ok(status) if status.success() => ExitCode::SUCCESS, Ok(status) => fail(&format!("cargo clippy exited with {status}\n")), Err(error) => fail(&format!("failed to run cargo clippy: {error}\n")), } } fn cargo_nextest(extra: &[OsString]) -> ExitCode { let args = scoped(&["--locked", "--workspace", "--all-features"], extra); nextest_run_os(&args) } /// End-to-end tier: the built binary through cli, eval, and rpc; test /// binaries named `e2e_*`, selected by the `e2e` nextest profile. fn cargo_e2e(extra: &[OsString]) -> ExitCode { let args = scoped( &[ "--locked", "--workspace", "--all-features", "--profile", "e2e", ], extra, ); nextest_run_os(&args) } /// Coverage over every tier; informational, never a percentage gate. /// Writes `target/coverage/lcov.info` and prints the per-file summary. fn cargo_coverage(extra: &[OsString]) -> ExitCode { let ran = coverage_tiers(extra); if ran != ExitCode::SUCCESS { return ran; } let lcov = target_dir().join("coverage").join("lcov.info"); let wrote = coverage_report(&["--lcov", "--output-path"], &lcov); if wrote != ExitCode::SUCCESS { return wrote; } run_os( "cargo", &["llvm-cov", "report", "--summary-only"].map(OsString::from), ) } /// Run both nextest tiers instrumented, accumulating one profile set. fn coverage_tiers(extra: &[OsString]) -> ExitCode { let installed = install_llvm_cov(&[]); if installed != ExitCode::SUCCESS { return installed; } if std::fs::create_dir_all(target_dir().join("coverage")).is_err() { return fail("cannot create target/coverage\n"); } let tiers: [&[&str]; 2] = [&[], &["--profile", "e2e"]]; let base = [ "llvm-cov", "nextest", "--no-report", "--locked", "--workspace", "--all-features", ]; let clean = run_os( "cargo", &["llvm-cov", "clean", "--workspace"].map(OsString::from), ); if clean != ExitCode::SUCCESS { return clean; } for tier in tiers { let mut args: Vec<&str> = base.to_vec(); args.extend_from_slice(tier); // A tier with no matching tests is a report, not a failure. args.push("--no-tests=warn"); let ran = run_os("cargo", &scoped(&args, extra)); if ran != ExitCode::SUCCESS { return ran; } } ExitCode::SUCCESS } /// One `cargo llvm-cov report` invocation with an output path. fn coverage_report(flags: &[&str], output: &Path) -> ExitCode { let mut args: Vec = ["llvm-cov", "report"].map(OsString::from).to_vec(); args.extend(flags.iter().map(OsString::from)); args.push(output.as_os_str().to_owned()); run_os("cargo", &args) } /// Every tier's evidence in one bundle under `target/report/`: `JUnit` per /// tier, lcov plus HTML coverage, the last mutation run, the last /// instruction-budget table, hyperfine wall-clock rows, and an `index.md` /// that summarizes them. Mutation and perf results are read, never run /// here; `mutants` and `perf` run first. Passthrough scopes the test tiers. fn report(extra: &[OsString]) -> ExitCode { use std::fmt::Write as _; let ran = coverage_tiers(extra); if ran != ExitCode::SUCCESS { return ran; } let out = target_dir().join("report"); // `cargo x perf` writes into the bundle directory; keep its table // across the reset. let perf = fs::read_to_string(out.join("perf.md")).ok(); let _ = std::fs::remove_dir_all(&out); if std::fs::create_dir_all(out.join("junit")).is_err() { return fail("cannot create target/report\n"); } let lcov = coverage_report(&["--lcov", "--output-path"], &out.join("lcov.info")); if lcov != ExitCode::SUCCESS { return lcov; } let html = coverage_report(&["--html", "--output-dir"], &out.join("coverage")); if html != ExitCode::SUCCESS { return html; } let summary = Command::new("cargo") .args(["llvm-cov", "report", "--summary-only"]) .output() .map_or_else( |error| format!("summary unavailable: {error}"), |output| String::from_utf8_lossy(&output.stdout).into_owned(), ); let mut index = String::from("# Smith test report\n\n"); index.push_str("## Tiers\n\n| tier | tests | failures | errors |\n|---|---|---|---|\n"); for tier in ["default", "e2e"] { // nextest keeps its store under the workspace target dir even when // llvm-cov redirects build output. let source = target_dir().join("nextest").join(tier).join("junit.xml"); let copied = out.join("junit").join(format!("{tier}.xml")); let row = std::fs::read_to_string(&source).map_or_else( |_| format!("| {tier} | no junit.xml | | |\n"), |xml| { let _ = std::fs::write(&copied, &xml); let [tests, failures, errors] = junit_counts(&xml); format!("| {tier} | {tests} | {failures} | {errors} |\n") }, ); index.push_str(&row); } index.push_str( "\n## Coverage\n\nHTML: `coverage/html/index.html`; lcov: `lcov.info`.\n\n```text\n", ); index.push_str(summary.trim_end()); index.push_str("\n```\n\n## Mutation\n\n"); let mutants = target_dir().join("mutants.out"); match mutants_counts(&mutants) { Some([caught, missed, timeout, unviable]) => { let _ = write!( index, "| caught | missed | timeout | unviable |\n|---|---|---|---|\n| {caught} | {missed} | {timeout} | {unviable} |\n" ); for name in ["missed.txt", "caught.txt", "timeout.txt", "unviable.txt"] { let _ = std::fs::copy(mutants.join(name), out.join(name)); } if missed > 0 { index.push_str("\nMissed mutants are the test backlog; see `missed.txt`.\n"); } } None => index.push_str("No mutation run found; run `cargo x mutants` first.\n"), } index.push_str("\n## Instructions\n\n"); match perf { Some(table) => { let _ = std::fs::write(out.join("perf.md"), &table); index.push_str(&table); } None => index.push_str("No perf run found; run `cargo x perf` first.\n"), } index.push_str("\n## Wall clock\n\n"); index.push_str(&wall_clock(&out)); if std::fs::write(out.join("index.md"), index).is_err() { return fail("cannot write target/report/index.md\n"); } let _ignored = std::io::stdout().write_all(b"report: target/report/index.md\n"); ExitCode::SUCCESS } /// Wall-clock rows of the release binary through hyperfine, isolated like /// the e2e tests. Informational only: the numbers hold for this machine /// and this run, so a missing tool or a failed run is a note, never a /// report failure. fn wall_clock(out: &Path) -> String { let present = Command::new("hyperfine") .arg("--version") .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .status() .is_ok_and(|status| status.success()); if !present { return "hyperfine not on PATH; no wall-clock rows.\n".to_string(); } let Ok(smith) = release_smith() else { return "release build of smith failed; no wall-clock rows.\n".to_string(); }; let home = target_dir().join("wall-clock-home"); let markdown = out.join("hyperfine.md"); let mut command = Command::new("hyperfine"); command .args(["--warmup", "3", "--runs", "20", "--export-json"]) .arg(out.join("hyperfine.json")) .arg("--export-markdown") .arg(&markdown) // A fresh profile per run: `eval` appends to its session otherwise. .arg("--prepare") .arg(format!("rm -rf '{}'", home.display())) .args(["--command-name", "smith --help"]) .args(["--command-name", "smith eval --mock hello"]) .arg(format!("'{}' --help", smith.display())) .arg(format!("'{}' eval --mock hello", smith.display())); for (key, dir) in dev_home_dirs() { command.env(key, home.join(dir)); } for var in PROVIDER_VARS { command.env_remove(var); } let ran = command.status().is_ok_and(|status| status.success()); match fs::read_to_string(&markdown) { Ok(rows) if ran => format!( "Informational, never a gate: {}-{}, {} hardware threads, this run only; raw data in `hyperfine.json`.\n\n{rows}", env::consts::OS, env::consts::ARCH, std::thread::available_parallelism().map_or(0, std::num::NonZero::get), ), _ => "hyperfine run failed; no wall-clock rows.\n".to_string(), } } /// `tests`, `failures`, `errors` from a `JUnit` `` root. fn junit_counts(xml: &str) -> [u64; 3] { let root = xml .find("').map(|end| &rest[..end])) .unwrap_or(""); ["tests", "failures", "errors"].map(|attribute| { let key = format!(" {attribute}=\""); root.find(&key) .map(|at| &root[at + key.len()..]) .and_then(|rest| rest.split('"').next()) .and_then(|value| value.parse().ok()) .unwrap_or(0) }) } /// Line counts of a cargo-mutants outcome directory: caught, missed, /// timeout, unviable; `None` without a run. fn mutants_counts(dir: &Path) -> Option<[usize; 4]> { let count = |name: &str| { std::fs::read_to_string(dir.join(name)) .map(|text| text.lines().filter(|line| !line.trim().is_empty()).count()) }; Some([ count("caught.txt").ok()?, count("missed.txt").ok()?, count("timeout.txt").unwrap_or(0), count("unviable.txt").unwrap_or(0), ]) } /// nextest has no warning-deny flag, so config-drift warnings are turned into /// a gate failure by scanning its stderr. fn nextest_run_os(args: &[OsString]) -> ExitCode { let mut child = match Command::new("cargo") .arg("nextest") .arg("run") .args(args) .stdout(std::process::Stdio::inherit()) .stderr(std::process::Stdio::piped()) .spawn() { Ok(child) => child, Err(error) => return fail(&format!("failed to run cargo nextest: {error}\n")), }; let mut warned = false; if let Some(stderr) = child.stderr.take() { let reader = std::io::BufReader::new(stderr); for line in reader.lines() { let Ok(line) = line else { break; }; if line.contains("warning: in config file") { warned = true; } let _ = std::io::stderr().write_all(format!("{line}\n").as_bytes()); } } match child.wait() { Ok(status) if status.success() && !warned => ExitCode::SUCCESS, Ok(status) if status.success() => fail("nextest produced config file warnings\n"), Ok(status) => fail(&format!( "cargo exited with exit status: {}\n", status.code().unwrap_or_default() )), Err(error) => fail(&format!("failed to wait for cargo nextest: {error}\n")), } } fn cargo_machete(_extra: &[OsString]) -> ExitCode { let installed = tool_present_bin("cargo-machete", MACHETE_VERSION, || { cargo(&[ "install", "--locked", "--version", MACHETE_VERSION, "cargo-machete", ]) }); if installed != ExitCode::SUCCESS { return installed; } run("cargo-machete", &[]) } /// Install the external gate tools that Cargo cannot ship. /// /// `cargo-nextest` arrives as a prebuilt binary through `cargo-binstall`. /// The step is idempotent. /// Installs tools; scoping does not apply. fn bootstrap(extra: &[OsString]) -> ExitCode { if !extra.is_empty() { return fail("bootstrap installs tools and admits no scoping\n"); } let steps: [Step; 8] = [ install_binstall, install_nextest, install_machete, install_deny, install_audit, install_mutants, install_llvm_cov, install_gungraun_runner, ]; sequence(&steps, extra) } fn install_machete(_extra: &[OsString]) -> ExitCode { tool_present_bin("cargo-machete", MACHETE_VERSION, || { cargo(&[ "install", "--locked", "--version", MACHETE_VERSION, "cargo-machete", ]) }) } fn install_deny(_extra: &[OsString]) -> ExitCode { binstall("cargo-deny", DENY_VERSION) } fn install_audit(_extra: &[OsString]) -> ExitCode { binstall("cargo-audit", AUDIT_VERSION) } fn install_mutants(_extra: &[OsString]) -> ExitCode { binstall("cargo-mutants", MUTANTS_VERSION) } fn install_gungraun_runner(_extra: &[OsString]) -> ExitCode { binstall("gungraun-runner", GUNGRAUN_VERSION) } fn install_llvm_cov(_extra: &[OsString]) -> ExitCode { let tool = binstall("cargo-llvm-cov", LLVM_COV_VERSION); if tool != ExitCode::SUCCESS { return tool; } run_os( "rustup", &["component", "add", "llvm-tools"].map(OsString::from), ) } fn install_binstall(_extra: &[OsString]) -> ExitCode { tool_present_bin("cargo-binstall", BINSTALL_VERSION, || { cargo(&[ "install", "cargo-binstall", "--version", BINSTALL_VERSION, // Freeze binstall's own dependency graph for the same reason // crates.io drift cannot break the gate. "--locked", ]) }) } fn install_nextest(_extra: &[OsString]) -> ExitCode { binstall("cargo-nextest", NEXTEST_VERSION) } /// Install the prebuilt `tool` at `version` through `cargo-binstall` /// unless that version is already on PATH. fn binstall(tool: &str, version: &str) -> ExitCode { tool_present_bin(tool, version, || { cargo(&["binstall", "--no-confirm", &format!("{tool}@{version}")]) }) } /// Run `probe` with cargo; when it fails, run `install` and return its code. /// The probe pins the tool so a stale unpinned binary is replaced. /// Probes an external binary directly and enforces the pinned version; /// mismatch or absence runs the installer, so drift self-heals to the pin. fn tool_present_bin(program: &str, version: &str, install: impl FnOnce() -> ExitCode) -> ExitCode { let probe = Command::new(program) .arg("--version") .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::null()) .output(); let pinned = probe.is_ok_and(|output| { output.status.success() && String::from_utf8_lossy(&output.stdout).contains(version) }); if pinned { ExitCode::SUCCESS } else { install() } } fn cargo_doc(extra: &[OsString]) -> ExitCode { let args = scoped(&["doc", "--locked", "--workspace", "--no-deps"], extra); run_os("cargo", &args) } fn cargo_deny(_extra: &[OsString]) -> ExitCode { cargo(&["deny", "--config", ".config/deny.toml", "check"]) } fn cargo_audit(_extra: &[OsString]) -> ExitCode { cargo(&["audit", "--deny", "warnings"]) } fn cargo(args: &[&str]) -> ExitCode { run("cargo", args) } /// Builds the CLI, then launches it against an isolated dev home. fn dev(args: &[OsString]) -> ExitCode { let code = cargo(&["build", "--locked", "-p", "smith-cli", "--bin", "smith"]); if code != ExitCode::SUCCESS { return code; } let target = target_dir(); let binary = target.join("debug").join(binary_name()); if !binary.is_file() { return fail(&format!("missing dev binary: {}\n", binary.display())); } let home = target.join("dev-home"); let mut command = Command::new(&binary); for (key, dir) in dev_home_dirs() { let path = home.join(dir); if let Err(error) = fs::create_dir_all(&path) { return fail(&format!("failed to create {}: {error}\n", path.display())); } command.env(key, &path); } command.args(args); note(&format!( "running {} (home {})\n", binary.display(), home.display() )); match command.status() { Ok(status) => match status.code() { Some(0) => ExitCode::SUCCESS, Some(value) => ExitCode::from(u8::try_from(value).unwrap_or(1)), None => fail("smith terminated by signal\n"), }, Err(error) => fail(&format!("failed to run {}: {error}\n", binary.display())), } } /// Environment overrides that keep dev runs out of the real user profile. const fn dev_home_dirs() -> &'static [(&'static str, &'static str)] { &[ ("XDG_CONFIG_HOME", "config"), ("XDG_DATA_HOME", "data"), ("XDG_STATE_HOME", "state"), ("XDG_CACHE_HOME", "cache"), ] } fn target_dir() -> PathBuf { env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from) } const fn binary_name() -> &'static str { if cfg!(windows) { "smith.exe" } else { "smith" } } fn note(message: &str) { let _ignored = std::io::stderr().write_all(message.as_bytes()); } fn run_os(program: &str, args: &[OsString]) -> ExitCode { match Command::new(program).args(args).status() { Ok(status) if status.success() => ExitCode::SUCCESS, Ok(status) => fail(&format!("{program} exited with {status}\n")), Err(error) => fail(&format!("failed to run {program}: {error}\n")), } } fn run(program: &str, args: &[&str]) -> ExitCode { match Command::new(program).args(args).status() { Ok(status) if status.success() => ExitCode::SUCCESS, Ok(status) => fail(&format!("{program} exited with {status}\n")), Err(error) => fail(&format!("failed to run {program}: {error}\n")), } } fn arch(extra: &[OsString]) -> ExitCode { let wanted: Vec = extra .iter() .map(|arg| arg.to_string_lossy().into_owned()) .collect(); for crate_name in CRATES { if !Path::new(crate_name).join("Cargo.toml").is_file() { return fail(&format!("missing crate manifest: {crate_name}\n")); } } for rule in DEPENDENCY_RULES { if !wanted.is_empty() && !wanted.iter().any(|name| name == rule.crate_name) { continue; } let manifest = match fs::read_to_string(Path::new(rule.crate_name).join("Cargo.toml")) { Ok(value) => value, Err(error) => { return fail(&format!( "failed to read {} Cargo.toml: {error}\n", rule.crate_name )); } }; let denied = CRATES .iter() .filter(|name| **name != rule.crate_name && !rule.allowed.contains(name)); for denied in denied { if has_dependency(&manifest, denied) { return fail(&format!( "{} must not depend on {denied}\n", rule.crate_name )); } } if rule.crate_name != "smith-bench" && names_dependency(&manifest, "smith-bench") { return fail(&format!( "{} must not depend on smith-bench, not even from tests\n", rule.crate_name )); } if LIBRARY_CRATES.contains(&rule.crate_name) && let Some(offender) = mutex_in_production(Path::new(rule.crate_name)) { return fail(&format!( "{}: Mutex in library code (concurrency law): {}\n", rule.crate_name, offender.display() )); } } if let Some(offender) = missing_spec_reference() { return fail(&format!( "{}: test code without an SMH-SPEC- citation in its first 20 lines or `//!` header (RULES.md Testing)\n", offender.display() )); } ExitCode::SUCCESS } /// The first `.rs` file under any workspace crate that carries test code /// (`#[test]` or a `tests/` directory) but cites no `SMH-SPEC-…` id in its /// first 20 lines or its `//!` header block. fn missing_spec_reference() -> Option { for crate_name in CRATES { let mut files = Vec::new(); collect_rs_files(Path::new(crate_name), &mut files); for path in files { let Ok(source) = fs::read_to_string(&path) else { continue; }; if is_test_bearing(&path, &source) && !cites_a_spec(&source) { return Some(path); } } } None } /// `.rs` files under `dir`, deterministically ordered, skipping `target`. fn collect_rs_files(dir: &Path, out: &mut Vec) { let Ok(entries) = fs::read_dir(dir) else { return; }; let mut entries: Vec = entries.flatten().map(|entry| entry.path()).collect(); entries.sort(); for path in entries { if path.is_dir() { if path.file_name().is_some_and(|name| name == "target") { continue; } collect_rs_files(&path, out); } else if path.extension().is_some_and(|ext| ext == "rs") { out.push(path); } } } /// A file carries test code when it literally declares `#[test]` or lives /// under a `tests/` directory (integration test crates have no attribute). fn is_test_bearing(path: &Path, source: &str) -> bool { source.contains("#[test]") || path.components().any(|part| part.as_os_str() == "tests") } /// Whether `SMH-SPEC-` appears in the first 20 lines or the leading `//!` /// doc-comment block, whichever the file has. fn cites_a_spec(source: &str) -> bool { let first_twenty: Vec<&str> = source.lines().take(20).collect(); first_twenty.iter().any(|line| line.contains("SMH-SPEC-")) || leading_doc_block(source).contains("SMH-SPEC-") } /// The contiguous run of `//!` lines starting the file, attributes and /// blank lines before it skipped, concatenated. fn leading_doc_block(source: &str) -> String { let mut block = String::new(); for line in source.lines() { let trimmed = line.trim_start(); if trimmed.starts_with("//!") { block.push_str(trimmed); block.push('\n'); } else if !block.is_empty() { break; } } block } /// Crates under the "no Mutex in library crates" law; binaries are exempt. const LIBRARY_CRATES: &[&str] = &[ "smith", "smith-core", "smith-ai", "smith-ui", "smith-tui", "smith-rpc", "smith-harness", "smith-alloc", ]; /// The first `.rs` file under `crate_dir/src` whose production slice /// (everything before `#[cfg(test)]`) mentions `Mutex`. fn mutex_in_production(crate_dir: &Path) -> Option { let mut pending = vec![crate_dir.join("src")]; while let Some(dir) = pending.pop() { let Ok(entries) = fs::read_dir(&dir) else { continue; }; for entry in entries.flatten() { let path = entry.path(); if path.is_dir() { pending.push(path); } else if path.extension().is_some_and(|ext| ext == "rs") && fs::read_to_string(&path) .is_ok_and(|source| production_slice(&source).contains("Mutex")) { return Some(path); } } } None } /// Source text before the first `#[cfg(test)]`; tests may lock freely. fn production_slice(source: &str) -> &str { source.split("#[cfg(test)]").next().unwrap_or(source) } /// Library edges a crate may have; every other workspace crate is denied. struct DependencyRule { crate_name: &'static str, allowed: &'static [&'static str], } /// The allowed-deps table of `.system/RULES.md`, architecture section. const DEPENDENCY_RULES: &[DependencyRule] = &[ DependencyRule { crate_name: "smith", allowed: &[], }, DependencyRule { crate_name: "smith-ui", allowed: &["smith"], }, DependencyRule { crate_name: "smith-core", allowed: &["smith"], }, DependencyRule { crate_name: "smith-ai", allowed: &["smith"], }, DependencyRule { crate_name: "smith-tui", allowed: &["smith", "smith-ui"], }, DependencyRule { crate_name: "smith-rpc", allowed: &["smith", "smith-core"], }, DependencyRule { crate_name: "smith-harness", allowed: &["smith", "smith-core", "smith-ai", "smith-tui", "smith-ui"], }, DependencyRule { crate_name: "smith-cli", allowed: &["smith", "smith-harness", "smith-rpc", "smith-alloc"], }, DependencyRule { crate_name: "smith-alloc", allowed: &[], }, DependencyRule { crate_name: "smith-bench", allowed: &[ "smith", "smith-core", "smith-ai", "smith-ui", "smith-tui", "smith-rpc", "smith-harness", "smith-cli", "smith-alloc", "xtask", ], }, DependencyRule { crate_name: "xtask", allowed: &[], }, ]; fn has_dependency(manifest: &str, crate_name: &str) -> bool { // Library edges only: the manifest up to the dev-dependency table. let library = manifest .split("[dev-dependencies]") .next() .unwrap_or(manifest); let table = format!("[dependencies.{crate_name}]"); let assignment = format!("{crate_name} ="); library.lines().any(|line| { let trimmed = line.trim_start(); trimmed.starts_with(&table) || trimmed.starts_with(&assignment) }) } /// Whether any dependency table of `manifest`, dev and target tables /// included, names `crate_name`. fn names_dependency(manifest: &str, crate_name: &str) -> bool { let assignment = format!("{crate_name} ="); let table = format!(".{crate_name}]"); manifest.lines().any(|line| { let trimmed = line.trim(); trimmed.starts_with(&assignment) || (trimmed.starts_with('[') && trimmed.ends_with(&table)) }) } fn fail(message: &str) -> ExitCode { let _ignored = std::io::stderr().write_all(message.as_bytes()); ExitCode::FAILURE } #[cfg(test)] mod tests { #[test] fn mutants_refuse_to_run_unscoped() { let scoped = |args: &[&str]| { let args: Vec = args.iter().map(std::ffi::OsString::from).collect(); super::mutants_scoped(&args) }; assert!(scoped(&["-f", "smith-core/src/session.rs"])); assert!(scoped(&["--shard=0/8"])); assert!(scoped(&["-D", "/tmp/d.patch", "-j2"])); assert!(scoped(&["--list"])); assert!(!scoped(&[])); assert!(!scoped(&["-j4", "--baseline", "run"])); } #[test] fn local_mutants_reject_a_job_count() { let flag = |arg: &str| super::jobs_flag(&std::ffi::OsString::from(arg)); assert!(flag("-j")); assert!(flag("-j4")); assert!(flag("--jobs")); assert!(flag("--jobs=2")); assert!(!flag("-jx")); assert!(!flag("--shard")); } #[test] fn junit_counts_read_the_testsuites_root() { let xml = r#""#; assert_eq!(super::junit_counts(xml), [19, 1, 0]); assert_eq!(super::junit_counts(""), [0, 0, 0]); } #[test] fn mutants_counts_need_a_run() { let dir = std::env::temp_dir().join(format!("smith_mutants_probe_{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); assert_eq!(super::mutants_counts(&dir), None); std::fs::create_dir_all(&dir).unwrap(); std::fs::write(dir.join("caught.txt"), "a\nb\n").unwrap(); std::fs::write(dir.join("missed.txt"), "c\n").unwrap(); assert_eq!(super::mutants_counts(&dir), Some([2, 1, 0, 0])); let _ = std::fs::remove_dir_all(&dir); } #[test] fn production_slice_stops_at_the_test_module() { let source = "use std::sync::Arc;\n#[cfg(test)]\nmod tests { use std::sync::Mutex; }"; assert!(!super::production_slice(source).contains("Mutex")); assert!(super::production_slice("static M: Mutex;").contains("Mutex")); } use super::{cites_a_spec, is_test_bearing}; use std::path::Path; #[test] fn test_bearing_by_attribute_or_by_directory() { assert!(is_test_bearing( Path::new("smith-core/src/agent.rs"), "mod tests {\n#[test]\nfn it_works() {}\n}" )); assert!(is_test_bearing( Path::new("smith-cli/tests/e2e_eval.rs"), "fn helper() {}" )); assert!(!is_test_bearing( Path::new("smith-core/src/lib.rs"), "fn helper() {}" )); } #[test] fn spec_citation_in_first_twenty_lines_or_doc_block() { assert!(cites_a_spec("//! docs (`SMH-SPEC-SPEC0001`, Tools).\n")); let padded = format!("{}fn f() {{}}\n", "//! line\n".repeat(19)); assert!(!cites_a_spec(&padded)); let past_twenty = format!( "{}//! cites `SMH-SPEC-SPEC0001` here\nfn f() {{}}\n", "//! line\n".repeat(25) ); assert!(cites_a_spec(&past_twenty)); assert!(!cites_a_spec("// no doc block\nfn f() {}\n")); } use super::{CRATES, DEPENDENCY_RULES, binary_name, dev_home_dirs, has_dependency, target_dir}; use std::path::PathBuf; #[test] fn every_crate_has_one_rule_over_known_crates() { for name in CRATES { let rules = DEPENDENCY_RULES .iter() .filter(|rule| rule.crate_name == *name); assert_eq!(rules.count(), 1, "{name}"); } assert_eq!(DEPENDENCY_RULES.len(), CRATES.len()); for rule in DEPENDENCY_RULES { assert!( rule.allowed .iter() .all(|dep| CRATES.contains(dep) && *dep != rule.crate_name) ); } } #[test] fn dev_home_covers_all_xdg_roots() { let keys: Vec<&str> = dev_home_dirs().iter().map(|(key, _)| *key).collect(); assert_eq!( keys, vec![ "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME" ] ); } #[test] fn dev_binary_path_is_platform_correct() { let path = target_dir().join("debug").join(binary_name()); assert!(path.ends_with(if cfg!(windows) { "smith.exe" } else { "smith" })); assert!(path.starts_with(env_target())); } fn env_target() -> PathBuf { std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from) } use super::{ BTreeMap, bench_ir, budget_violations, delta, names_dependency, parse_budgets, render_budgets, without_separator, }; const FIRST_RUN: &str = r#"{"baselines":[null,null],"details":"(linear(1000))","function_name":"encode","id":"linear_1000","kind":"LibraryBenchmark","module_path":"instructions::session::encode","profiles":[{"summaries":{"parts":[{"details":{"Left":{"pid":1,"thread":1}},"metrics_summary":{"Callgrind":{"Ir":{"diffs":null,"metrics":{"Left":{"Int":999}}}}}}],"total":{"regressions":[],"summary":{"Callgrind":{"Ir":{"diffs":null,"metrics":{"Left":{"Int":5736448}}}}}}},"tool":"Callgrind"}],"version":"6"}"#; #[test] fn pinned_platform_reads_the_header() { let text = "# x\n# Pinned with rustc 1.98.1 (a 2026) | glibc 2.41.\n[budgets]\n"; assert_eq!( super::pinned_platform(text).as_deref(), Some("rustc 1.98.1 (a 2026) | glibc 2.41") ); assert_eq!(super::pinned_platform("[budgets]\n"), None); } #[test] fn bench_ir_reads_the_total_of_first_and_repeated_runs() { assert_eq!( bench_ir(FIRST_RUN), Some(( "instructions::session::encode::linear_1000".to_string(), 5_736_448 )) ); let repeated = FIRST_RUN.replace( r#""total":{"regressions":[],"summary":{"Callgrind":{"Ir":{"diffs":null,"metrics":{"Left":{"Int":5736448}}}}}}"#, r#""total":{"regressions":[],"summary":{"Callgrind":{"Ir":{"diffs":{"diff_pct":0.1},"metrics":{"Both":[{"Int":5736500},{"Int":5736448}]}}}}}"#, ); assert_eq!(bench_ir(&repeated).map(|(_, ir)| ir), Some(5_736_500)); assert_eq!(bench_ir(r#"{"module_path":"a","id":"b"}"#), None); } #[test] fn budgets_round_trip_sorted_and_reject_other_shapes() { let pins = BTreeMap::from([ ("b::g::f::x".to_string(), 7), ("a::g::f::y".to_string(), 42), ]); let text = render_budgets(&pins, "rustc 1.0.0"); assert!(text.find("a::g::f::y").unwrap() < text.find("b::g::f::x").unwrap()); assert_eq!(parse_budgets(&text).unwrap(), pins); assert!( parse_budgets("\"a\" = { ir = 1 }\n").is_err(), "outside [budgets]" ); assert!(parse_budgets("[budgets]\n\"a\" = 1\n").is_err()); assert!(parse_budgets("[budgets]\n\"a\" = { ir = 1 }\n\"a\" = { ir = 2 }\n").is_err()); } #[test] fn violations_cover_both_directions_missing_and_stale_pins() { let pins = BTreeMap::from([ ("low".to_string(), 1000), ("edge".to_string(), 1000), ("high".to_string(), 1000), ("stale".to_string(), 5), ]); let measured = BTreeMap::from([ ("low".to_string(), 979), ("edge".to_string(), 1020), ("high".to_string(), 1021), ("new".to_string(), 3), ]); let found = budget_violations(&measured, &pins, true).concat(); for bench in ["low:", "high:", "new:", "stale:"] { assert!(found.contains(bench), "{bench} in {found}"); } assert!(!found.contains("edge:"), "{found}"); let filtered = budget_violations(&measured, &pins, false).concat(); assert!(!filtered.contains("stale:"), "{filtered}"); } #[test] fn delta_keeps_the_sign_below_one_percent() { assert_eq!(delta(1000, 995), "-0.50 %"); assert_eq!(delta(1000, 1025), "+2.50 %"); assert_eq!(delta(0, 5), "n/a"); } #[test] fn separator_is_stripped_once() { let args = ["--", "--", "x"].map(std::ffi::OsString::from); assert_eq!(without_separator(&args), &args[1..]); assert_eq!(without_separator(&args[2..]), &args[2..]); } #[test] fn names_dependency_sees_dev_and_target_tables() { assert!(names_dependency( "[dev-dependencies]\nsmith-bench = { path = \"..\" }", "smith-bench" )); assert!(names_dependency( "[target.'cfg(unix)'.dev-dependencies.smith-bench]", "smith-bench" )); assert!(!names_dependency("name = \"smith-bench\"", "smith-bench")); } #[test] fn detects_inline_and_table_dependencies() { assert!(has_dependency( "smith-core = { path = \"../smith-core\" }", "smith-core" )); assert!(has_dependency( "[dependencies.smith-ai]\npath = \"../smith-ai\"", "smith-ai" )); } #[test] fn ignores_non_dependency_mentions() { assert!(!has_dependency("# smith-core = forbidden", "smith-core")); assert!(!has_dependency("smith_core = \"0.1\"", "smith-core")); } }