repositories / smith
smith
There are many coding harnesses - but this one is fast
owned by admin
xtask/src/main.rs
Raw#![forbid(unsafe_code)]
#![warn(missing_docs)]
//! Cargo-only Smith project automation (`SMH-SPEC-SPEC0001`, acceptance:
//! quality, architecture, documentation gates).
//!
//! Gate tools follow one rule: install immutable artifacts or pinned
//! versions with `--locked`, never live dependency resolution, so upstream
//! releases cannot break a gate without a change in this repository.
use std::{
env,
ffi::OsString,
fs,
io::{BufRead, Write},
path::{Path, PathBuf},
process::Command,
process::ExitCode,
};
use std::collections::BTreeMap;
const HELP: &str = concat!(
"usage: cargo x <check|lint|fmt|test|e2e|coverage|report|doc|arch|mutants [args...]|audit|ci|release|bootstrap>\n",
" cargo x perf [--pin] [args...] instruction budgets (callgrind), gated against smith-bench/budgets.toml\n",
" cargo x bench [args...] wall-clock criterion benches; never gate\n",
" cargo x dev [args...] build and launch the dev smith binary\n",
);
/// Version of `cargo-nextest` used by the test gate.
const NEXTEST_VERSION: &str = "0.9.146";
/// Version of `cargo-binstall` used to install `cargo-nextest`.
const BINSTALL_VERSION: &str = "1.23.0";
const MACHETE_VERSION: &str = "0.9.2";
const DENY_VERSION: &str = "0.20.2";
const AUDIT_VERSION: &str = "0.22.2";
const MUTANTS_VERSION: &str = "27.1.0";
const LLVM_COV_VERSION: &str = "0.9.1";
/// Version of `gungraun-runner`; must match the `gungraun` crate in the lockfile.
const GUNGRAUN_VERSION: &str = "0.19.4";
/// gungraun bench targets of `smith-bench`, gated by `cargo x perf`.
const INSTRUCTION_BENCHES: &[&str] = &["instructions", "startup"];
/// criterion bench target of `smith-bench`, run by `cargo x bench`.
const WALL_CLOCK_BENCH: &str = "wall_clock";
/// Instruction pins, one per gungraun bench.
const BUDGETS: &str = "smith-bench/budgets.toml";
/// Allowed deviation of a measured `Ir` from its pin, either direction.
const BAND_PERCENT: u64 = 2;
/// Provider selection and credential variables that never reach a
/// measured `smith` run.
const PROVIDER_VARS: &[&str] = &[
"SMITH_PROVIDER",
"SMITH_MODEL",
"SMITH_BASE_URL",
"ANTHROPIC_API_KEY",
"OPENAI_API_KEY",
"GEMINI_API_KEY",
];
const CRATES: &[&str] = &[
"smith",
"smith-core",
"smith-ai",
"smith-ui",
"smith-tui",
"smith-rpc",
"smith-harness",
"smith-cli",
"smith-alloc",
"smith-bench",
"xtask",
];
fn main() -> ExitCode {
let mut args = env::args_os();
let _program = args.next();
let Some(command) = args.next() else {
return fail(HELP);
};
let rest: Vec<OsString> = args.collect();
if command.to_str() == Some("dev") {
return dev(&rest);
}
match command.to_str() {
Some("check") => check(&rest),
Some("lint") => lint(&rest),
Some("fmt") => cargo_fmt_step(&rest),
Some("test") => cargo_nextest(&rest),
Some("e2e") => cargo_e2e(&rest),
Some("coverage") => cargo_coverage(&rest),
Some("report") => report(&rest),
Some("doc") => cargo_doc(&rest),
Some("arch") => arch(&rest),
Some("mutants") => cargo_mutants(&rest),
Some("perf") => perf(&rest),
Some("bench") => bench(&rest),
Some("audit") => audit(&rest),
Some("ci") => ci(&rest),
Some("release") => release(&rest),
Some("bootstrap") => bootstrap(&rest),
_ => fail(HELP),
}
}
fn check(extra: &[OsString]) -> ExitCode {
sequence(
&[
cargo_fmt_step,
cargo_clippy,
arch,
cargo_nextest,
cargo_doc,
cargo_machete,
],
extra,
)
}
type Step = fn(&[OsString]) -> ExitCode;
fn sequence(steps: &[Step], extra: &[OsString]) -> ExitCode {
for step in steps {
let code = step(extra);
if code != ExitCode::SUCCESS {
return code;
}
}
ExitCode::SUCCESS
}
/// Append passthrough args; package scoping overrides workspace defaults,
/// so `-p crate` drops the `--workspace` flag rather than conflicting.
fn scoped(base: &[&str], extra: &[OsString]) -> Vec<OsString> {
let mut args: Vec<OsString> = base.iter().map(OsString::from).collect();
let narrows = extra.iter().any(|arg| {
arg == "-p" || arg == "--package" || arg.to_string_lossy().starts_with("--package=")
});
if narrows {
args.retain(|arg| arg != "--workspace");
}
args.extend_from_slice(extra);
args
}
fn lint(extra: &[OsString]) -> ExitCode {
sequence(&[cargo_fmt_step, cargo_clippy], extra)
}
/// Audit gates read the lockfile; scoping does not apply.
fn audit(extra: &[OsString]) -> ExitCode {
if !extra.is_empty() {
return fail("audit is lockfile-wide and admits no scoping\n");
}
sequence(
&[install_deny, install_audit, cargo_deny, cargo_audit],
extra,
)
}
/// The merge-gate pipeline: every gate, machine context, fixed order.
/// Passthrough is rejected so workflows cannot drift from the definition.
fn ci(extra: &[OsString]) -> ExitCode {
if !extra.is_empty() {
return fail("ci is a fixed pipeline and admits no scoping\n");
}
sequence(&[bootstrap, check, audit], extra)
}
/// Release gates are defined by the release-readiness plan; the command
/// exists so workflows reference a stable name today.
fn release(extra: &[OsString]) -> ExitCode {
let _ = extra;
fail("release gates are not defined yet; see SMH-PLAN-RELS0001\n")
}
/// Mutation testing runs on demand, never in the blocking check chain.
///
/// Split of concerns, kept CI-sensible:
/// - `.cargo/mutants.toml` owns tree properties (scope, tool, profiles)
/// so bare `cargo mutants` behaves identically anywhere.
/// - this step owns run context: disk-backed scratch (system tmpfs is
/// 16G RAM and tree copies crawl the machine), a modest default job
/// count overridable through `CARGO_MUTANTS_JOBS`, and a skipped
/// baseline because `cargo x check` owns test verification.
fn cargo_mutants(extra: &[OsString]) -> ExitCode {
let installed = install_mutants(&[]);
if installed != ExitCode::SUCCESS {
return installed;
}
let scratch = target_dir().join("mutants-tmp");
if std::fs::create_dir_all(&scratch).is_err() {
return fail("cannot create target/mutants-tmp\n");
}
// `cargo x mutants -- -f x` and `cargo x mutants -f x` are equivalent,
// so cargo-run habits cannot turn scope flags into stray nextest test
// filters.
let extra = without_separator(extra);
let all = extra.iter().any(|arg| arg == "--all");
if !all && !mutants_scoped(extra) {
return fail(MUTANTS_SCOPE_HELP);
}
let passthrough: Vec<&OsString> = extra.iter().filter(|arg| *arg != "--all").collect();
// Local runs are pinned to one job and a third of the cores for both
// the build and the test phase: four jobs and a full fan-out crashed
// this machine. Only `--all` (CI) reads `CARGO_MUTANTS_JOBS`.
if !all && passthrough.iter().any(|arg| jobs_flag(arg)) {
return fail("cargo x mutants runs one job locally; drop -j/--jobs\n");
}
let jobs = if all {
std::env::var("CARGO_MUTANTS_JOBS").unwrap_or_else(|_| "1".to_string())
} else {
"1".to_string()
};
let cores = std::thread::available_parallelism().map_or(2, |cores| (cores.get() / 3).max(2));
let mut command = Command::new("nice");
command
.args(["-n", "19", "cargo-mutants"])
.env("TMPDIR", &scratch)
.args([
"mutants",
"--output",
"target",
"--jobs",
&jobs,
"--baseline",
"skip",
"--timeout",
"120",
// A mutant once hung rustc itself; builds are bounded too.
"--build-timeout",
"300",
"--no-shuffle",
]);
if !all {
command
.env("NEXTEST_TEST_THREADS", cores.to_string())
.env("CARGO_BUILD_JOBS", cores.to_string());
}
// Passthrough args come last and override the defaults above.
command.args(passthrough);
match command.status() {
Ok(status) if status.success() => ExitCode::SUCCESS,
Ok(status) if status.code() == Some(2) => fail(
"missed mutants: tests did not catch everything (see target/mutants.out/missed.txt)\n",
),
Ok(status) => fail(&format!("cargo mutants exited with {status}\n")),
Err(error) => fail(&format!("failed to run cargo mutants: {error}\n")),
}
}
const MUTANTS_SCOPE_HELP: &str = "\
cargo x mutants needs a scope: a workspace-wide run copies the tree per job and \
has crashed this machine before.
-f <glob> one file or directory, e.g. -f smith-core/src/session.rs
-D <patch> mutants on changed lines only (jj diff --git > /tmp/d.patch)
-p <crate> one crate
--shard k/n one of n equal shards, e.g. --shard 0/8
-F/-E <regex> mutant-name filters
--list inventory only, no build
--all the whole workspace; CI only
";
fn jobs_flag(arg: &OsString) -> bool {
let arg = arg.to_string_lossy();
arg == "-j"
|| arg == "--jobs"
|| arg.starts_with("-j") && arg[2..].bytes().all(|b| b.is_ascii_digit())
|| arg.starts_with("--jobs=")
}
/// Whether the passthrough arguments narrow a mutants run.
fn mutants_scoped(extra: &[OsString]) -> bool {
extra.iter().any(|arg| {
let arg = arg.to_string_lossy();
matches!(
arg.as_ref(),
"-f" | "--file"
| "-D"
| "--in-diff"
| "-p"
| "--package"
| "--shard"
| "-F"
| "--re"
| "-E"
| "--exclude-re"
| "--list"
) || arg.starts_with("--file=")
|| arg.starts_with("--in-diff=")
|| arg.starts_with("--package=")
|| arg.starts_with("--shard=")
|| arg.starts_with("--re=")
|| arg.starts_with("--exclude-re=")
})
}
/// Instruction budgets: every gungraun bench's callgrind `Ir` against its
/// pin in `smith-bench/budgets.toml`, within ±2 % in either direction, so
/// a regression fails and an improvement fails until re-pinned.
///
/// `--pin` rewrites the pins from this run instead of gating. Other
/// arguments go to the gungraun runner; a filter narrows the run, and pins
/// of benches it skipped are kept rather than reported stale. Every run
/// writes `target/report/perf.md`.
fn perf(extra: &[OsString]) -> ExitCode {
let repin = extra.iter().any(|arg| arg == "--pin");
let passthrough: Vec<OsString> = without_separator(extra)
.iter()
.filter(|arg| *arg != "--pin")
.cloned()
.collect();
let installed = install_gungraun_runner(&[]);
if installed != ExitCode::SUCCESS {
return installed;
}
let smith = match release_smith() {
Ok(path) => path,
Err(code) => return code,
};
let measured = match instruction_counts(&smith, &passthrough) {
Ok(measured) => measured,
Err(message) => return fail(&message),
};
let pins = match fs::read_to_string(BUDGETS) {
Ok(text) => parse_budgets(&text),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(BTreeMap::new()),
Err(error) => Err(format!("cannot read {BUDGETS}: {error}")),
};
let pins = match pins {
Ok(pins) => pins,
Err(message) => return fail(&format!("{message}\n")),
};
let complete = passthrough.is_empty();
let platform = platform();
let written = write_perf_report(&measured, &pins, complete, &platform);
if written != ExitCode::SUCCESS {
return written;
}
if repin {
let mut pinned = if complete { BTreeMap::new() } else { pins };
pinned.extend(measured);
if fs::write(BUDGETS, render_budgets(&pinned, &platform)).is_err() {
return fail(&format!("cannot write {BUDGETS}\n"));
}
note(&format!(
"perf: pinned {} benches in {BUDGETS}\n",
pinned.len()
));
return ExitCode::SUCCESS;
}
// Instruction counts belong to one toolchain and libc: the gate runs
// where the pins were taken (the CI image); elsewhere the run reports.
let pinned_on = fs::read_to_string(BUDGETS)
.ok()
.and_then(|text| pinned_platform(&text));
if pinned_on.as_deref() != Some(platform.as_str()) {
note(&format!(
"perf: pins belong to `{}`, this run is `{platform}`; counts reported in target/report/perf.md, gate skipped\n",
pinned_on.as_deref().unwrap_or("no platform")
));
return ExitCode::SUCCESS;
}
let violations = budget_violations(&measured, &pins, complete);
if violations.is_empty() {
note(&format!(
"perf: {} benches within ±{BAND_PERCENT} % of their pins\n",
measured.len()
));
ExitCode::SUCCESS
} else {
fail(&format!(
"instruction budgets violated (re-pin deliberately with `cargo x perf --pin`):\n{}",
violations.concat()
))
}
}
/// Wall clock: the criterion benches, for A/B comparisons of two builds
/// on one machine in one session. Numbers are never committed and never
/// gate. Arguments go to criterion, e.g. `--save-baseline <change>`.
fn bench(extra: &[OsString]) -> ExitCode {
let mut args: Vec<OsString> = [
"bench",
"--locked",
"-p",
"smith-bench",
"--bench",
WALL_CLOCK_BENCH,
"--",
]
.map(OsString::from)
.to_vec();
args.extend_from_slice(without_separator(extra));
run_os("cargo", &args)
}
/// Build the release `smith` binary and return its absolute path.
fn release_smith() -> Result<PathBuf, ExitCode> {
let built = cargo(&[
"build",
"--locked",
"--release",
"-p",
"smith-cli",
"--bin",
"smith",
]);
if built != ExitCode::SUCCESS {
return Err(built);
}
let binary = target_dir().join("release").join(binary_name());
std::path::absolute(&binary)
.map_err(|error| fail(&format!("cannot resolve {}: {error}\n", binary.display())))
}
/// Run the gungraun benches against `smith` and collect every bench's
/// `Ir` from the JSON summary lines on standard output.
fn instruction_counts(
smith: &Path,
passthrough: &[OsString],
) -> Result<BTreeMap<String, u64>, String> {
let mut command = Command::new("cargo");
command.args(["bench", "--locked", "-p", "smith-bench"]);
for bench in INSTRUCTION_BENCHES {
command.args(["--bench", bench]);
}
command
// ASLR stays on: containers refuse the personality switch, and
// cache simulation is off, so instruction counts are unaffected.
.args([
"--",
"--save-summary=json",
"--output-format=json",
"--allow-aslr=true",
])
.args(passthrough)
.env("SMITH_BIN", smith)
.stderr(std::process::Stdio::inherit());
let output = command
.output()
.map_err(|error| format!("failed to run cargo bench: {error}\n"))?;
if !output.status.success() {
// Exit code 3 is gungraun's own regression verdict from limits
// passed through; anything else is a build or runner failure.
return Err(format!("cargo bench exited with {}\n", output.status));
}
let stdout = String::from_utf8_lossy(&output.stdout);
let mut measured = BTreeMap::new();
for line in stdout.lines().filter(|line| line.starts_with('{')) {
let Some((bench, ir)) = bench_ir(line) else {
return Err(format!("unrecognized gungraun summary line: {line}\n"));
};
measured.insert(bench, ir);
}
if measured.is_empty() {
return Err("gungraun reported no benches\n".to_string());
}
Ok(measured)
}
/// `(<module_path>::<id>, Ir)` of one gungraun `--output-format=json`
/// line (summary schema v6). `module_path` and `id` are top-level strings;
/// the total `Ir` is the first integer under `total` → `Ir` → `metrics`,
/// which reads `{"Left":{"Int":n}}` on a first run and
/// `{"Both":[{"Int":n},{"Int":previous}]}` once a previous run exists.
fn bench_ir(line: &str) -> Option<(String, u64)> {
let module = json_string(line, "module_path")?;
let id = json_string(line, "id")?;
let after = |text: &'_ str, anchor: &str| -> Option<usize> {
text.find(anchor).map(|at| at + anchor.len())
};
let total = &line[after(line, "\"total\":")?..];
let ir = &total[after(total, "\"Ir\":")?..];
let metrics = &ir[after(ir, "\"metrics\":")?..];
let int = &metrics[after(metrics, "{\"Int\":")?..];
let digits = &int[..int.find(|c: char| !c.is_ascii_digit())?];
Some((format!("{module}::{id}"), digits.parse().ok()?))
}
/// The first string value of `key`; bench names carry no escapes.
fn json_string<'a>(line: &'a str, key: &str) -> Option<&'a str> {
let anchor = format!("\"{key}\":\"");
let rest = &line[line.find(&anchor)? + anchor.len()..];
rest.find('"').map(|end| &rest[..end])
}
/// Pins of `budgets.toml`: a `[budgets]` table of
/// `"<file>::<group>::<function>::<id>" = { ir = <n> }` lines, the shape
/// `--pin` writes; any other line is rejected rather than guessed at.
fn parse_budgets(text: &str) -> Result<BTreeMap<String, u64>, String> {
let mut pins = BTreeMap::new();
let mut in_table = false;
for (number, line) in text.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
if line == "[budgets]" {
in_table = true;
continue;
}
let Some((bench, ir)) = in_table.then(|| budget_line(line)).flatten() else {
return Err(format!(
"{BUDGETS}:{}: expected `\"<bench>\" = {{ ir = <n> }}` under [budgets]",
number + 1
));
};
if pins.insert(bench.to_string(), ir).is_some() {
return Err(format!("{BUDGETS}:{}: {bench} pinned twice", number + 1));
}
}
Ok(pins)
}
fn budget_line(line: &str) -> Option<(&str, u64)> {
let (bench, rest) = line.strip_prefix('"')?.split_once('"')?;
let table = rest.trim_start().strip_prefix('=')?.trim();
let fields = table.strip_prefix('{')?.strip_suffix('}')?.trim();
let ir = fields
.strip_prefix("ir")?
.trim_start()
.strip_prefix('=')?
.trim();
Some((bench, ir.parse().ok()?))
}
/// The platform named in a budgets file header, if any.
fn pinned_platform(text: &str) -> Option<String> {
text.lines()
.find_map(|line| line.trim().strip_prefix("# Pinned with "))
.map(|rest| rest.trim_end_matches('.').to_string())
}
fn render_budgets(pins: &BTreeMap<String, u64>, platform: &str) -> String {
use std::fmt::Write as _;
let mut out = format!(
"# Callgrind instruction budgets (Ir) per bench, gated within ±{BAND_PERCENT} % by `cargo x perf`.\n\
# Written by `cargo x perf --pin`; a re-pin is a deliberate commit that says why.\n\
# Pinned with {platform}.\n\n[budgets]\n"
);
for (bench, ir) in pins {
let _ = writeln!(out, "\"{bench}\" = {{ ir = {ir} }}");
}
out
}
/// One line per bench outside its band, without a pin, or, when every
/// bench ran, pinned but no longer measured.
fn budget_violations(
measured: &BTreeMap<String, u64>,
pins: &BTreeMap<String, u64>,
complete: bool,
) -> Vec<String> {
let mut violations = Vec::new();
for (bench, ir) in measured {
match pins.get(bench) {
None => violations.push(format!(" {bench}: measured {ir}, no pin\n")),
Some(pin) if !within_band(*pin, *ir) => violations.push(format!(
" {bench}: pin {pin}, measured {ir}, delta {} (band ±{BAND_PERCENT} %)\n",
delta(*pin, *ir)
)),
Some(_) => {}
}
}
if complete {
for (bench, pin) in pins {
if !measured.contains_key(bench) {
violations.push(format!(" {bench}: pin {pin}, not measured (stale pin)\n"));
}
}
}
violations
}
fn within_band(pin: u64, ir: u64) -> bool {
u128::from(ir.abs_diff(pin)) * 100 <= u128::from(pin) * u128::from(BAND_PERCENT)
}
/// Signed change of `ir` against `pin` in percent, two decimals.
fn delta(pin: u64, ir: u64) -> String {
if pin == 0 {
return "n/a".to_string();
}
let basis_points = (i128::from(ir) - i128::from(pin)) * 10_000 / i128::from(pin);
let sign = if basis_points < 0 { '-' } else { '+' };
let magnitude = basis_points.unsigned_abs();
format!("{sign}{}.{:02} %", magnitude / 100, magnitude % 100)
}
/// `target/report/perf.md`: every measured bench against its pin, with the
/// toolchain the counts belong to.
fn write_perf_report(
measured: &BTreeMap<String, u64>,
pins: &BTreeMap<String, u64>,
complete: bool,
platform: &str,
) -> ExitCode {
use std::fmt::Write as _;
let mut table = format!(
"Callgrind instruction counts (`Ir`) against `{BUDGETS}`, band ±{BAND_PERCENT} %.\n\n\
Platform: `{platform}`.\n\n| bench | ir | pin | delta |\n|---|---|---|---|\n"
);
for (bench, ir) in measured {
let (pin, change) = pins.get(bench).map_or_else(
|| ("none".to_string(), "n/a".to_string()),
|pin| (pin.to_string(), delta(*pin, *ir)),
);
let _ = writeln!(table, "| `{bench}` | {ir} | {pin} | {change} |");
}
if complete {
for (bench, pin) in pins
.iter()
.filter(|(bench, _)| !measured.contains_key(*bench))
{
let _ = writeln!(table, "| `{bench}` | not measured | {pin} | n/a |");
}
}
let out = target_dir().join("report");
if fs::create_dir_all(&out).is_err() || fs::write(out.join("perf.md"), table).is_err() {
return fail("cannot write target/report/perf.md\n");
}
ExitCode::SUCCESS
}
/// What instruction counts depend on: the toolchain building the benches
/// and the libc they link, e.g. `rustc 1.98.1 (…) | glibc 2.41`.
fn platform() -> String {
format!("{} | {}", tool_first_line("rustc"), libc_version())
}
fn tool_first_line(program: &str) -> String {
Command::new(program)
.arg("--version")
.output()
.ok()
.filter(|output| output.status.success())
.and_then(|output| {
String::from_utf8_lossy(&output.stdout)
.lines()
.next()
.map(str::to_string)
})
.unwrap_or_else(|| format!("{program} version unavailable"))
}
/// `glibc <version>` from `ldd --version`, whose first line ends with the
/// version on every glibc distribution; other libcs report as unknown.
fn libc_version() -> String {
let line = tool_first_line("ldd");
line.rsplit(' ')
.next()
.filter(|last| last.starts_with(|c: char| c.is_ascii_digit()))
.map_or_else(
|| "libc unknown".to_string(),
|version| format!("glibc {version}"),
)
}
/// `extra` without one leading `--`: `cargo x cmd -- args` and
/// `cargo x cmd args` pass the same arguments on.
fn without_separator(extra: &[OsString]) -> &[OsString] {
match extra.split_first() {
Some((first, rest)) if first == "--" => rest,
_ => extra,
}
}
fn cargo_fmt_step(extra: &[OsString]) -> ExitCode {
let args = scoped(&["fmt", "--check"], extra);
run_os("cargo", &args)
}
fn cargo_clippy(extra: &[OsString]) -> ExitCode {
sequence(&[clippy_prod, clippy_tests, clippy_guests], extra)
}
/// Production targets carry the full strict gate.
fn clippy_prod(extra: &[OsString]) -> ExitCode {
clippy_phase(&prod_targets(extra), &[], extra)
}
/// `--lib` and `--bins` as the scoped package actually has them; cargo
/// rejects a target flag the package cannot satisfy.
fn prod_targets(extra: &[OsString]) -> Vec<&'static str> {
let package = extra
.iter()
.position(|arg| arg == "-p" || arg == "--package")
.and_then(|at| extra.get(at + 1))
.map(|name| name.to_string_lossy().into_owned());
let Some(package) = package else {
return vec!["--lib", "--bins"];
};
let src = Path::new(&package).join("src");
let mut targets = Vec::new();
if src.join("lib.rs").is_file() {
targets.push("--lib");
}
if src.join("main.rs").is_file() || Path::new(&package).join("src/bin").is_dir() {
targets.push("--bins");
}
if targets.is_empty() {
targets = vec!["--lib", "--bins"];
}
targets
}
/// Test and bench targets may assert, panic, and print: outcomes are the
/// point. Pedantic relaxes too; correctness lints stay on so test quality
/// is still policed.
fn clippy_tests(extra: &[OsString]) -> ExitCode {
clippy_phase(
&["--tests", "--benches"],
&[
"-A",
"clippy::unwrap_used",
"-A",
"clippy::expect_used",
"-A",
"clippy::panic",
"-A",
"clippy::todo",
"-A",
"clippy::unimplemented",
"-A",
"clippy::print_stdout",
"-A",
"clippy::print_stderr",
"-A",
"clippy::dbg_macro",
"-A",
"clippy::pedantic",
],
extra,
)
}
/// Guest crates stay strict, cross-checked on their wasm target; the
/// crate-level expectations on generated surfaces carry the exceptions.
fn clippy_guests(extra: &[OsString]) -> ExitCode {
if extra.iter().any(|arg| arg == "-p" || arg == "--package") {
return ExitCode::SUCCESS;
}
clippy_phase(
&[
"-p",
"smith-plugin-echo",
"--target",
"wasm32-wasip2",
"--lib",
],
&[],
extra,
)
}
fn clippy_phase(targets: &[&str], lint_args: &[&str], extra: &[OsString]) -> ExitCode {
let mut base = vec!["clippy", "--locked"];
base.extend_from_slice(targets);
base.push("--all-features");
let args = scoped(&base, extra);
let mut command = Command::new("cargo");
command.env("CLIPPY_CONF_DIR", ".config").args(&args);
if !lint_args.is_empty() {
command.arg("--");
command.args(lint_args);
}
match command.status() {
Ok(status) if status.success() => ExitCode::SUCCESS,
Ok(status) => fail(&format!("cargo clippy exited with {status}\n")),
Err(error) => fail(&format!("failed to run cargo clippy: {error}\n")),
}
}
fn cargo_nextest(extra: &[OsString]) -> ExitCode {
let args = scoped(&["--locked", "--workspace", "--all-features"], extra);
nextest_run_os(&args)
}
/// End-to-end tier: the built binary through cli, eval, and rpc; test
/// binaries named `e2e_*`, selected by the `e2e` nextest profile.
fn cargo_e2e(extra: &[OsString]) -> ExitCode {
let args = scoped(
&[
"--locked",
"--workspace",
"--all-features",
"--profile",
"e2e",
],
extra,
);
nextest_run_os(&args)
}
/// Coverage over every tier; informational, never a percentage gate.
/// Writes `target/coverage/lcov.info` and prints the per-file summary.
fn cargo_coverage(extra: &[OsString]) -> ExitCode {
let ran = coverage_tiers(extra);
if ran != ExitCode::SUCCESS {
return ran;
}
let lcov = target_dir().join("coverage").join("lcov.info");
let wrote = coverage_report(&["--lcov", "--output-path"], &lcov);
if wrote != ExitCode::SUCCESS {
return wrote;
}
run_os(
"cargo",
&["llvm-cov", "report", "--summary-only"].map(OsString::from),
)
}
/// Run both nextest tiers instrumented, accumulating one profile set.
fn coverage_tiers(extra: &[OsString]) -> ExitCode {
let installed = install_llvm_cov(&[]);
if installed != ExitCode::SUCCESS {
return installed;
}
if std::fs::create_dir_all(target_dir().join("coverage")).is_err() {
return fail("cannot create target/coverage\n");
}
let tiers: [&[&str]; 2] = [&[], &["--profile", "e2e"]];
let base = [
"llvm-cov",
"nextest",
"--no-report",
"--locked",
"--workspace",
"--all-features",
];
let clean = run_os(
"cargo",
&["llvm-cov", "clean", "--workspace"].map(OsString::from),
);
if clean != ExitCode::SUCCESS {
return clean;
}
for tier in tiers {
let mut args: Vec<&str> = base.to_vec();
args.extend_from_slice(tier);
// A tier with no matching tests is a report, not a failure.
args.push("--no-tests=warn");
let ran = run_os("cargo", &scoped(&args, extra));
if ran != ExitCode::SUCCESS {
return ran;
}
}
ExitCode::SUCCESS
}
/// One `cargo llvm-cov report` invocation with an output path.
fn coverage_report(flags: &[&str], output: &Path) -> ExitCode {
let mut args: Vec<OsString> = ["llvm-cov", "report"].map(OsString::from).to_vec();
args.extend(flags.iter().map(OsString::from));
args.push(output.as_os_str().to_owned());
run_os("cargo", &args)
}
/// Every tier's evidence in one bundle under `target/report/`: `JUnit` per
/// tier, lcov plus HTML coverage, the last mutation run, the last
/// instruction-budget table, hyperfine wall-clock rows, and an `index.md`
/// that summarizes them. Mutation and perf results are read, never run
/// here; `mutants` and `perf` run first. Passthrough scopes the test tiers.
fn report(extra: &[OsString]) -> ExitCode {
use std::fmt::Write as _;
let ran = coverage_tiers(extra);
if ran != ExitCode::SUCCESS {
return ran;
}
let out = target_dir().join("report");
// `cargo x perf` writes into the bundle directory; keep its table
// across the reset.
let perf = fs::read_to_string(out.join("perf.md")).ok();
let _ = std::fs::remove_dir_all(&out);
if std::fs::create_dir_all(out.join("junit")).is_err() {
return fail("cannot create target/report\n");
}
let lcov = coverage_report(&["--lcov", "--output-path"], &out.join("lcov.info"));
if lcov != ExitCode::SUCCESS {
return lcov;
}
let html = coverage_report(&["--html", "--output-dir"], &out.join("coverage"));
if html != ExitCode::SUCCESS {
return html;
}
let summary = Command::new("cargo")
.args(["llvm-cov", "report", "--summary-only"])
.output()
.map_or_else(
|error| format!("summary unavailable: {error}"),
|output| String::from_utf8_lossy(&output.stdout).into_owned(),
);
let mut index = String::from("# Smith test report\n\n");
index.push_str("## Tiers\n\n| tier | tests | failures | errors |\n|---|---|---|---|\n");
for tier in ["default", "e2e"] {
// nextest keeps its store under the workspace target dir even when
// llvm-cov redirects build output.
let source = target_dir().join("nextest").join(tier).join("junit.xml");
let copied = out.join("junit").join(format!("{tier}.xml"));
let row = std::fs::read_to_string(&source).map_or_else(
|_| format!("| {tier} | no junit.xml | | |\n"),
|xml| {
let _ = std::fs::write(&copied, &xml);
let [tests, failures, errors] = junit_counts(&xml);
format!("| {tier} | {tests} | {failures} | {errors} |\n")
},
);
index.push_str(&row);
}
index.push_str(
"\n## Coverage\n\nHTML: `coverage/html/index.html`; lcov: `lcov.info`.\n\n```text\n",
);
index.push_str(summary.trim_end());
index.push_str("\n```\n\n## Mutation\n\n");
let mutants = target_dir().join("mutants.out");
match mutants_counts(&mutants) {
Some([caught, missed, timeout, unviable]) => {
let _ = write!(
index,
"| caught | missed | timeout | unviable |\n|---|---|---|---|\n| {caught} | {missed} | {timeout} | {unviable} |\n"
);
for name in ["missed.txt", "caught.txt", "timeout.txt", "unviable.txt"] {
let _ = std::fs::copy(mutants.join(name), out.join(name));
}
if missed > 0 {
index.push_str("\nMissed mutants are the test backlog; see `missed.txt`.\n");
}
}
None => index.push_str("No mutation run found; run `cargo x mutants` first.\n"),
}
index.push_str("\n## Instructions\n\n");
match perf {
Some(table) => {
let _ = std::fs::write(out.join("perf.md"), &table);
index.push_str(&table);
}
None => index.push_str("No perf run found; run `cargo x perf` first.\n"),
}
index.push_str("\n## Wall clock\n\n");
index.push_str(&wall_clock(&out));
if std::fs::write(out.join("index.md"), index).is_err() {
return fail("cannot write target/report/index.md\n");
}
let _ignored = std::io::stdout().write_all(b"report: target/report/index.md\n");
ExitCode::SUCCESS
}
/// Wall-clock rows of the release binary through hyperfine, isolated like
/// the e2e tests. Informational only: the numbers hold for this machine
/// and this run, so a missing tool or a failed run is a note, never a
/// report failure.
fn wall_clock(out: &Path) -> String {
let present = Command::new("hyperfine")
.arg("--version")
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.is_ok_and(|status| status.success());
if !present {
return "hyperfine not on PATH; no wall-clock rows.\n".to_string();
}
let Ok(smith) = release_smith() else {
return "release build of smith failed; no wall-clock rows.\n".to_string();
};
let home = target_dir().join("wall-clock-home");
let markdown = out.join("hyperfine.md");
let mut command = Command::new("hyperfine");
command
.args(["--warmup", "3", "--runs", "20", "--export-json"])
.arg(out.join("hyperfine.json"))
.arg("--export-markdown")
.arg(&markdown)
// A fresh profile per run: `eval` appends to its session otherwise.
.arg("--prepare")
.arg(format!("rm -rf '{}'", home.display()))
.args(["--command-name", "smith --help"])
.args(["--command-name", "smith eval --mock hello"])
.arg(format!("'{}' --help", smith.display()))
.arg(format!("'{}' eval --mock hello", smith.display()));
for (key, dir) in dev_home_dirs() {
command.env(key, home.join(dir));
}
for var in PROVIDER_VARS {
command.env_remove(var);
}
let ran = command.status().is_ok_and(|status| status.success());
match fs::read_to_string(&markdown) {
Ok(rows) if ran => format!(
"Informational, never a gate: {}-{}, {} hardware threads, this run only; raw data in `hyperfine.json`.\n\n{rows}",
env::consts::OS,
env::consts::ARCH,
std::thread::available_parallelism().map_or(0, std::num::NonZero::get),
),
_ => "hyperfine run failed; no wall-clock rows.\n".to_string(),
}
}
/// `tests`, `failures`, `errors` from a `JUnit` `<testsuites>` root.
fn junit_counts(xml: &str) -> [u64; 3] {
let root = xml
.find("<testsuites")
.map(|at| &xml[at..])
.and_then(|rest| rest.find('>').map(|end| &rest[..end]))
.unwrap_or("");
["tests", "failures", "errors"].map(|attribute| {
let key = format!(" {attribute}=\"");
root.find(&key)
.map(|at| &root[at + key.len()..])
.and_then(|rest| rest.split('"').next())
.and_then(|value| value.parse().ok())
.unwrap_or(0)
})
}
/// Line counts of a cargo-mutants outcome directory: caught, missed,
/// timeout, unviable; `None` without a run.
fn mutants_counts(dir: &Path) -> Option<[usize; 4]> {
let count = |name: &str| {
std::fs::read_to_string(dir.join(name))
.map(|text| text.lines().filter(|line| !line.trim().is_empty()).count())
};
Some([
count("caught.txt").ok()?,
count("missed.txt").ok()?,
count("timeout.txt").unwrap_or(0),
count("unviable.txt").unwrap_or(0),
])
}
/// nextest has no warning-deny flag, so config-drift warnings are turned into
/// a gate failure by scanning its stderr.
fn nextest_run_os(args: &[OsString]) -> ExitCode {
let mut child = match Command::new("cargo")
.arg("nextest")
.arg("run")
.args(args)
.stdout(std::process::Stdio::inherit())
.stderr(std::process::Stdio::piped())
.spawn()
{
Ok(child) => child,
Err(error) => return fail(&format!("failed to run cargo nextest: {error}\n")),
};
let mut warned = false;
if let Some(stderr) = child.stderr.take() {
let reader = std::io::BufReader::new(stderr);
for line in reader.lines() {
let Ok(line) = line else {
break;
};
if line.contains("warning: in config file") {
warned = true;
}
let _ = std::io::stderr().write_all(format!("{line}\n").as_bytes());
}
}
match child.wait() {
Ok(status) if status.success() && !warned => ExitCode::SUCCESS,
Ok(status) if status.success() => fail("nextest produced config file warnings\n"),
Ok(status) => fail(&format!(
"cargo exited with exit status: {}\n",
status.code().unwrap_or_default()
)),
Err(error) => fail(&format!("failed to wait for cargo nextest: {error}\n")),
}
}
fn cargo_machete(_extra: &[OsString]) -> ExitCode {
let installed = tool_present_bin("cargo-machete", MACHETE_VERSION, || {
cargo(&[
"install",
"--locked",
"--version",
MACHETE_VERSION,
"cargo-machete",
])
});
if installed != ExitCode::SUCCESS {
return installed;
}
run("cargo-machete", &[])
}
/// Install the external gate tools that Cargo cannot ship.
///
/// `cargo-nextest` arrives as a prebuilt binary through `cargo-binstall`.
/// The step is idempotent.
/// Installs tools; scoping does not apply.
fn bootstrap(extra: &[OsString]) -> ExitCode {
if !extra.is_empty() {
return fail("bootstrap installs tools and admits no scoping\n");
}
let steps: [Step; 8] = [
install_binstall,
install_nextest,
install_machete,
install_deny,
install_audit,
install_mutants,
install_llvm_cov,
install_gungraun_runner,
];
sequence(&steps, extra)
}
fn install_machete(_extra: &[OsString]) -> ExitCode {
tool_present_bin("cargo-machete", MACHETE_VERSION, || {
cargo(&[
"install",
"--locked",
"--version",
MACHETE_VERSION,
"cargo-machete",
])
})
}
fn install_deny(_extra: &[OsString]) -> ExitCode {
binstall("cargo-deny", DENY_VERSION)
}
fn install_audit(_extra: &[OsString]) -> ExitCode {
binstall("cargo-audit", AUDIT_VERSION)
}
fn install_mutants(_extra: &[OsString]) -> ExitCode {
binstall("cargo-mutants", MUTANTS_VERSION)
}
fn install_gungraun_runner(_extra: &[OsString]) -> ExitCode {
binstall("gungraun-runner", GUNGRAUN_VERSION)
}
fn install_llvm_cov(_extra: &[OsString]) -> ExitCode {
let tool = binstall("cargo-llvm-cov", LLVM_COV_VERSION);
if tool != ExitCode::SUCCESS {
return tool;
}
run_os(
"rustup",
&["component", "add", "llvm-tools"].map(OsString::from),
)
}
fn install_binstall(_extra: &[OsString]) -> ExitCode {
tool_present_bin("cargo-binstall", BINSTALL_VERSION, || {
cargo(&[
"install",
"cargo-binstall",
"--version",
BINSTALL_VERSION,
// Freeze binstall's own dependency graph for the same reason
// crates.io drift cannot break the gate.
"--locked",
])
})
}
fn install_nextest(_extra: &[OsString]) -> ExitCode {
binstall("cargo-nextest", NEXTEST_VERSION)
}
/// Install the prebuilt `tool` at `version` through `cargo-binstall`
/// unless that version is already on PATH.
fn binstall(tool: &str, version: &str) -> ExitCode {
tool_present_bin(tool, version, || {
cargo(&["binstall", "--no-confirm", &format!("{tool}@{version}")])
})
}
/// Run `probe` with cargo; when it fails, run `install` and return its code.
/// The probe pins the tool so a stale unpinned binary is replaced.
/// Probes an external binary directly and enforces the pinned version;
/// mismatch or absence runs the installer, so drift self-heals to the pin.
fn tool_present_bin(program: &str, version: &str, install: impl FnOnce() -> ExitCode) -> ExitCode {
let probe = Command::new(program)
.arg("--version")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
.output();
let pinned = probe.is_ok_and(|output| {
output.status.success() && String::from_utf8_lossy(&output.stdout).contains(version)
});
if pinned { ExitCode::SUCCESS } else { install() }
}
fn cargo_doc(extra: &[OsString]) -> ExitCode {
let args = scoped(&["doc", "--locked", "--workspace", "--no-deps"], extra);
run_os("cargo", &args)
}
fn cargo_deny(_extra: &[OsString]) -> ExitCode {
cargo(&["deny", "--config", ".config/deny.toml", "check"])
}
fn cargo_audit(_extra: &[OsString]) -> ExitCode {
cargo(&["audit", "--deny", "warnings"])
}
fn cargo(args: &[&str]) -> ExitCode {
run("cargo", args)
}
/// Builds the CLI, then launches it against an isolated dev home.
fn dev(args: &[OsString]) -> ExitCode {
let code = cargo(&["build", "--locked", "-p", "smith-cli", "--bin", "smith"]);
if code != ExitCode::SUCCESS {
return code;
}
let target = target_dir();
let binary = target.join("debug").join(binary_name());
if !binary.is_file() {
return fail(&format!("missing dev binary: {}\n", binary.display()));
}
let home = target.join("dev-home");
let mut command = Command::new(&binary);
for (key, dir) in dev_home_dirs() {
let path = home.join(dir);
if let Err(error) = fs::create_dir_all(&path) {
return fail(&format!("failed to create {}: {error}\n", path.display()));
}
command.env(key, &path);
}
command.args(args);
note(&format!(
"running {} (home {})\n",
binary.display(),
home.display()
));
match command.status() {
Ok(status) => match status.code() {
Some(0) => ExitCode::SUCCESS,
Some(value) => ExitCode::from(u8::try_from(value).unwrap_or(1)),
None => fail("smith terminated by signal\n"),
},
Err(error) => fail(&format!("failed to run {}: {error}\n", binary.display())),
}
}
/// Environment overrides that keep dev runs out of the real user profile.
const fn dev_home_dirs() -> &'static [(&'static str, &'static str)] {
&[
("XDG_CONFIG_HOME", "config"),
("XDG_DATA_HOME", "data"),
("XDG_STATE_HOME", "state"),
("XDG_CACHE_HOME", "cache"),
]
}
fn target_dir() -> PathBuf {
env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from)
}
const fn binary_name() -> &'static str {
if cfg!(windows) { "smith.exe" } else { "smith" }
}
fn note(message: &str) {
let _ignored = std::io::stderr().write_all(message.as_bytes());
}
fn run_os(program: &str, args: &[OsString]) -> ExitCode {
match Command::new(program).args(args).status() {
Ok(status) if status.success() => ExitCode::SUCCESS,
Ok(status) => fail(&format!("{program} exited with {status}\n")),
Err(error) => fail(&format!("failed to run {program}: {error}\n")),
}
}
fn run(program: &str, args: &[&str]) -> ExitCode {
match Command::new(program).args(args).status() {
Ok(status) if status.success() => ExitCode::SUCCESS,
Ok(status) => fail(&format!("{program} exited with {status}\n")),
Err(error) => fail(&format!("failed to run {program}: {error}\n")),
}
}
fn arch(extra: &[OsString]) -> ExitCode {
let wanted: Vec<String> = extra
.iter()
.map(|arg| arg.to_string_lossy().into_owned())
.collect();
for crate_name in CRATES {
if !Path::new(crate_name).join("Cargo.toml").is_file() {
return fail(&format!("missing crate manifest: {crate_name}\n"));
}
}
for rule in DEPENDENCY_RULES {
if !wanted.is_empty() && !wanted.iter().any(|name| name == rule.crate_name) {
continue;
}
let manifest = match fs::read_to_string(Path::new(rule.crate_name).join("Cargo.toml")) {
Ok(value) => value,
Err(error) => {
return fail(&format!(
"failed to read {} Cargo.toml: {error}\n",
rule.crate_name
));
}
};
let denied = CRATES
.iter()
.filter(|name| **name != rule.crate_name && !rule.allowed.contains(name));
for denied in denied {
if has_dependency(&manifest, denied) {
return fail(&format!(
"{} must not depend on {denied}\n",
rule.crate_name
));
}
}
if rule.crate_name != "smith-bench" && names_dependency(&manifest, "smith-bench") {
return fail(&format!(
"{} must not depend on smith-bench, not even from tests\n",
rule.crate_name
));
}
if LIBRARY_CRATES.contains(&rule.crate_name)
&& let Some(offender) = mutex_in_production(Path::new(rule.crate_name))
{
return fail(&format!(
"{}: Mutex in library code (concurrency law): {}\n",
rule.crate_name,
offender.display()
));
}
}
if let Some(offender) = missing_spec_reference() {
return fail(&format!(
"{}: test code without an SMH-SPEC- citation in its first 20 lines or `//!` header (RULES.md Testing)\n",
offender.display()
));
}
ExitCode::SUCCESS
}
/// The first `.rs` file under any workspace crate that carries test code
/// (`#[test]` or a `tests/` directory) but cites no `SMH-SPEC-…` id in its
/// first 20 lines or its `//!` header block.
fn missing_spec_reference() -> Option<PathBuf> {
for crate_name in CRATES {
let mut files = Vec::new();
collect_rs_files(Path::new(crate_name), &mut files);
for path in files {
let Ok(source) = fs::read_to_string(&path) else {
continue;
};
if is_test_bearing(&path, &source) && !cites_a_spec(&source) {
return Some(path);
}
}
}
None
}
/// `.rs` files under `dir`, deterministically ordered, skipping `target`.
fn collect_rs_files(dir: &Path, out: &mut Vec<PathBuf>) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
let mut entries: Vec<PathBuf> = entries.flatten().map(|entry| entry.path()).collect();
entries.sort();
for path in entries {
if path.is_dir() {
if path.file_name().is_some_and(|name| name == "target") {
continue;
}
collect_rs_files(&path, out);
} else if path.extension().is_some_and(|ext| ext == "rs") {
out.push(path);
}
}
}
/// A file carries test code when it literally declares `#[test]` or lives
/// under a `tests/` directory (integration test crates have no attribute).
fn is_test_bearing(path: &Path, source: &str) -> bool {
source.contains("#[test]") || path.components().any(|part| part.as_os_str() == "tests")
}
/// Whether `SMH-SPEC-` appears in the first 20 lines or the leading `//!`
/// doc-comment block, whichever the file has.
fn cites_a_spec(source: &str) -> bool {
let first_twenty: Vec<&str> = source.lines().take(20).collect();
first_twenty.iter().any(|line| line.contains("SMH-SPEC-"))
|| leading_doc_block(source).contains("SMH-SPEC-")
}
/// The contiguous run of `//!` lines starting the file, attributes and
/// blank lines before it skipped, concatenated.
fn leading_doc_block(source: &str) -> String {
let mut block = String::new();
for line in source.lines() {
let trimmed = line.trim_start();
if trimmed.starts_with("//!") {
block.push_str(trimmed);
block.push('\n');
} else if !block.is_empty() {
break;
}
}
block
}
/// Crates under the "no Mutex in library crates" law; binaries are exempt.
const LIBRARY_CRATES: &[&str] = &[
"smith",
"smith-core",
"smith-ai",
"smith-ui",
"smith-tui",
"smith-rpc",
"smith-harness",
"smith-alloc",
];
/// The first `.rs` file under `crate_dir/src` whose production slice
/// (everything before `#[cfg(test)]`) mentions `Mutex`.
fn mutex_in_production(crate_dir: &Path) -> Option<PathBuf> {
let mut pending = vec![crate_dir.join("src")];
while let Some(dir) = pending.pop() {
let Ok(entries) = fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
pending.push(path);
} else if path.extension().is_some_and(|ext| ext == "rs")
&& fs::read_to_string(&path)
.is_ok_and(|source| production_slice(&source).contains("Mutex"))
{
return Some(path);
}
}
}
None
}
/// Source text before the first `#[cfg(test)]`; tests may lock freely.
fn production_slice(source: &str) -> &str {
source.split("#[cfg(test)]").next().unwrap_or(source)
}
/// Library edges a crate may have; every other workspace crate is denied.
struct DependencyRule {
crate_name: &'static str,
allowed: &'static [&'static str],
}
/// The allowed-deps table of `.system/RULES.md`, architecture section.
const DEPENDENCY_RULES: &[DependencyRule] = &[
DependencyRule {
crate_name: "smith",
allowed: &[],
},
DependencyRule {
crate_name: "smith-ui",
allowed: &["smith"],
},
DependencyRule {
crate_name: "smith-core",
allowed: &["smith"],
},
DependencyRule {
crate_name: "smith-ai",
allowed: &["smith"],
},
DependencyRule {
crate_name: "smith-tui",
allowed: &["smith", "smith-ui"],
},
DependencyRule {
crate_name: "smith-rpc",
allowed: &["smith", "smith-core"],
},
DependencyRule {
crate_name: "smith-harness",
allowed: &["smith", "smith-core", "smith-ai", "smith-tui", "smith-ui"],
},
DependencyRule {
crate_name: "smith-cli",
allowed: &["smith", "smith-harness", "smith-rpc", "smith-alloc"],
},
DependencyRule {
crate_name: "smith-alloc",
allowed: &[],
},
DependencyRule {
crate_name: "smith-bench",
allowed: &[
"smith",
"smith-core",
"smith-ai",
"smith-ui",
"smith-tui",
"smith-rpc",
"smith-harness",
"smith-cli",
"smith-alloc",
"xtask",
],
},
DependencyRule {
crate_name: "xtask",
allowed: &[],
},
];
fn has_dependency(manifest: &str, crate_name: &str) -> bool {
// Library edges only: the manifest up to the dev-dependency table.
let library = manifest
.split("[dev-dependencies]")
.next()
.unwrap_or(manifest);
let table = format!("[dependencies.{crate_name}]");
let assignment = format!("{crate_name} =");
library.lines().any(|line| {
let trimmed = line.trim_start();
trimmed.starts_with(&table) || trimmed.starts_with(&assignment)
})
}
/// Whether any dependency table of `manifest`, dev and target tables
/// included, names `crate_name`.
fn names_dependency(manifest: &str, crate_name: &str) -> bool {
let assignment = format!("{crate_name} =");
let table = format!(".{crate_name}]");
manifest.lines().any(|line| {
let trimmed = line.trim();
trimmed.starts_with(&assignment) || (trimmed.starts_with('[') && trimmed.ends_with(&table))
})
}
fn fail(message: &str) -> ExitCode {
let _ignored = std::io::stderr().write_all(message.as_bytes());
ExitCode::FAILURE
}
#[cfg(test)]
mod tests {
#[test]
fn mutants_refuse_to_run_unscoped() {
let scoped = |args: &[&str]| {
let args: Vec<std::ffi::OsString> = args.iter().map(std::ffi::OsString::from).collect();
super::mutants_scoped(&args)
};
assert!(scoped(&["-f", "smith-core/src/session.rs"]));
assert!(scoped(&["--shard=0/8"]));
assert!(scoped(&["-D", "/tmp/d.patch", "-j2"]));
assert!(scoped(&["--list"]));
assert!(!scoped(&[]));
assert!(!scoped(&["-j4", "--baseline", "run"]));
}
#[test]
fn local_mutants_reject_a_job_count() {
let flag = |arg: &str| super::jobs_flag(&std::ffi::OsString::from(arg));
assert!(flag("-j"));
assert!(flag("-j4"));
assert!(flag("--jobs"));
assert!(flag("--jobs=2"));
assert!(!flag("-jx"));
assert!(!flag("--shard"));
}
#[test]
fn junit_counts_read_the_testsuites_root() {
let xml = r#"<?xml version="1.0"?><testsuites name="nextest-run" tests="19" failures="1" errors="0"><testsuite tests="7"/></testsuites>"#;
assert_eq!(super::junit_counts(xml), [19, 1, 0]);
assert_eq!(super::junit_counts("<nothing/>"), [0, 0, 0]);
}
#[test]
fn mutants_counts_need_a_run() {
let dir = std::env::temp_dir().join(format!("smith_mutants_probe_{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(super::mutants_counts(&dir), None);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("caught.txt"), "a\nb\n").unwrap();
std::fs::write(dir.join("missed.txt"), "c\n").unwrap();
assert_eq!(super::mutants_counts(&dir), Some([2, 1, 0, 0]));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn production_slice_stops_at_the_test_module() {
let source = "use std::sync::Arc;\n#[cfg(test)]\nmod tests { use std::sync::Mutex; }";
assert!(!super::production_slice(source).contains("Mutex"));
assert!(super::production_slice("static M: Mutex<u8>;").contains("Mutex"));
}
use super::{cites_a_spec, is_test_bearing};
use std::path::Path;
#[test]
fn test_bearing_by_attribute_or_by_directory() {
assert!(is_test_bearing(
Path::new("smith-core/src/agent.rs"),
"mod tests {\n#[test]\nfn it_works() {}\n}"
));
assert!(is_test_bearing(
Path::new("smith-cli/tests/e2e_eval.rs"),
"fn helper() {}"
));
assert!(!is_test_bearing(
Path::new("smith-core/src/lib.rs"),
"fn helper() {}"
));
}
#[test]
fn spec_citation_in_first_twenty_lines_or_doc_block() {
assert!(cites_a_spec("//! docs (`SMH-SPEC-SPEC0001`, Tools).\n"));
let padded = format!("{}fn f() {{}}\n", "//! line\n".repeat(19));
assert!(!cites_a_spec(&padded));
let past_twenty = format!(
"{}//! cites `SMH-SPEC-SPEC0001` here\nfn f() {{}}\n",
"//! line\n".repeat(25)
);
assert!(cites_a_spec(&past_twenty));
assert!(!cites_a_spec("// no doc block\nfn f() {}\n"));
}
use super::{CRATES, DEPENDENCY_RULES, binary_name, dev_home_dirs, has_dependency, target_dir};
use std::path::PathBuf;
#[test]
fn every_crate_has_one_rule_over_known_crates() {
for name in CRATES {
let rules = DEPENDENCY_RULES
.iter()
.filter(|rule| rule.crate_name == *name);
assert_eq!(rules.count(), 1, "{name}");
}
assert_eq!(DEPENDENCY_RULES.len(), CRATES.len());
for rule in DEPENDENCY_RULES {
assert!(
rule.allowed
.iter()
.all(|dep| CRATES.contains(dep) && *dep != rule.crate_name)
);
}
}
#[test]
fn dev_home_covers_all_xdg_roots() {
let keys: Vec<&str> = dev_home_dirs().iter().map(|(key, _)| *key).collect();
assert_eq!(
keys,
vec![
"XDG_CONFIG_HOME",
"XDG_DATA_HOME",
"XDG_STATE_HOME",
"XDG_CACHE_HOME"
]
);
}
#[test]
fn dev_binary_path_is_platform_correct() {
let path = target_dir().join("debug").join(binary_name());
assert!(path.ends_with(if cfg!(windows) { "smith.exe" } else { "smith" }));
assert!(path.starts_with(env_target()));
}
fn env_target() -> PathBuf {
std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from)
}
use super::{
BTreeMap, bench_ir, budget_violations, delta, names_dependency, parse_budgets,
render_budgets, without_separator,
};
const FIRST_RUN: &str = r#"{"baselines":[null,null],"details":"(linear(1000))","function_name":"encode","id":"linear_1000","kind":"LibraryBenchmark","module_path":"instructions::session::encode","profiles":[{"summaries":{"parts":[{"details":{"Left":{"pid":1,"thread":1}},"metrics_summary":{"Callgrind":{"Ir":{"diffs":null,"metrics":{"Left":{"Int":999}}}}}}],"total":{"regressions":[],"summary":{"Callgrind":{"Ir":{"diffs":null,"metrics":{"Left":{"Int":5736448}}}}}}},"tool":"Callgrind"}],"version":"6"}"#;
#[test]
fn pinned_platform_reads_the_header() {
let text = "# x\n# Pinned with rustc 1.98.1 (a 2026) | glibc 2.41.\n[budgets]\n";
assert_eq!(
super::pinned_platform(text).as_deref(),
Some("rustc 1.98.1 (a 2026) | glibc 2.41")
);
assert_eq!(super::pinned_platform("[budgets]\n"), None);
}
#[test]
fn bench_ir_reads_the_total_of_first_and_repeated_runs() {
assert_eq!(
bench_ir(FIRST_RUN),
Some((
"instructions::session::encode::linear_1000".to_string(),
5_736_448
))
);
let repeated = FIRST_RUN.replace(
r#""total":{"regressions":[],"summary":{"Callgrind":{"Ir":{"diffs":null,"metrics":{"Left":{"Int":5736448}}}}}}"#,
r#""total":{"regressions":[],"summary":{"Callgrind":{"Ir":{"diffs":{"diff_pct":0.1},"metrics":{"Both":[{"Int":5736500},{"Int":5736448}]}}}}}"#,
);
assert_eq!(bench_ir(&repeated).map(|(_, ir)| ir), Some(5_736_500));
assert_eq!(bench_ir(r#"{"module_path":"a","id":"b"}"#), None);
}
#[test]
fn budgets_round_trip_sorted_and_reject_other_shapes() {
let pins = BTreeMap::from([
("b::g::f::x".to_string(), 7),
("a::g::f::y".to_string(), 42),
]);
let text = render_budgets(&pins, "rustc 1.0.0");
assert!(text.find("a::g::f::y").unwrap() < text.find("b::g::f::x").unwrap());
assert_eq!(parse_budgets(&text).unwrap(), pins);
assert!(
parse_budgets("\"a\" = { ir = 1 }\n").is_err(),
"outside [budgets]"
);
assert!(parse_budgets("[budgets]\n\"a\" = 1\n").is_err());
assert!(parse_budgets("[budgets]\n\"a\" = { ir = 1 }\n\"a\" = { ir = 2 }\n").is_err());
}
#[test]
fn violations_cover_both_directions_missing_and_stale_pins() {
let pins = BTreeMap::from([
("low".to_string(), 1000),
("edge".to_string(), 1000),
("high".to_string(), 1000),
("stale".to_string(), 5),
]);
let measured = BTreeMap::from([
("low".to_string(), 979),
("edge".to_string(), 1020),
("high".to_string(), 1021),
("new".to_string(), 3),
]);
let found = budget_violations(&measured, &pins, true).concat();
for bench in ["low:", "high:", "new:", "stale:"] {
assert!(found.contains(bench), "{bench} in {found}");
}
assert!(!found.contains("edge:"), "{found}");
let filtered = budget_violations(&measured, &pins, false).concat();
assert!(!filtered.contains("stale:"), "{filtered}");
}
#[test]
fn delta_keeps_the_sign_below_one_percent() {
assert_eq!(delta(1000, 995), "-0.50 %");
assert_eq!(delta(1000, 1025), "+2.50 %");
assert_eq!(delta(0, 5), "n/a");
}
#[test]
fn separator_is_stripped_once() {
let args = ["--", "--", "x"].map(std::ffi::OsString::from);
assert_eq!(without_separator(&args), &args[1..]);
assert_eq!(without_separator(&args[2..]), &args[2..]);
}
#[test]
fn names_dependency_sees_dev_and_target_tables() {
assert!(names_dependency(
"[dev-dependencies]\nsmith-bench = { path = \"..\" }",
"smith-bench"
));
assert!(names_dependency(
"[target.'cfg(unix)'.dev-dependencies.smith-bench]",
"smith-bench"
));
assert!(!names_dependency("name = \"smith-bench\"", "smith-bench"));
}
#[test]
fn detects_inline_and_table_dependencies() {
assert!(has_dependency(
"smith-core = { path = \"../smith-core\" }",
"smith-core"
));
assert!(has_dependency(
"[dependencies.smith-ai]\npath = \"../smith-ai\"",
"smith-ai"
));
}
#[test]
fn ignores_non_dependency_mentions() {
assert!(!has_dependency("# smith-core = forbidden", "smith-core"));
assert!(!has_dependency("smith_core = \"0.1\"", "smith-core"));
}
}