//! Staged nightly CI (`SMH-SPEC-SPEC0001`, testing acceptance). use std::{ffi::OsString, fs, path::Path, process::ExitCode}; const SHARDS: usize = 8; const LISTS: [&str; 4] = ["caught.txt", "missed.txt", "timeout.txt", "unviable.txt"]; /// Each Luci matrix child restores the same job's caches. Bootstrap clears /// evidence; shard finding codes defer failure; report publishes before verdict. pub(crate) fn run(extra: &[OsString]) -> ExitCode { let Some((phase, extra)) = crate::without_separator(extra).split_first() else { return crate::fail("nightly needs bootstrap, 0..7, report, or verdict\n"); }; let Some(phase) = phase.to_str() else { return crate::fail("nightly phase must be UTF-8\n"); }; execute(&crate::target_dir(), phase, extra, |step, args| step(args)) } fn execute( target: &Path, phase: &str, extra: &[OsString], mut call: impl FnMut(crate::Step, &[OsString]) -> ExitCode, ) -> ExitCode { let stages = target.join("nightly"); let summary = target.join("nightly-summary"); let output = stages.join(phase); let result = (|| -> std::io::Result { match phase { "bootstrap" => { if stages.exists() { fs::remove_dir_all(&stages)?; } if summary.exists() { fs::remove_dir_all(&summary)?; } for (step, args) in [ (crate::bootstrap as crate::Step, &[][..]), (crate::cargo_nextest, extra), (crate::ub, extra), ] { let status = call(step, args); if status != ExitCode::SUCCESS { return Ok(status); } } fs::create_dir_all(&output)?; fs::write( output.join("index.md"), "# Nightly bootstrap\n\nBaseline tests and Miri passed.\n", )?; } "report" => { aggregate(&stages, &summary)?; let status = call(report_stage, extra); if status != ExitCode::SUCCESS { return Ok(status); } let report = target.join("report"); for name in LISTS { fs::copy(summary.join(name), report.join(name))?; } if output.exists() { fs::remove_dir_all(&output)?; } fs::rename(report, &output)?; } "verdict" => { let report = stages.join("report"); // No report means no complete run, even if stale shard lists exist. if !report.join("index.md").is_file() || !LISTS.iter().all(|name| report.join(name).is_file()) { return Ok(crate::fail("nightly verdict requires the report stage\n")); } let Some([_, missed, timeout, _]) = crate::mutants_counts(&report) else { return Ok(crate::fail("nightly verdict requires mutation evidence\n")); }; fs::create_dir_all(&output)?; fs::write( output.join("index.md"), format!("# Nightly verdict\n\nMissed: {missed}; timeout: {timeout}.\n"), )?; if missed != 0 || timeout != 0 { return Ok(crate::fail( "nightly mutation findings; see the published report\n", )); } } _ => { let shard = phase .parse::() .ok() .filter(|&shard| shard < SHARDS && shard.to_string() == phase); let Some(shard) = shard else { return Ok(crate::fail("unknown nightly phase\n")); }; if output.exists() { fs::remove_dir_all(&output)?; } let mut args = Vec::with_capacity(extra.len() + 5); args.push(OsString::from("--all")); args.extend_from_slice(extra); args.extend([ OsString::from("--shard"), format!("{shard}/{SHARDS}").into(), OsString::from("--output"), output.into_os_string(), ]); let status = call(crate::cargo_mutants, &args); if ![ExitCode::SUCCESS, ExitCode::from(2), ExitCode::from(3)].contains(&status) { return Ok(status); } let evidence = stages.join(phase).join("mutants.out"); for name in LISTS.into_iter().chain(["outcomes.json"]) { fs::metadata(evidence.join(name))?; } } } Ok(ExitCode::SUCCESS) })(); result.unwrap_or_else(|error| crate::fail(&format!("nightly {phase}: {error}\n"))) } fn report_stage(extra: &[OsString]) -> ExitCode { crate::report_with_mutants(extra, &crate::target_dir().join("nightly-summary")) } /// Preserve each shard's JSON evidence; merge only the four text worklists /// consumed by the existing report. Missing shards are infrastructure failure. fn aggregate(stages: &Path, summary: &Path) -> std::io::Result<()> { fs::create_dir_all(summary)?; for name in LISTS { let mut combined = String::new(); for shard in 0..SHARDS { let source = stages.join(shard.to_string()).join("mutants.out"); let list = fs::read_to_string(source.join(name))?; combined.push_str(&list); if !list.is_empty() && !list.ends_with('\n') { combined.push('\n'); } } fs::write(summary.join(name), combined)?; } for shard in 0..SHARDS { fs::copy( stages .join(shard.to_string()) .join("mutants.out/outcomes.json"), summary.join(format!("shard-{shard}.json")), )?; } Ok(()) } #[cfg(test)] mod tests { use std::{ffi::OsString, fs, path::Path, process::ExitCode}; fn fixture(name: &str) -> std::path::PathBuf { let root = std::env::temp_dir().join(format!("smith-nightly-{name}-{}", std::process::id())); let _ = fs::remove_dir_all(&root); fs::create_dir_all(&root).unwrap(); root } fn evidence(root: &Path, shard: usize, missed: bool) { let dir = root .join("nightly") .join(shard.to_string()) .join("mutants.out"); fs::create_dir_all(&dir).unwrap(); for name in ["caught", "missed", "timeout", "unviable"] { fs::write( dir.join(format!("{name}.txt")), if name == "missed" && missed { "miss\n" } else { "" }, ) .unwrap(); } fs::write(dir.join("outcomes.json"), "{}").unwrap(); } #[test] fn findings_keep_shards_running_but_fail_the_verdict() { let root = fixture("findings"); for shard in 0..8 { let status = super::execute(&root, &shard.to_string(), &[], |step, args| { assert!(std::ptr::fn_addr_eq( step, crate::cargo_mutants as crate::Step )); assert_eq!( args, &[ OsString::from("--all"), OsString::from("--shard"), OsString::from(format!("{shard}/8")), OsString::from("--output"), root.join("nightly") .join(shard.to_string()) .into_os_string(), ] ); evidence(&root, shard, shard == 0); if shard == 0 { ExitCode::from(2) } else { ExitCode::SUCCESS } }); assert_eq!(status, ExitCode::SUCCESS); } let status = super::execute(&root, "report", &[], |_, _| { let out = root.join("report"); fs::create_dir_all(&out).unwrap(); fs::write(out.join("index.md"), "report").unwrap(); ExitCode::SUCCESS }); assert_eq!(status, ExitCode::SUCCESS); assert!(root.join("nightly/report/index.md").exists()); assert_eq!( fs::read_to_string(root.join("nightly/report/missed.txt")).unwrap(), "miss\n" ); assert_eq!( super::execute(&root, "verdict", &[], |_, _| panic!( "verdict runs no tools" )), ExitCode::FAILURE ); fs::remove_dir_all(root).unwrap(); } #[test] fn infrastructure_errors_are_not_findings() { for (name, status, write) in [ ("infra", ExitCode::FAILURE, true), ("missing", ExitCode::SUCCESS, false), ("timeout", ExitCode::from(3), true), ] { let root = fixture(name); let result = super::execute(&root, "0", &[], |_, _| { if write { evidence(&root, 0, false); } status }); assert_eq!( result, if write && status == ExitCode::from(3) { ExitCode::SUCCESS } else { ExitCode::FAILURE } ); fs::remove_dir_all(root).unwrap(); } } #[test] fn report_requires_every_shard_without_reusing_old_evidence() { let root = fixture("incomplete"); evidence(&root, 0, false); assert_eq!( super::execute(&root, "report", &[], |_, _| panic!( "incomplete shards must stop before coverage" )), ExitCode::FAILURE ); assert_eq!( super::execute(&root, "bootstrap", &[], |_, _| ExitCode::SUCCESS), ExitCode::SUCCESS ); assert!(!root.join("nightly/0").exists()); assert_eq!( super::execute(&root, "verdict", &[], |_, _| panic!( "verdict runs no tools" )), ExitCode::FAILURE ); fs::remove_dir_all(root).unwrap(); } #[test] fn bootstrap_verifies_baseline_before_miri_and_stops_on_failure() { let root = fixture("baseline"); let extra = [OsString::from("-p"), OsString::from("xtask")]; let mut calls = 0; let status = super::execute(&root, "bootstrap", &extra, |step, args| { calls += 1; match calls { 1 => { assert!(std::ptr::fn_addr_eq(step, crate::bootstrap as crate::Step)); assert!(args.is_empty()); ExitCode::SUCCESS } 2 => { assert!(std::ptr::fn_addr_eq( step, crate::cargo_nextest as crate::Step )); assert_eq!(args, extra); ExitCode::FAILURE } _ => panic!("Miri must not run after failing baseline"), } }); assert_eq!(calls, 2); assert_eq!(status, ExitCode::FAILURE); fs::remove_dir_all(root).unwrap(); } #[test] fn complete_run_passes_but_timeouts_fail_the_verdict() { let root = fixture("verdict"); for shard in 0..8 { evidence(&root, shard, false); } let report = || { assert_eq!( super::execute(&root, "report", &[], |_, _| { fs::create_dir_all(root.join("report")).unwrap(); fs::write(root.join("report/index.md"), "report").unwrap(); ExitCode::SUCCESS }), ExitCode::SUCCESS ); }; report(); assert_eq!( super::execute(&root, "verdict", &[], |_, _| panic!("no tools")), ExitCode::SUCCESS ); fs::write(root.join("nightly/7/mutants.out/timeout.txt"), "timeout").unwrap(); report(); assert_eq!( fs::read_to_string(root.join("nightly/report/timeout.txt")).unwrap(), "timeout\n" ); assert_eq!( super::execute(&root, "verdict", &[], |_, _| panic!("no tools")), ExitCode::FAILURE ); fs::remove_dir_all(root).unwrap(); } #[test] fn native_exit_codes_distinguish_findings_from_failed_runs() { for code in [0, 1, 2, 3, 4, 5, 6, 70] { let expected = match code { 0 => ExitCode::SUCCESS, 2 => ExitCode::from(2), 3 => ExitCode::from(3), _ => ExitCode::FAILURE, }; assert_eq!(crate::mutants_exit(Some(code)), expected); } assert_eq!(crate::mutants_exit(None), ExitCode::FAILURE); } #[test] fn invalid_phases_do_not_run_tools() { let root = fixture("invalid"); for phase in ["", "8", "-1", "00", "whatever"] { assert_eq!( super::execute(&root, phase, &[], |_, _| panic!( "invalid phases must not run tools" )), ExitCode::FAILURE ); } fs::remove_dir_all(root).unwrap(); } #[test] fn kdl_stages_share_one_job_in_execution_order() { let config = include_str!("../../.ci/nightly.kdl"); assert!(config.contains("matrix {\n phase \"bootstrap\" \"0\" \"1\" \"2\" \"3\" \"4\" \"5\" \"6\" \"7\" \"report\" \"verdict\"\n }")); assert!(config.contains("cargo run --locked -p xtask -- nightly ${phase}")); assert!(config.contains("from \"target/nightly/${phase}\"")); assert!(!config.contains("|| echo")); } }