#!/usr/bin/env bash set -uo pipefail # Root-run weekly maintenance freshness checks on klops. # Prints one TSV line per check (status, name, detail) and exits nonzero on # any failure so OnFailure=bugabinga-alert@ can notify. service_user="oli" ci_user="ci" toad_user="toad" rollback_image="localhost/bugabinga-luci:rollback" gc_max_age=$((8 * 24 * 3600)) update_max_age=$((25 * 3600)) disk_max_percent=90 failures=0 pass() { printf 'ok\t%s\t%s\n' "$1" "$2"; } fail() { printf 'fail\t%s\t%s\n' "$1" "$2" failures=$((failures + 1)) } user_shell() { local user="$1" command="$2" # cd / keeps the invocation independent of an unreadable caller cwd. cd / runuser -u "${user}" -- /usr/bin/bash -c "export HOME=\$(getent passwd ${user} | cut -d: -f6) XDG_RUNTIME_DIR=/run/user/\$(id -u ${user}) DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/\$(id -u ${user})/bus; exec ${command}" } check_success_age() { local name="$1" unit="$2" max_age="$3" age='' now='' timestamp='' timestamp="$(systemctl show "${unit}" --property=ExecMainExitTimestamp --value)" if [[ -n "${timestamp}" ]] && age="$(date -d "${timestamp}" +%s 2>/dev/null)"; then now="$(date +%s)" if ((now - age <= max_age)); then pass "${name}" "last success ${timestamp}" else fail "${name}" "last success ${timestamp} is stale" fi else fail "${name}" "no recorded successful run" fi } check_disk() { local name="$1" path="$2" usage='' usage="$(df -P "${path}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" if [[ "${usage}" =~ ^[0-9]+$ ]] && ((usage < disk_max_percent)); then pass "${name}" "${usage}% used on ${path}" else fail "${name}" "${usage:-unknown}% used on ${path} exceeds ${disk_max_percent}%" fi } check_success_age "gc-${service_user}" "bugabinga-gc@${service_user}.service" "${gc_max_age}" check_success_age "gc-${ci_user}" "bugabinga-gc@${ci_user}.service" "${gc_max_age}" check_success_age "gc-${toad_user}" "bugabinga-gc@${toad_user}.service" "${gc_max_age}" check_success_age "gc-host" "bugabinga-gc-host.service" "${gc_max_age}" update_trigger="$(user_shell "${service_user}" 'systemctl --user show podman-auto-update.timer --property=LastTriggerUSec --value')" update_age='' if [[ -n "${update_trigger}" ]] && update_age="$(date -d "${update_trigger}" +%s 2>/dev/null)"; then now="$(date +%s)" if ((now - update_age <= update_max_age)); then pass "auto-update" "last trigger ${update_trigger}" else fail "auto-update" "last trigger ${update_trigger} is stale" fi else fail "auto-update" "no recorded trigger" fi if user_shell "${ci_user}" "podman image exists '${rollback_image}'" >/dev/null 2>&1; then pass "luci-rollback" "${rollback_image} exists" else fail "luci-rollback" "${rollback_image} missing" fi check_disk "disk-data-ci" "/data/ci" check_disk "disk-root" "/" exit "$((failures > 0 ? 1 : 0))"