bugabinga.net

#!/usr/bin/env bash
set -uo pipefail

# Root-run weekly maintenance freshness checks on klops.
# Prints one TSV line per check (status, name, detail) and exits nonzero on
# any failure so OnFailure=bugabinga-alert@ can notify.

service_user="oli"
ci_user="ci"
toad_user="toad"
rollback_image="localhost/bugabinga-luci:rollback"
gc_max_age=$((8 * 24 * 3600))
update_max_age=$((25 * 3600))
disk_max_percent=90
failures=0

pass() { printf 'ok\t%s\t%s\n' "$1" "$2"; }
fail() {
  printf 'fail\t%s\t%s\n' "$1" "$2"
  failures=$((failures + 1))
}

user_shell() {
  local user="$1" command="$2"
  # cd / keeps the invocation independent of an unreadable caller cwd.
  cd /
  runuser -u "${user}" -- /usr/bin/bash -c "export HOME=\$(getent passwd ${user} | cut -d: -f6) XDG_RUNTIME_DIR=/run/user/\$(id -u ${user}) DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/\$(id -u ${user})/bus; exec ${command}"
}

check_success_age() {
  local name="$1" unit="$2" max_age="$3" age='' now='' timestamp=''
  timestamp="$(systemctl show "${unit}" --property=ExecMainExitTimestamp --value)"
  if [[ -n "${timestamp}" ]] && age="$(date -d "${timestamp}" +%s 2>/dev/null)"; then
    now="$(date +%s)"
    if ((now - age <= max_age)); then
      pass "${name}" "last success ${timestamp}"
    else
      fail "${name}" "last success ${timestamp} is stale"
    fi
  else
    fail "${name}" "no recorded successful run"
  fi
}

check_disk() {
  local name="$1" path="$2" usage=''
  usage="$(df -P "${path}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')"
  if [[ "${usage}" =~ ^[0-9]+$ ]] && ((usage < disk_max_percent)); then
    pass "${name}" "${usage}% used on ${path}"
  else
    fail "${name}" "${usage:-unknown}% used on ${path} exceeds ${disk_max_percent}%"
  fi
}

check_success_age "gc-${service_user}" "bugabinga-gc@${service_user}.service" "${gc_max_age}"
check_success_age "gc-${ci_user}" "bugabinga-gc@${ci_user}.service" "${gc_max_age}"
check_success_age "gc-${toad_user}" "bugabinga-gc@${toad_user}.service" "${gc_max_age}"
check_success_age "gc-host" "bugabinga-gc-host.service" "${gc_max_age}"

update_trigger="$(user_shell "${service_user}" 'systemctl --user show podman-auto-update.timer --property=LastTriggerUSec --value')"
update_age=''
if [[ -n "${update_trigger}" ]] && update_age="$(date -d "${update_trigger}" +%s 2>/dev/null)"; then
  now="$(date +%s)"
  if ((now - update_age <= update_max_age)); then
    pass "auto-update" "last trigger ${update_trigger}"
  else
    fail "auto-update" "last trigger ${update_trigger} is stale"
  fi
else
  fail "auto-update" "no recorded trigger"
fi

if user_shell "${ci_user}" "podman image exists '${rollback_image}'" >/dev/null 2>&1; then
  pass "luci-rollback" "${rollback_image} exists"
else
  fail "luci-rollback" "${rollback_image} missing"
fi

check_disk "disk-data-ci" "/data/ci"
check_disk "disk-root" "/"

exit "$((failures > 0 ? 1 : 0))"