#!/bin/bash
# M1 dead-man watchdog (the Pi is the only box outside the M1 alert chain).
# 3 consecutive failed probes of M1 (node-exporter :9101 + coordinator :8443
# tcp) -> high-priority push via Pi-local ntfy. One recovery push when back.
STATE=/var/lib/ahel-watchdog/state
mkdir -p "$(dirname "$STATE")"
fails=$(cat "$STATE" 2>/dev/null || echo 0)
ok=0
curl -sm5 -o /dev/null http://192.168.8.168:9101/metrics && ok=1
[ "$ok" = 0 ] && timeout 5 bash -c "</dev/tcp/192.168.8.168/8443" 2>/dev/null && ok=1
if [ "$ok" = 1 ]; then
  if [ "$fails" -ge 3 ]; then
    curl -sm5 -H "Priority: high" -H "Title: M1 recovered" -d "M1 answers again after $fails failed checks." http://127.0.0.1:8099/erebor >/dev/null
  fi
  echo 0 > "$STATE"
else
  fails=$((fails+1)); echo "$fails" > "$STATE"
  if [ "$fails" = 3 ]; then
    curl -sm5 -H "Priority: urgent" -H "Tags: rotating_light" -H "Title: M1 IS DOWN" -d "The gateway (192.168.8.168) failed 3 checks in a row. Coordinator, Bifrost, knowledge and bridge are likely offline." http://127.0.0.1:8099/erebor >/dev/null
  fi
fi
