173 lines
13 KiB
Bash
Executable File
173 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# SanadR1 post-install health check — run ON the R1 backpack after install.sh.
|
|
#
|
|
# ./healthcheck.sh # checks the local container on :8001
|
|
# ./healthcheck.sh http://HOST:8001 # check a remote backpack's dashboard
|
|
#
|
|
# Verifies the whole stack end-to-end: Docker container + image + systemd,
|
|
# the dashboard, the robot DDS link (all 18 subsystems + fresh lowstate),
|
|
# Gemini voice, the camera pipeline, and audio. Exits 0 only if every CRITICAL
|
|
# check passes (WARN is allowed); exits 1 on any FAIL so it can gate CI / a
|
|
# deploy script.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
set -uo pipefail
|
|
BASE="${1:-http://127.0.0.1:8001}"
|
|
LOCAL=1; case "$BASE" in http://127.0.0.1:*|http://localhost:*) LOCAL=1;; *) LOCAL=0;; esac
|
|
|
|
PASS=0; WARN=0; FAIL=0
|
|
g(){ printf '\033[0;32m[ PASS ]\033[0m %-34s %s\n' "$1" "${2:-}"; PASS=$((PASS+1)); }
|
|
w(){ printf '\033[0;33m[ WARN ]\033[0m %-34s %s\n' "$1" "${2:-}"; WARN=$((WARN+1)); }
|
|
b(){ printf '\033[0;31m[ FAIL ]\033[0m %-34s %s\n' "$1" "${2:-}"; FAIL=$((FAIL+1)); }
|
|
sec(){ printf '\n\033[1;36m── %s ──\033[0m\n' "$1"; }
|
|
|
|
http(){ curl -s -m 6 -o /dev/null -w '%{http_code}' "$1" 2>/dev/null; }
|
|
# jget <file> <python-expr over d> — prints value or empty on error
|
|
jget(){ python3 -c "import sys,json
|
|
try:
|
|
d=json.load(open('$1'))
|
|
v=($2)
|
|
print('' if v is None else v)
|
|
except Exception:
|
|
print('')" 2>/dev/null; }
|
|
|
|
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
INFO="$TMP/info.json"; VOICE="$TMP/voice.json"; CAM="$TMP/cam.json"
|
|
|
|
echo "SanadR1 health check → $BASE ($(date '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo now))"
|
|
|
|
# ── 1. Docker / image / container / systemd (local only) ────────────────────
|
|
if [ "$LOCAL" = 1 ]; then
|
|
sec "Container & host"
|
|
if command -v docker >/dev/null && docker info >/dev/null 2>&1; then g "docker daemon" "reachable"
|
|
else b "docker daemon" "not reachable"; fi
|
|
docker image inspect sanadr1:latest >/dev/null 2>&1 && g "image sanadr1:latest" "present" || b "image sanadr1:latest" "missing"
|
|
st="$(docker inspect -f '{{.State.Status}}' sanadr1 2>/dev/null)"
|
|
[ "$st" = "running" ] && g "container sanadr1" "running" || b "container sanadr1" "state=${st:-absent}"
|
|
# container image must match the loaded tag (catches a stale container after an image reload)
|
|
cimg="$(docker inspect -f '{{.Image}}' sanadr1 2>/dev/null)"; limg="$(docker image inspect -f '{{.Id}}' sanadr1:latest 2>/dev/null)"
|
|
if [ -n "$cimg" ] && [ "$cimg" = "$limg" ]; then g "image up-to-date" "container matches sanadr1:latest"
|
|
else w "image up-to-date" "container != sanadr1:latest — re-run run.sh to recreate"; fi
|
|
hs="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' sanadr1 2>/dev/null)"
|
|
case "$hs" in
|
|
healthy) g "container health" "healthy";;
|
|
starting) w "container health" "still starting (grace period)";;
|
|
unhealthy) b "container health" "unhealthy";;
|
|
none) w "container health" "no healthcheck defined";;
|
|
*) b "container health" "${hs:-unknown}";;
|
|
esac
|
|
en="$(systemctl is-enabled sanadr1 2>/dev/null)"; [ "$en" = "enabled" ] && g "systemd enabled" "auto-starts on boot" || w "systemd enabled" "is-enabled=$en"
|
|
ac="$(systemctl is-active sanadr1 2>/dev/null)"; [ "$ac" = "active" ] && g "systemd active" "" || w "systemd active" "is-active=$ac"
|
|
rp="$(docker inspect -f '{{.HostConfig.RestartPolicy.Name}}' sanadr1 2>/dev/null)"; rp_ok=0; case "$rp" in unless-stopped|always) rp_ok=1;; esac
|
|
[ "$rp_ok" = 1 ] && g "restart policy" "$rp" || w "restart policy" "${rp:-none}"
|
|
# reboot survival = systemd-enabled OR docker restart policy; both absent → dead after power-cycle
|
|
{ [ "$en" = "enabled" ] || [ "$rp_ok" = 1 ]; } && g "reboot survival" "auto-starts on power-cycle" \
|
|
|| b "reboot survival" "no systemd enable AND no restart policy — will NOT auto-start after power-cycle"
|
|
for v in sanadr1_data sanadr1_logs; do docker volume inspect "$v" >/dev/null 2>&1 && g "volume $v" "mounted" || b "volume $v" "missing"; done
|
|
# disk headroom on the docker root (a full data/logs volume fails writes while the API still looks healthy)
|
|
root="$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || echo /var/lib/docker)"; use="$(df -P "$root" 2>/dev/null | awk 'NR==2{print $5+0}')"
|
|
if [ -n "$use" ]; then
|
|
if [ "$use" -lt 90 ]; then g "disk space" "${use}% used on $root"
|
|
elif [ "$use" -lt 97 ]; then w "disk space" "${use}% used — volumes may soon fail writes"
|
|
else b "disk space" "${use}% used — writes will fail"; fi
|
|
fi
|
|
# clock sanity — the Orin has no RTC; a wrong system clock breaks Gemini's TLS handshake
|
|
yr="$(date +%Y 2>/dev/null)"
|
|
if [ "${yr:-0}" -ge 2025 ] 2>/dev/null; then g "system clock" "$(date '+%F %T')"
|
|
else w "system clock" "year=${yr:-?} looks wrong — TLS/Gemini may fail (no RTC; needs NTP)"; fi
|
|
fi
|
|
|
|
# ── 2. Dashboard reachability ───────────────────────────────────────────────
|
|
sec "Dashboard"
|
|
code="$(http "$BASE/")"; [ "$code" = 200 ] && g "dashboard /" "HTTP 200" || b "dashboard /" "HTTP ${code:-timeout}"
|
|
code="$(http "$BASE/api/system/info")"
|
|
if [ "$code" = 200 ]; then curl -s -m6 "$BASE/api/system/info" >"$INFO" 2>/dev/null; g "api/system/info" "HTTP 200"
|
|
else b "api/system/info" "HTTP ${code:-timeout}"; fi
|
|
|
|
# ── 3. Robot DDS link + subsystems ──────────────────────────────────────────
|
|
sec "Robot link (DDS) & subsystems"
|
|
if [ -s "$INFO" ]; then
|
|
ddsif="$(jget "$INFO" 'd["dds"]["interface"]')"
|
|
[ -n "$ddsif" ] && g "DDS interface" "$ddsif" || b "DDS interface" "unset"
|
|
ifup="$(jget "$INFO" 'next((i["is_up"] and i["ip"].startswith("192.168.123.") for i in d["network"]["interfaces"] if i["name"]==d["dds"]["interface"]), False)')"
|
|
[ "$ifup" = "True" ] && g "robot NIC up" "$ddsif has 192.168.123.x" || b "robot NIC up" "$ddsif down or no 192.168.123.x IP (robot cable?)"
|
|
tot="$(jget "$INFO" 'd["subsystems"]["total"]')"; con="$(jget "$INFO" 'd["subsystems"]["connected"]')"; dis="$(jget "$INFO" 'd["subsystems"]["disconnected"]')"
|
|
# require the full registry (>=18) so an empty/short registry (total=0 after a core
|
|
# import failure) can NEVER read as green "0/0 connected".
|
|
if [ -n "$con" ] && [ "${tot:-0}" -ge 18 ] 2>/dev/null && [ "$con" = "$tot" ] && [ "${dis:-0}" = 0 ]; then g "subsystems" "$con/$tot connected"
|
|
else down="$(jget "$INFO" '",".join(s["name"] for s in d["subsystems"]["list"] if not s["connected"])')"; b "subsystems" "${con:-?}/${tot:-?} connected (expect >=18) — down: ${down:-registry empty}"; fi
|
|
# arm lowstate freshness = proof DDS is actually RECEIVING from the robot (subsystem
|
|
# "connected" only means the object was constructed, not that data is flowing) → FAIL if stale.
|
|
age="$(jget "$INFO" 'next((s["status"].get("state_age_sec") for s in d["subsystems"]["list"] if s["name"]=="arm"), None)')"
|
|
if [ -n "$age" ] && awk -v a="$age" 'BEGIN{exit !(a+0<5)}'; then g "lowstate fresh" "arm state_age=${age}s (robot state flowing)"
|
|
else b "lowstate fresh" "arm state_age=${age:-n/a}s — robot state NOT flowing (robot powered off / cable?)"; fi
|
|
else w "Robot link" "skipped (no system/info)"; fi
|
|
|
|
# ── 4. Gemini voice ─────────────────────────────────────────────────────────
|
|
# The live session is lazy-started (connects on a wake-word / Live Voice toggle),
|
|
# so on an idle boot connected=False is NORMAL. We therefore verify readiness:
|
|
# connected -> PASS (a session is already up)
|
|
# reconnect_attempts>0 -> FAIL (actively trying + failing = bad key / network)
|
|
# else (lazy) -> PASS if the key is in the container AND the Gemini API
|
|
# is reachable; otherwise FAIL.
|
|
sec "Voice (Gemini)"
|
|
gc=""; ra=""
|
|
if [ "$(http "$BASE/api/voice/status")" = 200 ]; then
|
|
curl -s -m6 "$BASE/api/voice/status" >"$VOICE" 2>/dev/null
|
|
gc="$(jget "$VOICE" 'd["gemini"]["connected"]')"
|
|
ra="$(jget "$VOICE" 'd["gemini"].get("reconnect_attempts")')"
|
|
fi
|
|
if [ "$gc" = "True" ]; then
|
|
g "Gemini connected" "$(jget "$VOICE" 'd["gemini"]["model"]')"
|
|
elif [ "${ra:-0}" -gt 0 ] 2>/dev/null; then
|
|
b "Gemini connected" "failing after $ra reconnect attempts — bad key or network"
|
|
elif [ "$LOCAL" = 1 ]; then
|
|
docker exec sanadr1 sh -c '[ -n "$SANAD_GEMINI_API_KEY" ]' 2>/dev/null \
|
|
&& g "Gemini key" "present in container" || b "Gemini key" "SANAD_GEMINI_API_KEY not set in container"
|
|
gcode="$(docker exec sanadr1 sh -c 'curl -s -m6 -o /dev/null -w "%{http_code}" https://generativelanguage.googleapis.com/v1beta/models' 2>/dev/null)"
|
|
if [ -n "$gcode" ] && [ "$gcode" != 000 ]; then g "Gemini reachable" "API responded ($gcode) — lazy-connects on wake/live-voice"
|
|
else b "Gemini reachable" "no internet route to the Gemini API (http=${gcode:-timeout})"; fi
|
|
elif [ -s "$VOICE" ]; then
|
|
w "Gemini connected" "no live session yet (lazy-start — run locally to verify key+reachability)"
|
|
else
|
|
b "Gemini status" "api/voice/status unreachable"
|
|
fi
|
|
|
|
# ── 5. Camera pipeline ──────────────────────────────────────────────────────
|
|
sec "Camera"
|
|
if [ "$(http "$BASE/api/recognition/state")" = 200 ]; then
|
|
curl -s -m6 "$BASE/api/recognition/state" >"$CAM" 2>/dev/null
|
|
src="$(jget "$CAM" 'd["camera"].get("source")')"; port="$(jget "$CAM" 'd["camera"].get("r1_port")')"
|
|
cerr="$(jget "$CAM" 'd["camera"].get("error")')"; dw="$(jget "$CAM" '"yes" if d["camera"].get("dewarp") else "no"')"
|
|
if [ -n "$src" ] && [ -n "$port" ]; then g "camera code" "source=$src port=$port dewarp=$dw (fixes present)"
|
|
elif [ "$cerr" = "camera subsystem unavailable" ]; then b "camera code" "camera subsystem unavailable (not loaded)"
|
|
else b "camera code" "r1_port/source missing — old image without the camera fixes"; fi
|
|
code="$(http "$BASE/api/recognition/stream.mjpg")"; [ "$code" = 200 ] && g "MJPEG endpoint" "HTTP 200" || b "MJPEG endpoint" "HTTP ${code:-timeout}"
|
|
run="$(jget "$CAM" '"yes" if d["camera"].get("running") else "no"')"
|
|
[ "$run" = yes ] && g "camera capturing" "backend=$(jget "$CAM" 'd["camera"].get("backend")')" \
|
|
|| w "camera capturing" "off — toggle Vision ON + start 'Stereo patch PC1' in the app"
|
|
else b "api/recognition/state" "unreachable"; fi
|
|
|
|
# ── 6. Audio ────────────────────────────────────────────────────────────────
|
|
sec "Audio"
|
|
if [ -s "$INFO" ]; then
|
|
pa="$(jget "$INFO" 'd["audio"]["pactl_available"]')"; [ "$pa" = "True" ] && g "PulseAudio" "available in container" || w "PulseAudio" "unavailable"
|
|
prof="$(jget "$INFO" '(d["audio"]["current"].get("profile") or {}).get("label")')"
|
|
[ -n "$prof" ] && g "audio profile" "$prof" || w "audio profile" "(none)"
|
|
sk="$(jget "$INFO" 'd["audio"]["current"].get("source_kind")')"
|
|
[ -n "$sk" ] && g "audio input" "source_kind=$sk" || w "audio input" "no active mic source — wake/ASR deaf"
|
|
else w "Audio" "skipped (no system/info)"; fi
|
|
|
|
# ── 7. Container logs (local) ───────────────────────────────────────────────
|
|
if [ "$LOCAL" = 1 ]; then
|
|
sec "Logs"
|
|
errs="$(docker logs --since 10m sanadr1 2>&1 | grep -iE 'traceback|critica|modulenotfound|importerror|address already in use' | grep -viE 'no frame|no backend|reconnect|stream off|no camera|RealSense unavailable|USB camera|no working|local_tts' | tail -3)"
|
|
[ -z "$errs" ] && g "no fatal errors" "clean startup logs" || w "log errors" "$(echo "$errs" | head -1)"
|
|
fi
|
|
|
|
# ── summary ─────────────────────────────────────────────────────────────────
|
|
printf '\n\033[1m── Summary ──\033[0m \033[0;32m%d PASS\033[0m \033[0;33m%d WARN\033[0m \033[0;31m%d FAIL\033[0m\n' "$PASS" "$WARN" "$FAIL"
|
|
if [ "$FAIL" -gt 0 ]; then echo "RESULT: FAIL — see [FAIL] lines above."; exit 1; fi
|
|
[ "$WARN" -gt 0 ] && echo "RESULT: OK (with warnings — usually the camera stream / TTS model, both optional)." || echo "RESULT: ALL GREEN ✓"
|
|
exit 0
|