119 lines
4.5 KiB
Python
119 lines
4.5 KiB
Python
"""
|
|
Getting the robot back after a power cycle.
|
|
|
|
The X2 cannot reliably bring the dashboard agent up on its own:
|
|
|
|
* `systemd --user` units only run while the user has a login session, and the
|
|
`agi` account is not allowed to enable lingering (`loginctl enable-linger`
|
|
is denied, and sudo forbids running as root).
|
|
* The cron fallback does not fire either - the robot's clock jumps backwards
|
|
by several hours shortly after boot (RTC vs NTP), and cron stalls on a
|
|
backward jump.
|
|
|
|
The dashboard host is the dependable machine in this setup, so recovery lives
|
|
here instead: find the robot wherever DHCP put it, and start the agent over SSH
|
|
if it is not already running.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
|
|
from . import netinfo, x2_spec
|
|
|
|
|
|
class RecoveryError(Exception):
|
|
pass
|
|
|
|
|
|
async def find_robot(agent_port: int, ssh_port: int = 22,
|
|
networks: list[str] | None = None) -> list[dict]:
|
|
"""
|
|
Sweep the current subnets for anything that looks like the robot.
|
|
|
|
A host already running the agent is the strongest signal; a host answering
|
|
on SSH is a candidate we may be able to start the agent on.
|
|
"""
|
|
cidrs = networks or netinfo.local_networks()
|
|
found: list[dict] = []
|
|
|
|
for cidr in cidrs[:3]:
|
|
try:
|
|
hosts = await netinfo.scan_subnet(cidr, ports=[agent_port, ssh_port], timeout=0.4)
|
|
except (ValueError, Exception):
|
|
continue
|
|
for host in hosts:
|
|
open_ports = host.get("open_ports") or []
|
|
found.append({
|
|
"host": host["host"],
|
|
"hostname": host.get("hostname"),
|
|
"agent": agent_port in open_ports,
|
|
"ssh": ssh_port in open_ports,
|
|
"latency_ms": host.get("latency_ms"),
|
|
})
|
|
|
|
# Anything already running the agent sorts first.
|
|
found.sort(key=lambda h: (not h["agent"], not h["ssh"], h["host"]))
|
|
return found
|
|
|
|
|
|
def _start_over_ssh_blocking(host: str, port: int, user: str, password: str,
|
|
command: str, timeout: float = 25.0) -> tuple[bool, str]:
|
|
try:
|
|
import paramiko
|
|
except ImportError:
|
|
return False, ("paramiko is not installed on the dashboard host - "
|
|
"run: pip install -r requirements.txt")
|
|
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
try:
|
|
client.connect(host, port=port, username=user, password=password,
|
|
timeout=timeout, allow_agent=False, look_for_keys=False,
|
|
banner_timeout=timeout, auth_timeout=timeout)
|
|
except Exception as exc:
|
|
return False, f"SSH to {user}@{host}:{port} failed: {type(exc).__name__}: {exc}"
|
|
|
|
try:
|
|
# The helper is idempotent: it exits immediately if the agent is already
|
|
# listening, so running it when we raced a manual start is harmless.
|
|
stdin, stdout, stderr = client.exec_command(command, timeout=timeout)
|
|
stdout.channel.recv_exit_status()
|
|
err = stderr.read().decode("utf-8", "replace").strip()
|
|
if err and "warning" not in err.lower():
|
|
return True, f"start command ran with stderr: {err[:200]}"
|
|
return True, "start command ran"
|
|
except Exception as exc:
|
|
return False, f"Running the start command failed: {exc}"
|
|
finally:
|
|
client.close()
|
|
|
|
|
|
async def start_agent(host: str, config: dict) -> tuple[bool, str]:
|
|
"""Log in and run the agent starter. Returns (attempted_ok, message)."""
|
|
user = (config.get("robot_ssh_user") or "").strip()
|
|
password = config.get("robot_ssh_password") or ""
|
|
command = (config.get("agent_start_command") or "").strip()
|
|
ssh_port = int(config.get("robot_ssh_port") or 22)
|
|
|
|
if not user or not command:
|
|
return False, "SSH user or start command is not configured"
|
|
if not password:
|
|
return False, ("No SSH password saved. Add it in Settings so the dashboard can "
|
|
"start the agent after the robot reboots.")
|
|
|
|
return await asyncio.to_thread(
|
|
_start_over_ssh_blocking, host, ssh_port, user, password, command)
|
|
|
|
|
|
async def wait_for_agent(host: str, port: int, timeout: float = 45.0) -> bool:
|
|
"""Poll until the agent's port opens, or give up."""
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
info = await netinfo.probe_host(host, ports=[port], timeout=1.0)
|
|
if info.get("reachable"):
|
|
return True
|
|
await asyncio.sleep(2.0)
|
|
return False
|