"""Local live subprocess supervisor. Spawns `voice/sanad_voice.py` as a managed child with `SANAD_VOICE_BRAIN=local`, tails the child's stdout, and extracts state transitions + user transcripts from the log markers emitted by `local/script.py:LocalBrain`. Mirror of `gemini/subprocess.py`. Lives separately so the two supervisors stay decoupled — adding a new model does not touch this file. """ from __future__ import annotations import os import signal import subprocess import sys import threading from collections import deque from datetime import datetime from pathlib import Path from typing import Any from Project.Sanad.config import BASE_DIR, LOGS_DIR, SCRIPTS_DIR, LIVE_TUNE from Project.Sanad.core.config_loader import section as _cfg_section from Project.Sanad.core.logger import get_logger log = get_logger("local_subprocess") _LS_CFG = _cfg_section("local", "subprocess") def _resolve_live_script() -> Path: """Locate the voice script to run as subprocess (same as Gemini's).""" override = os.environ.get("SANAD_LIVE_SCRIPT", "").strip() if override: p = Path(override).expanduser() if p.exists(): return p for c in (BASE_DIR / "voice" / "sanad_voice.py", SCRIPTS_DIR / "sanad_voice.py"): if c.exists(): return c return SCRIPTS_DIR / "sanad_voice.py" LIVE_SCRIPT = _resolve_live_script() LOG_TAIL_SIZE = _LS_CFG.get("log_tail_size", 2000) TRANSCRIPT_TAIL_SIZE = _LS_CFG.get("transcript_tail_size", 30) LIVE_LOG_DIR = LOGS_DIR LIVE_LOG_NAME = _LS_CFG.get("log_name", "local_subprocess") # Python binary for the child process. The local pipeline runs in a # separate conda env (Python 3.8 + Jetson CUDA torch + CosyVoice/Whisper); # the dashboard stays in gemini_sdk (Python 3.10). Override with # SANAD_LOCAL_PYTHON env var at runtime. LOCAL_PYTHON_BIN = os.environ.get( "SANAD_LOCAL_PYTHON", _LS_CFG.get("python_bin", sys.executable), ) _STOP_TIMEOUT_SEC = _LS_CFG.get("stop_timeout_sec", 5.0) _TERMINATE_TIMEOUT_SEC = _LS_CFG.get("terminate_timeout_sec", 3.0) _NOISY_PREFIXES = tuple(_LS_CFG.get("noisy_prefixes", [ "ALSA lib ", "Expression 'alsa_", "Cannot connect to server socket", "jack server is not running", ])) _NOISY_FRAGMENTS = tuple(_LS_CFG.get("noisy_fragments", [ "Unknown PCM", "Evaluate error", "snd_pcm_open_noupdate", "PaAlsaStream", ])) class LocalSubprocess: def __init__(self): self._lock = threading.Lock() self.process: subprocess.Popen | None = None self.log_tail: deque[str] = deque(maxlen=LOG_TAIL_SIZE) self.user_transcript: deque[str] = deque(maxlen=TRANSCRIPT_TAIL_SIZE) self._reader_thread: threading.Thread | None = None self._log_file = None self.state = "stopped" self.state_message = "Idle." self.last_user_text = "" self.suppressed_noise = 0 # ─── log I/O ────────────────────────────────────────── def _open_session_log(self, pid: int): try: LIVE_LOG_DIR.mkdir(parents=True, exist_ok=True) fname = f"{LIVE_LOG_NAME}_{datetime.now().strftime('%Y%m%d')}.log" fh = open(LIVE_LOG_DIR / fname, "a", encoding="utf-8", buffering=1) fh.write( f"\n===== local subprocess start " f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} pid={pid} =====\n" ) return fh except Exception as exc: log.warning("Could not open local subprocess log file: %s", exc) return None def _is_noisy(self, line: str) -> bool: return line.startswith(_NOISY_PREFIXES) or any(f in line for f in _NOISY_FRAGMENTS) def _set_state(self, state: str, msg: str): self.state = state self.state_message = msg def _track_line(self, line: str): """Parse log markers emitted by `local/script.py:LocalBrain`. Must stay in lock-step with the `log.info(...)` strings there. """ if "connecting to local pipeline" in line: self._set_state("connecting", line) elif " USER: " in line or line.strip().startswith("USER:"): text = line.split("USER:", 1)[1].strip() if text: self.last_user_text = text self.user_transcript.append(text) self._set_state("hearing", f"User: {text}") elif " BOT: " in line or line.strip().startswith("BOT:"): self._set_state("speaking", line.split("BOT:", 1)[1].strip()[:80]) elif "BARGE-IN (local)" in line: self._set_state("interrupting", line) elif "session error" in line: self._set_state("error", line) elif "local pipeline stopped" in line or "cancelled — stopping" in line: self._set_state("stopped", line) elif "listening" in line.lower() and "no speech" not in line: self._set_state("listening", "Listening for speech.") def _reader_loop(self): proc = self.process if proc is None or proc.stdout is None: return fh = self._open_session_log(proc.pid) self._log_file = fh for line in proc.stdout: clean = line.rstrip() if not clean: continue if fh is not None: try: fh.write(clean + "\n") except Exception: pass with self._lock: if self._is_noisy(clean): self.suppressed_noise += 1 continue self.log_tail.append(clean) self._track_line(clean) with self._lock: self.log_tail.append("Local pipeline process exited.") self._set_state("stopped", "Process exited.") if fh is not None: try: fh.write( f"===== local subprocess exit " f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} =====\n" ) fh.close() except Exception: pass self._log_file = None # ─── lifecycle ──────────────────────────────────────── def is_running(self) -> bool: with self._lock: return self.process is not None and self.process.poll() is None def start(self) -> dict[str, Any]: with self._lock: if self.process is not None and self.process.poll() is None: return {"started": False, "message": "Already running.", "pid": self.process.pid} self._set_state("starting", "Starting local pipeline (loading models)...") script = LIVE_SCRIPT if not script.exists(): raise RuntimeError(f"Script not found: {script}") env = os.environ.copy() env.update({ "PYTHONUNBUFFERED": "1", **LIVE_TUNE, "SANAD_VOICE_BRAIN": "local", }) dds_iface = env.get("SANAD_DDS_INTERFACE", "eth0") # Use the `local` env's Python so CUDA torch + CosyVoice are available. # Fall back to sys.executable only if the configured bin doesn't exist. py_bin = LOCAL_PYTHON_BIN if not Path(py_bin).exists(): log.warning("LOCAL_PYTHON_BIN=%s not found, falling back to %s", py_bin, sys.executable) py_bin = sys.executable cmd = [py_bin, str(script), dds_iface] proc = subprocess.Popen( cmd, cwd=str(script.parent), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, env=env, ) with self._lock: self.process = proc self.log_tail.append(f"Started: pid={proc.pid}") self._set_state("starting", f"pid={proc.pid}") self._reader_thread = threading.Thread(target=self._reader_loop, daemon=True) self._reader_thread.start() log.info("Local subprocess started: pid=%d", proc.pid) return {"started": True, "pid": proc.pid} def stop(self) -> dict[str, Any]: with self._lock: proc = self.process if proc is None or proc.poll() is not None: return {"stopped": False, "message": "Not running."} self._set_state("stopping", "Stopping...") try: proc.send_signal(signal.SIGINT) proc.wait(timeout=_STOP_TIMEOUT_SEC) except subprocess.TimeoutExpired: proc.terminate() try: proc.wait(timeout=_TERMINATE_TIMEOUT_SEC) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=_TERMINATE_TIMEOUT_SEC) rc = proc.returncode with self._lock: self.process = None self.log_tail.append("Stopped.") self._set_state("stopped", "Stopped.") log.info("Local subprocess stopped (rc=%s)", rc) return {"stopped": True, "returncode": rc} def status(self) -> dict[str, Any]: with self._lock: running = self.process is not None and self.process.poll() is None return { "running": running, "pid": self.process.pid if running and self.process else None, "state": self.state, "state_message": self.state_message, "last_user_text": self.last_user_text, "user_transcript": list(self.user_transcript), "log_tail": list(self.log_tail), "suppressed_noise": self.suppressed_noise, }