262 lines
9.6 KiB
Python
262 lines
9.6 KiB
Python
"""Local live subprocess supervisor.
|
|
|
|
Spawns `voice/sanad_voice.py` as a managed child with
|
|
`SANAD_VOICE_BRAIN=local`, tails the child's stdout, and extracts state
|
|
transitions + user transcripts from the log markers emitted by
|
|
`local/script.py:LocalBrain`.
|
|
|
|
Mirror of `gemini/subprocess.py`. Lives separately so the two supervisors
|
|
stay decoupled — adding a new model does not touch this file.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from collections import deque
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from Project.Sanad.config import BASE_DIR, LOGS_DIR, SCRIPTS_DIR, LIVE_TUNE
|
|
from Project.Sanad.core.config_loader import section as _cfg_section
|
|
from Project.Sanad.core.logger import get_logger
|
|
|
|
log = get_logger("local_subprocess")
|
|
|
|
_LS_CFG = _cfg_section("local", "subprocess")
|
|
|
|
|
|
def _resolve_live_script() -> Path:
|
|
"""Locate the voice script to run as subprocess (same as Gemini's)."""
|
|
override = os.environ.get("SANAD_LIVE_SCRIPT", "").strip()
|
|
if override:
|
|
p = Path(override).expanduser()
|
|
if p.exists():
|
|
return p
|
|
for c in (BASE_DIR / "voice" / "sanad_voice.py",
|
|
SCRIPTS_DIR / "sanad_voice.py"):
|
|
if c.exists():
|
|
return c
|
|
return SCRIPTS_DIR / "sanad_voice.py"
|
|
|
|
|
|
LIVE_SCRIPT = _resolve_live_script()
|
|
LOG_TAIL_SIZE = _LS_CFG.get("log_tail_size", 2000)
|
|
TRANSCRIPT_TAIL_SIZE = _LS_CFG.get("transcript_tail_size", 30)
|
|
LIVE_LOG_DIR = LOGS_DIR
|
|
LIVE_LOG_NAME = _LS_CFG.get("log_name", "local_subprocess")
|
|
|
|
# Python binary for the child process. The local pipeline runs in a
|
|
# separate conda env (Python 3.8 + Jetson CUDA torch + CosyVoice/Whisper);
|
|
# the dashboard stays in gemini_sdk (Python 3.10). Override with
|
|
# SANAD_LOCAL_PYTHON env var at runtime.
|
|
LOCAL_PYTHON_BIN = os.environ.get(
|
|
"SANAD_LOCAL_PYTHON",
|
|
_LS_CFG.get("python_bin", sys.executable),
|
|
)
|
|
|
|
_STOP_TIMEOUT_SEC = _LS_CFG.get("stop_timeout_sec", 5.0)
|
|
_TERMINATE_TIMEOUT_SEC = _LS_CFG.get("terminate_timeout_sec", 3.0)
|
|
|
|
_NOISY_PREFIXES = tuple(_LS_CFG.get("noisy_prefixes", [
|
|
"ALSA lib ", "Expression 'alsa_", "Cannot connect to server socket",
|
|
"jack server is not running",
|
|
]))
|
|
_NOISY_FRAGMENTS = tuple(_LS_CFG.get("noisy_fragments", [
|
|
"Unknown PCM", "Evaluate error", "snd_pcm_open_noupdate", "PaAlsaStream",
|
|
]))
|
|
|
|
|
|
class LocalSubprocess:
|
|
def __init__(self):
|
|
self._lock = threading.Lock()
|
|
self.process: subprocess.Popen | None = None
|
|
self.log_tail: deque[str] = deque(maxlen=LOG_TAIL_SIZE)
|
|
self.user_transcript: deque[str] = deque(maxlen=TRANSCRIPT_TAIL_SIZE)
|
|
self._reader_thread: threading.Thread | None = None
|
|
self._log_file = None
|
|
self.state = "stopped"
|
|
self.state_message = "Idle."
|
|
self.last_user_text = ""
|
|
self.suppressed_noise = 0
|
|
|
|
# ─── log I/O ──────────────────────────────────────────
|
|
|
|
def _open_session_log(self, pid: int):
|
|
try:
|
|
LIVE_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
fname = f"{LIVE_LOG_NAME}_{datetime.now().strftime('%Y%m%d')}.log"
|
|
fh = open(LIVE_LOG_DIR / fname, "a", encoding="utf-8", buffering=1)
|
|
fh.write(
|
|
f"\n===== local subprocess start "
|
|
f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} pid={pid} =====\n"
|
|
)
|
|
return fh
|
|
except Exception as exc:
|
|
log.warning("Could not open local subprocess log file: %s", exc)
|
|
return None
|
|
|
|
def _is_noisy(self, line: str) -> bool:
|
|
return line.startswith(_NOISY_PREFIXES) or any(f in line for f in _NOISY_FRAGMENTS)
|
|
|
|
def _set_state(self, state: str, msg: str):
|
|
self.state = state
|
|
self.state_message = msg
|
|
|
|
def _track_line(self, line: str):
|
|
"""Parse log markers emitted by `local/script.py:LocalBrain`.
|
|
|
|
Must stay in lock-step with the `log.info(...)` strings there.
|
|
"""
|
|
if "connecting to local pipeline" in line:
|
|
self._set_state("connecting", line)
|
|
elif " USER: " in line or line.strip().startswith("USER:"):
|
|
text = line.split("USER:", 1)[1].strip()
|
|
if text:
|
|
self.last_user_text = text
|
|
self.user_transcript.append(text)
|
|
self._set_state("hearing", f"User: {text}")
|
|
elif " BOT: " in line or line.strip().startswith("BOT:"):
|
|
self._set_state("speaking", line.split("BOT:", 1)[1].strip()[:80])
|
|
elif "BARGE-IN (local)" in line:
|
|
self._set_state("interrupting", line)
|
|
elif "session error" in line:
|
|
self._set_state("error", line)
|
|
elif "local pipeline stopped" in line or "cancelled — stopping" in line:
|
|
self._set_state("stopped", line)
|
|
elif "listening" in line.lower() and "no speech" not in line:
|
|
self._set_state("listening", "Listening for speech.")
|
|
|
|
def _reader_loop(self):
|
|
proc = self.process
|
|
if proc is None or proc.stdout is None:
|
|
return
|
|
fh = self._open_session_log(proc.pid)
|
|
self._log_file = fh
|
|
for line in proc.stdout:
|
|
clean = line.rstrip()
|
|
if not clean:
|
|
continue
|
|
if fh is not None:
|
|
try:
|
|
fh.write(clean + "\n")
|
|
except Exception:
|
|
pass
|
|
with self._lock:
|
|
if self._is_noisy(clean):
|
|
self.suppressed_noise += 1
|
|
continue
|
|
self.log_tail.append(clean)
|
|
self._track_line(clean)
|
|
with self._lock:
|
|
self.log_tail.append("Local pipeline process exited.")
|
|
self._set_state("stopped", "Process exited.")
|
|
if fh is not None:
|
|
try:
|
|
fh.write(
|
|
f"===== local subprocess exit "
|
|
f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} =====\n"
|
|
)
|
|
fh.close()
|
|
except Exception:
|
|
pass
|
|
self._log_file = None
|
|
|
|
# ─── lifecycle ────────────────────────────────────────
|
|
|
|
def is_running(self) -> bool:
|
|
with self._lock:
|
|
return self.process is not None and self.process.poll() is None
|
|
|
|
def start(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
if self.process is not None and self.process.poll() is None:
|
|
return {"started": False, "message": "Already running.", "pid": self.process.pid}
|
|
self._set_state("starting", "Starting local pipeline (loading models)...")
|
|
|
|
script = LIVE_SCRIPT
|
|
if not script.exists():
|
|
raise RuntimeError(f"Script not found: {script}")
|
|
|
|
env = os.environ.copy()
|
|
env.update({
|
|
"PYTHONUNBUFFERED": "1",
|
|
**LIVE_TUNE,
|
|
"SANAD_VOICE_BRAIN": "local",
|
|
})
|
|
|
|
dds_iface = env.get("SANAD_DDS_INTERFACE", "eth0")
|
|
# Use the `local` env's Python so CUDA torch + CosyVoice are available.
|
|
# Fall back to sys.executable only if the configured bin doesn't exist.
|
|
py_bin = LOCAL_PYTHON_BIN
|
|
if not Path(py_bin).exists():
|
|
log.warning("LOCAL_PYTHON_BIN=%s not found, falling back to %s",
|
|
py_bin, sys.executable)
|
|
py_bin = sys.executable
|
|
cmd = [py_bin, str(script), dds_iface]
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
cwd=str(script.parent),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
bufsize=1,
|
|
env=env,
|
|
)
|
|
|
|
with self._lock:
|
|
self.process = proc
|
|
self.log_tail.append(f"Started: pid={proc.pid}")
|
|
self._set_state("starting", f"pid={proc.pid}")
|
|
self._reader_thread = threading.Thread(target=self._reader_loop, daemon=True)
|
|
self._reader_thread.start()
|
|
|
|
log.info("Local subprocess started: pid=%d", proc.pid)
|
|
return {"started": True, "pid": proc.pid}
|
|
|
|
def stop(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
proc = self.process
|
|
if proc is None or proc.poll() is not None:
|
|
return {"stopped": False, "message": "Not running."}
|
|
self._set_state("stopping", "Stopping...")
|
|
|
|
try:
|
|
proc.send_signal(signal.SIGINT)
|
|
proc.wait(timeout=_STOP_TIMEOUT_SEC)
|
|
except subprocess.TimeoutExpired:
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=_TERMINATE_TIMEOUT_SEC)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
proc.wait(timeout=_TERMINATE_TIMEOUT_SEC)
|
|
|
|
rc = proc.returncode
|
|
|
|
with self._lock:
|
|
self.process = None
|
|
self.log_tail.append("Stopped.")
|
|
self._set_state("stopped", "Stopped.")
|
|
|
|
log.info("Local subprocess stopped (rc=%s)", rc)
|
|
return {"stopped": True, "returncode": rc}
|
|
|
|
def status(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
running = self.process is not None and self.process.poll() is None
|
|
return {
|
|
"running": running,
|
|
"pid": self.process.pid if running and self.process else None,
|
|
"state": self.state,
|
|
"state_message": self.state_message,
|
|
"last_user_text": self.last_user_text,
|
|
"user_transcript": list(self.user_transcript),
|
|
"log_tail": list(self.log_tail),
|
|
"suppressed_noise": self.suppressed_noise,
|
|
}
|