Sanad Lite d9b2d5427f Voice fidelity, Live Gemini tab, and dashboard fixes
Replay now matches the robots and no longer cuts words:
- read the turn to turnComplete, not generationComplete, and drain the
  socket before each send; breaking early truncated every sentence and
  left frames that the next turn mis-read as its own reply
- accept a take only if the model's own transcript covers the text AND
  the audio is long enough to contain it (the transcript reports the
  full text even for a 0.8s clip)
- pitch gate: reject an off-tone take and re-ask, per voice, using a
  pure-Python F0 estimator (no numpy on the host)
- continuation: speak the words a voice skipped instead of retrying a
  line it stops on deterministically
- fresh Live session per replay; delivery drifts as turns accumulate

Live Gemini tab: browser talks to Gemini directly (the reverse proxy
cannot upgrade a WebSocket), with a persona library - named personas,
per-robot selection, built-ins that cannot be overwritten.

Dashboard: records search + voice filter, log panel falls back to
polling, sign-in history with CSV/JSON export, and JS errors now show
on the page instead of silently blanking a tab.
2026-09-02 22:56:18 +04:00

263 lines
10 KiB
Python

"""Live Gemini — browser-side realtime conversation with a chosen robot voice.
Why the browser talks to Gemini directly instead of through this server:
this deployment sits behind an Apache `[P]` rewrite that cannot upgrade a
WebSocket (measured — the same handshake answers 101 straight to uvicorn and
404 through the proxy), so a server-side relay of live audio is impossible
here. Instead the server mints a short-lived **ephemeral auth token** and the
browser opens its own socket to Gemini with that. The real API key never
reaches the page.
Endpoints:
GET /api/live/config voices, model, and the persona for each
GET /api/live/persona one voice's persona text
POST /api/live/persona edit it (persisted, survives restarts)
POST /api/live/token mint an ephemeral token for a session
"""
from __future__ import annotations
import json
import urllib.error
import urllib.request
from datetime import datetime, timedelta
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from Project.Sanad.config import BASE_DIR
from Project.Sanad.core.logger import get_logger
router = APIRouter()
log = get_logger("live_route")
PERSONA_FILE = BASE_DIR / "data" / "live_personas.json"
try:
from Project.Sanad.core.config_loader import section as _cfg_section
_LIVE_CFG = _cfg_section("voice", "live") or {}
except Exception:
_LIVE_CFG = {}
# OFF by default: enabling it puts the real API key in the browser. See
# mint_token() for why the safer ephemeral path is unavailable on this key.
_ALLOW_DIRECT_KEY = bool(_LIVE_CFG.get("allow_direct_key", False))
# Ephemeral tokens are minted per session and expire quickly; `uses` covers the
# single connect. Keep the window tight — the token is handed to a browser.
_TOKEN_TTL_MIN = 30
_SESSION_START_WINDOW_MIN = 2
_REST_BASE = "https://generativelanguage.googleapis.com"
# Ephemeral tokens work ONLY on v1beta (documented). Minting on v1alpha
# succeeds but the resulting token is rejected at connect time with
# "API key not valid" / "unregistered callers", which is what the first
# attempt at this feature hit.
_API_VERSION = "v1beta"
def _voice_options() -> list[dict]:
"""Reuse the voice picker's table so both tabs name the robots identically."""
try:
from Project.Sanad.dashboard.routes.voice import VOICE_OPTIONS
return [dict(o) for o in VOICE_OPTIONS]
except Exception:
return [{"voice": "Charon", "label": "Unitree G1"},
{"voice": "Puck", "label": "Unitree R1"},
{"voice": "Kore", "label": "Agibot x2"}]
def _store():
from Project.Sanad.dashboard.routes.live_personas import load_store
return load_store()
@router.get("/config")
async def live_config():
"""Voices, model, and which persona each robot is currently using."""
from Project.Sanad.config import GEMINI_MODEL
from Project.Sanad.dashboard.routes import live_personas as lp
store = _store()
voices = []
for opt in _voice_options():
pid = lp.active_for(store, opt["voice"])
voices.append(dict(opt,
persona_id=pid,
persona_name=lp.persona_name(store, pid),
persona=lp.persona_text(store, pid)))
return {"model": GEMINI_MODEL, "api_version": _API_VERSION, "voices": voices}
@router.get("/personas")
async def list_personas():
"""The whole library, plus the active selection per robot."""
from Project.Sanad.dashboard.routes import live_personas as lp
store = _store()
return {"personas": lp.all_personas(store),
"active": {o["voice"]: lp.active_for(store, o["voice"])
for o in _voice_options()}}
class PersonaSave(BaseModel):
id: str = "" # empty -> create a new one
name: str
text: str
@router.post("/personas")
async def save_persona(payload: PersonaSave):
"""Create a persona, or update a saved one. Built-ins are never modified —
editing one saves a copy instead, so the original stays available."""
from Project.Sanad.dashboard.routes import live_personas as lp
from datetime import datetime
name = payload.name.strip() or "Untitled persona"
text = payload.text.strip()
if not text:
raise HTTPException(400, "Persona text cannot be empty")
if len(text.encode("utf-8")) > 20000:
raise HTTPException(413, "Persona too large (max 20000 bytes)")
store = _store()
personas = store.setdefault("personas", {})
pid = payload.id.strip()
if not pid or pid in lp.BUILTIN:
pid = lp.new_id(name, personas)
personas[pid] = {"name": name, "text": text,
"updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
lp.save_store(store)
log.info("persona saved: %s (%s, %d chars)", name, pid, len(text))
return {"ok": True, "id": pid, "name": name}
class PersonaSelect(BaseModel):
voice: str
id: str
@router.post("/personas/select")
async def select_persona(payload: PersonaSelect):
"""Choose which persona a robot uses. Applies on the next connection."""
from Project.Sanad.dashboard.routes import live_personas as lp
store = _store()
known = {p["id"] for p in lp.all_personas(store)}
if payload.id not in known:
raise HTTPException(404, f"No persona {payload.id!r}")
store.setdefault("active", {})[payload.voice] = payload.id
lp.save_store(store)
log.info("persona for %s set to %s", payload.voice, payload.id)
return {"ok": True, "voice": payload.voice, "id": payload.id,
"name": lp.persona_name(store, payload.id)}
class PersonaDelete(BaseModel):
id: str
@router.post("/personas/delete")
async def delete_persona(payload: PersonaDelete):
"""Remove a saved persona. Any robot using it falls back to its built-in."""
from Project.Sanad.dashboard.routes import live_personas as lp
store = _store()
if payload.id in lp.BUILTIN:
raise HTTPException(400, "Built-in personas cannot be deleted")
if payload.id not in (store.get("personas") or {}):
raise HTTPException(404, f"No persona {payload.id!r}")
del store["personas"][payload.id]
reassigned = []
for voice, pid in list((store.get("active") or {}).items()):
if pid == payload.id:
store["active"][voice] = lp._DEFAULT_FOR_VOICE.get(voice, "")
reassigned.append(voice)
lp.save_store(store)
return {"ok": True, "deleted": payload.id, "reassigned": reassigned}
def _mint_token_sync(api_key: str) -> dict:
"""Blocking POST to auth_tokens. Returns the raw API response."""
now = datetime.utcnow()
body = json.dumps({
"uses": 1,
"expireTime": (now + timedelta(minutes=_TOKEN_TTL_MIN)).strftime("%Y-%m-%dT%H:%M:%SZ"),
"newSessionExpireTime":
(now + timedelta(minutes=_SESSION_START_WINDOW_MIN)).strftime("%Y-%m-%dT%H:%M:%SZ"),
}).encode("utf-8")
url = f"{_REST_BASE}/{_API_VERSION}/auth_tokens?key={api_key}"
req = urllib.request.Request(url, data=body,
headers={"Content-Type": "application/json"},
method="POST")
try:
with urllib.request.urlopen(req, timeout=20) as resp:
return json.loads(resp.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = ""
try:
detail = json.loads(exc.read().decode("utf-8")).get("error", {}).get("message", "")
except Exception:
pass
raise RuntimeError(f"HTTP {exc.code}: {detail or exc.reason}")
except urllib.error.URLError as exc:
raise RuntimeError(f"request failed: {exc.reason}")
class TokenPayload(BaseModel):
voice: str = "Charon"
@router.post("/token")
async def mint_token(payload: TokenPayload):
"""Credential for the browser's own Gemini socket.
Ephemeral tokens are the safe path — the key stays on the server — but they
are NOT accepted for keys of the "AQ." AI-Studio express kind that this
deployment uses. Measured: the express key opens a Live session directly on
both v1alpha and v1beta, while a token minted from it is refused in all four
documented presentations (access_token encoded/raw, Authorization: Token,
key=). So live conversation here needs the real key in the browser, which is
a deliberate decision and stays OFF until switched on in config:
voice_config.json > live.allow_direct_key = true
Turning it on means anyone who can open the dashboard (or its dev tools) can
read the key and spend against it. Getting a standard `AIza…` key instead
would restore the ephemeral path and make this unnecessary.
"""
import asyncio
from Project.Sanad.config import GEMINI_MODEL
import Project.Sanad.gemini.client as _gc
api_key = getattr(_gc, "GEMINI_API_KEY", "") or ""
if not api_key:
raise HTTPException(503, "No Gemini API key configured.")
from Project.Sanad.dashboard.routes import live_personas as lp
store = _store()
voice = payload.voice or "Charon"
persona_id = lp.active_for(store, voice)
common = {
"model": GEMINI_MODEL,
"api_version": _API_VERSION,
"voice": voice,
"persona": lp.persona_text(store, persona_id),
"persona_name": lp.persona_name(store, persona_id),
}
if _ALLOW_DIRECT_KEY:
log.warning("live session: handing the API key to the browser "
"(live.allow_direct_key is enabled)")
return dict(common, token=api_key, auth_param="key",
api_version="v1alpha", direct_key=True, expires_in_sec=0)
try:
data = await asyncio.to_thread(_mint_token_sync, api_key)
except RuntimeError as exc:
msg = str(exc)
log.error("could not mint ephemeral token: %s", msg)
if "credits are depleted" in msg or "billing" in msg.lower():
raise HTTPException(503, "Gemini API credits are depleted — top up billing.")
raise HTTPException(502, f"Could not start a live session: {msg}")
name = data.get("name", "")
if not name:
raise HTTPException(502, "Gemini returned no token")
log.info("live session token minted for %s", voice)
return dict(common, token=name, auth_param="access_token", direct_key=False,
expires_in_sec=_TOKEN_TTL_MIN * 60)