Replay now matches the robots and no longer cuts words: - read the turn to turnComplete, not generationComplete, and drain the socket before each send; breaking early truncated every sentence and left frames that the next turn mis-read as its own reply - accept a take only if the model's own transcript covers the text AND the audio is long enough to contain it (the transcript reports the full text even for a 0.8s clip) - pitch gate: reject an off-tone take and re-ask, per voice, using a pure-Python F0 estimator (no numpy on the host) - continuation: speak the words a voice skipped instead of retrying a line it stops on deterministically - fresh Live session per replay; delivery drifts as turns accumulate Live Gemini tab: browser talks to Gemini directly (the reverse proxy cannot upgrade a WebSocket), with a persona library - named personas, per-robot selection, built-ins that cannot be overwritten. Dashboard: records search + voice filter, log panel falls back to polling, sign-in history with CSV/JSON export, and JS errors now show on the page instead of silently blanking a tab.
263 lines
10 KiB
Python
263 lines
10 KiB
Python
"""Live Gemini — browser-side realtime conversation with a chosen robot voice.
|
|
|
|
Why the browser talks to Gemini directly instead of through this server:
|
|
this deployment sits behind an Apache `[P]` rewrite that cannot upgrade a
|
|
WebSocket (measured — the same handshake answers 101 straight to uvicorn and
|
|
404 through the proxy), so a server-side relay of live audio is impossible
|
|
here. Instead the server mints a short-lived **ephemeral auth token** and the
|
|
browser opens its own socket to Gemini with that. The real API key never
|
|
reaches the page.
|
|
|
|
Endpoints:
|
|
GET /api/live/config voices, model, and the persona for each
|
|
GET /api/live/persona one voice's persona text
|
|
POST /api/live/persona edit it (persisted, survives restarts)
|
|
POST /api/live/token mint an ephemeral token for a session
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timedelta
|
|
|
|
from fastapi import APIRouter, HTTPException
|
|
from pydantic import BaseModel
|
|
|
|
from Project.Sanad.config import BASE_DIR
|
|
from Project.Sanad.core.logger import get_logger
|
|
|
|
router = APIRouter()
|
|
log = get_logger("live_route")
|
|
|
|
PERSONA_FILE = BASE_DIR / "data" / "live_personas.json"
|
|
|
|
try:
|
|
from Project.Sanad.core.config_loader import section as _cfg_section
|
|
_LIVE_CFG = _cfg_section("voice", "live") or {}
|
|
except Exception:
|
|
_LIVE_CFG = {}
|
|
# OFF by default: enabling it puts the real API key in the browser. See
|
|
# mint_token() for why the safer ephemeral path is unavailable on this key.
|
|
_ALLOW_DIRECT_KEY = bool(_LIVE_CFG.get("allow_direct_key", False))
|
|
|
|
# Ephemeral tokens are minted per session and expire quickly; `uses` covers the
|
|
# single connect. Keep the window tight — the token is handed to a browser.
|
|
_TOKEN_TTL_MIN = 30
|
|
_SESSION_START_WINDOW_MIN = 2
|
|
_REST_BASE = "https://generativelanguage.googleapis.com"
|
|
# Ephemeral tokens work ONLY on v1beta (documented). Minting on v1alpha
|
|
# succeeds but the resulting token is rejected at connect time with
|
|
# "API key not valid" / "unregistered callers", which is what the first
|
|
# attempt at this feature hit.
|
|
_API_VERSION = "v1beta"
|
|
|
|
def _voice_options() -> list[dict]:
|
|
"""Reuse the voice picker's table so both tabs name the robots identically."""
|
|
try:
|
|
from Project.Sanad.dashboard.routes.voice import VOICE_OPTIONS
|
|
return [dict(o) for o in VOICE_OPTIONS]
|
|
except Exception:
|
|
return [{"voice": "Charon", "label": "Unitree G1"},
|
|
{"voice": "Puck", "label": "Unitree R1"},
|
|
{"voice": "Kore", "label": "Agibot x2"}]
|
|
|
|
|
|
def _store():
|
|
from Project.Sanad.dashboard.routes.live_personas import load_store
|
|
return load_store()
|
|
|
|
|
|
@router.get("/config")
|
|
async def live_config():
|
|
"""Voices, model, and which persona each robot is currently using."""
|
|
from Project.Sanad.config import GEMINI_MODEL
|
|
from Project.Sanad.dashboard.routes import live_personas as lp
|
|
store = _store()
|
|
voices = []
|
|
for opt in _voice_options():
|
|
pid = lp.active_for(store, opt["voice"])
|
|
voices.append(dict(opt,
|
|
persona_id=pid,
|
|
persona_name=lp.persona_name(store, pid),
|
|
persona=lp.persona_text(store, pid)))
|
|
return {"model": GEMINI_MODEL, "api_version": _API_VERSION, "voices": voices}
|
|
|
|
|
|
@router.get("/personas")
|
|
async def list_personas():
|
|
"""The whole library, plus the active selection per robot."""
|
|
from Project.Sanad.dashboard.routes import live_personas as lp
|
|
store = _store()
|
|
return {"personas": lp.all_personas(store),
|
|
"active": {o["voice"]: lp.active_for(store, o["voice"])
|
|
for o in _voice_options()}}
|
|
|
|
|
|
class PersonaSave(BaseModel):
|
|
id: str = "" # empty -> create a new one
|
|
name: str
|
|
text: str
|
|
|
|
|
|
@router.post("/personas")
|
|
async def save_persona(payload: PersonaSave):
|
|
"""Create a persona, or update a saved one. Built-ins are never modified —
|
|
editing one saves a copy instead, so the original stays available."""
|
|
from Project.Sanad.dashboard.routes import live_personas as lp
|
|
from datetime import datetime
|
|
name = payload.name.strip() or "Untitled persona"
|
|
text = payload.text.strip()
|
|
if not text:
|
|
raise HTTPException(400, "Persona text cannot be empty")
|
|
if len(text.encode("utf-8")) > 20000:
|
|
raise HTTPException(413, "Persona too large (max 20000 bytes)")
|
|
|
|
store = _store()
|
|
personas = store.setdefault("personas", {})
|
|
pid = payload.id.strip()
|
|
if not pid or pid in lp.BUILTIN:
|
|
pid = lp.new_id(name, personas)
|
|
personas[pid] = {"name": name, "text": text,
|
|
"updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
|
|
lp.save_store(store)
|
|
log.info("persona saved: %s (%s, %d chars)", name, pid, len(text))
|
|
return {"ok": True, "id": pid, "name": name}
|
|
|
|
|
|
class PersonaSelect(BaseModel):
|
|
voice: str
|
|
id: str
|
|
|
|
|
|
@router.post("/personas/select")
|
|
async def select_persona(payload: PersonaSelect):
|
|
"""Choose which persona a robot uses. Applies on the next connection."""
|
|
from Project.Sanad.dashboard.routes import live_personas as lp
|
|
store = _store()
|
|
known = {p["id"] for p in lp.all_personas(store)}
|
|
if payload.id not in known:
|
|
raise HTTPException(404, f"No persona {payload.id!r}")
|
|
store.setdefault("active", {})[payload.voice] = payload.id
|
|
lp.save_store(store)
|
|
log.info("persona for %s set to %s", payload.voice, payload.id)
|
|
return {"ok": True, "voice": payload.voice, "id": payload.id,
|
|
"name": lp.persona_name(store, payload.id)}
|
|
|
|
|
|
class PersonaDelete(BaseModel):
|
|
id: str
|
|
|
|
|
|
@router.post("/personas/delete")
|
|
async def delete_persona(payload: PersonaDelete):
|
|
"""Remove a saved persona. Any robot using it falls back to its built-in."""
|
|
from Project.Sanad.dashboard.routes import live_personas as lp
|
|
store = _store()
|
|
if payload.id in lp.BUILTIN:
|
|
raise HTTPException(400, "Built-in personas cannot be deleted")
|
|
if payload.id not in (store.get("personas") or {}):
|
|
raise HTTPException(404, f"No persona {payload.id!r}")
|
|
del store["personas"][payload.id]
|
|
reassigned = []
|
|
for voice, pid in list((store.get("active") or {}).items()):
|
|
if pid == payload.id:
|
|
store["active"][voice] = lp._DEFAULT_FOR_VOICE.get(voice, "")
|
|
reassigned.append(voice)
|
|
lp.save_store(store)
|
|
return {"ok": True, "deleted": payload.id, "reassigned": reassigned}
|
|
|
|
|
|
def _mint_token_sync(api_key: str) -> dict:
|
|
"""Blocking POST to auth_tokens. Returns the raw API response."""
|
|
now = datetime.utcnow()
|
|
body = json.dumps({
|
|
"uses": 1,
|
|
"expireTime": (now + timedelta(minutes=_TOKEN_TTL_MIN)).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"newSessionExpireTime":
|
|
(now + timedelta(minutes=_SESSION_START_WINDOW_MIN)).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
}).encode("utf-8")
|
|
url = f"{_REST_BASE}/{_API_VERSION}/auth_tokens?key={api_key}"
|
|
req = urllib.request.Request(url, data=body,
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST")
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=20) as resp:
|
|
return json.loads(resp.read().decode("utf-8"))
|
|
except urllib.error.HTTPError as exc:
|
|
detail = ""
|
|
try:
|
|
detail = json.loads(exc.read().decode("utf-8")).get("error", {}).get("message", "")
|
|
except Exception:
|
|
pass
|
|
raise RuntimeError(f"HTTP {exc.code}: {detail or exc.reason}")
|
|
except urllib.error.URLError as exc:
|
|
raise RuntimeError(f"request failed: {exc.reason}")
|
|
|
|
|
|
class TokenPayload(BaseModel):
|
|
voice: str = "Charon"
|
|
|
|
|
|
@router.post("/token")
|
|
async def mint_token(payload: TokenPayload):
|
|
"""Credential for the browser's own Gemini socket.
|
|
|
|
Ephemeral tokens are the safe path — the key stays on the server — but they
|
|
are NOT accepted for keys of the "AQ." AI-Studio express kind that this
|
|
deployment uses. Measured: the express key opens a Live session directly on
|
|
both v1alpha and v1beta, while a token minted from it is refused in all four
|
|
documented presentations (access_token encoded/raw, Authorization: Token,
|
|
key=). So live conversation here needs the real key in the browser, which is
|
|
a deliberate decision and stays OFF until switched on in config:
|
|
|
|
voice_config.json > live.allow_direct_key = true
|
|
|
|
Turning it on means anyone who can open the dashboard (or its dev tools) can
|
|
read the key and spend against it. Getting a standard `AIza…` key instead
|
|
would restore the ephemeral path and make this unnecessary.
|
|
"""
|
|
import asyncio
|
|
|
|
from Project.Sanad.config import GEMINI_MODEL
|
|
import Project.Sanad.gemini.client as _gc
|
|
|
|
api_key = getattr(_gc, "GEMINI_API_KEY", "") or ""
|
|
if not api_key:
|
|
raise HTTPException(503, "No Gemini API key configured.")
|
|
|
|
from Project.Sanad.dashboard.routes import live_personas as lp
|
|
store = _store()
|
|
voice = payload.voice or "Charon"
|
|
persona_id = lp.active_for(store, voice)
|
|
common = {
|
|
"model": GEMINI_MODEL,
|
|
"api_version": _API_VERSION,
|
|
"voice": voice,
|
|
"persona": lp.persona_text(store, persona_id),
|
|
"persona_name": lp.persona_name(store, persona_id),
|
|
}
|
|
|
|
if _ALLOW_DIRECT_KEY:
|
|
log.warning("live session: handing the API key to the browser "
|
|
"(live.allow_direct_key is enabled)")
|
|
return dict(common, token=api_key, auth_param="key",
|
|
api_version="v1alpha", direct_key=True, expires_in_sec=0)
|
|
|
|
try:
|
|
data = await asyncio.to_thread(_mint_token_sync, api_key)
|
|
except RuntimeError as exc:
|
|
msg = str(exc)
|
|
log.error("could not mint ephemeral token: %s", msg)
|
|
if "credits are depleted" in msg or "billing" in msg.lower():
|
|
raise HTTPException(503, "Gemini API credits are depleted — top up billing.")
|
|
raise HTTPException(502, f"Could not start a live session: {msg}")
|
|
|
|
name = data.get("name", "")
|
|
if not name:
|
|
raise HTTPException(502, "Gemini returned no token")
|
|
|
|
log.info("live session token minted for %s", voice)
|
|
return dict(common, token=name, auth_param="access_token", direct_key=False,
|
|
expires_in_sec=_TOKEN_TTL_MIN * 60)
|