d9b2d5427f
Replay now matches the robots and no longer cuts words: - read the turn to turnComplete, not generationComplete, and drain the socket before each send; breaking early truncated every sentence and left frames that the next turn mis-read as its own reply - accept a take only if the model's own transcript covers the text AND the audio is long enough to contain it (the transcript reports the full text even for a 0.8s clip) - pitch gate: reject an off-tone take and re-ask, per voice, using a pure-Python F0 estimator (no numpy on the host) - continuation: speak the words a voice skipped instead of retrying a line it stops on deterministically - fresh Live session per replay; delivery drifts as turns accumulate Live Gemini tab: browser talks to Gemini directly (the reverse proxy cannot upgrade a WebSocket), with a persona library - named personas, per-robot selection, built-ins that cannot be overwritten. Dashboard: records search + voice filter, log panel falls back to polling, sign-in history with CSV/JSON export, and JS errors now show on the page instead of silently blanking a tab.
390 lines
15 KiB
Python
390 lines
15 KiB
Python
"""Voice endpoints — Gemini interaction, local TTS, prompt management."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
from fastapi import APIRouter, HTTPException
|
|
from pydantic import BaseModel
|
|
|
|
from Project.Sanad.core.config_loader import section as _cfg_section
|
|
from Project.Sanad.core.logger import get_logger
|
|
|
|
log = get_logger("voice_route")
|
|
|
|
router = APIRouter()
|
|
|
|
_VR = _cfg_section("dashboard", "voice_route")
|
|
_API = _cfg_section("dashboard", "api_input")
|
|
# MAX_TEXT_LEN — SINGLE SOURCE in dashboard.api_input
|
|
MAX_TEXT_LEN = _API.get("max_text_len", 2000)
|
|
_API_KEY_MASK_VISIBLE = _VR.get("api_key_mask_visible", 4)
|
|
|
|
|
|
def _mask_api_key(key: str) -> str:
|
|
"""Mask an API key for display — keeps 4 chars on each end.
|
|
|
|
Examples:
|
|
"" → ""
|
|
"AIza123" → "*******" (≤8 chars = full mask)
|
|
"AIzaSy...kqf8" → "AIza***...kqf8" (>8 chars = partial mask)
|
|
"""
|
|
if not key:
|
|
return ""
|
|
if len(key) <= 8:
|
|
return "*" * len(key)
|
|
return f"{key[:4]}{'*' * (len(key) - 8)}{key[-4:]}"
|
|
|
|
|
|
class TextPayload(BaseModel):
|
|
text: str
|
|
engine: str = "gemini" # "gemini" | "local"
|
|
|
|
|
|
@router.get("/status")
|
|
async def voice_status():
|
|
from Project.Sanad.main import voice_client, local_tts
|
|
return {
|
|
"gemini": voice_client.status() if voice_client else {},
|
|
"local_tts": local_tts.status() if local_tts else {},
|
|
}
|
|
|
|
|
|
@router.post("/generate")
|
|
async def generate_speech(payload: TextPayload):
|
|
"""Generate speech from text using Gemini or local TTS."""
|
|
if not payload.text.strip():
|
|
raise HTTPException(400, "Text cannot be empty.")
|
|
if len(payload.text) > MAX_TEXT_LEN:
|
|
raise HTTPException(413, f"Text too long (max {MAX_TEXT_LEN} chars).")
|
|
|
|
from Project.Sanad.main import voice_client, local_tts, audio_mgr
|
|
|
|
if payload.engine == "local":
|
|
if local_tts is None:
|
|
raise HTTPException(503, "Local TTS not available.")
|
|
pcm = await asyncio.to_thread(local_tts.synthesize, payload.text)
|
|
if audio_mgr:
|
|
await asyncio.to_thread(audio_mgr.play_pcm, pcm, 1, 16000, 2)
|
|
return {
|
|
"ok": True,
|
|
"engine": "local",
|
|
"duration_sec": round(len(pcm) / (16000 * 2), 3),
|
|
}
|
|
else:
|
|
if voice_client is None:
|
|
raise HTTPException(503, "Voice client not initialized.")
|
|
if not voice_client.connected:
|
|
try:
|
|
await voice_client.connect()
|
|
except Exception:
|
|
log.exception("Gemini reconnect failed in /generate")
|
|
raise HTTPException(503, "Gemini not connected and reconnect failed.")
|
|
# Check session ownership — TypedReplay or live loop may hold it
|
|
if voice_client.session_owner is not None:
|
|
raise HTTPException(
|
|
409,
|
|
f"Voice session busy (owned by {voice_client.session_owner})",
|
|
)
|
|
try:
|
|
audio_bytes, text_parts = await voice_client.send_text(
|
|
payload.text, owner="voice_route"
|
|
)
|
|
except RuntimeError as exc:
|
|
raise HTTPException(503, str(exc))
|
|
except Exception as exc:
|
|
raise HTTPException(502, f"Gemini communication error: {exc}")
|
|
if audio_bytes and audio_mgr:
|
|
await asyncio.to_thread(audio_mgr.play_pcm, audio_bytes, 1, 24000, 2)
|
|
return {
|
|
"ok": True,
|
|
"engine": "gemini",
|
|
"has_audio": bool(audio_bytes),
|
|
"text_response": text_parts,
|
|
}
|
|
|
|
|
|
@router.post("/connect")
|
|
async def connect_gemini():
|
|
from Project.Sanad.main import voice_client
|
|
if voice_client is None:
|
|
raise HTTPException(503, "Voice client not initialized.")
|
|
try:
|
|
await voice_client.connect()
|
|
except Exception as exc:
|
|
raise HTTPException(502, f"Gemini connection failed: {exc}")
|
|
return {"connected": voice_client.connected}
|
|
|
|
|
|
@router.post("/disconnect")
|
|
async def disconnect_gemini():
|
|
from Project.Sanad.main import voice_client
|
|
if voice_client:
|
|
await voice_client.disconnect()
|
|
return {"connected": False}
|
|
|
|
|
|
# ─────────────────────── Gemini API key management ───────────────────────
|
|
|
|
class ApiKeyPayload(BaseModel):
|
|
api_key: str
|
|
|
|
|
|
@router.get("/api-key")
|
|
async def get_api_key():
|
|
"""Return the current Gemini API key in masked form.
|
|
|
|
The key now comes EXCLUSIVELY from the `api` env var (sourced from
|
|
cPanel "Setup Python App → Environment variables"). This endpoint
|
|
is read-only.
|
|
"""
|
|
import Project.Sanad.config as cfg_mod
|
|
key = getattr(cfg_mod, "GEMINI_API_KEY", "") or ""
|
|
return {
|
|
"has_key": bool(key),
|
|
"masked": _mask_api_key(key),
|
|
"length": len(key),
|
|
"source": "env:api",
|
|
"read_only": True,
|
|
}
|
|
|
|
|
|
@router.post("/api-key")
|
|
async def update_api_key(payload: ApiKeyPayload):
|
|
"""API key updates via dashboard are DISABLED.
|
|
|
|
Source of truth is the cPanel "Setup Python App → Environment variables"
|
|
form (env var `api`). To rotate the key, edit that form and restart
|
|
uvicorn. The dashboard intentionally cannot bypass this.
|
|
"""
|
|
raise HTTPException(
|
|
403,
|
|
"API key updates are disabled. Edit it in cPanel → Setup Python App "
|
|
"→ Environment variables → 'api', then restart uvicorn."
|
|
)
|
|
|
|
|
|
# Original update logic kept below for reference but no longer reachable.
|
|
async def _legacy_update_api_key(payload: ApiKeyPayload):
|
|
"""Update the Gemini API key — persists to data/motions/config.json and
|
|
hot-swaps the in-memory value so the next Gemini connect uses it.
|
|
|
|
Also disconnects any currently-connected Gemini session so that the
|
|
next reconnect picks up the new key cleanly. Returns the NEW masked
|
|
key + a flag telling the dashboard to trigger a reconnect.
|
|
"""
|
|
key = payload.api_key.strip()
|
|
if not key:
|
|
raise HTTPException(400, "API key cannot be empty.")
|
|
if len(key) < 20:
|
|
raise HTTPException(400, "API key looks too short.")
|
|
if not key.startswith("AIza"):
|
|
raise HTTPException(
|
|
400,
|
|
"Gemini API keys normally start with 'AIza'. "
|
|
"Double-check you're pasting a Google AI Studio key.",
|
|
)
|
|
|
|
# Persist to data/motions/config.json (atomic temp-then-replace)
|
|
try:
|
|
from Project.Sanad.config import load_config, save_config
|
|
cfg = load_config() or {}
|
|
gemini_cfg = cfg.get("gemini") if isinstance(cfg.get("gemini"), dict) else {}
|
|
gemini_cfg["api_key"] = key
|
|
cfg["gemini"] = gemini_cfg
|
|
save_config(cfg)
|
|
except Exception as exc:
|
|
log.exception("Failed to persist API key to config.json")
|
|
raise HTTPException(500, f"Could not save config: {exc}")
|
|
|
|
# Hot-swap the in-memory module globals.
|
|
# Both Project.Sanad.config AND Project.Sanad.gemini.client
|
|
# have their OWN reference to GEMINI_API_KEY (the latter was created
|
|
# at `from Project.Sanad.config import GEMINI_API_KEY` at import time).
|
|
# Python's `from X import Y` binds a local name — updating config.Y
|
|
# alone does NOT propagate to the importer, so we must patch both.
|
|
try:
|
|
import Project.Sanad.config as _cfg_mod
|
|
_cfg_mod.GEMINI_API_KEY = key
|
|
except Exception:
|
|
log.exception("could not patch config.GEMINI_API_KEY")
|
|
|
|
try:
|
|
import Project.Sanad.gemini.client as _gc
|
|
_gc.GEMINI_API_KEY = key
|
|
except Exception:
|
|
log.exception("could not patch gemini.client.GEMINI_API_KEY")
|
|
|
|
# Disconnect any live session so reconnect uses the new key.
|
|
from Project.Sanad.main import voice_client
|
|
was_connected = False
|
|
if voice_client is not None:
|
|
was_connected = bool(getattr(voice_client, "connected", False))
|
|
if was_connected:
|
|
try:
|
|
await voice_client.disconnect()
|
|
except Exception:
|
|
log.exception("disconnect during api-key swap failed")
|
|
|
|
log.info("Gemini API key updated (length=%d) source=config_file", len(key))
|
|
|
|
return {
|
|
"ok": True,
|
|
"masked": _mask_api_key(key),
|
|
"length": len(key),
|
|
"source": "config_file",
|
|
"was_connected": was_connected,
|
|
"message": (
|
|
"API key saved. Click 'Connect' to reopen the Gemini session with "
|
|
"the new key. Any running Live Gemini subprocess must be restarted "
|
|
"separately (Stop → Start) to pick up the new key."
|
|
),
|
|
}
|
|
|
|
|
|
# ─────────────────────── Gemini voice (robot) selection ───────────────────────
|
|
# Friendly robot labels mapped to Gemini prebuilt voice names. Source of truth
|
|
# for the dashboard voice picker — each robot maps to one Gemini voice.
|
|
# Optionally overridable from core_config.json → gemini_defaults.voice_options
|
|
# (a list of {"voice","label"} dicts) without touching code.
|
|
_DEFAULT_VOICE_OPTIONS = [
|
|
{"voice": "Charon", "label": "Unitree G1"},
|
|
{"voice": "Puck", "label": "Unitree R1"},
|
|
{"voice": "Kore", "label": "Agibot x2"},
|
|
]
|
|
|
|
|
|
def _load_voice_options() -> list[dict]:
|
|
raw = _cfg_section("core", "gemini_defaults").get("voice_options")
|
|
if isinstance(raw, list) and raw and all(
|
|
isinstance(o, dict) and o.get("voice") and o.get("label") for o in raw
|
|
):
|
|
return [{"voice": str(o["voice"]), "label": str(o["label"])} for o in raw]
|
|
return list(_DEFAULT_VOICE_OPTIONS)
|
|
|
|
|
|
VOICE_OPTIONS = _load_voice_options()
|
|
_VOICE_BY_NAME = {o["voice"]: o for o in VOICE_OPTIONS}
|
|
|
|
|
|
def _current_voice() -> str:
|
|
"""The voice the Gemini client will actually use on its next connect()."""
|
|
try:
|
|
import Project.Sanad.gemini.client as _gc
|
|
return getattr(_gc, "GEMINI_VOICE", "") or ""
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
class VoicePayload(BaseModel):
|
|
voice: str
|
|
|
|
|
|
@router.get("/voices")
|
|
async def list_voices():
|
|
"""List selectable Gemini voices with robot labels + which one is active."""
|
|
cur = _current_voice()
|
|
cur_opt = _VOICE_BY_NAME.get(cur)
|
|
return {
|
|
"current": cur,
|
|
"current_label": cur_opt["label"] if cur_opt else cur,
|
|
"options": [
|
|
{"voice": o["voice"], "label": o["label"], "active": o["voice"] == cur}
|
|
for o in VOICE_OPTIONS
|
|
],
|
|
}
|
|
|
|
|
|
@router.post("/voice")
|
|
async def set_voice(payload: VoicePayload):
|
|
"""Switch the Gemini voice, persist it, and apply on the next connect.
|
|
|
|
Mirrors the api-key hot-swap: persist to data/motions/config.json
|
|
(gemini.voice) so it survives a restart, patch the in-memory GEMINI_VOICE
|
|
in BOTH config and the gemini client module (each holds its own binding
|
|
from `from ... import GEMINI_VOICE`), then disconnect any idle Gemini
|
|
session so the next connect() (e.g. the next Typed Replay) opens with it.
|
|
"""
|
|
voice = (payload.voice or "").strip()
|
|
if voice not in _VOICE_BY_NAME:
|
|
raise HTTPException(
|
|
400,
|
|
"Unknown voice %r. Allowed: %s"
|
|
% (voice, ", ".join(o["voice"] for o in VOICE_OPTIONS)),
|
|
)
|
|
|
|
# 1. Persist (atomic temp-then-replace) to data/motions/config.json.
|
|
# Run the blocking file I/O off the event-loop thread.
|
|
try:
|
|
from Project.Sanad.config import load_config, save_config
|
|
cfg = (await asyncio.to_thread(load_config)) or {}
|
|
gemini_cfg = cfg.get("gemini") if isinstance(cfg.get("gemini"), dict) else {}
|
|
gemini_cfg["voice"] = voice
|
|
cfg["gemini"] = gemini_cfg
|
|
await asyncio.to_thread(save_config, cfg)
|
|
except Exception as exc:
|
|
log.exception("Failed to persist voice to config.json")
|
|
raise HTTPException(500, f"Could not save voice: {exc}")
|
|
|
|
# 2. Hot-swap the in-memory globals in both modules that hold GEMINI_VOICE.
|
|
try:
|
|
import Project.Sanad.config as _cfg_mod
|
|
_cfg_mod.GEMINI_VOICE = voice
|
|
except Exception:
|
|
log.exception("could not patch config.GEMINI_VOICE")
|
|
try:
|
|
import Project.Sanad.gemini.client as _gc
|
|
_gc.GEMINI_VOICE = voice
|
|
except Exception:
|
|
log.exception("could not patch gemini.client.GEMINI_VOICE")
|
|
|
|
# 3. Reopen the Gemini session so the new voice actually takes effect. The
|
|
# voice is baked into connect()'s setup handshake and the persistent
|
|
# session is only rebuilt when connected is False, so we must drop it —
|
|
# otherwise the next Generate & Play reuses the still-open OLD-voice
|
|
# session (voice never changes). Acquire the session lock first so we
|
|
# WAIT for any in-flight generation to finish (it keeps the old voice,
|
|
# which is fine) instead of cutting it off mid-stream; then disconnect.
|
|
# Bound the wait so a stuck turn can't hang this request — the persisted
|
|
# + patched voice still applies on the next reconnect either way.
|
|
from Project.Sanad.main import voice_client
|
|
was_connected = False
|
|
if voice_client is not None:
|
|
was_connected = bool(getattr(voice_client, "connected", False))
|
|
if was_connected:
|
|
async def _reset_session():
|
|
async with voice_client.acquire_session("voice_switch"):
|
|
await voice_client.disconnect()
|
|
try:
|
|
await asyncio.wait_for(_reset_session(), timeout=45)
|
|
except asyncio.TimeoutError:
|
|
log.warning("voice swap: session busy too long; new voice applies on next reconnect")
|
|
except Exception:
|
|
# The polite path can fail for reasons that have nothing to do
|
|
# with the socket — notably the session lock being bound to a
|
|
# different event loop than the request runs on (py3.8 binds
|
|
# asyncio.Lock at construction, and voice_client is built at
|
|
# import time, before uvicorn's loop exists), which raises
|
|
# "got Future attached to a different loop" the moment the lock
|
|
# is contended. Swallowing that left the OLD-voice socket open,
|
|
# so the next Generate & Play spoke in the PREVIOUS robot's
|
|
# voice. Drop the socket unconditionally instead — worst case a
|
|
# concurrent generation reconnects.
|
|
log.warning("voice swap: graceful session reset failed — "
|
|
"forcing disconnect so the new voice applies",
|
|
exc_info=True)
|
|
try:
|
|
await voice_client.disconnect()
|
|
except Exception:
|
|
log.exception("voice swap: forced disconnect failed too")
|
|
|
|
label = _VOICE_BY_NAME[voice]["label"]
|
|
log.info("Gemini voice switched to %s (%s)", voice, label)
|
|
return {
|
|
"ok": True,
|
|
"voice": voice,
|
|
"label": label,
|
|
"was_connected": was_connected,
|
|
"message": f"Voice set to {label} ({voice}) — applies on the next Generate & Play.",
|
|
}
|