Voice fidelity, Live Gemini tab, and dashboard fixes
Replay now matches the robots and no longer cuts words: - read the turn to turnComplete, not generationComplete, and drain the socket before each send; breaking early truncated every sentence and left frames that the next turn mis-read as its own reply - accept a take only if the model's own transcript covers the text AND the audio is long enough to contain it (the transcript reports the full text even for a 0.8s clip) - pitch gate: reject an off-tone take and re-ask, per voice, using a pure-Python F0 estimator (no numpy on the host) - continuation: speak the words a voice skipped instead of retrying a line it stops on deterministically - fresh Live session per replay; delivery drifts as turns accumulate Live Gemini tab: browser talks to Gemini directly (the reverse proxy cannot upgrade a WebSocket), with a persona library - named personas, per-robot selection, built-ins that cannot be overwritten. Dashboard: records search + voice filter, log panel falls back to polling, sign-in history with CSV/JSON export, and JS errors now show on the page instead of silently blanking a tab.
This commit is contained in:
Sanad Lite
parent
a40b8fca0b
commit
d9b2d5427f
@ -1,6 +1,5 @@
|
||||
{
|
||||
"_description": "Tunables for core/* modules. Loaded via core.config_loader.load('core').",
|
||||
|
||||
"brain": {
|
||||
"allowed_callback_prefixes": [
|
||||
"Project.Sanad.voice.",
|
||||
@ -8,7 +7,6 @@
|
||||
],
|
||||
"gestural_speaking_default": false
|
||||
},
|
||||
|
||||
"logger": {
|
||||
"log_level": "INFO",
|
||||
"format": "%(asctime)s [%(name)s] %(levelname)-7s %(message)s",
|
||||
@ -16,11 +14,9 @@
|
||||
"file_max_bytes": 10485760,
|
||||
"file_backup_count": 7
|
||||
},
|
||||
|
||||
"event_bus": {
|
||||
"emit_timeout_sec": 0.5
|
||||
},
|
||||
|
||||
"paths": {
|
||||
"_comment": "Path roots — resolved against BASE_DIR in core/config.py",
|
||||
"data": "data",
|
||||
@ -31,7 +27,6 @@
|
||||
"motion_recordings": "data/recordings/motion",
|
||||
"motions": "data/motions"
|
||||
},
|
||||
|
||||
"gemini_defaults": {
|
||||
"_comment": "Baseline Gemini API config — SINGLE SOURCE OF TRUTH. All voice modules read from here.",
|
||||
"api_key": "",
|
||||
@ -39,28 +34,28 @@
|
||||
"model_ws_uri": "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent",
|
||||
"voice_name": "Charon",
|
||||
"ws_timeout_sec": 30,
|
||||
"default_system_prompt": "You are Sanad (Bousandah), a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'."
|
||||
"default_system_prompt": "You are Sanad (Bousandah), a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'.",
|
||||
"_comment_voice_prompts": "Per-voice system prompts. Charon (G1) and Puck (R1) are DELIBERATELY absent: they must keep the robots' verbatim TTS prompt so the site matches Sanadv3/SanadR1. Only Kore (Agibot x2), which has no robot to match, is steered here.",
|
||||
"voice_system_prompts": {
|
||||
"Kore": "أنتِ \"موزة\" (Muza) — روبوت إماراتي احترافي تابع لشركة YS Lootah Robotics، تتكلمين باللهجة الإماراتية (الخليجية) بنبرة احترافية راقية، هادئة وواثقة، بدون مبالغة ولا عبارات عاطفية زائدة.\nفي وضع الإعادة هذا مهمتكِ الوحيدة: انطقي النص الذي يعطيكِ إياه المستخدم كما هو تماماً، حرفياً، كلمة بكلمة، من أول كلمة إلى آخر كلمة، بنطقكِ الإماراتي الطبيعي.\nلا تجاوبي على النص، ولا تترجميه، ولا تلخصيه، ولا تعيدي صياغته، ولا تضيفي أي تحية أو تعليق أو كلمة زائدة، ولا تفكري بصوت مسموع.\nإذا كان النص بلغة أخرى فانطقيه بنفس لغته بدون تغيير.\nأخرجي الصوت المنطوق فقط، ثم اسكتي."
|
||||
}
|
||||
},
|
||||
|
||||
"g1_hardware": {
|
||||
"_comment": "G1 humanoid hardware constants — shared by every motion/voice module that talks to the arm.",
|
||||
"num_motor": 29,
|
||||
"enable_arm_sdk_index": 29,
|
||||
"replay_hz": 60.0
|
||||
},
|
||||
|
||||
"script_files": {
|
||||
"_comment": "Filenames (under scripts/) used across voice + dashboard",
|
||||
"persona": "sanad_script.txt",
|
||||
"rules": "sanad_rule.txt"
|
||||
},
|
||||
|
||||
"dashboard_defaults": {
|
||||
"host": null,
|
||||
"port": 8000,
|
||||
"interface": "wlan0"
|
||||
},
|
||||
|
||||
"audio_defaults": {
|
||||
"_comment": "Host PulseAudio fallback only — the G1 deployment uses UDP multicast mic + AudioClient.PlayStream speaker (see SANAD_USE_G1_MIC in config.py LIVE_TUNE). Default here is the Jetson/G1 built-in platform-sound chip.",
|
||||
"send_sample_rate": 16000,
|
||||
@ -70,11 +65,9 @@
|
||||
"sink": "alsa_output.platform-sound.analog-stereo",
|
||||
"source": "alsa_input.platform-sound.analog-stereo"
|
||||
},
|
||||
|
||||
"dds": {
|
||||
"network_interface_default": "eth0"
|
||||
},
|
||||
|
||||
"auth": {
|
||||
"_comment": "Dashboard login credentials. CHANGE before any non-LAN deployment.",
|
||||
"username": "lkasjda213h",
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
{
|
||||
"_description": "Tunables for gemini/client.py. All keys are optional — defaults live in code. Gemini credentials (api_key, model_live, voice_name) come from core_config.json's gemini_defaults — single source of truth.",
|
||||
|
||||
"client": {
|
||||
"_comment": "gemini/client.py — Gemini Live WebSocket client used by the typed-replay engine for one-shot TTS calls.",
|
||||
"recv_timeout_sec": 30,
|
||||
"reconnect_max_attempts": 3,
|
||||
"reconnect_initial_delay_sec": 1.0,
|
||||
"reconnect_max_delay_sec": 10.0
|
||||
}
|
||||
{
|
||||
"_description": "Tunables for gemini/client.py. All keys are optional — defaults live in code. Gemini credentials (api_key, model_live, voice_name) come from core_config.json\u0027s gemini_defaults — single source of truth.",
|
||||
"client": {
|
||||
"_comment": "gemini/client.py — Gemini Live WebSocket client used by the typed-replay engine for one-shot TTS calls.",
|
||||
"recv_timeout_sec": 30,
|
||||
"reconnect_max_attempts": 3,
|
||||
"reconnect_initial_delay_sec": 1.0,
|
||||
"reconnect_max_delay_sec": 10.0,
|
||||
"post_generation_grace_sec": 3,
|
||||
"output_transcription": true,
|
||||
"_comment_grace": "Seconds to keep reading after generationComplete, reset on every frame. 1.2 was too short: reads got cut and the leftover tail bled into the NEXT generation."
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,46 +1,46 @@
|
||||
{
|
||||
"_description": "Tunables for voice/* modules. Loaded via core.config_loader.load('voice').",
|
||||
|
||||
"speaker": {
|
||||
"_comment": "G1 built-in speaker — AudioClient.PlayStream wrapper",
|
||||
"app_name": "sanad",
|
||||
"begin_stream_pause_sec": 0.15,
|
||||
"wait_finish_margin_sec": 0.3
|
||||
},
|
||||
|
||||
"vad": {
|
||||
"_comment": "Gemini Live server-side voice-activity-detection config",
|
||||
"start_sensitivity": "START_SENSITIVITY_HIGH",
|
||||
"end_sensitivity": "END_SENSITIVITY_LOW",
|
||||
"prefix_padding_ms": 20,
|
||||
"silence_duration_ms": 200
|
||||
},
|
||||
|
||||
"barge_in": {
|
||||
"threshold": 500,
|
||||
"loud_chunks_needed": 3,
|
||||
"cooldown_sec": 0.3,
|
||||
"echo_suppress_below": 500,
|
||||
"ai_speak_grace_sec": 0.15
|
||||
},
|
||||
|
||||
"recording": {
|
||||
"enabled": true,
|
||||
"dir_relative": "data/recordings"
|
||||
},
|
||||
|
||||
"typed_replay": {
|
||||
"_comment": "voice/typed_replay.py — max_text_len comes from dashboard.api_input",
|
||||
"monitor_chunk_size": 512,
|
||||
"monitor_tail_sec": 0.2
|
||||
},
|
||||
|
||||
"local_tts": {
|
||||
"_comment": "voice/local_tts.py — offline Coqui TTS",
|
||||
"model_subdir": "speecht5_tts_clartts_ar",
|
||||
"vocoder_subdir": "speecht5_hifigan",
|
||||
"xvector_filename": "arabic_xvector_embedding.pt",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1
|
||||
}
|
||||
}
|
||||
"_description": "Tunables for voice/* modules. Loaded via core.config_loader.load('voice').",
|
||||
"speaker": {
|
||||
"_comment": "G1 built-in speaker — AudioClient.PlayStream wrapper",
|
||||
"app_name": "sanad",
|
||||
"begin_stream_pause_sec": 0.15,
|
||||
"wait_finish_margin_sec": 0.3
|
||||
},
|
||||
"vad": {
|
||||
"_comment": "Gemini Live server-side voice-activity-detection config",
|
||||
"start_sensitivity": "START_SENSITIVITY_HIGH",
|
||||
"end_sensitivity": "END_SENSITIVITY_LOW",
|
||||
"prefix_padding_ms": 20,
|
||||
"silence_duration_ms": 200
|
||||
},
|
||||
"barge_in": {
|
||||
"threshold": 500,
|
||||
"loud_chunks_needed": 3,
|
||||
"cooldown_sec": 0.3,
|
||||
"echo_suppress_below": 500,
|
||||
"ai_speak_grace_sec": 0.15
|
||||
},
|
||||
"recording": {
|
||||
"enabled": true,
|
||||
"dir_relative": "data/recordings"
|
||||
},
|
||||
"typed_replay": {
|
||||
"_comment": "voice/typed_replay.py — max_text_len comes from dashboard.api_input",
|
||||
"monitor_chunk_size": 512,
|
||||
"monitor_tail_sec": 0.2,
|
||||
"fresh_session_per_replay": true,
|
||||
"warm_session_voices": []
|
||||
},
|
||||
"local_tts": {
|
||||
"_comment": "voice/local_tts.py — offline Coqui TTS",
|
||||
"model_subdir": "speecht5_tts_clartts_ar",
|
||||
"vocoder_subdir": "speecht5_hifigan",
|
||||
"xvector_filename": "arabic_xvector_embedding.pt",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1
|
||||
},
|
||||
"live": {
|
||||
"allow_direct_key": true,
|
||||
"_comment": "User-authorised: hands the Gemini API key to the browser for live conversation. Ephemeral tokens are refused for AI-Studio AQ. express keys (verified: the key opens a Live session directly, a token minted from it is refused in all four documented forms). While this is on, anyone who can open the dashboard can read the key."
|
||||
}
|
||||
}
|
||||
@ -87,6 +87,7 @@ _REST_ROUTES: list[tuple[str, str, str]] = [
|
||||
("records", "/api/records", "records"),
|
||||
("prompt", "/api/prompt", "prompt"),
|
||||
("typed_replay", "/api/typed-replay", "typed-replay"),
|
||||
("live", "/api/live", "live"),
|
||||
]
|
||||
|
||||
_WS_ROUTES: list[str] = ["log_stream"]
|
||||
@ -146,7 +147,15 @@ async def root():
|
||||
if index.exists():
|
||||
from fastapi.responses import HTMLResponse
|
||||
try:
|
||||
return HTMLResponse(index.read_text(encoding="utf-8"))
|
||||
# Never cache the page itself. Static assets are served with a
|
||||
# 7-day max-age, so the ONLY thing that tells a browser about a new
|
||||
# live.js is the ?v= stamp inside this HTML — if the HTML is stale,
|
||||
# the browser keeps loading the old (possibly broken) asset and no
|
||||
# amount of reloading helps.
|
||||
return HTMLResponse(
|
||||
index.read_text(encoding="utf-8"),
|
||||
headers={"Cache-Control": "no-store, must-revalidate",
|
||||
"Pragma": "no-cache"})
|
||||
except OSError as exc:
|
||||
return {"error": f"Could not read index.html: {exc}"}
|
||||
return {
|
||||
|
||||
@ -7,7 +7,7 @@ The session is signed by Starlette's SessionMiddleware (stateless cookie).
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from Project.Sanad.config import BASE_DIR
|
||||
@ -28,6 +28,98 @@ def is_authed(request: Request) -> bool:
|
||||
return bool(request.session.get("user"))
|
||||
|
||||
|
||||
# ── login history ────────────────────────────────────────────────────
|
||||
# Kept in a small JSON file rather than only in the log, so the dashboard can
|
||||
# show "last login, from which device" without parsing log text.
|
||||
LOGIN_HISTORY = BASE_DIR / "data" / "logins.json"
|
||||
_MAX_HISTORY = 200
|
||||
|
||||
|
||||
def _client_ip(request: Request) -> str:
|
||||
"""Real client IP. The site sits behind Cloudflare and then an Apache
|
||||
reverse proxy, so request.client is always 127.0.0.1 — the forwarded
|
||||
headers are the only source of the actual address."""
|
||||
for header in ("cf-connecting-ip", "x-forwarded-for", "x-real-ip"):
|
||||
value = request.headers.get(header, "")
|
||||
if value:
|
||||
return value.split(",")[0].strip()
|
||||
return getattr(request.client, "host", "") or "?"
|
||||
|
||||
|
||||
def _describe_device(request: Request) -> str:
|
||||
"""Short human label for the User-Agent — "Chrome on Windows", not 200
|
||||
characters of tokens."""
|
||||
ua = request.headers.get("user-agent", "")
|
||||
if not ua:
|
||||
return "unknown device"
|
||||
low = ua.lower()
|
||||
if "android" in low:
|
||||
platform = "Android"
|
||||
elif "iphone" in low:
|
||||
platform = "iPhone"
|
||||
elif "ipad" in low:
|
||||
platform = "iPad"
|
||||
elif "windows" in low:
|
||||
platform = "Windows"
|
||||
elif "mac os" in low or "macintosh" in low:
|
||||
platform = "Mac"
|
||||
elif "linux" in low:
|
||||
platform = "Linux"
|
||||
else:
|
||||
platform = "unknown OS"
|
||||
# order matters: Edge/Opera also contain "chrome", Chrome contains "safari"
|
||||
if "edg/" in low:
|
||||
browser = "Edge"
|
||||
elif "opr/" in low or "opera" in low:
|
||||
browser = "Opera"
|
||||
elif "firefox" in low:
|
||||
browser = "Firefox"
|
||||
elif "chrome" in low or "crios" in low:
|
||||
browser = "Chrome"
|
||||
elif "safari" in low:
|
||||
browser = "Safari"
|
||||
elif "curl" in low:
|
||||
browser = "curl"
|
||||
elif "powershell" in low or "winhttp" in low:
|
||||
browser = "PowerShell"
|
||||
else:
|
||||
browser = "unknown browser"
|
||||
return f"{browser} on {platform}"
|
||||
|
||||
|
||||
def _load_logins() -> list:
|
||||
try:
|
||||
import json
|
||||
return json.loads(LOGIN_HISTORY.read_text(encoding="utf-8")) or []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _record_login(request: Request, username: str, ok: bool) -> None:
|
||||
"""Append one sign-in attempt. Never raises: a logging problem must not
|
||||
stop someone from signing in."""
|
||||
try:
|
||||
import json
|
||||
from datetime import datetime
|
||||
entries = _load_logins()
|
||||
entries.append({
|
||||
"at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"user": username,
|
||||
"ok": bool(ok),
|
||||
"device": _describe_device(request),
|
||||
"ip": _client_ip(request),
|
||||
"user_agent": request.headers.get("user-agent", "")[:300],
|
||||
})
|
||||
del entries[:-_MAX_HISTORY]
|
||||
LOGIN_HISTORY.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = LOGIN_HISTORY.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(entries, ensure_ascii=False, indent=1),
|
||||
encoding="utf-8")
|
||||
tmp.replace(LOGIN_HISTORY)
|
||||
except Exception:
|
||||
log.exception("could not record login history")
|
||||
|
||||
|
||||
class LoginPayload(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
@ -46,12 +138,68 @@ async def login_page(request: Request):
|
||||
async def login(request: Request, payload: LoginPayload):
|
||||
if payload.username == USERNAME and payload.password == PASSWORD:
|
||||
request.session["user"] = payload.username
|
||||
log.info("login OK: %s", payload.username)
|
||||
log.info("login OK: %s (%s)", payload.username, _describe_device(request))
|
||||
_record_login(request, payload.username, True)
|
||||
return {"ok": True, "user": payload.username}
|
||||
log.warning("login FAILED: %s", payload.username)
|
||||
log.warning("login FAILED: %s (%s)", payload.username, _describe_device(request))
|
||||
_record_login(request, payload.username, False)
|
||||
raise HTTPException(401, "Invalid username or password")
|
||||
|
||||
|
||||
@router.get("/api/auth/logins/export")
|
||||
async def export_logins(request: Request, format: str = "csv"):
|
||||
"""Download the full sign-in history (not just the rows on screen).
|
||||
|
||||
CSV is written with a UTF-8 BOM so Excel opens it with the columns split
|
||||
and any non-ASCII intact; without it Excel mangles both.
|
||||
"""
|
||||
if not is_authed(request):
|
||||
raise HTTPException(401, "Not authenticated")
|
||||
import csv
|
||||
import io as _io
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
entries = list(reversed(_load_logins())) # newest first
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
if format.lower() == "json":
|
||||
return Response(
|
||||
content=json.dumps(entries, ensure_ascii=False, indent=1),
|
||||
media_type="application/json; charset=utf-8",
|
||||
headers={"Content-Disposition":
|
||||
f'attachment; filename="sanadlite-signins-{stamp}.json"'})
|
||||
|
||||
buf = _io.StringIO()
|
||||
writer = csv.writer(buf, lineterminator="\n")
|
||||
writer.writerow(["when", "user", "result", "device", "ip", "user_agent"])
|
||||
for e in entries:
|
||||
writer.writerow([e.get("at", ""), e.get("user", ""),
|
||||
"ok" if e.get("ok") else "failed",
|
||||
e.get("device", ""), e.get("ip", ""),
|
||||
e.get("user_agent", "")])
|
||||
return Response(
|
||||
content="" + buf.getvalue(),
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={"Content-Disposition":
|
||||
f'attachment; filename="sanadlite-signins-{stamp}.csv"'})
|
||||
|
||||
|
||||
@router.get("/api/auth/logins")
|
||||
async def login_history(request: Request, limit: int = 20):
|
||||
"""Recent sign-ins: when, from which device, and from which IP.
|
||||
|
||||
Failed attempts are included too — an unexplained failure from an unknown
|
||||
device is the thing worth noticing on a dashboard that is open to the
|
||||
internet.
|
||||
"""
|
||||
if not is_authed(request):
|
||||
raise HTTPException(401, "Not authenticated")
|
||||
entries = _load_logins()[-max(1, min(int(limit), 200)):]
|
||||
entries.reverse()
|
||||
return {"logins": entries, "total": len(_load_logins())}
|
||||
|
||||
|
||||
@router.post("/api/auth/logout")
|
||||
async def logout(request: Request):
|
||||
user = request.session.pop("user", None)
|
||||
|
||||
262
dashboard/routes/live.py
Normal file
262
dashboard/routes/live.py
Normal file
@ -0,0 +1,262 @@
|
||||
"""Live Gemini — browser-side realtime conversation with a chosen robot voice.
|
||||
|
||||
Why the browser talks to Gemini directly instead of through this server:
|
||||
this deployment sits behind an Apache `[P]` rewrite that cannot upgrade a
|
||||
WebSocket (measured — the same handshake answers 101 straight to uvicorn and
|
||||
404 through the proxy), so a server-side relay of live audio is impossible
|
||||
here. Instead the server mints a short-lived **ephemeral auth token** and the
|
||||
browser opens its own socket to Gemini with that. The real API key never
|
||||
reaches the page.
|
||||
|
||||
Endpoints:
|
||||
GET /api/live/config voices, model, and the persona for each
|
||||
GET /api/live/persona one voice's persona text
|
||||
POST /api/live/persona edit it (persisted, survives restarts)
|
||||
POST /api/live/token mint an ephemeral token for a session
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from Project.Sanad.config import BASE_DIR
|
||||
from Project.Sanad.core.logger import get_logger
|
||||
|
||||
router = APIRouter()
|
||||
log = get_logger("live_route")
|
||||
|
||||
PERSONA_FILE = BASE_DIR / "data" / "live_personas.json"
|
||||
|
||||
try:
|
||||
from Project.Sanad.core.config_loader import section as _cfg_section
|
||||
_LIVE_CFG = _cfg_section("voice", "live") or {}
|
||||
except Exception:
|
||||
_LIVE_CFG = {}
|
||||
# OFF by default: enabling it puts the real API key in the browser. See
|
||||
# mint_token() for why the safer ephemeral path is unavailable on this key.
|
||||
_ALLOW_DIRECT_KEY = bool(_LIVE_CFG.get("allow_direct_key", False))
|
||||
|
||||
# Ephemeral tokens are minted per session and expire quickly; `uses` covers the
|
||||
# single connect. Keep the window tight — the token is handed to a browser.
|
||||
_TOKEN_TTL_MIN = 30
|
||||
_SESSION_START_WINDOW_MIN = 2
|
||||
_REST_BASE = "https://generativelanguage.googleapis.com"
|
||||
# Ephemeral tokens work ONLY on v1beta (documented). Minting on v1alpha
|
||||
# succeeds but the resulting token is rejected at connect time with
|
||||
# "API key not valid" / "unregistered callers", which is what the first
|
||||
# attempt at this feature hit.
|
||||
_API_VERSION = "v1beta"
|
||||
|
||||
def _voice_options() -> list[dict]:
|
||||
"""Reuse the voice picker's table so both tabs name the robots identically."""
|
||||
try:
|
||||
from Project.Sanad.dashboard.routes.voice import VOICE_OPTIONS
|
||||
return [dict(o) for o in VOICE_OPTIONS]
|
||||
except Exception:
|
||||
return [{"voice": "Charon", "label": "Unitree G1"},
|
||||
{"voice": "Puck", "label": "Unitree R1"},
|
||||
{"voice": "Kore", "label": "Agibot x2"}]
|
||||
|
||||
|
||||
def _store():
|
||||
from Project.Sanad.dashboard.routes.live_personas import load_store
|
||||
return load_store()
|
||||
|
||||
|
||||
@router.get("/config")
|
||||
async def live_config():
|
||||
"""Voices, model, and which persona each robot is currently using."""
|
||||
from Project.Sanad.config import GEMINI_MODEL
|
||||
from Project.Sanad.dashboard.routes import live_personas as lp
|
||||
store = _store()
|
||||
voices = []
|
||||
for opt in _voice_options():
|
||||
pid = lp.active_for(store, opt["voice"])
|
||||
voices.append(dict(opt,
|
||||
persona_id=pid,
|
||||
persona_name=lp.persona_name(store, pid),
|
||||
persona=lp.persona_text(store, pid)))
|
||||
return {"model": GEMINI_MODEL, "api_version": _API_VERSION, "voices": voices}
|
||||
|
||||
|
||||
@router.get("/personas")
|
||||
async def list_personas():
|
||||
"""The whole library, plus the active selection per robot."""
|
||||
from Project.Sanad.dashboard.routes import live_personas as lp
|
||||
store = _store()
|
||||
return {"personas": lp.all_personas(store),
|
||||
"active": {o["voice"]: lp.active_for(store, o["voice"])
|
||||
for o in _voice_options()}}
|
||||
|
||||
|
||||
class PersonaSave(BaseModel):
|
||||
id: str = "" # empty -> create a new one
|
||||
name: str
|
||||
text: str
|
||||
|
||||
|
||||
@router.post("/personas")
|
||||
async def save_persona(payload: PersonaSave):
|
||||
"""Create a persona, or update a saved one. Built-ins are never modified —
|
||||
editing one saves a copy instead, so the original stays available."""
|
||||
from Project.Sanad.dashboard.routes import live_personas as lp
|
||||
from datetime import datetime
|
||||
name = payload.name.strip() or "Untitled persona"
|
||||
text = payload.text.strip()
|
||||
if not text:
|
||||
raise HTTPException(400, "Persona text cannot be empty")
|
||||
if len(text.encode("utf-8")) > 20000:
|
||||
raise HTTPException(413, "Persona too large (max 20000 bytes)")
|
||||
|
||||
store = _store()
|
||||
personas = store.setdefault("personas", {})
|
||||
pid = payload.id.strip()
|
||||
if not pid or pid in lp.BUILTIN:
|
||||
pid = lp.new_id(name, personas)
|
||||
personas[pid] = {"name": name, "text": text,
|
||||
"updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
|
||||
lp.save_store(store)
|
||||
log.info("persona saved: %s (%s, %d chars)", name, pid, len(text))
|
||||
return {"ok": True, "id": pid, "name": name}
|
||||
|
||||
|
||||
class PersonaSelect(BaseModel):
|
||||
voice: str
|
||||
id: str
|
||||
|
||||
|
||||
@router.post("/personas/select")
|
||||
async def select_persona(payload: PersonaSelect):
|
||||
"""Choose which persona a robot uses. Applies on the next connection."""
|
||||
from Project.Sanad.dashboard.routes import live_personas as lp
|
||||
store = _store()
|
||||
known = {p["id"] for p in lp.all_personas(store)}
|
||||
if payload.id not in known:
|
||||
raise HTTPException(404, f"No persona {payload.id!r}")
|
||||
store.setdefault("active", {})[payload.voice] = payload.id
|
||||
lp.save_store(store)
|
||||
log.info("persona for %s set to %s", payload.voice, payload.id)
|
||||
return {"ok": True, "voice": payload.voice, "id": payload.id,
|
||||
"name": lp.persona_name(store, payload.id)}
|
||||
|
||||
|
||||
class PersonaDelete(BaseModel):
|
||||
id: str
|
||||
|
||||
|
||||
@router.post("/personas/delete")
|
||||
async def delete_persona(payload: PersonaDelete):
|
||||
"""Remove a saved persona. Any robot using it falls back to its built-in."""
|
||||
from Project.Sanad.dashboard.routes import live_personas as lp
|
||||
store = _store()
|
||||
if payload.id in lp.BUILTIN:
|
||||
raise HTTPException(400, "Built-in personas cannot be deleted")
|
||||
if payload.id not in (store.get("personas") or {}):
|
||||
raise HTTPException(404, f"No persona {payload.id!r}")
|
||||
del store["personas"][payload.id]
|
||||
reassigned = []
|
||||
for voice, pid in list((store.get("active") or {}).items()):
|
||||
if pid == payload.id:
|
||||
store["active"][voice] = lp._DEFAULT_FOR_VOICE.get(voice, "")
|
||||
reassigned.append(voice)
|
||||
lp.save_store(store)
|
||||
return {"ok": True, "deleted": payload.id, "reassigned": reassigned}
|
||||
|
||||
|
||||
def _mint_token_sync(api_key: str) -> dict:
|
||||
"""Blocking POST to auth_tokens. Returns the raw API response."""
|
||||
now = datetime.utcnow()
|
||||
body = json.dumps({
|
||||
"uses": 1,
|
||||
"expireTime": (now + timedelta(minutes=_TOKEN_TTL_MIN)).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"newSessionExpireTime":
|
||||
(now + timedelta(minutes=_SESSION_START_WINDOW_MIN)).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
}).encode("utf-8")
|
||||
url = f"{_REST_BASE}/{_API_VERSION}/auth_tokens?key={api_key}"
|
||||
req = urllib.request.Request(url, data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = ""
|
||||
try:
|
||||
detail = json.loads(exc.read().decode("utf-8")).get("error", {}).get("message", "")
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError(f"HTTP {exc.code}: {detail or exc.reason}")
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f"request failed: {exc.reason}")
|
||||
|
||||
|
||||
class TokenPayload(BaseModel):
|
||||
voice: str = "Charon"
|
||||
|
||||
|
||||
@router.post("/token")
|
||||
async def mint_token(payload: TokenPayload):
|
||||
"""Credential for the browser's own Gemini socket.
|
||||
|
||||
Ephemeral tokens are the safe path — the key stays on the server — but they
|
||||
are NOT accepted for keys of the "AQ." AI-Studio express kind that this
|
||||
deployment uses. Measured: the express key opens a Live session directly on
|
||||
both v1alpha and v1beta, while a token minted from it is refused in all four
|
||||
documented presentations (access_token encoded/raw, Authorization: Token,
|
||||
key=). So live conversation here needs the real key in the browser, which is
|
||||
a deliberate decision and stays OFF until switched on in config:
|
||||
|
||||
voice_config.json > live.allow_direct_key = true
|
||||
|
||||
Turning it on means anyone who can open the dashboard (or its dev tools) can
|
||||
read the key and spend against it. Getting a standard `AIza…` key instead
|
||||
would restore the ephemeral path and make this unnecessary.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from Project.Sanad.config import GEMINI_MODEL
|
||||
import Project.Sanad.gemini.client as _gc
|
||||
|
||||
api_key = getattr(_gc, "GEMINI_API_KEY", "") or ""
|
||||
if not api_key:
|
||||
raise HTTPException(503, "No Gemini API key configured.")
|
||||
|
||||
from Project.Sanad.dashboard.routes import live_personas as lp
|
||||
store = _store()
|
||||
voice = payload.voice or "Charon"
|
||||
persona_id = lp.active_for(store, voice)
|
||||
common = {
|
||||
"model": GEMINI_MODEL,
|
||||
"api_version": _API_VERSION,
|
||||
"voice": voice,
|
||||
"persona": lp.persona_text(store, persona_id),
|
||||
"persona_name": lp.persona_name(store, persona_id),
|
||||
}
|
||||
|
||||
if _ALLOW_DIRECT_KEY:
|
||||
log.warning("live session: handing the API key to the browser "
|
||||
"(live.allow_direct_key is enabled)")
|
||||
return dict(common, token=api_key, auth_param="key",
|
||||
api_version="v1alpha", direct_key=True, expires_in_sec=0)
|
||||
|
||||
try:
|
||||
data = await asyncio.to_thread(_mint_token_sync, api_key)
|
||||
except RuntimeError as exc:
|
||||
msg = str(exc)
|
||||
log.error("could not mint ephemeral token: %s", msg)
|
||||
if "credits are depleted" in msg or "billing" in msg.lower():
|
||||
raise HTTPException(503, "Gemini API credits are depleted — top up billing.")
|
||||
raise HTTPException(502, f"Could not start a live session: {msg}")
|
||||
|
||||
name = data.get("name", "")
|
||||
if not name:
|
||||
raise HTTPException(502, "Gemini returned no token")
|
||||
|
||||
log.info("live session token minted for %s", voice)
|
||||
return dict(common, token=name, auth_param="access_token", direct_key=False,
|
||||
expires_in_sec=_TOKEN_TTL_MIN * 60)
|
||||
165
dashboard/routes/live_personas.py
Normal file
165
dashboard/routes/live_personas.py
Normal file
@ -0,0 +1,165 @@
|
||||
"""Persona library for Live Gemini.
|
||||
|
||||
Personas used to be one blob per robot voice. This stores a *library* of named
|
||||
personas plus which one is active for each robot, so several can be kept side
|
||||
by side (e.g. a formal receptionist and a playful tour guide for the same
|
||||
robot) and switched without retyping.
|
||||
|
||||
Shape on disk (data/live_personas.json):
|
||||
|
||||
{
|
||||
"personas": {"<id>": {"name": str, "text": str, "updated": str}},
|
||||
"active": {"<voiceName>": "<id>"}
|
||||
}
|
||||
|
||||
The previous format — a flat {voice: text} mapping — is migrated on first read,
|
||||
so nothing the user already wrote is lost.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from Project.Sanad.config import BASE_DIR
|
||||
from Project.Sanad.core.logger import get_logger
|
||||
|
||||
log = get_logger("live_personas")
|
||||
|
||||
STORE = BASE_DIR / "data" / "live_personas.json"
|
||||
|
||||
# Built-in personas, seeded from each robot's own character. They are always
|
||||
# present, cannot be deleted, and act as the fallback when nothing is selected.
|
||||
BUILTIN: dict[str, dict] = {
|
||||
"builtin:charon": {
|
||||
"name": "Sanad — Unitree G1 (default)",
|
||||
"voice": "Charon",
|
||||
"text": (
|
||||
"أنت \"سند\" — روبوت إماراتي من شركة YS Lootah Robotics تعمل على منصة Unitree G1.\n"
|
||||
"تكلم باللهجة الإماراتية (الخليجية) الأصيلة بشكل طبيعي وواضح، بدون مبالغة.\n"
|
||||
"ردودك قصيرة ومسموعة: من جملة إلى ثلاث جمل، لأن كلامك يُنطق بصوت وليس مقروءاً.\n"
|
||||
"لا تستخدم رموزاً ولا قوائم مرقّمة ولا علامات تنسيق — فقط جُمل يسهل نطقها.\n"
|
||||
"إذا تحدث المستخدم بلغة أخرى، افهمه وردّ عليه بنفس لغته.\n"
|
||||
"كن مهذباً وواثقاً ومباشراً، وإذا ما تعرف الجواب قل ذلك بصراحة."
|
||||
),
|
||||
},
|
||||
"builtin:puck": {
|
||||
"name": "سوبر دبي — Unitree R1 (default)",
|
||||
"voice": "Puck",
|
||||
"text": (
|
||||
"أنت \"سوبر دبي\" (super-dubai) — روبوت إماراتي ذكي تابع لشركة لوتاه تيك، "
|
||||
"تعمل على منصة Unitree R1.\n"
|
||||
"تكلم باللهجة الإماراتية بشكل طبيعي وراقٍ ومفهوم، ونوّع بداياتك "
|
||||
"(مرحبابك، أبشر بعزك، حياك الله، زين، تم).\n"
|
||||
"إذا استخدم المستخدم لغة ثانية، بدّل فوراً وردّ بنفس اللغة.\n"
|
||||
"ردودك قصيرة ومركزة على الزبدة والحل العملي، بدون رموز أو تنسيق.\n"
|
||||
"كن ودوداً ومحترماً ومباشراً."
|
||||
),
|
||||
},
|
||||
"builtin:kore": {
|
||||
"name": "موزة — Agibot x2 (default)",
|
||||
"voice": "Kore",
|
||||
"text": (
|
||||
"أنتِ \"موزة\" (Muza) — روبوت إماراتي احترافي تابع لشركة YS Lootah Robotics "
|
||||
"تعمل على منصة Agibot X2.\n"
|
||||
"تتحدثين حصراً باللهجة الإماراتية (الخليجية) في كل رد، بأسلوب احترافي راقٍ "
|
||||
"وهادئ وواثق، بدون مبالغة في الودّ.\n"
|
||||
"ردودك قصيرة: من جملة إلى ثلاث جمل، بدون رموز ولا قوائم.\n"
|
||||
"ابدئي الجلسة بتحية واحدة قصيرة: \"حياك الله\".\n"
|
||||
"إذا سُئلتِ عن الشركة، عرّفي بها باختصار وبدقة، ولا تختلقي أرقاماً أو التزامات."
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
_DEFAULT_FOR_VOICE = {v["voice"]: pid for pid, v in BUILTIN.items()}
|
||||
|
||||
|
||||
def _blank_store() -> dict:
|
||||
return {"personas": {}, "active": dict(_DEFAULT_FOR_VOICE)}
|
||||
|
||||
|
||||
def _migrate_flat(old: dict) -> dict:
|
||||
"""Convert the old {voice: text} mapping into the library format."""
|
||||
store = _blank_store()
|
||||
for voice, text in old.items():
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
continue
|
||||
builtin_id = _DEFAULT_FOR_VOICE.get(voice)
|
||||
if builtin_id and text.strip() == BUILTIN[builtin_id]["text"].strip():
|
||||
continue # unchanged default: nothing to keep
|
||||
pid = f"saved:{voice.lower()}"
|
||||
store["personas"][pid] = {
|
||||
"name": f"{voice} (saved)",
|
||||
"text": text,
|
||||
"updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
store["active"][voice] = pid
|
||||
log.info("migrated %d persona(s) from the old format", len(store["personas"]))
|
||||
return store
|
||||
|
||||
|
||||
def load_store() -> dict:
|
||||
try:
|
||||
raw = json.loads(STORE.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return _blank_store()
|
||||
except Exception:
|
||||
log.exception("could not read persona store — starting fresh")
|
||||
return _blank_store()
|
||||
|
||||
if isinstance(raw, dict) and "personas" in raw and "active" in raw:
|
||||
raw.setdefault("personas", {})
|
||||
active = raw.setdefault("active", {})
|
||||
for voice, pid in _DEFAULT_FOR_VOICE.items():
|
||||
active.setdefault(voice, pid)
|
||||
return raw
|
||||
if isinstance(raw, dict):
|
||||
store = _migrate_flat(raw)
|
||||
save_store(store)
|
||||
return store
|
||||
return _blank_store()
|
||||
|
||||
|
||||
def save_store(store: dict) -> None:
|
||||
STORE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = STORE.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(store, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
tmp.replace(STORE)
|
||||
|
||||
|
||||
def all_personas(store: dict) -> list[dict]:
|
||||
"""Built-ins first, then saved ones, newest last."""
|
||||
out = [{"id": pid, "name": v["name"], "text": v["text"], "builtin": True,
|
||||
"voice": v["voice"]}
|
||||
for pid, v in BUILTIN.items()]
|
||||
for pid, v in (store.get("personas") or {}).items():
|
||||
out.append({"id": pid, "name": v.get("name", pid), "text": v.get("text", ""),
|
||||
"builtin": False, "updated": v.get("updated", "")})
|
||||
return out
|
||||
|
||||
|
||||
def persona_text(store: dict, persona_id: str) -> str:
|
||||
if persona_id in BUILTIN:
|
||||
return BUILTIN[persona_id]["text"]
|
||||
entry = (store.get("personas") or {}).get(persona_id)
|
||||
return entry.get("text", "") if entry else ""
|
||||
|
||||
|
||||
def persona_name(store: dict, persona_id: str) -> str:
|
||||
if persona_id in BUILTIN:
|
||||
return BUILTIN[persona_id]["name"]
|
||||
entry = (store.get("personas") or {}).get(persona_id)
|
||||
return entry.get("name", persona_id) if entry else persona_id
|
||||
|
||||
|
||||
def active_for(store: dict, voice: str) -> str:
|
||||
return (store.get("active") or {}).get(voice) or _DEFAULT_FOR_VOICE.get(voice, "")
|
||||
|
||||
|
||||
def new_id(name: str, existing: dict) -> str:
|
||||
base = re.sub(r"[^a-z0-9]+", "-", (name or "persona").lower()).strip("-") or "persona"
|
||||
pid, n = base, 2
|
||||
while pid in existing or pid in BUILTIN:
|
||||
pid = f"{base}-{n}"
|
||||
n += 1
|
||||
return pid
|
||||
@ -20,6 +20,20 @@ from Project.Sanad.dashboard.routes._safe_io import safe_path_under
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/live")
|
||||
async def live_tail(cursor: int = -1, limit: int = 300):
|
||||
"""Polling fallback for the live log panel.
|
||||
|
||||
The dashboard's /ws/logs WebSocket cannot reach this deployment: the site
|
||||
is served through an Apache `[P]` rewrite which forwards the upgrade as a
|
||||
plain GET, so the app answers 404 and the panel stays empty forever. The
|
||||
same ring buffer is exposed here so the browser can poll it instead.
|
||||
"""
|
||||
from Project.Sanad.dashboard.websockets.log_stream import recent_since
|
||||
lines, new_cursor = recent_since(int(cursor), max(1, min(int(limit), 1000)))
|
||||
return {"lines": lines, "cursor": new_cursor}
|
||||
|
||||
|
||||
def _list_logs_sync():
|
||||
LOGS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
files = []
|
||||
|
||||
@ -7,11 +7,13 @@ from pydantic import BaseModel
|
||||
|
||||
from Project.Sanad.config import SCRIPTS_DIR
|
||||
from Project.Sanad.core.config_loader import section as _cfg_section
|
||||
from Project.Sanad.core.logger import get_logger
|
||||
from Project.Sanad.dashboard.routes._safe_io import (
|
||||
atomic_write_text, MAX_UPLOAD_BYTES,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
log = get_logger("prompt_route")
|
||||
|
||||
# Filenames — SINGLE SOURCE in core.script_files
|
||||
_SCRIPTS = _cfg_section("core", "script_files")
|
||||
@ -96,3 +98,47 @@ async def reload_prompts():
|
||||
"script_path": str(SCRIPT_PROMPT_PATH),
|
||||
"rule_path": str(RULE_PROMPT_PATH),
|
||||
}
|
||||
|
||||
|
||||
# ── Voice replay rule — the [REPLAY_SYSTEM_PROMPT] that shapes the spoken voice ──
|
||||
|
||||
class RulePayload(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
@router.get("/rule")
|
||||
async def get_rule():
|
||||
"""The current voice replay rule (how typed text is spoken: accent, verbatim, gender)."""
|
||||
return {"content": _load_rule_prompts().get("replay_prompt", ""),
|
||||
"rule_path": str(RULE_PROMPT_PATH)}
|
||||
|
||||
|
||||
@router.post("/rule")
|
||||
async def update_rule(payload: RulePayload):
|
||||
"""Save the replay rule to sanad_rule.txt AND apply it live (hot-swap + reconnect)."""
|
||||
content = payload.content.strip()
|
||||
if not content:
|
||||
raise HTTPException(400, "Rule cannot be empty.")
|
||||
if len(content.encode("utf-8")) > MAX_PROMPT_BYTES:
|
||||
raise HTTPException(413, f"Rule too large (max {MAX_PROMPT_BYTES} bytes).")
|
||||
try:
|
||||
SCRIPTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_text(
|
||||
RULE_PROMPT_PATH, "[REPLAY_SYSTEM_PROMPT]\n" + content.rstrip() + "\n")
|
||||
except OSError as exc:
|
||||
raise HTTPException(500, f"Could not write rule: {exc}")
|
||||
# Apply immediately: hot-swap the live voice_client's system prompt and drop the
|
||||
# session so the next Generate & Play reconnects with the new rule.
|
||||
applied = False
|
||||
try:
|
||||
from Project.Sanad.main import voice_client
|
||||
if voice_client is not None:
|
||||
voice_client.system_prompt = content
|
||||
if getattr(voice_client, "connected", False):
|
||||
await voice_client.disconnect()
|
||||
applied = True
|
||||
except Exception:
|
||||
log.exception("could not hot-swap the voice rule")
|
||||
return {"ok": True, "applied": applied,
|
||||
"message": ("Rule saved and applied — takes effect on your next Generate & Play."
|
||||
if applied else "Rule saved (restart to apply).")}
|
||||
|
||||
@ -240,3 +240,150 @@ async def _legacy_update_api_key(payload: ApiKeyPayload):
|
||||
"separately (Stop → Start) to pick up the new key."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────── Gemini voice (robot) selection ───────────────────────
|
||||
# Friendly robot labels mapped to Gemini prebuilt voice names. Source of truth
|
||||
# for the dashboard voice picker — each robot maps to one Gemini voice.
|
||||
# Optionally overridable from core_config.json → gemini_defaults.voice_options
|
||||
# (a list of {"voice","label"} dicts) without touching code.
|
||||
_DEFAULT_VOICE_OPTIONS = [
|
||||
{"voice": "Charon", "label": "Unitree G1"},
|
||||
{"voice": "Puck", "label": "Unitree R1"},
|
||||
{"voice": "Kore", "label": "Agibot x2"},
|
||||
]
|
||||
|
||||
|
||||
def _load_voice_options() -> list[dict]:
|
||||
raw = _cfg_section("core", "gemini_defaults").get("voice_options")
|
||||
if isinstance(raw, list) and raw and all(
|
||||
isinstance(o, dict) and o.get("voice") and o.get("label") for o in raw
|
||||
):
|
||||
return [{"voice": str(o["voice"]), "label": str(o["label"])} for o in raw]
|
||||
return list(_DEFAULT_VOICE_OPTIONS)
|
||||
|
||||
|
||||
VOICE_OPTIONS = _load_voice_options()
|
||||
_VOICE_BY_NAME = {o["voice"]: o for o in VOICE_OPTIONS}
|
||||
|
||||
|
||||
def _current_voice() -> str:
|
||||
"""The voice the Gemini client will actually use on its next connect()."""
|
||||
try:
|
||||
import Project.Sanad.gemini.client as _gc
|
||||
return getattr(_gc, "GEMINI_VOICE", "") or ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
class VoicePayload(BaseModel):
|
||||
voice: str
|
||||
|
||||
|
||||
@router.get("/voices")
|
||||
async def list_voices():
|
||||
"""List selectable Gemini voices with robot labels + which one is active."""
|
||||
cur = _current_voice()
|
||||
cur_opt = _VOICE_BY_NAME.get(cur)
|
||||
return {
|
||||
"current": cur,
|
||||
"current_label": cur_opt["label"] if cur_opt else cur,
|
||||
"options": [
|
||||
{"voice": o["voice"], "label": o["label"], "active": o["voice"] == cur}
|
||||
for o in VOICE_OPTIONS
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/voice")
|
||||
async def set_voice(payload: VoicePayload):
|
||||
"""Switch the Gemini voice, persist it, and apply on the next connect.
|
||||
|
||||
Mirrors the api-key hot-swap: persist to data/motions/config.json
|
||||
(gemini.voice) so it survives a restart, patch the in-memory GEMINI_VOICE
|
||||
in BOTH config and the gemini client module (each holds its own binding
|
||||
from `from ... import GEMINI_VOICE`), then disconnect any idle Gemini
|
||||
session so the next connect() (e.g. the next Typed Replay) opens with it.
|
||||
"""
|
||||
voice = (payload.voice or "").strip()
|
||||
if voice not in _VOICE_BY_NAME:
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Unknown voice %r. Allowed: %s"
|
||||
% (voice, ", ".join(o["voice"] for o in VOICE_OPTIONS)),
|
||||
)
|
||||
|
||||
# 1. Persist (atomic temp-then-replace) to data/motions/config.json.
|
||||
# Run the blocking file I/O off the event-loop thread.
|
||||
try:
|
||||
from Project.Sanad.config import load_config, save_config
|
||||
cfg = (await asyncio.to_thread(load_config)) or {}
|
||||
gemini_cfg = cfg.get("gemini") if isinstance(cfg.get("gemini"), dict) else {}
|
||||
gemini_cfg["voice"] = voice
|
||||
cfg["gemini"] = gemini_cfg
|
||||
await asyncio.to_thread(save_config, cfg)
|
||||
except Exception as exc:
|
||||
log.exception("Failed to persist voice to config.json")
|
||||
raise HTTPException(500, f"Could not save voice: {exc}")
|
||||
|
||||
# 2. Hot-swap the in-memory globals in both modules that hold GEMINI_VOICE.
|
||||
try:
|
||||
import Project.Sanad.config as _cfg_mod
|
||||
_cfg_mod.GEMINI_VOICE = voice
|
||||
except Exception:
|
||||
log.exception("could not patch config.GEMINI_VOICE")
|
||||
try:
|
||||
import Project.Sanad.gemini.client as _gc
|
||||
_gc.GEMINI_VOICE = voice
|
||||
except Exception:
|
||||
log.exception("could not patch gemini.client.GEMINI_VOICE")
|
||||
|
||||
# 3. Reopen the Gemini session so the new voice actually takes effect. The
|
||||
# voice is baked into connect()'s setup handshake and the persistent
|
||||
# session is only rebuilt when connected is False, so we must drop it —
|
||||
# otherwise the next Generate & Play reuses the still-open OLD-voice
|
||||
# session (voice never changes). Acquire the session lock first so we
|
||||
# WAIT for any in-flight generation to finish (it keeps the old voice,
|
||||
# which is fine) instead of cutting it off mid-stream; then disconnect.
|
||||
# Bound the wait so a stuck turn can't hang this request — the persisted
|
||||
# + patched voice still applies on the next reconnect either way.
|
||||
from Project.Sanad.main import voice_client
|
||||
was_connected = False
|
||||
if voice_client is not None:
|
||||
was_connected = bool(getattr(voice_client, "connected", False))
|
||||
if was_connected:
|
||||
async def _reset_session():
|
||||
async with voice_client.acquire_session("voice_switch"):
|
||||
await voice_client.disconnect()
|
||||
try:
|
||||
await asyncio.wait_for(_reset_session(), timeout=45)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("voice swap: session busy too long; new voice applies on next reconnect")
|
||||
except Exception:
|
||||
# The polite path can fail for reasons that have nothing to do
|
||||
# with the socket — notably the session lock being bound to a
|
||||
# different event loop than the request runs on (py3.8 binds
|
||||
# asyncio.Lock at construction, and voice_client is built at
|
||||
# import time, before uvicorn's loop exists), which raises
|
||||
# "got Future attached to a different loop" the moment the lock
|
||||
# is contended. Swallowing that left the OLD-voice socket open,
|
||||
# so the next Generate & Play spoke in the PREVIOUS robot's
|
||||
# voice. Drop the socket unconditionally instead — worst case a
|
||||
# concurrent generation reconnects.
|
||||
log.warning("voice swap: graceful session reset failed — "
|
||||
"forcing disconnect so the new voice applies",
|
||||
exc_info=True)
|
||||
try:
|
||||
await voice_client.disconnect()
|
||||
except Exception:
|
||||
log.exception("voice swap: forced disconnect failed too")
|
||||
|
||||
label = _VOICE_BY_NAME[voice]["label"]
|
||||
log.info("Gemini voice switched to %s (%s)", voice, label)
|
||||
return {
|
||||
"ok": True,
|
||||
"voice": voice,
|
||||
"label": label,
|
||||
"was_connected": was_connected,
|
||||
"message": f"Voice set to {label} ({voice}) — applies on the next Generate & Play.",
|
||||
}
|
||||
|
||||
@ -102,6 +102,30 @@
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="js-error" style="display:none;position:sticky;top:0;z-index:9999;
|
||||
background:#7f1d1d;color:#fff;padding:.5rem .75rem;font:12px/1.5 monospace;
|
||||
white-space:pre-wrap"></div>
|
||||
<script>
|
||||
// Installed before every other script so load-time failures are visible.
|
||||
// Without this a ReferenceError in live.js silently blanks the Live tab.
|
||||
(function () {
|
||||
function show(text) {
|
||||
var el = document.getElementById('js-error');
|
||||
if (!el) { return; }
|
||||
el.style.display = 'block';
|
||||
el.textContent = (el.textContent ? el.textContent + '\n' : '') + text;
|
||||
}
|
||||
window.addEventListener('error', function (e) {
|
||||
show('JS error: ' + (e.message || e.type)
|
||||
+ (e.filename ? ' [' + e.filename.split('/').pop() + ':' + e.lineno + ']' : ''));
|
||||
});
|
||||
window.addEventListener('unhandledrejection', function (e) {
|
||||
var r = e.reason;
|
||||
show('Unhandled promise: ' + ((r && (r.message || r)) || 'unknown'));
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
|
||||
<div id="toast-box"></div>
|
||||
|
||||
<!-- Header -->
|
||||
@ -121,6 +145,7 @@
|
||||
<!-- Tabs -->
|
||||
<div class="tabs">
|
||||
<div class="tab active" onclick="switchTab('voice')">Voice & Audio</div>
|
||||
<div class="tab" onclick="switchTab('live')">Live Gemini</div>
|
||||
<div class="tab" onclick="switchTab('recordings')">Recordings</div>
|
||||
<div class="tab" onclick="switchTab('settings')">Settings & Logs</div>
|
||||
</div>
|
||||
@ -147,6 +172,19 @@
|
||||
<div id="gm-key-msg" style="font-size:.7rem;margin-top:.3rem;color:var(--muted)"></div>
|
||||
</div>
|
||||
|
||||
<!-- Robot Voice (Gemini prebuilt voice picker) -->
|
||||
<div class="card card-full">
|
||||
<h3>Robot Voice</h3>
|
||||
<div style="font-size:.72rem;color:var(--muted);margin-bottom:.5rem">
|
||||
Pick which robot's voice Gemini speaks with. Applies on the next
|
||||
<strong>Generate & Play</strong> and is saved for next time.
|
||||
</div>
|
||||
<div class="row" id="voice-options" style="gap:.4rem;flex-wrap:wrap">
|
||||
<span style="color:var(--muted);font-size:.72rem">Loading…</span>
|
||||
</div>
|
||||
<div id="voice-current" style="font-size:.7rem;margin-top:.4rem;color:var(--muted)"></div>
|
||||
</div>
|
||||
|
||||
<!-- Typed Replay -->
|
||||
<div class="card card-full">
|
||||
<h3>Typed Replay Engine</h3>
|
||||
@ -166,7 +204,8 @@
|
||||
</div>
|
||||
<div style="flex:1;min-width:200px">
|
||||
<label>Session</label>
|
||||
<div id="tr-session" style="font-size:.72rem;color:var(--muted);margin-top:.3rem;line-height:1.6"></div>
|
||||
<div id="tr-warning" style="display:none;margin:.4rem 0;padding:.45rem .6rem;border-radius:4px;background:rgba(224,160,48,.12);border:1px solid rgba(224,160,48,.55);color:#e0a030;font-size:.75rem;line-height:1.5"></div>
|
||||
<div id="tr-session" style="font-size:.72rem;color:var(--muted);margin-top:.3rem;line-height:1.6"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -174,6 +213,64 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ==================== TAB: Live Gemini ==================== -->
|
||||
<div class="tab-content" id="tab-live">
|
||||
<div class="grid">
|
||||
|
||||
<div class="card card-full">
|
||||
<h3>Live Conversation <span id="live-status" style="font-size:.7rem;font-weight:400;color:var(--dim)">idle</span></h3>
|
||||
<p style="font-size:.72rem;color:var(--dim);margin:.2rem 0 .5rem">
|
||||
Talk to the robot in real time. Audio goes straight from this browser to Gemini —
|
||||
the server only issues a short-lived session token, so the API key is never exposed.
|
||||
Needs microphone permission.
|
||||
</p>
|
||||
|
||||
<label>Robot</label>
|
||||
<div id="live-voices" class="row" style="gap:.3rem;flex-wrap:wrap;margin:.2rem 0 .6rem"></div>
|
||||
|
||||
<div class="row" style="gap:.4rem;align-items:center;flex-wrap:wrap;margin-bottom:.4rem">
|
||||
<button class="btn btn-ghost btn-sm" onclick="liveRequestMic()">Allow microphone</button>
|
||||
<button class="btn btn-ghost btn-sm" onclick="liveTestSpeaker()">Test speaker</button>
|
||||
<span id="live-perm" style="font-size:.7rem;color:var(--dim)"></span>
|
||||
</div>
|
||||
<div class="row" style="gap:.4rem;align-items:center;margin-bottom:.5rem">
|
||||
<button class="btn btn-primary" id="live-connect" onclick="liveToggle()">Connect</button>
|
||||
<span style="font-size:.68rem;color:var(--dim)">model: <span id="live-model">--</span></span>
|
||||
</div>
|
||||
|
||||
<div id="live-diag" style="font-size:.68rem;color:var(--dim);margin:.1rem 0 .5rem;font-family:monospace"></div>
|
||||
|
||||
<label>Conversation</label>
|
||||
<div class="log-box" id="live-transcript" style="height:220px;color:#e5e7eb"></div>
|
||||
</div>
|
||||
|
||||
<div class="card card-full">
|
||||
<h3>Persona <span id="persona-for" style="font-size:.75rem;font-weight:400;color:#38bdf8"></span></h3>
|
||||
<p style="font-size:.72rem;color:var(--dim);margin:.2rem 0 .5rem">
|
||||
How the robot selected above behaves in conversation — its character, dialect and
|
||||
answering style. Pick any persona from the library and press
|
||||
<em>Use for this robot</em>; it applies on the next connection.
|
||||
</p>
|
||||
<div class="row" style="gap:.35rem;flex-wrap:wrap;align-items:center;margin-bottom:.4rem">
|
||||
<select id="persona-picker" onchange="livePickPersona(this.value)"
|
||||
style="flex:1 1 240px;max-width:420px;width:auto"></select>
|
||||
<button class="btn btn-primary btn-sm" onclick="liveUsePersona()">Use for this robot</button>
|
||||
<span id="persona-active" style="font-size:.72rem;color:#4ade80"></span>
|
||||
</div>
|
||||
<div id="persona-note" style="font-size:.72rem;color:#f87171;margin:.1rem 0"></div>
|
||||
<input id="persona-name" placeholder="Persona name" style="width:100%;margin-bottom:.3rem">
|
||||
<textarea id="live-persona" rows="10" spellcheck="false"
|
||||
style="width:100%;font-size:.78rem;line-height:1.7"></textarea>
|
||||
<div class="row" style="margin-top:.35rem;gap:.3rem;flex-wrap:wrap">
|
||||
<button class="btn btn-success btn-sm" onclick="liveSavePersona()">Save</button>
|
||||
<button class="btn btn-ghost btn-sm" onclick="liveSavePersonaAs()">Save as new</button>
|
||||
<button class="btn btn-danger btn-sm" onclick="liveDeletePersona()" style="margin-left:auto">Delete</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ==================== TAB: Recordings ==================== -->
|
||||
<div class="tab-content" id="tab-recordings">
|
||||
<div class="grid">
|
||||
@ -181,6 +278,22 @@
|
||||
<!-- Saved Records -->
|
||||
<div class="card card-full">
|
||||
<h3>Saved Records</h3>
|
||||
<div class="row" style="gap:.35rem;margin-bottom:.45rem;flex-wrap:wrap;align-items:center">
|
||||
<input id="records-search" type="search" oninput="renderRecords()"
|
||||
placeholder="Search name or text… بحث بالاسم أو النص"
|
||||
style="flex:1;min-width:190px">
|
||||
<select id="records-voice" onchange="renderRecords()" style="min-width:140px">
|
||||
<option value="">All voices</option>
|
||||
</select>
|
||||
<select id="records-sort" onchange="renderRecords()" style="min-width:150px">
|
||||
<option value="new">Newest first</option>
|
||||
<option value="old">Oldest first</option>
|
||||
<option value="name">Name A–Z</option>
|
||||
<option value="long">Longest audio</option>
|
||||
<option value="plays">Most replayed</option>
|
||||
</select>
|
||||
<button class="btn btn-ghost btn-sm" onclick="clearRecordSearch()">Clear</button>
|
||||
</div>
|
||||
<div id="records-list"><div class="empty">No records saved</div></div>
|
||||
<div class="row" style="margin-top:.3rem;gap:.3rem">
|
||||
<button class="btn btn-ghost btn-sm" onclick="refreshRecords()">Refresh</button>
|
||||
@ -196,33 +309,34 @@
|
||||
<div class="tab-content" id="tab-settings">
|
||||
<div class="grid">
|
||||
|
||||
<!-- Scripts -->
|
||||
<div class="card">
|
||||
<h3>Scripts Manager</h3>
|
||||
<div class="row"><select id="script-select" style="flex:1" onchange="loadScript(this.value)"><option value="">-- select --</option></select><button class="btn btn-ghost btn-sm" onclick="refreshScripts()">Refresh</button></div>
|
||||
<textarea id="script-content" placeholder="Script content..." style="min-height:100px"></textarea>
|
||||
<!-- Voice Rule — edit the replay instruction that shapes how the robot speaks -->
|
||||
<div class="card card-full">
|
||||
<h3>Voice Rule</h3>
|
||||
<div style="font-size:.72rem;color:var(--muted);margin-bottom:.4rem">
|
||||
The instruction that shapes how the robot speaks your typed text (accent, verbatim, gender).
|
||||
<strong>Save & Apply</strong> takes effect on the next Generate & Play.
|
||||
</div>
|
||||
<textarea id="rule-content" placeholder="Voice rule…" dir="auto" style="min-height:130px"></textarea>
|
||||
<div class="row" style="margin-top:.3rem">
|
||||
<button class="btn btn-primary btn-sm" onclick="saveScript()">Save</button>
|
||||
<input id="script-new-name" placeholder="new_file.txt" style="flex:1">
|
||||
<button class="btn btn-success btn-sm" onclick="createScript()">Create</button>
|
||||
<button class="btn btn-danger btn-sm" onclick="deleteScript()">Delete</button>
|
||||
<button class="btn btn-primary btn-sm" onclick="saveRule(this)">Save & Apply</button>
|
||||
<button class="btn btn-ghost btn-sm" onclick="refreshRule()">Reload</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Prompt -->
|
||||
<div class="card">
|
||||
<h3>Prompt Management</h3>
|
||||
<div id="prompt-info" style="font-size:.7rem;color:var(--dim);margin-bottom:.3rem"></div>
|
||||
<textarea id="prompt-content" placeholder="System prompt..." style="min-height:100px"></textarea>
|
||||
<div class="row" style="margin-top:.3rem">
|
||||
<button class="btn btn-primary btn-sm" onclick="updatePrompt()">Save</button>
|
||||
<button class="btn btn-ghost btn-sm" onclick="reloadPrompt()">Reload from Disk</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Logs -->
|
||||
<div class="card card-full">
|
||||
<h3>Live Logs</h3>
|
||||
<h3>Sign-in History</h3>
|
||||
<div id="login-list"><div class="empty">Loading…</div></div>
|
||||
<div class="row" style="margin-top:.3rem">
|
||||
<button class="btn btn-ghost btn-sm" onclick="refreshLogins()">Refresh</button>
|
||||
<button class="btn btn-success btn-sm" onclick="exportLogins('csv')">Export CSV</button>
|
||||
<button class="btn btn-ghost btn-sm" onclick="exportLogins('json')">JSON</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card card-full">
|
||||
<h3>Live Logs <span id="log-mode" style="font-size:.68rem;font-weight:400;color:var(--dim)">connecting…</span></h3>
|
||||
<div class="row" style="margin-bottom:.3rem;flex-wrap:wrap;gap:.3rem">
|
||||
<button class="btn btn-ghost btn-sm" onclick="saveLogSnapshot()" title="Save a timestamped copy of all .log files under logs/">Save Snapshot</button>
|
||||
<button class="btn btn-primary btn-sm" onclick="copyAllLogs(this)" title="Fetch system status + every log file and copy to clipboard">Copy All</button>
|
||||
@ -242,7 +356,7 @@ function toast(m,t='info'){const b=document.getElementById('toast-box'),e=docume
|
||||
function esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');}
|
||||
function btnLoad(b){if(b&&b.classList)b.classList.add('loading');}
|
||||
function btnDone(b){if(b&&b.classList)b.classList.remove('loading');}
|
||||
async function api(m,p,b){const o={method:m,headers:{'Content-Type':'application/json'},credentials:'same-origin'};if(b)o.body=JSON.stringify(b);const r=await fetch(API+p,o);if(r.status===401){location.href='/login?next='+encodeURIComponent(location.pathname);throw new Error('Not authenticated');}const j=await r.json();if(!r.ok){toast(j.detail||j.error||'Error '+r.status,'err');throw new Error(j.detail||j.error);}return j;}
|
||||
async function api(m,p,b){const o={method:m,headers:{'Content-Type':'application/json'},credentials:'same-origin'};if(b)o.body=JSON.stringify(b);const r=await fetch(API+p,o);if(r.status===401){location.href='/login?next='+encodeURIComponent(location.pathname);throw new Error('Not authenticated');}let j={};const txt=await r.text();if(txt){try{j=JSON.parse(txt);}catch(e){const msg=r.ok?'Unexpected non-JSON response from server':('Server error '+r.status+' (timeout or gateway)');toast(msg,'err');throw new Error(msg);}}if(!r.ok){toast(j.detail||j.error||('Error '+r.status),'err');throw new Error(j.detail||j.error||('Error '+r.status));}return j;}
|
||||
|
||||
// Tabs
|
||||
function switchTab(name){document.querySelectorAll('.tab').forEach(t=>t.classList.toggle('active',t.textContent.toLowerCase().includes(name.slice(0,4))));document.querySelectorAll('.tab-content').forEach(c=>c.classList.toggle('active',c.id==='tab-'+name));}
|
||||
@ -307,40 +421,41 @@ async function refreshAudio(){
|
||||
async function toggleMic(){try{await api('POST','/api/audio/mic/mute');}catch(e){}refreshAudio();}
|
||||
async function toggleSpeaker(){try{await api('POST','/api/audio/speaker/mute');}catch(e){}refreshAudio();}
|
||||
|
||||
// Scripts
|
||||
async function refreshScripts(){try{const r=await api('GET','/api/scripts/');const sel=document.getElementById('script-select');sel.innerHTML='<option value="">-- select --</option>'+(r.files||[]).map(f=>`<option value="${esc(f.name)}">${esc(f.name)} (${f.size_bytes}B)</option>`).join('');}catch(e){}}
|
||||
async function loadScript(name){if(!name)return;try{const r=await api('POST','/api/scripts/load',{name});document.getElementById('script-content').value=r.content||'';}catch(e){}}
|
||||
async function saveScript(){const name=document.getElementById('script-select').value,content=document.getElementById('script-content').value;if(!name)return toast('Select file','err');try{await api('POST','/api/scripts/save',{name,content});toast('Saved','ok');refreshScripts();}catch(e){}}
|
||||
async function createScript(){const name=document.getElementById('script-new-name').value,content=document.getElementById('script-content').value;if(!name)return toast('Enter filename','err');try{await api('POST','/api/scripts/create',{name,content});toast('Created: '+name,'ok');refreshScripts();}catch(e){}}
|
||||
async function deleteScript(){const name=document.getElementById('script-select').value;if(!name)return;if(confirm('Delete '+name+'?'))try{await api('POST','/api/scripts/delete',{name});toast('Deleted','ok');document.getElementById('script-content').value='';refreshScripts();}catch(e){}}
|
||||
|
||||
// Prompt
|
||||
async function refreshPrompt(){try{const r=await api('GET','/api/prompt/');document.getElementById('prompt-content').value=r.system_prompt||'';document.getElementById('prompt-info').textContent=`Script: ${r.script_path} | Rule: ${r.rule_path}`;}catch(e){}}
|
||||
async function updatePrompt(){try{await api('POST','/api/prompt/update',{content:document.getElementById('prompt-content').value});toast('Saved','ok');}catch(e){}}
|
||||
async function reloadPrompt(){try{const r=await api('POST','/api/prompt/reload');document.getElementById('prompt-content').value=r.system_prompt||'';toast('Reloaded','ok');}catch(e){}}
|
||||
// Voice Rule — edit the replay instruction (sanad_rule.txt [REPLAY_SYSTEM_PROMPT]).
|
||||
async function refreshRule(){try{const r=await api('GET','/api/prompt/rule');document.getElementById('rule-content').value=r.content||'';}catch(e){}}
|
||||
async function saveRule(b){btnLoad(b);try{const r=await api('POST','/api/prompt/rule',{content:document.getElementById('rule-content').value});toast(r.message||'Rule saved','ok');}catch(e){}btnDone(b);}
|
||||
|
||||
// Typed Replay — audio plays in YOUR browser (not the server's speaker).
|
||||
let _trAudio=null;
|
||||
let _trAudio=null,_trGenerating=false;
|
||||
function _playTRInBrowser(){
|
||||
try{if(_trAudio){_trAudio.pause();_trAudio.src='';}}catch(e){}
|
||||
_trAudio=new Audio(API+'/api/typed-replay/audio/last?t='+Date.now());
|
||||
return _trAudio.play();
|
||||
}
|
||||
async function trGenerate(b){
|
||||
if(_trGenerating)return toast('Still generating — please wait…','info');
|
||||
const t=document.getElementById('tr-text').value;
|
||||
if(!t)return toast('Enter text','err');
|
||||
btnLoad(b);
|
||||
_trGenerating=true;btnLoad(b);
|
||||
try{
|
||||
await api('POST','/api/typed-replay/say',{
|
||||
const res=await api('POST','/api/typed-replay/say',{
|
||||
text:t,
|
||||
record:document.getElementById('tr-capture').checked,
|
||||
record_name:document.getElementById('tr-name').value,
|
||||
});
|
||||
// Surface an off-voice / partial take instead of leaving it in the log.
|
||||
const warnEl=document.getElementById('tr-warning');
|
||||
if(warnEl){
|
||||
if(res&&res.voice_warning){warnEl.textContent='\u26a0 '+res.voice_warning;warnEl.style.display='';}
|
||||
else{warnEl.style.display='none';warnEl.textContent='';}
|
||||
}
|
||||
await _playTRInBrowser();
|
||||
toast('Generated — playing in your browser','ok');
|
||||
const hz=(res&&res.pitch_hz)?` — ${Math.round(res.pitch_hz)} Hz`:'';
|
||||
toast((res&&res.voice_warning)?('Generated, but check the warning'+hz):('Generated — playing in your browser'+hz),
|
||||
(res&&res.voice_warning)?'err':'ok');
|
||||
refreshTR();
|
||||
}catch(e){toast('Play failed: '+(e&&e.message||e),'err');}
|
||||
btnDone(b);
|
||||
finally{_trGenerating=false;btnDone(b);}
|
||||
}
|
||||
async function trReplayLast(b){
|
||||
btnLoad(b);
|
||||
@ -356,7 +471,63 @@ async function trSaveLast(b){btnLoad(b);try{await api('POST','/api/typed-replay/
|
||||
async function refreshTR(){try{const r=await api('GET','/api/typed-replay/status');const s=r.session||{};document.getElementById('tr-session').innerHTML=`<strong>Text:</strong> ${esc(s.text||'--')}<br><strong>Audio:</strong> ${s.has_audio?'Yes':'No'} | <strong>Capture:</strong> ${s.has_capture?'Yes':'No'}<br><strong>Replays:</strong> ${s.replay_count||0}<br><strong>Generated:</strong> ${s.generated_at||'--'}<br><strong>Saved:</strong> ${esc(s.saved_as||'--')}`;}catch(e){}}
|
||||
|
||||
// Records
|
||||
async function refreshRecords(){try{const r=await api('GET','/api/records/');const el=document.getElementById('records-list');if(!(r.records||[]).length){el.innerHTML='<div class="empty">No records saved</div>';return;}el.innerHTML=`<div style="font-size:.7rem;color:var(--dim);margin-bottom:.3rem">Total: ${r.total_records} | Updated: ${r.last_updated||'--'}</div><table><tr><th>Name</th><th>Text</th><th>Replays</th><th></th></tr>`+(r.records||[]).map(rec=>{const n=esc(rec.record_name);return`<tr><td>${n}</td><td style="max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${esc(rec.text||'')}</td><td>${rec.replay_count||0}</td><td><button class="btn btn-primary btn-sm" onclick="playRecord('${n}','speaker')">Play</button> <button class="btn btn-ghost btn-sm" onclick="playRecord('${n}','raw')">Raw</button> <button class="btn btn-success btn-sm" onclick="downloadRecord('${n}','speaker')">Download</button> <button class="btn btn-danger btn-sm" onclick="deleteRecord('${n}')">Del</button></td></tr>`;}).join('')+'</table>';}catch(e){}}
|
||||
let _records=[];
|
||||
// Fold Arabic so a search for "سلام" also matches "السلام"/"سَلام": strip
|
||||
// diacritics and tatweel, then normalise the alef/ya/ta-marbuta variants.
|
||||
function _norm(s){return (s==null?'':String(s)).toLowerCase()
|
||||
.replace(/[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06ED\u0640]/g,'')
|
||||
.replace(/[\u0623\u0625\u0622\u0671]/g,'\u0627')
|
||||
.replace(/\u0629/g,'\u0647').replace(/\u0649/g,'\u064A')
|
||||
.replace(/\u0624/g,'\u0648').replace(/\u0626/g,'\u064A');}
|
||||
function _dur(rec){const f=(rec.files||{});const a=f.speaker_recording||f.gemini_raw_output||{};return a.duration_seconds||0;}
|
||||
function _saved(rec){return ((rec.timeline||{}).saved_at)||((rec.timeline||{}).audio_generated_at)||'';}
|
||||
function clearRecordSearch(){const b=document.getElementById('records-search');if(b)b.value='';const v=document.getElementById('records-voice');if(v)v.value='';renderRecords();}
|
||||
function renderRecords(){
|
||||
const el=document.getElementById('records-list');if(!el)return;
|
||||
if(!_records.length){el.innerHTML='<div class="empty">No records saved</div>';return;}
|
||||
const q=_norm((document.getElementById('records-search')||{}).value||'').trim();
|
||||
const terms=q?q.split(/\s+/):[];
|
||||
// every term must appear somewhere in the name or the text
|
||||
const voiceSel=document.getElementById('records-voice');
|
||||
const want=(voiceSel||{}).value||'';
|
||||
let list=_records.filter(r=>{
|
||||
if(want && (r.voice||'')!==want) return false;
|
||||
const hay=_norm(r.record_name)+' '+_norm(r.text)+' '+_norm(r.voice_label)+' '+_norm(r.voice);
|
||||
return terms.every(t=>hay.indexOf(t)>=0);});
|
||||
const sort=((document.getElementById('records-sort')||{}).value)||'new';
|
||||
const cmp={new:(a,b)=>_saved(b).localeCompare(_saved(a)),
|
||||
old:(a,b)=>_saved(a).localeCompare(_saved(b)),
|
||||
name:(a,b)=>String(a.record_name).localeCompare(String(b.record_name)),
|
||||
long:(a,b)=>_dur(b)-_dur(a),
|
||||
plays:(a,b)=>(b.replay_count||0)-(a.replay_count||0)}[sort];
|
||||
list=list.slice().sort(cmp);
|
||||
const head=`<div style="font-size:.7rem;color:var(--dim);margin-bottom:.3rem">Showing ${list.length} of ${_records.length}${terms.length?' — filtered':''}</div>`;
|
||||
if(!list.length){el.innerHTML=head+'<div class="empty">Nothing matches that search</div>';return;}
|
||||
el.innerHTML=head+'<table><tr><th>Name</th><th>Voice</th><th>Text</th><th>Saved</th><th>Dur</th><th>Pitch</th><th>Plays</th><th></th></tr>'+
|
||||
list.map(rec=>{const n=esc(rec.record_name);const d=_dur(rec);
|
||||
const vl=rec.voice_label||rec.voice||'--';
|
||||
return `<tr><td>${n}</td>`+
|
||||
`<td style="white-space:nowrap;font-size:.72rem">${esc(vl)}</td>`+
|
||||
`<td style="max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="${esc(rec.text||'')}">${esc(rec.text||'')}</td>`+
|
||||
`<td style="white-space:nowrap;font-size:.7rem;color:var(--dim)">${esc((_saved(rec)||'--').slice(0,16))}</td>`+
|
||||
`<td style="white-space:nowrap">${d?d.toFixed(1)+'s':'--'}</td>`+
|
||||
`<td style="white-space:nowrap;font-size:.72rem${rec.voice_warning?';color:#e0a030':''}" title="${esc(rec.voice_warning||'')}">${rec.pitch_hz?Math.round(rec.pitch_hz)+' Hz':'--'}${rec.voice_warning?' \u26a0':''}</td>`+
|
||||
`<td>${rec.replay_count||0}</td>`+
|
||||
`<td><button class="btn btn-primary btn-sm" onclick="playRecord('${n}','speaker')">Play</button> `+
|
||||
`<button class="btn btn-ghost btn-sm" onclick="playRecord('${n}','raw')">Raw</button> `+
|
||||
`<button class="btn btn-success btn-sm" onclick="downloadRecord('${n}','speaker')">Download</button> `+
|
||||
`<button class="btn btn-danger btn-sm" onclick="deleteRecord('${n}')">Del</button></td></tr>`;}).join('')+'</table>';
|
||||
}
|
||||
function _syncVoiceFilter(){
|
||||
const sel=document.getElementById('records-voice');if(!sel)return;
|
||||
const seen=new Map();
|
||||
_records.forEach(r=>{if(r.voice)seen.set(r.voice,r.voice_label||r.voice);});
|
||||
const keep=sel.value;
|
||||
sel.innerHTML='<option value="">All voices</option>'+
|
||||
[...seen.entries()].map(([v,l])=>`<option value="${esc(v)}">${esc(l)}</option>`).join('');
|
||||
if([...seen.keys()].includes(keep))sel.value=keep;
|
||||
}
|
||||
async function refreshRecords(){try{const r=await api('GET','/api/records/');_records=r.records||[];_syncVoiceFilter();renderRecords();}catch(e){}}
|
||||
// Browser-side playback — streams the WAV from /api/records/audio/{name}
|
||||
// and plays it through the user's speakers (not the robot's).
|
||||
let _recordAudio=null;
|
||||
@ -473,11 +644,91 @@ function downloadLogBundle(){
|
||||
async function refreshStatus(){try{const s=await api('GET','/api/status');document.getElementById('status-dot').className='dot dot-ok';document.getElementById('status-text').textContent='Online';const gb=document.getElementById('gemini-badge');if(s.voice?.connected){gb.style.display='inline-flex';gb.className='hdr-badge hdr-badge-ok';gb.textContent='GEMINI';}else{gb.style.display='inline-flex';gb.className='hdr-badge hdr-badge-err';gb.textContent='GEMINI OFF';}}catch(e){document.getElementById('status-dot').className='dot dot-err';document.getElementById('status-text').textContent='Offline';}}
|
||||
|
||||
// WebSocket logs
|
||||
let logWs;function connectLogs(){const p=location.protocol==='https:'?'wss':'ws';logWs=new WebSocket(`${p}://${location.host}/ws/logs`);const box=document.getElementById('log-box');logWs.onmessage=e=>{box.textContent+=e.data+'\n';if(box.childNodes.length>1000)box.textContent=box.textContent.split('\n').slice(-500).join('\n');box.scrollTop=box.scrollHeight;};logWs.onclose=()=>setTimeout(connectLogs,3000);}
|
||||
let logWs, _logPoll=null, _logCursor=-1, _logMode='';
|
||||
function _logAppend(lines){
|
||||
const box=document.getElementById('log-box');if(!box||!lines||!lines.length)return;
|
||||
box.textContent+=lines.join('\n')+'\n';
|
||||
const all=box.textContent.split('\n');
|
||||
if(all.length>1000)box.textContent=all.slice(-500).join('\n');
|
||||
box.scrollTop=box.scrollHeight;
|
||||
}
|
||||
function _logStatus(mode){
|
||||
if(_logMode===mode)return; _logMode=mode;
|
||||
const el=document.getElementById('log-mode');
|
||||
if(el)el.textContent=mode==='ws'?'live (websocket)':(mode==='poll'?'live (polling)':'disconnected');
|
||||
}
|
||||
// Poll the ring buffer. Used whenever the WebSocket cannot be established —
|
||||
// which is always on the public URL, because the reverse proxy in front of
|
||||
// this app forwards the upgrade as a plain GET.
|
||||
async function _pollLogs(){
|
||||
try{
|
||||
const r=await api('GET','/api/logs/live?cursor='+_logCursor+'&limit=300');
|
||||
if(r&&r.lines){_logAppend(r.lines);_logCursor=r.cursor;_logStatus('poll');}
|
||||
}catch(e){_logStatus('');}
|
||||
}
|
||||
function _startPolling(){
|
||||
if(_logPoll)return;
|
||||
_pollLogs();
|
||||
_logPoll=setInterval(_pollLogs,2000);
|
||||
}
|
||||
function connectLogs(){
|
||||
const p=location.protocol==='https:'?'wss':'ws';
|
||||
let opened=false;
|
||||
try{ logWs=new WebSocket(`${p}://${location.host}/ws/logs`); }
|
||||
catch(e){ _startPolling(); return; }
|
||||
logWs.onopen=()=>{opened=true;if(_logPoll){clearInterval(_logPoll);_logPoll=null;}_logStatus('ws');};
|
||||
logWs.onmessage=e=>_logAppend([e.data]);
|
||||
logWs.onerror=()=>{try{logWs.close();}catch(_){}};
|
||||
logWs.onclose=()=>{
|
||||
// No WebSocket here: fall back rather than retrying forever into a proxy
|
||||
// that will never upgrade.
|
||||
_startPolling();
|
||||
if(opened)setTimeout(connectLogs,5000);
|
||||
};
|
||||
// Belt and braces: if it never opens, start polling anyway.
|
||||
setTimeout(()=>{if(!opened)_startPolling();},3000);
|
||||
}
|
||||
// ── sign-in history ────────────────────────────────────────────────
|
||||
// Exports the FULL history, not just the rows on screen. Navigating rather
|
||||
// than fetching lets the browser save the file and send the session cookie.
|
||||
function exportLogins(fmt){
|
||||
window.location='/api/auth/logins/export?format='+encodeURIComponent(fmt||'csv');
|
||||
}
|
||||
async function refreshLogins(){
|
||||
try{
|
||||
const r=await api('GET','/api/auth/logins?limit=20');
|
||||
const el=document.getElementById('login-list');if(!el)return;
|
||||
const rows=r.logins||[];
|
||||
if(!rows.length){el.innerHTML='<div class="empty">No sign-ins recorded yet</div>';return;}
|
||||
el.innerHTML='<table><tr><th>When</th><th>Device</th><th>IP</th><th></th></tr>'+
|
||||
rows.map(x=>`<tr>`+
|
||||
`<td style="white-space:nowrap">${esc(x.at||'')}</td>`+
|
||||
`<td title="${esc(x.user_agent||'')}">${esc(x.device||'')}</td>`+
|
||||
`<td style="font-size:.72rem;color:var(--dim)">${esc(x.ip||'')}</td>`+
|
||||
`<td>${x.ok?'<span style="color:#4ade80">ok</span>':'<span style="color:#f87171">failed</span>'}</td>`+
|
||||
`</tr>`).join('')+'</table>';
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
// Init — every audio feature in lite plays client-side via <audio> tags.
|
||||
refreshStatus();refreshAudio();refreshScripts();refreshPrompt();refreshTR();refreshApiKey();refreshRecords();connectLogs();
|
||||
// Robot Voice — map friendly robot labels to Gemini voices; switch + persist.
|
||||
async function refreshVoices(){
|
||||
try{
|
||||
const r=await api('GET','/api/voice/voices');
|
||||
const box=document.getElementById('voice-options');
|
||||
if(box)box.innerHTML=(r.options||[]).map(o=>`<button class="btn ${o.active?'btn-primary':'btn-ghost'} btn-sm" data-voice="${esc(o.voice)}" onclick="setVoice(this.dataset.voice,this)">${esc(o.label)}</button>`).join('')||'<span style="color:var(--muted);font-size:.72rem">No voices configured</span>';
|
||||
const cur=document.getElementById('voice-current');
|
||||
if(cur)cur.innerHTML=`Current: <strong>${esc(r.current_label||r.current||'--')}</strong>`+(r.current?` <span style="color:var(--dim)">(${esc(r.current)})</span>`:'');
|
||||
}catch(e){}
|
||||
}
|
||||
async function setVoice(voice,b){
|
||||
btnLoad(b);
|
||||
try{const r=await api('POST','/api/voice/voice',{voice});toast(r.message||('Voice set to '+(r.label||voice)),'ok');refreshVoices();}catch(e){}
|
||||
btnDone(b);
|
||||
}
|
||||
refreshStatus();refreshAudio();refreshTR();refreshApiKey();refreshVoices();refreshRule();refreshRecords();connectLogs();refreshLogins();if(window.liveInit)liveInit();
|
||||
setInterval(refreshStatus,5000);
|
||||
</script>
|
||||
<script src="/static/live.js?v=1787937410"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
480
dashboard/static/live.js
Normal file
480
dashboard/static/live.js
Normal file
@ -0,0 +1,480 @@
|
||||
/* Live Gemini — talk to a robot voice from the browser.
|
||||
*
|
||||
* The page opens its own WebSocket to Gemini rather than routing audio through
|
||||
* this server, because the Apache [P] rewrite in front of the app cannot
|
||||
* upgrade a WebSocket (verified: 101 straight to uvicorn, 404 through the
|
||||
* proxy). The server only mints a short-lived ephemeral token, so the API key
|
||||
* never reaches the browser.
|
||||
*
|
||||
* Audio contract of the Live API:
|
||||
* send 16 kHz signed 16-bit mono PCM, base64, as realtimeInput
|
||||
* receive 24 kHz signed 16-bit mono PCM, base64, in serverContent parts
|
||||
*/
|
||||
(function () {
|
||||
const SEND_RATE = 16000;
|
||||
const RECV_RATE = 24000;
|
||||
|
||||
let ws = null; // socket to Gemini
|
||||
let micStream = null; // MediaStream from getUserMedia
|
||||
let micCtx = null; // AudioContext for capture
|
||||
let playCtx = null; // AudioContext for playback
|
||||
let processor = null;
|
||||
let playHead = 0; // when the next chunk should start, in playCtx time
|
||||
let connected = false;
|
||||
let sentChunks = 0, recvFrames = 0, playedChunks = 0, micPeak = 0, diagTimer = null;
|
||||
let cfg = null; // {voices:[{voice,label,persona}], model}
|
||||
let selectedVoice = 'Charon';
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
|
||||
function status(text, tone) {
|
||||
const el = $('live-status');
|
||||
if (!el) return;
|
||||
el.textContent = text;
|
||||
el.style.color = tone === 'err' ? '#f87171'
|
||||
: tone === 'ok' ? '#4ade80' : 'var(--dim)';
|
||||
}
|
||||
|
||||
function diag() {
|
||||
const el = document.getElementById('live-diag');
|
||||
if (!el) return;
|
||||
const mic = micCtx ? micCtx.state : '-';
|
||||
const play = playCtx ? playCtx.state : '-';
|
||||
el.textContent = `sent ${sentChunks} chunks (peak ${micPeak.toFixed(3)}) · `
|
||||
+ `received ${recvFrames} frames · played ${playedChunks} · `
|
||||
+ `mic ctx ${mic} · out ctx ${play}`;
|
||||
}
|
||||
|
||||
function addLine(who, text) {
|
||||
const box = $('live-transcript');
|
||||
if (!box || !text) return;
|
||||
const row = document.createElement('div');
|
||||
row.style.margin = '.15rem 0';
|
||||
row.innerHTML = `<span style="color:${who === 'you' ? '#7dd3fc' : '#a78bfa'}">`
|
||||
+ `${who === 'you' ? 'you' : 'robot'}:</span> `;
|
||||
row.appendChild(document.createTextNode(text));
|
||||
box.appendChild(row);
|
||||
box.scrollTop = box.scrollHeight;
|
||||
}
|
||||
|
||||
// ── config + persona ────────────────────────────────────────────
|
||||
async function loadConfig() {
|
||||
cfg = await api('GET', '/api/live/config');
|
||||
const box = $('live-voices');
|
||||
if (box) {
|
||||
box.innerHTML = (cfg.voices || []).map(v =>
|
||||
`<button class="btn ${v.voice === selectedVoice ? 'btn-primary' : 'btn-ghost'} btn-sm"
|
||||
data-voice="${esc(v.voice)}">${esc(v.label)}</button>`).join(' ');
|
||||
box.querySelectorAll('button').forEach(b => {
|
||||
b.onclick = () => selectVoice(b.dataset.voice);
|
||||
});
|
||||
}
|
||||
selectVoice(selectedVoice);
|
||||
const m = $('live-model');
|
||||
if (m) m.textContent = cfg.model || '';
|
||||
}
|
||||
|
||||
function selectVoice(voice) {
|
||||
selectedVoice = voice;
|
||||
const entry = (cfg && cfg.voices || []).find(v => v.voice === voice);
|
||||
const box = $('live-voices');
|
||||
if (box) box.querySelectorAll('button').forEach(b => {
|
||||
const on = b.dataset.voice === voice;
|
||||
b.classList.toggle('btn-primary', on);
|
||||
b.classList.toggle('btn-ghost', !on);
|
||||
});
|
||||
if (personas.length) {
|
||||
const sel = $('persona-picker');
|
||||
if (sel && activeIds[voice]) sel.value = activeIds[voice];
|
||||
livePickPersona((sel && sel.value) || '');
|
||||
}
|
||||
showActive();
|
||||
if (connected) status('Voice changes apply on the next connection', 'err');
|
||||
}
|
||||
|
||||
let personas = []; // whole library
|
||||
let activeIds = {}; // voice -> persona id in use
|
||||
let editingId = ''; // persona currently in the editor
|
||||
|
||||
async function loadPersonas(keepId) {
|
||||
const note = $('persona-note');
|
||||
let r;
|
||||
try {
|
||||
r = await api('GET', '/api/live/personas');
|
||||
} catch (e) {
|
||||
if (note) note.textContent = 'Could not load personas: ' + (e && e.message || e);
|
||||
return;
|
||||
}
|
||||
if (note) note.textContent = '';
|
||||
personas = r.personas || [];
|
||||
activeIds = r.active || {};
|
||||
const sel = $('persona-picker');
|
||||
if (sel) {
|
||||
sel.innerHTML = personas.map(p =>
|
||||
`<option value="${esc(p.id)}">${esc(p.name)}${p.builtin ? ' *' : ''}</option>`).join('');
|
||||
const want = keepId || activeIds[selectedVoice] || (personas[0] && personas[0].id);
|
||||
if (want) sel.value = want;
|
||||
}
|
||||
try {
|
||||
livePickPersona((sel && sel.value) || '');
|
||||
showActive();
|
||||
} catch (e) {
|
||||
window.__personaError = e;
|
||||
if (note) note.textContent = 'Persona render failed: ' + ((e && e.message) || e);
|
||||
}
|
||||
}
|
||||
|
||||
function showActive() {
|
||||
// Name the robot on the card so it is clear which one is being edited.
|
||||
const label = (cfg && cfg.voices || []).find(v => v.voice === selectedVoice);
|
||||
const forEl = $('persona-for');
|
||||
if (forEl) forEl.textContent = label ? '— ' + label.label : '';
|
||||
const el = $('persona-active');
|
||||
if (!el) return;
|
||||
const id = activeIds[selectedVoice];
|
||||
const p = personas.find(x => x.id === id);
|
||||
el.textContent = p ? `in use by ${label ? label.label : selectedVoice}: ${p.name}` : '';
|
||||
}
|
||||
|
||||
window.livePickPersona = (id) => {
|
||||
editingId = id;
|
||||
const p = personas.find(x => x.id === id);
|
||||
const ta = $('live-persona'), nm = $('persona-name');
|
||||
if (p) {
|
||||
if (ta) ta.value = p.text || '';
|
||||
if (nm) nm.value = p.builtin ? p.name.replace(/ \(default\)$/, '') + ' (copy)' : p.name;
|
||||
}
|
||||
const del = document.querySelector('[onclick="liveDeletePersona()"]');
|
||||
if (del) del.disabled = !!(p && p.builtin);
|
||||
};
|
||||
|
||||
window.liveUsePersona = async () => {
|
||||
const sel = $('persona-picker');
|
||||
if (!sel || !sel.value) return;
|
||||
try {
|
||||
const r = await api('POST', '/api/live/personas/select',
|
||||
{ voice: selectedVoice, id: sel.value });
|
||||
activeIds[selectedVoice] = r.id;
|
||||
showActive();
|
||||
toast(`${r.name} — applies on the next connection`, 'ok');
|
||||
} catch (e) { toast('Could not select: ' + (e && e.message || e), 'err'); }
|
||||
};
|
||||
|
||||
async function persistPersona(forceNew) {
|
||||
const ta = $('live-persona'), nm = $('persona-name');
|
||||
if (!ta) return;
|
||||
const current = personas.find(x => x.id === editingId);
|
||||
// Built-ins are read-only: editing one always creates a copy.
|
||||
const asNew = forceNew || !editingId || (current && current.builtin);
|
||||
try {
|
||||
const r = await api('POST', '/api/live/personas', {
|
||||
id: asNew ? '' : editingId,
|
||||
name: (nm && nm.value) || 'Untitled persona',
|
||||
text: ta.value,
|
||||
});
|
||||
await loadPersonas(r.id);
|
||||
toast(asNew ? `Saved as "${r.name}"` : `Saved "${r.name}"`, 'ok');
|
||||
} catch (e) { toast('Save failed: ' + (e && e.message || e), 'err'); }
|
||||
}
|
||||
|
||||
window.liveSavePersona = () => persistPersona(false);
|
||||
window.liveSavePersonaAs = () => persistPersona(true);
|
||||
|
||||
window.liveDeletePersona = async () => {
|
||||
const p = personas.find(x => x.id === editingId);
|
||||
if (!p) return;
|
||||
if (p.builtin) { toast('Built-in personas cannot be deleted', 'err'); return; }
|
||||
if (!confirm(`Delete persona "${p.name}"?`)) return;
|
||||
try {
|
||||
const r = await api('POST', '/api/live/personas/delete', { id: p.id });
|
||||
if ((r.reassigned || []).length) {
|
||||
toast(`Deleted — ${r.reassigned.join(', ')} fell back to the default`, 'ok');
|
||||
} else { toast('Deleted', 'ok'); }
|
||||
await loadPersonas();
|
||||
} catch (e) { toast('Delete failed: ' + (e && e.message || e), 'err'); }
|
||||
};
|
||||
|
||||
// ── audio helpers ───────────────────────────────────────────────
|
||||
function floatToPcm16(input) {
|
||||
const out = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, input[i]));
|
||||
out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function downsample(buffer, inRate, outRate) {
|
||||
if (outRate === inRate) return buffer;
|
||||
const ratio = inRate / outRate;
|
||||
const length = Math.round(buffer.length / ratio);
|
||||
const out = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (let i = 0; i < length; i++) {
|
||||
const next = Math.round((i + 1) * ratio);
|
||||
let sum = 0, count = 0;
|
||||
for (let j = offset; j < next && j < buffer.length; j++) { sum += buffer[j]; count++; }
|
||||
out[i] = count ? sum / count : 0;
|
||||
offset = next;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function b64FromBytes(bytes) {
|
||||
let bin = '';
|
||||
const chunk = 0x8000;
|
||||
for (let i = 0; i < bytes.length; i += chunk) {
|
||||
bin += String.fromCharCode.apply(null, bytes.subarray(i, i + chunk));
|
||||
}
|
||||
return btoa(bin);
|
||||
}
|
||||
|
||||
function playPcm(b64) {
|
||||
const bin = atob(b64);
|
||||
const bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
const pcm = new Int16Array(bytes.buffer);
|
||||
if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
const buf = playCtx.createBuffer(1, pcm.length, RECV_RATE);
|
||||
const ch = buf.getChannelData(0);
|
||||
for (let i = 0; i < pcm.length; i++) ch[i] = pcm[i] / 32768;
|
||||
const src = playCtx.createBufferSource();
|
||||
src.buffer = buf;
|
||||
src.connect(playCtx.destination);
|
||||
// Queue chunks back-to-back so speech does not overlap or gap.
|
||||
const now = playCtx.currentTime;
|
||||
if (playHead < now) playHead = now;
|
||||
src.start(playHead);
|
||||
playHead += buf.duration;
|
||||
}
|
||||
|
||||
// ── session ─────────────────────────────────────────────────────
|
||||
async function connect() {
|
||||
if (connected) return;
|
||||
status('Requesting microphone…');
|
||||
try {
|
||||
micStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
|
||||
});
|
||||
} catch (e) {
|
||||
status('Microphone denied — the browser must allow it', 'err');
|
||||
return;
|
||||
}
|
||||
|
||||
status('Getting a session token…');
|
||||
let session;
|
||||
try {
|
||||
session = await api('POST', '/api/live/token', { voice: selectedVoice });
|
||||
} catch (e) {
|
||||
status('Could not start: ' + (e && e.message || e), 'err');
|
||||
stop();
|
||||
return;
|
||||
}
|
||||
|
||||
// The server decides how to authenticate: an ephemeral token goes in
|
||||
// `access_token` (v1beta only), a real API key in `key`. Passing either in
|
||||
// the other's parameter is refused — "unregistered callers" one way,
|
||||
// "API key not valid" the other.
|
||||
const authParam = session.auth_param || 'access_token';
|
||||
const url = `wss://generativelanguage.googleapis.com/ws/`
|
||||
+ `google.ai.generativelanguage.${session.api_version}.GenerativeService`
|
||||
+ `.BidiGenerateContent?${authParam}=${encodeURIComponent(session.token)}`;
|
||||
status('Connecting to Gemini…');
|
||||
ws = new WebSocket(url);
|
||||
ws.binaryType = 'arraybuffer';
|
||||
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({
|
||||
setup: {
|
||||
model: session.model,
|
||||
generationConfig: {
|
||||
responseModalities: ['AUDIO'],
|
||||
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: session.voice } } },
|
||||
},
|
||||
systemInstruction: { parts: [{ text: session.persona || '' }] },
|
||||
inputAudioTranscription: {},
|
||||
outputAudioTranscription: {},
|
||||
// Mirrors Sanad_Package_5's VAD: LOW start-sensitivity so room noise
|
||||
// does not open a turn (on HIGH the model answers every rustle), and
|
||||
// a short silence window so replies still feel immediate.
|
||||
realtimeInputConfig: {
|
||||
automaticActivityDetection: {
|
||||
disabled: false,
|
||||
startOfSpeechSensitivity: 'START_SENSITIVITY_LOW',
|
||||
endOfSpeechSensitivity: 'END_SENSITIVITY_LOW',
|
||||
prefixPaddingMs: 300,
|
||||
silenceDurationMs: 400,
|
||||
},
|
||||
},
|
||||
},
|
||||
}));
|
||||
connected = true;
|
||||
status('Live — speak now', 'ok');
|
||||
startMic();
|
||||
const b = $('live-connect');
|
||||
if (b) { b.textContent = 'Stop'; b.classList.remove('btn-primary'); b.classList.add('btn-danger'); }
|
||||
};
|
||||
|
||||
ws.onmessage = async (ev) => {
|
||||
let text = ev.data;
|
||||
if (text instanceof ArrayBuffer) text = new TextDecoder().decode(text);
|
||||
else if (text instanceof Blob) text = await text.text();
|
||||
let msg;
|
||||
try { msg = JSON.parse(text); } catch (_) { return; }
|
||||
recvFrames++;
|
||||
const sc = msg.serverContent || {};
|
||||
(sc.modelTurn && sc.modelTurn.parts || []).forEach(p => {
|
||||
const inline = p.inlineData || p.inline_data;
|
||||
if (inline && inline.data) { playedChunks++; playPcm(inline.data); }
|
||||
});
|
||||
if (sc.outputTranscription && sc.outputTranscription.text) {
|
||||
addLine('robot', sc.outputTranscription.text);
|
||||
}
|
||||
if (sc.inputTranscription && sc.inputTranscription.text) {
|
||||
addLine('you', sc.inputTranscription.text);
|
||||
}
|
||||
if (msg.error) status('Gemini error: ' + JSON.stringify(msg.error), 'err');
|
||||
};
|
||||
|
||||
ws.onerror = () => status('Connection error', 'err');
|
||||
ws.onclose = (e) => {
|
||||
if (connected) status('Session ended' + (e && e.reason ? ' — ' + e.reason : ''), 'err');
|
||||
stop();
|
||||
};
|
||||
}
|
||||
|
||||
function startMic() {
|
||||
micCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
// A context created outside a user gesture starts suspended and its
|
||||
// processor never fires — silence with no error anywhere.
|
||||
if (micCtx.state === 'suspended') micCtx.resume();
|
||||
if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
if (playCtx.state === 'suspended') playCtx.resume();
|
||||
sentChunks = recvFrames = playedChunks = 0; micPeak = 0;
|
||||
if (diagTimer) clearInterval(diagTimer);
|
||||
diagTimer = setInterval(diag, 1000);
|
||||
const source = micCtx.createMediaStreamSource(micStream);
|
||||
processor = micCtx.createScriptProcessor(4096, 1, 1);
|
||||
source.connect(processor);
|
||||
processor.connect(micCtx.destination);
|
||||
processor.onaudioprocess = (ev) => {
|
||||
if (!connected || !ws || ws.readyState !== WebSocket.OPEN) return;
|
||||
const raw = ev.inputBuffer.getChannelData(0);
|
||||
let peak = 0;
|
||||
for (let i = 0; i < raw.length; i++) { const a = Math.abs(raw[i]); if (a > peak) peak = a; }
|
||||
if (peak > micPeak) micPeak = peak;
|
||||
const down = downsample(raw, micCtx.sampleRate, SEND_RATE);
|
||||
const pcm = floatToPcm16(down);
|
||||
sentChunks++;
|
||||
ws.send(JSON.stringify({
|
||||
realtimeInput: {
|
||||
mediaChunks: [{
|
||||
mimeType: 'audio/pcm;rate=' + SEND_RATE,
|
||||
data: b64FromBytes(new Uint8Array(pcm.buffer)),
|
||||
}],
|
||||
},
|
||||
}));
|
||||
};
|
||||
}
|
||||
|
||||
function stop() {
|
||||
connected = false;
|
||||
if (diagTimer) { clearInterval(diagTimer); diagTimer = null; }
|
||||
diag();
|
||||
try { if (processor) processor.disconnect(); } catch (_) {}
|
||||
try { if (micCtx) micCtx.close(); } catch (_) {}
|
||||
try { if (micStream) micStream.getTracks().forEach(t => t.stop()); } catch (_) {}
|
||||
try { if (ws && ws.readyState <= 1) ws.close(); } catch (_) {}
|
||||
processor = micCtx = micStream = ws = null;
|
||||
playHead = 0;
|
||||
const b = $('live-connect');
|
||||
if (b) { b.textContent = 'Connect'; b.classList.add('btn-primary'); b.classList.remove('btn-danger'); }
|
||||
}
|
||||
|
||||
// ── microphone / speaker ────────────────────────────────────────
|
||||
function permNote(text, tone) {
|
||||
const el = $('live-perm');
|
||||
if (!el) return;
|
||||
el.textContent = text;
|
||||
el.style.color = tone === 'err' ? '#f87171'
|
||||
: tone === 'ok' ? '#4ade80' : 'var(--dim)';
|
||||
}
|
||||
|
||||
async function permState() {
|
||||
// Not supported everywhere (Safari); absence is not an error.
|
||||
try {
|
||||
if (!navigator.permissions || !navigator.permissions.query) return 'unknown';
|
||||
const s = await navigator.permissions.query({ name: 'microphone' });
|
||||
return s.state; // granted | denied | prompt
|
||||
} catch (e) { return 'unknown'; }
|
||||
}
|
||||
|
||||
window.liveRefreshPerm = async () => {
|
||||
const state = await permState();
|
||||
if (state === 'granted') permNote('microphone: allowed', 'ok');
|
||||
else if (state === 'denied') permNote('microphone: blocked - use the padlock in the address bar to allow it', 'err');
|
||||
else if (state === 'prompt') permNote('microphone: not asked yet', '');
|
||||
else permNote('', '');
|
||||
};
|
||||
|
||||
window.liveRequestMic = async () => {
|
||||
permNote('asking for the microphone...', '');
|
||||
try {
|
||||
const s = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
// Release it immediately: this is only to obtain permission.
|
||||
s.getTracks().forEach(t => t.stop());
|
||||
let name = '';
|
||||
try {
|
||||
const devs = await navigator.mediaDevices.enumerateDevices();
|
||||
const mic = devs.find(d => d.kind === 'audioinput' && d.label);
|
||||
name = mic ? ' (' + mic.label + ')' : '';
|
||||
} catch (e) { /* labels need permission; ignore */ }
|
||||
permNote('microphone: allowed' + name, 'ok');
|
||||
} catch (e) {
|
||||
// Once refused, the browser will not prompt again for this site.
|
||||
const denied = e && (e.name === 'NotAllowedError' || e.name === 'SecurityError');
|
||||
permNote(denied
|
||||
? 'microphone blocked - click the padlock next to the address, set Microphone to Allow, then reload'
|
||||
: 'microphone unavailable: ' + ((e && e.message) || e), 'err');
|
||||
}
|
||||
};
|
||||
|
||||
window.liveTestSpeaker = async () => {
|
||||
try {
|
||||
if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||||
if (playCtx.state === 'suspended') await playCtx.resume();
|
||||
const osc = playCtx.createOscillator();
|
||||
const gain = playCtx.createGain();
|
||||
osc.frequency.value = 440;
|
||||
gain.gain.value = 0.15; // audible but not startling
|
||||
osc.connect(gain).connect(playCtx.destination);
|
||||
osc.start();
|
||||
osc.stop(playCtx.currentTime + 0.35);
|
||||
permNote('speaker: played a test tone (output ' + playCtx.state + ')', 'ok');
|
||||
} catch (e) {
|
||||
permNote('speaker failed: ' + ((e && e.message) || e), 'err');
|
||||
}
|
||||
};
|
||||
|
||||
window.liveToggle = () => (connected ? (stop(), status('Disconnected')) : connect());
|
||||
window.liveInit = () => {
|
||||
// Independent: a failure in one must not leave the other blank.
|
||||
loadConfig().catch((e) => status('Could not load config: ' + (e && e.message || e), 'err'));
|
||||
if (window.liveRefreshPerm) liveRefreshPerm();
|
||||
loadPersonas().catch((e) => {
|
||||
window.__personaError = e;
|
||||
const note = document.getElementById('persona-note');
|
||||
const msg = (e && (e.stack || e.message)) || String(e);
|
||||
if (note) note.textContent = 'Persona list failed: ' + msg;
|
||||
});
|
||||
};
|
||||
|
||||
// Self-initialise. This file loads at the end of <body>, AFTER the inline
|
||||
// script has already run its startup calls — so waiting to be called by it
|
||||
// left the robot list empty and the model showing "--".
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', window.liveInit);
|
||||
} else {
|
||||
window.liveInit();
|
||||
}
|
||||
})();
|
||||
@ -23,12 +23,38 @@ _watchers: set[asyncio.Queue] = set()
|
||||
_watchers_lock = threading.Lock()
|
||||
|
||||
|
||||
_seq = 0 # total lines ever pushed; _recent holds the tail of this sequence
|
||||
|
||||
|
||||
def recent_since(cursor: int, limit: int = 300) -> tuple[list[str], int]:
|
||||
"""Lines newer than `cursor`, plus the new cursor.
|
||||
|
||||
Backs the HTTP polling fallback: this deployment sits behind an Apache
|
||||
`[P]` rewrite that cannot upgrade WebSockets (verified — the same
|
||||
handshake returns 101 straight to uvicorn and 404 through the proxy), so
|
||||
the browser can never hold /ws/logs open and needs to poll instead.
|
||||
|
||||
A cursor of -1, or one so old its lines have already been evicted from the
|
||||
ring, returns the most recent `limit` lines.
|
||||
"""
|
||||
lines = list(_recent)
|
||||
first_seq = _seq - len(lines) + 1 # sequence number of lines[0]
|
||||
if cursor < 0 or cursor < first_seq - 1:
|
||||
tail = lines[-limit:]
|
||||
return tail, _seq
|
||||
skip = cursor - (first_seq - 1)
|
||||
fresh = lines[skip:][:limit]
|
||||
return fresh, cursor + len(fresh)
|
||||
|
||||
|
||||
def push_log_line(line: str):
|
||||
"""Called from the logging system to feed new lines.
|
||||
|
||||
May be called from any thread (logging is multi-threaded), so we
|
||||
snapshot the watchers under a lock before iterating.
|
||||
"""
|
||||
global _seq
|
||||
_seq += 1
|
||||
_recent.append(line)
|
||||
with _watchers_lock:
|
||||
snapshot = list(_watchers)
|
||||
|
||||
289
gemini/client.py
289
gemini/client.py
@ -13,6 +13,8 @@ import asyncio
|
||||
import base64
|
||||
import inspect
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
import websockets
|
||||
@ -37,11 +39,143 @@ _DEFAULT_SYSTEM_PROMPT = _cfg_section("core", "gemini_defaults").get(
|
||||
"You are Sanad (Bousandah), a wise and friendly Emirati assistant. "
|
||||
"Speak in UAE dialect (Khaleeji). Be helpful and concise."
|
||||
)
|
||||
|
||||
# TTS / typed-replay system prompt. The voice_client speaks TYPED text, so it
|
||||
# must read the text VERBATIM in its OWN language — NOT answer it and NOT force
|
||||
# Khaleeji. This is what makes the Live native-audio model return AUDIO instead
|
||||
# of "thinking" text. Copied from SanadR1/Sanadv3 so the lite dashboard speaks
|
||||
# with the exact same engine + behavior as the robots.
|
||||
TTS_SYSTEM_PROMPT = _cfg_section("core", "gemini_defaults").get(
|
||||
"tts_system_prompt",
|
||||
"You are a pure multilingual text-to-speech voice. The instant the user "
|
||||
"sends text, speak it aloud word for word in the SAME language it is "
|
||||
"written in, then stop. Output ONLY that spoken audio — no thinking, no "
|
||||
"commentary, no acknowledgements, no headers, no explanations, no "
|
||||
"greetings, no extra words. Never translate and never change the language: "
|
||||
"English stays English, Arabic stays Arabic, Urdu stays Urdu, Indonesian "
|
||||
"stays Indonesian. Your speech must be identical to the user's text, "
|
||||
"nothing more and nothing less."
|
||||
)
|
||||
# Per-voice system-prompt overrides, keyed by prebuilt voice name. Empty by
|
||||
# default, and INTENTIONALLY so for Charon (Unitree G1) and Puck (Unitree R1):
|
||||
# those two must keep speaking with the robots' own verbatim TTS prompt, since
|
||||
# sounding identical to Sanadv3/SanadR1 is the whole point. Only voices with no
|
||||
# robot to match — Agibot x2 / Kore — get a different instruction here.
|
||||
# Configured in core_config.json → gemini_defaults.voice_system_prompts.
|
||||
_VOICE_SYSTEM_PROMPTS = _cfg_section("core", "gemini_defaults").get(
|
||||
"voice_system_prompts", {}) or {}
|
||||
|
||||
_RECV_TIMEOUT_SEC = _GC.get("recv_timeout_sec", 30)
|
||||
_RECONNECT_MAX_ATTEMPTS = _GC.get("reconnect_max_attempts", 3)
|
||||
_RECONNECT_INITIAL_DELAY_SEC = _GC.get("reconnect_initial_delay_sec", 1.0)
|
||||
_RECONNECT_MAX_DELAY_SEC = _GC.get("reconnect_max_delay_sec", 10.0)
|
||||
|
||||
# Dedicated text-to-speech model (stateless REST generateContent). Far more
|
||||
# reliable than the Live native-audio model for pure TTS: the Live model often
|
||||
# returns its "thinking" reasoning text instead of speech for short prompts,
|
||||
# whereas this returns audio-only. Same prebuilt voices (Charon/Puck/Kore/...).
|
||||
GEMINI_TTS_MODEL = _GC.get("tts_model", "gemini-2.5-flash-preview-tts")
|
||||
_GEMINI_REST_BASE = _GC.get(
|
||||
"rest_base", "https://generativelanguage.googleapis.com/v1beta")
|
||||
# Spoken-directive preamble so the TTS model READS the text aloud rather than
|
||||
# answering it (bare short inputs like "مرحبا" otherwise 400 "Model tried to
|
||||
# generate text"), and nudges an Emirati/Gulf dialect. The directive is NOT
|
||||
# spoken (verified: audio length ~unchanged with/without it). Retries cover the
|
||||
# model's occasional empty response. All config-overridable via gemini_config.json.
|
||||
_TTS_PREAMBLE = _GC.get("tts_preamble", "بِاللهجة الإماراتية: ")
|
||||
_TTS_MAX_ATTEMPTS = int(_GC.get("tts_max_attempts", 3))
|
||||
# BCP-47 language/accent code for the TTS speechConfig. "ar-AE" = UAE Arabic —
|
||||
# the structured, reliable lever for an Emirati accent. "" omits the field.
|
||||
_TTS_LANGUAGE_CODE = _GC.get("tts_language_code", "ar-AE")
|
||||
# Sampling temperature. 0.0 makes the TTS DETERMINISTIC — the same text renders
|
||||
# the same audio every time (stable tone). The default (~1.0) re-performs with
|
||||
# random prosody on every call, which is why the tone kept drifting between
|
||||
# generations. The retry temperature is used only if temp-0 returns an empty
|
||||
# part for a given input (so we can still get audio for it).
|
||||
_TTS_TEMPERATURE = float(_GC.get("tts_temperature", 0.0))
|
||||
_TTS_RETRY_TEMPERATURE = float(_GC.get("tts_retry_temperature", 0.6))
|
||||
# Per-attempt HTTP timeout for the TTS call. A normal TTS response is ~2s; a
|
||||
# stuck call would otherwise hang the whole generation and hold the typed-replay
|
||||
# single-flight lock (blocking the UI), so keep this tight.
|
||||
_TTS_TIMEOUT = float(_GC.get("tts_timeout_sec", 20))
|
||||
# Ask the Live session to transcribe its own spoken output. Costs nothing
|
||||
# extra and gives typed_replay a way to verify the whole text was actually
|
||||
# read aloud. Set false in gemini_config.json to go back to blind reads.
|
||||
_OUTPUT_TRANSCRIPTION = bool(_GC.get("output_transcription", True))
|
||||
# How long to keep reading after `generationComplete` while waiting for
|
||||
# `turnComplete`. Only the tail of an already-generated turn arrives in this
|
||||
# window, so it is short.
|
||||
_POST_GENERATION_GRACE_SEC = float(_GC.get("post_generation_grace_sec", 3.0))
|
||||
|
||||
|
||||
class GeminiQuotaExhausted(RuntimeError):
|
||||
"""The API key itself is out of credits / over quota.
|
||||
|
||||
Distinct from every other failure because retrying is pointless: the Live
|
||||
socket, the reconnect chain and the REST TTS fallback all fail the same
|
||||
way. Without this, one Generate & Play burns 4 Live attempts (each with a
|
||||
reconnect chain) plus 3 REST retries before answering — long enough to
|
||||
blow the Apache proxy timeout, so the user sees a bare 503 and the cron
|
||||
health-check restarts a perfectly healthy app.
|
||||
"""
|
||||
|
||||
|
||||
# Substrings that mean "this key cannot make calls right now". Deliberately
|
||||
# narrow: a transient 429 rate-limit IS worth retrying, an empty wallet is not.
|
||||
_QUOTA_MARKERS = (
|
||||
"credits are depleted",
|
||||
"prepayment credits",
|
||||
"billing",
|
||||
"exceeded your current quota",
|
||||
)
|
||||
|
||||
|
||||
def _quota_message(exc: BaseException) -> str:
|
||||
"""Return a clean reason string if `exc` is a credits/quota failure, else ""."""
|
||||
text = f"{exc}".lower()
|
||||
if not any(m in text for m in _QUOTA_MARKERS):
|
||||
return ""
|
||||
if "credits are depleted" in text or "prepayment credits" in text:
|
||||
return ("Gemini API credits are depleted — top up billing in AI Studio "
|
||||
"for this project's key. No voice can be generated until then.")
|
||||
return ("Gemini API quota/billing error — this API key cannot generate "
|
||||
"audio right now.")
|
||||
|
||||
|
||||
def _tts_rest_blocking(url: str, body: dict, timeout: float) -> bytes:
|
||||
"""Blocking POST to the TTS generateContent endpoint → PCM bytes.
|
||||
|
||||
Runs in a worker thread (via asyncio.to_thread) so the event loop is never
|
||||
blocked. Returns the first inline audio part decoded from base64; raises
|
||||
RuntimeError carrying the API's message on an HTTP/transport error.
|
||||
"""
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url, data=data,
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = ""
|
||||
try:
|
||||
detail = json.loads(exc.read().decode("utf-8")).get("error", {}).get("message", "")
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError(f"Gemini TTS HTTP {exc.code}: {detail or exc.reason}")
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f"Gemini TTS request failed: {exc.reason}")
|
||||
payload = json.loads(raw)
|
||||
if isinstance(payload, dict) and payload.get("error"):
|
||||
msg = payload["error"].get("message", payload["error"])
|
||||
raise RuntimeError(f"Gemini TTS error: {msg}")
|
||||
for cand in payload.get("candidates", []):
|
||||
for part in cand.get("content", {}).get("parts", []):
|
||||
inline = part.get("inlineData") or part.get("inline_data")
|
||||
if inline and inline.get("data"):
|
||||
return base64.b64decode(inline["data"])
|
||||
return b""
|
||||
|
||||
|
||||
class GeminiVoiceClient:
|
||||
"""Manages one WebSocket session to the Gemini Bidi audio API.
|
||||
@ -63,6 +197,10 @@ class GeminiVoiceClient:
|
||||
self._connect_lock = asyncio.Lock() # serializes reconnect attempts
|
||||
self._owner: str | None = None
|
||||
self._reconnect_attempts = 0
|
||||
# Transcript of the audio returned by the most recent send_text(),
|
||||
# when output transcription is enabled. Written under the session
|
||||
# lock, so it always belongs to the call that just returned.
|
||||
self.last_output_transcript = ""
|
||||
|
||||
@property
|
||||
def connected(self) -> bool:
|
||||
@ -98,6 +236,10 @@ class GeminiVoiceClient:
|
||||
|
||||
async def connect(self):
|
||||
uri = f"{GEMINI_WS_URI}?key={GEMINI_API_KEY}"
|
||||
# The system instruction is baked into this handshake alongside the
|
||||
# voice, so per-voice prompts can only be applied here. Voices with no
|
||||
# override (Charon/Puck) keep self.system_prompt untouched.
|
||||
system_prompt = _VOICE_SYSTEM_PROMPTS.get(GEMINI_VOICE) or self.system_prompt
|
||||
try:
|
||||
self._ws = await websockets.connect(uri, **self._ws_kwargs())
|
||||
setup = {
|
||||
@ -111,9 +253,18 @@ class GeminiVoiceClient:
|
||||
}
|
||||
},
|
||||
},
|
||||
"systemInstruction": {"parts": [{"text": self.system_prompt}]},
|
||||
"systemInstruction": {"parts": [{"text": system_prompt}]},
|
||||
}
|
||||
}
|
||||
if _OUTPUT_TRANSCRIPTION:
|
||||
# Ask the server to transcribe the audio it actually speaks.
|
||||
# This is the only reliable way to tell a COMPLETE read from a
|
||||
# cut-off one: the native-audio model regularly speaks the
|
||||
# first few words, stops, and emits reasoning text instead of
|
||||
# the rest — and the audio it returns looks perfectly healthy.
|
||||
# typed_replay compares this transcript against the requested
|
||||
# text and retries when words are missing.
|
||||
setup["setup"]["outputAudioTranscription"] = {}
|
||||
# Guard the app-level setup handshake with a timeout. websockets'
|
||||
# own open_timeout only covers the HTTP upgrade, NOT this send/ACK.
|
||||
# A socket that opens but never ACKs would otherwise block this
|
||||
@ -124,16 +275,23 @@ class GeminiVoiceClient:
|
||||
await asyncio.wait_for(self._ws.recv(), timeout=GEMINI_WS_TIMEOUT) # ACK
|
||||
self._connected = True
|
||||
self._reconnect_attempts = 0
|
||||
log.info("Connected to Gemini (%s)", GEMINI_MODEL)
|
||||
log.info("Connected to Gemini (%s, voice=%s%s)", GEMINI_MODEL, GEMINI_VOICE,
|
||||
", voice-specific prompt" if GEMINI_VOICE in _VOICE_SYSTEM_PROMPTS else "")
|
||||
await bus.emit("voice.connected")
|
||||
except asyncio.TimeoutError:
|
||||
self._connected = False
|
||||
await self._safe_close_ws()
|
||||
log.warning("Gemini setup handshake timed out after %ss", GEMINI_WS_TIMEOUT)
|
||||
raise
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
self._connected = False
|
||||
await self._safe_close_ws()
|
||||
reason = _quota_message(exc)
|
||||
if reason:
|
||||
# Out of credits — say so once, plainly, and stop. Retrying
|
||||
# only stacks up latency until the proxy times out.
|
||||
log.error("Gemini refused the connection: %s", reason)
|
||||
raise GeminiQuotaExhausted(reason) from exc
|
||||
log.exception("Failed to connect to Gemini")
|
||||
raise
|
||||
|
||||
@ -166,6 +324,11 @@ class GeminiVoiceClient:
|
||||
log.warning("Reconnecting to Gemini (attempt %d/%d)", attempt + 1, max_attempts)
|
||||
await self.connect()
|
||||
return True
|
||||
except GeminiQuotaExhausted:
|
||||
# No amount of reconnecting refills the account — surface it
|
||||
# to the caller immediately instead of sleeping through the
|
||||
# whole backoff chain on every attempt.
|
||||
raise
|
||||
except Exception:
|
||||
self._reconnect_attempts += 1
|
||||
await asyncio.sleep(delay)
|
||||
@ -222,7 +385,16 @@ class GeminiVoiceClient:
|
||||
self._owner = owner
|
||||
try:
|
||||
return await self._send_text_inner(text)
|
||||
except (websockets.exceptions.ConnectionClosed, asyncio.TimeoutError):
|
||||
except (websockets.exceptions.ConnectionClosed, asyncio.TimeoutError) as exc:
|
||||
# A 1011 close carrying a billing message is the server telling
|
||||
# us the key is empty — the close reason is the only place that
|
||||
# information appears, so read it before treating this as a
|
||||
# routine drop and reconnecting into the same wall.
|
||||
reason = _quota_message(exc)
|
||||
if reason:
|
||||
self._connected = False
|
||||
log.error("Gemini closed the session: %s", reason)
|
||||
raise GeminiQuotaExhausted(reason) from exc
|
||||
log.warning("send_text: connection died/stalled on send — reconnecting once")
|
||||
self._connected = False
|
||||
if not await self._ensure_connected():
|
||||
@ -231,6 +403,28 @@ class GeminiVoiceClient:
|
||||
finally:
|
||||
self._owner = None
|
||||
|
||||
async def _drain_socket(self) -> int:
|
||||
"""Discard frames left over from an earlier turn. Returns how many.
|
||||
|
||||
If a previous turn ended without consuming everything the server sent,
|
||||
those frames sit in the socket and the NEXT send_text() reads them as
|
||||
its own reply — you get the tail of the last sentence instead of the
|
||||
new one, arriving implausibly fast. Clearing them first makes every
|
||||
turn start from a known-empty stream.
|
||||
"""
|
||||
dropped = 0
|
||||
while dropped < 500:
|
||||
try:
|
||||
await asyncio.wait_for(self._ws.recv(), timeout=0.01)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
break
|
||||
except Exception:
|
||||
break
|
||||
dropped += 1
|
||||
if dropped:
|
||||
log.warning("drained %d stale frame(s) from the previous turn", dropped)
|
||||
return dropped
|
||||
|
||||
async def _send_text_inner(self, text: str) -> tuple[bytes, list[str]]:
|
||||
"""Inner send/receive loop — caller must hold _session_lock."""
|
||||
request = {
|
||||
@ -240,16 +434,29 @@ class GeminiVoiceClient:
|
||||
}
|
||||
}
|
||||
async with self._send_lock:
|
||||
await self._drain_socket()
|
||||
await asyncio.wait_for(
|
||||
self._ws.send(json.dumps(request)), timeout=GEMINI_WS_TIMEOUT)
|
||||
|
||||
audio_chunks: list[bytes] = []
|
||||
text_parts: list[str] = []
|
||||
transcript_parts: list[str] = []
|
||||
self.last_output_transcript = ""
|
||||
|
||||
# `generationComplete` means the model stopped GENERATING — the server
|
||||
# still has audio and transcript to deliver, and only `turnComplete`
|
||||
# ends the turn. Breaking on the former truncated the tail of every
|
||||
# sentence AND left those frames in the socket for the next turn to
|
||||
# mis-read. Wait for turnComplete, with a short grace period after
|
||||
# generationComplete so a turn that never sends it can't stall us.
|
||||
gen_done = False
|
||||
while True:
|
||||
timeout = _POST_GENERATION_GRACE_SEC if gen_done else GEMINI_WS_TIMEOUT
|
||||
try:
|
||||
raw = await asyncio.wait_for(self._ws.recv(), timeout=GEMINI_WS_TIMEOUT)
|
||||
raw = await asyncio.wait_for(self._ws.recv(), timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
if gen_done:
|
||||
break # tail delivered, server just never closed the turn
|
||||
log.warning("send_text: recv timed out")
|
||||
break
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
@ -282,10 +489,20 @@ class GeminiVoiceClient:
|
||||
if input_tr.get("text"):
|
||||
await bus.emit("voice.user_said", text=input_tr["text"])
|
||||
|
||||
if sc.get("turnComplete") or sc.get("generationComplete"):
|
||||
# Transcript of the audio the model is speaking, streamed in
|
||||
# fragments alongside it. Accumulated verbatim; the caller decides
|
||||
# whether it covers the requested text.
|
||||
out_tr = sc.get("outputTranscription", {})
|
||||
if out_tr.get("text"):
|
||||
transcript_parts.append(out_tr["text"])
|
||||
|
||||
if sc.get("turnComplete"):
|
||||
break
|
||||
if sc.get("generationComplete"):
|
||||
gen_done = True
|
||||
|
||||
audio_bytes = b"".join(audio_chunks)
|
||||
self.last_output_transcript = "".join(transcript_parts).strip()
|
||||
if audio_bytes:
|
||||
await bus.emit("voice.gemini_spoke", audio_len=len(audio_bytes))
|
||||
return audio_bytes, text_parts
|
||||
@ -331,6 +548,66 @@ class GeminiVoiceClient:
|
||||
log.exception("raw_send failed")
|
||||
return False
|
||||
|
||||
async def synthesize_tts(self, text: str, voice: str | None = None) -> bytes:
|
||||
"""Text-to-speech via the dedicated Gemini TTS model (stateless REST).
|
||||
|
||||
Returns raw PCM bytes (24 kHz, 16-bit, mono). Reliable audio-only
|
||||
output — unlike the Live native-audio model, which frequently returns
|
||||
its reasoning text instead of speech for short prompts. Voice defaults
|
||||
to the current GEMINI_VOICE (hot-swappable from the dashboard). Runs the
|
||||
blocking HTTP call in a worker thread so the event loop stays free.
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return b""
|
||||
if not GEMINI_API_KEY:
|
||||
raise RuntimeError("No Gemini API key configured.")
|
||||
voice = voice or GEMINI_VOICE
|
||||
url = (f"{_GEMINI_REST_BASE}/models/{GEMINI_TTS_MODEL}"
|
||||
f":generateContent?key={GEMINI_API_KEY}")
|
||||
prompt = f"{_TTS_PREAMBLE}{text}" if _TTS_PREAMBLE else text
|
||||
speech_config = {
|
||||
"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}
|
||||
}
|
||||
if _TTS_LANGUAGE_CODE:
|
||||
speech_config["languageCode"] = _TTS_LANGUAGE_CODE
|
||||
body = {
|
||||
"contents": [{"parts": [{"text": prompt}]}],
|
||||
"generationConfig": {
|
||||
"responseModalities": ["AUDIO"],
|
||||
"speechConfig": speech_config,
|
||||
},
|
||||
}
|
||||
# Deterministic first (temperature 0) so the same text renders the same
|
||||
# audio → stable tone across generations. Only bump the temperature on a
|
||||
# retry if temp-0 returned an empty part for this input (some very short
|
||||
# inputs need it); those retries are the only ones whose tone can vary.
|
||||
temps = [_TTS_TEMPERATURE] + [_TTS_RETRY_TEMPERATURE] * max(0, _TTS_MAX_ATTEMPTS - 1)
|
||||
last_exc = None
|
||||
for attempt, temp in enumerate(temps, 1):
|
||||
body["generationConfig"]["temperature"] = temp
|
||||
try:
|
||||
audio = await asyncio.to_thread(
|
||||
_tts_rest_blocking, url, body, _TTS_TIMEOUT)
|
||||
except Exception as exc:
|
||||
reason = _quota_message(exc)
|
||||
if reason:
|
||||
log.error("Gemini REST TTS refused: %s", reason)
|
||||
raise GeminiQuotaExhausted(reason) from exc
|
||||
last_exc = exc
|
||||
log.warning("TTS attempt %d/%d (temp=%.1f) failed: %s",
|
||||
attempt, len(temps), temp, exc)
|
||||
continue
|
||||
if audio:
|
||||
if attempt > 1:
|
||||
log.info("TTS succeeded on attempt %d (temp=%.1f)", attempt, temp)
|
||||
return audio
|
||||
log.warning("TTS attempt %d/%d (temp=%.1f) returned no audio",
|
||||
attempt, len(temps), temp)
|
||||
if last_exc is not None:
|
||||
raise last_exc
|
||||
return b""
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
return {
|
||||
"connected": self._connected,
|
||||
|
||||
20
main.py
20
main.py
@ -114,7 +114,25 @@ GeminiVoiceClient = _safe_import("GeminiVoiceClient", lambda: __import__("
|
||||
|
||||
brain = _safe_construct("brain", Brain) if Brain else None
|
||||
audio_mgr = _safe_construct("audio_mgr", AudioManager)
|
||||
voice_client = _safe_construct("voice_client", GeminiVoiceClient)
|
||||
def _build_voice_client():
|
||||
# The website must sound EXACTLY like Sanadv3 (G1) and SanadR1 (R1), and
|
||||
# those robots build their replay voice_client with the neutral verbatim
|
||||
# TTS_SYSTEM_PROMPT — the system instruction shapes the delivery on a
|
||||
# native-audio model, so any other prompt here means a different sound.
|
||||
# scripts/sanad_rule.txt therefore SHIPS that exact prompt as its
|
||||
# [REPLAY_SYSTEM_PROMPT]; editing the Voice Rule card in the dashboard is
|
||||
# the deliberate way to opt out of matching the robots.
|
||||
from Project.Sanad.gemini.client import TTS_SYSTEM_PROMPT
|
||||
system_prompt = TTS_SYSTEM_PROMPT
|
||||
try:
|
||||
from Project.Sanad.dashboard.routes.prompt import _load_rule_prompts
|
||||
replay = (_load_rule_prompts() or {}).get("replay_prompt", "").strip()
|
||||
if replay:
|
||||
system_prompt = replay
|
||||
except Exception:
|
||||
log.exception("could not load REPLAY_SYSTEM_PROMPT; using neutral TTS prompt")
|
||||
return GeminiVoiceClient(system_prompt=system_prompt)
|
||||
voice_client = _safe_construct("voice_client", _build_voice_client if GeminiVoiceClient else None)
|
||||
local_tts = _safe_construct("local_tts", LocalTTSEngine)
|
||||
typed_replay = _safe_construct("typed_replay", (lambda: TypedReplayEngine(voice_client, audio_mgr)) if (TypedReplayEngine and voice_client) else None)
|
||||
|
||||
|
||||
@ -7,13 +7,4 @@ Do not over-explain.
|
||||
Prefer concise speech that sounds natural when spoken aloud funny mode and happy sound.
|
||||
|
||||
[REPLAY_SYSTEM_PROMPT]
|
||||
You are Sanad (Bousandah), using the same Emirati voice and personality.
|
||||
For replay mode, the user will provide text that you must speak exactly as written.
|
||||
You may sound warm and lively, but you must preserve the exact text.
|
||||
Do not translate it.
|
||||
Do not summarize it.
|
||||
Do not answer it.
|
||||
Do not rephrase it into another dialect or style.
|
||||
Do not add greetings, punctuation changes, comments, or extra words.
|
||||
Keep the same word order and language as the provided text.
|
||||
Your only task is to speak the exact user text verbatim.
|
||||
You are a pure multilingual text-to-speech voice. The instant the user sends text, speak it aloud word for word in the SAME language it is written in, then stop. Output ONLY that spoken audio — no thinking, no commentary, no acknowledgements, no headers, no explanations, no greetings, no extra words. Never translate and never change the language: English stays English, Arabic stays Arabic, Urdu stays Urdu, Indonesian stays Indonesian. Your speech must be identical to the user's text, nothing more and nothing less.
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
أنت "بوسنده" — روبوت إماراتي ذكي تابع لروبوت شركة لوتاه تيك Lootah Tech.
|
||||
أنت "بوسنده" — روبوت إماراتي ذكي تابع لشركة YS Lootah Robotics في دبي.
|
||||
|
||||
[أولاً: الروح والمعرفة]
|
||||
|
||||
|
||||
93
voice/pitch.py
Normal file
93
voice/pitch.py
Normal file
@ -0,0 +1,93 @@
|
||||
"""Median fundamental frequency of PCM audio, in pure Python.
|
||||
|
||||
Used by the typed-replay pitch gate to reject a take whose tone does not match
|
||||
the voice it is supposed to be. The shared host has no numpy, so this is
|
||||
written against the stdlib only and deliberately samples a handful of frames
|
||||
rather than the whole clip — the median only needs to be stable, not exact,
|
||||
and a full autocorrelation over a 17 s paragraph would cost seconds of CPU.
|
||||
|
||||
Accuracy target: within a few Hz of a numpy implementation, which is far
|
||||
tighter than the ~25 % drift the gate exists to catch.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import array
|
||||
|
||||
# Human speech range. 60-350 Hz covers every prebuilt voice in use
|
||||
# (male ~100-140, female ~160-215) with margin.
|
||||
_MIN_HZ = 60.0
|
||||
_MAX_HZ = 350.0
|
||||
# Decimate to this rate before correlating: 8 kHz keeps pitch information
|
||||
# (harmonics well above it are irrelevant) and cuts the work by 3x.
|
||||
_TARGET_RATE = 8000
|
||||
_FRAME_SEC = 0.040 # matches the reference implementation
|
||||
_MAX_FRAMES = 48 # sampled evenly across the clip
|
||||
_SILENCE_RMS = 500.0 # int16 amplitude; below this a frame is not speech
|
||||
_PEAK_RATIO = 0.3 # autocorrelation peak must be this fraction of lag 0
|
||||
|
||||
|
||||
def _rms(frame) -> float:
|
||||
total = 0
|
||||
for s in frame:
|
||||
total += s * s
|
||||
return (total / len(frame)) ** 0.5 if frame else 0.0
|
||||
|
||||
|
||||
def median_f0(pcm: bytes, sample_rate: int = 24000) -> float:
|
||||
"""Median F0 in Hz over voiced frames, or 0.0 if nothing voiced was found."""
|
||||
if not pcm or len(pcm) < 4:
|
||||
return 0.0
|
||||
samples = array.array("h")
|
||||
samples.frombytes(pcm[: len(pcm) // 2 * 2])
|
||||
|
||||
step = max(1, int(round(sample_rate / float(_TARGET_RATE))))
|
||||
rate = sample_rate // step
|
||||
if step > 1:
|
||||
samples = samples[::step]
|
||||
|
||||
frame_len = int(rate * _FRAME_SEC)
|
||||
if frame_len < 32 or len(samples) < frame_len * 2:
|
||||
return 0.0
|
||||
|
||||
lo = max(2, int(rate / _MAX_HZ))
|
||||
hi = min(frame_len - 1, int(rate / _MIN_HZ))
|
||||
if hi <= lo:
|
||||
return 0.0
|
||||
|
||||
starts = []
|
||||
span = len(samples) - frame_len
|
||||
count = min(_MAX_FRAMES, max(1, span // frame_len))
|
||||
for i in range(count):
|
||||
starts.append(int(span * i / float(count)) if count > 1 else 0)
|
||||
|
||||
found = []
|
||||
for start in starts:
|
||||
frame = samples[start:start + frame_len]
|
||||
if _rms(frame) < _SILENCE_RMS:
|
||||
continue
|
||||
mean = sum(frame) / float(frame_len)
|
||||
centred = [s - mean for s in frame]
|
||||
|
||||
energy = 0.0
|
||||
for v in centred:
|
||||
energy += v * v
|
||||
if energy <= 0:
|
||||
continue
|
||||
|
||||
best_lag, best_val = 0, 0.0
|
||||
for lag in range(lo, hi + 1):
|
||||
total = 0.0
|
||||
for i in range(frame_len - lag):
|
||||
total += centred[i] * centred[i + lag]
|
||||
if total > best_val:
|
||||
best_val, best_lag = total, lag
|
||||
if best_lag and best_val > _PEAK_RATIO * energy:
|
||||
found.append(rate / float(best_lag))
|
||||
|
||||
if not found:
|
||||
return 0.0
|
||||
found.sort()
|
||||
mid = len(found) // 2
|
||||
if len(found) % 2:
|
||||
return found[mid]
|
||||
return (found[mid - 1] + found[mid]) / 2.0
|
||||
@ -21,6 +21,7 @@ import tempfile
|
||||
import threading
|
||||
import time
|
||||
import wave
|
||||
from collections import Counter
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@ -36,6 +37,7 @@ from Project.Sanad.config import (
|
||||
MONITOR_SOURCE as DEFAULT_MONITOR_SOURCE,
|
||||
)
|
||||
from Project.Sanad.core.logger import get_logger
|
||||
from Project.Sanad.gemini.client import GeminiQuotaExhausted
|
||||
|
||||
try:
|
||||
import pyaudio
|
||||
@ -57,6 +59,207 @@ RECORD_INDEX_PATH = AUDIO_RECORDINGS_DIR / "records.json"
|
||||
MONITOR_CHUNK_SIZE = _TR.get("monitor_chunk_size", CHUNK_SIZE)
|
||||
MONITOR_TAIL_SEC = _TR.get("monitor_tail_sec", 0.2)
|
||||
MAX_TEXT_LEN = _TR.get("max_text_len", 2000)
|
||||
# How many times to ask the Live model before falling back to the REST TTS. The
|
||||
# Live native-audio model "thinks" instead of speaking on ~40-50% of attempts
|
||||
# (measured; temperature doesn't fix it), so several tries give ~99% chance of
|
||||
# the real robot-voice audio. Config-overridable via voice.typed_replay.
|
||||
_LIVE_ATTEMPTS = int(_TR.get("live_attempts", 6))
|
||||
# Fraction of the requested words that must appear in the model's own
|
||||
# transcript of what it spoke for a read to count as COMPLETE. The native-audio
|
||||
# model often speaks the opening words, stops, and "thinks" the rest — the
|
||||
# audio comes back looking healthy, just cut off mid-sentence. 1.0 = every word.
|
||||
_MIN_COVERAGE = float(_TR.get("min_spoken_coverage", 1.0))
|
||||
# Fastest believable speaking rate, in characters per second — a backstop for
|
||||
# the transcript check, which reports the FULL text as spoken even when the
|
||||
# audio is far too short to contain it (measured: identical transcripts for
|
||||
# takes of 0.80s–2.32s on the same 18-char phrase).
|
||||
#
|
||||
# Calibrated against the ROBOT, not against a guess. Measured reads:
|
||||
# GOOD 9.5 c/s (G1 :8014, 19 chars) GOOD 13.2 c/s (G1 :8014, 38 chars)
|
||||
# GOOD 7.5-10.7 c/s (website, both phrases)
|
||||
# CUT 14.8 c/s (Kore, 50% coverage) CUT 15.8-22.5 c/s (18-char phrase)
|
||||
# The good/cut boundary sits between 13.2 and 14.8, so the ceiling goes just
|
||||
# above the robot's fastest genuine read. Setting it at 11 (the first guess)
|
||||
# rejected takes AT THE ROBOT'S OWN PACE and made the site retry until it got
|
||||
# an unusually slow one — the website ended up ~24% slower than the G1, which
|
||||
# is audible and was reported as "not similar".
|
||||
_MAX_CHARS_PER_SEC_AR = float(_TR.get("max_chars_per_sec_ar", 14.0))
|
||||
_MAX_CHARS_PER_SEC_LATIN = float(_TR.get("max_chars_per_sec_latin", 20.0))
|
||||
# How many times to re-ask for the words a voice skipped. Some voices stop at a
|
||||
# natural boundary DETERMINISTICALLY (Kore ends after "سلام عليكم" on every
|
||||
# attempt), so retrying the whole line never completes it — only asking for the
|
||||
# remainder does. Each round costs one more Live call.
|
||||
_MAX_CONTINUATIONS = int(_TR.get("max_continuations", 3))
|
||||
# Pitch gate. The Live model re-performs each turn, so the SAME voice comes back
|
||||
# at a different pitch depending on input length — measured ~25% spread across
|
||||
# word/sentence/paragraph even with an explicit tone-lock prompt, and a prompt
|
||||
# cannot fix it (tested). Rejecting an off-tone take and asking again is what
|
||||
# actually keeps a set of recordings sounding like one robot.
|
||||
# Bands are per prebuilt voice, measured against the robots: G1/R1 read
|
||||
# ~109-120 Hz, Kore (female, no robot) ~165-205 Hz.
|
||||
_PITCH_GATE = bool(_TR.get("pitch_gate", True))
|
||||
# Inputs at or below this length get a doubled Live budget (see _live_read).
|
||||
_SHORT_TEXT_CHARS = int(_TR.get("short_text_chars", 12))
|
||||
_PITCH_BANDS = _TR.get("pitch_bands") or {
|
||||
"Charon": [100.0, 140.0], # Unitree G1
|
||||
"Puck": [100.0, 140.0], # Unitree R1
|
||||
"Kore": [165.0, 225.0], # Agibot x2 (Muza) — genuine reads reach ~216
|
||||
}
|
||||
|
||||
|
||||
def _pitch_band_for(voice: str):
|
||||
"""(low, high) Hz for `voice`, or None when it has no configured band."""
|
||||
if not _PITCH_GATE or not voice:
|
||||
return None
|
||||
band = _PITCH_BANDS.get(voice)
|
||||
if not band or len(band) != 2:
|
||||
return None
|
||||
return float(band[0]), float(band[1])
|
||||
|
||||
|
||||
def _pitch_check(audio: bytes, voice: str):
|
||||
"""(ok, f0, band). ok is True when there is no band or nothing measurable —
|
||||
the gate must never block audio just because pitch could not be estimated."""
|
||||
band = _pitch_band_for(voice)
|
||||
if not band:
|
||||
return True, 0.0, None
|
||||
try:
|
||||
from Project.Sanad.voice.pitch import median_f0
|
||||
f0 = median_f0(audio, RECEIVE_SAMPLE_RATE)
|
||||
except Exception:
|
||||
log.exception("pitch estimation failed — letting the take through")
|
||||
return True, 0.0, band
|
||||
if not f0:
|
||||
return True, 0.0, band
|
||||
return band[0] <= f0 <= band[1], f0, band
|
||||
# Drop and reopen the Live session before each replay — see the note in
|
||||
# generate_audio. Set false to reuse one long-lived socket (faster, but the
|
||||
# delivery drifts away from the robots' as turns accumulate).
|
||||
_FRESH_SESSION_PER_REPLAY = bool(_TR.get("fresh_session_per_replay", True))
|
||||
# Voices that must instead REUSE a long-lived session, because the robot they
|
||||
# imitate runs one. Session state changes delivery, and the two robots are in
|
||||
# different states: the G1's P4 sits disconnected (every take opens a fresh
|
||||
# session, ~13 chars/s) while the R1's sanadr1 holds one socket for days
|
||||
# (~10.9 chars/s). Measured on the same sentence with Puck: fresh 13.4 chars/s
|
||||
# vs warm 11.2-12.3 — so Puck matches the R1 only when kept warm, while Charon
|
||||
# needs the opposite (a warm session dropped it to 93 Hz, far from the G1).
|
||||
# Default EMPTY: keeping Puck warm was tried against the R1 and did not
|
||||
# reproduce (3.40s one run, 2.92s the next on identical config) and it made the
|
||||
# pitch worse (F0 100-106 vs 113-123 fresh, robot 118-120). The mechanism stays
|
||||
# available for anyone who wants to retry it with a bigger sample.
|
||||
_WARM_SESSION_VOICES = {v.strip() for v in _TR.get("warm_session_voices", []) if v}
|
||||
|
||||
|
||||
def _current_voice() -> tuple[str, str]:
|
||||
"""(voice_name, friendly_label) of the voice in force right now.
|
||||
|
||||
Read live from the client module because the dashboard hot-swaps that
|
||||
global; the label comes from the same table the voice picker uses, so a
|
||||
record shows "Unitree G1" rather than "Charon". Imports are local to avoid
|
||||
a package-level cycle (dashboard imports this module).
|
||||
"""
|
||||
voice = ""
|
||||
try:
|
||||
from Project.Sanad.gemini import client as _gc
|
||||
voice = getattr(_gc, "GEMINI_VOICE", "") or ""
|
||||
except Exception:
|
||||
pass
|
||||
label = ""
|
||||
try:
|
||||
from Project.Sanad.dashboard.routes.voice import _VOICE_BY_NAME
|
||||
label = (_VOICE_BY_NAME.get(voice) or {}).get("label", "")
|
||||
except Exception:
|
||||
pass
|
||||
return voice, label
|
||||
|
||||
|
||||
def _min_believable_duration(text: str) -> float:
|
||||
"""Shortest duration in which `text` could plausibly have been spoken."""
|
||||
stripped = (text or "").strip()
|
||||
if not stripped:
|
||||
return 0.0
|
||||
arabic = sum(1 for ch in stripped if "" <= ch <= "ۿ")
|
||||
rate = (_MAX_CHARS_PER_SEC_AR if arabic * 2 >= len(stripped)
|
||||
else _MAX_CHARS_PER_SEC_LATIN)
|
||||
return len(stripped) / rate
|
||||
|
||||
# Arabic normalisation for that comparison: strip diacritics/tatweel and fold
|
||||
# the alef/yaa/taa-marbuta variants, because the transcript spells them
|
||||
# differently from the typed text more often than not.
|
||||
# Spelled with explicit codepoints on purpose: written with literal Arabic
|
||||
# marks this class becomes ranges like \u0610-\u064B, which swallow every
|
||||
# Arabic LETTER — normalising all text to "" so every read scores as
|
||||
# complete and the whole cut-off check silently becomes a no-op.
|
||||
_AR_MARKS = re.compile("[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06ED\u0640]")
|
||||
_AR_FOLD = str.maketrans({"أ": "ا", "إ": "ا", "آ": "ا", "ٱ": "ا",
|
||||
"ة": "ه", "ى": "ي", "ؤ": "و", "ئ": "ي"})
|
||||
|
||||
|
||||
def _uncovered_tail(spoken: str, wanted: str) -> str:
|
||||
"""The part of `wanted` the voice never got to, in its ORIGINAL spelling.
|
||||
|
||||
Walks the requested words in order and consumes the transcript in order,
|
||||
stopping at the first word the transcript doesn't reach. Everything from
|
||||
there on is returned verbatim (original text, not the normalised form) so
|
||||
it can be sent straight back to the model as the continuation.
|
||||
"""
|
||||
orig = (wanted or "").split()
|
||||
want = _norm_words(wanted)
|
||||
pool = _norm_words(spoken)
|
||||
# Normalisation can merge/split tokens; only trust the word-for-word walk
|
||||
# when it lines up with the original word count.
|
||||
if len(orig) != len(want):
|
||||
return ""
|
||||
pos = 0
|
||||
for idx, w in enumerate(want):
|
||||
found = False
|
||||
while pos < len(pool):
|
||||
tok = pool[pos]
|
||||
pos += 1
|
||||
if tok == w or (len(w) >= 2 and len(tok) >= 2
|
||||
and (w in tok or tok in w)):
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
return " ".join(orig[idx:])
|
||||
return ""
|
||||
|
||||
|
||||
def _norm_words(text: str) -> list[str]:
|
||||
"""Normalised word list used for spoken-coverage comparison."""
|
||||
s = _AR_MARKS.sub("", text or "").translate(_AR_FOLD).lower()
|
||||
s = re.sub(r"[^\w\s]", " ", s, flags=re.UNICODE)
|
||||
return s.split()
|
||||
|
||||
|
||||
def _spoken_coverage(spoken: str, wanted: str) -> float:
|
||||
"""Fraction of `wanted`'s words that the transcript `spoken` contains.
|
||||
|
||||
Order-insensitive, and each spoken token may satisfy only one requested
|
||||
word — the point is to catch MISSING words (a cut-off read), not to police
|
||||
word order. Matching is deliberately loose: the transcript routinely
|
||||
spells a word differently from the typed text (it writes "السلام" where
|
||||
the user typed "سلام", drops a suffix, joins a clitic), and rejecting a
|
||||
COMPLETE read over spelling would retry forever. A word therefore counts
|
||||
as spoken when either token contains the other. Words genuinely cut off
|
||||
the end don't appear in any form, which is the case that matters.
|
||||
"""
|
||||
want = _norm_words(wanted)
|
||||
if not want:
|
||||
return 1.0
|
||||
pool = _norm_words(spoken)
|
||||
used = [False] * len(pool)
|
||||
hits = 0
|
||||
for w in want:
|
||||
for i, tok in enumerate(pool):
|
||||
if used[i]:
|
||||
continue
|
||||
if tok == w or (len(w) >= 2 and len(tok) >= 2
|
||||
and (w in tok or tok in w)):
|
||||
used[i] = True
|
||||
hits += 1
|
||||
break
|
||||
return hits / len(want)
|
||||
|
||||
|
||||
# ─── helpers ─────────────────────────────────────────────────────────
|
||||
@ -401,13 +604,13 @@ class TypedReplayEngine:
|
||||
# ── generation ───────────────────────────────────────────────
|
||||
|
||||
async def generate_audio(self, text: str) -> tuple[bytes, list[str]]:
|
||||
"""Route typed text through Gemini Live as the voice, first-try reliable.
|
||||
"""Speak `text` in full through Gemini Live, joining on any missing tail.
|
||||
|
||||
The session's system-prompt sets a persona ("You are Sanad…"),
|
||||
so the prompt that most reliably gets audio out is a direct
|
||||
address to the persona with the quoted text. A transparent
|
||||
retry chain covers the edge cases where the model still
|
||||
replies with text only.
|
||||
Some voices reliably stop at a natural boundary and never read the
|
||||
rest, no matter how many times you ask — Kore (Agibot) stops dead
|
||||
after "سلام عليكم" on every single attempt, while Charon and Puck read
|
||||
the same line fine. Retrying cannot fix a deterministic stop, so when a
|
||||
read is short we speak the WORDS THAT WERE MISSED and append that audio.
|
||||
"""
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
@ -415,32 +618,212 @@ class TypedReplayEngine:
|
||||
|
||||
if self.voice_client is None:
|
||||
raise RuntimeError("voice_client unavailable")
|
||||
if not self.voice_client.connected:
|
||||
await self.voice_client.connect()
|
||||
|
||||
# Ordered by empirical reliability — first variant wins ~95% of turns.
|
||||
# The quoted-phrase form is the most consistent trigger for an
|
||||
# audio-only response with the current Sanad persona prompt.
|
||||
attempts = [
|
||||
f'قل هذا بالضبط وبدون إضافات: "{stripped}"', # Arabic: "Say this exactly, no additions"
|
||||
f'Say this exactly, nothing else: "{stripped}"',
|
||||
# Start every replay from a CLEAN Live session. Delivery drifts as turns
|
||||
# accumulate in one socket: measured on the same sentence, a long-lived
|
||||
# session read it at 93 Hz / 14.2 chars/s, while a fresh session read it
|
||||
# at 99-104 Hz / 12.5-12.7 chars/s — matching the robot's 109-111 Hz /
|
||||
# 12.7-13.2 chars/s. The robots reconnect constantly (their client is
|
||||
# only opened on demand), so a session-per-replay is what actually
|
||||
# reproduces their sound. Costs one ~0.3s handshake per generation.
|
||||
# Read the CURRENT voice from the client module (the dashboard hot-swaps
|
||||
# that global), not an import-time copy.
|
||||
try:
|
||||
from Project.Sanad.gemini import client as _gc
|
||||
current_voice = getattr(_gc, "GEMINI_VOICE", "")
|
||||
except Exception:
|
||||
current_voice = ""
|
||||
if (_FRESH_SESSION_PER_REPLAY and self.voice_client.connected
|
||||
and current_voice not in _WARM_SESSION_VOICES):
|
||||
await self.voice_client.disconnect()
|
||||
self.last_voice_warning = ""
|
||||
self.last_f0 = 0.0
|
||||
audio, parts, spoken, coverage = await self._live_read(stripped)
|
||||
# Continue where the voice gave up, one missing chunk at a time.
|
||||
for round_no in range(1, _MAX_CONTINUATIONS + 1):
|
||||
if not audio or coverage >= _MIN_COVERAGE:
|
||||
break
|
||||
remaining = _uncovered_tail(spoken, stripped)
|
||||
if not remaining:
|
||||
break
|
||||
log.warning("continuation %d — voice stopped after %r; speaking the "
|
||||
"missing %r", round_no, spoken[:40], remaining[:40])
|
||||
tail_audio, tail_parts, tail_spoken, _ = await self._live_read(remaining)
|
||||
if not tail_audio:
|
||||
break
|
||||
audio += tail_audio
|
||||
parts += tail_parts
|
||||
spoken = f"{spoken} {tail_spoken}".strip()
|
||||
coverage = _spoken_coverage(spoken, stripped)
|
||||
self.last_spoken_transcript = spoken
|
||||
if audio:
|
||||
if coverage < _MIN_COVERAGE:
|
||||
log.warning("Serving %.0f%% of the text — the voice would not "
|
||||
"read the rest", coverage * 100)
|
||||
return audio, parts
|
||||
return await self._rest_fallback(stripped, parts)
|
||||
|
||||
async def _live_read(self, text: str) -> tuple[bytes, list[str], str, float]:
|
||||
"""One Live read of `text`. Returns (audio, parts, transcript, coverage).
|
||||
|
||||
Retries the model's coin-flip failures (no audio at all) and its
|
||||
cut-off reads; returns the fullest take it managed if none is complete.
|
||||
"""
|
||||
stripped = text.strip()
|
||||
|
||||
# The Live native-audio model is the SAME engine + voice the real robots
|
||||
# (Sanadv3 G1 = Charon, SanadR1 R1 = Puck) speak with, so this path must
|
||||
# behave EXACTLY like theirs to sound like them: take the FIRST attempt
|
||||
# that returns audio, whatever text parts came with it. Any extra
|
||||
# "quality gate" here (reject reads that also returned thinking text,
|
||||
# reject reads that look too long) only ends up routing the request to
|
||||
# the REST TTS model — a DIFFERENT engine with a different voice. That
|
||||
# is precisely what made the website not match the robots: every single
|
||||
# generation was being rejected and served from REST TTS instead.
|
||||
if not self.voice_client.connected:
|
||||
# A credits/quota failure here is final — let it propagate as-is so
|
||||
# the dashboard shows "top up billing" instead of a generic 503
|
||||
# eleven attempts later.
|
||||
await self.voice_client.connect()
|
||||
# Same three framings the robots use, cycled. The Live model is a
|
||||
# coin-flip to speak vs "think" on any single try, so the extra rounds
|
||||
# raise the odds of getting audio WITHOUT changing engine or voice.
|
||||
_base = [
|
||||
stripped,
|
||||
f"Read this aloud word for word, in its original language, "
|
||||
f"nothing else:\n{stripped}",
|
||||
f'"{stripped}"',
|
||||
]
|
||||
# Very short inputs are where the Live model refuses outright — it
|
||||
# "thinks" instead of speaking and returns no audio, which drops the
|
||||
# request onto the REST fallback (a different engine, audibly not the
|
||||
# robot). Give short text more chances on the real engine rather than
|
||||
# letting it fall through: measured, a single word needed >6 tries.
|
||||
budget = _LIVE_ATTEMPTS * 2 if len(stripped) <= _SHORT_TEXT_CHARS else _LIVE_ATTEMPTS
|
||||
attempts = [_base[i % len(_base)] for i in range(budget)]
|
||||
last_parts: list[str] = []
|
||||
# Best incomplete read so far, as (coverage, audio, parts) — used only
|
||||
# if no attempt manages a full read.
|
||||
# (coverage, audio, parts, transcript, duration, on_tone)
|
||||
best = (0.0, b"", [], "", 0.0, False)
|
||||
# The voice can be swapped between generations, so read it per read.
|
||||
current_voice = _current_voice()[0]
|
||||
for idx, wrapped in enumerate(attempts, start=1):
|
||||
try:
|
||||
audio_bytes, text_parts = await self.voice_client.send_text(
|
||||
wrapped, owner="typed_replay")
|
||||
except GeminiQuotaExhausted:
|
||||
# Empty wallet — every remaining Live attempt AND the REST
|
||||
# fallback would fail identically. Stop now so the user gets a
|
||||
# clear answer in ~1s instead of a proxy timeout.
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.warning("Gemini TTS attempt %d failed: %s", idx, exc)
|
||||
log.warning("Gemini Live TTS attempt %d failed: %s", idx, exc)
|
||||
continue
|
||||
if audio_bytes:
|
||||
if idx > 1:
|
||||
log.info("Gemini TTS succeeded on attempt %d", idx)
|
||||
return audio_bytes, text_parts
|
||||
last_parts = text_parts
|
||||
log.warning("Gemini TTS attempt %d returned no audio — parts: %s",
|
||||
idx, " | ".join(text_parts or [])[:120])
|
||||
if not audio_bytes:
|
||||
last_parts = text_parts
|
||||
log.warning("Gemini Live TTS attempt %d returned no audio — parts: %s",
|
||||
idx, " | ".join(text_parts or [])[:120])
|
||||
continue
|
||||
|
||||
dur = len(audio_bytes) / (RECEIVE_SAMPLE_RATE * 2.0)
|
||||
spoken = getattr(self.voice_client, "last_output_transcript", "") or ""
|
||||
self.last_spoken_transcript = spoken
|
||||
if not spoken:
|
||||
# No transcript (feature off, or the server sent none): the
|
||||
# coverage check is impossible, but duration and pitch still
|
||||
# apply and need no transcript. Skipping them here let a 0.16s
|
||||
# fragment through on a 5-character word.
|
||||
dur_ok = dur >= _min_believable_duration(stripped)
|
||||
tone_ok, f0_nt, band_nt = _pitch_check(audio_bytes, current_voice)
|
||||
if dur_ok and tone_ok:
|
||||
if idx > 1:
|
||||
log.info("Gemini Live TTS succeeded on attempt %d (%.1fs, %.0f Hz, "
|
||||
"no transcript to verify)", idx, dur, f0_nt)
|
||||
return audio_bytes, text_parts, "", 1.0
|
||||
log.warning("Gemini Live TTS attempt %d rejected without transcript "
|
||||
"(%.2fs, %.0f Hz) — retrying", idx, dur, f0_nt)
|
||||
if (0.0, tone_ok, dur) > (best[0], best[5], best[4]):
|
||||
best = (0.0, audio_bytes, text_parts, "", dur, tone_ok)
|
||||
continue
|
||||
|
||||
coverage = _spoken_coverage(spoken, stripped)
|
||||
long_enough = dur >= _min_believable_duration(stripped)
|
||||
on_tone, f0, band = _pitch_check(audio_bytes, current_voice)
|
||||
if coverage >= _MIN_COVERAGE and long_enough and on_tone:
|
||||
log.info("Gemini Live TTS complete read on attempt %d (%.1fs, %.1f chars/s, "
|
||||
"%.0f Hz)", idx, dur, len(stripped) / dur if dur else 0, f0)
|
||||
self.last_f0 = f0
|
||||
return audio_bytes, text_parts, spoken, coverage
|
||||
# Rank candidates: fully spoken first, then on-tone, then longest.
|
||||
if (coverage, on_tone, dur) > (best[0], best[5], best[4]):
|
||||
best = (coverage, audio_bytes, text_parts, spoken, dur, on_tone)
|
||||
if coverage >= _MIN_COVERAGE and long_enough and not on_tone:
|
||||
log.warning("Gemini Live TTS attempt %d OFF-TONE for %s — %.0f Hz outside "
|
||||
"%.0f-%.0f Hz — retrying", idx, current_voice, f0,
|
||||
band[0], band[1])
|
||||
continue
|
||||
if not long_enough:
|
||||
log.warning("Gemini Live TTS attempt %d TOO FAST to be complete — "
|
||||
"%.2fs for %d chars (%.1f chars/s, need >= %.2fs) — retrying",
|
||||
idx, dur, len(stripped), len(stripped) / dur if dur else 0,
|
||||
_min_believable_duration(stripped))
|
||||
else:
|
||||
log.warning("Gemini Live TTS attempt %d CUT — spoke %.0f%% of the text "
|
||||
"(%.1fs) %r — retrying", idx, coverage * 100, dur, spoken[:60])
|
||||
|
||||
# No complete read in _LIVE_ATTEMPTS tries. Return the fullest one we
|
||||
# got — still the robot's voice, which matters more than the last word,
|
||||
# and better than the REST engine's different voice.
|
||||
if best[1]:
|
||||
log.warning("No complete Live read of %r in %d attempts — fullest was "
|
||||
"%.0f%%", stripped[:40], _LIVE_ATTEMPTS, best[0] * 100)
|
||||
if not best[5]:
|
||||
self.last_voice_warning = (
|
||||
"Tone is off for this voice — the model would not produce an "
|
||||
"in-range take. Generate again for a better match.")
|
||||
elif best[0] < _MIN_COVERAGE:
|
||||
self.last_voice_warning = (
|
||||
"Only part of the text was spoken (%.0f%%)." % (best[0] * 100))
|
||||
return best[1], best[2], best[3], best[0]
|
||||
return b"", last_parts, "", 0.0
|
||||
|
||||
async def _rest_fallback(self, stripped: str,
|
||||
last_parts: list[str]) -> tuple[bytes, list[str]]:
|
||||
"""Last resort when Live produced no audio at all (see note above)."""
|
||||
|
||||
# Every Live attempt came back with ZERO audio — the robots would fail
|
||||
# outright here. Rather than erroring the page, fall back to the REST
|
||||
# TTS model, but log it as a WARNING: this is the one path whose output
|
||||
# does NOT sound like the robots, so it should be rare and visible.
|
||||
try:
|
||||
audio = await self.voice_client.synthesize_tts(stripped)
|
||||
if audio:
|
||||
# The REST model is a different engine: same voice NAMES, but a
|
||||
# measurably different timbre (its "Charon" sits ~50 Hz above the
|
||||
# Live one). Check it against the same band so an off-voice take
|
||||
# is at least reported rather than passed off as the robot.
|
||||
voice = _current_voice()[0]
|
||||
on_tone, f0, band = _pitch_check(audio, voice)
|
||||
self.last_f0 = f0
|
||||
if on_tone:
|
||||
log.warning("Live returned no audio — served REST TTS (%.0f Hz)", f0)
|
||||
self.last_voice_warning = (
|
||||
"Spoken by the backup engine, not the robot's live voice.")
|
||||
else:
|
||||
log.error("Live returned no audio and the REST TTS take is OFF-VOICE "
|
||||
"for %s (%.0f Hz, band %.0f-%.0f) — serving it anyway, but "
|
||||
"it will not sound like the robot", voice, f0,
|
||||
band[0], band[1])
|
||||
self.last_voice_warning = (
|
||||
"NOT the robot voice — the backup engine produced %.0f Hz, "
|
||||
"outside %s's %.0f-%.0f Hz range. Generate again."
|
||||
% (f0, voice or "this voice", band[0], band[1]))
|
||||
return audio, []
|
||||
except GeminiQuotaExhausted:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.warning("REST TTS fallback failed: %s", exc)
|
||||
return b"", last_parts
|
||||
|
||||
# ── playback + capture ───────────────────────────────────────
|
||||
@ -598,6 +981,9 @@ class TypedReplayEngine:
|
||||
"raw_duration_sec": round(
|
||||
audio_duration_seconds(audio_bytes, RECEIVE_SAMPLE_RATE,
|
||||
CHANNELS, self.sample_width()), 3),
|
||||
"spoken_transcript": getattr(self, "last_spoken_transcript", ""),
|
||||
"voice_warning": getattr(self, "last_voice_warning", ""),
|
||||
"pitch_hz": round(getattr(self, "last_f0", 0.0), 1),
|
||||
"captured_speaker_bytes": 0,
|
||||
"audio_url": "/api/typed-replay/audio/last",
|
||||
"recorded": False,
|
||||
@ -610,6 +996,18 @@ class TypedReplayEngine:
|
||||
result["recorded"] = True
|
||||
|
||||
return result
|
||||
except Exception:
|
||||
# Drop the cached generation on ANY failure. /audio/last serves
|
||||
# whatever is cached, and the browser fetches it right after a
|
||||
# say(), so a failed generation would otherwise replay the PREVIOUS
|
||||
# take — audio of the wrong words, presented as if it were the new
|
||||
# text. A 404 there is honest; the old take is still on disk if it
|
||||
# was saved as a record.
|
||||
self.session.audio_bytes = b""
|
||||
self.session.speaker_capture = b""
|
||||
self.session.generated_at = ""
|
||||
self.session.saved_as = ""
|
||||
raise
|
||||
finally:
|
||||
self._gen_lock.release()
|
||||
|
||||
@ -660,10 +1058,20 @@ class TypedReplayEngine:
|
||||
|
||||
self.save_audio(audio, raw_path, CHANNELS, RECEIVE_SAMPLE_RATE)
|
||||
|
||||
voice, voice_label = _current_voice()
|
||||
entry = {
|
||||
"record_name": base.name,
|
||||
"text": self.session.text,
|
||||
"replay_count": self.session.replay_count,
|
||||
# Which robot voice spoke this. Recorded at save time because the
|
||||
# dashboard can switch voices between generations, so it cannot be
|
||||
# reconstructed later from anything else in the file.
|
||||
"voice": voice,
|
||||
"voice_label": voice_label,
|
||||
# Measured pitch of this take, and any reason it may not sound like
|
||||
# the robot — so a bad recording is identifiable in the list later.
|
||||
"pitch_hz": round(getattr(self, "last_f0", 0.0), 1),
|
||||
"voice_warning": getattr(self, "last_voice_warning", ""),
|
||||
"timeline": {
|
||||
"audio_generated_at": self.session.generated_at,
|
||||
"last_playback_finished_at": self.session.last_playback_at,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user