From cfdc0556a4124dc6afb28ca2ffc11983fbdf402c Mon Sep 17 00:00:00 2001 From: kassam Date: Mon, 20 Jul 2026 10:30:48 +0400 Subject: [PATCH] Update 2026-07-20 10:30:47 --- config/core_config.json | 4 +- config/voice_config.json | 15 +- dashboard/routes/live_subprocess.py | 234 +++++++++++++++++++++ dashboard/routes/motion.py | 40 +++- dashboard/routes/voice.py | 6 +- dashboard/static/index.html | 140 ++++++++++++- gemini/script.py | 309 ++++++++++++++++++++++++++-- gemini/subprocess.py | 16 +- motion/arm_controller.py | 138 +++++++------ motion/r1_arm_action_client.py | 67 ++++++ scripts/sanad_script.txt | 2 +- tests/test_smoke.py | 4 +- voice/audio_devices.py | 13 ++ voice/audio_io.py | 111 +++++++++- voice/sanad_voice.py | 2 +- 15 files changed, 993 insertions(+), 108 deletions(-) create mode 100644 motion/r1_arm_action_client.py diff --git a/config/core_config.json b/config/core_config.json index 9c56a21..b46d36a 100644 --- a/config/core_config.json +++ b/config/core_config.json @@ -39,9 +39,9 @@ "api_key": "", "model_live": "gemini-2.5-flash-native-audio-preview-12-2025", "model_ws_uri": "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent", - "voice_name": "Charon", + "voice_name": "Puck", "ws_timeout_sec": 30, - "default_system_prompt": "You are Bousandah, a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'." + "default_system_prompt": "You are super-dubai (سوبر دبي), a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'." }, "g1_hardware": { diff --git a/config/voice_config.json b/config/voice_config.json index 00e151d..c453e34 100644 --- a/config/voice_config.json +++ b/config/voice_config.json @@ -10,7 +10,9 @@ "session_timeout_sec": 660, "max_reconnect_delay_sec": 30, "max_consecutive_errors": 10, - "no_messages_timeout_sec": 30 + "no_messages_timeout_sec": 180, + "pure_voice": false, + "internet_search": false }, "mic_udp": { @@ -26,7 +28,10 @@ "_comment": "G1 built-in speaker — AudioClient.PlayStream wrapper", "app_name": "sanad", "begin_stream_pause_sec": 0.15, - "wait_finish_margin_sec": 0.3 + "wait_finish_margin_sec": 0.3, + "jitter_prebuffer_sec": 0.3, + "jitter_first_flush_sec": 0.6, + "jitter_max_lead_sec": 0.6 }, "vad": { @@ -34,7 +39,8 @@ "start_sensitivity": "START_SENSITIVITY_HIGH", "end_sensitivity": "END_SENSITIVITY_LOW", "prefix_padding_ms": 20, - "silence_duration_ms": 200 + "silence_duration_ms": 200, + "direct_answer": false }, "barge_in": { @@ -42,7 +48,8 @@ "loud_chunks_needed": 3, "cooldown_sec": 0.3, "echo_suppress_below": 500, - "ai_speak_grace_sec": 0.15 + "ai_speak_grace_sec": 0.15, + "external_barge_in": false }, "recording": { diff --git a/dashboard/routes/live_subprocess.py b/dashboard/routes/live_subprocess.py index 43e1610..f300b2b 100644 --- a/dashboard/routes/live_subprocess.py +++ b/dashboard/routes/live_subprocess.py @@ -41,6 +41,240 @@ async def set_record(on: bool = Query(...)): return {"ok": True, "record_enabled": st.record_enabled} +_VOICE_OPTS_PATH = BASE_DIR / "data" / ".voice_options.json" + + +def _read_voice_opts() -> dict: + import json + try: + with open(_VOICE_OPTS_PATH, encoding="utf-8") as f: + d = json.load(f) + return d if isinstance(d, dict) else {} + except Exception: + return {} + + +@router.get("/interrupt-option") +async def get_interrupt_option(): + """Voice interrupt (barge-in) — GLOBAL, all speakers. OFF: Sanad always + finishes his sentence on every profile (chest/Anker/JBL/Beats Pill — mic + fully gated while he speaks). ON: loud sustained speech cuts him. + Resolution: dashboard toggle (data/.voice_options.json) > + SANAD_EXTERNAL_BARGE_IN env > voice_config.json barge_in.external_barge_in + > off.""" + import os + opts = _read_voice_opts() + if "external_barge_in" in opts: + return {"enabled": bool(opts["external_barge_in"]), "source": "dashboard"} + env = os.environ.get("SANAD_EXTERNAL_BARGE_IN", "").strip().lower() + if env in ("1", "true", "yes"): + return {"enabled": True, "source": "env"} + try: + from Project.Sanad.core.config_loader import section + if bool((section("voice", "barge_in") or {}).get("external_barge_in", False)): + return {"enabled": True, "source": "config"} + except Exception: + pass + return {"enabled": False, "source": "default"} + + +@router.post("/interrupt-option") +async def set_interrupt_option(on: bool = Query(...)): + """Persist the interrupt option (survives rebuilds — data/ is mounted) and + restart the live session so the voice child picks it up.""" + import json + import tempfile + import os as _os + opts = _read_voice_opts() + opts["external_barge_in"] = bool(on) + _VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent), + prefix=".voice_options.", suffix=".tmp") + with _os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(opts, f, indent=2) + _os.replace(tmp, _VOICE_OPTS_PATH) + + restarted = False + from Project.Sanad.main import live_sub + if live_sub is not None: + try: + if (live_sub.status() or {}).get("running"): + await asyncio.to_thread(live_sub.stop) + await asyncio.to_thread(live_sub.start) + restarted = True + except Exception: + pass + return {"ok": True, "enabled": bool(on), "restarted": restarted} + + +@router.get("/direct-answer-option") +async def get_direct_answer_option(): + """'Direct answer' = Gemini replies as soon as the speaker pauses (Live-API + end-of-speech sensitivity HIGH). Resolution: dashboard > env > config.""" + import os + opts = _read_voice_opts() + if "direct_answer" in opts: + return {"enabled": bool(opts["direct_answer"]), "source": "dashboard"} + if os.environ.get("SANAD_DIRECT_ANSWER", "").strip().lower() in ("1", "true", "yes"): + return {"enabled": True, "source": "env"} + try: + from Project.Sanad.core.config_loader import section + if bool((section("voice", "vad") or {}).get("direct_answer", False)): + return {"enabled": True, "source": "config"} + except Exception: + pass + return {"enabled": False, "source": "default"} + + +@router.post("/direct-answer-option") +async def set_direct_answer_option(on: bool = Query(...)): + """Persist the direct-answer option and restart the live session.""" + import json + import tempfile + import os as _os + opts = _read_voice_opts() + opts["direct_answer"] = bool(on) + _VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent), + prefix=".voice_options.", suffix=".tmp") + with _os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(opts, f, indent=2) + _os.replace(tmp, _VOICE_OPTS_PATH) + restarted = False + from Project.Sanad.main import live_sub + if live_sub is not None: + try: + if (live_sub.status() or {}).get("running"): + await asyncio.to_thread(live_sub.stop) + await asyncio.to_thread(live_sub.start) + restarted = True + except Exception: + pass + return {"ok": True, "enabled": bool(on), "restarted": restarted} + + +@router.get("/pure-voice-option") +async def get_pure_voice_option(): + """PURE VOICE = the live session built exactly like standalone Sanadv1: + no tools, no state injections, persona only, no lip-sync markers — the + fastest conversation pipe. Resolution: dashboard > env > config.""" + import os + opts = _read_voice_opts() + if "pure_voice" in opts: + return {"enabled": bool(opts["pure_voice"]), "source": "dashboard"} + if os.environ.get("SANAD_PURE_VOICE", "").strip().lower() in ("1", "true", "yes"): + return {"enabled": True, "source": "env"} + try: + from Project.Sanad.core.config_loader import section + if bool((section("voice", "sanad_voice") or {}).get("pure_voice", False)): + return {"enabled": True, "source": "config"} + except Exception: + pass + return {"enabled": False, "source": "default"} + + +@router.post("/pure-voice-option") +async def set_pure_voice_option(on: bool = Query(...)): + """Persist the pure-voice option and restart the live session.""" + import json + import tempfile + import os as _os + opts = _read_voice_opts() + opts["pure_voice"] = bool(on) + _VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent), + prefix=".voice_options.", suffix=".tmp") + with _os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(opts, f, indent=2) + _os.replace(tmp, _VOICE_OPTS_PATH) + restarted = False + from Project.Sanad.main import live_sub + if live_sub is not None: + try: + if (live_sub.status() or {}).get("running"): + await asyncio.to_thread(live_sub.stop) + await asyncio.to_thread(live_sub.start) + restarted = True + except Exception: + pass + return {"ok": True, "enabled": bool(on), "restarted": restarted} + + +@router.get("/internet-search-option") +async def get_internet_search_option(): + """INTERNET SEARCH = the live session gets the search_internet tool so Gemini + can look up current facts (prices, latest phones, live offers, news) via + Google Search grounding, announcing the search first. Resolution: + dashboard > env > config.""" + import os + opts = _read_voice_opts() + if "internet_search" in opts: + return {"enabled": bool(opts["internet_search"]), "source": "dashboard"} + if os.environ.get("SANAD_INTERNET_SEARCH", "").strip().lower() in ("1", "true", "yes"): + return {"enabled": True, "source": "env"} + try: + from Project.Sanad.core.config_loader import section + if bool((section("voice", "sanad_voice") or {}).get("internet_search", False)): + return {"enabled": True, "source": "config"} + except Exception: + pass + return {"enabled": False, "source": "default"} + + +@router.post("/internet-search-option") +async def set_internet_search_option(on: bool = Query(...)): + """Persist the internet-search option and restart the live session so the + tool set (and persona addendum) is rebaked.""" + import json + import tempfile + import os as _os + opts = _read_voice_opts() + opts["internet_search"] = bool(on) + _VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent), + prefix=".voice_options.", suffix=".tmp") + with _os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(opts, f, indent=2) + _os.replace(tmp, _VOICE_OPTS_PATH) + restarted = False + from Project.Sanad.main import live_sub + if live_sub is not None: + try: + if (live_sub.status() or {}).get("running"): + await asyncio.to_thread(live_sub.stop) + await asyncio.to_thread(live_sub.start) + restarted = True + except Exception: + pass + return {"ok": True, "enabled": bool(on), "restarted": restarted} + + +@router.post("/answer-now") +async def answer_now(): + """Force Gemini to reply immediately instead of waiting out the pause + detection (no-op while it is already speaking).""" + sub = _sub_or_503() + if not (sub.status() or {}).get("running"): + raise HTTPException(409, "Live session not running") + if not hasattr(sub, "send_answer_now"): + raise HTTPException(501, "engine build lacks send_answer_now") + await asyncio.to_thread(sub.send_answer_now) + return {"ok": True} + + +@router.post("/interrupt") +async def interrupt_speech(): + """Instantly stop the AI mid-sentence and go back to listening (works + regardless of the barge-in option; no-op if it isn't speaking).""" + sub = _sub_or_503() + if not (sub.status() or {}).get("running"): + raise HTTPException(409, "Live session not running") + if not hasattr(sub, "send_interrupt"): + raise HTTPException(501, "engine build lacks send_interrupt") + await asyncio.to_thread(sub.send_interrupt) + return {"ok": True} + + @router.post("/start") async def start_subprocess(): live_sub = _sub_or_503() diff --git a/dashboard/routes/motion.py b/dashboard/routes/motion.py index 58cea95..06fba9c 100644 --- a/dashboard/routes/motion.py +++ b/dashboard/routes/motion.py @@ -4,7 +4,7 @@ from __future__ import annotations import asyncio -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Query from pydantic import BaseModel router = APIRouter() @@ -50,7 +50,19 @@ async def trigger_action(payload: TriggerPayload): from Project.Sanad.main import arm if arm is None: raise HTTPException(503, "Arm controller not attached.") - _block_if_movement_armed() + + # Only the legacy JSONL replays (direct rt/arm_sdk streaming) conflict with + # the walker. R1 preset gestures go through the arm service, which moves the + # arms while the firmware balances — so those are allowed while movement is + # armed. Gate JSONL only. + acts = arm.list_actions() if hasattr(arm, "list_actions") else [] + resolved = None + if payload.action_id is not None: + resolved = next((a for a in acts if a.get("id") == payload.action_id), None) + elif payload.action_name: + resolved = next((a for a in acts if a.get("name") == payload.action_name), None) + if resolved and resolved.get("file"): + _block_if_movement_armed() speed = max(0.1, min(payload.speed, 5.0)) @@ -73,6 +85,30 @@ async def trigger_action(payload: TriggerPayload): raise HTTPException(400, "Provide action_id or action_name.") +# ── R1 head-look menu (recorded teach motions via the arm service) ────────── + +@router.get("/head-actions") +async def head_actions(): + """The recorded head-look names for the Controller-tab HEAD menu.""" + from Project.Sanad.main import arm + return {"looks": arm.head_looks() if arm else []} + + +@router.post("/head") +async def head_look(name: str = Query(...)): + """Replay a recorded head rotation (R1 arm-service teach action). NOT gated by + the movement lock: the R1 arm service moves the head while the firmware keeps + the body balanced (that is the whole point of the canteen pattern), so head + looks are safe — and useful — while locomotion/teleop is armed.""" + from Project.Sanad.main import arm + if arm is None: + raise HTTPException(503, "Arm controller not attached.") + res = await asyncio.to_thread(arm.run_head_look, name) + if not res.get("ok"): + raise HTTPException(502, res.get("error") or f"head look failed (rc={res.get('rc')})") + return res + + @router.post("/cancel") async def cancel_motion(): from Project.Sanad.main import arm diff --git a/dashboard/routes/voice.py b/dashboard/routes/voice.py index a3d1b88..8edf2f3 100644 --- a/dashboard/routes/voice.py +++ b/dashboard/routes/voice.py @@ -173,10 +173,12 @@ async def update_api_key(payload: ApiKeyPayload): raise HTTPException(400, "API key cannot be empty.") if len(key) < 20: raise HTTPException(400, "API key looks too short.") - if not key.startswith("AIza"): + # Google AI Studio issues classic "AIza..." keys and (since late 2025) + # new-format "AQ...." keys — both are plain x-goog-api-key API keys. + if not key.startswith(("AIza", "AQ.")): raise HTTPException( 400, - "Gemini API keys normally start with 'AIza'. " + "Gemini API keys start with 'AIza' (classic) or 'AQ.' (new format). " "Double-check you're pasting a Google AI Studio key.", ) diff --git a/dashboard/static/index.html b/dashboard/static/index.html index dd29511..0db1461 100644 --- a/dashboard/static/index.html +++ b/dashboard/static/index.html @@ -352,7 +352,7 @@
-
+ @@ -401,9 +401,15 @@
+ + + Pure: … + Direct: … + Interrupt: … + Search: … Pause: Auto @@ -504,7 +510,7 @@
-
+
@@ -883,6 +889,18 @@

Diagnostics — joints 12–28


     
+ + +
+

Head — recorded looks

+
+ Replays your recorded head rotations via the R1 arm service. Firmware keeps the body balanced. + Gated by the same movement lock as arm actions. +
+
+ loading… +
+
@@ -1617,7 +1635,10 @@ async function renderActions(arm){ _armBusy=arm.busy||false; try{ const r=await api('GET','/api/motion/actions'); - const acts=r.actions||[]; + // R1 gestures only — the JSONL replay buttons are dropped (arm control now + // goes through the R1 'arm' service; recorded head looks live in the HEAD + // menu on the Controller tab). + const acts=(r.actions||[]).filter(a=>!a.file); _renderChips(acts); _renderList(acts); ['','2'].forEach(sfx=>{const bb=document.getElementById('arm-busy-badge'+sfx);if(bb)bb.style.display=_armBusy?'inline-flex':'none';}); @@ -2014,6 +2035,92 @@ async function reassertLiveHold(){try{const s=await api('GET','/api/records/play async function stopLiveSub(b){btnLoad(b);try{await api('POST','/api/live-subprocess/stop');toast('Stopped','info');}catch(e){}btnDone(b);refreshLiveSub();} async function refreshLiveSub(){try{const r=await api('GET','/api/live-subprocess/status');const st=document.getElementById('ls-state');st.textContent=r.state||'stopped';st.className='badge '+(r.running?'badge-ok':'badge-warn');document.getElementById('ls-msg').textContent=r.state_message||'--';document.getElementById('ls-user').textContent=r.last_user_text||'--';document.getElementById('ls-log').textContent=(r.log_tail||[]).slice(-25).join('\n');const rb=document.getElementById('ls-rec-btn');if(rb){const on=!!r.record_enabled;rb.textContent='Rec: '+(on?'ON':'OFF');rb.className='btn btn-sm '+(on?'btn-success':'btn-ghost');}}catch(e){}} async function toggleAutoRecord(b){const cur=(b&&b.textContent||'').includes('ON');const next=!cur;btnLoad(b);try{const r=await api('POST','/api/live-subprocess/record?on='+(next?'1':'0'));toast('Auto-recording '+(r.record_enabled?'ON':'OFF'),'ok');}catch(e){toast('Toggle failed: '+(e.message||e),'err');}btnDone(b);refreshLiveSub();} +// Voice-option badges (Pure / Search / Direct / Interrupt) + instant buttons — +// ported from Sanad G1; backed by /api/live-subprocess/*-option endpoints. +let _pureOpt=null; +function _paintPureBadge(){ + const b=document.getElementById('ls-pure'); + if(!b)return; + b.textContent='Pure: '+(_pureOpt===null?'…':(_pureOpt?'ON':'OFF')); + b.className='badge '+(_pureOpt?'badge-ok':''); +} +async function refreshPureOpt(){ + try{const r=await api('GET','/api/live-subprocess/pure-voice-option');_pureOpt=!!r.enabled;_paintPureBadge();}catch(e){} +} +async function togglePureOpt(){ + const want=!_pureOpt; + try{ + const r=await api('POST','/api/live-subprocess/pure-voice-option?on='+want); + _pureOpt=!!r.enabled;_paintPureBadge(); + toast('Pure voice '+(_pureOpt?'ON — Sanadv1-style session':'OFF — full features')+(r.restarted?' (session restarted)':'')); + }catch(e){toast('Pure toggle failed: '+(e.message||e));} +} +setTimeout(refreshPureOpt,1000); +let _searchOpt=null; +function _paintSearchBadge(){ + const b=document.getElementById('ls-search'); + if(!b)return; + b.textContent='Search: '+(_searchOpt===null?'…':(_searchOpt?'ON':'OFF')); + b.className='badge '+(_searchOpt?'badge-ok':''); +} +async function refreshSearchOpt(){ + try{const r=await api('GET','/api/live-subprocess/internet-search-option');_searchOpt=!!r.enabled;_paintSearchBadge();}catch(e){} +} +async function toggleSearchOpt(){ + const want=!_searchOpt; + try{ + const r=await api('POST','/api/live-subprocess/internet-search-option?on='+want); + _searchOpt=!!r.enabled;_paintSearchBadge(); + toast('Internet search '+(_searchOpt?'ON — Sanad can look up current info':'OFF — pure voice, no search')+(r.restarted?' (session restarted)':'')); + }catch(e){toast('Search toggle failed: '+(e.message||e));} +} +setTimeout(refreshSearchOpt,1050); +let _dirOpt=null; +function _paintDirectBadge(){ + const b=document.getElementById('ls-direct'); + if(!b)return; + b.textContent='Direct: '+(_dirOpt===null?'…':(_dirOpt?'ON':'OFF')); + b.className='badge '+(_dirOpt?'badge-ok':''); +} +async function refreshDirectOpt(){ + try{const r=await api('GET','/api/live-subprocess/direct-answer-option');_dirOpt=!!r.enabled;_paintDirectBadge();}catch(e){} +} +async function toggleDirectOpt(){ + const want=!_dirOpt; + try{ + const r=await api('POST','/api/live-subprocess/direct-answer-option?on='+want); + _dirOpt=!!r.enabled;_paintDirectBadge(); + toast('Direct answer '+(_dirOpt?'ON':'OFF')+(r.restarted?' — live session restarted':'')); + }catch(e){toast('Direct toggle failed: '+(e.message||e));} +} +setTimeout(refreshDirectOpt,950); +let _intOpt=null; +function _paintInterruptBadge(){ + const b=document.getElementById('ls-interrupt'); + if(!b)return; + b.textContent='Interrupt: '+(_intOpt===null?'…':(_intOpt?'ON':'OFF')); + b.className='badge '+(_intOpt?'badge-ok':''); +} +async function refreshInterruptOpt(){ + try{const r=await api('GET','/api/live-subprocess/interrupt-option');_intOpt=!!r.enabled;_paintInterruptBadge();}catch(e){} +} +async function answerNow(){ + try{await api('POST','/api/live-subprocess/answer-now');toast('Answering now…');} + catch(e){toast('Answer-now failed: '+(e.message||e));} +} +async function instantInterrupt(){ + try{await api('POST','/api/live-subprocess/interrupt');toast('Interrupted — listening');} + catch(e){toast('Interrupt failed: '+(e.message||e));} +} +async function toggleInterruptOpt(){ + const want=!_intOpt; + try{ + const r=await api('POST','/api/live-subprocess/interrupt-option?on='+want); + _intOpt=!!r.enabled;_paintInterruptBadge(); + toast('Interrupt '+(_intOpt?'ON':'OFF')+(r.restarted?' — live session restarted':'')); + }catch(e){toast('Interrupt toggle failed: '+(e.message||e));} +} +setTimeout(refreshInterruptOpt,900); // Typed Replay async function trGenerate(b){const t=document.getElementById('tr-text').value;if(!t)return toast('Enter text','err');btnLoad(b);try{await api('POST','/api/typed-replay/say',{text:t,record:document.getElementById('tr-capture').checked,record_name:document.getElementById('tr-name').value});toast('Generated & played','ok');refreshTR();}catch(e){}btnDone(b);} @@ -2365,7 +2472,31 @@ function stopRecPreview(){ img.src=''; // closes the MJPEG connection } } -// Hook into tab switch — start/stop preview when recognition tab is active. +// R1 head-look menu (Controller tab) — replays your recorded head rotations via +// the R1 arm service (ExecuteCustomAction). Loaded once when the tab opens. +let _ctrlHeadLoaded=false; +async function ctrlLoadHead(){ + const el=document.getElementById('ctrl-head-menu'); if(!el) return; + try{ + const r=await api('GET','/api/motion/head-actions'); + const looks=r.looks||[]; + const label={up_look:'↑ up',down_look:'↓ down',left_look:'← left',right_look:'→ right', + up_down_look:'↕ up-down',left_right_look:'⇄ left-right',right_left_look:'⇄ right-left',center:'⊙ center'}; + el.innerHTML=looks.length + ? looks.map(n=>``).join('') + : 'No recorded head looks'; + _ctrlHeadLoaded=true; + }catch(e){ el.innerHTML='head menu unavailable'; } +} +async function ctrlHeadLook(name,btn){ + if(btn) btnLoad(btn); + try{ await api('POST','/api/motion/head?name='+encodeURIComponent(name)); toast('Head → '+name.replace(/_/g,' '),'ok'); } + catch(e){ toast('Head look failed: '+(e.message||e),'err'); } + if(btn) btnDone(btn); +} + +// Hook into tab switch — start/stop preview when recognition tab is active, +// and lazy-load the Controller head menu on first open. (function(){ const origSwitchTab=window.switchTab; window.switchTab=function(name){ @@ -2373,6 +2504,7 @@ function stopRecPreview(){ _recTabActive=(name==='recognition'); if(name==='recognition'){refreshRecognition();refreshFaces();refreshZones();startRecPreview();} else{stopRecPreview();} + if(name==='controller'&&!_ctrlHeadLoaded) ctrlLoadHead(); }; })(); // Face CRUD stubs — filled in milestone 5 diff --git a/gemini/script.py b/gemini/script.py index 83ebeb2..a7c9ab5 100644 --- a/gemini/script.py +++ b/gemini/script.py @@ -55,11 +55,23 @@ _MODEL = os.environ.get( "SANAD_GEMINI_MODEL", "gemini-2.5-flash-native-audio-preview-12-2025", ) -_MIC_GAIN = _SV.get("mic_gain", 1.0) +# Mic gain: amplifies mic samples before they reach Gemini — the single +# biggest lever for slow/failed speech detection at a distance (weak audio +# transcribes but often yields NO reply turn). Env override for per-robot +# tuning; watch for clipping if the speaker stands close (>3.0 rarely wise). +try: + _MIC_GAIN = float(os.environ.get("SANAD_MIC_GAIN", "") or _SV.get("mic_gain", 1.0)) +except ValueError: + _MIC_GAIN = _SV.get("mic_gain", 1.0) _SESSION_TIMEOUT = _SV.get("session_timeout_sec", 660) _MAX_RECONNECT_DELAY = _SV.get("max_reconnect_delay_sec", 30) _MAX_CONSECUTIVE_ERRORS = _SV.get("max_consecutive_errors", 10) -_NO_MESSAGES_TIMEOUT = _SV.get("no_messages_timeout_sec", 30) +# How long with ZERO messages from Gemini before we declare the session dead. +# In a QUIET room Gemini legitimately sends nothing — 30s was killing healthy +# sessions every ~40s (reconnect churn + unsolicited greeting + a 1-3s deaf +# window each time). REAL deaths are caught fast by the send path ("mic send +# failed: 1011 ..."), so this can be patient. +_NO_MESSAGES_TIMEOUT = _SV.get("no_messages_timeout_sec", 180) # Extra mic-gate time after the AI stops, on loud external-speaker profiles # (JBL) — covers the speaker buffer + room reverb so it doesn't hear its tail. _ECHO_TAIL_SEC = _SV.get("echo_tail_sec", 0.6) @@ -77,6 +89,70 @@ _JBL_BARGE_CHUNKS = _SV.get("jbl_barge_chunks", 9) # FADED (a gap between words/numbers). In that window barge-in drops to a low, # sensitive bar so the user can interrupt; while audio is flowing it stays high. _JBL_BLEED_FADE_SEC = _SV.get("jbl_bleed_fade_sec", 0.5) +# OPTION: voice interrupt (barge-in) — GLOBAL, all speakers. OFF (default): +# Sanad always finishes his sentence — the mic to Gemini is fully gated while +# he speaks (+ echo tail) on EVERY profile (chest/Anker/JBL/Beats Pill), so +# neither the local energy detector nor Gemini's server VAD can cut him. +# ON: loud sustained speech interrupts him — builtin/Anker use the normal +# barge parameters; loud external speakers (JBL / Beats Pill) use the stricter +# external parameters and stay fully gated for echo safety. The dashboard +# ⏹ Interrupt-now button works regardless. Env > config; default OFF. +_EXTERNAL_BARGE_IN = ( + os.environ.get("SANAD_EXTERNAL_BARGE_IN", "").strip().lower() in ("1", "true", "yes") + or bool(_BI.get("external_barge_in", False)) +) +# OPTION: "direct answer" — reply as soon as the speaker pauses. Flips the +# Live-API end-of-speech sensitivity from LOW (conservative: waits to be sure +# you finished, feels slow) to HIGH (immediate turn-taking; may cut off very +# hesitant speakers mid-thought). Env > config; dashboard file has final say. +_DIRECT_ANSWER = ( + os.environ.get("SANAD_DIRECT_ANSWER", "").strip().lower() in ("1", "true", "yes") + or bool(_VAD.get("direct_answer", False)) +) + +# OPTION: PURE VOICE — make the live session EXACTLY like standalone Sanadv1: +# no tools (no function-calling deliberation), no state injections, persona +# only, no lip-sync markers, no frame/state loops. The fastest possible +# conversation pipe; nav/vision hooks are dormant. +_PURE_VOICE = ( + os.environ.get("SANAD_PURE_VOICE", "").strip().lower() in ("1", "true", "yes") + or bool(_SV.get("pure_voice", False)) +) + +# OPTION: INTERNET SEARCH (Google Search grounding). When ON, the live session +# gets ONE extra tool, `search_internet(query)`, so the model can look up +# CURRENT facts (today's prices, the newest phones, live Etisalat offers, news). +# The tool description tells the model to SAY it will check first, then call the +# tool — so the robot announces "let me search the internet", the parent runs +# the grounded generateContent call in a worker thread, and the model speaks the +# result. Default OFF → pure voice stays tool-free and Sanadv1-fast; only when +# a search actually fires does the (announced) grounding delay occur. +_INTERNET_SEARCH = ( + os.environ.get("SANAD_INTERNET_SEARCH", "").strip().lower() in ("1", "true", "yes") + or bool(_SV.get("internet_search", False)) +) +# Model used for the grounded web-search call (generateContent, NOT the live +# native-audio model). gemini-3.5-flash grounds well and cites the source site. +_SEARCH_MODEL = os.environ.get("SANAD_SEARCH_MODEL", "gemini-3.5-flash") + +# The dashboard toggles (data/.voice_options.json, written by the +# /api/live-subprocess/*-option endpoints, which then restart this child) +# have the FINAL say over env/config. +try: + import json as _vo_json + with open(BASE_DIR / "data" / ".voice_options.json", encoding="utf-8") as _vo_f: + _vo = _vo_json.load(_vo_f) + if isinstance(_vo, dict): + if "external_barge_in" in _vo: + _EXTERNAL_BARGE_IN = bool(_vo["external_barge_in"]) + if "direct_answer" in _vo: + _DIRECT_ANSWER = bool(_vo["direct_answer"]) + if "pure_voice" in _vo: + _PURE_VOICE = bool(_vo["pure_voice"]) + if "internet_search" in _vo: + _INTERNET_SEARCH = bool(_vo["internet_search"]) +except Exception: + pass _CHUNK_BYTES = CHUNK_SIZE * 2 _SILENCE_PCM = b"\x00" * _CHUNK_BYTES @@ -88,6 +164,15 @@ _SILENCE_PCM = b"\x00" * _CHUNK_BYTES # safe to read from the asyncio loops. _INPUT_PAUSED = threading.Event() +# Set by an "interrupt:" stdin command (the dashboard's Interrupt-now button). +# The mic loop consumes it: stops the speaker mid-sentence, back to listening. +_MANUAL_INTERRUPT = threading.Event() + +# Set by an "answernow:" stdin command (the dashboard's Answer-now button). +# The mic loop feeds ~0.6s of silence so the server VAD closes the user's turn +# immediately → Gemini answers right away instead of waiting out the pause. +_FORCE_ANSWER = threading.Event() + # ── Recognition (camera + face gallery) tunables ── _RECOG_STATE_PATH = Path(os.environ.get( "SANAD_RECOGNITION_STATE_PATH", @@ -195,6 +280,76 @@ def _nav_function_declarations() -> list: ] +def _search_function_declaration() -> list: + """Live tool: search the internet (Google Search grounding) for current facts.""" + S, T = types.Schema, types.Type + return [ + types.FunctionDeclaration( + name="search_internet", + description=( + "Search the LIVE internet for CURRENT, real-time facts you cannot " + "otherwise know: today's prices, the newest phone / device models, " + "current Etisalat / e& offers and promotions, product availability, " + "or recent news. IMPORTANT: BEFORE you call this, first SAY one " + "short sentence out loud telling the user you will check — e.g. " + "\"Let me search the internet for that.\" / " + "\"دعني أبحث لك على الإنترنت.\" — " + "then call this with a clear English search query. After the " + "result returns, answer the user naturally from it. Do NOT say the " + "tool name out loud. Only use this for genuinely current / " + "time-sensitive questions, not for things you already know." + ), + parameters=S(type=T.OBJECT, properties={ + "query": S(type=T.STRING, + description="The web-search query, phrased in English."), + }, required=["query"]), + ), + ] + + +# Appended to the persona ONLY when internet search is enabled, so the model +# knows it can (and how to) look things up. Kept short — the tool description +# carries the detail. +_SEARCH_PROMPT_ADDENDUM = ( + "\n\nYou can look things up on the live internet with the search_internet " + "tool. Use it whenever the user asks about current prices, the latest phones " + "or devices, today's Etisalat/e& offers, availability, or recent news — " + "anything you can't be certain is up to date. ALWAYS say a short 'let me " + "check the internet' line first (in the user's language), then call the tool, " + "then answer from the result. Keep answers concise and, for exact prices, add " + "that a sales advisor can confirm the final figure. " + "CRITICAL: after the search result returns, CONTINUE where you left off — " + "NEVER repeat or re-say any sentence you already spoke, including the 'let me " + "check the internet' lead-in. If your reply is complete, stop and wait." +) + + +def _run_grounding_search(query: str, api_key: str) -> dict: + """Blocking Google-Search-grounded generateContent call (run in a thread). + Returns {ok, answer} for the model to speak, or {ok: False, reason}.""" + try: + client = genai.Client(api_key=api_key) + resp = client.models.generate_content( + model=_SEARCH_MODEL, + contents=query, + config=types.GenerateContentConfig( + tools=[types.Tool(google_search=types.GoogleSearch())], + ), + ) + answer = (getattr(resp, "text", "") or "").strip() + if not answer: + return {"ok": False, "reason": "empty_result"} + # Cap length so the spoken reply stays reasonable. + return {"ok": True, "answer": answer[:1500]} + except Exception as exc: + log.warning("internet search failed: %s", exc) + return {"ok": False, "reason": "search_failed", "detail": str(exc)[:200]} + + +# NOTE (SanadR1): the LED-mask face/social-QR tools (_FACE_EMOTIONS, +# set_expression, show_social) were removed in the R1 port — the R1 has no mask. + + # ── stdin push channel (Marcus pattern) ────────────────────── # The GeminiSubprocess supervisor writes two line types to this process's # stdin: @@ -233,7 +388,7 @@ _PROFILE_LOCK = threading.Lock() _PROFILE_PENDING: dict = {"id": None, "reason": ""} _VALID_PROFILES = {"builtin", "anker", "anker_powerconf", - "hollyland_builtin", "jbl_builtin_mic"} + "hollyland_builtin", "jbl_builtin_mic", "beats_pill"} def _stdin_watcher() -> None: @@ -266,6 +421,10 @@ def _stdin_watcher() -> None: with _LATEST_FRAME_LOCK: _LATEST_FRAME["bytes"] = data _LATEST_FRAME["ts"] = time.time() + elif line.startswith("interrupt:"): + _MANUAL_INTERRUPT.set() + elif line.startswith("answernow:"): + _FORCE_ANSWER.set() elif line.startswith("state:"): try: payload = json.loads(line[len("state:"):]) @@ -349,7 +508,12 @@ class GeminiBrain: self._swap_lock: Optional[asyncio.Lock] = None # built in run() self._recorder = recorder self._voice = voice_name or GEMINI_VOICE + # R1: no LED-mask face, so no face addendum in any mode. self._system_prompt = system_prompt or "" + # Internet-search addendum is independent of pure voice — the search + # tool is available in either mode when the option is enabled. + if _INTERNET_SEARCH: + self._system_prompt += _SEARCH_PROMPT_ADDENDUM self._api_key = GEMINI_API_KEY self._stop_flag = asyncio.Event() # per-session state (reset in the outer reconnect loop) @@ -443,6 +607,10 @@ class GeminiBrain: if self._swap_lock is None: self._swap_lock = asyncio.Lock() + # PURE VOICE is TOGGLE-AWARE: all loops run (cheap when + # their feature is off) — pure-ness is enforced per-feature + # (announce guards + tools) so enabling Camera/Movement in + # the dashboard works live, and disabling returns to pure. try: await asyncio.wait_for( asyncio.gather( @@ -505,10 +673,15 @@ class GeminiBrain: types.StartSensitivity, _VAD.get("start_sensitivity", "START_SENSITIVITY_HIGH"), ), - end_of_speech_sensitivity=getattr( - types.EndSensitivity, - _VAD.get("end_sensitivity", "END_SENSITIVITY_LOW"), - ), + # "Direct answer" option: HIGH = reply the moment the + # speaker pauses; otherwise the configured (LOW) default + # waits longer to be sure the speaker finished. + end_of_speech_sensitivity=( + types.EndSensitivity.END_SENSITIVITY_HIGH + if _DIRECT_ANSWER else getattr( + types.EndSensitivity, + _VAD.get("end_sensitivity", "END_SENSITIVITY_LOW"), + )), prefix_padding_ms=_VAD.get("prefix_padding_ms", 20), silence_duration_ms=_VAD.get("silence_duration_ms", 200), ), @@ -518,13 +691,26 @@ class GeminiBrain: system_instruction=types.Content( parts=[types.Part(text=self._system_prompt)], ), - # Native function-calling: Gemini can drive the robot to saved - # places (navigate_to_place / list_places / where_am_i / - # stop_navigation). Disable with SANAD_NAV_TOOLS=0. - tools=([types.Tool(function_declarations=_nav_function_declarations())] - if _NAV_TOOLS_ENABLED else []), + # Native function-calling: nav tools (if enabled) + the internet- + # search tool (if enabled). No face tools on the R1 (no mask). + # PURE VOICE: no tools (the reply-latency tax) — EXCEPT nav tools + # when movement is enabled (walking commands need them). + tools=self._build_tools(), ) + def _build_tools(self) -> Any: + """Assemble the live-session function tools. PURE VOICE keeps this None + (the reply-latency tax) UNLESS movement is on (nav tools) or internet + search is on (the search tool). R1: no face tools (no mask).""" + decls: list = [] + # Nav tools: whenever NOT (pure and not moving) and nav is enabled. + if _NAV_TOOLS_ENABLED and (self._movement_enabled or not _PURE_VOICE): + decls += _nav_function_declarations() + # Internet-search tool: whenever the option is enabled (either mode). + if _INTERNET_SEARCH: + decls += _search_function_declaration() + return [types.Tool(function_declarations=decls)] if decls else None + # ─── state helpers ──────────────────────────────────── def _reset_turn_state(self) -> None: @@ -564,6 +750,7 @@ class GeminiBrain: loop = asyncio.get_event_loop() loud_count = 0 last_activity = time.time() + force_silence_until = 0.0 # Answer-now button: silence burst window while not self._done.is_set() and not self._stop_flag.is_set(): try: @@ -578,6 +765,27 @@ class GeminiBrain: data = samples.tobytes() energy = _audio_energy(data) now = time.time() + + # Dashboard Answer-now button: feed the server VAD a silence burst + # so it closes the user's turn NOW → immediate reply. + if _FORCE_ANSWER.is_set(): + _FORCE_ANSWER.clear() + if not self._speaking: + force_silence_until = now + 0.6 + log.info("ANSWER NOW (dashboard) — forcing turn end") + + # Dashboard Interrupt-now button: cut speech instantly AND stay + # SILENT until the user speaks — arming the greeting suppression + # also swallows any in-flight or reconnect-greeting audio, so a + # press while "listening" can never be followed by Sanad talking + # on his own. + if _MANUAL_INTERRUPT.is_set(): + _MANUAL_INTERRUPT.clear() + log.info("MANUAL INTERRUPT (dashboard)%s", + "" if self._speaking else " — idle; muted until user speaks") + self._interrupt("manual") + self._barge_block_until = now + cooldown + self._suppress_greeting = True # On the JBL (loud external speaker) the head mic hears the robot's # OWN voice as loud as the user. We FULLY gate the mic to Gemini while # it speaks (+ a short echo tail) so it NEVER hears itself, and we @@ -586,24 +794,47 @@ class GeminiBrain: # model. (Reliable JBL interrupt needs AEC; the only PulseAudio mic is # dead, so that's separate work.) The chest speaker (builtin) keeps # light quiet-frame suppression + working barge-in (firmware AEC). - full_gate = "jbl" in (self._current_profile_id or "") + # The Beats Pill (USB) is the same regime as the JBL: loud external + # speaker with a raw AEC-less mic inches away — the mic feed to + # Gemini is ALWAYS fully gated while speaking (echo-safe). Voice + # barge-in on external speakers is an OPTION (_EXTERNAL_BARGE_IN): + # the LOCAL detector cuts the speaker using stricter parameters. + external = any(t in (self._current_profile_id or "") + for t in ("jbl", "beats")) + # INTERRUPT OFF → NOBODY interrupts, on ANY speaker: every profile + # gets the Beats-Pill regime (mic fully gated while speaking + tail, + # so Gemini hears silence → no server-side cut; local barge-in + # disabled → no energy cut). Sanad always finishes his sentence. + # INTERRUPT ON → local barge-in works everywhere: builtin/Anker use + # the normal parameters, loud external speakers (JBL / Beats Pill) + # use the stricter external parameters and STAY fully gated (echo + # safety — the bleed is as loud as a user). The dashboard + # ⏹ Interrupt-now button always works regardless of this option. + full_gate = external or not _EXTERNAL_BARGE_IN + allow_barge = _EXTERNAL_BARGE_IN - # Barge-in: sustained user energy cuts the AI — chest profile only. - if self._speaking and not full_gate and now >= self._barge_block_until: - if (now - self._ai_speak_start) >= grace: - if energy > threshold: + # Barge-in: sustained user energy cuts the AI. + if self._speaking and allow_barge and now >= self._barge_block_until: + eff_grace = _JBL_BARGE_GRACE if external else grace + eff_threshold = threshold * _JBL_BLEED_MARGIN if external else threshold + eff_chunks = _JBL_BARGE_CHUNKS if external else chunks_needed + if (now - self._ai_speak_start) >= eff_grace: + if energy > eff_threshold: loud_count += 1 else: loud_count = max(0, loud_count - 1) - if loud_count > chunks_needed: - log.info("BARGE-IN (e=%d)", energy) + if loud_count > eff_chunks: + log.info("BARGE-IN (e=%d, external=%s)", energy, external) self._interrupt("barge-in") loud_count = 0 self._barge_block_until = now + cooldown # Echo suppression: mask the mic so the model doesn't hear its own bleed. send_data = data - if _INPUT_PAUSED.is_set(): + if now < force_silence_until: + # Answer-now: mask the mic so the server VAD sees pure silence. + send_data = _SILENCE_PCM + elif _INPUT_PAUSED.is_set(): # Paused for a record playback — feed silence so Gemini neither # hears the record nor keeps talking over it. send_data = _SILENCE_PCM @@ -764,7 +995,8 @@ class GeminiBrain: # mouth-open level (0..3) from this chunk's RMS, # throttled. Parsed by GeminiSubprocess._reader_loop. _mnow = time.time() - if _mnow - getattr(self, "_mouth_t", 0.0) >= 0.08: + if not _PURE_VOICE and \ + _mnow - getattr(self, "_mouth_t", 0.0) >= 0.08: _rms = (float(np.sqrt(np.mean( audio.astype(np.float32) ** 2))) if audio.size else 0.0) # Lower thresholds bias the mouth more open @@ -821,6 +1053,10 @@ class GeminiBrain: async def _announce_vision_state(self, session: Any, enabled: bool, is_toggle: bool) -> None: + if _PURE_VOICE and not enabled and not is_toggle: + # PURE: skip the "disabled" standing-context injection — only + # ENABLED features get announced/used (toggles always announce). + return if is_toggle and enabled: text = ( "[VISION ON] Your camera was just enabled — you can now see " @@ -864,6 +1100,10 @@ class GeminiBrain: async def _announce_facerec_state(self, session: Any, enabled: bool, is_toggle: bool) -> None: + if _PURE_VOICE and not enabled and not is_toggle: + # PURE: skip the "disabled" standing-context injection — only + # ENABLED features get announced/used (toggles always announce). + return if is_toggle and enabled: text = ( "[FACE RECOGNITION ON] Face recognition was just enabled — " @@ -908,6 +1148,10 @@ class GeminiBrain: async def _announce_zonerec_state(self, session: Any, enabled: bool, is_toggle: bool) -> None: + if _PURE_VOICE and not enabled and not is_toggle: + # PURE: skip the "disabled" standing-context injection — only + # ENABLED features get announced/used (toggles always announce). + return if is_toggle and enabled: text = ( "[ZONE RECOGNITION ON] You were just given the zones and places " @@ -1069,6 +1313,13 @@ class GeminiBrain: "places": r.get("places", [])} if name == "stop_navigation": return await asyncio.to_thread(_nav_api, "POST", "/api/nav/cancel", None) + if name == "search_internet": + query = str(args.get("query") or "").strip() + if not query: + return {"ok": False, "reason": "no_query"} + log.info("[[SEARCH:%s]]", query) # breadcrumb for the dashboard + return await asyncio.to_thread( + _run_grounding_search, query, self._api_key) return {"ok": False, "reason": "unknown_tool"} except Exception as exc: log.warning("tool %s error: %s", name, exc) @@ -1081,6 +1332,10 @@ class GeminiBrain: async def _announce_movement_state(self, session: Any, enabled: bool, is_toggle: bool) -> None: + if _PURE_VOICE and not enabled and not is_toggle: + # PURE: skip the "disabled" standing-context injection — only + # ENABLED features get announced/used (toggles always announce). + return if is_toggle and enabled: text = ( "[MOVEMENT ON] Walking is now enabled — you can move when the " @@ -1406,6 +1661,12 @@ class GeminiBrain: await self._announce_movement_state( session, self._movement_enabled, is_toggle=True, ) + if _PURE_VOICE: + # PURE: tools are baked per-session (nav tools only while + # movement is on) — cycle the session so the toggle takes + # effect on the function-calling side too. + log.info("pure-voice: cycling session to rebake nav tools") + self._done.set() # Auto-record toggle — flip the recorder live (no session restart). if new_state.record_enabled != last_state.record_enabled: @@ -1436,6 +1697,12 @@ class GeminiBrain: await asyncio.sleep(max(period, backoff)) if not self._vision_enabled: continue + # Never compete with the INCOMING audio stream while the AI is + # speaking — a frame upload on the shared websocket delays audio + # chunks, which breaks the chest speaker (tiny firmware buffer; + # the pulse/Pill path hides it behind its own buffering). + if self._speaking or (time.time() - self._last_ai_audio) < 0.5: + continue with _LATEST_FRAME_LOCK: data = _LATEST_FRAME.get("bytes") ts = _LATEST_FRAME.get("ts", 0.0) diff --git a/gemini/subprocess.py b/gemini/subprocess.py index 76d8c04..0b0047f 100644 --- a/gemini/subprocess.py +++ b/gemini/subprocess.py @@ -38,7 +38,9 @@ _FRAME_FORWARD_INTERVAL_S = float(_LS_CFG.get("frame_forward_interval_sec", 0.5) # Audio profile watcher — poll pactl for the Anker USB device at this # interval, send "profile:" to the child on every state change. -_AUDIO_WATCH_INTERVAL_S = float(_LS_CFG.get("audio_watch_interval_sec", 1.5)) +# 5s: each poll shells out to pactl several times (detect_plugged_profiles) — +# 1.5s was a constant subprocess storm for a rare event (device hot-plug). +_AUDIO_WATCH_INTERVAL_S = float(_LS_CFG.get("audio_watch_interval_sec", 5.0)) # The Anker profile id, as defined in voice/audio_devices.py. When this # profile is fully plugged (both sink + source present), we switch the @@ -515,7 +517,7 @@ class GeminiSubprocess: if the process isn't running or stdin is closed.""" pid = (profile_id or "").strip().lower() if pid not in {"builtin", "anker", "anker_powerconf", - "hollyland_builtin", "jbl_builtin_mic"}: + "hollyland_builtin", "jbl_builtin_mic", "beats_pill"}: log.warning("send_profile: ignoring unknown profile %r", profile_id) return payload: dict[str, Any] = {"id": pid} @@ -533,6 +535,16 @@ class GeminiSubprocess: owns the chest speaker, then resumes. No-op if not running.""" self._send_stdin("pause:%d\n" % (1 if paused else 0)) + def send_interrupt(self) -> None: + """Instantly stop the AI's current speech (dashboard Interrupt-now + button) — the child stops the speaker and returns to listening.""" + self._send_stdin("interrupt:1\n") + + def send_answer_now(self) -> None: + """Force Gemini to answer immediately (dashboard Answer-now button) — + the child feeds the server VAD a short silence burst to close the turn.""" + self._send_stdin("answernow:1\n") + def _audio_watcher(self) -> None: """Background thread — poll pactl for the Anker USB device, signal the child on every plug/unplug edge transition. diff --git a/motion/arm_controller.py b/motion/arm_controller.py index fc01748..8de46aa 100644 --- a/motion/arm_controller.py +++ b/motion/arm_controller.py @@ -82,20 +82,18 @@ except ImportError: _make_low_cmd = None log.warning("Unitree SDK not available — running in simulation mode") -# G1 arm-action client for built-in arm moves (wave, shake_hand, hug, …). -# NOTE: do NOT use LocoClient here — LocoClient is the locomotion/body-move -# client and its ExecuteAction() doesn't recognise arm-action IDs, so arm -# commands become silent no-ops. The correct client is the arm-specific -# G1ArmActionClient with the SDK's action_map (name → opcode lookup). +# R1-native arm-action RPC client for built-in gestures + recorded "teach" +# motions. The R1 firmware does NOT expose live joint streaming of the head/arms; +# the 'arm' service owns rt/arm_sdk and clients drive the upper body ONLY through +# this RPC: ExecuteAction(id) for Unitree preset gestures, ExecuteCustomAction( +# name) for motions recorded via the app's teaching (incl. the head looks). This +# replaces the old G1ArmActionClient — the R1 has no G1 arm-action service, so +# those gestures were silent no-ops on the R1. try: - from unitree_sdk2py.g1.arm.g1_arm_action_client import ( - G1ArmActionClient, - action_map as _ARM_ACTION_MAP, - ) + from Project.Sanad.motion.r1_arm_action_client import R1ArmActionClient _HAS_ARM_CLIENT = True except ImportError: - G1ArmActionClient = None - _ARM_ACTION_MAP = {} + R1ArmActionClient = None _HAS_ARM_CLIENT = False @@ -107,24 +105,41 @@ class Action: category: str = "sdk" # "sdk" | "jsonl" -# -- SDK actions (fixed — built into Unitree firmware) -- +# -- SDK actions (R1 'arm'-service preset gestures) -- +# The Action.id IS the R1 arm-service opcode → executed directly via +# R1ArmActionClient.ExecuteAction(id). Names/ids captured from the robot's +# GetActionList (2026-07-14). Friendliest gestures first. SDK_ACTIONS: list[Action] = [ - Action("release_arm", 0, category="sdk"), - Action("shake_hand", 1, category="sdk"), - Action("high_five", 2, category="sdk"), - Action("hug", 3, category="sdk"), - Action("high_wave", 4, category="sdk"), - Action("clap", 5, category="sdk"), - Action("face_wave", 6, category="sdk"), - Action("left_kiss", 7, category="sdk"), - Action("heart", 8, category="sdk"), - Action("right_heart", 9, category="sdk"), - Action("hands_up", 10, category="sdk"), - Action("x_ray", 11, category="sdk"), - Action("right_hand_up", 12, category="sdk"), - Action("reject", 13, category="sdk"), - Action("right_kiss", 14, category="sdk"), - Action("two_hand_kiss", 15, category="sdk"), + Action("release_arm", 99, category="sdk"), + Action("shake_hand", 27, category="sdk"), + Action("high_five", 18, category="sdk"), + Action("hug", 19, category="sdk"), + Action("both_hands_up", 15, category="sdk"), + Action("wave_above_head", 26, category="sdk"), + Action("wave_under_head", 25, category="sdk"), + Action("blow_kiss_both_hands", 11, category="sdk"), + Action("blow_kiss_left_hand", 12, category="sdk"), + Action("blow_kiss_right_hand", 13, category="sdk"), + Action("right_hand_up", 23, category="sdk"), + Action("right_hand_on_heart", 33, category="sdk"), + Action("clamp", 17, category="sdk"), + Action("refuse", 22, category="sdk"), + Action("emphasize", 35, category="sdk"), + Action("forward_push", 36, category="sdk"), + Action("extend_right_arm_forward", 31, category="sdk"), + Action("both_hands_up_deviate_right", 34, category="sdk"), + Action("ultraman_ray", 24, category="sdk"), + Action("box_left_hand_win", 28, category="sdk"), + Action("box_right_hand_win", 29, category="sdk"), + Action("box_both_hand_win", 30, category="sdk"), +] + +# R1 head-look "teach" motions (recorded via the app's teaching) — replayed by +# NAME via R1ArmActionClient.ExecuteCustomAction. Shown as a simple button list +# in the Controller tab's HEAD card. These are your recorded head rotations. +HEAD_LOOKS: list[str] = [ + "up_look", "down_look", "left_look", "right_look", + "up_down_look", "left_right_look", "right_left_look", "center", ] # Next auto-ID for JSONL actions starts after SDK range. @@ -285,16 +300,16 @@ class ArmController: ) self._crc = CRC() - # Arm-specific action client for built-in moves + # R1 arm-service client for preset gestures + recorded teach/head moves if _HAS_ARM_CLIENT: try: - self._arm_client = G1ArmActionClient() + self._arm_client = R1ArmActionClient() self._arm_client.SetTimeout(10.0) self._arm_client.Init() - log.info("G1ArmActionClient initialized (%d actions) — built-in moves available", - len(_ARM_ACTION_MAP)) + log.info("R1ArmActionClient initialized — %d preset gestures + %d head looks", + len(SDK_ACTIONS), len(HEAD_LOOKS)) except Exception as exc: - log.warning("G1ArmActionClient init failed: %s — built-in actions disabled", exc) + log.warning("R1ArmActionClient init failed: %s — arm actions disabled", exc) self._arm_client = None self._initialized = True @@ -869,39 +884,42 @@ class ArmController: return if self._arm_client is None: log.warning( - "SDK action %s requested but G1ArmActionClient not available — skipping", + "SDK action %s requested but R1ArmActionClient not available — skipping", action.name, ) return - # Sanad's registry uses underscored names ("shake_hand", "x_ray"); - # the SDK's action_map is keyed by human-readable forms that mix - # spaces and hyphens ("shake hand", "x-ray", "two-hand kiss"). - # Try each candidate in turn. - name = action.name - candidates = [ - name, - name.replace("_", " "), # shake_hand → shake hand - name.replace("_", "-"), # x_ray → x-ray - # two-word with specific hyphenation: first token with hyphen, - # rest with spaces (matches SDK's "two-hand kiss" pattern) - name.replace("_", "-", 1).replace("_", " "), - ] - sdk_name = next((c for c in candidates if c in _ARM_ACTION_MAP), None) - if sdk_name is None: - log.warning( - "SDK action %s not in G1ArmActionClient action_map — tried %s. keys=%s", - action.name, candidates, sorted(_ARM_ACTION_MAP.keys())[:12], - ) - return - opcode = _ARM_ACTION_MAP[sdk_name] - log.info("SDK action: %s (opcode=%s)", action.name, opcode) + # R1 preset gestures execute by opcode directly — Action.id IS the R1 + # arm-service opcode (see SDK_ACTIONS). No name→opcode lookup needed. + log.info("R1 gesture: %s (opcode=%s)", action.name, action.id) try: - self._arm_client.ExecuteAction(opcode) - # Built-in arm actions block on the robot side for ~3s; the SDK - # call returns immediately. Sleep so we don't hammer it back-to-back. + rc = self._arm_client.ExecuteAction(action.id) + if rc != 0: + log.warning("R1 gesture %s returned rc=%s", action.name, rc) + # The arm service blocks on the robot for ~3s; the RPC returns fast. time.sleep(3.0) except Exception as exc: - log.error("SDK action %s failed: %s", action.name, exc) + log.error("R1 gesture %s failed: %s", action.name, exc) + + def run_head_look(self, name: str) -> dict[str, Any]: + """Replay a recorded head-look 'teach' motion by name via the R1 arm + service (ExecuteCustomAction). Used by the Controller-tab HEAD menu. + Returns {ok, rc}. Never raises — reports failure in the result.""" + if name not in HEAD_LOOKS: + return {"ok": False, "error": f"unknown head look '{name}'"} + if not _HAS_SDK or self._arm_client is None: + log.info("[SIM] head look: %s", name) + return {"ok": _HAS_SDK is False, "rc": None, "sim": True} + log.info("R1 head look: %s", name) + try: + rc = self._arm_client.ExecuteCustomAction(name) + return {"ok": rc == 0, "rc": rc} + except Exception as exc: + log.error("head look %s failed: %s", name, exc) + return {"ok": False, "error": str(exc)} + + def head_looks(self) -> list[str]: + """The recorded head-look names for the dashboard menu.""" + return list(HEAD_LOOKS) def status(self) -> dict[str, Any]: return { diff --git a/motion/r1_arm_action_client.py b/motion/r1_arm_action_client.py new file mode 100644 index 0000000..b4a0d0b --- /dev/null +++ b/motion/r1_arm_action_client.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +""" +R1 arm-action RPC client — the R1-NATIVE way to move the upper body (arms + head) +while the firmware keeps balancing the legs. + +Why this and not rt/arm_sdk: the R1 firmware does NOT expose live joint streaming +of the head/arms. The 'arm' service (a.k.a. r1_arm_example) owns rt/arm_sdk +internally; clients drive the upper body ONLY through this RPC — preset gesture +actions (by id) and custom "teach" actions (by name, recorded via the app's +teaching). Captured from the app's own traffic (2026-07-06): + release_arm -> ExecuteAction(99); replay recorded motion -> ExecuteCustomAction(name). + +Service "arm", api version "1.0.0.14". Same Client base as R1LocoClient. +State is published on rt/arm/action/state: {"holding":bool,"id":int,"name":str}. +""" +import json +from unitree_sdk2py.rpc.client import Client + +ARM_ACTION_SERVICE_NAME = "arm" +ARM_ACTION_API_VERSION = "1.0.0.14" + +API_EXECUTE_ACTION = 7106 # {"action_id": N} preset actions +API_GET_ACTION_LIST = 7107 # -> JSON list +API_EXECUTE_CUSTOM_ACTION = 7108 # {"action_name": "..."} recorded teach actions +API_RECORD = 7110 # start = {"action_name":NAME} ; stop+save = "" (empty str) +API_STOP_CUSTOM_ACTION = 7113 + +RELEASE_ARM = 99 # go compliant / release a held action + + +class R1ArmActionClient(Client): + def __init__(self): + super().__init__(ARM_ACTION_SERVICE_NAME, False) + + def Init(self): + self._SetApiVerson(ARM_ACTION_API_VERSION) # SDK spells it "Verson" + for api in (API_EXECUTE_ACTION, API_GET_ACTION_LIST, + API_EXECUTE_CUSTOM_ACTION, API_RECORD, API_STOP_CUSTOM_ACTION): + self._RegistApi(api, 0) + + def ExecuteAction(self, action_id: int): + """Run a preset action by id (99 = release). Returns rc (0 = ok).""" + code, _ = self._Call(API_EXECUTE_ACTION, json.dumps({"action_id": int(action_id)})) + return code + + def ExecuteCustomAction(self, action_name: str): + """Replay a recorded teach action by name (from the app's teaching).""" + code, _ = self._Call(API_EXECUTE_CUSTOM_ACTION, json.dumps({"action_name": str(action_name)})) + return code + + def StopCustomAction(self): + code, _ = self._Call(API_STOP_CUSTOM_ACTION, "{}") + return code + + def ReleaseArm(self): + return self.ExecuteAction(RELEASE_ARM) + + def GetActionList(self): + """Return (rc, data) — data is the JSON list of available actions.""" + code, data = self._Call(API_GET_ACTION_LIST, "") + parsed = None + if code == 0 and data: + try: + parsed = json.loads(data) + except (ValueError, TypeError): + parsed = data + return code, parsed diff --git a/scripts/sanad_script.txt b/scripts/sanad_script.txt index a6b38e4..2c4cd51 100644 --- a/scripts/sanad_script.txt +++ b/scripts/sanad_script.txt @@ -1,4 +1,4 @@ -أنت "سند iron man" — روبوت إماراتي ذكي تابع لروبوت شركة لوتاه تيك Lootah Tech. +أنت "سوبر دبي super-dubai" — روبوت إماراتي ذكي تابع لروبوت شركة لوتاه تيك Lootah Tech. [أولاً: الروح والمعرفة] diff --git a/tests/test_smoke.py b/tests/test_smoke.py index bce4a3b..b686107 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -22,7 +22,9 @@ class TestConfig(unittest.TestCase): def test_base_dir_auto_detect(self): from Project.Sanad.config import BASE_DIR self.assertTrue(BASE_DIR.exists(), f"BASE_DIR missing: {BASE_DIR}") - self.assertEqual(BASE_DIR.name, "Sanad") + # Workstation checkout is SanadR1; the deployed container aliases it + # as Project/Sanad, so both names are valid. + self.assertIn(BASE_DIR.name, ("Sanad", "SanadR1")) def test_data_dirs_exist(self): from Project.Sanad.config import ( diff --git a/voice/audio_devices.py b/voice/audio_devices.py index 8de2b61..9c92a73 100644 --- a/voice/audio_devices.py +++ b/voice/audio_devices.py @@ -106,6 +106,18 @@ PROFILES: list[AudioProfile] = [ source_pattern="powerconf,anker", description="Anker PowerConf USB conference unit — mic + speaker on the same device.", ), + AudioProfile( + id="beats_pill", + label="Beats Pill (mic + speaker)", + # USB-C: kernel names the card "Pill"; pulse sinks/sources carry + # "Beats_Pill" (auto-probed or the manual module-alsa-sink/-source + # loads from /etc/pulse/default.pa — pulse fails to probe the Pill's + # output profile on some kernels). Bluetooth: bluez sink. + sink_pattern="beats_pill,beats,pill", + source_pattern="beats_pill,beats,pill", + description="Beats Pill (USB-C or Bluetooth) — speaker + speakerphone mic.", + udp_mic_fallback=True, # mic falls back to the R1/G1 UDP mic if absent + ), AudioProfile( id="jbl_builtin_mic", label="JBL speaker + built-in mic", @@ -115,6 +127,7 @@ PROFILES: list[AudioProfile] = [ # The JBL has NO microphone → input stays on the G1 built-in mic. source_pattern="alsa_input.platform-sound", description="JBL Bluetooth speaker for output + the G1 built-in microphone for input (the JBL has no mic).", + udp_mic_fallback=True, # builtin mic = R1/G1 UDP stream on units without the alsa source ), ] diff --git a/voice/audio_io.py b/voice/audio_io.py index c2b0b09..59c208d 100644 --- a/voice/audio_io.py +++ b/voice/audio_io.py @@ -31,6 +31,7 @@ import json import socket import struct import subprocess +import queue import threading import time from abc import ABC, abstractmethod @@ -211,7 +212,16 @@ class BuiltinMic(Mic): class BuiltinSpeaker(Speaker): - """G1 robot's built-in speaker via AudioClient.PlayStream (16 kHz mono).""" + """G1 robot's built-in speaker via AudioClient.PlayStream (16 kHz mono). + + JITTER-BUFFERED: chunks are enqueued and a dedicated writer thread sends + them to the firmware, prebuffering ~0.3s before the first RPC and never + running more than ~0.6s ahead of real time. Gemini's audio arrives in + BURSTS (WiFi jitter, shared-websocket contention) — the old direct + per-chunk RPC mapped every arrival gap to an audible dropout on the + chest speaker's tiny firmware buffer, while the pulse (Pill/Anker) path + hid it behind PulseAudio's own buffering. This gives the chest the same + tolerance.""" HARDWARE_RATE = 16_000 @@ -224,10 +234,16 @@ class BuiltinSpeaker(Speaker): self._app_name = app_name or _SP_CFG.get("app_name", "sanad") self._begin_pause = _SP_CFG.get("begin_stream_pause_sec", 0.15) self._finish_margin = _SP_CFG.get("wait_finish_margin_sec", 0.3) + # Jitter buffer tuning (see class docstring). + self._prebuffer_sec = _SP_CFG.get("jitter_prebuffer_sec", 0.30) + self._first_flush_sec = _SP_CFG.get("jitter_first_flush_sec", 0.60) + self._max_lead_sec = _SP_CFG.get("jitter_max_lead_sec", 0.60) self._stop_flag = threading.Event() self._stream_id: Optional[str] = None self._total_sent = 0.0 self._play_start = 0.0 + self._q: "queue.Queue[Optional[bytes]]" = queue.Queue() + self._writer: Optional[threading.Thread] = None def _stop_play_api(self) -> None: try: @@ -241,13 +257,79 @@ class BuiltinSpeaker(Speaker): except Exception: log.warning("BuiltinSpeaker AUDIO_STOP_PLAY failed") + def _drain_queue(self) -> None: + try: + while True: + self._q.get_nowait() + except queue.Empty: + pass + def begin_stream(self) -> None: + # Tear down any previous writer cleanly before starting a new turn. + self._stop_flag.set() + self._drain_queue() + if self._writer is not None and self._writer.is_alive(): + self._q.put(None) + self._writer.join(timeout=1.0) self._stop_flag.clear() + self._drain_queue() self._stop_play_api() time.sleep(self._begin_pause) self._stream_id = f"s_{int(time.time() * 1000)}" self._total_sent = 0.0 self._play_start = time.time() + self._writer = threading.Thread( + target=self._writer_loop, args=(self._stream_id,), + name="builtin-speaker-writer", daemon=True) + self._writer.start() + + def _writer_loop(self, stream_id: str) -> None: + rate = float(self.HARDWARE_RATE) + sent_sec = 0.0 + started = False + t0 = time.time() + batch: list = [] + buffered = 0.0 + while not self._stop_flag.is_set() and self._stream_id == stream_id: + try: + item = self._q.get(timeout=0.05) + except queue.Empty: + item = () + if item is None: + return + if not started: + if item: + batch.append(item) + buffered += len(item) / 2 / rate + if buffered >= self._prebuffer_sec or ( + batch and (time.time() - t0) >= self._first_flush_sec): + self._play_start = time.time() + started = True + for b in batch: + if self._stop_flag.is_set() or self._stream_id != stream_id: + return + try: + self._ac.PlayStream(self._app_name, stream_id, b) + except Exception as exc: + log.warning("PlayStream failed: %s", exc) + sent_sec += len(b) / 2 / rate + batch = [] + continue + if not item: + continue + # Pace: never run more than max_lead ahead of real time — keeps + # the firmware queue short (fast interrupts) while OUR queue is + # the elastic reservoir that absorbs bursty arrival. + lead = sent_sec - (time.time() - self._play_start) + if lead > self._max_lead_sec: + time.sleep(min(lead - self._max_lead_sec, 0.25)) + if self._stop_flag.is_set() or self._stream_id != stream_id: + return + try: + self._ac.PlayStream(self._app_name, stream_id, item) + except Exception as exc: + log.warning("PlayStream failed: %s", exc) + sent_sec += len(item) / 2 / rate def send_chunk(self, pcm: PCMLike, source_rate: int) -> None: if self._stop_flag.is_set(): @@ -256,20 +338,22 @@ class BuiltinSpeaker(Speaker): if arr.size < 10: return hw = _resample_int16(arr, source_rate, self.HARDWARE_RATE) - self._ac.PlayStream(self._app_name, self._stream_id, hw.tobytes()) + self._q.put(hw.tobytes()) self._total_sent += len(hw) / self.HARDWARE_RATE def wait_finish(self) -> None: - elapsed = time.time() - self._play_start - remaining = self._total_sent - elapsed + self._finish_margin - waited = 0.0 - while waited < remaining and not self._stop_flag.is_set(): + # Wait for the queue to drain AND real-time playback to elapse. + while not self._stop_flag.is_set(): + elapsed = time.time() - self._play_start + remaining = self._total_sent - elapsed + self._finish_margin + if self._q.empty() and remaining <= 0: + break time.sleep(0.1) - waited += 0.1 self._stop_play_api() def stop(self) -> None: self._stop_flag.set() + self._drain_queue() self._stop_play_api() @property @@ -815,9 +899,13 @@ _PROFILE_ALIASES = { "hollyland_builtin": "hollyland_builtin", "jbl": "jbl_builtin_mic", "jbl_builtin_mic": "jbl_builtin_mic", + "beats": "beats_pill", + "beats_pill": "beats_pill", + "pill": "beats_pill", } -SUPPORTED_PROFILES = ("builtin", "anker", "hollyland_builtin", "jbl_builtin_mic") +SUPPORTED_PROFILES = ("builtin", "anker", "hollyland_builtin", "jbl_builtin_mic", + "beats_pill") @dataclass @@ -885,6 +973,13 @@ class AudioIO: # pacat is used because PyAudio's 'pulse' device is unavailable in # this env. Neither backend needs the AudioClient. return BuiltinMic(), PulseStreamSpeaker(label="JBL", sink_pattern="jbl,bluez") + if resolved == "beats_pill": + # Beats Pill speaker via pacat → its PulseAudio sink + the G1 + # built-in DDS mic — same regime as the JBL (loud external speaker, + # full mic-gate while speaking; see gemini/script.py). The Pill's + # own USB mic stays available to pulse for dashboard recordings. + return BuiltinMic(), PulseStreamSpeaker( + label="Beats Pill", sink_pattern="beats_pill,beats,pill") raise AssertionError(f"unhandled resolved profile: {resolved!r}") @classmethod diff --git a/voice/sanad_voice.py b/voice/sanad_voice.py index dc64fc5..b389f90 100644 --- a/voice/sanad_voice.py +++ b/voice/sanad_voice.py @@ -146,7 +146,7 @@ _MOVEMENT_PROMPT_RULES = ( "(you receive a [MOVEMENT ON] or [MOVEMENT STATUS] note). When movement is " "OFF, never confirm a motion — tell the user to enable movement from the " "dashboard.\n" - "When movement is ON and the user addresses you by name (Sanad iron man / سند) AND " + "When movement is ON and the user addresses you by name (super-dubai / سوبر دبي) AND " "asks you to move, reply with ONE short confirmation phrase per requested " "motion, in the SAME language, in the order asked. Use these EXACT shapes — " "they are what triggers the motion:\n"