Update 2026-07-20 10:30:47

This commit is contained in:
kassam 2026-07-20 10:30:48 +04:00
parent 44f547f754
commit cfdc0556a4
15 changed files with 993 additions and 108 deletions

View File

@ -39,9 +39,9 @@
"api_key": "", "api_key": "",
"model_live": "gemini-2.5-flash-native-audio-preview-12-2025", "model_live": "gemini-2.5-flash-native-audio-preview-12-2025",
"model_ws_uri": "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent", "model_ws_uri": "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent",
"voice_name": "Charon", "voice_name": "Puck",
"ws_timeout_sec": 30, "ws_timeout_sec": 30,
"default_system_prompt": "You are Bousandah, a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'." "default_system_prompt": "You are super-dubai (سوبر دبي), a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'."
}, },
"g1_hardware": { "g1_hardware": {

View File

@ -10,7 +10,9 @@
"session_timeout_sec": 660, "session_timeout_sec": 660,
"max_reconnect_delay_sec": 30, "max_reconnect_delay_sec": 30,
"max_consecutive_errors": 10, "max_consecutive_errors": 10,
"no_messages_timeout_sec": 30 "no_messages_timeout_sec": 180,
"pure_voice": false,
"internet_search": false
}, },
"mic_udp": { "mic_udp": {
@ -26,7 +28,10 @@
"_comment": "G1 built-in speaker — AudioClient.PlayStream wrapper", "_comment": "G1 built-in speaker — AudioClient.PlayStream wrapper",
"app_name": "sanad", "app_name": "sanad",
"begin_stream_pause_sec": 0.15, "begin_stream_pause_sec": 0.15,
"wait_finish_margin_sec": 0.3 "wait_finish_margin_sec": 0.3,
"jitter_prebuffer_sec": 0.3,
"jitter_first_flush_sec": 0.6,
"jitter_max_lead_sec": 0.6
}, },
"vad": { "vad": {
@ -34,7 +39,8 @@
"start_sensitivity": "START_SENSITIVITY_HIGH", "start_sensitivity": "START_SENSITIVITY_HIGH",
"end_sensitivity": "END_SENSITIVITY_LOW", "end_sensitivity": "END_SENSITIVITY_LOW",
"prefix_padding_ms": 20, "prefix_padding_ms": 20,
"silence_duration_ms": 200 "silence_duration_ms": 200,
"direct_answer": false
}, },
"barge_in": { "barge_in": {
@ -42,7 +48,8 @@
"loud_chunks_needed": 3, "loud_chunks_needed": 3,
"cooldown_sec": 0.3, "cooldown_sec": 0.3,
"echo_suppress_below": 500, "echo_suppress_below": 500,
"ai_speak_grace_sec": 0.15 "ai_speak_grace_sec": 0.15,
"external_barge_in": false
}, },
"recording": { "recording": {

View File

@ -41,6 +41,240 @@ async def set_record(on: bool = Query(...)):
return {"ok": True, "record_enabled": st.record_enabled} return {"ok": True, "record_enabled": st.record_enabled}
_VOICE_OPTS_PATH = BASE_DIR / "data" / ".voice_options.json"
def _read_voice_opts() -> dict:
import json
try:
with open(_VOICE_OPTS_PATH, encoding="utf-8") as f:
d = json.load(f)
return d if isinstance(d, dict) else {}
except Exception:
return {}
@router.get("/interrupt-option")
async def get_interrupt_option():
"""Voice interrupt (barge-in) — GLOBAL, all speakers. OFF: Sanad always
finishes his sentence on every profile (chest/Anker/JBL/Beats Pill mic
fully gated while he speaks). ON: loud sustained speech cuts him.
Resolution: dashboard toggle (data/.voice_options.json) >
SANAD_EXTERNAL_BARGE_IN env > voice_config.json barge_in.external_barge_in
> off."""
import os
opts = _read_voice_opts()
if "external_barge_in" in opts:
return {"enabled": bool(opts["external_barge_in"]), "source": "dashboard"}
env = os.environ.get("SANAD_EXTERNAL_BARGE_IN", "").strip().lower()
if env in ("1", "true", "yes"):
return {"enabled": True, "source": "env"}
try:
from Project.Sanad.core.config_loader import section
if bool((section("voice", "barge_in") or {}).get("external_barge_in", False)):
return {"enabled": True, "source": "config"}
except Exception:
pass
return {"enabled": False, "source": "default"}
@router.post("/interrupt-option")
async def set_interrupt_option(on: bool = Query(...)):
"""Persist the interrupt option (survives rebuilds — data/ is mounted) and
restart the live session so the voice child picks it up."""
import json
import tempfile
import os as _os
opts = _read_voice_opts()
opts["external_barge_in"] = bool(on)
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
prefix=".voice_options.", suffix=".tmp")
with _os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(opts, f, indent=2)
_os.replace(tmp, _VOICE_OPTS_PATH)
restarted = False
from Project.Sanad.main import live_sub
if live_sub is not None:
try:
if (live_sub.status() or {}).get("running"):
await asyncio.to_thread(live_sub.stop)
await asyncio.to_thread(live_sub.start)
restarted = True
except Exception:
pass
return {"ok": True, "enabled": bool(on), "restarted": restarted}
@router.get("/direct-answer-option")
async def get_direct_answer_option():
"""'Direct answer' = Gemini replies as soon as the speaker pauses (Live-API
end-of-speech sensitivity HIGH). Resolution: dashboard > env > config."""
import os
opts = _read_voice_opts()
if "direct_answer" in opts:
return {"enabled": bool(opts["direct_answer"]), "source": "dashboard"}
if os.environ.get("SANAD_DIRECT_ANSWER", "").strip().lower() in ("1", "true", "yes"):
return {"enabled": True, "source": "env"}
try:
from Project.Sanad.core.config_loader import section
if bool((section("voice", "vad") or {}).get("direct_answer", False)):
return {"enabled": True, "source": "config"}
except Exception:
pass
return {"enabled": False, "source": "default"}
@router.post("/direct-answer-option")
async def set_direct_answer_option(on: bool = Query(...)):
"""Persist the direct-answer option and restart the live session."""
import json
import tempfile
import os as _os
opts = _read_voice_opts()
opts["direct_answer"] = bool(on)
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
prefix=".voice_options.", suffix=".tmp")
with _os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(opts, f, indent=2)
_os.replace(tmp, _VOICE_OPTS_PATH)
restarted = False
from Project.Sanad.main import live_sub
if live_sub is not None:
try:
if (live_sub.status() or {}).get("running"):
await asyncio.to_thread(live_sub.stop)
await asyncio.to_thread(live_sub.start)
restarted = True
except Exception:
pass
return {"ok": True, "enabled": bool(on), "restarted": restarted}
@router.get("/pure-voice-option")
async def get_pure_voice_option():
"""PURE VOICE = the live session built exactly like standalone Sanadv1:
no tools, no state injections, persona only, no lip-sync markers the
fastest conversation pipe. Resolution: dashboard > env > config."""
import os
opts = _read_voice_opts()
if "pure_voice" in opts:
return {"enabled": bool(opts["pure_voice"]), "source": "dashboard"}
if os.environ.get("SANAD_PURE_VOICE", "").strip().lower() in ("1", "true", "yes"):
return {"enabled": True, "source": "env"}
try:
from Project.Sanad.core.config_loader import section
if bool((section("voice", "sanad_voice") or {}).get("pure_voice", False)):
return {"enabled": True, "source": "config"}
except Exception:
pass
return {"enabled": False, "source": "default"}
@router.post("/pure-voice-option")
async def set_pure_voice_option(on: bool = Query(...)):
"""Persist the pure-voice option and restart the live session."""
import json
import tempfile
import os as _os
opts = _read_voice_opts()
opts["pure_voice"] = bool(on)
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
prefix=".voice_options.", suffix=".tmp")
with _os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(opts, f, indent=2)
_os.replace(tmp, _VOICE_OPTS_PATH)
restarted = False
from Project.Sanad.main import live_sub
if live_sub is not None:
try:
if (live_sub.status() or {}).get("running"):
await asyncio.to_thread(live_sub.stop)
await asyncio.to_thread(live_sub.start)
restarted = True
except Exception:
pass
return {"ok": True, "enabled": bool(on), "restarted": restarted}
@router.get("/internet-search-option")
async def get_internet_search_option():
"""INTERNET SEARCH = the live session gets the search_internet tool so Gemini
can look up current facts (prices, latest phones, live offers, news) via
Google Search grounding, announcing the search first. Resolution:
dashboard > env > config."""
import os
opts = _read_voice_opts()
if "internet_search" in opts:
return {"enabled": bool(opts["internet_search"]), "source": "dashboard"}
if os.environ.get("SANAD_INTERNET_SEARCH", "").strip().lower() in ("1", "true", "yes"):
return {"enabled": True, "source": "env"}
try:
from Project.Sanad.core.config_loader import section
if bool((section("voice", "sanad_voice") or {}).get("internet_search", False)):
return {"enabled": True, "source": "config"}
except Exception:
pass
return {"enabled": False, "source": "default"}
@router.post("/internet-search-option")
async def set_internet_search_option(on: bool = Query(...)):
"""Persist the internet-search option and restart the live session so the
tool set (and persona addendum) is rebaked."""
import json
import tempfile
import os as _os
opts = _read_voice_opts()
opts["internet_search"] = bool(on)
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
prefix=".voice_options.", suffix=".tmp")
with _os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(opts, f, indent=2)
_os.replace(tmp, _VOICE_OPTS_PATH)
restarted = False
from Project.Sanad.main import live_sub
if live_sub is not None:
try:
if (live_sub.status() or {}).get("running"):
await asyncio.to_thread(live_sub.stop)
await asyncio.to_thread(live_sub.start)
restarted = True
except Exception:
pass
return {"ok": True, "enabled": bool(on), "restarted": restarted}
@router.post("/answer-now")
async def answer_now():
"""Force Gemini to reply immediately instead of waiting out the pause
detection (no-op while it is already speaking)."""
sub = _sub_or_503()
if not (sub.status() or {}).get("running"):
raise HTTPException(409, "Live session not running")
if not hasattr(sub, "send_answer_now"):
raise HTTPException(501, "engine build lacks send_answer_now")
await asyncio.to_thread(sub.send_answer_now)
return {"ok": True}
@router.post("/interrupt")
async def interrupt_speech():
"""Instantly stop the AI mid-sentence and go back to listening (works
regardless of the barge-in option; no-op if it isn't speaking)."""
sub = _sub_or_503()
if not (sub.status() or {}).get("running"):
raise HTTPException(409, "Live session not running")
if not hasattr(sub, "send_interrupt"):
raise HTTPException(501, "engine build lacks send_interrupt")
await asyncio.to_thread(sub.send_interrupt)
return {"ok": True}
@router.post("/start") @router.post("/start")
async def start_subprocess(): async def start_subprocess():
live_sub = _sub_or_503() live_sub = _sub_or_503()

View File

@ -4,7 +4,7 @@ from __future__ import annotations
import asyncio import asyncio
from fastapi import APIRouter, HTTPException from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel from pydantic import BaseModel
router = APIRouter() router = APIRouter()
@ -50,7 +50,19 @@ async def trigger_action(payload: TriggerPayload):
from Project.Sanad.main import arm from Project.Sanad.main import arm
if arm is None: if arm is None:
raise HTTPException(503, "Arm controller not attached.") raise HTTPException(503, "Arm controller not attached.")
_block_if_movement_armed()
# Only the legacy JSONL replays (direct rt/arm_sdk streaming) conflict with
# the walker. R1 preset gestures go through the arm service, which moves the
# arms while the firmware balances — so those are allowed while movement is
# armed. Gate JSONL only.
acts = arm.list_actions() if hasattr(arm, "list_actions") else []
resolved = None
if payload.action_id is not None:
resolved = next((a for a in acts if a.get("id") == payload.action_id), None)
elif payload.action_name:
resolved = next((a for a in acts if a.get("name") == payload.action_name), None)
if resolved and resolved.get("file"):
_block_if_movement_armed()
speed = max(0.1, min(payload.speed, 5.0)) speed = max(0.1, min(payload.speed, 5.0))
@ -73,6 +85,30 @@ async def trigger_action(payload: TriggerPayload):
raise HTTPException(400, "Provide action_id or action_name.") raise HTTPException(400, "Provide action_id or action_name.")
# ── R1 head-look menu (recorded teach motions via the arm service) ──────────
@router.get("/head-actions")
async def head_actions():
"""The recorded head-look names for the Controller-tab HEAD menu."""
from Project.Sanad.main import arm
return {"looks": arm.head_looks() if arm else []}
@router.post("/head")
async def head_look(name: str = Query(...)):
"""Replay a recorded head rotation (R1 arm-service teach action). NOT gated by
the movement lock: the R1 arm service moves the head while the firmware keeps
the body balanced (that is the whole point of the canteen pattern), so head
looks are safe and useful while locomotion/teleop is armed."""
from Project.Sanad.main import arm
if arm is None:
raise HTTPException(503, "Arm controller not attached.")
res = await asyncio.to_thread(arm.run_head_look, name)
if not res.get("ok"):
raise HTTPException(502, res.get("error") or f"head look failed (rc={res.get('rc')})")
return res
@router.post("/cancel") @router.post("/cancel")
async def cancel_motion(): async def cancel_motion():
from Project.Sanad.main import arm from Project.Sanad.main import arm

View File

@ -173,10 +173,12 @@ async def update_api_key(payload: ApiKeyPayload):
raise HTTPException(400, "API key cannot be empty.") raise HTTPException(400, "API key cannot be empty.")
if len(key) < 20: if len(key) < 20:
raise HTTPException(400, "API key looks too short.") raise HTTPException(400, "API key looks too short.")
if not key.startswith("AIza"): # Google AI Studio issues classic "AIza..." keys and (since late 2025)
# new-format "AQ...." keys — both are plain x-goog-api-key API keys.
if not key.startswith(("AIza", "AQ.")):
raise HTTPException( raise HTTPException(
400, 400,
"Gemini API keys normally start with 'AIza'. " "Gemini API keys start with 'AIza' (classic) or 'AQ.' (new format). "
"Double-check you're pasting a Google AI Studio key.", "Double-check you're pasting a Google AI Studio key.",
) )

View File

@ -352,7 +352,7 @@
</div> </div>
<div id="running-action" style="font-size:.75rem;color:var(--accent);margin-bottom:.3rem;display:none"></div> <div id="running-action" style="font-size:.75rem;color:var(--accent);margin-bottom:.3rem;display:none"></div>
<div id="sdk-actions" style="display:flex;flex-wrap:wrap;gap:3px;margin-top:.2rem"></div> <div id="sdk-actions" style="display:flex;flex-wrap:wrap;gap:3px;margin-top:.2rem"></div>
<div id="jsonl-actions" style="display:flex;flex-wrap:wrap;gap:3px;margin-top:.4rem"></div> <div id="jsonl-actions" style="display:none;flex-wrap:wrap;gap:3px;margin-top:.4rem"></div>
</div> </div>
</div> </div>
@ -401,9 +401,15 @@
<div class="row"> <div class="row">
<button class="btn btn-success" onclick="startLiveSub(this)">Start</button> <button class="btn btn-success" onclick="startLiveSub(this)">Start</button>
<button class="btn btn-danger" onclick="stopLiveSub(this)">Stop</button> <button class="btn btn-danger" onclick="stopLiveSub(this)">Stop</button>
<button class="btn" style="background:#7c3aed;color:#fff" onclick="instantInterrupt()" title="Instantly stop Sanad mid-sentence and go back to listening (works even with the Interrupt voice option off).">⏹ Interrupt now</button>
<button class="btn" style="background:#0e9f6e;color:#fff" onclick="answerNow()" title="Make Sanad answer RIGHT NOW instead of waiting for the end-of-speech pause.">➤ Answer now</button>
<button id="ls-cam-btn" class="btn btn-sm btn-ghost" onclick="toggleGeminiCamera(this)" title="Stream camera frames to Gemini Live — same toggle as the Recognition tab">Camera: --</button> <button id="ls-cam-btn" class="btn btn-sm btn-ghost" onclick="toggleGeminiCamera(this)" title="Stream camera frames to Gemini Live — same toggle as the Recognition tab">Camera: --</button>
<button id="ls-rec-btn" class="btn btn-sm btn-ghost" onclick="toggleAutoRecord(this)" title="Auto-save every conversation turn to data/recordings/. Toggling takes effect live — no session restart.">Rec: --</button> <button id="ls-rec-btn" class="btn btn-sm btn-ghost" onclick="toggleAutoRecord(this)" title="Auto-save every conversation turn to data/recordings/. Toggling takes effect live — no session restart.">Rec: --</button>
<span id="ls-state" class="badge"></span> <span id="ls-state" class="badge"></span>
<span id="ls-pure" class="badge" onclick="togglePureOpt()" style="cursor:pointer" title="Pure voice: fastest replies (Sanadv1-style — no tools, no standing state, persona only). TOGGLE-AWARE: features you enable (Camera vision, Movement) still work live and announce themselves; disable them and it returns to pure speech.">Pure: …</span>
<span id="ls-direct" class="badge" onclick="toggleDirectOpt()" style="cursor:pointer" title="Direct answer: Sanad replies the moment you pause (fast turn-taking). OFF = waits longer to be sure you finished (better for slow/hesitant speakers). Toggling restarts the live session.">Direct: …</span>
<span id="ls-interrupt" class="badge" onclick="toggleInterruptOpt()" style="cursor:pointer" title="Voice interrupt (barge-in) — ALL speakers. OFF: Sanad always finishes his sentence, nobody can talk over him (R1 chest / JBL / Anker all behave the same). ON: loud sustained speech (~1s) cuts him mid-sentence. Echo-safe either way; the ⏹ Interrupt-now button always works. Toggling restarts the live session.">Interrupt: …</span>
<span id="ls-search" class="badge" onclick="toggleSearchOpt()" style="cursor:pointer" title="Internet search (Google grounding): Sanad can look up CURRENT facts — today's prices, the latest phones, live offers, news. When he needs to, he says 'let me search the internet', looks it up, then answers. OFF = pure voice, no search, fastest replies (recommended for a busy event). Toggling restarts the live session.">Search: …</span>
<span id="ls-pausemode" class="badge" onclick="toggleLiveHoldBadge()" style="cursor:pointer" title="Pause mode (same as the Saved Records 'Keep Gemini paused' toggle). Auto = pause only during a record, resume after. Manual = Gemini stays paused until you switch back. Click to toggle.">Pause: Auto</span> <span id="ls-pausemode" class="badge" onclick="toggleLiveHoldBadge()" style="cursor:pointer" title="Pause mode (same as the Saved Records 'Keep Gemini paused' toggle). Auto = pause only during a record, resume after. Manual = Gemini stays paused until you switch back. Click to toggle.">Pause: Auto</span>
<button class="btn btn-sm mic-mute-shortcut btn-success" onclick="toggleMic()" style="margin-left:auto">Mic: LIVE</button> <button class="btn btn-sm mic-mute-shortcut btn-success" onclick="toggleMic()" style="margin-left:auto">Mic: LIVE</button>
<button class="btn btn-sm spk-mute-shortcut btn-success" onclick="toggleSpeaker()">Speaker: LIVE</button> <button class="btn btn-sm spk-mute-shortcut btn-success" onclick="toggleSpeaker()">Speaker: LIVE</button>
@ -504,7 +510,7 @@
</div> </div>
<div id="sdk-actions-2" class="action-list"></div> <div id="sdk-actions-2" class="action-list"></div>
</div> </div>
<div style="flex:1;min-width:260px"> <div style="flex:1;min-width:260px;display:none">
<div class="row" style="justify-content:space-between;align-items:center;margin:0 0 .3rem 0"> <div class="row" style="justify-content:space-between;align-items:center;margin:0 0 .3rem 0">
<label style="margin:0">JSONL Replays (recorded)</label> <label style="margin:0">JSONL Replays (recorded)</label>
<button id="play-jsonl-btn" class="btn btn-primary btn-sm" onclick="playSelectedAction('jsonl')" disabled>Play</button> <button id="play-jsonl-btn" class="btn btn-primary btn-sm" onclick="playSelectedAction('jsonl')" disabled>Play</button>
@ -883,6 +889,18 @@
<h3>Diagnostics — joints 1228</h3> <h3>Diagnostics — joints 1228</h3>
<pre id="ctrl-joints" style="font-size:.66rem;max-height:240px;overflow:auto;background:var(--panel2);border-radius:6px;padding:.5rem;margin:0"></pre> <pre id="ctrl-joints" style="font-size:.66rem;max-height:240px;overflow:auto;background:var(--panel2);border-radius:6px;padding:.5rem;margin:0"></pre>
</div> </div>
<!-- Head — recorded looks (R1 arm-service teach motions) -->
<div class="card">
<h3>Head — recorded looks</h3>
<div style="font-size:.7rem;color:var(--muted);margin-bottom:.45rem">
Replays your recorded head rotations via the R1 arm service. Firmware keeps the body balanced.
Gated by the same movement lock as arm actions.
</div>
<div class="row" id="ctrl-head-menu" style="flex-wrap:wrap;gap:.3rem">
<span class="empty">loading…</span>
</div>
</div>
</div> </div>
</div> </div>
@ -1617,7 +1635,10 @@ async function renderActions(arm){
_armBusy=arm.busy||false; _armBusy=arm.busy||false;
try{ try{
const r=await api('GET','/api/motion/actions'); const r=await api('GET','/api/motion/actions');
const acts=r.actions||[]; // R1 gestures only — the JSONL replay buttons are dropped (arm control now
// goes through the R1 'arm' service; recorded head looks live in the HEAD
// menu on the Controller tab).
const acts=(r.actions||[]).filter(a=>!a.file);
_renderChips(acts); _renderChips(acts);
_renderList(acts); _renderList(acts);
['','2'].forEach(sfx=>{const bb=document.getElementById('arm-busy-badge'+sfx);if(bb)bb.style.display=_armBusy?'inline-flex':'none';}); ['','2'].forEach(sfx=>{const bb=document.getElementById('arm-busy-badge'+sfx);if(bb)bb.style.display=_armBusy?'inline-flex':'none';});
@ -2014,6 +2035,92 @@ async function reassertLiveHold(){try{const s=await api('GET','/api/records/play
async function stopLiveSub(b){btnLoad(b);try{await api('POST','/api/live-subprocess/stop');toast('Stopped','info');}catch(e){}btnDone(b);refreshLiveSub();} async function stopLiveSub(b){btnLoad(b);try{await api('POST','/api/live-subprocess/stop');toast('Stopped','info');}catch(e){}btnDone(b);refreshLiveSub();}
async function refreshLiveSub(){try{const r=await api('GET','/api/live-subprocess/status');const st=document.getElementById('ls-state');st.textContent=r.state||'stopped';st.className='badge '+(r.running?'badge-ok':'badge-warn');document.getElementById('ls-msg').textContent=r.state_message||'--';document.getElementById('ls-user').textContent=r.last_user_text||'--';document.getElementById('ls-log').textContent=(r.log_tail||[]).slice(-25).join('\n');const rb=document.getElementById('ls-rec-btn');if(rb){const on=!!r.record_enabled;rb.textContent='Rec: '+(on?'ON':'OFF');rb.className='btn btn-sm '+(on?'btn-success':'btn-ghost');}}catch(e){}} async function refreshLiveSub(){try{const r=await api('GET','/api/live-subprocess/status');const st=document.getElementById('ls-state');st.textContent=r.state||'stopped';st.className='badge '+(r.running?'badge-ok':'badge-warn');document.getElementById('ls-msg').textContent=r.state_message||'--';document.getElementById('ls-user').textContent=r.last_user_text||'--';document.getElementById('ls-log').textContent=(r.log_tail||[]).slice(-25).join('\n');const rb=document.getElementById('ls-rec-btn');if(rb){const on=!!r.record_enabled;rb.textContent='Rec: '+(on?'ON':'OFF');rb.className='btn btn-sm '+(on?'btn-success':'btn-ghost');}}catch(e){}}
async function toggleAutoRecord(b){const cur=(b&&b.textContent||'').includes('ON');const next=!cur;btnLoad(b);try{const r=await api('POST','/api/live-subprocess/record?on='+(next?'1':'0'));toast('Auto-recording '+(r.record_enabled?'ON':'OFF'),'ok');}catch(e){toast('Toggle failed: '+(e.message||e),'err');}btnDone(b);refreshLiveSub();} async function toggleAutoRecord(b){const cur=(b&&b.textContent||'').includes('ON');const next=!cur;btnLoad(b);try{const r=await api('POST','/api/live-subprocess/record?on='+(next?'1':'0'));toast('Auto-recording '+(r.record_enabled?'ON':'OFF'),'ok');}catch(e){toast('Toggle failed: '+(e.message||e),'err');}btnDone(b);refreshLiveSub();}
// Voice-option badges (Pure / Search / Direct / Interrupt) + instant buttons —
// ported from Sanad G1; backed by /api/live-subprocess/*-option endpoints.
let _pureOpt=null;
function _paintPureBadge(){
const b=document.getElementById('ls-pure');
if(!b)return;
b.textContent='Pure: '+(_pureOpt===null?'…':(_pureOpt?'ON':'OFF'));
b.className='badge '+(_pureOpt?'badge-ok':'');
}
async function refreshPureOpt(){
try{const r=await api('GET','/api/live-subprocess/pure-voice-option');_pureOpt=!!r.enabled;_paintPureBadge();}catch(e){}
}
async function togglePureOpt(){
const want=!_pureOpt;
try{
const r=await api('POST','/api/live-subprocess/pure-voice-option?on='+want);
_pureOpt=!!r.enabled;_paintPureBadge();
toast('Pure voice '+(_pureOpt?'ON — Sanadv1-style session':'OFF — full features')+(r.restarted?' (session restarted)':''));
}catch(e){toast('Pure toggle failed: '+(e.message||e));}
}
setTimeout(refreshPureOpt,1000);
let _searchOpt=null;
function _paintSearchBadge(){
const b=document.getElementById('ls-search');
if(!b)return;
b.textContent='Search: '+(_searchOpt===null?'…':(_searchOpt?'ON':'OFF'));
b.className='badge '+(_searchOpt?'badge-ok':'');
}
async function refreshSearchOpt(){
try{const r=await api('GET','/api/live-subprocess/internet-search-option');_searchOpt=!!r.enabled;_paintSearchBadge();}catch(e){}
}
async function toggleSearchOpt(){
const want=!_searchOpt;
try{
const r=await api('POST','/api/live-subprocess/internet-search-option?on='+want);
_searchOpt=!!r.enabled;_paintSearchBadge();
toast('Internet search '+(_searchOpt?'ON — Sanad can look up current info':'OFF — pure voice, no search')+(r.restarted?' (session restarted)':''));
}catch(e){toast('Search toggle failed: '+(e.message||e));}
}
setTimeout(refreshSearchOpt,1050);
let _dirOpt=null;
function _paintDirectBadge(){
const b=document.getElementById('ls-direct');
if(!b)return;
b.textContent='Direct: '+(_dirOpt===null?'…':(_dirOpt?'ON':'OFF'));
b.className='badge '+(_dirOpt?'badge-ok':'');
}
async function refreshDirectOpt(){
try{const r=await api('GET','/api/live-subprocess/direct-answer-option');_dirOpt=!!r.enabled;_paintDirectBadge();}catch(e){}
}
async function toggleDirectOpt(){
const want=!_dirOpt;
try{
const r=await api('POST','/api/live-subprocess/direct-answer-option?on='+want);
_dirOpt=!!r.enabled;_paintDirectBadge();
toast('Direct answer '+(_dirOpt?'ON':'OFF')+(r.restarted?' — live session restarted':''));
}catch(e){toast('Direct toggle failed: '+(e.message||e));}
}
setTimeout(refreshDirectOpt,950);
let _intOpt=null;
function _paintInterruptBadge(){
const b=document.getElementById('ls-interrupt');
if(!b)return;
b.textContent='Interrupt: '+(_intOpt===null?'…':(_intOpt?'ON':'OFF'));
b.className='badge '+(_intOpt?'badge-ok':'');
}
async function refreshInterruptOpt(){
try{const r=await api('GET','/api/live-subprocess/interrupt-option');_intOpt=!!r.enabled;_paintInterruptBadge();}catch(e){}
}
async function answerNow(){
try{await api('POST','/api/live-subprocess/answer-now');toast('Answering now…');}
catch(e){toast('Answer-now failed: '+(e.message||e));}
}
async function instantInterrupt(){
try{await api('POST','/api/live-subprocess/interrupt');toast('Interrupted — listening');}
catch(e){toast('Interrupt failed: '+(e.message||e));}
}
async function toggleInterruptOpt(){
const want=!_intOpt;
try{
const r=await api('POST','/api/live-subprocess/interrupt-option?on='+want);
_intOpt=!!r.enabled;_paintInterruptBadge();
toast('Interrupt '+(_intOpt?'ON':'OFF')+(r.restarted?' — live session restarted':''));
}catch(e){toast('Interrupt toggle failed: '+(e.message||e));}
}
setTimeout(refreshInterruptOpt,900);
// Typed Replay // Typed Replay
async function trGenerate(b){const t=document.getElementById('tr-text').value;if(!t)return toast('Enter text','err');btnLoad(b);try{await api('POST','/api/typed-replay/say',{text:t,record:document.getElementById('tr-capture').checked,record_name:document.getElementById('tr-name').value});toast('Generated & played','ok');refreshTR();}catch(e){}btnDone(b);} async function trGenerate(b){const t=document.getElementById('tr-text').value;if(!t)return toast('Enter text','err');btnLoad(b);try{await api('POST','/api/typed-replay/say',{text:t,record:document.getElementById('tr-capture').checked,record_name:document.getElementById('tr-name').value});toast('Generated & played','ok');refreshTR();}catch(e){}btnDone(b);}
@ -2365,7 +2472,31 @@ function stopRecPreview(){
img.src=''; // closes the MJPEG connection img.src=''; // closes the MJPEG connection
} }
} }
// Hook into tab switch — start/stop preview when recognition tab is active. // R1 head-look menu (Controller tab) — replays your recorded head rotations via
// the R1 arm service (ExecuteCustomAction). Loaded once when the tab opens.
let _ctrlHeadLoaded=false;
async function ctrlLoadHead(){
const el=document.getElementById('ctrl-head-menu'); if(!el) return;
try{
const r=await api('GET','/api/motion/head-actions');
const looks=r.looks||[];
const label={up_look:'↑ up',down_look:'↓ down',left_look:'← left',right_look:'→ right',
up_down_look:'↕ up-down',left_right_look:'⇄ left-right',right_left_look:'⇄ right-left',center:'⊙ center'};
el.innerHTML=looks.length
? looks.map(n=>`<button class="btn btn-ghost btn-sm" onclick="ctrlHeadLook('${esc(n)}',this)">${label[n]||esc(n).replace(/_/g,' ')}</button>`).join('')
: '<span class="empty">No recorded head looks</span>';
_ctrlHeadLoaded=true;
}catch(e){ el.innerHTML='<span class="empty">head menu unavailable</span>'; }
}
async function ctrlHeadLook(name,btn){
if(btn) btnLoad(btn);
try{ await api('POST','/api/motion/head?name='+encodeURIComponent(name)); toast('Head → '+name.replace(/_/g,' '),'ok'); }
catch(e){ toast('Head look failed: '+(e.message||e),'err'); }
if(btn) btnDone(btn);
}
// Hook into tab switch — start/stop preview when recognition tab is active,
// and lazy-load the Controller head menu on first open.
(function(){ (function(){
const origSwitchTab=window.switchTab; const origSwitchTab=window.switchTab;
window.switchTab=function(name){ window.switchTab=function(name){
@ -2373,6 +2504,7 @@ function stopRecPreview(){
_recTabActive=(name==='recognition'); _recTabActive=(name==='recognition');
if(name==='recognition'){refreshRecognition();refreshFaces();refreshZones();startRecPreview();} if(name==='recognition'){refreshRecognition();refreshFaces();refreshZones();startRecPreview();}
else{stopRecPreview();} else{stopRecPreview();}
if(name==='controller'&&!_ctrlHeadLoaded) ctrlLoadHead();
}; };
})(); })();
// Face CRUD stubs — filled in milestone 5 // Face CRUD stubs — filled in milestone 5

View File

@ -55,11 +55,23 @@ _MODEL = os.environ.get(
"SANAD_GEMINI_MODEL", "SANAD_GEMINI_MODEL",
"gemini-2.5-flash-native-audio-preview-12-2025", "gemini-2.5-flash-native-audio-preview-12-2025",
) )
_MIC_GAIN = _SV.get("mic_gain", 1.0) # Mic gain: amplifies mic samples before they reach Gemini — the single
# biggest lever for slow/failed speech detection at a distance (weak audio
# transcribes but often yields NO reply turn). Env override for per-robot
# tuning; watch for clipping if the speaker stands close (>3.0 rarely wise).
try:
_MIC_GAIN = float(os.environ.get("SANAD_MIC_GAIN", "") or _SV.get("mic_gain", 1.0))
except ValueError:
_MIC_GAIN = _SV.get("mic_gain", 1.0)
_SESSION_TIMEOUT = _SV.get("session_timeout_sec", 660) _SESSION_TIMEOUT = _SV.get("session_timeout_sec", 660)
_MAX_RECONNECT_DELAY = _SV.get("max_reconnect_delay_sec", 30) _MAX_RECONNECT_DELAY = _SV.get("max_reconnect_delay_sec", 30)
_MAX_CONSECUTIVE_ERRORS = _SV.get("max_consecutive_errors", 10) _MAX_CONSECUTIVE_ERRORS = _SV.get("max_consecutive_errors", 10)
_NO_MESSAGES_TIMEOUT = _SV.get("no_messages_timeout_sec", 30) # How long with ZERO messages from Gemini before we declare the session dead.
# In a QUIET room Gemini legitimately sends nothing — 30s was killing healthy
# sessions every ~40s (reconnect churn + unsolicited greeting + a 1-3s deaf
# window each time). REAL deaths are caught fast by the send path ("mic send
# failed: 1011 ..."), so this can be patient.
_NO_MESSAGES_TIMEOUT = _SV.get("no_messages_timeout_sec", 180)
# Extra mic-gate time after the AI stops, on loud external-speaker profiles # Extra mic-gate time after the AI stops, on loud external-speaker profiles
# (JBL) — covers the speaker buffer + room reverb so it doesn't hear its tail. # (JBL) — covers the speaker buffer + room reverb so it doesn't hear its tail.
_ECHO_TAIL_SEC = _SV.get("echo_tail_sec", 0.6) _ECHO_TAIL_SEC = _SV.get("echo_tail_sec", 0.6)
@ -77,6 +89,70 @@ _JBL_BARGE_CHUNKS = _SV.get("jbl_barge_chunks", 9)
# FADED (a gap between words/numbers). In that window barge-in drops to a low, # FADED (a gap between words/numbers). In that window barge-in drops to a low,
# sensitive bar so the user can interrupt; while audio is flowing it stays high. # sensitive bar so the user can interrupt; while audio is flowing it stays high.
_JBL_BLEED_FADE_SEC = _SV.get("jbl_bleed_fade_sec", 0.5) _JBL_BLEED_FADE_SEC = _SV.get("jbl_bleed_fade_sec", 0.5)
# OPTION: voice interrupt (barge-in) — GLOBAL, all speakers. OFF (default):
# Sanad always finishes his sentence — the mic to Gemini is fully gated while
# he speaks (+ echo tail) on EVERY profile (chest/Anker/JBL/Beats Pill), so
# neither the local energy detector nor Gemini's server VAD can cut him.
# ON: loud sustained speech interrupts him — builtin/Anker use the normal
# barge parameters; loud external speakers (JBL / Beats Pill) use the stricter
# external parameters and stay fully gated for echo safety. The dashboard
# ⏹ Interrupt-now button works regardless. Env > config; default OFF.
_EXTERNAL_BARGE_IN = (
os.environ.get("SANAD_EXTERNAL_BARGE_IN", "").strip().lower() in ("1", "true", "yes")
or bool(_BI.get("external_barge_in", False))
)
# OPTION: "direct answer" — reply as soon as the speaker pauses. Flips the
# Live-API end-of-speech sensitivity from LOW (conservative: waits to be sure
# you finished, feels slow) to HIGH (immediate turn-taking; may cut off very
# hesitant speakers mid-thought). Env > config; dashboard file has final say.
_DIRECT_ANSWER = (
os.environ.get("SANAD_DIRECT_ANSWER", "").strip().lower() in ("1", "true", "yes")
or bool(_VAD.get("direct_answer", False))
)
# OPTION: PURE VOICE — make the live session EXACTLY like standalone Sanadv1:
# no tools (no function-calling deliberation), no state injections, persona
# only, no lip-sync markers, no frame/state loops. The fastest possible
# conversation pipe; nav/vision hooks are dormant.
_PURE_VOICE = (
os.environ.get("SANAD_PURE_VOICE", "").strip().lower() in ("1", "true", "yes")
or bool(_SV.get("pure_voice", False))
)
# OPTION: INTERNET SEARCH (Google Search grounding). When ON, the live session
# gets ONE extra tool, `search_internet(query)`, so the model can look up
# CURRENT facts (today's prices, the newest phones, live Etisalat offers, news).
# The tool description tells the model to SAY it will check first, then call the
# tool — so the robot announces "let me search the internet", the parent runs
# the grounded generateContent call in a worker thread, and the model speaks the
# result. Default OFF → pure voice stays tool-free and Sanadv1-fast; only when
# a search actually fires does the (announced) grounding delay occur.
_INTERNET_SEARCH = (
os.environ.get("SANAD_INTERNET_SEARCH", "").strip().lower() in ("1", "true", "yes")
or bool(_SV.get("internet_search", False))
)
# Model used for the grounded web-search call (generateContent, NOT the live
# native-audio model). gemini-3.5-flash grounds well and cites the source site.
_SEARCH_MODEL = os.environ.get("SANAD_SEARCH_MODEL", "gemini-3.5-flash")
# The dashboard toggles (data/.voice_options.json, written by the
# /api/live-subprocess/*-option endpoints, which then restart this child)
# have the FINAL say over env/config.
try:
import json as _vo_json
with open(BASE_DIR / "data" / ".voice_options.json", encoding="utf-8") as _vo_f:
_vo = _vo_json.load(_vo_f)
if isinstance(_vo, dict):
if "external_barge_in" in _vo:
_EXTERNAL_BARGE_IN = bool(_vo["external_barge_in"])
if "direct_answer" in _vo:
_DIRECT_ANSWER = bool(_vo["direct_answer"])
if "pure_voice" in _vo:
_PURE_VOICE = bool(_vo["pure_voice"])
if "internet_search" in _vo:
_INTERNET_SEARCH = bool(_vo["internet_search"])
except Exception:
pass
_CHUNK_BYTES = CHUNK_SIZE * 2 _CHUNK_BYTES = CHUNK_SIZE * 2
_SILENCE_PCM = b"\x00" * _CHUNK_BYTES _SILENCE_PCM = b"\x00" * _CHUNK_BYTES
@ -88,6 +164,15 @@ _SILENCE_PCM = b"\x00" * _CHUNK_BYTES
# safe to read from the asyncio loops. # safe to read from the asyncio loops.
_INPUT_PAUSED = threading.Event() _INPUT_PAUSED = threading.Event()
# Set by an "interrupt:" stdin command (the dashboard's Interrupt-now button).
# The mic loop consumes it: stops the speaker mid-sentence, back to listening.
_MANUAL_INTERRUPT = threading.Event()
# Set by an "answernow:" stdin command (the dashboard's Answer-now button).
# The mic loop feeds ~0.6s of silence so the server VAD closes the user's turn
# immediately → Gemini answers right away instead of waiting out the pause.
_FORCE_ANSWER = threading.Event()
# ── Recognition (camera + face gallery) tunables ── # ── Recognition (camera + face gallery) tunables ──
_RECOG_STATE_PATH = Path(os.environ.get( _RECOG_STATE_PATH = Path(os.environ.get(
"SANAD_RECOGNITION_STATE_PATH", "SANAD_RECOGNITION_STATE_PATH",
@ -195,6 +280,76 @@ def _nav_function_declarations() -> list:
] ]
def _search_function_declaration() -> list:
"""Live tool: search the internet (Google Search grounding) for current facts."""
S, T = types.Schema, types.Type
return [
types.FunctionDeclaration(
name="search_internet",
description=(
"Search the LIVE internet for CURRENT, real-time facts you cannot "
"otherwise know: today's prices, the newest phone / device models, "
"current Etisalat / e& offers and promotions, product availability, "
"or recent news. IMPORTANT: BEFORE you call this, first SAY one "
"short sentence out loud telling the user you will check — e.g. "
"\"Let me search the internet for that.\" / "
"\"دعني أبحث لك على الإنترنت.\""
"then call this with a clear English search query. After the "
"result returns, answer the user naturally from it. Do NOT say the "
"tool name out loud. Only use this for genuinely current / "
"time-sensitive questions, not for things you already know."
),
parameters=S(type=T.OBJECT, properties={
"query": S(type=T.STRING,
description="The web-search query, phrased in English."),
}, required=["query"]),
),
]
# Appended to the persona ONLY when internet search is enabled, so the model
# knows it can (and how to) look things up. Kept short — the tool description
# carries the detail.
_SEARCH_PROMPT_ADDENDUM = (
"\n\nYou can look things up on the live internet with the search_internet "
"tool. Use it whenever the user asks about current prices, the latest phones "
"or devices, today's Etisalat/e& offers, availability, or recent news — "
"anything you can't be certain is up to date. ALWAYS say a short 'let me "
"check the internet' line first (in the user's language), then call the tool, "
"then answer from the result. Keep answers concise and, for exact prices, add "
"that a sales advisor can confirm the final figure. "
"CRITICAL: after the search result returns, CONTINUE where you left off — "
"NEVER repeat or re-say any sentence you already spoke, including the 'let me "
"check the internet' lead-in. If your reply is complete, stop and wait."
)
def _run_grounding_search(query: str, api_key: str) -> dict:
"""Blocking Google-Search-grounded generateContent call (run in a thread).
Returns {ok, answer} for the model to speak, or {ok: False, reason}."""
try:
client = genai.Client(api_key=api_key)
resp = client.models.generate_content(
model=_SEARCH_MODEL,
contents=query,
config=types.GenerateContentConfig(
tools=[types.Tool(google_search=types.GoogleSearch())],
),
)
answer = (getattr(resp, "text", "") or "").strip()
if not answer:
return {"ok": False, "reason": "empty_result"}
# Cap length so the spoken reply stays reasonable.
return {"ok": True, "answer": answer[:1500]}
except Exception as exc:
log.warning("internet search failed: %s", exc)
return {"ok": False, "reason": "search_failed", "detail": str(exc)[:200]}
# NOTE (SanadR1): the LED-mask face/social-QR tools (_FACE_EMOTIONS,
# set_expression, show_social) were removed in the R1 port — the R1 has no mask.
# ── stdin push channel (Marcus pattern) ────────────────────── # ── stdin push channel (Marcus pattern) ──────────────────────
# The GeminiSubprocess supervisor writes two line types to this process's # The GeminiSubprocess supervisor writes two line types to this process's
# stdin: # stdin:
@ -233,7 +388,7 @@ _PROFILE_LOCK = threading.Lock()
_PROFILE_PENDING: dict = {"id": None, "reason": ""} _PROFILE_PENDING: dict = {"id": None, "reason": ""}
_VALID_PROFILES = {"builtin", "anker", "anker_powerconf", _VALID_PROFILES = {"builtin", "anker", "anker_powerconf",
"hollyland_builtin", "jbl_builtin_mic"} "hollyland_builtin", "jbl_builtin_mic", "beats_pill"}
def _stdin_watcher() -> None: def _stdin_watcher() -> None:
@ -266,6 +421,10 @@ def _stdin_watcher() -> None:
with _LATEST_FRAME_LOCK: with _LATEST_FRAME_LOCK:
_LATEST_FRAME["bytes"] = data _LATEST_FRAME["bytes"] = data
_LATEST_FRAME["ts"] = time.time() _LATEST_FRAME["ts"] = time.time()
elif line.startswith("interrupt:"):
_MANUAL_INTERRUPT.set()
elif line.startswith("answernow:"):
_FORCE_ANSWER.set()
elif line.startswith("state:"): elif line.startswith("state:"):
try: try:
payload = json.loads(line[len("state:"):]) payload = json.loads(line[len("state:"):])
@ -349,7 +508,12 @@ class GeminiBrain:
self._swap_lock: Optional[asyncio.Lock] = None # built in run() self._swap_lock: Optional[asyncio.Lock] = None # built in run()
self._recorder = recorder self._recorder = recorder
self._voice = voice_name or GEMINI_VOICE self._voice = voice_name or GEMINI_VOICE
# R1: no LED-mask face, so no face addendum in any mode.
self._system_prompt = system_prompt or "" self._system_prompt = system_prompt or ""
# Internet-search addendum is independent of pure voice — the search
# tool is available in either mode when the option is enabled.
if _INTERNET_SEARCH:
self._system_prompt += _SEARCH_PROMPT_ADDENDUM
self._api_key = GEMINI_API_KEY self._api_key = GEMINI_API_KEY
self._stop_flag = asyncio.Event() self._stop_flag = asyncio.Event()
# per-session state (reset in the outer reconnect loop) # per-session state (reset in the outer reconnect loop)
@ -443,6 +607,10 @@ class GeminiBrain:
if self._swap_lock is None: if self._swap_lock is None:
self._swap_lock = asyncio.Lock() self._swap_lock = asyncio.Lock()
# PURE VOICE is TOGGLE-AWARE: all loops run (cheap when
# their feature is off) — pure-ness is enforced per-feature
# (announce guards + tools) so enabling Camera/Movement in
# the dashboard works live, and disabling returns to pure.
try: try:
await asyncio.wait_for( await asyncio.wait_for(
asyncio.gather( asyncio.gather(
@ -505,10 +673,15 @@ class GeminiBrain:
types.StartSensitivity, types.StartSensitivity,
_VAD.get("start_sensitivity", "START_SENSITIVITY_HIGH"), _VAD.get("start_sensitivity", "START_SENSITIVITY_HIGH"),
), ),
end_of_speech_sensitivity=getattr( # "Direct answer" option: HIGH = reply the moment the
types.EndSensitivity, # speaker pauses; otherwise the configured (LOW) default
_VAD.get("end_sensitivity", "END_SENSITIVITY_LOW"), # waits longer to be sure the speaker finished.
), end_of_speech_sensitivity=(
types.EndSensitivity.END_SENSITIVITY_HIGH
if _DIRECT_ANSWER else getattr(
types.EndSensitivity,
_VAD.get("end_sensitivity", "END_SENSITIVITY_LOW"),
)),
prefix_padding_ms=_VAD.get("prefix_padding_ms", 20), prefix_padding_ms=_VAD.get("prefix_padding_ms", 20),
silence_duration_ms=_VAD.get("silence_duration_ms", 200), silence_duration_ms=_VAD.get("silence_duration_ms", 200),
), ),
@ -518,13 +691,26 @@ class GeminiBrain:
system_instruction=types.Content( system_instruction=types.Content(
parts=[types.Part(text=self._system_prompt)], parts=[types.Part(text=self._system_prompt)],
), ),
# Native function-calling: Gemini can drive the robot to saved # Native function-calling: nav tools (if enabled) + the internet-
# places (navigate_to_place / list_places / where_am_i / # search tool (if enabled). No face tools on the R1 (no mask).
# stop_navigation). Disable with SANAD_NAV_TOOLS=0. # PURE VOICE: no tools (the reply-latency tax) — EXCEPT nav tools
tools=([types.Tool(function_declarations=_nav_function_declarations())] # when movement is enabled (walking commands need them).
if _NAV_TOOLS_ENABLED else []), tools=self._build_tools(),
) )
def _build_tools(self) -> Any:
"""Assemble the live-session function tools. PURE VOICE keeps this None
(the reply-latency tax) UNLESS movement is on (nav tools) or internet
search is on (the search tool). R1: no face tools (no mask)."""
decls: list = []
# Nav tools: whenever NOT (pure and not moving) and nav is enabled.
if _NAV_TOOLS_ENABLED and (self._movement_enabled or not _PURE_VOICE):
decls += _nav_function_declarations()
# Internet-search tool: whenever the option is enabled (either mode).
if _INTERNET_SEARCH:
decls += _search_function_declaration()
return [types.Tool(function_declarations=decls)] if decls else None
# ─── state helpers ──────────────────────────────────── # ─── state helpers ────────────────────────────────────
def _reset_turn_state(self) -> None: def _reset_turn_state(self) -> None:
@ -564,6 +750,7 @@ class GeminiBrain:
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loud_count = 0 loud_count = 0
last_activity = time.time() last_activity = time.time()
force_silence_until = 0.0 # Answer-now button: silence burst window
while not self._done.is_set() and not self._stop_flag.is_set(): while not self._done.is_set() and not self._stop_flag.is_set():
try: try:
@ -578,6 +765,27 @@ class GeminiBrain:
data = samples.tobytes() data = samples.tobytes()
energy = _audio_energy(data) energy = _audio_energy(data)
now = time.time() now = time.time()
# Dashboard Answer-now button: feed the server VAD a silence burst
# so it closes the user's turn NOW → immediate reply.
if _FORCE_ANSWER.is_set():
_FORCE_ANSWER.clear()
if not self._speaking:
force_silence_until = now + 0.6
log.info("ANSWER NOW (dashboard) — forcing turn end")
# Dashboard Interrupt-now button: cut speech instantly AND stay
# SILENT until the user speaks — arming the greeting suppression
# also swallows any in-flight or reconnect-greeting audio, so a
# press while "listening" can never be followed by Sanad talking
# on his own.
if _MANUAL_INTERRUPT.is_set():
_MANUAL_INTERRUPT.clear()
log.info("MANUAL INTERRUPT (dashboard)%s",
"" if self._speaking else " — idle; muted until user speaks")
self._interrupt("manual")
self._barge_block_until = now + cooldown
self._suppress_greeting = True
# On the JBL (loud external speaker) the head mic hears the robot's # On the JBL (loud external speaker) the head mic hears the robot's
# OWN voice as loud as the user. We FULLY gate the mic to Gemini while # OWN voice as loud as the user. We FULLY gate the mic to Gemini while
# it speaks (+ a short echo tail) so it NEVER hears itself, and we # it speaks (+ a short echo tail) so it NEVER hears itself, and we
@ -586,24 +794,47 @@ class GeminiBrain:
# model. (Reliable JBL interrupt needs AEC; the only PulseAudio mic is # model. (Reliable JBL interrupt needs AEC; the only PulseAudio mic is
# dead, so that's separate work.) The chest speaker (builtin) keeps # dead, so that's separate work.) The chest speaker (builtin) keeps
# light quiet-frame suppression + working barge-in (firmware AEC). # light quiet-frame suppression + working barge-in (firmware AEC).
full_gate = "jbl" in (self._current_profile_id or "") # The Beats Pill (USB) is the same regime as the JBL: loud external
# speaker with a raw AEC-less mic inches away — the mic feed to
# Gemini is ALWAYS fully gated while speaking (echo-safe). Voice
# barge-in on external speakers is an OPTION (_EXTERNAL_BARGE_IN):
# the LOCAL detector cuts the speaker using stricter parameters.
external = any(t in (self._current_profile_id or "")
for t in ("jbl", "beats"))
# INTERRUPT OFF → NOBODY interrupts, on ANY speaker: every profile
# gets the Beats-Pill regime (mic fully gated while speaking + tail,
# so Gemini hears silence → no server-side cut; local barge-in
# disabled → no energy cut). Sanad always finishes his sentence.
# INTERRUPT ON → local barge-in works everywhere: builtin/Anker use
# the normal parameters, loud external speakers (JBL / Beats Pill)
# use the stricter external parameters and STAY fully gated (echo
# safety — the bleed is as loud as a user). The dashboard
# ⏹ Interrupt-now button always works regardless of this option.
full_gate = external or not _EXTERNAL_BARGE_IN
allow_barge = _EXTERNAL_BARGE_IN
# Barge-in: sustained user energy cuts the AI — chest profile only. # Barge-in: sustained user energy cuts the AI.
if self._speaking and not full_gate and now >= self._barge_block_until: if self._speaking and allow_barge and now >= self._barge_block_until:
if (now - self._ai_speak_start) >= grace: eff_grace = _JBL_BARGE_GRACE if external else grace
if energy > threshold: eff_threshold = threshold * _JBL_BLEED_MARGIN if external else threshold
eff_chunks = _JBL_BARGE_CHUNKS if external else chunks_needed
if (now - self._ai_speak_start) >= eff_grace:
if energy > eff_threshold:
loud_count += 1 loud_count += 1
else: else:
loud_count = max(0, loud_count - 1) loud_count = max(0, loud_count - 1)
if loud_count > chunks_needed: if loud_count > eff_chunks:
log.info("BARGE-IN (e=%d)", energy) log.info("BARGE-IN (e=%d, external=%s)", energy, external)
self._interrupt("barge-in") self._interrupt("barge-in")
loud_count = 0 loud_count = 0
self._barge_block_until = now + cooldown self._barge_block_until = now + cooldown
# Echo suppression: mask the mic so the model doesn't hear its own bleed. # Echo suppression: mask the mic so the model doesn't hear its own bleed.
send_data = data send_data = data
if _INPUT_PAUSED.is_set(): if now < force_silence_until:
# Answer-now: mask the mic so the server VAD sees pure silence.
send_data = _SILENCE_PCM
elif _INPUT_PAUSED.is_set():
# Paused for a record playback — feed silence so Gemini neither # Paused for a record playback — feed silence so Gemini neither
# hears the record nor keeps talking over it. # hears the record nor keeps talking over it.
send_data = _SILENCE_PCM send_data = _SILENCE_PCM
@ -764,7 +995,8 @@ class GeminiBrain:
# mouth-open level (0..3) from this chunk's RMS, # mouth-open level (0..3) from this chunk's RMS,
# throttled. Parsed by GeminiSubprocess._reader_loop. # throttled. Parsed by GeminiSubprocess._reader_loop.
_mnow = time.time() _mnow = time.time()
if _mnow - getattr(self, "_mouth_t", 0.0) >= 0.08: if not _PURE_VOICE and \
_mnow - getattr(self, "_mouth_t", 0.0) >= 0.08:
_rms = (float(np.sqrt(np.mean( _rms = (float(np.sqrt(np.mean(
audio.astype(np.float32) ** 2))) if audio.size else 0.0) audio.astype(np.float32) ** 2))) if audio.size else 0.0)
# Lower thresholds bias the mouth more open # Lower thresholds bias the mouth more open
@ -821,6 +1053,10 @@ class GeminiBrain:
async def _announce_vision_state(self, session: Any, enabled: bool, async def _announce_vision_state(self, session: Any, enabled: bool,
is_toggle: bool) -> None: is_toggle: bool) -> None:
if _PURE_VOICE and not enabled and not is_toggle:
# PURE: skip the "disabled" standing-context injection — only
# ENABLED features get announced/used (toggles always announce).
return
if is_toggle and enabled: if is_toggle and enabled:
text = ( text = (
"[VISION ON] Your camera was just enabled — you can now see " "[VISION ON] Your camera was just enabled — you can now see "
@ -864,6 +1100,10 @@ class GeminiBrain:
async def _announce_facerec_state(self, session: Any, enabled: bool, async def _announce_facerec_state(self, session: Any, enabled: bool,
is_toggle: bool) -> None: is_toggle: bool) -> None:
if _PURE_VOICE and not enabled and not is_toggle:
# PURE: skip the "disabled" standing-context injection — only
# ENABLED features get announced/used (toggles always announce).
return
if is_toggle and enabled: if is_toggle and enabled:
text = ( text = (
"[FACE RECOGNITION ON] Face recognition was just enabled — " "[FACE RECOGNITION ON] Face recognition was just enabled — "
@ -908,6 +1148,10 @@ class GeminiBrain:
async def _announce_zonerec_state(self, session: Any, enabled: bool, async def _announce_zonerec_state(self, session: Any, enabled: bool,
is_toggle: bool) -> None: is_toggle: bool) -> None:
if _PURE_VOICE and not enabled and not is_toggle:
# PURE: skip the "disabled" standing-context injection — only
# ENABLED features get announced/used (toggles always announce).
return
if is_toggle and enabled: if is_toggle and enabled:
text = ( text = (
"[ZONE RECOGNITION ON] You were just given the zones and places " "[ZONE RECOGNITION ON] You were just given the zones and places "
@ -1069,6 +1313,13 @@ class GeminiBrain:
"places": r.get("places", [])} "places": r.get("places", [])}
if name == "stop_navigation": if name == "stop_navigation":
return await asyncio.to_thread(_nav_api, "POST", "/api/nav/cancel", None) return await asyncio.to_thread(_nav_api, "POST", "/api/nav/cancel", None)
if name == "search_internet":
query = str(args.get("query") or "").strip()
if not query:
return {"ok": False, "reason": "no_query"}
log.info("[[SEARCH:%s]]", query) # breadcrumb for the dashboard
return await asyncio.to_thread(
_run_grounding_search, query, self._api_key)
return {"ok": False, "reason": "unknown_tool"} return {"ok": False, "reason": "unknown_tool"}
except Exception as exc: except Exception as exc:
log.warning("tool %s error: %s", name, exc) log.warning("tool %s error: %s", name, exc)
@ -1081,6 +1332,10 @@ class GeminiBrain:
async def _announce_movement_state(self, session: Any, enabled: bool, async def _announce_movement_state(self, session: Any, enabled: bool,
is_toggle: bool) -> None: is_toggle: bool) -> None:
if _PURE_VOICE and not enabled and not is_toggle:
# PURE: skip the "disabled" standing-context injection — only
# ENABLED features get announced/used (toggles always announce).
return
if is_toggle and enabled: if is_toggle and enabled:
text = ( text = (
"[MOVEMENT ON] Walking is now enabled — you can move when the " "[MOVEMENT ON] Walking is now enabled — you can move when the "
@ -1406,6 +1661,12 @@ class GeminiBrain:
await self._announce_movement_state( await self._announce_movement_state(
session, self._movement_enabled, is_toggle=True, session, self._movement_enabled, is_toggle=True,
) )
if _PURE_VOICE:
# PURE: tools are baked per-session (nav tools only while
# movement is on) — cycle the session so the toggle takes
# effect on the function-calling side too.
log.info("pure-voice: cycling session to rebake nav tools")
self._done.set()
# Auto-record toggle — flip the recorder live (no session restart). # Auto-record toggle — flip the recorder live (no session restart).
if new_state.record_enabled != last_state.record_enabled: if new_state.record_enabled != last_state.record_enabled:
@ -1436,6 +1697,12 @@ class GeminiBrain:
await asyncio.sleep(max(period, backoff)) await asyncio.sleep(max(period, backoff))
if not self._vision_enabled: if not self._vision_enabled:
continue continue
# Never compete with the INCOMING audio stream while the AI is
# speaking — a frame upload on the shared websocket delays audio
# chunks, which breaks the chest speaker (tiny firmware buffer;
# the pulse/Pill path hides it behind its own buffering).
if self._speaking or (time.time() - self._last_ai_audio) < 0.5:
continue
with _LATEST_FRAME_LOCK: with _LATEST_FRAME_LOCK:
data = _LATEST_FRAME.get("bytes") data = _LATEST_FRAME.get("bytes")
ts = _LATEST_FRAME.get("ts", 0.0) ts = _LATEST_FRAME.get("ts", 0.0)

View File

@ -38,7 +38,9 @@ _FRAME_FORWARD_INTERVAL_S = float(_LS_CFG.get("frame_forward_interval_sec", 0.5)
# Audio profile watcher — poll pactl for the Anker USB device at this # Audio profile watcher — poll pactl for the Anker USB device at this
# interval, send "profile:<json>" to the child on every state change. # interval, send "profile:<json>" to the child on every state change.
_AUDIO_WATCH_INTERVAL_S = float(_LS_CFG.get("audio_watch_interval_sec", 1.5)) # 5s: each poll shells out to pactl several times (detect_plugged_profiles) —
# 1.5s was a constant subprocess storm for a rare event (device hot-plug).
_AUDIO_WATCH_INTERVAL_S = float(_LS_CFG.get("audio_watch_interval_sec", 5.0))
# The Anker profile id, as defined in voice/audio_devices.py. When this # The Anker profile id, as defined in voice/audio_devices.py. When this
# profile is fully plugged (both sink + source present), we switch the # profile is fully plugged (both sink + source present), we switch the
@ -515,7 +517,7 @@ class GeminiSubprocess:
if the process isn't running or stdin is closed.""" if the process isn't running or stdin is closed."""
pid = (profile_id or "").strip().lower() pid = (profile_id or "").strip().lower()
if pid not in {"builtin", "anker", "anker_powerconf", if pid not in {"builtin", "anker", "anker_powerconf",
"hollyland_builtin", "jbl_builtin_mic"}: "hollyland_builtin", "jbl_builtin_mic", "beats_pill"}:
log.warning("send_profile: ignoring unknown profile %r", profile_id) log.warning("send_profile: ignoring unknown profile %r", profile_id)
return return
payload: dict[str, Any] = {"id": pid} payload: dict[str, Any] = {"id": pid}
@ -533,6 +535,16 @@ class GeminiSubprocess:
owns the chest speaker, then resumes. No-op if not running.""" owns the chest speaker, then resumes. No-op if not running."""
self._send_stdin("pause:%d\n" % (1 if paused else 0)) self._send_stdin("pause:%d\n" % (1 if paused else 0))
def send_interrupt(self) -> None:
"""Instantly stop the AI's current speech (dashboard Interrupt-now
button) the child stops the speaker and returns to listening."""
self._send_stdin("interrupt:1\n")
def send_answer_now(self) -> None:
"""Force Gemini to answer immediately (dashboard Answer-now button) —
the child feeds the server VAD a short silence burst to close the turn."""
self._send_stdin("answernow:1\n")
def _audio_watcher(self) -> None: def _audio_watcher(self) -> None:
"""Background thread — poll pactl for the Anker USB device, signal """Background thread — poll pactl for the Anker USB device, signal
the child on every plug/unplug edge transition. the child on every plug/unplug edge transition.

View File

@ -82,20 +82,18 @@ except ImportError:
_make_low_cmd = None _make_low_cmd = None
log.warning("Unitree SDK not available — running in simulation mode") log.warning("Unitree SDK not available — running in simulation mode")
# G1 arm-action client for built-in arm moves (wave, shake_hand, hug, …). # R1-native arm-action RPC client for built-in gestures + recorded "teach"
# NOTE: do NOT use LocoClient here — LocoClient is the locomotion/body-move # motions. The R1 firmware does NOT expose live joint streaming of the head/arms;
# client and its ExecuteAction() doesn't recognise arm-action IDs, so arm # the 'arm' service owns rt/arm_sdk and clients drive the upper body ONLY through
# commands become silent no-ops. The correct client is the arm-specific # this RPC: ExecuteAction(id) for Unitree preset gestures, ExecuteCustomAction(
# G1ArmActionClient with the SDK's action_map (name → opcode lookup). # name) for motions recorded via the app's teaching (incl. the head looks). This
# replaces the old G1ArmActionClient — the R1 has no G1 arm-action service, so
# those gestures were silent no-ops on the R1.
try: try:
from unitree_sdk2py.g1.arm.g1_arm_action_client import ( from Project.Sanad.motion.r1_arm_action_client import R1ArmActionClient
G1ArmActionClient,
action_map as _ARM_ACTION_MAP,
)
_HAS_ARM_CLIENT = True _HAS_ARM_CLIENT = True
except ImportError: except ImportError:
G1ArmActionClient = None R1ArmActionClient = None
_ARM_ACTION_MAP = {}
_HAS_ARM_CLIENT = False _HAS_ARM_CLIENT = False
@ -107,24 +105,41 @@ class Action:
category: str = "sdk" # "sdk" | "jsonl" category: str = "sdk" # "sdk" | "jsonl"
# -- SDK actions (fixed — built into Unitree firmware) -- # -- SDK actions (R1 'arm'-service preset gestures) --
# The Action.id IS the R1 arm-service opcode → executed directly via
# R1ArmActionClient.ExecuteAction(id). Names/ids captured from the robot's
# GetActionList (2026-07-14). Friendliest gestures first.
SDK_ACTIONS: list[Action] = [ SDK_ACTIONS: list[Action] = [
Action("release_arm", 0, category="sdk"), Action("release_arm", 99, category="sdk"),
Action("shake_hand", 1, category="sdk"), Action("shake_hand", 27, category="sdk"),
Action("high_five", 2, category="sdk"), Action("high_five", 18, category="sdk"),
Action("hug", 3, category="sdk"), Action("hug", 19, category="sdk"),
Action("high_wave", 4, category="sdk"), Action("both_hands_up", 15, category="sdk"),
Action("clap", 5, category="sdk"), Action("wave_above_head", 26, category="sdk"),
Action("face_wave", 6, category="sdk"), Action("wave_under_head", 25, category="sdk"),
Action("left_kiss", 7, category="sdk"), Action("blow_kiss_both_hands", 11, category="sdk"),
Action("heart", 8, category="sdk"), Action("blow_kiss_left_hand", 12, category="sdk"),
Action("right_heart", 9, category="sdk"), Action("blow_kiss_right_hand", 13, category="sdk"),
Action("hands_up", 10, category="sdk"), Action("right_hand_up", 23, category="sdk"),
Action("x_ray", 11, category="sdk"), Action("right_hand_on_heart", 33, category="sdk"),
Action("right_hand_up", 12, category="sdk"), Action("clamp", 17, category="sdk"),
Action("reject", 13, category="sdk"), Action("refuse", 22, category="sdk"),
Action("right_kiss", 14, category="sdk"), Action("emphasize", 35, category="sdk"),
Action("two_hand_kiss", 15, category="sdk"), Action("forward_push", 36, category="sdk"),
Action("extend_right_arm_forward", 31, category="sdk"),
Action("both_hands_up_deviate_right", 34, category="sdk"),
Action("ultraman_ray", 24, category="sdk"),
Action("box_left_hand_win", 28, category="sdk"),
Action("box_right_hand_win", 29, category="sdk"),
Action("box_both_hand_win", 30, category="sdk"),
]
# R1 head-look "teach" motions (recorded via the app's teaching) — replayed by
# NAME via R1ArmActionClient.ExecuteCustomAction. Shown as a simple button list
# in the Controller tab's HEAD card. These are your recorded head rotations.
HEAD_LOOKS: list[str] = [
"up_look", "down_look", "left_look", "right_look",
"up_down_look", "left_right_look", "right_left_look", "center",
] ]
# Next auto-ID for JSONL actions starts after SDK range. # Next auto-ID for JSONL actions starts after SDK range.
@ -285,16 +300,16 @@ class ArmController:
) )
self._crc = CRC() self._crc = CRC()
# Arm-specific action client for built-in moves # R1 arm-service client for preset gestures + recorded teach/head moves
if _HAS_ARM_CLIENT: if _HAS_ARM_CLIENT:
try: try:
self._arm_client = G1ArmActionClient() self._arm_client = R1ArmActionClient()
self._arm_client.SetTimeout(10.0) self._arm_client.SetTimeout(10.0)
self._arm_client.Init() self._arm_client.Init()
log.info("G1ArmActionClient initialized (%d actions) — built-in moves available", log.info("R1ArmActionClient initialized — %d preset gestures + %d head looks",
len(_ARM_ACTION_MAP)) len(SDK_ACTIONS), len(HEAD_LOOKS))
except Exception as exc: except Exception as exc:
log.warning("G1ArmActionClient init failed: %s — built-in actions disabled", exc) log.warning("R1ArmActionClient init failed: %s — arm actions disabled", exc)
self._arm_client = None self._arm_client = None
self._initialized = True self._initialized = True
@ -869,39 +884,42 @@ class ArmController:
return return
if self._arm_client is None: if self._arm_client is None:
log.warning( log.warning(
"SDK action %s requested but G1ArmActionClient not available — skipping", "SDK action %s requested but R1ArmActionClient not available — skipping",
action.name, action.name,
) )
return return
# Sanad's registry uses underscored names ("shake_hand", "x_ray"); # R1 preset gestures execute by opcode directly — Action.id IS the R1
# the SDK's action_map is keyed by human-readable forms that mix # arm-service opcode (see SDK_ACTIONS). No name→opcode lookup needed.
# spaces and hyphens ("shake hand", "x-ray", "two-hand kiss"). log.info("R1 gesture: %s (opcode=%s)", action.name, action.id)
# Try each candidate in turn.
name = action.name
candidates = [
name,
name.replace("_", " "), # shake_hand → shake hand
name.replace("_", "-"), # x_ray → x-ray
# two-word with specific hyphenation: first token with hyphen,
# rest with spaces (matches SDK's "two-hand kiss" pattern)
name.replace("_", "-", 1).replace("_", " "),
]
sdk_name = next((c for c in candidates if c in _ARM_ACTION_MAP), None)
if sdk_name is None:
log.warning(
"SDK action %s not in G1ArmActionClient action_map — tried %s. keys=%s",
action.name, candidates, sorted(_ARM_ACTION_MAP.keys())[:12],
)
return
opcode = _ARM_ACTION_MAP[sdk_name]
log.info("SDK action: %s (opcode=%s)", action.name, opcode)
try: try:
self._arm_client.ExecuteAction(opcode) rc = self._arm_client.ExecuteAction(action.id)
# Built-in arm actions block on the robot side for ~3s; the SDK if rc != 0:
# call returns immediately. Sleep so we don't hammer it back-to-back. log.warning("R1 gesture %s returned rc=%s", action.name, rc)
# The arm service blocks on the robot for ~3s; the RPC returns fast.
time.sleep(3.0) time.sleep(3.0)
except Exception as exc: except Exception as exc:
log.error("SDK action %s failed: %s", action.name, exc) log.error("R1 gesture %s failed: %s", action.name, exc)
def run_head_look(self, name: str) -> dict[str, Any]:
"""Replay a recorded head-look 'teach' motion by name via the R1 arm
service (ExecuteCustomAction). Used by the Controller-tab HEAD menu.
Returns {ok, rc}. Never raises reports failure in the result."""
if name not in HEAD_LOOKS:
return {"ok": False, "error": f"unknown head look '{name}'"}
if not _HAS_SDK or self._arm_client is None:
log.info("[SIM] head look: %s", name)
return {"ok": _HAS_SDK is False, "rc": None, "sim": True}
log.info("R1 head look: %s", name)
try:
rc = self._arm_client.ExecuteCustomAction(name)
return {"ok": rc == 0, "rc": rc}
except Exception as exc:
log.error("head look %s failed: %s", name, exc)
return {"ok": False, "error": str(exc)}
def head_looks(self) -> list[str]:
"""The recorded head-look names for the dashboard menu."""
return list(HEAD_LOOKS)
def status(self) -> dict[str, Any]: def status(self) -> dict[str, Any]:
return { return {

View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""
R1 arm-action RPC client the R1-NATIVE way to move the upper body (arms + head)
while the firmware keeps balancing the legs.
Why this and not rt/arm_sdk: the R1 firmware does NOT expose live joint streaming
of the head/arms. The 'arm' service (a.k.a. r1_arm_example) owns rt/arm_sdk
internally; clients drive the upper body ONLY through this RPC preset gesture
actions (by id) and custom "teach" actions (by name, recorded via the app's
teaching). Captured from the app's own traffic (2026-07-06):
release_arm -> ExecuteAction(99); replay recorded motion -> ExecuteCustomAction(name).
Service "arm", api version "1.0.0.14". Same Client base as R1LocoClient.
State is published on rt/arm/action/state: {"holding":bool,"id":int,"name":str}.
"""
import json
from unitree_sdk2py.rpc.client import Client
ARM_ACTION_SERVICE_NAME = "arm"
ARM_ACTION_API_VERSION = "1.0.0.14"
API_EXECUTE_ACTION = 7106 # {"action_id": N} preset actions
API_GET_ACTION_LIST = 7107 # -> JSON list
API_EXECUTE_CUSTOM_ACTION = 7108 # {"action_name": "..."} recorded teach actions
API_RECORD = 7110 # start = {"action_name":NAME} ; stop+save = "" (empty str)
API_STOP_CUSTOM_ACTION = 7113
RELEASE_ARM = 99 # go compliant / release a held action
class R1ArmActionClient(Client):
def __init__(self):
super().__init__(ARM_ACTION_SERVICE_NAME, False)
def Init(self):
self._SetApiVerson(ARM_ACTION_API_VERSION) # SDK spells it "Verson"
for api in (API_EXECUTE_ACTION, API_GET_ACTION_LIST,
API_EXECUTE_CUSTOM_ACTION, API_RECORD, API_STOP_CUSTOM_ACTION):
self._RegistApi(api, 0)
def ExecuteAction(self, action_id: int):
"""Run a preset action by id (99 = release). Returns rc (0 = ok)."""
code, _ = self._Call(API_EXECUTE_ACTION, json.dumps({"action_id": int(action_id)}))
return code
def ExecuteCustomAction(self, action_name: str):
"""Replay a recorded teach action by name (from the app's teaching)."""
code, _ = self._Call(API_EXECUTE_CUSTOM_ACTION, json.dumps({"action_name": str(action_name)}))
return code
def StopCustomAction(self):
code, _ = self._Call(API_STOP_CUSTOM_ACTION, "{}")
return code
def ReleaseArm(self):
return self.ExecuteAction(RELEASE_ARM)
def GetActionList(self):
"""Return (rc, data) — data is the JSON list of available actions."""
code, data = self._Call(API_GET_ACTION_LIST, "")
parsed = None
if code == 0 and data:
try:
parsed = json.loads(data)
except (ValueError, TypeError):
parsed = data
return code, parsed

View File

@ -1,4 +1,4 @@
أنت "سند iron man" — روبوت إماراتي ذكي تابع لروبوت شركة لوتاه تيك Lootah Tech. أنت "سوبر دبي super-dubai" — روبوت إماراتي ذكي تابع لروبوت شركة لوتاه تيك Lootah Tech.
[أولاً: الروح والمعرفة] [أولاً: الروح والمعرفة]

View File

@ -22,7 +22,9 @@ class TestConfig(unittest.TestCase):
def test_base_dir_auto_detect(self): def test_base_dir_auto_detect(self):
from Project.Sanad.config import BASE_DIR from Project.Sanad.config import BASE_DIR
self.assertTrue(BASE_DIR.exists(), f"BASE_DIR missing: {BASE_DIR}") self.assertTrue(BASE_DIR.exists(), f"BASE_DIR missing: {BASE_DIR}")
self.assertEqual(BASE_DIR.name, "Sanad") # Workstation checkout is SanadR1; the deployed container aliases it
# as Project/Sanad, so both names are valid.
self.assertIn(BASE_DIR.name, ("Sanad", "SanadR1"))
def test_data_dirs_exist(self): def test_data_dirs_exist(self):
from Project.Sanad.config import ( from Project.Sanad.config import (

View File

@ -106,6 +106,18 @@ PROFILES: list[AudioProfile] = [
source_pattern="powerconf,anker", source_pattern="powerconf,anker",
description="Anker PowerConf USB conference unit — mic + speaker on the same device.", description="Anker PowerConf USB conference unit — mic + speaker on the same device.",
), ),
AudioProfile(
id="beats_pill",
label="Beats Pill (mic + speaker)",
# USB-C: kernel names the card "Pill"; pulse sinks/sources carry
# "Beats_Pill" (auto-probed or the manual module-alsa-sink/-source
# loads from /etc/pulse/default.pa — pulse fails to probe the Pill's
# output profile on some kernels). Bluetooth: bluez sink.
sink_pattern="beats_pill,beats,pill",
source_pattern="beats_pill,beats,pill",
description="Beats Pill (USB-C or Bluetooth) — speaker + speakerphone mic.",
udp_mic_fallback=True, # mic falls back to the R1/G1 UDP mic if absent
),
AudioProfile( AudioProfile(
id="jbl_builtin_mic", id="jbl_builtin_mic",
label="JBL speaker + built-in mic", label="JBL speaker + built-in mic",
@ -115,6 +127,7 @@ PROFILES: list[AudioProfile] = [
# The JBL has NO microphone → input stays on the G1 built-in mic. # The JBL has NO microphone → input stays on the G1 built-in mic.
source_pattern="alsa_input.platform-sound", source_pattern="alsa_input.platform-sound",
description="JBL Bluetooth speaker for output + the G1 built-in microphone for input (the JBL has no mic).", description="JBL Bluetooth speaker for output + the G1 built-in microphone for input (the JBL has no mic).",
udp_mic_fallback=True, # builtin mic = R1/G1 UDP stream on units without the alsa source
), ),
] ]

View File

@ -31,6 +31,7 @@ import json
import socket import socket
import struct import struct
import subprocess import subprocess
import queue
import threading import threading
import time import time
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
@ -211,7 +212,16 @@ class BuiltinMic(Mic):
class BuiltinSpeaker(Speaker): class BuiltinSpeaker(Speaker):
"""G1 robot's built-in speaker via AudioClient.PlayStream (16 kHz mono).""" """G1 robot's built-in speaker via AudioClient.PlayStream (16 kHz mono).
JITTER-BUFFERED: chunks are enqueued and a dedicated writer thread sends
them to the firmware, prebuffering ~0.3s before the first RPC and never
running more than ~0.6s ahead of real time. Gemini's audio arrives in
BURSTS (WiFi jitter, shared-websocket contention) the old direct
per-chunk RPC mapped every arrival gap to an audible dropout on the
chest speaker's tiny firmware buffer, while the pulse (Pill/Anker) path
hid it behind PulseAudio's own buffering. This gives the chest the same
tolerance."""
HARDWARE_RATE = 16_000 HARDWARE_RATE = 16_000
@ -224,10 +234,16 @@ class BuiltinSpeaker(Speaker):
self._app_name = app_name or _SP_CFG.get("app_name", "sanad") self._app_name = app_name or _SP_CFG.get("app_name", "sanad")
self._begin_pause = _SP_CFG.get("begin_stream_pause_sec", 0.15) self._begin_pause = _SP_CFG.get("begin_stream_pause_sec", 0.15)
self._finish_margin = _SP_CFG.get("wait_finish_margin_sec", 0.3) self._finish_margin = _SP_CFG.get("wait_finish_margin_sec", 0.3)
# Jitter buffer tuning (see class docstring).
self._prebuffer_sec = _SP_CFG.get("jitter_prebuffer_sec", 0.30)
self._first_flush_sec = _SP_CFG.get("jitter_first_flush_sec", 0.60)
self._max_lead_sec = _SP_CFG.get("jitter_max_lead_sec", 0.60)
self._stop_flag = threading.Event() self._stop_flag = threading.Event()
self._stream_id: Optional[str] = None self._stream_id: Optional[str] = None
self._total_sent = 0.0 self._total_sent = 0.0
self._play_start = 0.0 self._play_start = 0.0
self._q: "queue.Queue[Optional[bytes]]" = queue.Queue()
self._writer: Optional[threading.Thread] = None
def _stop_play_api(self) -> None: def _stop_play_api(self) -> None:
try: try:
@ -241,13 +257,79 @@ class BuiltinSpeaker(Speaker):
except Exception: except Exception:
log.warning("BuiltinSpeaker AUDIO_STOP_PLAY failed") log.warning("BuiltinSpeaker AUDIO_STOP_PLAY failed")
def _drain_queue(self) -> None:
try:
while True:
self._q.get_nowait()
except queue.Empty:
pass
def begin_stream(self) -> None: def begin_stream(self) -> None:
# Tear down any previous writer cleanly before starting a new turn.
self._stop_flag.set()
self._drain_queue()
if self._writer is not None and self._writer.is_alive():
self._q.put(None)
self._writer.join(timeout=1.0)
self._stop_flag.clear() self._stop_flag.clear()
self._drain_queue()
self._stop_play_api() self._stop_play_api()
time.sleep(self._begin_pause) time.sleep(self._begin_pause)
self._stream_id = f"s_{int(time.time() * 1000)}" self._stream_id = f"s_{int(time.time() * 1000)}"
self._total_sent = 0.0 self._total_sent = 0.0
self._play_start = time.time() self._play_start = time.time()
self._writer = threading.Thread(
target=self._writer_loop, args=(self._stream_id,),
name="builtin-speaker-writer", daemon=True)
self._writer.start()
def _writer_loop(self, stream_id: str) -> None:
rate = float(self.HARDWARE_RATE)
sent_sec = 0.0
started = False
t0 = time.time()
batch: list = []
buffered = 0.0
while not self._stop_flag.is_set() and self._stream_id == stream_id:
try:
item = self._q.get(timeout=0.05)
except queue.Empty:
item = ()
if item is None:
return
if not started:
if item:
batch.append(item)
buffered += len(item) / 2 / rate
if buffered >= self._prebuffer_sec or (
batch and (time.time() - t0) >= self._first_flush_sec):
self._play_start = time.time()
started = True
for b in batch:
if self._stop_flag.is_set() or self._stream_id != stream_id:
return
try:
self._ac.PlayStream(self._app_name, stream_id, b)
except Exception as exc:
log.warning("PlayStream failed: %s", exc)
sent_sec += len(b) / 2 / rate
batch = []
continue
if not item:
continue
# Pace: never run more than max_lead ahead of real time — keeps
# the firmware queue short (fast interrupts) while OUR queue is
# the elastic reservoir that absorbs bursty arrival.
lead = sent_sec - (time.time() - self._play_start)
if lead > self._max_lead_sec:
time.sleep(min(lead - self._max_lead_sec, 0.25))
if self._stop_flag.is_set() or self._stream_id != stream_id:
return
try:
self._ac.PlayStream(self._app_name, stream_id, item)
except Exception as exc:
log.warning("PlayStream failed: %s", exc)
sent_sec += len(item) / 2 / rate
def send_chunk(self, pcm: PCMLike, source_rate: int) -> None: def send_chunk(self, pcm: PCMLike, source_rate: int) -> None:
if self._stop_flag.is_set(): if self._stop_flag.is_set():
@ -256,20 +338,22 @@ class BuiltinSpeaker(Speaker):
if arr.size < 10: if arr.size < 10:
return return
hw = _resample_int16(arr, source_rate, self.HARDWARE_RATE) hw = _resample_int16(arr, source_rate, self.HARDWARE_RATE)
self._ac.PlayStream(self._app_name, self._stream_id, hw.tobytes()) self._q.put(hw.tobytes())
self._total_sent += len(hw) / self.HARDWARE_RATE self._total_sent += len(hw) / self.HARDWARE_RATE
def wait_finish(self) -> None: def wait_finish(self) -> None:
elapsed = time.time() - self._play_start # Wait for the queue to drain AND real-time playback to elapse.
remaining = self._total_sent - elapsed + self._finish_margin while not self._stop_flag.is_set():
waited = 0.0 elapsed = time.time() - self._play_start
while waited < remaining and not self._stop_flag.is_set(): remaining = self._total_sent - elapsed + self._finish_margin
if self._q.empty() and remaining <= 0:
break
time.sleep(0.1) time.sleep(0.1)
waited += 0.1
self._stop_play_api() self._stop_play_api()
def stop(self) -> None: def stop(self) -> None:
self._stop_flag.set() self._stop_flag.set()
self._drain_queue()
self._stop_play_api() self._stop_play_api()
@property @property
@ -815,9 +899,13 @@ _PROFILE_ALIASES = {
"hollyland_builtin": "hollyland_builtin", "hollyland_builtin": "hollyland_builtin",
"jbl": "jbl_builtin_mic", "jbl": "jbl_builtin_mic",
"jbl_builtin_mic": "jbl_builtin_mic", "jbl_builtin_mic": "jbl_builtin_mic",
"beats": "beats_pill",
"beats_pill": "beats_pill",
"pill": "beats_pill",
} }
SUPPORTED_PROFILES = ("builtin", "anker", "hollyland_builtin", "jbl_builtin_mic") SUPPORTED_PROFILES = ("builtin", "anker", "hollyland_builtin", "jbl_builtin_mic",
"beats_pill")
@dataclass @dataclass
@ -885,6 +973,13 @@ class AudioIO:
# pacat is used because PyAudio's 'pulse' device is unavailable in # pacat is used because PyAudio's 'pulse' device is unavailable in
# this env. Neither backend needs the AudioClient. # this env. Neither backend needs the AudioClient.
return BuiltinMic(), PulseStreamSpeaker(label="JBL", sink_pattern="jbl,bluez") return BuiltinMic(), PulseStreamSpeaker(label="JBL", sink_pattern="jbl,bluez")
if resolved == "beats_pill":
# Beats Pill speaker via pacat → its PulseAudio sink + the G1
# built-in DDS mic — same regime as the JBL (loud external speaker,
# full mic-gate while speaking; see gemini/script.py). The Pill's
# own USB mic stays available to pulse for dashboard recordings.
return BuiltinMic(), PulseStreamSpeaker(
label="Beats Pill", sink_pattern="beats_pill,beats,pill")
raise AssertionError(f"unhandled resolved profile: {resolved!r}") raise AssertionError(f"unhandled resolved profile: {resolved!r}")
@classmethod @classmethod

View File

@ -146,7 +146,7 @@ _MOVEMENT_PROMPT_RULES = (
"(you receive a [MOVEMENT ON] or [MOVEMENT STATUS] note). When movement is " "(you receive a [MOVEMENT ON] or [MOVEMENT STATUS] note). When movement is "
"OFF, never confirm a motion — tell the user to enable movement from the " "OFF, never confirm a motion — tell the user to enable movement from the "
"dashboard.\n" "dashboard.\n"
"When movement is ON and the user addresses you by name (Sanad iron man / سند) AND " "When movement is ON and the user addresses you by name (super-dubai / سوبر دبي) AND "
"asks you to move, reply with ONE short confirmation phrase per requested " "asks you to move, reply with ONE short confirmation phrase per requested "
"motion, in the SAME language, in the order asked. Use these EXACT shapes — " "motion, in the SAME language, in the order asked. Use these EXACT shapes — "
"they are what triggers the motion:\n" "they are what triggers the motion:\n"