Update 2026-07-20 10:30:47
This commit is contained in:
parent
44f547f754
commit
cfdc0556a4
@ -39,9 +39,9 @@
|
|||||||
"api_key": "",
|
"api_key": "",
|
||||||
"model_live": "gemini-2.5-flash-native-audio-preview-12-2025",
|
"model_live": "gemini-2.5-flash-native-audio-preview-12-2025",
|
||||||
"model_ws_uri": "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent",
|
"model_ws_uri": "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent",
|
||||||
"voice_name": "Charon",
|
"voice_name": "Puck",
|
||||||
"ws_timeout_sec": 30,
|
"ws_timeout_sec": 30,
|
||||||
"default_system_prompt": "You are Bousandah, a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'."
|
"default_system_prompt": "You are super-dubai (سوبر دبي), a wise and friendly Emirati assistant. Speak strictly in the UAE dialect (Khaleeji). Be helpful, concise, and use local greetings like 'Marhaba' and 'Ya Khoy'."
|
||||||
},
|
},
|
||||||
|
|
||||||
"g1_hardware": {
|
"g1_hardware": {
|
||||||
|
|||||||
@ -10,7 +10,9 @@
|
|||||||
"session_timeout_sec": 660,
|
"session_timeout_sec": 660,
|
||||||
"max_reconnect_delay_sec": 30,
|
"max_reconnect_delay_sec": 30,
|
||||||
"max_consecutive_errors": 10,
|
"max_consecutive_errors": 10,
|
||||||
"no_messages_timeout_sec": 30
|
"no_messages_timeout_sec": 180,
|
||||||
|
"pure_voice": false,
|
||||||
|
"internet_search": false
|
||||||
},
|
},
|
||||||
|
|
||||||
"mic_udp": {
|
"mic_udp": {
|
||||||
@ -26,7 +28,10 @@
|
|||||||
"_comment": "G1 built-in speaker — AudioClient.PlayStream wrapper",
|
"_comment": "G1 built-in speaker — AudioClient.PlayStream wrapper",
|
||||||
"app_name": "sanad",
|
"app_name": "sanad",
|
||||||
"begin_stream_pause_sec": 0.15,
|
"begin_stream_pause_sec": 0.15,
|
||||||
"wait_finish_margin_sec": 0.3
|
"wait_finish_margin_sec": 0.3,
|
||||||
|
"jitter_prebuffer_sec": 0.3,
|
||||||
|
"jitter_first_flush_sec": 0.6,
|
||||||
|
"jitter_max_lead_sec": 0.6
|
||||||
},
|
},
|
||||||
|
|
||||||
"vad": {
|
"vad": {
|
||||||
@ -34,7 +39,8 @@
|
|||||||
"start_sensitivity": "START_SENSITIVITY_HIGH",
|
"start_sensitivity": "START_SENSITIVITY_HIGH",
|
||||||
"end_sensitivity": "END_SENSITIVITY_LOW",
|
"end_sensitivity": "END_SENSITIVITY_LOW",
|
||||||
"prefix_padding_ms": 20,
|
"prefix_padding_ms": 20,
|
||||||
"silence_duration_ms": 200
|
"silence_duration_ms": 200,
|
||||||
|
"direct_answer": false
|
||||||
},
|
},
|
||||||
|
|
||||||
"barge_in": {
|
"barge_in": {
|
||||||
@ -42,7 +48,8 @@
|
|||||||
"loud_chunks_needed": 3,
|
"loud_chunks_needed": 3,
|
||||||
"cooldown_sec": 0.3,
|
"cooldown_sec": 0.3,
|
||||||
"echo_suppress_below": 500,
|
"echo_suppress_below": 500,
|
||||||
"ai_speak_grace_sec": 0.15
|
"ai_speak_grace_sec": 0.15,
|
||||||
|
"external_barge_in": false
|
||||||
},
|
},
|
||||||
|
|
||||||
"recording": {
|
"recording": {
|
||||||
|
|||||||
@ -41,6 +41,240 @@ async def set_record(on: bool = Query(...)):
|
|||||||
return {"ok": True, "record_enabled": st.record_enabled}
|
return {"ok": True, "record_enabled": st.record_enabled}
|
||||||
|
|
||||||
|
|
||||||
|
_VOICE_OPTS_PATH = BASE_DIR / "data" / ".voice_options.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _read_voice_opts() -> dict:
|
||||||
|
import json
|
||||||
|
try:
|
||||||
|
with open(_VOICE_OPTS_PATH, encoding="utf-8") as f:
|
||||||
|
d = json.load(f)
|
||||||
|
return d if isinstance(d, dict) else {}
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/interrupt-option")
|
||||||
|
async def get_interrupt_option():
|
||||||
|
"""Voice interrupt (barge-in) — GLOBAL, all speakers. OFF: Sanad always
|
||||||
|
finishes his sentence on every profile (chest/Anker/JBL/Beats Pill — mic
|
||||||
|
fully gated while he speaks). ON: loud sustained speech cuts him.
|
||||||
|
Resolution: dashboard toggle (data/.voice_options.json) >
|
||||||
|
SANAD_EXTERNAL_BARGE_IN env > voice_config.json barge_in.external_barge_in
|
||||||
|
> off."""
|
||||||
|
import os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
if "external_barge_in" in opts:
|
||||||
|
return {"enabled": bool(opts["external_barge_in"]), "source": "dashboard"}
|
||||||
|
env = os.environ.get("SANAD_EXTERNAL_BARGE_IN", "").strip().lower()
|
||||||
|
if env in ("1", "true", "yes"):
|
||||||
|
return {"enabled": True, "source": "env"}
|
||||||
|
try:
|
||||||
|
from Project.Sanad.core.config_loader import section
|
||||||
|
if bool((section("voice", "barge_in") or {}).get("external_barge_in", False)):
|
||||||
|
return {"enabled": True, "source": "config"}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"enabled": False, "source": "default"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/interrupt-option")
|
||||||
|
async def set_interrupt_option(on: bool = Query(...)):
|
||||||
|
"""Persist the interrupt option (survives rebuilds — data/ is mounted) and
|
||||||
|
restart the live session so the voice child picks it up."""
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import os as _os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
opts["external_barge_in"] = bool(on)
|
||||||
|
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
|
||||||
|
prefix=".voice_options.", suffix=".tmp")
|
||||||
|
with _os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(opts, f, indent=2)
|
||||||
|
_os.replace(tmp, _VOICE_OPTS_PATH)
|
||||||
|
|
||||||
|
restarted = False
|
||||||
|
from Project.Sanad.main import live_sub
|
||||||
|
if live_sub is not None:
|
||||||
|
try:
|
||||||
|
if (live_sub.status() or {}).get("running"):
|
||||||
|
await asyncio.to_thread(live_sub.stop)
|
||||||
|
await asyncio.to_thread(live_sub.start)
|
||||||
|
restarted = True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"ok": True, "enabled": bool(on), "restarted": restarted}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/direct-answer-option")
|
||||||
|
async def get_direct_answer_option():
|
||||||
|
"""'Direct answer' = Gemini replies as soon as the speaker pauses (Live-API
|
||||||
|
end-of-speech sensitivity HIGH). Resolution: dashboard > env > config."""
|
||||||
|
import os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
if "direct_answer" in opts:
|
||||||
|
return {"enabled": bool(opts["direct_answer"]), "source": "dashboard"}
|
||||||
|
if os.environ.get("SANAD_DIRECT_ANSWER", "").strip().lower() in ("1", "true", "yes"):
|
||||||
|
return {"enabled": True, "source": "env"}
|
||||||
|
try:
|
||||||
|
from Project.Sanad.core.config_loader import section
|
||||||
|
if bool((section("voice", "vad") or {}).get("direct_answer", False)):
|
||||||
|
return {"enabled": True, "source": "config"}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"enabled": False, "source": "default"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/direct-answer-option")
|
||||||
|
async def set_direct_answer_option(on: bool = Query(...)):
|
||||||
|
"""Persist the direct-answer option and restart the live session."""
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import os as _os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
opts["direct_answer"] = bool(on)
|
||||||
|
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
|
||||||
|
prefix=".voice_options.", suffix=".tmp")
|
||||||
|
with _os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(opts, f, indent=2)
|
||||||
|
_os.replace(tmp, _VOICE_OPTS_PATH)
|
||||||
|
restarted = False
|
||||||
|
from Project.Sanad.main import live_sub
|
||||||
|
if live_sub is not None:
|
||||||
|
try:
|
||||||
|
if (live_sub.status() or {}).get("running"):
|
||||||
|
await asyncio.to_thread(live_sub.stop)
|
||||||
|
await asyncio.to_thread(live_sub.start)
|
||||||
|
restarted = True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"ok": True, "enabled": bool(on), "restarted": restarted}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/pure-voice-option")
|
||||||
|
async def get_pure_voice_option():
|
||||||
|
"""PURE VOICE = the live session built exactly like standalone Sanadv1:
|
||||||
|
no tools, no state injections, persona only, no lip-sync markers — the
|
||||||
|
fastest conversation pipe. Resolution: dashboard > env > config."""
|
||||||
|
import os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
if "pure_voice" in opts:
|
||||||
|
return {"enabled": bool(opts["pure_voice"]), "source": "dashboard"}
|
||||||
|
if os.environ.get("SANAD_PURE_VOICE", "").strip().lower() in ("1", "true", "yes"):
|
||||||
|
return {"enabled": True, "source": "env"}
|
||||||
|
try:
|
||||||
|
from Project.Sanad.core.config_loader import section
|
||||||
|
if bool((section("voice", "sanad_voice") or {}).get("pure_voice", False)):
|
||||||
|
return {"enabled": True, "source": "config"}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"enabled": False, "source": "default"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/pure-voice-option")
|
||||||
|
async def set_pure_voice_option(on: bool = Query(...)):
|
||||||
|
"""Persist the pure-voice option and restart the live session."""
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import os as _os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
opts["pure_voice"] = bool(on)
|
||||||
|
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
|
||||||
|
prefix=".voice_options.", suffix=".tmp")
|
||||||
|
with _os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(opts, f, indent=2)
|
||||||
|
_os.replace(tmp, _VOICE_OPTS_PATH)
|
||||||
|
restarted = False
|
||||||
|
from Project.Sanad.main import live_sub
|
||||||
|
if live_sub is not None:
|
||||||
|
try:
|
||||||
|
if (live_sub.status() or {}).get("running"):
|
||||||
|
await asyncio.to_thread(live_sub.stop)
|
||||||
|
await asyncio.to_thread(live_sub.start)
|
||||||
|
restarted = True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"ok": True, "enabled": bool(on), "restarted": restarted}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/internet-search-option")
|
||||||
|
async def get_internet_search_option():
|
||||||
|
"""INTERNET SEARCH = the live session gets the search_internet tool so Gemini
|
||||||
|
can look up current facts (prices, latest phones, live offers, news) via
|
||||||
|
Google Search grounding, announcing the search first. Resolution:
|
||||||
|
dashboard > env > config."""
|
||||||
|
import os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
if "internet_search" in opts:
|
||||||
|
return {"enabled": bool(opts["internet_search"]), "source": "dashboard"}
|
||||||
|
if os.environ.get("SANAD_INTERNET_SEARCH", "").strip().lower() in ("1", "true", "yes"):
|
||||||
|
return {"enabled": True, "source": "env"}
|
||||||
|
try:
|
||||||
|
from Project.Sanad.core.config_loader import section
|
||||||
|
if bool((section("voice", "sanad_voice") or {}).get("internet_search", False)):
|
||||||
|
return {"enabled": True, "source": "config"}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"enabled": False, "source": "default"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/internet-search-option")
|
||||||
|
async def set_internet_search_option(on: bool = Query(...)):
|
||||||
|
"""Persist the internet-search option and restart the live session so the
|
||||||
|
tool set (and persona addendum) is rebaked."""
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import os as _os
|
||||||
|
opts = _read_voice_opts()
|
||||||
|
opts["internet_search"] = bool(on)
|
||||||
|
_VOICE_OPTS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp = tempfile.mkstemp(dir=str(_VOICE_OPTS_PATH.parent),
|
||||||
|
prefix=".voice_options.", suffix=".tmp")
|
||||||
|
with _os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(opts, f, indent=2)
|
||||||
|
_os.replace(tmp, _VOICE_OPTS_PATH)
|
||||||
|
restarted = False
|
||||||
|
from Project.Sanad.main import live_sub
|
||||||
|
if live_sub is not None:
|
||||||
|
try:
|
||||||
|
if (live_sub.status() or {}).get("running"):
|
||||||
|
await asyncio.to_thread(live_sub.stop)
|
||||||
|
await asyncio.to_thread(live_sub.start)
|
||||||
|
restarted = True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {"ok": True, "enabled": bool(on), "restarted": restarted}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/answer-now")
|
||||||
|
async def answer_now():
|
||||||
|
"""Force Gemini to reply immediately instead of waiting out the pause
|
||||||
|
detection (no-op while it is already speaking)."""
|
||||||
|
sub = _sub_or_503()
|
||||||
|
if not (sub.status() or {}).get("running"):
|
||||||
|
raise HTTPException(409, "Live session not running")
|
||||||
|
if not hasattr(sub, "send_answer_now"):
|
||||||
|
raise HTTPException(501, "engine build lacks send_answer_now")
|
||||||
|
await asyncio.to_thread(sub.send_answer_now)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/interrupt")
|
||||||
|
async def interrupt_speech():
|
||||||
|
"""Instantly stop the AI mid-sentence and go back to listening (works
|
||||||
|
regardless of the barge-in option; no-op if it isn't speaking)."""
|
||||||
|
sub = _sub_or_503()
|
||||||
|
if not (sub.status() or {}).get("running"):
|
||||||
|
raise HTTPException(409, "Live session not running")
|
||||||
|
if not hasattr(sub, "send_interrupt"):
|
||||||
|
raise HTTPException(501, "engine build lacks send_interrupt")
|
||||||
|
await asyncio.to_thread(sub.send_interrupt)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/start")
|
@router.post("/start")
|
||||||
async def start_subprocess():
|
async def start_subprocess():
|
||||||
live_sub = _sub_or_503()
|
live_sub = _sub_or_503()
|
||||||
|
|||||||
@ -4,7 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
@ -50,7 +50,19 @@ async def trigger_action(payload: TriggerPayload):
|
|||||||
from Project.Sanad.main import arm
|
from Project.Sanad.main import arm
|
||||||
if arm is None:
|
if arm is None:
|
||||||
raise HTTPException(503, "Arm controller not attached.")
|
raise HTTPException(503, "Arm controller not attached.")
|
||||||
_block_if_movement_armed()
|
|
||||||
|
# Only the legacy JSONL replays (direct rt/arm_sdk streaming) conflict with
|
||||||
|
# the walker. R1 preset gestures go through the arm service, which moves the
|
||||||
|
# arms while the firmware balances — so those are allowed while movement is
|
||||||
|
# armed. Gate JSONL only.
|
||||||
|
acts = arm.list_actions() if hasattr(arm, "list_actions") else []
|
||||||
|
resolved = None
|
||||||
|
if payload.action_id is not None:
|
||||||
|
resolved = next((a for a in acts if a.get("id") == payload.action_id), None)
|
||||||
|
elif payload.action_name:
|
||||||
|
resolved = next((a for a in acts if a.get("name") == payload.action_name), None)
|
||||||
|
if resolved and resolved.get("file"):
|
||||||
|
_block_if_movement_armed()
|
||||||
|
|
||||||
speed = max(0.1, min(payload.speed, 5.0))
|
speed = max(0.1, min(payload.speed, 5.0))
|
||||||
|
|
||||||
@ -73,6 +85,30 @@ async def trigger_action(payload: TriggerPayload):
|
|||||||
raise HTTPException(400, "Provide action_id or action_name.")
|
raise HTTPException(400, "Provide action_id or action_name.")
|
||||||
|
|
||||||
|
|
||||||
|
# ── R1 head-look menu (recorded teach motions via the arm service) ──────────
|
||||||
|
|
||||||
|
@router.get("/head-actions")
|
||||||
|
async def head_actions():
|
||||||
|
"""The recorded head-look names for the Controller-tab HEAD menu."""
|
||||||
|
from Project.Sanad.main import arm
|
||||||
|
return {"looks": arm.head_looks() if arm else []}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/head")
|
||||||
|
async def head_look(name: str = Query(...)):
|
||||||
|
"""Replay a recorded head rotation (R1 arm-service teach action). NOT gated by
|
||||||
|
the movement lock: the R1 arm service moves the head while the firmware keeps
|
||||||
|
the body balanced (that is the whole point of the canteen pattern), so head
|
||||||
|
looks are safe — and useful — while locomotion/teleop is armed."""
|
||||||
|
from Project.Sanad.main import arm
|
||||||
|
if arm is None:
|
||||||
|
raise HTTPException(503, "Arm controller not attached.")
|
||||||
|
res = await asyncio.to_thread(arm.run_head_look, name)
|
||||||
|
if not res.get("ok"):
|
||||||
|
raise HTTPException(502, res.get("error") or f"head look failed (rc={res.get('rc')})")
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
@router.post("/cancel")
|
@router.post("/cancel")
|
||||||
async def cancel_motion():
|
async def cancel_motion():
|
||||||
from Project.Sanad.main import arm
|
from Project.Sanad.main import arm
|
||||||
|
|||||||
@ -173,10 +173,12 @@ async def update_api_key(payload: ApiKeyPayload):
|
|||||||
raise HTTPException(400, "API key cannot be empty.")
|
raise HTTPException(400, "API key cannot be empty.")
|
||||||
if len(key) < 20:
|
if len(key) < 20:
|
||||||
raise HTTPException(400, "API key looks too short.")
|
raise HTTPException(400, "API key looks too short.")
|
||||||
if not key.startswith("AIza"):
|
# Google AI Studio issues classic "AIza..." keys and (since late 2025)
|
||||||
|
# new-format "AQ...." keys — both are plain x-goog-api-key API keys.
|
||||||
|
if not key.startswith(("AIza", "AQ.")):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
400,
|
400,
|
||||||
"Gemini API keys normally start with 'AIza'. "
|
"Gemini API keys start with 'AIza' (classic) or 'AQ.' (new format). "
|
||||||
"Double-check you're pasting a Google AI Studio key.",
|
"Double-check you're pasting a Google AI Studio key.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@ -352,7 +352,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div id="running-action" style="font-size:.75rem;color:var(--accent);margin-bottom:.3rem;display:none"></div>
|
<div id="running-action" style="font-size:.75rem;color:var(--accent);margin-bottom:.3rem;display:none"></div>
|
||||||
<div id="sdk-actions" style="display:flex;flex-wrap:wrap;gap:3px;margin-top:.2rem"></div>
|
<div id="sdk-actions" style="display:flex;flex-wrap:wrap;gap:3px;margin-top:.2rem"></div>
|
||||||
<div id="jsonl-actions" style="display:flex;flex-wrap:wrap;gap:3px;margin-top:.4rem"></div>
|
<div id="jsonl-actions" style="display:none;flex-wrap:wrap;gap:3px;margin-top:.4rem"></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
@ -401,9 +401,15 @@
|
|||||||
<div class="row">
|
<div class="row">
|
||||||
<button class="btn btn-success" onclick="startLiveSub(this)">Start</button>
|
<button class="btn btn-success" onclick="startLiveSub(this)">Start</button>
|
||||||
<button class="btn btn-danger" onclick="stopLiveSub(this)">Stop</button>
|
<button class="btn btn-danger" onclick="stopLiveSub(this)">Stop</button>
|
||||||
|
<button class="btn" style="background:#7c3aed;color:#fff" onclick="instantInterrupt()" title="Instantly stop Sanad mid-sentence and go back to listening (works even with the Interrupt voice option off).">⏹ Interrupt now</button>
|
||||||
|
<button class="btn" style="background:#0e9f6e;color:#fff" onclick="answerNow()" title="Make Sanad answer RIGHT NOW instead of waiting for the end-of-speech pause.">➤ Answer now</button>
|
||||||
<button id="ls-cam-btn" class="btn btn-sm btn-ghost" onclick="toggleGeminiCamera(this)" title="Stream camera frames to Gemini Live — same toggle as the Recognition tab">Camera: --</button>
|
<button id="ls-cam-btn" class="btn btn-sm btn-ghost" onclick="toggleGeminiCamera(this)" title="Stream camera frames to Gemini Live — same toggle as the Recognition tab">Camera: --</button>
|
||||||
<button id="ls-rec-btn" class="btn btn-sm btn-ghost" onclick="toggleAutoRecord(this)" title="Auto-save every conversation turn to data/recordings/. Toggling takes effect live — no session restart.">Rec: --</button>
|
<button id="ls-rec-btn" class="btn btn-sm btn-ghost" onclick="toggleAutoRecord(this)" title="Auto-save every conversation turn to data/recordings/. Toggling takes effect live — no session restart.">Rec: --</button>
|
||||||
<span id="ls-state" class="badge"></span>
|
<span id="ls-state" class="badge"></span>
|
||||||
|
<span id="ls-pure" class="badge" onclick="togglePureOpt()" style="cursor:pointer" title="Pure voice: fastest replies (Sanadv1-style — no tools, no standing state, persona only). TOGGLE-AWARE: features you enable (Camera vision, Movement) still work live and announce themselves; disable them and it returns to pure speech.">Pure: …</span>
|
||||||
|
<span id="ls-direct" class="badge" onclick="toggleDirectOpt()" style="cursor:pointer" title="Direct answer: Sanad replies the moment you pause (fast turn-taking). OFF = waits longer to be sure you finished (better for slow/hesitant speakers). Toggling restarts the live session.">Direct: …</span>
|
||||||
|
<span id="ls-interrupt" class="badge" onclick="toggleInterruptOpt()" style="cursor:pointer" title="Voice interrupt (barge-in) — ALL speakers. OFF: Sanad always finishes his sentence, nobody can talk over him (R1 chest / JBL / Anker all behave the same). ON: loud sustained speech (~1s) cuts him mid-sentence. Echo-safe either way; the ⏹ Interrupt-now button always works. Toggling restarts the live session.">Interrupt: …</span>
|
||||||
|
<span id="ls-search" class="badge" onclick="toggleSearchOpt()" style="cursor:pointer" title="Internet search (Google grounding): Sanad can look up CURRENT facts — today's prices, the latest phones, live offers, news. When he needs to, he says 'let me search the internet', looks it up, then answers. OFF = pure voice, no search, fastest replies (recommended for a busy event). Toggling restarts the live session.">Search: …</span>
|
||||||
<span id="ls-pausemode" class="badge" onclick="toggleLiveHoldBadge()" style="cursor:pointer" title="Pause mode (same as the Saved Records 'Keep Gemini paused' toggle). Auto = pause only during a record, resume after. Manual = Gemini stays paused until you switch back. Click to toggle.">Pause: Auto</span>
|
<span id="ls-pausemode" class="badge" onclick="toggleLiveHoldBadge()" style="cursor:pointer" title="Pause mode (same as the Saved Records 'Keep Gemini paused' toggle). Auto = pause only during a record, resume after. Manual = Gemini stays paused until you switch back. Click to toggle.">Pause: Auto</span>
|
||||||
<button class="btn btn-sm mic-mute-shortcut btn-success" onclick="toggleMic()" style="margin-left:auto">Mic: LIVE</button>
|
<button class="btn btn-sm mic-mute-shortcut btn-success" onclick="toggleMic()" style="margin-left:auto">Mic: LIVE</button>
|
||||||
<button class="btn btn-sm spk-mute-shortcut btn-success" onclick="toggleSpeaker()">Speaker: LIVE</button>
|
<button class="btn btn-sm spk-mute-shortcut btn-success" onclick="toggleSpeaker()">Speaker: LIVE</button>
|
||||||
@ -504,7 +510,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div id="sdk-actions-2" class="action-list"></div>
|
<div id="sdk-actions-2" class="action-list"></div>
|
||||||
</div>
|
</div>
|
||||||
<div style="flex:1;min-width:260px">
|
<div style="flex:1;min-width:260px;display:none">
|
||||||
<div class="row" style="justify-content:space-between;align-items:center;margin:0 0 .3rem 0">
|
<div class="row" style="justify-content:space-between;align-items:center;margin:0 0 .3rem 0">
|
||||||
<label style="margin:0">JSONL Replays (recorded)</label>
|
<label style="margin:0">JSONL Replays (recorded)</label>
|
||||||
<button id="play-jsonl-btn" class="btn btn-primary btn-sm" onclick="playSelectedAction('jsonl')" disabled>Play</button>
|
<button id="play-jsonl-btn" class="btn btn-primary btn-sm" onclick="playSelectedAction('jsonl')" disabled>Play</button>
|
||||||
@ -883,6 +889,18 @@
|
|||||||
<h3>Diagnostics — joints 12–28</h3>
|
<h3>Diagnostics — joints 12–28</h3>
|
||||||
<pre id="ctrl-joints" style="font-size:.66rem;max-height:240px;overflow:auto;background:var(--panel2);border-radius:6px;padding:.5rem;margin:0"></pre>
|
<pre id="ctrl-joints" style="font-size:.66rem;max-height:240px;overflow:auto;background:var(--panel2);border-radius:6px;padding:.5rem;margin:0"></pre>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Head — recorded looks (R1 arm-service teach motions) -->
|
||||||
|
<div class="card">
|
||||||
|
<h3>Head — recorded looks</h3>
|
||||||
|
<div style="font-size:.7rem;color:var(--muted);margin-bottom:.45rem">
|
||||||
|
Replays your recorded head rotations via the R1 arm service. Firmware keeps the body balanced.
|
||||||
|
Gated by the same movement lock as arm actions.
|
||||||
|
</div>
|
||||||
|
<div class="row" id="ctrl-head-menu" style="flex-wrap:wrap;gap:.3rem">
|
||||||
|
<span class="empty">loading…</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@ -1617,7 +1635,10 @@ async function renderActions(arm){
|
|||||||
_armBusy=arm.busy||false;
|
_armBusy=arm.busy||false;
|
||||||
try{
|
try{
|
||||||
const r=await api('GET','/api/motion/actions');
|
const r=await api('GET','/api/motion/actions');
|
||||||
const acts=r.actions||[];
|
// R1 gestures only — the JSONL replay buttons are dropped (arm control now
|
||||||
|
// goes through the R1 'arm' service; recorded head looks live in the HEAD
|
||||||
|
// menu on the Controller tab).
|
||||||
|
const acts=(r.actions||[]).filter(a=>!a.file);
|
||||||
_renderChips(acts);
|
_renderChips(acts);
|
||||||
_renderList(acts);
|
_renderList(acts);
|
||||||
['','2'].forEach(sfx=>{const bb=document.getElementById('arm-busy-badge'+sfx);if(bb)bb.style.display=_armBusy?'inline-flex':'none';});
|
['','2'].forEach(sfx=>{const bb=document.getElementById('arm-busy-badge'+sfx);if(bb)bb.style.display=_armBusy?'inline-flex':'none';});
|
||||||
@ -2014,6 +2035,92 @@ async function reassertLiveHold(){try{const s=await api('GET','/api/records/play
|
|||||||
async function stopLiveSub(b){btnLoad(b);try{await api('POST','/api/live-subprocess/stop');toast('Stopped','info');}catch(e){}btnDone(b);refreshLiveSub();}
|
async function stopLiveSub(b){btnLoad(b);try{await api('POST','/api/live-subprocess/stop');toast('Stopped','info');}catch(e){}btnDone(b);refreshLiveSub();}
|
||||||
async function refreshLiveSub(){try{const r=await api('GET','/api/live-subprocess/status');const st=document.getElementById('ls-state');st.textContent=r.state||'stopped';st.className='badge '+(r.running?'badge-ok':'badge-warn');document.getElementById('ls-msg').textContent=r.state_message||'--';document.getElementById('ls-user').textContent=r.last_user_text||'--';document.getElementById('ls-log').textContent=(r.log_tail||[]).slice(-25).join('\n');const rb=document.getElementById('ls-rec-btn');if(rb){const on=!!r.record_enabled;rb.textContent='Rec: '+(on?'ON':'OFF');rb.className='btn btn-sm '+(on?'btn-success':'btn-ghost');}}catch(e){}}
|
async function refreshLiveSub(){try{const r=await api('GET','/api/live-subprocess/status');const st=document.getElementById('ls-state');st.textContent=r.state||'stopped';st.className='badge '+(r.running?'badge-ok':'badge-warn');document.getElementById('ls-msg').textContent=r.state_message||'--';document.getElementById('ls-user').textContent=r.last_user_text||'--';document.getElementById('ls-log').textContent=(r.log_tail||[]).slice(-25).join('\n');const rb=document.getElementById('ls-rec-btn');if(rb){const on=!!r.record_enabled;rb.textContent='Rec: '+(on?'ON':'OFF');rb.className='btn btn-sm '+(on?'btn-success':'btn-ghost');}}catch(e){}}
|
||||||
async function toggleAutoRecord(b){const cur=(b&&b.textContent||'').includes('ON');const next=!cur;btnLoad(b);try{const r=await api('POST','/api/live-subprocess/record?on='+(next?'1':'0'));toast('Auto-recording '+(r.record_enabled?'ON':'OFF'),'ok');}catch(e){toast('Toggle failed: '+(e.message||e),'err');}btnDone(b);refreshLiveSub();}
|
async function toggleAutoRecord(b){const cur=(b&&b.textContent||'').includes('ON');const next=!cur;btnLoad(b);try{const r=await api('POST','/api/live-subprocess/record?on='+(next?'1':'0'));toast('Auto-recording '+(r.record_enabled?'ON':'OFF'),'ok');}catch(e){toast('Toggle failed: '+(e.message||e),'err');}btnDone(b);refreshLiveSub();}
|
||||||
|
// Voice-option badges (Pure / Search / Direct / Interrupt) + instant buttons —
|
||||||
|
// ported from Sanad G1; backed by /api/live-subprocess/*-option endpoints.
|
||||||
|
let _pureOpt=null;
|
||||||
|
function _paintPureBadge(){
|
||||||
|
const b=document.getElementById('ls-pure');
|
||||||
|
if(!b)return;
|
||||||
|
b.textContent='Pure: '+(_pureOpt===null?'…':(_pureOpt?'ON':'OFF'));
|
||||||
|
b.className='badge '+(_pureOpt?'badge-ok':'');
|
||||||
|
}
|
||||||
|
async function refreshPureOpt(){
|
||||||
|
try{const r=await api('GET','/api/live-subprocess/pure-voice-option');_pureOpt=!!r.enabled;_paintPureBadge();}catch(e){}
|
||||||
|
}
|
||||||
|
async function togglePureOpt(){
|
||||||
|
const want=!_pureOpt;
|
||||||
|
try{
|
||||||
|
const r=await api('POST','/api/live-subprocess/pure-voice-option?on='+want);
|
||||||
|
_pureOpt=!!r.enabled;_paintPureBadge();
|
||||||
|
toast('Pure voice '+(_pureOpt?'ON — Sanadv1-style session':'OFF — full features')+(r.restarted?' (session restarted)':''));
|
||||||
|
}catch(e){toast('Pure toggle failed: '+(e.message||e));}
|
||||||
|
}
|
||||||
|
setTimeout(refreshPureOpt,1000);
|
||||||
|
let _searchOpt=null;
|
||||||
|
function _paintSearchBadge(){
|
||||||
|
const b=document.getElementById('ls-search');
|
||||||
|
if(!b)return;
|
||||||
|
b.textContent='Search: '+(_searchOpt===null?'…':(_searchOpt?'ON':'OFF'));
|
||||||
|
b.className='badge '+(_searchOpt?'badge-ok':'');
|
||||||
|
}
|
||||||
|
async function refreshSearchOpt(){
|
||||||
|
try{const r=await api('GET','/api/live-subprocess/internet-search-option');_searchOpt=!!r.enabled;_paintSearchBadge();}catch(e){}
|
||||||
|
}
|
||||||
|
async function toggleSearchOpt(){
|
||||||
|
const want=!_searchOpt;
|
||||||
|
try{
|
||||||
|
const r=await api('POST','/api/live-subprocess/internet-search-option?on='+want);
|
||||||
|
_searchOpt=!!r.enabled;_paintSearchBadge();
|
||||||
|
toast('Internet search '+(_searchOpt?'ON — Sanad can look up current info':'OFF — pure voice, no search')+(r.restarted?' (session restarted)':''));
|
||||||
|
}catch(e){toast('Search toggle failed: '+(e.message||e));}
|
||||||
|
}
|
||||||
|
setTimeout(refreshSearchOpt,1050);
|
||||||
|
let _dirOpt=null;
|
||||||
|
function _paintDirectBadge(){
|
||||||
|
const b=document.getElementById('ls-direct');
|
||||||
|
if(!b)return;
|
||||||
|
b.textContent='Direct: '+(_dirOpt===null?'…':(_dirOpt?'ON':'OFF'));
|
||||||
|
b.className='badge '+(_dirOpt?'badge-ok':'');
|
||||||
|
}
|
||||||
|
async function refreshDirectOpt(){
|
||||||
|
try{const r=await api('GET','/api/live-subprocess/direct-answer-option');_dirOpt=!!r.enabled;_paintDirectBadge();}catch(e){}
|
||||||
|
}
|
||||||
|
async function toggleDirectOpt(){
|
||||||
|
const want=!_dirOpt;
|
||||||
|
try{
|
||||||
|
const r=await api('POST','/api/live-subprocess/direct-answer-option?on='+want);
|
||||||
|
_dirOpt=!!r.enabled;_paintDirectBadge();
|
||||||
|
toast('Direct answer '+(_dirOpt?'ON':'OFF')+(r.restarted?' — live session restarted':''));
|
||||||
|
}catch(e){toast('Direct toggle failed: '+(e.message||e));}
|
||||||
|
}
|
||||||
|
setTimeout(refreshDirectOpt,950);
|
||||||
|
let _intOpt=null;
|
||||||
|
function _paintInterruptBadge(){
|
||||||
|
const b=document.getElementById('ls-interrupt');
|
||||||
|
if(!b)return;
|
||||||
|
b.textContent='Interrupt: '+(_intOpt===null?'…':(_intOpt?'ON':'OFF'));
|
||||||
|
b.className='badge '+(_intOpt?'badge-ok':'');
|
||||||
|
}
|
||||||
|
async function refreshInterruptOpt(){
|
||||||
|
try{const r=await api('GET','/api/live-subprocess/interrupt-option');_intOpt=!!r.enabled;_paintInterruptBadge();}catch(e){}
|
||||||
|
}
|
||||||
|
async function answerNow(){
|
||||||
|
try{await api('POST','/api/live-subprocess/answer-now');toast('Answering now…');}
|
||||||
|
catch(e){toast('Answer-now failed: '+(e.message||e));}
|
||||||
|
}
|
||||||
|
async function instantInterrupt(){
|
||||||
|
try{await api('POST','/api/live-subprocess/interrupt');toast('Interrupted — listening');}
|
||||||
|
catch(e){toast('Interrupt failed: '+(e.message||e));}
|
||||||
|
}
|
||||||
|
async function toggleInterruptOpt(){
|
||||||
|
const want=!_intOpt;
|
||||||
|
try{
|
||||||
|
const r=await api('POST','/api/live-subprocess/interrupt-option?on='+want);
|
||||||
|
_intOpt=!!r.enabled;_paintInterruptBadge();
|
||||||
|
toast('Interrupt '+(_intOpt?'ON':'OFF')+(r.restarted?' — live session restarted':''));
|
||||||
|
}catch(e){toast('Interrupt toggle failed: '+(e.message||e));}
|
||||||
|
}
|
||||||
|
setTimeout(refreshInterruptOpt,900);
|
||||||
|
|
||||||
// Typed Replay
|
// Typed Replay
|
||||||
async function trGenerate(b){const t=document.getElementById('tr-text').value;if(!t)return toast('Enter text','err');btnLoad(b);try{await api('POST','/api/typed-replay/say',{text:t,record:document.getElementById('tr-capture').checked,record_name:document.getElementById('tr-name').value});toast('Generated & played','ok');refreshTR();}catch(e){}btnDone(b);}
|
async function trGenerate(b){const t=document.getElementById('tr-text').value;if(!t)return toast('Enter text','err');btnLoad(b);try{await api('POST','/api/typed-replay/say',{text:t,record:document.getElementById('tr-capture').checked,record_name:document.getElementById('tr-name').value});toast('Generated & played','ok');refreshTR();}catch(e){}btnDone(b);}
|
||||||
@ -2365,7 +2472,31 @@ function stopRecPreview(){
|
|||||||
img.src=''; // closes the MJPEG connection
|
img.src=''; // closes the MJPEG connection
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Hook into tab switch — start/stop preview when recognition tab is active.
|
// R1 head-look menu (Controller tab) — replays your recorded head rotations via
|
||||||
|
// the R1 arm service (ExecuteCustomAction). Loaded once when the tab opens.
|
||||||
|
let _ctrlHeadLoaded=false;
|
||||||
|
async function ctrlLoadHead(){
|
||||||
|
const el=document.getElementById('ctrl-head-menu'); if(!el) return;
|
||||||
|
try{
|
||||||
|
const r=await api('GET','/api/motion/head-actions');
|
||||||
|
const looks=r.looks||[];
|
||||||
|
const label={up_look:'↑ up',down_look:'↓ down',left_look:'← left',right_look:'→ right',
|
||||||
|
up_down_look:'↕ up-down',left_right_look:'⇄ left-right',right_left_look:'⇄ right-left',center:'⊙ center'};
|
||||||
|
el.innerHTML=looks.length
|
||||||
|
? looks.map(n=>`<button class="btn btn-ghost btn-sm" onclick="ctrlHeadLook('${esc(n)}',this)">${label[n]||esc(n).replace(/_/g,' ')}</button>`).join('')
|
||||||
|
: '<span class="empty">No recorded head looks</span>';
|
||||||
|
_ctrlHeadLoaded=true;
|
||||||
|
}catch(e){ el.innerHTML='<span class="empty">head menu unavailable</span>'; }
|
||||||
|
}
|
||||||
|
async function ctrlHeadLook(name,btn){
|
||||||
|
if(btn) btnLoad(btn);
|
||||||
|
try{ await api('POST','/api/motion/head?name='+encodeURIComponent(name)); toast('Head → '+name.replace(/_/g,' '),'ok'); }
|
||||||
|
catch(e){ toast('Head look failed: '+(e.message||e),'err'); }
|
||||||
|
if(btn) btnDone(btn);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hook into tab switch — start/stop preview when recognition tab is active,
|
||||||
|
// and lazy-load the Controller head menu on first open.
|
||||||
(function(){
|
(function(){
|
||||||
const origSwitchTab=window.switchTab;
|
const origSwitchTab=window.switchTab;
|
||||||
window.switchTab=function(name){
|
window.switchTab=function(name){
|
||||||
@ -2373,6 +2504,7 @@ function stopRecPreview(){
|
|||||||
_recTabActive=(name==='recognition');
|
_recTabActive=(name==='recognition');
|
||||||
if(name==='recognition'){refreshRecognition();refreshFaces();refreshZones();startRecPreview();}
|
if(name==='recognition'){refreshRecognition();refreshFaces();refreshZones();startRecPreview();}
|
||||||
else{stopRecPreview();}
|
else{stopRecPreview();}
|
||||||
|
if(name==='controller'&&!_ctrlHeadLoaded) ctrlLoadHead();
|
||||||
};
|
};
|
||||||
})();
|
})();
|
||||||
// Face CRUD stubs — filled in milestone 5
|
// Face CRUD stubs — filled in milestone 5
|
||||||
|
|||||||
309
gemini/script.py
309
gemini/script.py
@ -55,11 +55,23 @@ _MODEL = os.environ.get(
|
|||||||
"SANAD_GEMINI_MODEL",
|
"SANAD_GEMINI_MODEL",
|
||||||
"gemini-2.5-flash-native-audio-preview-12-2025",
|
"gemini-2.5-flash-native-audio-preview-12-2025",
|
||||||
)
|
)
|
||||||
_MIC_GAIN = _SV.get("mic_gain", 1.0)
|
# Mic gain: amplifies mic samples before they reach Gemini — the single
|
||||||
|
# biggest lever for slow/failed speech detection at a distance (weak audio
|
||||||
|
# transcribes but often yields NO reply turn). Env override for per-robot
|
||||||
|
# tuning; watch for clipping if the speaker stands close (>3.0 rarely wise).
|
||||||
|
try:
|
||||||
|
_MIC_GAIN = float(os.environ.get("SANAD_MIC_GAIN", "") or _SV.get("mic_gain", 1.0))
|
||||||
|
except ValueError:
|
||||||
|
_MIC_GAIN = _SV.get("mic_gain", 1.0)
|
||||||
_SESSION_TIMEOUT = _SV.get("session_timeout_sec", 660)
|
_SESSION_TIMEOUT = _SV.get("session_timeout_sec", 660)
|
||||||
_MAX_RECONNECT_DELAY = _SV.get("max_reconnect_delay_sec", 30)
|
_MAX_RECONNECT_DELAY = _SV.get("max_reconnect_delay_sec", 30)
|
||||||
_MAX_CONSECUTIVE_ERRORS = _SV.get("max_consecutive_errors", 10)
|
_MAX_CONSECUTIVE_ERRORS = _SV.get("max_consecutive_errors", 10)
|
||||||
_NO_MESSAGES_TIMEOUT = _SV.get("no_messages_timeout_sec", 30)
|
# How long with ZERO messages from Gemini before we declare the session dead.
|
||||||
|
# In a QUIET room Gemini legitimately sends nothing — 30s was killing healthy
|
||||||
|
# sessions every ~40s (reconnect churn + unsolicited greeting + a 1-3s deaf
|
||||||
|
# window each time). REAL deaths are caught fast by the send path ("mic send
|
||||||
|
# failed: 1011 ..."), so this can be patient.
|
||||||
|
_NO_MESSAGES_TIMEOUT = _SV.get("no_messages_timeout_sec", 180)
|
||||||
# Extra mic-gate time after the AI stops, on loud external-speaker profiles
|
# Extra mic-gate time after the AI stops, on loud external-speaker profiles
|
||||||
# (JBL) — covers the speaker buffer + room reverb so it doesn't hear its tail.
|
# (JBL) — covers the speaker buffer + room reverb so it doesn't hear its tail.
|
||||||
_ECHO_TAIL_SEC = _SV.get("echo_tail_sec", 0.6)
|
_ECHO_TAIL_SEC = _SV.get("echo_tail_sec", 0.6)
|
||||||
@ -77,6 +89,70 @@ _JBL_BARGE_CHUNKS = _SV.get("jbl_barge_chunks", 9)
|
|||||||
# FADED (a gap between words/numbers). In that window barge-in drops to a low,
|
# FADED (a gap between words/numbers). In that window barge-in drops to a low,
|
||||||
# sensitive bar so the user can interrupt; while audio is flowing it stays high.
|
# sensitive bar so the user can interrupt; while audio is flowing it stays high.
|
||||||
_JBL_BLEED_FADE_SEC = _SV.get("jbl_bleed_fade_sec", 0.5)
|
_JBL_BLEED_FADE_SEC = _SV.get("jbl_bleed_fade_sec", 0.5)
|
||||||
|
# OPTION: voice interrupt (barge-in) — GLOBAL, all speakers. OFF (default):
|
||||||
|
# Sanad always finishes his sentence — the mic to Gemini is fully gated while
|
||||||
|
# he speaks (+ echo tail) on EVERY profile (chest/Anker/JBL/Beats Pill), so
|
||||||
|
# neither the local energy detector nor Gemini's server VAD can cut him.
|
||||||
|
# ON: loud sustained speech interrupts him — builtin/Anker use the normal
|
||||||
|
# barge parameters; loud external speakers (JBL / Beats Pill) use the stricter
|
||||||
|
# external parameters and stay fully gated for echo safety. The dashboard
|
||||||
|
# ⏹ Interrupt-now button works regardless. Env > config; default OFF.
|
||||||
|
_EXTERNAL_BARGE_IN = (
|
||||||
|
os.environ.get("SANAD_EXTERNAL_BARGE_IN", "").strip().lower() in ("1", "true", "yes")
|
||||||
|
or bool(_BI.get("external_barge_in", False))
|
||||||
|
)
|
||||||
|
# OPTION: "direct answer" — reply as soon as the speaker pauses. Flips the
|
||||||
|
# Live-API end-of-speech sensitivity from LOW (conservative: waits to be sure
|
||||||
|
# you finished, feels slow) to HIGH (immediate turn-taking; may cut off very
|
||||||
|
# hesitant speakers mid-thought). Env > config; dashboard file has final say.
|
||||||
|
_DIRECT_ANSWER = (
|
||||||
|
os.environ.get("SANAD_DIRECT_ANSWER", "").strip().lower() in ("1", "true", "yes")
|
||||||
|
or bool(_VAD.get("direct_answer", False))
|
||||||
|
)
|
||||||
|
|
||||||
|
# OPTION: PURE VOICE — make the live session EXACTLY like standalone Sanadv1:
|
||||||
|
# no tools (no function-calling deliberation), no state injections, persona
|
||||||
|
# only, no lip-sync markers, no frame/state loops. The fastest possible
|
||||||
|
# conversation pipe; nav/vision hooks are dormant.
|
||||||
|
_PURE_VOICE = (
|
||||||
|
os.environ.get("SANAD_PURE_VOICE", "").strip().lower() in ("1", "true", "yes")
|
||||||
|
or bool(_SV.get("pure_voice", False))
|
||||||
|
)
|
||||||
|
|
||||||
|
# OPTION: INTERNET SEARCH (Google Search grounding). When ON, the live session
|
||||||
|
# gets ONE extra tool, `search_internet(query)`, so the model can look up
|
||||||
|
# CURRENT facts (today's prices, the newest phones, live Etisalat offers, news).
|
||||||
|
# The tool description tells the model to SAY it will check first, then call the
|
||||||
|
# tool — so the robot announces "let me search the internet", the parent runs
|
||||||
|
# the grounded generateContent call in a worker thread, and the model speaks the
|
||||||
|
# result. Default OFF → pure voice stays tool-free and Sanadv1-fast; only when
|
||||||
|
# a search actually fires does the (announced) grounding delay occur.
|
||||||
|
_INTERNET_SEARCH = (
|
||||||
|
os.environ.get("SANAD_INTERNET_SEARCH", "").strip().lower() in ("1", "true", "yes")
|
||||||
|
or bool(_SV.get("internet_search", False))
|
||||||
|
)
|
||||||
|
# Model used for the grounded web-search call (generateContent, NOT the live
|
||||||
|
# native-audio model). gemini-3.5-flash grounds well and cites the source site.
|
||||||
|
_SEARCH_MODEL = os.environ.get("SANAD_SEARCH_MODEL", "gemini-3.5-flash")
|
||||||
|
|
||||||
|
# The dashboard toggles (data/.voice_options.json, written by the
|
||||||
|
# /api/live-subprocess/*-option endpoints, which then restart this child)
|
||||||
|
# have the FINAL say over env/config.
|
||||||
|
try:
|
||||||
|
import json as _vo_json
|
||||||
|
with open(BASE_DIR / "data" / ".voice_options.json", encoding="utf-8") as _vo_f:
|
||||||
|
_vo = _vo_json.load(_vo_f)
|
||||||
|
if isinstance(_vo, dict):
|
||||||
|
if "external_barge_in" in _vo:
|
||||||
|
_EXTERNAL_BARGE_IN = bool(_vo["external_barge_in"])
|
||||||
|
if "direct_answer" in _vo:
|
||||||
|
_DIRECT_ANSWER = bool(_vo["direct_answer"])
|
||||||
|
if "pure_voice" in _vo:
|
||||||
|
_PURE_VOICE = bool(_vo["pure_voice"])
|
||||||
|
if "internet_search" in _vo:
|
||||||
|
_INTERNET_SEARCH = bool(_vo["internet_search"])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
_CHUNK_BYTES = CHUNK_SIZE * 2
|
_CHUNK_BYTES = CHUNK_SIZE * 2
|
||||||
_SILENCE_PCM = b"\x00" * _CHUNK_BYTES
|
_SILENCE_PCM = b"\x00" * _CHUNK_BYTES
|
||||||
@ -88,6 +164,15 @@ _SILENCE_PCM = b"\x00" * _CHUNK_BYTES
|
|||||||
# safe to read from the asyncio loops.
|
# safe to read from the asyncio loops.
|
||||||
_INPUT_PAUSED = threading.Event()
|
_INPUT_PAUSED = threading.Event()
|
||||||
|
|
||||||
|
# Set by an "interrupt:" stdin command (the dashboard's Interrupt-now button).
|
||||||
|
# The mic loop consumes it: stops the speaker mid-sentence, back to listening.
|
||||||
|
_MANUAL_INTERRUPT = threading.Event()
|
||||||
|
|
||||||
|
# Set by an "answernow:" stdin command (the dashboard's Answer-now button).
|
||||||
|
# The mic loop feeds ~0.6s of silence so the server VAD closes the user's turn
|
||||||
|
# immediately → Gemini answers right away instead of waiting out the pause.
|
||||||
|
_FORCE_ANSWER = threading.Event()
|
||||||
|
|
||||||
# ── Recognition (camera + face gallery) tunables ──
|
# ── Recognition (camera + face gallery) tunables ──
|
||||||
_RECOG_STATE_PATH = Path(os.environ.get(
|
_RECOG_STATE_PATH = Path(os.environ.get(
|
||||||
"SANAD_RECOGNITION_STATE_PATH",
|
"SANAD_RECOGNITION_STATE_PATH",
|
||||||
@ -195,6 +280,76 @@ def _nav_function_declarations() -> list:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _search_function_declaration() -> list:
|
||||||
|
"""Live tool: search the internet (Google Search grounding) for current facts."""
|
||||||
|
S, T = types.Schema, types.Type
|
||||||
|
return [
|
||||||
|
types.FunctionDeclaration(
|
||||||
|
name="search_internet",
|
||||||
|
description=(
|
||||||
|
"Search the LIVE internet for CURRENT, real-time facts you cannot "
|
||||||
|
"otherwise know: today's prices, the newest phone / device models, "
|
||||||
|
"current Etisalat / e& offers and promotions, product availability, "
|
||||||
|
"or recent news. IMPORTANT: BEFORE you call this, first SAY one "
|
||||||
|
"short sentence out loud telling the user you will check — e.g. "
|
||||||
|
"\"Let me search the internet for that.\" / "
|
||||||
|
"\"دعني أبحث لك على الإنترنت.\" — "
|
||||||
|
"then call this with a clear English search query. After the "
|
||||||
|
"result returns, answer the user naturally from it. Do NOT say the "
|
||||||
|
"tool name out loud. Only use this for genuinely current / "
|
||||||
|
"time-sensitive questions, not for things you already know."
|
||||||
|
),
|
||||||
|
parameters=S(type=T.OBJECT, properties={
|
||||||
|
"query": S(type=T.STRING,
|
||||||
|
description="The web-search query, phrased in English."),
|
||||||
|
}, required=["query"]),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# Appended to the persona ONLY when internet search is enabled, so the model
|
||||||
|
# knows it can (and how to) look things up. Kept short — the tool description
|
||||||
|
# carries the detail.
|
||||||
|
_SEARCH_PROMPT_ADDENDUM = (
|
||||||
|
"\n\nYou can look things up on the live internet with the search_internet "
|
||||||
|
"tool. Use it whenever the user asks about current prices, the latest phones "
|
||||||
|
"or devices, today's Etisalat/e& offers, availability, or recent news — "
|
||||||
|
"anything you can't be certain is up to date. ALWAYS say a short 'let me "
|
||||||
|
"check the internet' line first (in the user's language), then call the tool, "
|
||||||
|
"then answer from the result. Keep answers concise and, for exact prices, add "
|
||||||
|
"that a sales advisor can confirm the final figure. "
|
||||||
|
"CRITICAL: after the search result returns, CONTINUE where you left off — "
|
||||||
|
"NEVER repeat or re-say any sentence you already spoke, including the 'let me "
|
||||||
|
"check the internet' lead-in. If your reply is complete, stop and wait."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_grounding_search(query: str, api_key: str) -> dict:
|
||||||
|
"""Blocking Google-Search-grounded generateContent call (run in a thread).
|
||||||
|
Returns {ok, answer} for the model to speak, or {ok: False, reason}."""
|
||||||
|
try:
|
||||||
|
client = genai.Client(api_key=api_key)
|
||||||
|
resp = client.models.generate_content(
|
||||||
|
model=_SEARCH_MODEL,
|
||||||
|
contents=query,
|
||||||
|
config=types.GenerateContentConfig(
|
||||||
|
tools=[types.Tool(google_search=types.GoogleSearch())],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
answer = (getattr(resp, "text", "") or "").strip()
|
||||||
|
if not answer:
|
||||||
|
return {"ok": False, "reason": "empty_result"}
|
||||||
|
# Cap length so the spoken reply stays reasonable.
|
||||||
|
return {"ok": True, "answer": answer[:1500]}
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("internet search failed: %s", exc)
|
||||||
|
return {"ok": False, "reason": "search_failed", "detail": str(exc)[:200]}
|
||||||
|
|
||||||
|
|
||||||
|
# NOTE (SanadR1): the LED-mask face/social-QR tools (_FACE_EMOTIONS,
|
||||||
|
# set_expression, show_social) were removed in the R1 port — the R1 has no mask.
|
||||||
|
|
||||||
|
|
||||||
# ── stdin push channel (Marcus pattern) ──────────────────────
|
# ── stdin push channel (Marcus pattern) ──────────────────────
|
||||||
# The GeminiSubprocess supervisor writes two line types to this process's
|
# The GeminiSubprocess supervisor writes two line types to this process's
|
||||||
# stdin:
|
# stdin:
|
||||||
@ -233,7 +388,7 @@ _PROFILE_LOCK = threading.Lock()
|
|||||||
_PROFILE_PENDING: dict = {"id": None, "reason": ""}
|
_PROFILE_PENDING: dict = {"id": None, "reason": ""}
|
||||||
|
|
||||||
_VALID_PROFILES = {"builtin", "anker", "anker_powerconf",
|
_VALID_PROFILES = {"builtin", "anker", "anker_powerconf",
|
||||||
"hollyland_builtin", "jbl_builtin_mic"}
|
"hollyland_builtin", "jbl_builtin_mic", "beats_pill"}
|
||||||
|
|
||||||
|
|
||||||
def _stdin_watcher() -> None:
|
def _stdin_watcher() -> None:
|
||||||
@ -266,6 +421,10 @@ def _stdin_watcher() -> None:
|
|||||||
with _LATEST_FRAME_LOCK:
|
with _LATEST_FRAME_LOCK:
|
||||||
_LATEST_FRAME["bytes"] = data
|
_LATEST_FRAME["bytes"] = data
|
||||||
_LATEST_FRAME["ts"] = time.time()
|
_LATEST_FRAME["ts"] = time.time()
|
||||||
|
elif line.startswith("interrupt:"):
|
||||||
|
_MANUAL_INTERRUPT.set()
|
||||||
|
elif line.startswith("answernow:"):
|
||||||
|
_FORCE_ANSWER.set()
|
||||||
elif line.startswith("state:"):
|
elif line.startswith("state:"):
|
||||||
try:
|
try:
|
||||||
payload = json.loads(line[len("state:"):])
|
payload = json.loads(line[len("state:"):])
|
||||||
@ -349,7 +508,12 @@ class GeminiBrain:
|
|||||||
self._swap_lock: Optional[asyncio.Lock] = None # built in run()
|
self._swap_lock: Optional[asyncio.Lock] = None # built in run()
|
||||||
self._recorder = recorder
|
self._recorder = recorder
|
||||||
self._voice = voice_name or GEMINI_VOICE
|
self._voice = voice_name or GEMINI_VOICE
|
||||||
|
# R1: no LED-mask face, so no face addendum in any mode.
|
||||||
self._system_prompt = system_prompt or ""
|
self._system_prompt = system_prompt or ""
|
||||||
|
# Internet-search addendum is independent of pure voice — the search
|
||||||
|
# tool is available in either mode when the option is enabled.
|
||||||
|
if _INTERNET_SEARCH:
|
||||||
|
self._system_prompt += _SEARCH_PROMPT_ADDENDUM
|
||||||
self._api_key = GEMINI_API_KEY
|
self._api_key = GEMINI_API_KEY
|
||||||
self._stop_flag = asyncio.Event()
|
self._stop_flag = asyncio.Event()
|
||||||
# per-session state (reset in the outer reconnect loop)
|
# per-session state (reset in the outer reconnect loop)
|
||||||
@ -443,6 +607,10 @@ class GeminiBrain:
|
|||||||
if self._swap_lock is None:
|
if self._swap_lock is None:
|
||||||
self._swap_lock = asyncio.Lock()
|
self._swap_lock = asyncio.Lock()
|
||||||
|
|
||||||
|
# PURE VOICE is TOGGLE-AWARE: all loops run (cheap when
|
||||||
|
# their feature is off) — pure-ness is enforced per-feature
|
||||||
|
# (announce guards + tools) so enabling Camera/Movement in
|
||||||
|
# the dashboard works live, and disabling returns to pure.
|
||||||
try:
|
try:
|
||||||
await asyncio.wait_for(
|
await asyncio.wait_for(
|
||||||
asyncio.gather(
|
asyncio.gather(
|
||||||
@ -505,10 +673,15 @@ class GeminiBrain:
|
|||||||
types.StartSensitivity,
|
types.StartSensitivity,
|
||||||
_VAD.get("start_sensitivity", "START_SENSITIVITY_HIGH"),
|
_VAD.get("start_sensitivity", "START_SENSITIVITY_HIGH"),
|
||||||
),
|
),
|
||||||
end_of_speech_sensitivity=getattr(
|
# "Direct answer" option: HIGH = reply the moment the
|
||||||
types.EndSensitivity,
|
# speaker pauses; otherwise the configured (LOW) default
|
||||||
_VAD.get("end_sensitivity", "END_SENSITIVITY_LOW"),
|
# waits longer to be sure the speaker finished.
|
||||||
),
|
end_of_speech_sensitivity=(
|
||||||
|
types.EndSensitivity.END_SENSITIVITY_HIGH
|
||||||
|
if _DIRECT_ANSWER else getattr(
|
||||||
|
types.EndSensitivity,
|
||||||
|
_VAD.get("end_sensitivity", "END_SENSITIVITY_LOW"),
|
||||||
|
)),
|
||||||
prefix_padding_ms=_VAD.get("prefix_padding_ms", 20),
|
prefix_padding_ms=_VAD.get("prefix_padding_ms", 20),
|
||||||
silence_duration_ms=_VAD.get("silence_duration_ms", 200),
|
silence_duration_ms=_VAD.get("silence_duration_ms", 200),
|
||||||
),
|
),
|
||||||
@ -518,13 +691,26 @@ class GeminiBrain:
|
|||||||
system_instruction=types.Content(
|
system_instruction=types.Content(
|
||||||
parts=[types.Part(text=self._system_prompt)],
|
parts=[types.Part(text=self._system_prompt)],
|
||||||
),
|
),
|
||||||
# Native function-calling: Gemini can drive the robot to saved
|
# Native function-calling: nav tools (if enabled) + the internet-
|
||||||
# places (navigate_to_place / list_places / where_am_i /
|
# search tool (if enabled). No face tools on the R1 (no mask).
|
||||||
# stop_navigation). Disable with SANAD_NAV_TOOLS=0.
|
# PURE VOICE: no tools (the reply-latency tax) — EXCEPT nav tools
|
||||||
tools=([types.Tool(function_declarations=_nav_function_declarations())]
|
# when movement is enabled (walking commands need them).
|
||||||
if _NAV_TOOLS_ENABLED else []),
|
tools=self._build_tools(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _build_tools(self) -> Any:
|
||||||
|
"""Assemble the live-session function tools. PURE VOICE keeps this None
|
||||||
|
(the reply-latency tax) UNLESS movement is on (nav tools) or internet
|
||||||
|
search is on (the search tool). R1: no face tools (no mask)."""
|
||||||
|
decls: list = []
|
||||||
|
# Nav tools: whenever NOT (pure and not moving) and nav is enabled.
|
||||||
|
if _NAV_TOOLS_ENABLED and (self._movement_enabled or not _PURE_VOICE):
|
||||||
|
decls += _nav_function_declarations()
|
||||||
|
# Internet-search tool: whenever the option is enabled (either mode).
|
||||||
|
if _INTERNET_SEARCH:
|
||||||
|
decls += _search_function_declaration()
|
||||||
|
return [types.Tool(function_declarations=decls)] if decls else None
|
||||||
|
|
||||||
# ─── state helpers ────────────────────────────────────
|
# ─── state helpers ────────────────────────────────────
|
||||||
|
|
||||||
def _reset_turn_state(self) -> None:
|
def _reset_turn_state(self) -> None:
|
||||||
@ -564,6 +750,7 @@ class GeminiBrain:
|
|||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
loud_count = 0
|
loud_count = 0
|
||||||
last_activity = time.time()
|
last_activity = time.time()
|
||||||
|
force_silence_until = 0.0 # Answer-now button: silence burst window
|
||||||
|
|
||||||
while not self._done.is_set() and not self._stop_flag.is_set():
|
while not self._done.is_set() and not self._stop_flag.is_set():
|
||||||
try:
|
try:
|
||||||
@ -578,6 +765,27 @@ class GeminiBrain:
|
|||||||
data = samples.tobytes()
|
data = samples.tobytes()
|
||||||
energy = _audio_energy(data)
|
energy = _audio_energy(data)
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
|
||||||
|
# Dashboard Answer-now button: feed the server VAD a silence burst
|
||||||
|
# so it closes the user's turn NOW → immediate reply.
|
||||||
|
if _FORCE_ANSWER.is_set():
|
||||||
|
_FORCE_ANSWER.clear()
|
||||||
|
if not self._speaking:
|
||||||
|
force_silence_until = now + 0.6
|
||||||
|
log.info("ANSWER NOW (dashboard) — forcing turn end")
|
||||||
|
|
||||||
|
# Dashboard Interrupt-now button: cut speech instantly AND stay
|
||||||
|
# SILENT until the user speaks — arming the greeting suppression
|
||||||
|
# also swallows any in-flight or reconnect-greeting audio, so a
|
||||||
|
# press while "listening" can never be followed by Sanad talking
|
||||||
|
# on his own.
|
||||||
|
if _MANUAL_INTERRUPT.is_set():
|
||||||
|
_MANUAL_INTERRUPT.clear()
|
||||||
|
log.info("MANUAL INTERRUPT (dashboard)%s",
|
||||||
|
"" if self._speaking else " — idle; muted until user speaks")
|
||||||
|
self._interrupt("manual")
|
||||||
|
self._barge_block_until = now + cooldown
|
||||||
|
self._suppress_greeting = True
|
||||||
# On the JBL (loud external speaker) the head mic hears the robot's
|
# On the JBL (loud external speaker) the head mic hears the robot's
|
||||||
# OWN voice as loud as the user. We FULLY gate the mic to Gemini while
|
# OWN voice as loud as the user. We FULLY gate the mic to Gemini while
|
||||||
# it speaks (+ a short echo tail) so it NEVER hears itself, and we
|
# it speaks (+ a short echo tail) so it NEVER hears itself, and we
|
||||||
@ -586,24 +794,47 @@ class GeminiBrain:
|
|||||||
# model. (Reliable JBL interrupt needs AEC; the only PulseAudio mic is
|
# model. (Reliable JBL interrupt needs AEC; the only PulseAudio mic is
|
||||||
# dead, so that's separate work.) The chest speaker (builtin) keeps
|
# dead, so that's separate work.) The chest speaker (builtin) keeps
|
||||||
# light quiet-frame suppression + working barge-in (firmware AEC).
|
# light quiet-frame suppression + working barge-in (firmware AEC).
|
||||||
full_gate = "jbl" in (self._current_profile_id or "")
|
# The Beats Pill (USB) is the same regime as the JBL: loud external
|
||||||
|
# speaker with a raw AEC-less mic inches away — the mic feed to
|
||||||
|
# Gemini is ALWAYS fully gated while speaking (echo-safe). Voice
|
||||||
|
# barge-in on external speakers is an OPTION (_EXTERNAL_BARGE_IN):
|
||||||
|
# the LOCAL detector cuts the speaker using stricter parameters.
|
||||||
|
external = any(t in (self._current_profile_id or "")
|
||||||
|
for t in ("jbl", "beats"))
|
||||||
|
# INTERRUPT OFF → NOBODY interrupts, on ANY speaker: every profile
|
||||||
|
# gets the Beats-Pill regime (mic fully gated while speaking + tail,
|
||||||
|
# so Gemini hears silence → no server-side cut; local barge-in
|
||||||
|
# disabled → no energy cut). Sanad always finishes his sentence.
|
||||||
|
# INTERRUPT ON → local barge-in works everywhere: builtin/Anker use
|
||||||
|
# the normal parameters, loud external speakers (JBL / Beats Pill)
|
||||||
|
# use the stricter external parameters and STAY fully gated (echo
|
||||||
|
# safety — the bleed is as loud as a user). The dashboard
|
||||||
|
# ⏹ Interrupt-now button always works regardless of this option.
|
||||||
|
full_gate = external or not _EXTERNAL_BARGE_IN
|
||||||
|
allow_barge = _EXTERNAL_BARGE_IN
|
||||||
|
|
||||||
# Barge-in: sustained user energy cuts the AI — chest profile only.
|
# Barge-in: sustained user energy cuts the AI.
|
||||||
if self._speaking and not full_gate and now >= self._barge_block_until:
|
if self._speaking and allow_barge and now >= self._barge_block_until:
|
||||||
if (now - self._ai_speak_start) >= grace:
|
eff_grace = _JBL_BARGE_GRACE if external else grace
|
||||||
if energy > threshold:
|
eff_threshold = threshold * _JBL_BLEED_MARGIN if external else threshold
|
||||||
|
eff_chunks = _JBL_BARGE_CHUNKS if external else chunks_needed
|
||||||
|
if (now - self._ai_speak_start) >= eff_grace:
|
||||||
|
if energy > eff_threshold:
|
||||||
loud_count += 1
|
loud_count += 1
|
||||||
else:
|
else:
|
||||||
loud_count = max(0, loud_count - 1)
|
loud_count = max(0, loud_count - 1)
|
||||||
if loud_count > chunks_needed:
|
if loud_count > eff_chunks:
|
||||||
log.info("BARGE-IN (e=%d)", energy)
|
log.info("BARGE-IN (e=%d, external=%s)", energy, external)
|
||||||
self._interrupt("barge-in")
|
self._interrupt("barge-in")
|
||||||
loud_count = 0
|
loud_count = 0
|
||||||
self._barge_block_until = now + cooldown
|
self._barge_block_until = now + cooldown
|
||||||
|
|
||||||
# Echo suppression: mask the mic so the model doesn't hear its own bleed.
|
# Echo suppression: mask the mic so the model doesn't hear its own bleed.
|
||||||
send_data = data
|
send_data = data
|
||||||
if _INPUT_PAUSED.is_set():
|
if now < force_silence_until:
|
||||||
|
# Answer-now: mask the mic so the server VAD sees pure silence.
|
||||||
|
send_data = _SILENCE_PCM
|
||||||
|
elif _INPUT_PAUSED.is_set():
|
||||||
# Paused for a record playback — feed silence so Gemini neither
|
# Paused for a record playback — feed silence so Gemini neither
|
||||||
# hears the record nor keeps talking over it.
|
# hears the record nor keeps talking over it.
|
||||||
send_data = _SILENCE_PCM
|
send_data = _SILENCE_PCM
|
||||||
@ -764,7 +995,8 @@ class GeminiBrain:
|
|||||||
# mouth-open level (0..3) from this chunk's RMS,
|
# mouth-open level (0..3) from this chunk's RMS,
|
||||||
# throttled. Parsed by GeminiSubprocess._reader_loop.
|
# throttled. Parsed by GeminiSubprocess._reader_loop.
|
||||||
_mnow = time.time()
|
_mnow = time.time()
|
||||||
if _mnow - getattr(self, "_mouth_t", 0.0) >= 0.08:
|
if not _PURE_VOICE and \
|
||||||
|
_mnow - getattr(self, "_mouth_t", 0.0) >= 0.08:
|
||||||
_rms = (float(np.sqrt(np.mean(
|
_rms = (float(np.sqrt(np.mean(
|
||||||
audio.astype(np.float32) ** 2))) if audio.size else 0.0)
|
audio.astype(np.float32) ** 2))) if audio.size else 0.0)
|
||||||
# Lower thresholds bias the mouth more open
|
# Lower thresholds bias the mouth more open
|
||||||
@ -821,6 +1053,10 @@ class GeminiBrain:
|
|||||||
|
|
||||||
async def _announce_vision_state(self, session: Any, enabled: bool,
|
async def _announce_vision_state(self, session: Any, enabled: bool,
|
||||||
is_toggle: bool) -> None:
|
is_toggle: bool) -> None:
|
||||||
|
if _PURE_VOICE and not enabled and not is_toggle:
|
||||||
|
# PURE: skip the "disabled" standing-context injection — only
|
||||||
|
# ENABLED features get announced/used (toggles always announce).
|
||||||
|
return
|
||||||
if is_toggle and enabled:
|
if is_toggle and enabled:
|
||||||
text = (
|
text = (
|
||||||
"[VISION ON] Your camera was just enabled — you can now see "
|
"[VISION ON] Your camera was just enabled — you can now see "
|
||||||
@ -864,6 +1100,10 @@ class GeminiBrain:
|
|||||||
|
|
||||||
async def _announce_facerec_state(self, session: Any, enabled: bool,
|
async def _announce_facerec_state(self, session: Any, enabled: bool,
|
||||||
is_toggle: bool) -> None:
|
is_toggle: bool) -> None:
|
||||||
|
if _PURE_VOICE and not enabled and not is_toggle:
|
||||||
|
# PURE: skip the "disabled" standing-context injection — only
|
||||||
|
# ENABLED features get announced/used (toggles always announce).
|
||||||
|
return
|
||||||
if is_toggle and enabled:
|
if is_toggle and enabled:
|
||||||
text = (
|
text = (
|
||||||
"[FACE RECOGNITION ON] Face recognition was just enabled — "
|
"[FACE RECOGNITION ON] Face recognition was just enabled — "
|
||||||
@ -908,6 +1148,10 @@ class GeminiBrain:
|
|||||||
|
|
||||||
async def _announce_zonerec_state(self, session: Any, enabled: bool,
|
async def _announce_zonerec_state(self, session: Any, enabled: bool,
|
||||||
is_toggle: bool) -> None:
|
is_toggle: bool) -> None:
|
||||||
|
if _PURE_VOICE and not enabled and not is_toggle:
|
||||||
|
# PURE: skip the "disabled" standing-context injection — only
|
||||||
|
# ENABLED features get announced/used (toggles always announce).
|
||||||
|
return
|
||||||
if is_toggle and enabled:
|
if is_toggle and enabled:
|
||||||
text = (
|
text = (
|
||||||
"[ZONE RECOGNITION ON] You were just given the zones and places "
|
"[ZONE RECOGNITION ON] You were just given the zones and places "
|
||||||
@ -1069,6 +1313,13 @@ class GeminiBrain:
|
|||||||
"places": r.get("places", [])}
|
"places": r.get("places", [])}
|
||||||
if name == "stop_navigation":
|
if name == "stop_navigation":
|
||||||
return await asyncio.to_thread(_nav_api, "POST", "/api/nav/cancel", None)
|
return await asyncio.to_thread(_nav_api, "POST", "/api/nav/cancel", None)
|
||||||
|
if name == "search_internet":
|
||||||
|
query = str(args.get("query") or "").strip()
|
||||||
|
if not query:
|
||||||
|
return {"ok": False, "reason": "no_query"}
|
||||||
|
log.info("[[SEARCH:%s]]", query) # breadcrumb for the dashboard
|
||||||
|
return await asyncio.to_thread(
|
||||||
|
_run_grounding_search, query, self._api_key)
|
||||||
return {"ok": False, "reason": "unknown_tool"}
|
return {"ok": False, "reason": "unknown_tool"}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("tool %s error: %s", name, exc)
|
log.warning("tool %s error: %s", name, exc)
|
||||||
@ -1081,6 +1332,10 @@ class GeminiBrain:
|
|||||||
|
|
||||||
async def _announce_movement_state(self, session: Any, enabled: bool,
|
async def _announce_movement_state(self, session: Any, enabled: bool,
|
||||||
is_toggle: bool) -> None:
|
is_toggle: bool) -> None:
|
||||||
|
if _PURE_VOICE and not enabled and not is_toggle:
|
||||||
|
# PURE: skip the "disabled" standing-context injection — only
|
||||||
|
# ENABLED features get announced/used (toggles always announce).
|
||||||
|
return
|
||||||
if is_toggle and enabled:
|
if is_toggle and enabled:
|
||||||
text = (
|
text = (
|
||||||
"[MOVEMENT ON] Walking is now enabled — you can move when the "
|
"[MOVEMENT ON] Walking is now enabled — you can move when the "
|
||||||
@ -1406,6 +1661,12 @@ class GeminiBrain:
|
|||||||
await self._announce_movement_state(
|
await self._announce_movement_state(
|
||||||
session, self._movement_enabled, is_toggle=True,
|
session, self._movement_enabled, is_toggle=True,
|
||||||
)
|
)
|
||||||
|
if _PURE_VOICE:
|
||||||
|
# PURE: tools are baked per-session (nav tools only while
|
||||||
|
# movement is on) — cycle the session so the toggle takes
|
||||||
|
# effect on the function-calling side too.
|
||||||
|
log.info("pure-voice: cycling session to rebake nav tools")
|
||||||
|
self._done.set()
|
||||||
|
|
||||||
# Auto-record toggle — flip the recorder live (no session restart).
|
# Auto-record toggle — flip the recorder live (no session restart).
|
||||||
if new_state.record_enabled != last_state.record_enabled:
|
if new_state.record_enabled != last_state.record_enabled:
|
||||||
@ -1436,6 +1697,12 @@ class GeminiBrain:
|
|||||||
await asyncio.sleep(max(period, backoff))
|
await asyncio.sleep(max(period, backoff))
|
||||||
if not self._vision_enabled:
|
if not self._vision_enabled:
|
||||||
continue
|
continue
|
||||||
|
# Never compete with the INCOMING audio stream while the AI is
|
||||||
|
# speaking — a frame upload on the shared websocket delays audio
|
||||||
|
# chunks, which breaks the chest speaker (tiny firmware buffer;
|
||||||
|
# the pulse/Pill path hides it behind its own buffering).
|
||||||
|
if self._speaking or (time.time() - self._last_ai_audio) < 0.5:
|
||||||
|
continue
|
||||||
with _LATEST_FRAME_LOCK:
|
with _LATEST_FRAME_LOCK:
|
||||||
data = _LATEST_FRAME.get("bytes")
|
data = _LATEST_FRAME.get("bytes")
|
||||||
ts = _LATEST_FRAME.get("ts", 0.0)
|
ts = _LATEST_FRAME.get("ts", 0.0)
|
||||||
|
|||||||
@ -38,7 +38,9 @@ _FRAME_FORWARD_INTERVAL_S = float(_LS_CFG.get("frame_forward_interval_sec", 0.5)
|
|||||||
|
|
||||||
# Audio profile watcher — poll pactl for the Anker USB device at this
|
# Audio profile watcher — poll pactl for the Anker USB device at this
|
||||||
# interval, send "profile:<json>" to the child on every state change.
|
# interval, send "profile:<json>" to the child on every state change.
|
||||||
_AUDIO_WATCH_INTERVAL_S = float(_LS_CFG.get("audio_watch_interval_sec", 1.5))
|
# 5s: each poll shells out to pactl several times (detect_plugged_profiles) —
|
||||||
|
# 1.5s was a constant subprocess storm for a rare event (device hot-plug).
|
||||||
|
_AUDIO_WATCH_INTERVAL_S = float(_LS_CFG.get("audio_watch_interval_sec", 5.0))
|
||||||
|
|
||||||
# The Anker profile id, as defined in voice/audio_devices.py. When this
|
# The Anker profile id, as defined in voice/audio_devices.py. When this
|
||||||
# profile is fully plugged (both sink + source present), we switch the
|
# profile is fully plugged (both sink + source present), we switch the
|
||||||
@ -515,7 +517,7 @@ class GeminiSubprocess:
|
|||||||
if the process isn't running or stdin is closed."""
|
if the process isn't running or stdin is closed."""
|
||||||
pid = (profile_id or "").strip().lower()
|
pid = (profile_id or "").strip().lower()
|
||||||
if pid not in {"builtin", "anker", "anker_powerconf",
|
if pid not in {"builtin", "anker", "anker_powerconf",
|
||||||
"hollyland_builtin", "jbl_builtin_mic"}:
|
"hollyland_builtin", "jbl_builtin_mic", "beats_pill"}:
|
||||||
log.warning("send_profile: ignoring unknown profile %r", profile_id)
|
log.warning("send_profile: ignoring unknown profile %r", profile_id)
|
||||||
return
|
return
|
||||||
payload: dict[str, Any] = {"id": pid}
|
payload: dict[str, Any] = {"id": pid}
|
||||||
@ -533,6 +535,16 @@ class GeminiSubprocess:
|
|||||||
owns the chest speaker, then resumes. No-op if not running."""
|
owns the chest speaker, then resumes. No-op if not running."""
|
||||||
self._send_stdin("pause:%d\n" % (1 if paused else 0))
|
self._send_stdin("pause:%d\n" % (1 if paused else 0))
|
||||||
|
|
||||||
|
def send_interrupt(self) -> None:
|
||||||
|
"""Instantly stop the AI's current speech (dashboard Interrupt-now
|
||||||
|
button) — the child stops the speaker and returns to listening."""
|
||||||
|
self._send_stdin("interrupt:1\n")
|
||||||
|
|
||||||
|
def send_answer_now(self) -> None:
|
||||||
|
"""Force Gemini to answer immediately (dashboard Answer-now button) —
|
||||||
|
the child feeds the server VAD a short silence burst to close the turn."""
|
||||||
|
self._send_stdin("answernow:1\n")
|
||||||
|
|
||||||
def _audio_watcher(self) -> None:
|
def _audio_watcher(self) -> None:
|
||||||
"""Background thread — poll pactl for the Anker USB device, signal
|
"""Background thread — poll pactl for the Anker USB device, signal
|
||||||
the child on every plug/unplug edge transition.
|
the child on every plug/unplug edge transition.
|
||||||
|
|||||||
@ -82,20 +82,18 @@ except ImportError:
|
|||||||
_make_low_cmd = None
|
_make_low_cmd = None
|
||||||
log.warning("Unitree SDK not available — running in simulation mode")
|
log.warning("Unitree SDK not available — running in simulation mode")
|
||||||
|
|
||||||
# G1 arm-action client for built-in arm moves (wave, shake_hand, hug, …).
|
# R1-native arm-action RPC client for built-in gestures + recorded "teach"
|
||||||
# NOTE: do NOT use LocoClient here — LocoClient is the locomotion/body-move
|
# motions. The R1 firmware does NOT expose live joint streaming of the head/arms;
|
||||||
# client and its ExecuteAction() doesn't recognise arm-action IDs, so arm
|
# the 'arm' service owns rt/arm_sdk and clients drive the upper body ONLY through
|
||||||
# commands become silent no-ops. The correct client is the arm-specific
|
# this RPC: ExecuteAction(id) for Unitree preset gestures, ExecuteCustomAction(
|
||||||
# G1ArmActionClient with the SDK's action_map (name → opcode lookup).
|
# name) for motions recorded via the app's teaching (incl. the head looks). This
|
||||||
|
# replaces the old G1ArmActionClient — the R1 has no G1 arm-action service, so
|
||||||
|
# those gestures were silent no-ops on the R1.
|
||||||
try:
|
try:
|
||||||
from unitree_sdk2py.g1.arm.g1_arm_action_client import (
|
from Project.Sanad.motion.r1_arm_action_client import R1ArmActionClient
|
||||||
G1ArmActionClient,
|
|
||||||
action_map as _ARM_ACTION_MAP,
|
|
||||||
)
|
|
||||||
_HAS_ARM_CLIENT = True
|
_HAS_ARM_CLIENT = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
G1ArmActionClient = None
|
R1ArmActionClient = None
|
||||||
_ARM_ACTION_MAP = {}
|
|
||||||
_HAS_ARM_CLIENT = False
|
_HAS_ARM_CLIENT = False
|
||||||
|
|
||||||
|
|
||||||
@ -107,24 +105,41 @@ class Action:
|
|||||||
category: str = "sdk" # "sdk" | "jsonl"
|
category: str = "sdk" # "sdk" | "jsonl"
|
||||||
|
|
||||||
|
|
||||||
# -- SDK actions (fixed — built into Unitree firmware) --
|
# -- SDK actions (R1 'arm'-service preset gestures) --
|
||||||
|
# The Action.id IS the R1 arm-service opcode → executed directly via
|
||||||
|
# R1ArmActionClient.ExecuteAction(id). Names/ids captured from the robot's
|
||||||
|
# GetActionList (2026-07-14). Friendliest gestures first.
|
||||||
SDK_ACTIONS: list[Action] = [
|
SDK_ACTIONS: list[Action] = [
|
||||||
Action("release_arm", 0, category="sdk"),
|
Action("release_arm", 99, category="sdk"),
|
||||||
Action("shake_hand", 1, category="sdk"),
|
Action("shake_hand", 27, category="sdk"),
|
||||||
Action("high_five", 2, category="sdk"),
|
Action("high_five", 18, category="sdk"),
|
||||||
Action("hug", 3, category="sdk"),
|
Action("hug", 19, category="sdk"),
|
||||||
Action("high_wave", 4, category="sdk"),
|
Action("both_hands_up", 15, category="sdk"),
|
||||||
Action("clap", 5, category="sdk"),
|
Action("wave_above_head", 26, category="sdk"),
|
||||||
Action("face_wave", 6, category="sdk"),
|
Action("wave_under_head", 25, category="sdk"),
|
||||||
Action("left_kiss", 7, category="sdk"),
|
Action("blow_kiss_both_hands", 11, category="sdk"),
|
||||||
Action("heart", 8, category="sdk"),
|
Action("blow_kiss_left_hand", 12, category="sdk"),
|
||||||
Action("right_heart", 9, category="sdk"),
|
Action("blow_kiss_right_hand", 13, category="sdk"),
|
||||||
Action("hands_up", 10, category="sdk"),
|
Action("right_hand_up", 23, category="sdk"),
|
||||||
Action("x_ray", 11, category="sdk"),
|
Action("right_hand_on_heart", 33, category="sdk"),
|
||||||
Action("right_hand_up", 12, category="sdk"),
|
Action("clamp", 17, category="sdk"),
|
||||||
Action("reject", 13, category="sdk"),
|
Action("refuse", 22, category="sdk"),
|
||||||
Action("right_kiss", 14, category="sdk"),
|
Action("emphasize", 35, category="sdk"),
|
||||||
Action("two_hand_kiss", 15, category="sdk"),
|
Action("forward_push", 36, category="sdk"),
|
||||||
|
Action("extend_right_arm_forward", 31, category="sdk"),
|
||||||
|
Action("both_hands_up_deviate_right", 34, category="sdk"),
|
||||||
|
Action("ultraman_ray", 24, category="sdk"),
|
||||||
|
Action("box_left_hand_win", 28, category="sdk"),
|
||||||
|
Action("box_right_hand_win", 29, category="sdk"),
|
||||||
|
Action("box_both_hand_win", 30, category="sdk"),
|
||||||
|
]
|
||||||
|
|
||||||
|
# R1 head-look "teach" motions (recorded via the app's teaching) — replayed by
|
||||||
|
# NAME via R1ArmActionClient.ExecuteCustomAction. Shown as a simple button list
|
||||||
|
# in the Controller tab's HEAD card. These are your recorded head rotations.
|
||||||
|
HEAD_LOOKS: list[str] = [
|
||||||
|
"up_look", "down_look", "left_look", "right_look",
|
||||||
|
"up_down_look", "left_right_look", "right_left_look", "center",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Next auto-ID for JSONL actions starts after SDK range.
|
# Next auto-ID for JSONL actions starts after SDK range.
|
||||||
@ -285,16 +300,16 @@ class ArmController:
|
|||||||
)
|
)
|
||||||
self._crc = CRC()
|
self._crc = CRC()
|
||||||
|
|
||||||
# Arm-specific action client for built-in moves
|
# R1 arm-service client for preset gestures + recorded teach/head moves
|
||||||
if _HAS_ARM_CLIENT:
|
if _HAS_ARM_CLIENT:
|
||||||
try:
|
try:
|
||||||
self._arm_client = G1ArmActionClient()
|
self._arm_client = R1ArmActionClient()
|
||||||
self._arm_client.SetTimeout(10.0)
|
self._arm_client.SetTimeout(10.0)
|
||||||
self._arm_client.Init()
|
self._arm_client.Init()
|
||||||
log.info("G1ArmActionClient initialized (%d actions) — built-in moves available",
|
log.info("R1ArmActionClient initialized — %d preset gestures + %d head looks",
|
||||||
len(_ARM_ACTION_MAP))
|
len(SDK_ACTIONS), len(HEAD_LOOKS))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("G1ArmActionClient init failed: %s — built-in actions disabled", exc)
|
log.warning("R1ArmActionClient init failed: %s — arm actions disabled", exc)
|
||||||
self._arm_client = None
|
self._arm_client = None
|
||||||
|
|
||||||
self._initialized = True
|
self._initialized = True
|
||||||
@ -869,39 +884,42 @@ class ArmController:
|
|||||||
return
|
return
|
||||||
if self._arm_client is None:
|
if self._arm_client is None:
|
||||||
log.warning(
|
log.warning(
|
||||||
"SDK action %s requested but G1ArmActionClient not available — skipping",
|
"SDK action %s requested but R1ArmActionClient not available — skipping",
|
||||||
action.name,
|
action.name,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
# Sanad's registry uses underscored names ("shake_hand", "x_ray");
|
# R1 preset gestures execute by opcode directly — Action.id IS the R1
|
||||||
# the SDK's action_map is keyed by human-readable forms that mix
|
# arm-service opcode (see SDK_ACTIONS). No name→opcode lookup needed.
|
||||||
# spaces and hyphens ("shake hand", "x-ray", "two-hand kiss").
|
log.info("R1 gesture: %s (opcode=%s)", action.name, action.id)
|
||||||
# Try each candidate in turn.
|
|
||||||
name = action.name
|
|
||||||
candidates = [
|
|
||||||
name,
|
|
||||||
name.replace("_", " "), # shake_hand → shake hand
|
|
||||||
name.replace("_", "-"), # x_ray → x-ray
|
|
||||||
# two-word with specific hyphenation: first token with hyphen,
|
|
||||||
# rest with spaces (matches SDK's "two-hand kiss" pattern)
|
|
||||||
name.replace("_", "-", 1).replace("_", " "),
|
|
||||||
]
|
|
||||||
sdk_name = next((c for c in candidates if c in _ARM_ACTION_MAP), None)
|
|
||||||
if sdk_name is None:
|
|
||||||
log.warning(
|
|
||||||
"SDK action %s not in G1ArmActionClient action_map — tried %s. keys=%s",
|
|
||||||
action.name, candidates, sorted(_ARM_ACTION_MAP.keys())[:12],
|
|
||||||
)
|
|
||||||
return
|
|
||||||
opcode = _ARM_ACTION_MAP[sdk_name]
|
|
||||||
log.info("SDK action: %s (opcode=%s)", action.name, opcode)
|
|
||||||
try:
|
try:
|
||||||
self._arm_client.ExecuteAction(opcode)
|
rc = self._arm_client.ExecuteAction(action.id)
|
||||||
# Built-in arm actions block on the robot side for ~3s; the SDK
|
if rc != 0:
|
||||||
# call returns immediately. Sleep so we don't hammer it back-to-back.
|
log.warning("R1 gesture %s returned rc=%s", action.name, rc)
|
||||||
|
# The arm service blocks on the robot for ~3s; the RPC returns fast.
|
||||||
time.sleep(3.0)
|
time.sleep(3.0)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.error("SDK action %s failed: %s", action.name, exc)
|
log.error("R1 gesture %s failed: %s", action.name, exc)
|
||||||
|
|
||||||
|
def run_head_look(self, name: str) -> dict[str, Any]:
|
||||||
|
"""Replay a recorded head-look 'teach' motion by name via the R1 arm
|
||||||
|
service (ExecuteCustomAction). Used by the Controller-tab HEAD menu.
|
||||||
|
Returns {ok, rc}. Never raises — reports failure in the result."""
|
||||||
|
if name not in HEAD_LOOKS:
|
||||||
|
return {"ok": False, "error": f"unknown head look '{name}'"}
|
||||||
|
if not _HAS_SDK or self._arm_client is None:
|
||||||
|
log.info("[SIM] head look: %s", name)
|
||||||
|
return {"ok": _HAS_SDK is False, "rc": None, "sim": True}
|
||||||
|
log.info("R1 head look: %s", name)
|
||||||
|
try:
|
||||||
|
rc = self._arm_client.ExecuteCustomAction(name)
|
||||||
|
return {"ok": rc == 0, "rc": rc}
|
||||||
|
except Exception as exc:
|
||||||
|
log.error("head look %s failed: %s", name, exc)
|
||||||
|
return {"ok": False, "error": str(exc)}
|
||||||
|
|
||||||
|
def head_looks(self) -> list[str]:
|
||||||
|
"""The recorded head-look names for the dashboard menu."""
|
||||||
|
return list(HEAD_LOOKS)
|
||||||
|
|
||||||
def status(self) -> dict[str, Any]:
|
def status(self) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
|
|||||||
67
motion/r1_arm_action_client.py
Normal file
67
motion/r1_arm_action_client.py
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
R1 arm-action RPC client — the R1-NATIVE way to move the upper body (arms + head)
|
||||||
|
while the firmware keeps balancing the legs.
|
||||||
|
|
||||||
|
Why this and not rt/arm_sdk: the R1 firmware does NOT expose live joint streaming
|
||||||
|
of the head/arms. The 'arm' service (a.k.a. r1_arm_example) owns rt/arm_sdk
|
||||||
|
internally; clients drive the upper body ONLY through this RPC — preset gesture
|
||||||
|
actions (by id) and custom "teach" actions (by name, recorded via the app's
|
||||||
|
teaching). Captured from the app's own traffic (2026-07-06):
|
||||||
|
release_arm -> ExecuteAction(99); replay recorded motion -> ExecuteCustomAction(name).
|
||||||
|
|
||||||
|
Service "arm", api version "1.0.0.14". Same Client base as R1LocoClient.
|
||||||
|
State is published on rt/arm/action/state: {"holding":bool,"id":int,"name":str}.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
from unitree_sdk2py.rpc.client import Client
|
||||||
|
|
||||||
|
ARM_ACTION_SERVICE_NAME = "arm"
|
||||||
|
ARM_ACTION_API_VERSION = "1.0.0.14"
|
||||||
|
|
||||||
|
API_EXECUTE_ACTION = 7106 # {"action_id": N} preset actions
|
||||||
|
API_GET_ACTION_LIST = 7107 # -> JSON list
|
||||||
|
API_EXECUTE_CUSTOM_ACTION = 7108 # {"action_name": "..."} recorded teach actions
|
||||||
|
API_RECORD = 7110 # start = {"action_name":NAME} ; stop+save = "" (empty str)
|
||||||
|
API_STOP_CUSTOM_ACTION = 7113
|
||||||
|
|
||||||
|
RELEASE_ARM = 99 # go compliant / release a held action
|
||||||
|
|
||||||
|
|
||||||
|
class R1ArmActionClient(Client):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(ARM_ACTION_SERVICE_NAME, False)
|
||||||
|
|
||||||
|
def Init(self):
|
||||||
|
self._SetApiVerson(ARM_ACTION_API_VERSION) # SDK spells it "Verson"
|
||||||
|
for api in (API_EXECUTE_ACTION, API_GET_ACTION_LIST,
|
||||||
|
API_EXECUTE_CUSTOM_ACTION, API_RECORD, API_STOP_CUSTOM_ACTION):
|
||||||
|
self._RegistApi(api, 0)
|
||||||
|
|
||||||
|
def ExecuteAction(self, action_id: int):
|
||||||
|
"""Run a preset action by id (99 = release). Returns rc (0 = ok)."""
|
||||||
|
code, _ = self._Call(API_EXECUTE_ACTION, json.dumps({"action_id": int(action_id)}))
|
||||||
|
return code
|
||||||
|
|
||||||
|
def ExecuteCustomAction(self, action_name: str):
|
||||||
|
"""Replay a recorded teach action by name (from the app's teaching)."""
|
||||||
|
code, _ = self._Call(API_EXECUTE_CUSTOM_ACTION, json.dumps({"action_name": str(action_name)}))
|
||||||
|
return code
|
||||||
|
|
||||||
|
def StopCustomAction(self):
|
||||||
|
code, _ = self._Call(API_STOP_CUSTOM_ACTION, "{}")
|
||||||
|
return code
|
||||||
|
|
||||||
|
def ReleaseArm(self):
|
||||||
|
return self.ExecuteAction(RELEASE_ARM)
|
||||||
|
|
||||||
|
def GetActionList(self):
|
||||||
|
"""Return (rc, data) — data is the JSON list of available actions."""
|
||||||
|
code, data = self._Call(API_GET_ACTION_LIST, "")
|
||||||
|
parsed = None
|
||||||
|
if code == 0 and data:
|
||||||
|
try:
|
||||||
|
parsed = json.loads(data)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
parsed = data
|
||||||
|
return code, parsed
|
||||||
@ -1,4 +1,4 @@
|
|||||||
أنت "سند iron man" — روبوت إماراتي ذكي تابع لروبوت شركة لوتاه تيك Lootah Tech.
|
أنت "سوبر دبي super-dubai" — روبوت إماراتي ذكي تابع لروبوت شركة لوتاه تيك Lootah Tech.
|
||||||
|
|
||||||
[أولاً: الروح والمعرفة]
|
[أولاً: الروح والمعرفة]
|
||||||
|
|
||||||
|
|||||||
@ -22,7 +22,9 @@ class TestConfig(unittest.TestCase):
|
|||||||
def test_base_dir_auto_detect(self):
|
def test_base_dir_auto_detect(self):
|
||||||
from Project.Sanad.config import BASE_DIR
|
from Project.Sanad.config import BASE_DIR
|
||||||
self.assertTrue(BASE_DIR.exists(), f"BASE_DIR missing: {BASE_DIR}")
|
self.assertTrue(BASE_DIR.exists(), f"BASE_DIR missing: {BASE_DIR}")
|
||||||
self.assertEqual(BASE_DIR.name, "Sanad")
|
# Workstation checkout is SanadR1; the deployed container aliases it
|
||||||
|
# as Project/Sanad, so both names are valid.
|
||||||
|
self.assertIn(BASE_DIR.name, ("Sanad", "SanadR1"))
|
||||||
|
|
||||||
def test_data_dirs_exist(self):
|
def test_data_dirs_exist(self):
|
||||||
from Project.Sanad.config import (
|
from Project.Sanad.config import (
|
||||||
|
|||||||
@ -106,6 +106,18 @@ PROFILES: list[AudioProfile] = [
|
|||||||
source_pattern="powerconf,anker",
|
source_pattern="powerconf,anker",
|
||||||
description="Anker PowerConf USB conference unit — mic + speaker on the same device.",
|
description="Anker PowerConf USB conference unit — mic + speaker on the same device.",
|
||||||
),
|
),
|
||||||
|
AudioProfile(
|
||||||
|
id="beats_pill",
|
||||||
|
label="Beats Pill (mic + speaker)",
|
||||||
|
# USB-C: kernel names the card "Pill"; pulse sinks/sources carry
|
||||||
|
# "Beats_Pill" (auto-probed or the manual module-alsa-sink/-source
|
||||||
|
# loads from /etc/pulse/default.pa — pulse fails to probe the Pill's
|
||||||
|
# output profile on some kernels). Bluetooth: bluez sink.
|
||||||
|
sink_pattern="beats_pill,beats,pill",
|
||||||
|
source_pattern="beats_pill,beats,pill",
|
||||||
|
description="Beats Pill (USB-C or Bluetooth) — speaker + speakerphone mic.",
|
||||||
|
udp_mic_fallback=True, # mic falls back to the R1/G1 UDP mic if absent
|
||||||
|
),
|
||||||
AudioProfile(
|
AudioProfile(
|
||||||
id="jbl_builtin_mic",
|
id="jbl_builtin_mic",
|
||||||
label="JBL speaker + built-in mic",
|
label="JBL speaker + built-in mic",
|
||||||
@ -115,6 +127,7 @@ PROFILES: list[AudioProfile] = [
|
|||||||
# The JBL has NO microphone → input stays on the G1 built-in mic.
|
# The JBL has NO microphone → input stays on the G1 built-in mic.
|
||||||
source_pattern="alsa_input.platform-sound",
|
source_pattern="alsa_input.platform-sound",
|
||||||
description="JBL Bluetooth speaker for output + the G1 built-in microphone for input (the JBL has no mic).",
|
description="JBL Bluetooth speaker for output + the G1 built-in microphone for input (the JBL has no mic).",
|
||||||
|
udp_mic_fallback=True, # builtin mic = R1/G1 UDP stream on units without the alsa source
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@ -31,6 +31,7 @@ import json
|
|||||||
import socket
|
import socket
|
||||||
import struct
|
import struct
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import queue
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
@ -211,7 +212,16 @@ class BuiltinMic(Mic):
|
|||||||
|
|
||||||
|
|
||||||
class BuiltinSpeaker(Speaker):
|
class BuiltinSpeaker(Speaker):
|
||||||
"""G1 robot's built-in speaker via AudioClient.PlayStream (16 kHz mono)."""
|
"""G1 robot's built-in speaker via AudioClient.PlayStream (16 kHz mono).
|
||||||
|
|
||||||
|
JITTER-BUFFERED: chunks are enqueued and a dedicated writer thread sends
|
||||||
|
them to the firmware, prebuffering ~0.3s before the first RPC and never
|
||||||
|
running more than ~0.6s ahead of real time. Gemini's audio arrives in
|
||||||
|
BURSTS (WiFi jitter, shared-websocket contention) — the old direct
|
||||||
|
per-chunk RPC mapped every arrival gap to an audible dropout on the
|
||||||
|
chest speaker's tiny firmware buffer, while the pulse (Pill/Anker) path
|
||||||
|
hid it behind PulseAudio's own buffering. This gives the chest the same
|
||||||
|
tolerance."""
|
||||||
|
|
||||||
HARDWARE_RATE = 16_000
|
HARDWARE_RATE = 16_000
|
||||||
|
|
||||||
@ -224,10 +234,16 @@ class BuiltinSpeaker(Speaker):
|
|||||||
self._app_name = app_name or _SP_CFG.get("app_name", "sanad")
|
self._app_name = app_name or _SP_CFG.get("app_name", "sanad")
|
||||||
self._begin_pause = _SP_CFG.get("begin_stream_pause_sec", 0.15)
|
self._begin_pause = _SP_CFG.get("begin_stream_pause_sec", 0.15)
|
||||||
self._finish_margin = _SP_CFG.get("wait_finish_margin_sec", 0.3)
|
self._finish_margin = _SP_CFG.get("wait_finish_margin_sec", 0.3)
|
||||||
|
# Jitter buffer tuning (see class docstring).
|
||||||
|
self._prebuffer_sec = _SP_CFG.get("jitter_prebuffer_sec", 0.30)
|
||||||
|
self._first_flush_sec = _SP_CFG.get("jitter_first_flush_sec", 0.60)
|
||||||
|
self._max_lead_sec = _SP_CFG.get("jitter_max_lead_sec", 0.60)
|
||||||
self._stop_flag = threading.Event()
|
self._stop_flag = threading.Event()
|
||||||
self._stream_id: Optional[str] = None
|
self._stream_id: Optional[str] = None
|
||||||
self._total_sent = 0.0
|
self._total_sent = 0.0
|
||||||
self._play_start = 0.0
|
self._play_start = 0.0
|
||||||
|
self._q: "queue.Queue[Optional[bytes]]" = queue.Queue()
|
||||||
|
self._writer: Optional[threading.Thread] = None
|
||||||
|
|
||||||
def _stop_play_api(self) -> None:
|
def _stop_play_api(self) -> None:
|
||||||
try:
|
try:
|
||||||
@ -241,13 +257,79 @@ class BuiltinSpeaker(Speaker):
|
|||||||
except Exception:
|
except Exception:
|
||||||
log.warning("BuiltinSpeaker AUDIO_STOP_PLAY failed")
|
log.warning("BuiltinSpeaker AUDIO_STOP_PLAY failed")
|
||||||
|
|
||||||
|
def _drain_queue(self) -> None:
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
self._q.get_nowait()
|
||||||
|
except queue.Empty:
|
||||||
|
pass
|
||||||
|
|
||||||
def begin_stream(self) -> None:
|
def begin_stream(self) -> None:
|
||||||
|
# Tear down any previous writer cleanly before starting a new turn.
|
||||||
|
self._stop_flag.set()
|
||||||
|
self._drain_queue()
|
||||||
|
if self._writer is not None and self._writer.is_alive():
|
||||||
|
self._q.put(None)
|
||||||
|
self._writer.join(timeout=1.0)
|
||||||
self._stop_flag.clear()
|
self._stop_flag.clear()
|
||||||
|
self._drain_queue()
|
||||||
self._stop_play_api()
|
self._stop_play_api()
|
||||||
time.sleep(self._begin_pause)
|
time.sleep(self._begin_pause)
|
||||||
self._stream_id = f"s_{int(time.time() * 1000)}"
|
self._stream_id = f"s_{int(time.time() * 1000)}"
|
||||||
self._total_sent = 0.0
|
self._total_sent = 0.0
|
||||||
self._play_start = time.time()
|
self._play_start = time.time()
|
||||||
|
self._writer = threading.Thread(
|
||||||
|
target=self._writer_loop, args=(self._stream_id,),
|
||||||
|
name="builtin-speaker-writer", daemon=True)
|
||||||
|
self._writer.start()
|
||||||
|
|
||||||
|
def _writer_loop(self, stream_id: str) -> None:
|
||||||
|
rate = float(self.HARDWARE_RATE)
|
||||||
|
sent_sec = 0.0
|
||||||
|
started = False
|
||||||
|
t0 = time.time()
|
||||||
|
batch: list = []
|
||||||
|
buffered = 0.0
|
||||||
|
while not self._stop_flag.is_set() and self._stream_id == stream_id:
|
||||||
|
try:
|
||||||
|
item = self._q.get(timeout=0.05)
|
||||||
|
except queue.Empty:
|
||||||
|
item = ()
|
||||||
|
if item is None:
|
||||||
|
return
|
||||||
|
if not started:
|
||||||
|
if item:
|
||||||
|
batch.append(item)
|
||||||
|
buffered += len(item) / 2 / rate
|
||||||
|
if buffered >= self._prebuffer_sec or (
|
||||||
|
batch and (time.time() - t0) >= self._first_flush_sec):
|
||||||
|
self._play_start = time.time()
|
||||||
|
started = True
|
||||||
|
for b in batch:
|
||||||
|
if self._stop_flag.is_set() or self._stream_id != stream_id:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._ac.PlayStream(self._app_name, stream_id, b)
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("PlayStream failed: %s", exc)
|
||||||
|
sent_sec += len(b) / 2 / rate
|
||||||
|
batch = []
|
||||||
|
continue
|
||||||
|
if not item:
|
||||||
|
continue
|
||||||
|
# Pace: never run more than max_lead ahead of real time — keeps
|
||||||
|
# the firmware queue short (fast interrupts) while OUR queue is
|
||||||
|
# the elastic reservoir that absorbs bursty arrival.
|
||||||
|
lead = sent_sec - (time.time() - self._play_start)
|
||||||
|
if lead > self._max_lead_sec:
|
||||||
|
time.sleep(min(lead - self._max_lead_sec, 0.25))
|
||||||
|
if self._stop_flag.is_set() or self._stream_id != stream_id:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._ac.PlayStream(self._app_name, stream_id, item)
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("PlayStream failed: %s", exc)
|
||||||
|
sent_sec += len(item) / 2 / rate
|
||||||
|
|
||||||
def send_chunk(self, pcm: PCMLike, source_rate: int) -> None:
|
def send_chunk(self, pcm: PCMLike, source_rate: int) -> None:
|
||||||
if self._stop_flag.is_set():
|
if self._stop_flag.is_set():
|
||||||
@ -256,20 +338,22 @@ class BuiltinSpeaker(Speaker):
|
|||||||
if arr.size < 10:
|
if arr.size < 10:
|
||||||
return
|
return
|
||||||
hw = _resample_int16(arr, source_rate, self.HARDWARE_RATE)
|
hw = _resample_int16(arr, source_rate, self.HARDWARE_RATE)
|
||||||
self._ac.PlayStream(self._app_name, self._stream_id, hw.tobytes())
|
self._q.put(hw.tobytes())
|
||||||
self._total_sent += len(hw) / self.HARDWARE_RATE
|
self._total_sent += len(hw) / self.HARDWARE_RATE
|
||||||
|
|
||||||
def wait_finish(self) -> None:
|
def wait_finish(self) -> None:
|
||||||
elapsed = time.time() - self._play_start
|
# Wait for the queue to drain AND real-time playback to elapse.
|
||||||
remaining = self._total_sent - elapsed + self._finish_margin
|
while not self._stop_flag.is_set():
|
||||||
waited = 0.0
|
elapsed = time.time() - self._play_start
|
||||||
while waited < remaining and not self._stop_flag.is_set():
|
remaining = self._total_sent - elapsed + self._finish_margin
|
||||||
|
if self._q.empty() and remaining <= 0:
|
||||||
|
break
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
waited += 0.1
|
|
||||||
self._stop_play_api()
|
self._stop_play_api()
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
self._stop_flag.set()
|
self._stop_flag.set()
|
||||||
|
self._drain_queue()
|
||||||
self._stop_play_api()
|
self._stop_play_api()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@ -815,9 +899,13 @@ _PROFILE_ALIASES = {
|
|||||||
"hollyland_builtin": "hollyland_builtin",
|
"hollyland_builtin": "hollyland_builtin",
|
||||||
"jbl": "jbl_builtin_mic",
|
"jbl": "jbl_builtin_mic",
|
||||||
"jbl_builtin_mic": "jbl_builtin_mic",
|
"jbl_builtin_mic": "jbl_builtin_mic",
|
||||||
|
"beats": "beats_pill",
|
||||||
|
"beats_pill": "beats_pill",
|
||||||
|
"pill": "beats_pill",
|
||||||
}
|
}
|
||||||
|
|
||||||
SUPPORTED_PROFILES = ("builtin", "anker", "hollyland_builtin", "jbl_builtin_mic")
|
SUPPORTED_PROFILES = ("builtin", "anker", "hollyland_builtin", "jbl_builtin_mic",
|
||||||
|
"beats_pill")
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@ -885,6 +973,13 @@ class AudioIO:
|
|||||||
# pacat is used because PyAudio's 'pulse' device is unavailable in
|
# pacat is used because PyAudio's 'pulse' device is unavailable in
|
||||||
# this env. Neither backend needs the AudioClient.
|
# this env. Neither backend needs the AudioClient.
|
||||||
return BuiltinMic(), PulseStreamSpeaker(label="JBL", sink_pattern="jbl,bluez")
|
return BuiltinMic(), PulseStreamSpeaker(label="JBL", sink_pattern="jbl,bluez")
|
||||||
|
if resolved == "beats_pill":
|
||||||
|
# Beats Pill speaker via pacat → its PulseAudio sink + the G1
|
||||||
|
# built-in DDS mic — same regime as the JBL (loud external speaker,
|
||||||
|
# full mic-gate while speaking; see gemini/script.py). The Pill's
|
||||||
|
# own USB mic stays available to pulse for dashboard recordings.
|
||||||
|
return BuiltinMic(), PulseStreamSpeaker(
|
||||||
|
label="Beats Pill", sink_pattern="beats_pill,beats,pill")
|
||||||
raise AssertionError(f"unhandled resolved profile: {resolved!r}")
|
raise AssertionError(f"unhandled resolved profile: {resolved!r}")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@ -146,7 +146,7 @@ _MOVEMENT_PROMPT_RULES = (
|
|||||||
"(you receive a [MOVEMENT ON] or [MOVEMENT STATUS] note). When movement is "
|
"(you receive a [MOVEMENT ON] or [MOVEMENT STATUS] note). When movement is "
|
||||||
"OFF, never confirm a motion — tell the user to enable movement from the "
|
"OFF, never confirm a motion — tell the user to enable movement from the "
|
||||||
"dashboard.\n"
|
"dashboard.\n"
|
||||||
"When movement is ON and the user addresses you by name (Sanad iron man / سند) AND "
|
"When movement is ON and the user addresses you by name (super-dubai / سوبر دبي) AND "
|
||||||
"asks you to move, reply with ONE short confirmation phrase per requested "
|
"asks you to move, reply with ONE short confirmation phrase per requested "
|
||||||
"motion, in the SAME language, in the order asked. Use these EXACT shapes — "
|
"motion, in the SAME language, in the order asked. Use these EXACT shapes — "
|
||||||
"they are what triggers the motion:\n"
|
"they are what triggers the motion:\n"
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user