diff --git a/dashboard/routes/macros.py b/dashboard/routes/macros.py index acfb9c1..43b99f8 100644 --- a/dashboard/routes/macros.py +++ b/dashboard/routes/macros.py @@ -2,9 +2,16 @@ from __future__ import annotations +import asyncio +from pathlib import Path + from fastapi import APIRouter, HTTPException from pydantic import BaseModel +from Project.Sanad.config import AUDIO_RECORDINGS_DIR, MOTIONS_DIR +from Project.Sanad.core.logger import get_logger + +log = get_logger("macros_route") router = APIRouter() @@ -12,6 +19,13 @@ class MacroName(BaseModel): name: str +class ComboPlayPayload(BaseModel): + audio_file: str = "" # filename under data/audio/ (or empty for none) + motion_file: str = "" # DEPRECATED — use action_id. Still accepted for bare JSONL by filename. + action_id: int | None = None # arm_controller action id (SDK built-in OR JSONL) — preferred + speed: float = 1.0 + + @router.get("/") async def list_macros(): from Project.Sanad.main import macro_play @@ -58,3 +72,118 @@ async def stop_macro(): if macro_play: macro_play.stop() return {"ok": True} + + +# ─── Ad-hoc audio + motion combined playback ───────────────────────── +# List the two catalogues so the dashboard can populate dropdowns, then +# play the chosen pair in parallel (asyncio.gather) — same scheme the +# Brain uses for `parallel`-mode skills, but ad-hoc instead of predefined. + +@router.get("/audio-files") +async def list_audio_files(): + """Enumerate playable audio files under data/audio/.""" + AUDIO_RECORDINGS_DIR.mkdir(parents=True, exist_ok=True) + files = [] + for p in sorted(AUDIO_RECORDINGS_DIR.glob("*.wav")): + try: + files.append({ + "name": p.name, + "size_kb": round(p.stat().st_size / 1024, 1), + }) + except OSError: + continue + return {"files": files, "dir": str(AUDIO_RECORDINGS_DIR)} + + +@router.get("/motion-files") +async def list_motion_files(): + """Enumerate playable .jsonl motions under data/motions/ (thin wrapper + so the Macro Recorder dropdown doesn't have to call the replay route).""" + MOTIONS_DIR.mkdir(parents=True, exist_ok=True) + files = [] + for p in sorted(MOTIONS_DIR.glob("*.jsonl")): + try: + files.append({ + "name": p.name, + "size_kb": round(p.stat().st_size / 1024, 1), + }) + except OSError: + continue + return {"files": files, "dir": str(MOTIONS_DIR)} + + +@router.post("/play-combined") +async def play_combined(payload: ComboPlayPayload): + """Fire a user-picked audio clip and arm action in parallel. + + Motion dispatch is via `arm.trigger_by_id(action_id)` which handles + BOTH SDK built-in actions (shake_hand, wave, …) and recorded JSONL + replays. Audio goes through `audio_mgr.play_wav` (routed to the G1 + chest speaker via DDS). Either side may be omitted. + """ + from Project.Sanad.main import audio_mgr, arm + + has_audio = bool(payload.audio_file) + has_motion = payload.action_id is not None or bool(payload.motion_file) + if not has_audio and not has_motion: + raise HTTPException(400, "pick at least one of audio_file / action_id / motion_file") + + tasks = [] + result: dict = { + "audio_file": payload.audio_file, + "action_id": payload.action_id, + "motion_file": payload.motion_file, + } + + if has_audio: + if audio_mgr is None: + raise HTTPException(503, "AudioManager not available") + audio_path = (AUDIO_RECORDINGS_DIR / payload.audio_file).resolve() + try: + audio_path.relative_to(AUDIO_RECORDINGS_DIR.resolve()) + except ValueError: + raise HTTPException(400, "audio_file path traversal denied") + if not audio_path.exists(): + raise HTTPException(404, f"audio not found: {payload.audio_file}") + + async def _play_audio(): + try: + await asyncio.to_thread(audio_mgr.play_wav, audio_path) + result["audio_played"] = audio_path.name + except Exception as exc: + log.exception("combined play: audio failed") + result["audio_error"] = str(exc) + tasks.append(_play_audio()) + + if has_motion: + if arm is None: + raise HTTPException(503, "ArmController not available") + + async def _play_motion(): + try: + if payload.action_id is not None: + # SDK built-in OR JSONL — arm.trigger_by_id handles both + await asyncio.to_thread(arm.trigger_by_id, + int(payload.action_id), + payload.speed) + result["motion_played"] = f"action_id={payload.action_id}" + else: + # Legacy path: bare JSONL filename + motion_path = (MOTIONS_DIR / payload.motion_file).resolve() + try: + motion_path.relative_to(MOTIONS_DIR.resolve()) + except ValueError: + result["motion_error"] = "motion_file path traversal denied" + return + if not motion_path.exists(): + result["motion_error"] = f"motion not found: {payload.motion_file}" + return + await asyncio.to_thread(arm.replay_file, str(motion_path), payload.speed) + result["motion_played"] = motion_path.name + except Exception as exc: + log.exception("combined play: motion failed") + result["motion_error"] = str(exc) + tasks.append(_play_motion()) + + await asyncio.gather(*tasks) + return {"ok": True, **result} diff --git a/dashboard/static/index.html b/dashboard/static/index.html index dbcda9d..51f0098 100644 --- a/dashboard/static/index.html +++ b/dashboard/static/index.html @@ -432,16 +432,37 @@

Macro Recorder (Audio + Motion)

-
-
- -
+ + + +
+ + + +
+ + + +
+
+
Voice (WAV)
+
-
- -
+
+
Motion (JSONL)
+ +
+
+
Speed
+ +
+
+ +
+ +
@@ -1109,6 +1130,70 @@ async function startMacro(b){const n=document.getElementById('macro-name').value async function stopMacro(b){btnLoad(b);try{const r=await api('POST','/api/macros/record/stop');toast('Saved','ok');document.getElementById('macro-status').textContent=`Saved: ${r.name} (${r.duration_sec}s)`;}catch(e){}btnDone(b);} async function playMacro(b){const n=document.getElementById('play-macro-name').value;if(!n)return toast('Enter name','err');btnLoad(b);try{await api('POST','/api/macros/play',{name:n});toast('Played: '+n,'ok');}catch(e){}btnDone(b);} +// Ad-hoc combined playback — select voice + motion, play in parallel. +// Motion list = SDK built-ins + JSONL replays (via /api/motion/actions), +// so the dropdown offers every arm action — not just recorded files. +async function refreshCombo(){ + try{ + const [av,am]=await Promise.all([ + api('GET','/api/macros/audio-files'), + api('GET','/api/motion/actions'), + ]); + const voiceSel=document.getElementById('combo-voice'); + const motionSel=document.getElementById('combo-motion'); + const prevV=voiceSel.value, prevM=motionSel.value; + + voiceSel.innerHTML='' + +(av.files||[]).map(f=>``).join(''); + + // Motion: group by category, SDK first then JSONL + const acts=am.actions||[]; + const sdk=acts.filter(a=>!a.file); + const jsl=acts.filter(a=>!!a.file); + let html=''; + if(sdk.length){ + html+=''; + for(const a of sdk){html+=``;} + html+=''; + } + if(jsl.length){ + html+=''; + for(const a of jsl){html+=``;} + html+=''; + } + motionSel.innerHTML=html; + + if(prevV)voiceSel.value=prevV; + if(prevM)motionSel.value=prevM; + }catch(e){toast('Could not load combined lists','err');} +} +async function playCombo(b){ + const v=document.getElementById('combo-voice').value; + const mRaw=document.getElementById('combo-motion').value; + const actionId=mRaw?parseInt(mRaw,10):null; + if(!v&&actionId==null)return toast('Pick a voice or motion (or both)','err'); + const speed=parseFloat(document.getElementById('combo-speed').value||'1.0'); + const st=document.getElementById('combo-status'); + const mLabel=mRaw?document.getElementById('combo-motion').selectedOptions[0].textContent:'(no motion)'; + st.textContent='Playing: '+[v||'(no voice)',mLabel].join(' + ')+'...'; + btnLoad(b); + try{ + const r=await api('POST','/api/macros/play-combined',{ + audio_file:v, + action_id:actionId, + speed, + }); + const parts=[]; + if(r.audio_played)parts.push('audio='+r.audio_played); + if(r.motion_played)parts.push('motion='+r.motion_played); + if(r.audio_error)parts.push('audio_err='+r.audio_error); + if(r.motion_error)parts.push('motion_err='+r.motion_error); + st.textContent='Done: '+parts.join(', '); + toast('Combined play done','ok'); + }catch(e){st.textContent='Failed';} + btnDone(b); +} + // Replay async function refreshReplayFiles(){try{const r=await api('GET','/api/replay/files');const el=document.getElementById('replay-files');if(!(r.files||[]).length){el.innerHTML='
No motion files
';return;}el.innerHTML=''+(r.files||[]).map(f=>``).join('')+'
FileFramesDurationSize
${esc(f.name)}${f.frames}${f.duration_sec}s${f.size_kb}KB
';}catch(e){}} async function testReplay(b){const n=document.getElementById('replay-name').value,s=parseFloat(document.getElementById('replay-speed').value);if(!n)return;btnLoad(b);try{await api('POST','/api/replay/test',{name:n,speed:s});toast('Replay: '+n,'ok');pollArmBusy();}catch(e){}btnDone(b);} @@ -1315,7 +1400,7 @@ async function autoStartLiveSub(){ } // Init — vision/camera/detector fetches removed; those endpoints were deleted. -refreshStatus();refreshSystem();refreshAudio();refreshAudioDevices();refreshSkills();refreshReplayFiles();refreshScripts();refreshPrompt();refreshRecords();populateGestureSelect();refreshLiveVoice();refreshLiveSub();refreshTR();refreshWakeActions();refreshApiKey();connectLogs(); +refreshStatus();refreshSystem();refreshAudio();refreshAudioDevices();refreshSkills();refreshReplayFiles();refreshScripts();refreshPrompt();refreshRecords();populateGestureSelect();refreshLiveVoice();refreshLiveSub();refreshTR();refreshWakeActions();refreshApiKey();refreshCombo();connectLogs(); setTimeout(autoConnectGemini,2000);setTimeout(autoStartLiveSub,3000); setInterval(refreshStatus,5000);setInterval(refreshSystem,30000);setInterval(refreshLiveVoice,5000);setInterval(refreshLiveSub,5000);