Update 2026-04-21 12:00:42

dashboard/routes/macros.py

@@ -2,9 +2,16 @@
 
 from __future__ import annotations
 
+import asyncio
+from pathlib import Path
+
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 
+from Project.Sanad.config import AUDIO_RECORDINGS_DIR, MOTIONS_DIR
+from Project.Sanad.core.logger import get_logger
+
+log = get_logger("macros_route")
 router = APIRouter()
 
 
@@ -12,6 +19,13 @@ class MacroName(BaseModel):
     name: str
 
 
+class ComboPlayPayload(BaseModel):
+    audio_file: str = ""      # filename under data/audio/ (or empty for none)
+    motion_file: str = ""     # DEPRECATED — use action_id. Still accepted for bare JSONL by filename.
+    action_id: int | None = None  # arm_controller action id (SDK built-in OR JSONL) — preferred
+    speed: float = 1.0
+
+
 @router.get("/")
 async def list_macros():
     from Project.Sanad.main import macro_play
@@ -58,3 +72,118 @@ async def stop_macro():
     if macro_play:
         macro_play.stop()
     return {"ok": True}
+
+
+# ─── Ad-hoc audio + motion combined playback ─────────────────────────
+# List the two catalogues so the dashboard can populate dropdowns, then
+# play the chosen pair in parallel (asyncio.gather) — same scheme the
+# Brain uses for `parallel`-mode skills, but ad-hoc instead of predefined.
+
+@router.get("/audio-files")
+async def list_audio_files():
+    """Enumerate playable audio files under data/audio/."""
+    AUDIO_RECORDINGS_DIR.mkdir(parents=True, exist_ok=True)
+    files = []
+    for p in sorted(AUDIO_RECORDINGS_DIR.glob("*.wav")):
+        try:
+            files.append({
+                "name": p.name,
+                "size_kb": round(p.stat().st_size / 1024, 1),
+            })
+        except OSError:
+            continue
+    return {"files": files, "dir": str(AUDIO_RECORDINGS_DIR)}
+
+
+@router.get("/motion-files")
+async def list_motion_files():
+    """Enumerate playable .jsonl motions under data/motions/ (thin wrapper
+    so the Macro Recorder dropdown doesn't have to call the replay route)."""
+    MOTIONS_DIR.mkdir(parents=True, exist_ok=True)
+    files = []
+    for p in sorted(MOTIONS_DIR.glob("*.jsonl")):
+        try:
+            files.append({
+                "name": p.name,
+                "size_kb": round(p.stat().st_size / 1024, 1),
+            })
+        except OSError:
+            continue
+    return {"files": files, "dir": str(MOTIONS_DIR)}
+
+
+@router.post("/play-combined")
+async def play_combined(payload: ComboPlayPayload):
+    """Fire a user-picked audio clip and arm action in parallel.
+
+    Motion dispatch is via `arm.trigger_by_id(action_id)` which handles
+    BOTH SDK built-in actions (shake_hand, wave, …) and recorded JSONL
+    replays. Audio goes through `audio_mgr.play_wav` (routed to the G1
+    chest speaker via DDS). Either side may be omitted.
+    """
+    from Project.Sanad.main import audio_mgr, arm
+
+    has_audio = bool(payload.audio_file)
+    has_motion = payload.action_id is not None or bool(payload.motion_file)
+    if not has_audio and not has_motion:
+        raise HTTPException(400, "pick at least one of audio_file / action_id / motion_file")
+
+    tasks = []
+    result: dict = {
+        "audio_file": payload.audio_file,
+        "action_id": payload.action_id,
+        "motion_file": payload.motion_file,
+    }
+
+    if has_audio:
+        if audio_mgr is None:
+            raise HTTPException(503, "AudioManager not available")
+        audio_path = (AUDIO_RECORDINGS_DIR / payload.audio_file).resolve()
+        try:
+            audio_path.relative_to(AUDIO_RECORDINGS_DIR.resolve())
+        except ValueError:
+            raise HTTPException(400, "audio_file path traversal denied")
+        if not audio_path.exists():
+            raise HTTPException(404, f"audio not found: {payload.audio_file}")
+
+        async def _play_audio():
+            try:
+                await asyncio.to_thread(audio_mgr.play_wav, audio_path)
+                result["audio_played"] = audio_path.name
+            except Exception as exc:
+                log.exception("combined play: audio failed")
+                result["audio_error"] = str(exc)
+        tasks.append(_play_audio())
+
+    if has_motion:
+        if arm is None:
+            raise HTTPException(503, "ArmController not available")
+
+        async def _play_motion():
+            try:
+                if payload.action_id is not None:
+                    # SDK built-in OR JSONL — arm.trigger_by_id handles both
+                    await asyncio.to_thread(arm.trigger_by_id,
+                                            int(payload.action_id),
+                                            payload.speed)
+                    result["motion_played"] = f"action_id={payload.action_id}"
+                else:
+                    # Legacy path: bare JSONL filename
+                    motion_path = (MOTIONS_DIR / payload.motion_file).resolve()
+                    try:
+                        motion_path.relative_to(MOTIONS_DIR.resolve())
+                    except ValueError:
+                        result["motion_error"] = "motion_file path traversal denied"
+                        return
+                    if not motion_path.exists():
+                        result["motion_error"] = f"motion not found: {payload.motion_file}"
+                        return
+                    await asyncio.to_thread(arm.replay_file, str(motion_path), payload.speed)
+                    result["motion_played"] = motion_path.name
+            except Exception as exc:
+                log.exception("combined play: motion failed")
+                result["motion_error"] = str(exc)
+        tasks.append(_play_motion())
+
+    await asyncio.gather(*tasks)
+    return {"ok": True, **result}

dashboard/static/index.html

@@ -432,16 +432,37 @@
   <!-- Macro Recorder -->
   <div class="card card-full">
     <h3>Macro Recorder (Audio + Motion)</h3>
-    <div style="display:flex;gap:1rem;flex-wrap:wrap">
+
-      <div style="flex:1">
+    <!-- Record -->
-        <label>Record</label>
+    <label>Record</label>
-        <div class="row" style="margin-top:.3rem"><input id="macro-name" placeholder="Macro name" style="flex:1"><button class="btn btn-primary btn-sm" onclick="startMacro(this)">Record</button><button class="btn btn-danger btn-sm" onclick="stopMacro(this)">Stop</button></div>
+    <div class="row" style="margin-top:.3rem">
+      <input id="macro-name" placeholder="Macro name" style="flex:1">
+      <button class="btn btn-primary btn-sm" onclick="startMacro(this)">Record</button>
+      <button class="btn btn-danger btn-sm" onclick="stopMacro(this)">Stop</button>
+    </div>
+
+    <!-- Play: pick a voice + motion (either optional), play in parallel -->
+    <label style="margin-top:.6rem;display:block">Play</label>
+    <div class="row" style="margin-top:.3rem;gap:.4rem;flex-wrap:wrap">
+      <div style="flex:1;min-width:200px">
+        <div style="font-size:.68rem;color:var(--muted);margin-bottom:.15rem">Voice (WAV)</div>
+        <select id="combo-voice" style="width:100%"><option value="">— none —</option></select>
       </div>
-      <div style="flex:1">
+      <div style="flex:1;min-width:200px">
-        <label>Playback</label>
+        <div style="font-size:.68rem;color:var(--muted);margin-bottom:.15rem">Motion (JSONL)</div>
-        <div class="row" style="margin-top:.3rem"><input id="play-macro-name" placeholder="Macro to play" style="flex:1"><button class="btn btn-success btn-sm" onclick="playMacro(this)">Play</button></div>
+        <select id="combo-motion" style="width:100%"><option value="">— none —</option></select>
+      </div>
+      <div style="align-self:flex-end">
+        <div style="font-size:.68rem;color:var(--muted);margin-bottom:.15rem">Speed</div>
+        <select id="combo-speed" style="width:75px"><option value="0.5">0.5x</option><option value="1.0" selected>1.0x</option><option value="1.5">1.5x</option><option value="2.0">2.0x</option></select>
+      </div>
+      <div style="align-self:flex-end;display:flex;gap:.3rem">
+        <button class="btn btn-ghost btn-sm" onclick="refreshCombo()" title="Reload file lists">↻</button>
+        <button class="btn btn-success btn-sm" onclick="playCombo(this)">Play</button>
       </div>
     </div>
+
+    <div id="combo-status" style="font-size:.7rem;color:var(--muted);margin-top:.3rem"></div>
     <div id="macro-status" style="font-size:.72rem;color:var(--muted);margin-top:.3rem"></div>
   </div>
 
@@ -1109,6 +1130,70 @@ async function startMacro(b){const n=document.getElementById('macro-name').value
 async function stopMacro(b){btnLoad(b);try{const r=await api('POST','/api/macros/record/stop');toast('Saved','ok');document.getElementById('macro-status').textContent=`Saved: ${r.name} (${r.duration_sec}s)`;}catch(e){}btnDone(b);}
 async function playMacro(b){const n=document.getElementById('play-macro-name').value;if(!n)return toast('Enter name','err');btnLoad(b);try{await api('POST','/api/macros/play',{name:n});toast('Played: '+n,'ok');}catch(e){}btnDone(b);}
 
+// Ad-hoc combined playback — select voice + motion, play in parallel.
+// Motion list = SDK built-ins + JSONL replays (via /api/motion/actions),
+// so the dropdown offers every arm action — not just recorded files.
+async function refreshCombo(){
+  try{
+    const [av,am]=await Promise.all([
+      api('GET','/api/macros/audio-files'),
+      api('GET','/api/motion/actions'),
+    ]);
+    const voiceSel=document.getElementById('combo-voice');
+    const motionSel=document.getElementById('combo-motion');
+    const prevV=voiceSel.value, prevM=motionSel.value;
+
+    voiceSel.innerHTML='<option value="">— none —</option>'
+      +(av.files||[]).map(f=>`<option value="${esc(f.name)}">${esc(f.name)} (${f.size_kb}KB)</option>`).join('');
+
+    // Motion: group by category, SDK first then JSONL
+    const acts=am.actions||[];
+    const sdk=acts.filter(a=>!a.file);
+    const jsl=acts.filter(a=>!!a.file);
+    let html='<option value="">— none —</option>';
+    if(sdk.length){
+      html+='<optgroup label="SDK built-ins">';
+      for(const a of sdk){html+=`<option value="${a.id}">${esc(a.name).replace(/_/g,' ')} (#${a.id})</option>`;}
+      html+='</optgroup>';
+    }
+    if(jsl.length){
+      html+='<optgroup label="JSONL replays">';
+      for(const a of jsl){html+=`<option value="${a.id}">${esc(a.file)} (#${a.id})</option>`;}
+      html+='</optgroup>';
+    }
+    motionSel.innerHTML=html;
+
+    if(prevV)voiceSel.value=prevV;
+    if(prevM)motionSel.value=prevM;
+  }catch(e){toast('Could not load combined lists','err');}
+}
+async function playCombo(b){
+  const v=document.getElementById('combo-voice').value;
+  const mRaw=document.getElementById('combo-motion').value;
+  const actionId=mRaw?parseInt(mRaw,10):null;
+  if(!v&&actionId==null)return toast('Pick a voice or motion (or both)','err');
+  const speed=parseFloat(document.getElementById('combo-speed').value||'1.0');
+  const st=document.getElementById('combo-status');
+  const mLabel=mRaw?document.getElementById('combo-motion').selectedOptions[0].textContent:'(no motion)';
+  st.textContent='Playing: '+[v||'(no voice)',mLabel].join(' + ')+'...';
+  btnLoad(b);
+  try{
+    const r=await api('POST','/api/macros/play-combined',{
+      audio_file:v,
+      action_id:actionId,
+      speed,
+    });
+    const parts=[];
+    if(r.audio_played)parts.push('audio='+r.audio_played);
+    if(r.motion_played)parts.push('motion='+r.motion_played);
+    if(r.audio_error)parts.push('audio_err='+r.audio_error);
+    if(r.motion_error)parts.push('motion_err='+r.motion_error);
+    st.textContent='Done: '+parts.join(', ');
+    toast('Combined play done','ok');
+  }catch(e){st.textContent='Failed';}
+  btnDone(b);
+}
+
 // Replay
 async function refreshReplayFiles(){try{const r=await api('GET','/api/replay/files');const el=document.getElementById('replay-files');if(!(r.files||[]).length){el.innerHTML='<div class="empty">No motion files</div>';return;}el.innerHTML='<table><tr><th>File</th><th>Frames</th><th>Duration</th><th>Size</th><th></th></tr>'+(r.files||[]).map(f=>`<tr><td>${esc(f.name)}</td><td>${f.frames}</td><td>${f.duration_sec}s</td><td>${f.size_kb}KB</td><td><button class="btn btn-primary btn-sm" onclick="document.getElementById('replay-name').value='${esc(f.name)}';testReplay()">Play</button> <button class="btn btn-danger btn-sm" onclick="deleteMotionFile('${esc(f.name)}')">Del</button></td></tr>`).join('')+'</table>';}catch(e){}}
 async function testReplay(b){const n=document.getElementById('replay-name').value,s=parseFloat(document.getElementById('replay-speed').value);if(!n)return;btnLoad(b);try{await api('POST','/api/replay/test',{name:n,speed:s});toast('Replay: '+n,'ok');pollArmBusy();}catch(e){}btnDone(b);}
@@ -1315,7 +1400,7 @@ async function autoStartLiveSub(){
 }
 
 // Init — vision/camera/detector fetches removed; those endpoints were deleted.
-refreshStatus();refreshSystem();refreshAudio();refreshAudioDevices();refreshSkills();refreshReplayFiles();refreshScripts();refreshPrompt();refreshRecords();populateGestureSelect();refreshLiveVoice();refreshLiveSub();refreshTR();refreshWakeActions();refreshApiKey();connectLogs();
+refreshStatus();refreshSystem();refreshAudio();refreshAudioDevices();refreshSkills();refreshReplayFiles();refreshScripts();refreshPrompt();refreshRecords();populateGestureSelect();refreshLiveVoice();refreshLiveSub();refreshTR();refreshWakeActions();refreshApiKey();refreshCombo();connectLogs();
 setTimeout(autoConnectGemini,2000);setTimeout(autoStartLiveSub,3000);
 setInterval(refreshStatus,5000);setInterval(refreshSystem,30000);setInterval(refreshLiveVoice,5000);setInterval(refreshLiveSub,5000);
 </script>