/* Interaction - speech, screen expression, LED strip. */ import { store, command, num, api, get, toast } from '../core.js'; import { el, card, pageHead, button, note, badge, range, select, textarea, field, segmented, emptyState, } from '../ui.js'; export default { id: 'interaction', label: 'Interaction', icon: 'interaction', async render() { const spec = store.spec; const root = el('div.stack'); root.appendChild(pageHead( 'Interaction', 'What the robot says, shows on its face, and signals with its light strip.', )); /* ================================================================== Conversation — the listening loop (Muza / Lumi) ================================================================== The robot only starts listening once a character, a language and a model have been chosen and Apply is pressed. Selections are staged locally in `draft` and committed in one request, because switching any of them means relaunching the voice process — doing that per-click would restart the pipeline three times on the way to one setup. */ let session = null; let applying = false; const draft = { enabled: false, gender: 'female', language: 'arabic', model: 'gemini' }; const sessionControlsHost = el('div.stack'); const sessionStatusHost = el('div'); const sessionDetailHost = el('div'); const sessionWarnHost = el('div'); const applyBtn = button('Apply', async () => { applying = true; applyBtn.disabled = true; applyBtn.textContent = draft.enabled ? 'Starting…' : 'Stopping…'; paintStatus(); try { // Deliberately not `command()`: a unit restart can outlast its 15s // default timeout, and a timed-out apply that actually succeeded is // the most confusing possible outcome here. const data = await api('/api/voice/session', { method: 'POST', body: { ...draft }, timeout: 90000, }); // Re-seed from what the server ACTUALLY applied, never from `draft`. // Showing the requested state instead of the achieved one is what made // the card read "muza_ar.txt / Kore" while Lumi was selected. syncFromServer(data.snapshot); if (data.snapshot?.profile?.enabled) { const who = `${session.persona.name_en} · ${session.persona.name_ar}`; toast('Starting', `${who} — the greeting plays once the voice ` + 'connects, usually within about ten seconds.', 'good'); } else { toast('Speaking off', 'The robot has stopped listening.', 'good'); } } catch (err) { toast('Could not apply', err.message, 'critical'); // Pull the real state back: a failed apply may still have changed it. get('/api/voice/session').then(syncFromServer).catch(() => {}); } finally { applying = false; applyBtn.disabled = false; applyBtn.textContent = 'Apply'; paintStatus(); } }, { style: 'primary' }); /* Controls are rebuilt only when the SERVER's state changes, not on every local click — rebuilding mid-interaction would tear down the widget the operator just clicked. Local clicks mutate `draft` and repaint only the read-only lines below it. */ function buildSessionControls() { const enableBox = el('input', { type: 'checkbox', checked: draft.enabled }); enableBox.addEventListener('change', () => { draft.enabled = enableBox.checked; paintDetail(); }); sessionControlsHost.replaceChildren( el('label.switch', {}, enableBox, el('span.switch-track'), el('span.switch-label', { text: 'Speaking on' })), field('Character', segmented(session.options.gender, draft.gender, (value) => { draft.gender = value; paintDetail(); }), 'Female speaks as Muza (موزة); male speaks as Lumi (لومي).'), field('Language', segmented(session.options.language, draft.language, (value) => { draft.language = value; paintDetail(); }), 'Arabic only is pure Emirati dialect. Multi-language still opens in ' + 'Emirati and follows the visitor from there.'), field('Model', segmented(session.options.model, draft.model, (value) => { draft.model = value; paintDetail(); }), 'Gemini Live is streaming speech-to-speech. LinkSoul routes through ' + 'the robot’s own agent, which keeps its native co-speech motion.'), el('div.row.between', {}, sessionWarnHost, applyBtn), ); } function paintStatus() { const running = session?.service?.running; sessionStatusHost.replaceChildren( applying ? badge('Starting…', 'accent') : badge(running ? 'Listening' : 'Off', running ? 'good' : 'default'), ); } function paintDetail() { if (!session) return; // Everything below describes the STAGED selection, so it has to be // derived from `draft` — reading persona/voice off the last snapshot is // what made the line disagree with the selected character. const persona = (session.personas || {})[`${draft.gender}_${draft.language}`]; const voice = draft.model === 'linksoul' ? session.voices?.edge?.[draft.gender] : session.voices?.gemini?.[draft.gender]; const staged = draft.enabled !== !!session.profile?.enabled || draft.gender !== session.profile?.gender || draft.language !== session.profile?.language || draft.model !== session.profile?.model; sessionDetailHost.replaceChildren( el('div.hint', { text: `${staged ? 'Will apply' : 'Running'}: persona ${persona || '—'} · ` + `voice ${voice || '—'} · unit ${session.service?.unit || '—'} ` + `(${session.service?.state || '—'})`, }), ); // Only warn about LinkSoul while LinkSoul is the staged choice — the // missing SDK is irrelevant noise when Gemini is selected. const ls = session.linksoul; sessionWarnHost.replaceChildren( draft.model === 'linksoul' && ls && !ls.ready ? note(`LinkSoul cannot start yet — ${ls.reasons.join('; ')}.`, 'warning') : el('span'), ); } function syncFromServer(snapshot) { if (!snapshot) return; session = snapshot; if (!snapshot.available) { sessionControlsHost.replaceChildren( note(`Voice session unavailable — ${snapshot.error}. Looked for Sanad at ` + `${snapshot.sanad_dir || 'the configured path'}.`, 'warning'), ); return; } // Seed the toggle from the SAVED INTENT, not from whether the unit // happens to be up. Reading service.running here meant that pressing // Apply during a restart (or any moment the unit was briefly down) sent // enabled:false and STOPPED the voice instead of starting it — which // reads exactly like "I pressed Apply and now it never answers". // paintStatus() still shows the real unit state, so a disagreement // between intent and reality stays visible without rewriting the intent. Object.assign(draft, snapshot.profile); buildSessionControls(); paintStatus(); paintDetail(); } const conversationCard = card('Conversation', { sub: 'Muza / Lumi', actions: [sessionStatusHost], }, el('div.stack', {}, sessionControlsHost, sessionDetailHost)); root.appendChild(conversationCard); get('/api/voice/session').then(syncFromServer).catch((err) => { sessionControlsHost.replaceChildren( note(`Could not read the voice session: ${err.message}`, 'warning'), ); }); /* ================================================================== Voice ================================================================== */ const speechBox = textarea({ placeholder: 'Type what the robot should say…', rows: 3, }); const prioritySelect = select( (spec?.tts_priorities || []).map((p) => ({ value: p.value, label: `${p.label} (${p.value}) — ${p.desc}` })), { value: 6 }, ); let interrupt = false; const speakBtn = button('Speak', async () => { const text = speechBox.value.trim(); if (!text) { speechBox.focus(); return; } const ok = await command('/api/speak', { text, priority: Number(prioritySelect.value), interrupt, }, { successTitle: 'Sent to TTS' }); if (ok) speechBox.value = ''; }, { style: 'primary' }); // Ctrl/Cmd+Enter is the expected shortcut in a text box that has a submit button. speechBox.addEventListener('keydown', (event) => { if (event.key === 'Enter' && (event.ctrlKey || event.metaKey)) speakBtn.click(); }); const quickPhrases = [ 'Hello, I am X2.', 'Please stand clear.', 'Starting the routine now.', 'Task complete.', 'Battery is low.', ]; const volumeRange = range({ min: 0, max: 100, step: 1, value: store.state?.volume ?? 60, precision: 0, unit: '%', onChange: (value) => command('/api/volume', { volume: value }, { silent: true }), }); const muteHost = el('div'); const ttsStatusHost = el('div'); const voiceCard = card('Speech', { sub: 'PlayTts' }, el('div.stack', {}, field('Text', speechBox, 'Ctrl + Enter to send'), el('div.row.tight', {}, ...quickPhrases.map((phrase) => button(phrase, () => { speechBox.value = phrase; speechBox.focus(); }, { size: 'sm', style: 'ghost' })), ), field('Priority', prioritySelect, 'Higher priorities pre-empt queued speech. Reserve 8–10 for genuine safety announcements.'), el('div.row.between', {}, (() => { const box = el('input', { type: 'checkbox' }); box.addEventListener('change', () => { interrupt = box.checked; }); return el('label.switch', {}, box, el('span.switch-track'), el('span.switch-label', { text: 'Interrupt current speech' })); })(), speakBtn, ), ), ); const audioCard = card('Audio', { sub: 'GetVolume / SetVolume / SetMute' }, el('div.stack', {}, el('div', {}, el('div', { text: 'Output volume', style: { fontSize: '12px', color: 'var(--text-2)', marginBottom: '5px' } }), volumeRange, ), muteHost, ttsStatusHost, note('Microphone source switching is not exposed as a service on this firmware, so it is ' + 'not offered here. Volume and mute are live.', 'info'), ), ); root.appendChild(el('div.grid.cols-2', {}, voiceCard, audioCard)); /* ================================================================== Screen / emoji ================================================================== */ let emojiMode = 1; const emojiHost = el('div.stack'); const emojiTiles = new Map(); const modeTabs = segmented( (spec?.emoji_modes || []).map((m) => ({ value: m.value, label: m.label })), emojiMode, (value) => { emojiMode = value; }, ); for (const group of spec?.emoji_groups || []) { const inGroup = (spec?.emojis || []).filter((e) => e.group === group.key); if (!inGroup.length) continue; const grid = el('div.emoji-grid'); for (const emoji of inGroup) { const tile = el('button.emoji-tile', { type: 'button', 'aria-pressed': 'false', title: `ID ${emoji.id}`, onclick: () => command('/api/emoji', { emotion_id: emoji.id, mode: emojiMode }, { successTitle: emoji.label }), }, el('span.emoji-glyph', { text: emoji.glyph }), el('span.emoji-name', { text: emoji.label }), ); emojiTiles.set(emoji.id, tile); grid.appendChild(tile); } emojiHost.appendChild(el('div', {}, el('div', { text: group.label, style: { fontSize: '11px', textTransform: 'uppercase', letterSpacing: '.07em', color: 'var(--text-3)', fontWeight: '600', margin: '2px 0 8px' }, }), grid, )); } root.appendChild(card('Face', { sub: `PlayEmoji · ${(spec?.emojis || []).length} expressions`, actions: [modeTabs], }, emojiHost)); /* ================================================================== LED strip ================================================================== */ let led = { mode: 0, r: 57, g: 135, b: 229 }; let ledKeep = true; const preview = el('div', { style: { width: '100%', height: '54px', borderRadius: 'var(--radius-sm)', border: '1px solid var(--border)', background: `rgb(${led.r}, ${led.g}, ${led.b})`, transition: 'background .2s', }, }); const colorInput = el('input.input', { type: 'color', value: rgbToHex(led) }); colorInput.addEventListener('input', () => { Object.assign(led, hexToRgb(colorInput.value)); preview.style.background = `rgb(${led.r}, ${led.g}, ${led.b})`; markSwatch(); }); const swatchHost = el('div.swatches'); const swatchNodes = []; for (const swatch of spec?.led_swatches || []) { const node = el('button.swatch', { type: 'button', title: swatch.label, 'aria-pressed': 'false', style: { background: `rgb(${swatch.r}, ${swatch.g}, ${swatch.b})`, borderColor: swatch.r + swatch.g + swatch.b === 0 ? 'var(--surface-hi)' : 'transparent', }, onclick: () => { led = { ...led, r: swatch.r, g: swatch.g, b: swatch.b }; colorInput.value = rgbToHex(led); preview.style.background = `rgb(${led.r}, ${led.g}, ${led.b})`; markSwatch(); }, }); node._swatch = swatch; swatchNodes.push(node); swatchHost.appendChild(node); } function markSwatch() { for (const node of swatchNodes) { const s = node._swatch; node.setAttribute('aria-pressed', String(s.r === led.r && s.g === led.g && s.b === led.b)); } } markSwatch(); const ledModeTabs = segmented( (spec?.led_modes || []).map((m) => ({ value: m.value, label: m.label })), led.mode, (value) => { led.mode = value; }, ); const ledCard = card('Light strip', { sub: 'SetPmuLed', actions: [ledModeTabs] }, el('div.stack', {}, preview, swatchHost, field('Custom colour', colorInput), (() => { const box = el('input', { type: 'checkbox', checked: ledKeep }); box.addEventListener('change', () => { ledKeep = box.checked; }); return el('div', {}, el('label.switch', {}, box, el('span.switch-track'), el('span.switch-label', { text: 'Keep it on' })), el('div.hint', { text: 'The robot\'s own task_manager reclaims the light strip after about a minute. ' + 'With this on, the dashboard re-applies your setting every 20 seconds so it ' + 'stays. Turn it off if you want a single one-off flash instead.', style: { marginTop: '5px' }, }), ); })(), el('div.row', {}, button('Apply', () => command('/api/led', { ...led, keep: ledKeep }, { successTitle: ledKeep ? 'LED set and held' : 'LED set once' }), { style: 'primary' }), button('Turn off', () => { led = { ...led, r: 0, g: 0, b: 0 }; colorInput.value = '#000000'; preview.style.background = '#000'; markSwatch(); // keep:false also cancels the keepalive, so "off" stays off. return command('/api/led', { ...led, mode: 0, keep: false }, { successTitle: 'LED off' }); }), ), el('div', {}, ...(spec?.led_modes || []).map((m) => el('div', { text: `${m.label} — ${m.desc}`, style: { fontSize: '11.5px', color: 'var(--text-3)', lineHeight: '1.6' }, })), ), ), ); const statusCard = card('Current output', {}, el('div', { id: 'interaction-status' }), ); root.appendChild(el('div.grid.cols-2', {}, ledCard, statusCard)); /* ================================================================== Live state ================================================================== */ function paint() { const s = store.state; if (!s) return; muteHost.replaceChildren((() => { const box = el('input', { type: 'checkbox', checked: s.muted }); box.addEventListener('change', () => command('/api/mute', { muted: box.checked }, { silent: true })); return el('label.switch', {}, box, el('span.switch-track'), el('span.switch-label', { text: s.muted ? 'Muted' : 'Sound on' })); })()); const faceStatus = s.custom?.face_status; const faceLabel = (spec?.face_status || {})[faceStatus]; ttsStatusHost.replaceChildren( el('div.row', {}, el('span', { text: 'Face playback', style: { fontSize: '12px', color: 'var(--text-2)' } }), badge(faceLabel || 'unknown', faceStatus === 2 ? 'good' : faceStatus === 0 ? 'default' : 'accent'), ), ); for (const [id, tile] of emojiTiles) { tile.setAttribute('aria-pressed', String(id === s.emoji_id)); } const activeEmoji = (spec?.emojis || []).find((e) => e.id === s.emoji_id); const modeLabel = (spec?.led_modes || []).find((m) => m.value === s.led?.mode)?.label || '—'; document.getElementById('interaction-status')?.replaceChildren( el('div.grid.cols-2', { style: { gap: '10px' } }, el('div.stat', {}, el('div.stat-label', { text: 'Face' }), el('div.row', { style: { gap: '8px', alignItems: 'center', marginTop: '4px' } }, el('span', { text: activeEmoji?.glyph || '—', style: { fontSize: '26px' } }), el('span', { text: activeEmoji?.label || 'nothing set', style: { fontSize: '13px' } }), ), ), el('div.stat', {}, el('div.stat-label', { text: 'Light strip' }), el('div.row', { style: { gap: '8px', alignItems: 'center', marginTop: '4px' } }, el('span', { style: { width: '22px', height: '22px', borderRadius: '6px', border: '1px solid var(--border)', background: `rgb(${s.led?.r ?? 0}, ${s.led?.g ?? 0}, ${s.led?.b ?? 0})`, }, }), el('span', { text: modeLabel, style: { fontSize: '13px' } }), ), ), el('div.stat', {}, el('div.stat-label', { text: 'Volume' }), el('div.stat-value', { text: `${s.volume ?? 0}` }, el('span.unit', { text: '%' })), ), el('div.stat', {}, el('div.stat-label', { text: 'Audio' }), el('div', { style: { marginTop: '6px' } }, badge(s.muted ? 'Muted' : 'Active', s.muted ? 'warning' : 'good'), ), ), ), ); } paint(); let last = 0; const unsubscribe = store.on('state', () => { const now = Date.now(); if (now - last < 800) return; last = now; paint(); }); return { node: root, dispose: unsubscribe }; }, }; function rgbToHex({ r, g, b }) { return `#${[r, g, b].map((v) => Math.max(0, Math.min(255, v | 0)).toString(16).padStart(2, '0')).join('')}`; } function hexToRgb(hex) { const clean = hex.replace('#', ''); return { r: parseInt(clean.slice(0, 2), 16) || 0, g: parseInt(clean.slice(2, 4), 16) || 0, b: parseInt(clean.slice(4, 6), 16) || 0, }; }