536 lines
20 KiB
JavaScript
536 lines
20 KiB
JavaScript
/* Interaction - speech, screen expression, LED strip. */
|
||
|
||
import { store, command, num, api, get, toast } from '../core.js';
|
||
import {
|
||
el, card, pageHead, button, note, badge, range, select, textarea, field,
|
||
segmented, emptyState,
|
||
} from '../ui.js';
|
||
|
||
export default {
|
||
id: 'interaction',
|
||
label: 'Interaction',
|
||
icon: 'interaction',
|
||
|
||
async render() {
|
||
const spec = store.spec;
|
||
const root = el('div.stack');
|
||
|
||
root.appendChild(pageHead(
|
||
'Interaction',
|
||
'What the robot says, shows on its face, and signals with its light strip.',
|
||
));
|
||
|
||
/* ==================================================================
|
||
Conversation — the listening loop (Muza / Lumi)
|
||
==================================================================
|
||
The robot only starts listening once a character, a language and a model
|
||
have been chosen and Apply is pressed. Selections are staged locally in
|
||
`draft` and committed in one request, because switching any of them means
|
||
relaunching the voice process — doing that per-click would restart the
|
||
pipeline three times on the way to one setup. */
|
||
|
||
let session = null;
|
||
let applying = false;
|
||
const draft = { enabled: false, gender: 'female', language: 'arabic', model: 'gemini' };
|
||
|
||
const sessionControlsHost = el('div.stack');
|
||
const sessionStatusHost = el('div');
|
||
const sessionDetailHost = el('div');
|
||
const sessionWarnHost = el('div');
|
||
|
||
const applyBtn = button('Apply', async () => {
|
||
applying = true;
|
||
applyBtn.disabled = true;
|
||
applyBtn.textContent = draft.enabled ? 'Starting…' : 'Stopping…';
|
||
paintStatus();
|
||
try {
|
||
// Deliberately not `command()`: a unit restart can outlast its 15s
|
||
// default timeout, and a timed-out apply that actually succeeded is
|
||
// the most confusing possible outcome here.
|
||
const data = await api('/api/voice/session', {
|
||
method: 'POST', body: { ...draft }, timeout: 90000,
|
||
});
|
||
// Re-seed from what the server ACTUALLY applied, never from `draft`.
|
||
// Showing the requested state instead of the achieved one is what made
|
||
// the card read "muza_ar.txt / Kore" while Lumi was selected.
|
||
syncFromServer(data.snapshot);
|
||
if (data.snapshot?.profile?.enabled) {
|
||
const who = `${session.persona.name_en} · ${session.persona.name_ar}`;
|
||
toast('Starting', `${who} — the greeting plays once the voice `
|
||
+ 'connects, usually within about ten seconds.', 'good');
|
||
} else {
|
||
toast('Speaking off', 'The robot has stopped listening.', 'good');
|
||
}
|
||
} catch (err) {
|
||
toast('Could not apply', err.message, 'critical');
|
||
// Pull the real state back: a failed apply may still have changed it.
|
||
get('/api/voice/session').then(syncFromServer).catch(() => {});
|
||
} finally {
|
||
applying = false;
|
||
applyBtn.disabled = false;
|
||
applyBtn.textContent = 'Apply';
|
||
paintStatus();
|
||
}
|
||
}, { style: 'primary' });
|
||
|
||
/* Controls are rebuilt only when the SERVER's state changes, not on every
|
||
local click — rebuilding mid-interaction would tear down the widget the
|
||
operator just clicked. Local clicks mutate `draft` and repaint only the
|
||
read-only lines below it. */
|
||
function buildSessionControls() {
|
||
const enableBox = el('input', { type: 'checkbox', checked: draft.enabled });
|
||
enableBox.addEventListener('change', () => {
|
||
draft.enabled = enableBox.checked;
|
||
paintDetail();
|
||
});
|
||
|
||
sessionControlsHost.replaceChildren(
|
||
el('label.switch', {}, enableBox, el('span.switch-track'),
|
||
el('span.switch-label', { text: 'Speaking on' })),
|
||
|
||
field('Character',
|
||
segmented(session.options.gender, draft.gender, (value) => {
|
||
draft.gender = value;
|
||
paintDetail();
|
||
}),
|
||
'Female speaks as Muza (موزة); male speaks as Lumi (لومي).'),
|
||
|
||
field('Language',
|
||
segmented(session.options.language, draft.language, (value) => {
|
||
draft.language = value;
|
||
paintDetail();
|
||
}),
|
||
'Arabic only is pure Emirati dialect. Multi-language still opens in '
|
||
+ 'Emirati and follows the visitor from there.'),
|
||
|
||
field('Model',
|
||
segmented(session.options.model, draft.model, (value) => {
|
||
draft.model = value;
|
||
paintDetail();
|
||
}),
|
||
'Gemini Live is streaming speech-to-speech. LinkSoul routes through '
|
||
+ 'the robot’s own agent, which keeps its native co-speech motion.'),
|
||
|
||
el('div.row.between', {}, sessionWarnHost, applyBtn),
|
||
);
|
||
}
|
||
|
||
function paintStatus() {
|
||
const running = session?.service?.running;
|
||
sessionStatusHost.replaceChildren(
|
||
applying ? badge('Starting…', 'accent')
|
||
: badge(running ? 'Listening' : 'Off', running ? 'good' : 'default'),
|
||
);
|
||
}
|
||
|
||
function paintDetail() {
|
||
if (!session) return;
|
||
|
||
// Everything below describes the STAGED selection, so it has to be
|
||
// derived from `draft` — reading persona/voice off the last snapshot is
|
||
// what made the line disagree with the selected character.
|
||
const persona = (session.personas || {})[`${draft.gender}_${draft.language}`];
|
||
const voice = draft.model === 'linksoul'
|
||
? session.voices?.edge?.[draft.gender]
|
||
: session.voices?.gemini?.[draft.gender];
|
||
|
||
const staged = draft.enabled !== !!session.profile?.enabled
|
||
|| draft.gender !== session.profile?.gender
|
||
|| draft.language !== session.profile?.language
|
||
|| draft.model !== session.profile?.model;
|
||
|
||
sessionDetailHost.replaceChildren(
|
||
el('div.hint', {
|
||
text: `${staged ? 'Will apply' : 'Running'}: persona ${persona || '—'} · `
|
||
+ `voice ${voice || '—'} · unit ${session.service?.unit || '—'} `
|
||
+ `(${session.service?.state || '—'})`,
|
||
}),
|
||
);
|
||
|
||
// Only warn about LinkSoul while LinkSoul is the staged choice — the
|
||
// missing SDK is irrelevant noise when Gemini is selected.
|
||
const ls = session.linksoul;
|
||
sessionWarnHost.replaceChildren(
|
||
draft.model === 'linksoul' && ls && !ls.ready
|
||
? note(`LinkSoul cannot start yet — ${ls.reasons.join('; ')}.`, 'warning')
|
||
: el('span'),
|
||
);
|
||
}
|
||
|
||
function syncFromServer(snapshot) {
|
||
if (!snapshot) return;
|
||
session = snapshot;
|
||
if (!snapshot.available) {
|
||
sessionControlsHost.replaceChildren(
|
||
note(`Voice session unavailable — ${snapshot.error}. Looked for Sanad at `
|
||
+ `${snapshot.sanad_dir || 'the configured path'}.`, 'warning'),
|
||
);
|
||
return;
|
||
}
|
||
// Seed the toggle from the SAVED INTENT, not from whether the unit
|
||
// happens to be up. Reading service.running here meant that pressing
|
||
// Apply during a restart (or any moment the unit was briefly down) sent
|
||
// enabled:false and STOPPED the voice instead of starting it — which
|
||
// reads exactly like "I pressed Apply and now it never answers".
|
||
// paintStatus() still shows the real unit state, so a disagreement
|
||
// between intent and reality stays visible without rewriting the intent.
|
||
Object.assign(draft, snapshot.profile);
|
||
buildSessionControls();
|
||
paintStatus();
|
||
paintDetail();
|
||
}
|
||
|
||
const conversationCard = card('Conversation', {
|
||
sub: 'Muza / Lumi',
|
||
actions: [sessionStatusHost],
|
||
}, el('div.stack', {}, sessionControlsHost, sessionDetailHost));
|
||
|
||
root.appendChild(conversationCard);
|
||
|
||
get('/api/voice/session').then(syncFromServer).catch((err) => {
|
||
sessionControlsHost.replaceChildren(
|
||
note(`Could not read the voice session: ${err.message}`, 'warning'),
|
||
);
|
||
});
|
||
|
||
/* ==================================================================
|
||
Voice
|
||
================================================================== */
|
||
|
||
const speechBox = textarea({
|
||
placeholder: 'Type what the robot should say…',
|
||
rows: 3,
|
||
});
|
||
|
||
const prioritySelect = select(
|
||
(spec?.tts_priorities || []).map((p) => ({ value: p.value, label: `${p.label} (${p.value}) — ${p.desc}` })),
|
||
{ value: 6 },
|
||
);
|
||
|
||
let interrupt = false;
|
||
|
||
const speakBtn = button('Speak', async () => {
|
||
const text = speechBox.value.trim();
|
||
if (!text) { speechBox.focus(); return; }
|
||
const ok = await command('/api/speak', {
|
||
text,
|
||
priority: Number(prioritySelect.value),
|
||
interrupt,
|
||
}, { successTitle: 'Sent to TTS' });
|
||
if (ok) speechBox.value = '';
|
||
}, { style: 'primary' });
|
||
|
||
// Ctrl/Cmd+Enter is the expected shortcut in a text box that has a submit button.
|
||
speechBox.addEventListener('keydown', (event) => {
|
||
if (event.key === 'Enter' && (event.ctrlKey || event.metaKey)) speakBtn.click();
|
||
});
|
||
|
||
const quickPhrases = [
|
||
'Hello, I am X2.',
|
||
'Please stand clear.',
|
||
'Starting the routine now.',
|
||
'Task complete.',
|
||
'Battery is low.',
|
||
];
|
||
|
||
const volumeRange = range({
|
||
min: 0, max: 100, step: 1, value: store.state?.volume ?? 60, precision: 0, unit: '%',
|
||
onChange: (value) => command('/api/volume', { volume: value }, { silent: true }),
|
||
});
|
||
|
||
const muteHost = el('div');
|
||
const ttsStatusHost = el('div');
|
||
|
||
const voiceCard = card('Speech', { sub: 'PlayTts' },
|
||
el('div.stack', {},
|
||
field('Text', speechBox, 'Ctrl + Enter to send'),
|
||
el('div.row.tight', {},
|
||
...quickPhrases.map((phrase) => button(phrase, () => {
|
||
speechBox.value = phrase;
|
||
speechBox.focus();
|
||
}, { size: 'sm', style: 'ghost' })),
|
||
),
|
||
field('Priority', prioritySelect,
|
||
'Higher priorities pre-empt queued speech. Reserve 8–10 for genuine safety announcements.'),
|
||
el('div.row.between', {},
|
||
(() => {
|
||
const box = el('input', { type: 'checkbox' });
|
||
box.addEventListener('change', () => { interrupt = box.checked; });
|
||
return el('label.switch', {}, box, el('span.switch-track'),
|
||
el('span.switch-label', { text: 'Interrupt current speech' }));
|
||
})(),
|
||
speakBtn,
|
||
),
|
||
),
|
||
);
|
||
|
||
const audioCard = card('Audio', { sub: 'GetVolume / SetVolume / SetMute' },
|
||
el('div.stack', {},
|
||
el('div', {},
|
||
el('div', { text: 'Output volume', style: { fontSize: '12px', color: 'var(--text-2)', marginBottom: '5px' } }),
|
||
volumeRange,
|
||
),
|
||
muteHost,
|
||
ttsStatusHost,
|
||
note('Microphone source switching is not exposed as a service on this firmware, so it is '
|
||
+ 'not offered here. Volume and mute are live.', 'info'),
|
||
),
|
||
);
|
||
|
||
root.appendChild(el('div.grid.cols-2', {}, voiceCard, audioCard));
|
||
|
||
/* ==================================================================
|
||
Screen / emoji
|
||
================================================================== */
|
||
|
||
let emojiMode = 1;
|
||
const emojiHost = el('div.stack');
|
||
const emojiTiles = new Map();
|
||
|
||
const modeTabs = segmented(
|
||
(spec?.emoji_modes || []).map((m) => ({ value: m.value, label: m.label })),
|
||
emojiMode,
|
||
(value) => { emojiMode = value; },
|
||
);
|
||
|
||
for (const group of spec?.emoji_groups || []) {
|
||
const inGroup = (spec?.emojis || []).filter((e) => e.group === group.key);
|
||
if (!inGroup.length) continue;
|
||
|
||
const grid = el('div.emoji-grid');
|
||
for (const emoji of inGroup) {
|
||
const tile = el('button.emoji-tile', {
|
||
type: 'button',
|
||
'aria-pressed': 'false',
|
||
title: `ID ${emoji.id}`,
|
||
onclick: () => command('/api/emoji', { emotion_id: emoji.id, mode: emojiMode },
|
||
{ successTitle: emoji.label }),
|
||
},
|
||
el('span.emoji-glyph', { text: emoji.glyph }),
|
||
el('span.emoji-name', { text: emoji.label }),
|
||
);
|
||
emojiTiles.set(emoji.id, tile);
|
||
grid.appendChild(tile);
|
||
}
|
||
|
||
emojiHost.appendChild(el('div', {},
|
||
el('div', {
|
||
text: group.label,
|
||
style: { fontSize: '11px', textTransform: 'uppercase', letterSpacing: '.07em',
|
||
color: 'var(--text-3)', fontWeight: '600', margin: '2px 0 8px' },
|
||
}),
|
||
grid,
|
||
));
|
||
}
|
||
|
||
root.appendChild(card('Face', {
|
||
sub: `PlayEmoji · ${(spec?.emojis || []).length} expressions`,
|
||
actions: [modeTabs],
|
||
}, emojiHost));
|
||
|
||
/* ==================================================================
|
||
LED strip
|
||
================================================================== */
|
||
|
||
let led = { mode: 0, r: 57, g: 135, b: 229 };
|
||
let ledKeep = true;
|
||
|
||
const preview = el('div', {
|
||
style: {
|
||
width: '100%', height: '54px', borderRadius: 'var(--radius-sm)',
|
||
border: '1px solid var(--border)',
|
||
background: `rgb(${led.r}, ${led.g}, ${led.b})`,
|
||
transition: 'background .2s',
|
||
},
|
||
});
|
||
|
||
const colorInput = el('input.input', { type: 'color', value: rgbToHex(led) });
|
||
colorInput.addEventListener('input', () => {
|
||
Object.assign(led, hexToRgb(colorInput.value));
|
||
preview.style.background = `rgb(${led.r}, ${led.g}, ${led.b})`;
|
||
markSwatch();
|
||
});
|
||
|
||
const swatchHost = el('div.swatches');
|
||
const swatchNodes = [];
|
||
for (const swatch of spec?.led_swatches || []) {
|
||
const node = el('button.swatch', {
|
||
type: 'button',
|
||
title: swatch.label,
|
||
'aria-pressed': 'false',
|
||
style: {
|
||
background: `rgb(${swatch.r}, ${swatch.g}, ${swatch.b})`,
|
||
borderColor: swatch.r + swatch.g + swatch.b === 0 ? 'var(--surface-hi)' : 'transparent',
|
||
},
|
||
onclick: () => {
|
||
led = { ...led, r: swatch.r, g: swatch.g, b: swatch.b };
|
||
colorInput.value = rgbToHex(led);
|
||
preview.style.background = `rgb(${led.r}, ${led.g}, ${led.b})`;
|
||
markSwatch();
|
||
},
|
||
});
|
||
node._swatch = swatch;
|
||
swatchNodes.push(node);
|
||
swatchHost.appendChild(node);
|
||
}
|
||
|
||
function markSwatch() {
|
||
for (const node of swatchNodes) {
|
||
const s = node._swatch;
|
||
node.setAttribute('aria-pressed', String(s.r === led.r && s.g === led.g && s.b === led.b));
|
||
}
|
||
}
|
||
markSwatch();
|
||
|
||
const ledModeTabs = segmented(
|
||
(spec?.led_modes || []).map((m) => ({ value: m.value, label: m.label })),
|
||
led.mode,
|
||
(value) => { led.mode = value; },
|
||
);
|
||
|
||
const ledCard = card('Light strip', { sub: 'SetPmuLed', actions: [ledModeTabs] },
|
||
el('div.stack', {},
|
||
preview,
|
||
swatchHost,
|
||
field('Custom colour', colorInput),
|
||
|
||
(() => {
|
||
const box = el('input', { type: 'checkbox', checked: ledKeep });
|
||
box.addEventListener('change', () => { ledKeep = box.checked; });
|
||
return el('div', {},
|
||
el('label.switch', {}, box, el('span.switch-track'),
|
||
el('span.switch-label', { text: 'Keep it on' })),
|
||
el('div.hint', {
|
||
text: 'The robot\'s own task_manager reclaims the light strip after about a minute. '
|
||
+ 'With this on, the dashboard re-applies your setting every 20 seconds so it '
|
||
+ 'stays. Turn it off if you want a single one-off flash instead.',
|
||
style: { marginTop: '5px' },
|
||
}),
|
||
);
|
||
})(),
|
||
|
||
el('div.row', {},
|
||
button('Apply', () => command('/api/led', { ...led, keep: ledKeep },
|
||
{ successTitle: ledKeep ? 'LED set and held' : 'LED set once' }),
|
||
{ style: 'primary' }),
|
||
button('Turn off', () => {
|
||
led = { ...led, r: 0, g: 0, b: 0 };
|
||
colorInput.value = '#000000';
|
||
preview.style.background = '#000';
|
||
markSwatch();
|
||
// keep:false also cancels the keepalive, so "off" stays off.
|
||
return command('/api/led', { ...led, mode: 0, keep: false },
|
||
{ successTitle: 'LED off' });
|
||
}),
|
||
),
|
||
el('div', {},
|
||
...(spec?.led_modes || []).map((m) => el('div', {
|
||
text: `${m.label} — ${m.desc}`,
|
||
style: { fontSize: '11.5px', color: 'var(--text-3)', lineHeight: '1.6' },
|
||
})),
|
||
),
|
||
),
|
||
);
|
||
|
||
const statusCard = card('Current output', {},
|
||
el('div', { id: 'interaction-status' }),
|
||
);
|
||
|
||
root.appendChild(el('div.grid.cols-2', {}, ledCard, statusCard));
|
||
|
||
/* ==================================================================
|
||
Live state
|
||
================================================================== */
|
||
|
||
function paint() {
|
||
const s = store.state;
|
||
if (!s) return;
|
||
|
||
muteHost.replaceChildren((() => {
|
||
const box = el('input', { type: 'checkbox', checked: s.muted });
|
||
box.addEventListener('change', () =>
|
||
command('/api/mute', { muted: box.checked }, { silent: true }));
|
||
return el('label.switch', {}, box, el('span.switch-track'),
|
||
el('span.switch-label', { text: s.muted ? 'Muted' : 'Sound on' }));
|
||
})());
|
||
|
||
const faceStatus = s.custom?.face_status;
|
||
const faceLabel = (spec?.face_status || {})[faceStatus];
|
||
ttsStatusHost.replaceChildren(
|
||
el('div.row', {},
|
||
el('span', { text: 'Face playback', style: { fontSize: '12px', color: 'var(--text-2)' } }),
|
||
badge(faceLabel || 'unknown',
|
||
faceStatus === 2 ? 'good' : faceStatus === 0 ? 'default' : 'accent'),
|
||
),
|
||
);
|
||
|
||
for (const [id, tile] of emojiTiles) {
|
||
tile.setAttribute('aria-pressed', String(id === s.emoji_id));
|
||
}
|
||
|
||
const activeEmoji = (spec?.emojis || []).find((e) => e.id === s.emoji_id);
|
||
const modeLabel = (spec?.led_modes || []).find((m) => m.value === s.led?.mode)?.label || '—';
|
||
|
||
document.getElementById('interaction-status')?.replaceChildren(
|
||
el('div.grid.cols-2', { style: { gap: '10px' } },
|
||
el('div.stat', {},
|
||
el('div.stat-label', { text: 'Face' }),
|
||
el('div.row', { style: { gap: '8px', alignItems: 'center', marginTop: '4px' } },
|
||
el('span', { text: activeEmoji?.glyph || '—', style: { fontSize: '26px' } }),
|
||
el('span', { text: activeEmoji?.label || 'nothing set', style: { fontSize: '13px' } }),
|
||
),
|
||
),
|
||
el('div.stat', {},
|
||
el('div.stat-label', { text: 'Light strip' }),
|
||
el('div.row', { style: { gap: '8px', alignItems: 'center', marginTop: '4px' } },
|
||
el('span', {
|
||
style: {
|
||
width: '22px', height: '22px', borderRadius: '6px',
|
||
border: '1px solid var(--border)',
|
||
background: `rgb(${s.led?.r ?? 0}, ${s.led?.g ?? 0}, ${s.led?.b ?? 0})`,
|
||
},
|
||
}),
|
||
el('span', { text: modeLabel, style: { fontSize: '13px' } }),
|
||
),
|
||
),
|
||
el('div.stat', {},
|
||
el('div.stat-label', { text: 'Volume' }),
|
||
el('div.stat-value', { text: `${s.volume ?? 0}` }, el('span.unit', { text: '%' })),
|
||
),
|
||
el('div.stat', {},
|
||
el('div.stat-label', { text: 'Audio' }),
|
||
el('div', { style: { marginTop: '6px' } },
|
||
badge(s.muted ? 'Muted' : 'Active', s.muted ? 'warning' : 'good'),
|
||
),
|
||
),
|
||
),
|
||
);
|
||
}
|
||
|
||
paint();
|
||
|
||
let last = 0;
|
||
const unsubscribe = store.on('state', () => {
|
||
const now = Date.now();
|
||
if (now - last < 800) return;
|
||
last = now;
|
||
paint();
|
||
});
|
||
|
||
return { node: root, dispose: unsubscribe };
|
||
},
|
||
};
|
||
|
||
function rgbToHex({ r, g, b }) {
|
||
return `#${[r, g, b].map((v) => Math.max(0, Math.min(255, v | 0)).toString(16).padStart(2, '0')).join('')}`;
|
||
}
|
||
|
||
function hexToRgb(hex) {
|
||
const clean = hex.replace('#', '');
|
||
return {
|
||
r: parseInt(clean.slice(0, 2), 16) || 0,
|
||
g: parseInt(clean.slice(2, 4), 16) || 0,
|
||
b: parseInt(clean.slice(4, 6), 16) || 0,
|
||
};
|
||
}
|