d9b2d5427f
Replay now matches the robots and no longer cuts words: - read the turn to turnComplete, not generationComplete, and drain the socket before each send; breaking early truncated every sentence and left frames that the next turn mis-read as its own reply - accept a take only if the model's own transcript covers the text AND the audio is long enough to contain it (the transcript reports the full text even for a 0.8s clip) - pitch gate: reject an off-tone take and re-ask, per voice, using a pure-Python F0 estimator (no numpy on the host) - continuation: speak the words a voice skipped instead of retrying a line it stops on deterministically - fresh Live session per replay; delivery drifts as turns accumulate Live Gemini tab: browser talks to Gemini directly (the reverse proxy cannot upgrade a WebSocket), with a persona library - named personas, per-robot selection, built-ins that cannot be overwritten. Dashboard: records search + voice filter, log panel falls back to polling, sign-in history with CSV/JSON export, and JS errors now show on the page instead of silently blanking a tab.
481 lines
19 KiB
JavaScript
481 lines
19 KiB
JavaScript
/* Live Gemini — talk to a robot voice from the browser.
|
|
*
|
|
* The page opens its own WebSocket to Gemini rather than routing audio through
|
|
* this server, because the Apache [P] rewrite in front of the app cannot
|
|
* upgrade a WebSocket (verified: 101 straight to uvicorn, 404 through the
|
|
* proxy). The server only mints a short-lived ephemeral token, so the API key
|
|
* never reaches the browser.
|
|
*
|
|
* Audio contract of the Live API:
|
|
* send 16 kHz signed 16-bit mono PCM, base64, as realtimeInput
|
|
* receive 24 kHz signed 16-bit mono PCM, base64, in serverContent parts
|
|
*/
|
|
(function () {
|
|
const SEND_RATE = 16000;
|
|
const RECV_RATE = 24000;
|
|
|
|
let ws = null; // socket to Gemini
|
|
let micStream = null; // MediaStream from getUserMedia
|
|
let micCtx = null; // AudioContext for capture
|
|
let playCtx = null; // AudioContext for playback
|
|
let processor = null;
|
|
let playHead = 0; // when the next chunk should start, in playCtx time
|
|
let connected = false;
|
|
let sentChunks = 0, recvFrames = 0, playedChunks = 0, micPeak = 0, diagTimer = null;
|
|
let cfg = null; // {voices:[{voice,label,persona}], model}
|
|
let selectedVoice = 'Charon';
|
|
|
|
const $ = (id) => document.getElementById(id);
|
|
|
|
function status(text, tone) {
|
|
const el = $('live-status');
|
|
if (!el) return;
|
|
el.textContent = text;
|
|
el.style.color = tone === 'err' ? '#f87171'
|
|
: tone === 'ok' ? '#4ade80' : 'var(--dim)';
|
|
}
|
|
|
|
function diag() {
|
|
const el = document.getElementById('live-diag');
|
|
if (!el) return;
|
|
const mic = micCtx ? micCtx.state : '-';
|
|
const play = playCtx ? playCtx.state : '-';
|
|
el.textContent = `sent ${sentChunks} chunks (peak ${micPeak.toFixed(3)}) · `
|
|
+ `received ${recvFrames} frames · played ${playedChunks} · `
|
|
+ `mic ctx ${mic} · out ctx ${play}`;
|
|
}
|
|
|
|
function addLine(who, text) {
|
|
const box = $('live-transcript');
|
|
if (!box || !text) return;
|
|
const row = document.createElement('div');
|
|
row.style.margin = '.15rem 0';
|
|
row.innerHTML = `<span style="color:${who === 'you' ? '#7dd3fc' : '#a78bfa'}">`
|
|
+ `${who === 'you' ? 'you' : 'robot'}:</span> `;
|
|
row.appendChild(document.createTextNode(text));
|
|
box.appendChild(row);
|
|
box.scrollTop = box.scrollHeight;
|
|
}
|
|
|
|
// ── config + persona ────────────────────────────────────────────
|
|
async function loadConfig() {
|
|
cfg = await api('GET', '/api/live/config');
|
|
const box = $('live-voices');
|
|
if (box) {
|
|
box.innerHTML = (cfg.voices || []).map(v =>
|
|
`<button class="btn ${v.voice === selectedVoice ? 'btn-primary' : 'btn-ghost'} btn-sm"
|
|
data-voice="${esc(v.voice)}">${esc(v.label)}</button>`).join(' ');
|
|
box.querySelectorAll('button').forEach(b => {
|
|
b.onclick = () => selectVoice(b.dataset.voice);
|
|
});
|
|
}
|
|
selectVoice(selectedVoice);
|
|
const m = $('live-model');
|
|
if (m) m.textContent = cfg.model || '';
|
|
}
|
|
|
|
function selectVoice(voice) {
|
|
selectedVoice = voice;
|
|
const entry = (cfg && cfg.voices || []).find(v => v.voice === voice);
|
|
const box = $('live-voices');
|
|
if (box) box.querySelectorAll('button').forEach(b => {
|
|
const on = b.dataset.voice === voice;
|
|
b.classList.toggle('btn-primary', on);
|
|
b.classList.toggle('btn-ghost', !on);
|
|
});
|
|
if (personas.length) {
|
|
const sel = $('persona-picker');
|
|
if (sel && activeIds[voice]) sel.value = activeIds[voice];
|
|
livePickPersona((sel && sel.value) || '');
|
|
}
|
|
showActive();
|
|
if (connected) status('Voice changes apply on the next connection', 'err');
|
|
}
|
|
|
|
let personas = []; // whole library
|
|
let activeIds = {}; // voice -> persona id in use
|
|
let editingId = ''; // persona currently in the editor
|
|
|
|
async function loadPersonas(keepId) {
|
|
const note = $('persona-note');
|
|
let r;
|
|
try {
|
|
r = await api('GET', '/api/live/personas');
|
|
} catch (e) {
|
|
if (note) note.textContent = 'Could not load personas: ' + (e && e.message || e);
|
|
return;
|
|
}
|
|
if (note) note.textContent = '';
|
|
personas = r.personas || [];
|
|
activeIds = r.active || {};
|
|
const sel = $('persona-picker');
|
|
if (sel) {
|
|
sel.innerHTML = personas.map(p =>
|
|
`<option value="${esc(p.id)}">${esc(p.name)}${p.builtin ? ' *' : ''}</option>`).join('');
|
|
const want = keepId || activeIds[selectedVoice] || (personas[0] && personas[0].id);
|
|
if (want) sel.value = want;
|
|
}
|
|
try {
|
|
livePickPersona((sel && sel.value) || '');
|
|
showActive();
|
|
} catch (e) {
|
|
window.__personaError = e;
|
|
if (note) note.textContent = 'Persona render failed: ' + ((e && e.message) || e);
|
|
}
|
|
}
|
|
|
|
function showActive() {
|
|
// Name the robot on the card so it is clear which one is being edited.
|
|
const label = (cfg && cfg.voices || []).find(v => v.voice === selectedVoice);
|
|
const forEl = $('persona-for');
|
|
if (forEl) forEl.textContent = label ? '— ' + label.label : '';
|
|
const el = $('persona-active');
|
|
if (!el) return;
|
|
const id = activeIds[selectedVoice];
|
|
const p = personas.find(x => x.id === id);
|
|
el.textContent = p ? `in use by ${label ? label.label : selectedVoice}: ${p.name}` : '';
|
|
}
|
|
|
|
window.livePickPersona = (id) => {
|
|
editingId = id;
|
|
const p = personas.find(x => x.id === id);
|
|
const ta = $('live-persona'), nm = $('persona-name');
|
|
if (p) {
|
|
if (ta) ta.value = p.text || '';
|
|
if (nm) nm.value = p.builtin ? p.name.replace(/ \(default\)$/, '') + ' (copy)' : p.name;
|
|
}
|
|
const del = document.querySelector('[onclick="liveDeletePersona()"]');
|
|
if (del) del.disabled = !!(p && p.builtin);
|
|
};
|
|
|
|
window.liveUsePersona = async () => {
|
|
const sel = $('persona-picker');
|
|
if (!sel || !sel.value) return;
|
|
try {
|
|
const r = await api('POST', '/api/live/personas/select',
|
|
{ voice: selectedVoice, id: sel.value });
|
|
activeIds[selectedVoice] = r.id;
|
|
showActive();
|
|
toast(`${r.name} — applies on the next connection`, 'ok');
|
|
} catch (e) { toast('Could not select: ' + (e && e.message || e), 'err'); }
|
|
};
|
|
|
|
async function persistPersona(forceNew) {
|
|
const ta = $('live-persona'), nm = $('persona-name');
|
|
if (!ta) return;
|
|
const current = personas.find(x => x.id === editingId);
|
|
// Built-ins are read-only: editing one always creates a copy.
|
|
const asNew = forceNew || !editingId || (current && current.builtin);
|
|
try {
|
|
const r = await api('POST', '/api/live/personas', {
|
|
id: asNew ? '' : editingId,
|
|
name: (nm && nm.value) || 'Untitled persona',
|
|
text: ta.value,
|
|
});
|
|
await loadPersonas(r.id);
|
|
toast(asNew ? `Saved as "${r.name}"` : `Saved "${r.name}"`, 'ok');
|
|
} catch (e) { toast('Save failed: ' + (e && e.message || e), 'err'); }
|
|
}
|
|
|
|
window.liveSavePersona = () => persistPersona(false);
|
|
window.liveSavePersonaAs = () => persistPersona(true);
|
|
|
|
window.liveDeletePersona = async () => {
|
|
const p = personas.find(x => x.id === editingId);
|
|
if (!p) return;
|
|
if (p.builtin) { toast('Built-in personas cannot be deleted', 'err'); return; }
|
|
if (!confirm(`Delete persona "${p.name}"?`)) return;
|
|
try {
|
|
const r = await api('POST', '/api/live/personas/delete', { id: p.id });
|
|
if ((r.reassigned || []).length) {
|
|
toast(`Deleted — ${r.reassigned.join(', ')} fell back to the default`, 'ok');
|
|
} else { toast('Deleted', 'ok'); }
|
|
await loadPersonas();
|
|
} catch (e) { toast('Delete failed: ' + (e && e.message || e), 'err'); }
|
|
};
|
|
|
|
// ── audio helpers ───────────────────────────────────────────────
|
|
function floatToPcm16(input) {
|
|
const out = new Int16Array(input.length);
|
|
for (let i = 0; i < input.length; i++) {
|
|
const s = Math.max(-1, Math.min(1, input[i]));
|
|
out[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function downsample(buffer, inRate, outRate) {
|
|
if (outRate === inRate) return buffer;
|
|
const ratio = inRate / outRate;
|
|
const length = Math.round(buffer.length / ratio);
|
|
const out = new Float32Array(length);
|
|
let offset = 0;
|
|
for (let i = 0; i < length; i++) {
|
|
const next = Math.round((i + 1) * ratio);
|
|
let sum = 0, count = 0;
|
|
for (let j = offset; j < next && j < buffer.length; j++) { sum += buffer[j]; count++; }
|
|
out[i] = count ? sum / count : 0;
|
|
offset = next;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function b64FromBytes(bytes) {
|
|
let bin = '';
|
|
const chunk = 0x8000;
|
|
for (let i = 0; i < bytes.length; i += chunk) {
|
|
bin += String.fromCharCode.apply(null, bytes.subarray(i, i + chunk));
|
|
}
|
|
return btoa(bin);
|
|
}
|
|
|
|
function playPcm(b64) {
|
|
const bin = atob(b64);
|
|
const bytes = new Uint8Array(bin.length);
|
|
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
|
const pcm = new Int16Array(bytes.buffer);
|
|
if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)();
|
|
const buf = playCtx.createBuffer(1, pcm.length, RECV_RATE);
|
|
const ch = buf.getChannelData(0);
|
|
for (let i = 0; i < pcm.length; i++) ch[i] = pcm[i] / 32768;
|
|
const src = playCtx.createBufferSource();
|
|
src.buffer = buf;
|
|
src.connect(playCtx.destination);
|
|
// Queue chunks back-to-back so speech does not overlap or gap.
|
|
const now = playCtx.currentTime;
|
|
if (playHead < now) playHead = now;
|
|
src.start(playHead);
|
|
playHead += buf.duration;
|
|
}
|
|
|
|
// ── session ─────────────────────────────────────────────────────
|
|
async function connect() {
|
|
if (connected) return;
|
|
status('Requesting microphone…');
|
|
try {
|
|
micStream = await navigator.mediaDevices.getUserMedia({
|
|
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
|
|
});
|
|
} catch (e) {
|
|
status('Microphone denied — the browser must allow it', 'err');
|
|
return;
|
|
}
|
|
|
|
status('Getting a session token…');
|
|
let session;
|
|
try {
|
|
session = await api('POST', '/api/live/token', { voice: selectedVoice });
|
|
} catch (e) {
|
|
status('Could not start: ' + (e && e.message || e), 'err');
|
|
stop();
|
|
return;
|
|
}
|
|
|
|
// The server decides how to authenticate: an ephemeral token goes in
|
|
// `access_token` (v1beta only), a real API key in `key`. Passing either in
|
|
// the other's parameter is refused — "unregistered callers" one way,
|
|
// "API key not valid" the other.
|
|
const authParam = session.auth_param || 'access_token';
|
|
const url = `wss://generativelanguage.googleapis.com/ws/`
|
|
+ `google.ai.generativelanguage.${session.api_version}.GenerativeService`
|
|
+ `.BidiGenerateContent?${authParam}=${encodeURIComponent(session.token)}`;
|
|
status('Connecting to Gemini…');
|
|
ws = new WebSocket(url);
|
|
ws.binaryType = 'arraybuffer';
|
|
|
|
ws.onopen = () => {
|
|
ws.send(JSON.stringify({
|
|
setup: {
|
|
model: session.model,
|
|
generationConfig: {
|
|
responseModalities: ['AUDIO'],
|
|
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: session.voice } } },
|
|
},
|
|
systemInstruction: { parts: [{ text: session.persona || '' }] },
|
|
inputAudioTranscription: {},
|
|
outputAudioTranscription: {},
|
|
// Mirrors Sanad_Package_5's VAD: LOW start-sensitivity so room noise
|
|
// does not open a turn (on HIGH the model answers every rustle), and
|
|
// a short silence window so replies still feel immediate.
|
|
realtimeInputConfig: {
|
|
automaticActivityDetection: {
|
|
disabled: false,
|
|
startOfSpeechSensitivity: 'START_SENSITIVITY_LOW',
|
|
endOfSpeechSensitivity: 'END_SENSITIVITY_LOW',
|
|
prefixPaddingMs: 300,
|
|
silenceDurationMs: 400,
|
|
},
|
|
},
|
|
},
|
|
}));
|
|
connected = true;
|
|
status('Live — speak now', 'ok');
|
|
startMic();
|
|
const b = $('live-connect');
|
|
if (b) { b.textContent = 'Stop'; b.classList.remove('btn-primary'); b.classList.add('btn-danger'); }
|
|
};
|
|
|
|
ws.onmessage = async (ev) => {
|
|
let text = ev.data;
|
|
if (text instanceof ArrayBuffer) text = new TextDecoder().decode(text);
|
|
else if (text instanceof Blob) text = await text.text();
|
|
let msg;
|
|
try { msg = JSON.parse(text); } catch (_) { return; }
|
|
recvFrames++;
|
|
const sc = msg.serverContent || {};
|
|
(sc.modelTurn && sc.modelTurn.parts || []).forEach(p => {
|
|
const inline = p.inlineData || p.inline_data;
|
|
if (inline && inline.data) { playedChunks++; playPcm(inline.data); }
|
|
});
|
|
if (sc.outputTranscription && sc.outputTranscription.text) {
|
|
addLine('robot', sc.outputTranscription.text);
|
|
}
|
|
if (sc.inputTranscription && sc.inputTranscription.text) {
|
|
addLine('you', sc.inputTranscription.text);
|
|
}
|
|
if (msg.error) status('Gemini error: ' + JSON.stringify(msg.error), 'err');
|
|
};
|
|
|
|
ws.onerror = () => status('Connection error', 'err');
|
|
ws.onclose = (e) => {
|
|
if (connected) status('Session ended' + (e && e.reason ? ' — ' + e.reason : ''), 'err');
|
|
stop();
|
|
};
|
|
}
|
|
|
|
function startMic() {
|
|
micCtx = new (window.AudioContext || window.webkitAudioContext)();
|
|
// A context created outside a user gesture starts suspended and its
|
|
// processor never fires — silence with no error anywhere.
|
|
if (micCtx.state === 'suspended') micCtx.resume();
|
|
if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)();
|
|
if (playCtx.state === 'suspended') playCtx.resume();
|
|
sentChunks = recvFrames = playedChunks = 0; micPeak = 0;
|
|
if (diagTimer) clearInterval(diagTimer);
|
|
diagTimer = setInterval(diag, 1000);
|
|
const source = micCtx.createMediaStreamSource(micStream);
|
|
processor = micCtx.createScriptProcessor(4096, 1, 1);
|
|
source.connect(processor);
|
|
processor.connect(micCtx.destination);
|
|
processor.onaudioprocess = (ev) => {
|
|
if (!connected || !ws || ws.readyState !== WebSocket.OPEN) return;
|
|
const raw = ev.inputBuffer.getChannelData(0);
|
|
let peak = 0;
|
|
for (let i = 0; i < raw.length; i++) { const a = Math.abs(raw[i]); if (a > peak) peak = a; }
|
|
if (peak > micPeak) micPeak = peak;
|
|
const down = downsample(raw, micCtx.sampleRate, SEND_RATE);
|
|
const pcm = floatToPcm16(down);
|
|
sentChunks++;
|
|
ws.send(JSON.stringify({
|
|
realtimeInput: {
|
|
mediaChunks: [{
|
|
mimeType: 'audio/pcm;rate=' + SEND_RATE,
|
|
data: b64FromBytes(new Uint8Array(pcm.buffer)),
|
|
}],
|
|
},
|
|
}));
|
|
};
|
|
}
|
|
|
|
function stop() {
|
|
connected = false;
|
|
if (diagTimer) { clearInterval(diagTimer); diagTimer = null; }
|
|
diag();
|
|
try { if (processor) processor.disconnect(); } catch (_) {}
|
|
try { if (micCtx) micCtx.close(); } catch (_) {}
|
|
try { if (micStream) micStream.getTracks().forEach(t => t.stop()); } catch (_) {}
|
|
try { if (ws && ws.readyState <= 1) ws.close(); } catch (_) {}
|
|
processor = micCtx = micStream = ws = null;
|
|
playHead = 0;
|
|
const b = $('live-connect');
|
|
if (b) { b.textContent = 'Connect'; b.classList.add('btn-primary'); b.classList.remove('btn-danger'); }
|
|
}
|
|
|
|
// ── microphone / speaker ────────────────────────────────────────
|
|
function permNote(text, tone) {
|
|
const el = $('live-perm');
|
|
if (!el) return;
|
|
el.textContent = text;
|
|
el.style.color = tone === 'err' ? '#f87171'
|
|
: tone === 'ok' ? '#4ade80' : 'var(--dim)';
|
|
}
|
|
|
|
async function permState() {
|
|
// Not supported everywhere (Safari); absence is not an error.
|
|
try {
|
|
if (!navigator.permissions || !navigator.permissions.query) return 'unknown';
|
|
const s = await navigator.permissions.query({ name: 'microphone' });
|
|
return s.state; // granted | denied | prompt
|
|
} catch (e) { return 'unknown'; }
|
|
}
|
|
|
|
window.liveRefreshPerm = async () => {
|
|
const state = await permState();
|
|
if (state === 'granted') permNote('microphone: allowed', 'ok');
|
|
else if (state === 'denied') permNote('microphone: blocked - use the padlock in the address bar to allow it', 'err');
|
|
else if (state === 'prompt') permNote('microphone: not asked yet', '');
|
|
else permNote('', '');
|
|
};
|
|
|
|
window.liveRequestMic = async () => {
|
|
permNote('asking for the microphone...', '');
|
|
try {
|
|
const s = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
// Release it immediately: this is only to obtain permission.
|
|
s.getTracks().forEach(t => t.stop());
|
|
let name = '';
|
|
try {
|
|
const devs = await navigator.mediaDevices.enumerateDevices();
|
|
const mic = devs.find(d => d.kind === 'audioinput' && d.label);
|
|
name = mic ? ' (' + mic.label + ')' : '';
|
|
} catch (e) { /* labels need permission; ignore */ }
|
|
permNote('microphone: allowed' + name, 'ok');
|
|
} catch (e) {
|
|
// Once refused, the browser will not prompt again for this site.
|
|
const denied = e && (e.name === 'NotAllowedError' || e.name === 'SecurityError');
|
|
permNote(denied
|
|
? 'microphone blocked - click the padlock next to the address, set Microphone to Allow, then reload'
|
|
: 'microphone unavailable: ' + ((e && e.message) || e), 'err');
|
|
}
|
|
};
|
|
|
|
window.liveTestSpeaker = async () => {
|
|
try {
|
|
if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)();
|
|
if (playCtx.state === 'suspended') await playCtx.resume();
|
|
const osc = playCtx.createOscillator();
|
|
const gain = playCtx.createGain();
|
|
osc.frequency.value = 440;
|
|
gain.gain.value = 0.15; // audible but not startling
|
|
osc.connect(gain).connect(playCtx.destination);
|
|
osc.start();
|
|
osc.stop(playCtx.currentTime + 0.35);
|
|
permNote('speaker: played a test tone (output ' + playCtx.state + ')', 'ok');
|
|
} catch (e) {
|
|
permNote('speaker failed: ' + ((e && e.message) || e), 'err');
|
|
}
|
|
};
|
|
|
|
window.liveToggle = () => (connected ? (stop(), status('Disconnected')) : connect());
|
|
window.liveInit = () => {
|
|
// Independent: a failure in one must not leave the other blank.
|
|
loadConfig().catch((e) => status('Could not load config: ' + (e && e.message || e), 'err'));
|
|
if (window.liveRefreshPerm) liveRefreshPerm();
|
|
loadPersonas().catch((e) => {
|
|
window.__personaError = e;
|
|
const note = document.getElementById('persona-note');
|
|
const msg = (e && (e.stack || e.message)) || String(e);
|
|
if (note) note.textContent = 'Persona list failed: ' + msg;
|
|
});
|
|
};
|
|
|
|
// Self-initialise. This file loads at the end of <body>, AFTER the inline
|
|
// script has already run its startup calls — so waiting to be called by it
|
|
// left the robot list empty and the model showing "--".
|
|
if (document.readyState === 'loading') {
|
|
document.addEventListener('DOMContentLoaded', window.liveInit);
|
|
} else {
|
|
window.liveInit();
|
|
}
|
|
})();
|