/* Live Gemini — talk to a robot voice from the browser. * * The page opens its own WebSocket to Gemini rather than routing audio through * this server, because the Apache [P] rewrite in front of the app cannot * upgrade a WebSocket (verified: 101 straight to uvicorn, 404 through the * proxy). The server only mints a short-lived ephemeral token, so the API key * never reaches the browser. * * Audio contract of the Live API: * send 16 kHz signed 16-bit mono PCM, base64, as realtimeInput * receive 24 kHz signed 16-bit mono PCM, base64, in serverContent parts */ (function () { const SEND_RATE = 16000; const RECV_RATE = 24000; let ws = null; // socket to Gemini let micStream = null; // MediaStream from getUserMedia let micCtx = null; // AudioContext for capture let playCtx = null; // AudioContext for playback let processor = null; let playHead = 0; // when the next chunk should start, in playCtx time let connected = false; let sentChunks = 0, recvFrames = 0, playedChunks = 0, micPeak = 0, diagTimer = null; let cfg = null; // {voices:[{voice,label,persona}], model} let selectedVoice = 'Charon'; const $ = (id) => document.getElementById(id); function status(text, tone) { const el = $('live-status'); if (!el) return; el.textContent = text; el.style.color = tone === 'err' ? '#f87171' : tone === 'ok' ? '#4ade80' : 'var(--dim)'; } function diag() { const el = document.getElementById('live-diag'); if (!el) return; const mic = micCtx ? micCtx.state : '-'; const play = playCtx ? playCtx.state : '-'; el.textContent = `sent ${sentChunks} chunks (peak ${micPeak.toFixed(3)}) · ` + `received ${recvFrames} frames · played ${playedChunks} · ` + `mic ctx ${mic} · out ctx ${play}`; } function addLine(who, text) { const box = $('live-transcript'); if (!box || !text) return; const row = document.createElement('div'); row.style.margin = '.15rem 0'; row.innerHTML = `` + `${who === 'you' ? 'you' : 'robot'}: `; row.appendChild(document.createTextNode(text)); box.appendChild(row); box.scrollTop = box.scrollHeight; } // ── config + persona ──────────────────────────────────────────── async function loadConfig() { cfg = await api('GET', '/api/live/config'); const box = $('live-voices'); if (box) { box.innerHTML = (cfg.voices || []).map(v => ``).join(' '); box.querySelectorAll('button').forEach(b => { b.onclick = () => selectVoice(b.dataset.voice); }); } selectVoice(selectedVoice); const m = $('live-model'); if (m) m.textContent = cfg.model || ''; } function selectVoice(voice) { selectedVoice = voice; const entry = (cfg && cfg.voices || []).find(v => v.voice === voice); const box = $('live-voices'); if (box) box.querySelectorAll('button').forEach(b => { const on = b.dataset.voice === voice; b.classList.toggle('btn-primary', on); b.classList.toggle('btn-ghost', !on); }); if (personas.length) { const sel = $('persona-picker'); if (sel && activeIds[voice]) sel.value = activeIds[voice]; livePickPersona((sel && sel.value) || ''); } showActive(); if (connected) status('Voice changes apply on the next connection', 'err'); } let personas = []; // whole library let activeIds = {}; // voice -> persona id in use let editingId = ''; // persona currently in the editor async function loadPersonas(keepId) { const note = $('persona-note'); let r; try { r = await api('GET', '/api/live/personas'); } catch (e) { if (note) note.textContent = 'Could not load personas: ' + (e && e.message || e); return; } if (note) note.textContent = ''; personas = r.personas || []; activeIds = r.active || {}; const sel = $('persona-picker'); if (sel) { sel.innerHTML = personas.map(p => ``).join(''); const want = keepId || activeIds[selectedVoice] || (personas[0] && personas[0].id); if (want) sel.value = want; } try { livePickPersona((sel && sel.value) || ''); showActive(); } catch (e) { window.__personaError = e; if (note) note.textContent = 'Persona render failed: ' + ((e && e.message) || e); } } function showActive() { // Name the robot on the card so it is clear which one is being edited. const label = (cfg && cfg.voices || []).find(v => v.voice === selectedVoice); const forEl = $('persona-for'); if (forEl) forEl.textContent = label ? '— ' + label.label : ''; const el = $('persona-active'); if (!el) return; const id = activeIds[selectedVoice]; const p = personas.find(x => x.id === id); el.textContent = p ? `in use by ${label ? label.label : selectedVoice}: ${p.name}` : ''; } window.livePickPersona = (id) => { editingId = id; const p = personas.find(x => x.id === id); const ta = $('live-persona'), nm = $('persona-name'); if (p) { if (ta) ta.value = p.text || ''; if (nm) nm.value = p.builtin ? p.name.replace(/ \(default\)$/, '') + ' (copy)' : p.name; } const del = document.querySelector('[onclick="liveDeletePersona()"]'); if (del) del.disabled = !!(p && p.builtin); }; window.liveUsePersona = async () => { const sel = $('persona-picker'); if (!sel || !sel.value) return; try { const r = await api('POST', '/api/live/personas/select', { voice: selectedVoice, id: sel.value }); activeIds[selectedVoice] = r.id; showActive(); toast(`${r.name} — applies on the next connection`, 'ok'); } catch (e) { toast('Could not select: ' + (e && e.message || e), 'err'); } }; async function persistPersona(forceNew) { const ta = $('live-persona'), nm = $('persona-name'); if (!ta) return; const current = personas.find(x => x.id === editingId); // Built-ins are read-only: editing one always creates a copy. const asNew = forceNew || !editingId || (current && current.builtin); try { const r = await api('POST', '/api/live/personas', { id: asNew ? '' : editingId, name: (nm && nm.value) || 'Untitled persona', text: ta.value, }); await loadPersonas(r.id); toast(asNew ? `Saved as "${r.name}"` : `Saved "${r.name}"`, 'ok'); } catch (e) { toast('Save failed: ' + (e && e.message || e), 'err'); } } window.liveSavePersona = () => persistPersona(false); window.liveSavePersonaAs = () => persistPersona(true); window.liveDeletePersona = async () => { const p = personas.find(x => x.id === editingId); if (!p) return; if (p.builtin) { toast('Built-in personas cannot be deleted', 'err'); return; } if (!confirm(`Delete persona "${p.name}"?`)) return; try { const r = await api('POST', '/api/live/personas/delete', { id: p.id }); if ((r.reassigned || []).length) { toast(`Deleted — ${r.reassigned.join(', ')} fell back to the default`, 'ok'); } else { toast('Deleted', 'ok'); } await loadPersonas(); } catch (e) { toast('Delete failed: ' + (e && e.message || e), 'err'); } }; // ── audio helpers ─────────────────────────────────────────────── function floatToPcm16(input) { const out = new Int16Array(input.length); for (let i = 0; i < input.length; i++) { const s = Math.max(-1, Math.min(1, input[i])); out[i] = s < 0 ? s * 0x8000 : s * 0x7fff; } return out; } function downsample(buffer, inRate, outRate) { if (outRate === inRate) return buffer; const ratio = inRate / outRate; const length = Math.round(buffer.length / ratio); const out = new Float32Array(length); let offset = 0; for (let i = 0; i < length; i++) { const next = Math.round((i + 1) * ratio); let sum = 0, count = 0; for (let j = offset; j < next && j < buffer.length; j++) { sum += buffer[j]; count++; } out[i] = count ? sum / count : 0; offset = next; } return out; } function b64FromBytes(bytes) { let bin = ''; const chunk = 0x8000; for (let i = 0; i < bytes.length; i += chunk) { bin += String.fromCharCode.apply(null, bytes.subarray(i, i + chunk)); } return btoa(bin); } function playPcm(b64) { const bin = atob(b64); const bytes = new Uint8Array(bin.length); for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); const pcm = new Int16Array(bytes.buffer); if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)(); const buf = playCtx.createBuffer(1, pcm.length, RECV_RATE); const ch = buf.getChannelData(0); for (let i = 0; i < pcm.length; i++) ch[i] = pcm[i] / 32768; const src = playCtx.createBufferSource(); src.buffer = buf; src.connect(playCtx.destination); // Queue chunks back-to-back so speech does not overlap or gap. const now = playCtx.currentTime; if (playHead < now) playHead = now; src.start(playHead); playHead += buf.duration; } // ── session ───────────────────────────────────────────────────── async function connect() { if (connected) return; status('Requesting microphone…'); try { micStream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }, }); } catch (e) { status('Microphone denied — the browser must allow it', 'err'); return; } status('Getting a session token…'); let session; try { session = await api('POST', '/api/live/token', { voice: selectedVoice }); } catch (e) { status('Could not start: ' + (e && e.message || e), 'err'); stop(); return; } // The server decides how to authenticate: an ephemeral token goes in // `access_token` (v1beta only), a real API key in `key`. Passing either in // the other's parameter is refused — "unregistered callers" one way, // "API key not valid" the other. const authParam = session.auth_param || 'access_token'; const url = `wss://generativelanguage.googleapis.com/ws/` + `google.ai.generativelanguage.${session.api_version}.GenerativeService` + `.BidiGenerateContent?${authParam}=${encodeURIComponent(session.token)}`; status('Connecting to Gemini…'); ws = new WebSocket(url); ws.binaryType = 'arraybuffer'; ws.onopen = () => { ws.send(JSON.stringify({ setup: { model: session.model, generationConfig: { responseModalities: ['AUDIO'], speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: session.voice } } }, }, systemInstruction: { parts: [{ text: session.persona || '' }] }, inputAudioTranscription: {}, outputAudioTranscription: {}, // Mirrors Sanad_Package_5's VAD: LOW start-sensitivity so room noise // does not open a turn (on HIGH the model answers every rustle), and // a short silence window so replies still feel immediate. realtimeInputConfig: { automaticActivityDetection: { disabled: false, startOfSpeechSensitivity: 'START_SENSITIVITY_LOW', endOfSpeechSensitivity: 'END_SENSITIVITY_LOW', prefixPaddingMs: 300, silenceDurationMs: 400, }, }, }, })); connected = true; status('Live — speak now', 'ok'); startMic(); const b = $('live-connect'); if (b) { b.textContent = 'Stop'; b.classList.remove('btn-primary'); b.classList.add('btn-danger'); } }; ws.onmessage = async (ev) => { let text = ev.data; if (text instanceof ArrayBuffer) text = new TextDecoder().decode(text); else if (text instanceof Blob) text = await text.text(); let msg; try { msg = JSON.parse(text); } catch (_) { return; } recvFrames++; const sc = msg.serverContent || {}; (sc.modelTurn && sc.modelTurn.parts || []).forEach(p => { const inline = p.inlineData || p.inline_data; if (inline && inline.data) { playedChunks++; playPcm(inline.data); } }); if (sc.outputTranscription && sc.outputTranscription.text) { addLine('robot', sc.outputTranscription.text); } if (sc.inputTranscription && sc.inputTranscription.text) { addLine('you', sc.inputTranscription.text); } if (msg.error) status('Gemini error: ' + JSON.stringify(msg.error), 'err'); }; ws.onerror = () => status('Connection error', 'err'); ws.onclose = (e) => { if (connected) status('Session ended' + (e && e.reason ? ' — ' + e.reason : ''), 'err'); stop(); }; } function startMic() { micCtx = new (window.AudioContext || window.webkitAudioContext)(); // A context created outside a user gesture starts suspended and its // processor never fires — silence with no error anywhere. if (micCtx.state === 'suspended') micCtx.resume(); if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)(); if (playCtx.state === 'suspended') playCtx.resume(); sentChunks = recvFrames = playedChunks = 0; micPeak = 0; if (diagTimer) clearInterval(diagTimer); diagTimer = setInterval(diag, 1000); const source = micCtx.createMediaStreamSource(micStream); processor = micCtx.createScriptProcessor(4096, 1, 1); source.connect(processor); processor.connect(micCtx.destination); processor.onaudioprocess = (ev) => { if (!connected || !ws || ws.readyState !== WebSocket.OPEN) return; const raw = ev.inputBuffer.getChannelData(0); let peak = 0; for (let i = 0; i < raw.length; i++) { const a = Math.abs(raw[i]); if (a > peak) peak = a; } if (peak > micPeak) micPeak = peak; const down = downsample(raw, micCtx.sampleRate, SEND_RATE); const pcm = floatToPcm16(down); sentChunks++; ws.send(JSON.stringify({ realtimeInput: { mediaChunks: [{ mimeType: 'audio/pcm;rate=' + SEND_RATE, data: b64FromBytes(new Uint8Array(pcm.buffer)), }], }, })); }; } function stop() { connected = false; if (diagTimer) { clearInterval(diagTimer); diagTimer = null; } diag(); try { if (processor) processor.disconnect(); } catch (_) {} try { if (micCtx) micCtx.close(); } catch (_) {} try { if (micStream) micStream.getTracks().forEach(t => t.stop()); } catch (_) {} try { if (ws && ws.readyState <= 1) ws.close(); } catch (_) {} processor = micCtx = micStream = ws = null; playHead = 0; const b = $('live-connect'); if (b) { b.textContent = 'Connect'; b.classList.add('btn-primary'); b.classList.remove('btn-danger'); } } // ── microphone / speaker ──────────────────────────────────────── function permNote(text, tone) { const el = $('live-perm'); if (!el) return; el.textContent = text; el.style.color = tone === 'err' ? '#f87171' : tone === 'ok' ? '#4ade80' : 'var(--dim)'; } async function permState() { // Not supported everywhere (Safari); absence is not an error. try { if (!navigator.permissions || !navigator.permissions.query) return 'unknown'; const s = await navigator.permissions.query({ name: 'microphone' }); return s.state; // granted | denied | prompt } catch (e) { return 'unknown'; } } window.liveRefreshPerm = async () => { const state = await permState(); if (state === 'granted') permNote('microphone: allowed', 'ok'); else if (state === 'denied') permNote('microphone: blocked - use the padlock in the address bar to allow it', 'err'); else if (state === 'prompt') permNote('microphone: not asked yet', ''); else permNote('', ''); }; window.liveRequestMic = async () => { permNote('asking for the microphone...', ''); try { const s = await navigator.mediaDevices.getUserMedia({ audio: true }); // Release it immediately: this is only to obtain permission. s.getTracks().forEach(t => t.stop()); let name = ''; try { const devs = await navigator.mediaDevices.enumerateDevices(); const mic = devs.find(d => d.kind === 'audioinput' && d.label); name = mic ? ' (' + mic.label + ')' : ''; } catch (e) { /* labels need permission; ignore */ } permNote('microphone: allowed' + name, 'ok'); } catch (e) { // Once refused, the browser will not prompt again for this site. const denied = e && (e.name === 'NotAllowedError' || e.name === 'SecurityError'); permNote(denied ? 'microphone blocked - click the padlock next to the address, set Microphone to Allow, then reload' : 'microphone unavailable: ' + ((e && e.message) || e), 'err'); } }; window.liveTestSpeaker = async () => { try { if (!playCtx) playCtx = new (window.AudioContext || window.webkitAudioContext)(); if (playCtx.state === 'suspended') await playCtx.resume(); const osc = playCtx.createOscillator(); const gain = playCtx.createGain(); osc.frequency.value = 440; gain.gain.value = 0.15; // audible but not startling osc.connect(gain).connect(playCtx.destination); osc.start(); osc.stop(playCtx.currentTime + 0.35); permNote('speaker: played a test tone (output ' + playCtx.state + ')', 'ok'); } catch (e) { permNote('speaker failed: ' + ((e && e.message) || e), 'err'); } }; window.liveToggle = () => (connected ? (stop(), status('Disconnected')) : connect()); window.liveInit = () => { // Independent: a failure in one must not leave the other blank. loadConfig().catch((e) => status('Could not load config: ' + (e && e.message || e), 'err')); if (window.liveRefreshPerm) liveRefreshPerm(); loadPersonas().catch((e) => { window.__personaError = e; const note = document.getElementById('persona-note'); const msg = (e && (e.stack || e.message)) || String(e); if (note) note.textContent = 'Persona list failed: ' + msg; }); }; // Self-initialise. This file loads at the end of , AFTER the inline // script has already run its startup calls — so waiting to be called by it // left the robot list empty and the model showing "--". if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', window.liveInit); } else { window.liveInit(); } })();