434 lines
17 KiB
Python
434 lines
17 KiB
Python
"""Gemini cloud voice for the simulator.
|
|
|
|
Gives the mock robot a natural neural voice instead of the flat built-in Windows
|
|
one, so a rehearsal sounds like the real thing.
|
|
|
|
**Simulator only.** The real AGIBOT A3 synthesises its own speech on-board;
|
|
nothing here is ever used against the robot.
|
|
|
|
API (verified against a live call on 2026-09-02, not copied from a doc summary):
|
|
|
|
POST https://generativelanguage.googleapis.com/v1beta/interactions
|
|
x-goog-api-key: <key>
|
|
{"model": "...-tts-preview", "input": "...",
|
|
"response_format": {"type": "audio"},
|
|
"generation_config": {"speech_config": [{"voice": "Puck"}]}}
|
|
|
|
-> steps[0].content[0].data base64 PCM
|
|
steps[0].content[0].mime_type audio/l16; rate=24000; channels=1
|
|
|
|
The audio lives under `steps[]`, not the `output_audio` field some docs describe
|
|
- that path was checked against a real response.
|
|
|
|
THE LATENCY PROBLEM, AND THE TWO THINGS DONE ABOUT IT
|
|
-----------------------------------------------------
|
|
Measured on this account: ~4 s to synthesise a short sentence, ~8-10 s for a long
|
|
paragraph. Unusable for a live demo if taken naively. So:
|
|
|
|
1. **Saved audio.** Every line is written to `audio_library/` as an ordinary
|
|
.wav named after its text, so a repeated line replays instantly, survives
|
|
restarts, works with no internet, and can be played outside this app.
|
|
Rehearsed lines can be built ahead of time - see scripts/warm_voice.py.
|
|
2. **One clip per utterance.** Text is NOT split by default: the whole line is
|
|
synthesised in a single take, so the delivery is continuous and the library
|
|
holds one file per thing you said. Optional sentence pipelining is available
|
|
for very long text via GEMINI_CHUNK_CHARS, at the cost of an audible seam.
|
|
|
|
And it must never break the demo: any failure - no network, bad key, quota -
|
|
falls back to the built-in system voice rather than producing silence.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import logging
|
|
import os
|
|
import platform
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
from ..services.audio_library import AudioLibrary
|
|
from .local_audio import LocalVoice
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ENDPOINT = "https://generativelanguage.googleapis.com/v1beta/interactions"
|
|
|
|
#: Prebuilt voices with the tone Google documents for each.
|
|
VOICES: Dict[str, str] = {
|
|
"Zephyr": "Bright", "Puck": "Upbeat", "Charon": "Informative", "Kore": "Firm",
|
|
"Fenrir": "Excitable", "Leda": "Youthful", "Orus": "Firm", "Aoede": "Breezy",
|
|
"Callirrhoe": "Easy-going", "Autonoe": "Bright", "Enceladus": "Breathy",
|
|
"Iapetus": "Clear", "Umbriel": "Easy-going", "Algieba": "Smooth",
|
|
"Despina": "Smooth", "Erinome": "Clear", "Algenib": "Gravelly",
|
|
"Rasalgethi": "Informative", "Laomedeia": "Upbeat", "Achernar": "Soft",
|
|
"Alnilam": "Firm", "Schedar": "Even", "Gacrux": "Mature",
|
|
"Pulcherrima": "Forward", "Achird": "Friendly", "Zubenelgenubi": "Casual",
|
|
"Vindemiatrix": "Gentle", "Sadachbia": "Lively", "Sadaltager": "Knowledgeable",
|
|
"Sulafat": "Warm",
|
|
}
|
|
|
|
#: 0 = never split. Every utterance is synthesised as ONE clip, in one take.
|
|
#:
|
|
#: Splitting was an optimisation: a request costs ~3.5 s fixed plus ~45 ms per
|
|
#: character, so starting playback after the first sentence reaches audio sooner
|
|
#: on long text. But each piece is a separate synthesis - separate delivery,
|
|
#: separate file - and that is audible. One continuous take is worth more than
|
|
#: a second or two of head start.
|
|
#:
|
|
#: Set GEMINI_CHUNK_CHARS in .env to a character count to re-enable pipelining
|
|
#: for very long text.
|
|
#:
|
|
#: NOTE: unrelated to the robot's own 1024-BYTE limit in aimdk_transport.py.
|
|
#: That one is a hard API constraint; this is a latency choice.
|
|
_CHUNK_TARGET_CHARS = 0
|
|
|
|
_SENTENCE_END = re.compile(r"(?<=[.!?。！？;:])\s+|\n+")
|
|
|
|
|
|
def wav_from_pcm(pcm: bytes, sample_rate: int = 24000, channels: int = 1,
|
|
bits: int = 16) -> bytes:
|
|
"""Wrap raw PCM in a 44-byte WAV header so ordinary players accept it."""
|
|
byte_rate = sample_rate * channels * bits // 8
|
|
block_align = channels * bits // 8
|
|
return b"".join([
|
|
b"RIFF", struct.pack("<I", 36 + len(pcm)), b"WAVE",
|
|
b"fmt ", struct.pack("<IHHIIHH", 16, 1, channels, sample_rate,
|
|
byte_rate, block_align, bits),
|
|
b"data", struct.pack("<I", len(pcm)), pcm,
|
|
])
|
|
|
|
|
|
def split_sentences(text: str, target: int = _CHUNK_TARGET_CHARS) -> List[str]:
|
|
"""Split into speakable chunks on sentence boundaries.
|
|
|
|
`target <= 0` disables splitting entirely: the whole utterance is synthesised
|
|
as ONE clip, in one continuous take. That is the default, because a split
|
|
line is synthesised as separate requests - each piece gets its own delivery
|
|
and its own file, and the join between them can be audible.
|
|
"""
|
|
text = " ".join((text or "").split())
|
|
if not text:
|
|
return []
|
|
if target <= 0 or len(text) <= target:
|
|
return [text]
|
|
|
|
pieces = [p.strip() for p in _SENTENCE_END.split(text) if p and p.strip()]
|
|
chunks: List[str] = []
|
|
current = ""
|
|
for piece in pieces:
|
|
if not current:
|
|
current = piece
|
|
elif len(current) + 1 + len(piece) <= target:
|
|
current += " " + piece
|
|
else:
|
|
chunks.append(current)
|
|
current = piece
|
|
if current:
|
|
chunks.append(current)
|
|
return chunks
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# playback
|
|
# --------------------------------------------------------------------------- #
|
|
class _Player:
|
|
"""Plays one WAV without blocking; can be stopped mid-playback."""
|
|
|
|
def __init__(self) -> None:
|
|
self._windows = platform.system().lower().startswith("win")
|
|
self._process: Optional[subprocess.Popen] = None
|
|
self._ends_at = 0.0
|
|
self._lock = threading.Lock()
|
|
|
|
def play(self, path: str, duration: float) -> None:
|
|
if self._windows:
|
|
import winsound
|
|
|
|
winsound.PlaySound(path, winsound.SND_FILENAME | winsound.SND_ASYNC)
|
|
# winsound has no completion callback, but the exact duration is
|
|
# known from the PCM length, so the deadline is precise.
|
|
with self._lock:
|
|
self._ends_at = time.monotonic() + duration
|
|
return
|
|
|
|
binary = "afplay" if platform.system().lower() == "darwin" else "aplay"
|
|
try:
|
|
with self._lock:
|
|
self._process = subprocess.Popen(
|
|
[binary, path], stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL)
|
|
except FileNotFoundError:
|
|
logger.warning("no audio player found (%s); install it to hear the simulator", binary)
|
|
|
|
def stop(self) -> None:
|
|
if self._windows:
|
|
try:
|
|
import winsound
|
|
|
|
winsound.PlaySound(None, winsound.SND_PURGE)
|
|
except Exception: # pragma: no cover
|
|
pass
|
|
with self._lock:
|
|
self._ends_at = 0.0
|
|
return
|
|
with self._lock:
|
|
process, self._process = self._process, None
|
|
if process is not None and process.poll() is None:
|
|
try:
|
|
process.kill()
|
|
except Exception: # pragma: no cover
|
|
pass
|
|
|
|
def is_playing(self) -> bool:
|
|
if self._windows:
|
|
with self._lock:
|
|
return time.monotonic() < self._ends_at
|
|
with self._lock:
|
|
process = self._process
|
|
return process is not None and process.poll() is None
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# the voice
|
|
# --------------------------------------------------------------------------- #
|
|
class GeminiVoice(LocalVoice):
|
|
"""Neural TTS through the Gemini API, saved to disk, with a local fallback."""
|
|
|
|
name = "gemini"
|
|
|
|
def __init__(self, api_key: str, model: str, voice: str, style: str = "",
|
|
timeout: float = 60.0, cache_dir: Optional[str] = None,
|
|
library: Optional[AudioLibrary] = None,
|
|
chunk_chars: int = _CHUNK_TARGET_CHARS,
|
|
fallback: Optional[LocalVoice] = None) -> None:
|
|
if not api_key:
|
|
raise ValueError("GEMINI_API_KEY is empty")
|
|
self._key = api_key
|
|
self._model = model
|
|
self._voice = voice
|
|
self._style = style
|
|
self._timeout = timeout
|
|
# 0 (the default) means never split - one utterance, one clip.
|
|
self._chunk_chars = max(0, int(chunk_chars))
|
|
self._fallback = fallback
|
|
self._using_fallback = False
|
|
self._warned = False
|
|
|
|
# Audio is saved as ordinary .wav files, so a line spoken once can be
|
|
# replayed instantly, played outside this app, or used offline.
|
|
if library is not None:
|
|
self.library = library
|
|
else:
|
|
root = cache_dir or os.path.join(
|
|
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
|
"audio_library",
|
|
)
|
|
self.library = AudioLibrary(Path(root))
|
|
|
|
self._player = _Player()
|
|
self._sequence_stop = threading.Event()
|
|
self._sequence: Optional[threading.Thread] = None
|
|
self._active = threading.Event()
|
|
|
|
# -- LocalVoice ---------------------------------------------------------- #
|
|
def prepare(self, text: str) -> bool:
|
|
"""Synthesise the first chunk (and warm the rest in the background).
|
|
|
|
Returns False when the cloud voice is unusable, so playback falls back
|
|
to the system voice.
|
|
"""
|
|
chunks = split_sentences(text, self._chunk_chars)
|
|
if not chunks:
|
|
return False
|
|
|
|
# Start the later chunks BEFORE synthesising the first one, so they are
|
|
# fetched in parallel. Doing it afterwards leaves an audible gap between
|
|
# sentences on a cold run, because chunk 2 only starts once chunk 1 is
|
|
# already playing and finishes after the audio has run out.
|
|
prefetch: Optional[threading.Thread] = None
|
|
if len(chunks) > 1:
|
|
prefetch = threading.Thread(
|
|
target=self._warm_rest, args=(chunks[1:],),
|
|
name="gemini-prefetch", daemon=True,
|
|
)
|
|
prefetch.start()
|
|
|
|
try:
|
|
self._ensure(chunks[0])
|
|
except Exception as exc:
|
|
self._using_fallback = True
|
|
if not self._warned:
|
|
self._warned = True
|
|
logger.warning(
|
|
"Gemini voice unavailable (%s) - falling back to the built-in "
|
|
"system voice. The demo continues.", exc)
|
|
else:
|
|
logger.debug("Gemini synthesis failed: %s", exc)
|
|
return False
|
|
|
|
self._using_fallback = False
|
|
return True
|
|
|
|
def start(self, text: str) -> None:
|
|
self.stop()
|
|
chunks = split_sentences(text, self._chunk_chars)
|
|
if not chunks:
|
|
return
|
|
|
|
if self._using_fallback or not self._is_cached(chunks[0]):
|
|
if not self.prepare(text):
|
|
if self._fallback is not None:
|
|
self._using_fallback = True
|
|
self._fallback.start(text)
|
|
return
|
|
|
|
self._sequence_stop.clear()
|
|
self._active.set()
|
|
self._sequence = threading.Thread(
|
|
target=self._play_sequence, args=(chunks,),
|
|
name="gemini-playback", daemon=True,
|
|
)
|
|
self._sequence.start()
|
|
|
|
def stop(self) -> None:
|
|
self._sequence_stop.set()
|
|
self._player.stop()
|
|
self._active.clear()
|
|
if self._fallback is not None:
|
|
try:
|
|
self._fallback.stop()
|
|
except Exception: # pragma: no cover
|
|
pass
|
|
|
|
def is_speaking(self) -> bool:
|
|
if self._using_fallback and self._fallback is not None:
|
|
return self._fallback.is_speaking()
|
|
return self._active.is_set()
|
|
|
|
def close(self) -> None:
|
|
self.stop()
|
|
if self._fallback is not None:
|
|
try:
|
|
self._fallback.close()
|
|
except Exception: # pragma: no cover
|
|
pass
|
|
|
|
# -- playback sequencing ------------------------------------------------- #
|
|
def _play_sequence(self, chunks: List[str]) -> None:
|
|
"""Play each chunk in turn, waiting for later ones to finish synthesis."""
|
|
try:
|
|
for chunk in chunks:
|
|
if self._sequence_stop.is_set():
|
|
return
|
|
try:
|
|
path, duration = self._ensure(chunk)
|
|
except Exception as exc:
|
|
logger.warning("Gemini synthesis failed mid-utterance: %s", exc)
|
|
return
|
|
if self._sequence_stop.is_set():
|
|
return
|
|
self._player.play(path, duration)
|
|
while self._player.is_playing():
|
|
if self._sequence_stop.is_set():
|
|
self._player.stop()
|
|
return
|
|
time.sleep(0.03)
|
|
finally:
|
|
self._active.clear()
|
|
|
|
def _warm_rest(self, chunks: List[str]) -> None:
|
|
for chunk in chunks:
|
|
if self._sequence_stop.is_set():
|
|
return
|
|
try:
|
|
self._ensure(chunk)
|
|
except Exception as exc: # pragma: no cover - retried at play time
|
|
logger.debug("prefetch failed: %s", exc)
|
|
return
|
|
|
|
# -- saved audio + synthesis --------------------------------------------- #
|
|
def _is_cached(self, text: str) -> bool:
|
|
return self.library.find(text, self._voice, self._model, self._style) is not None
|
|
|
|
def _ensure(self, text: str) -> Tuple[str, float]:
|
|
"""Return (wav path, duration), synthesising only if not already saved."""
|
|
entry = self.library.find(text, self._voice, self._model, self._style)
|
|
if entry is None:
|
|
pcm, sample_rate, channels = self._synthesise(text)
|
|
entry = self.library.save(
|
|
text, wav_from_pcm(pcm, sample_rate, channels),
|
|
self._voice, self._model, self._style,
|
|
)
|
|
return str(self.library.root / entry["file"]), entry["durationSeconds"]
|
|
|
|
def _synthesise(self, text: str) -> Tuple[bytes, int, int]:
|
|
prompt = "{0} {1}".format(self._style, text).strip() if self._style else text
|
|
body = json.dumps({
|
|
"model": self._model,
|
|
"input": prompt,
|
|
"response_format": {"type": "audio"},
|
|
"generation_config": {"speech_config": [{"voice": self._voice}]},
|
|
}).encode("utf-8")
|
|
|
|
request = urllib.request.Request(
|
|
ENDPOINT, data=body,
|
|
headers={"x-goog-api-key": self._key, "Content-Type": "application/json"},
|
|
)
|
|
started = time.perf_counter()
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=self._timeout) as response:
|
|
payload = json.loads(response.read())
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
raise RuntimeError("HTTP {0}: {1}".format(exc.code, detail)) from exc
|
|
|
|
pcm, sample_rate, channels = _extract_audio(payload)
|
|
logger.info(
|
|
"Gemini '%s': %.1fs of audio in %.2fs for %d chars",
|
|
self._voice, len(pcm) / float(sample_rate * channels * 2),
|
|
time.perf_counter() - started, len(text),
|
|
)
|
|
return pcm, sample_rate, channels
|
|
|
|
# -- helpers ------------------------------------------------------------- #
|
|
def warm(self, text: str) -> int:
|
|
"""Pre-synthesise every chunk of `text`. Returns how many were fetched."""
|
|
fetched = 0
|
|
for chunk in split_sentences(text, self._chunk_chars):
|
|
if not self._is_cached(chunk):
|
|
self._ensure(chunk)
|
|
fetched += 1
|
|
return fetched
|
|
|
|
def cache_stats(self) -> Dict[str, object]:
|
|
stats = self.library.stats()
|
|
return {"entries": stats["count"], "bytes": stats["bytes"], "dir": stats["dir"]}
|
|
|
|
def clear_cache(self) -> None:
|
|
self.library.clear()
|
|
|
|
|
|
|
|
def _extract_audio(payload: dict) -> Tuple[bytes, int, int]:
|
|
"""Pull PCM out of an interactions response.
|
|
|
|
Walks the steps rather than indexing a fixed path, so an extra step or a
|
|
reordered response does not break playback.
|
|
"""
|
|
for step in payload.get("steps", []) or []:
|
|
for part in step.get("content", []) or []:
|
|
if part.get("type") != "audio" or not part.get("data"):
|
|
continue
|
|
pcm = base64.b64decode(part["data"])
|
|
return pcm, int(part.get("sample_rate") or 24000), int(part.get("channels") or 1)
|
|
raise RuntimeError("no audio in response (status={0})".format(payload.get("status")))
|