d9b2d5427f
Replay now matches the robots and no longer cuts words: - read the turn to turnComplete, not generationComplete, and drain the socket before each send; breaking early truncated every sentence and left frames that the next turn mis-read as its own reply - accept a take only if the model's own transcript covers the text AND the audio is long enough to contain it (the transcript reports the full text even for a 0.8s clip) - pitch gate: reject an off-tone take and re-ask, per voice, using a pure-Python F0 estimator (no numpy on the host) - continuation: speak the words a voice skipped instead of retrying a line it stops on deterministically - fresh Live session per replay; delivery drifts as turns accumulate Live Gemini tab: browser talks to Gemini directly (the reverse proxy cannot upgrade a WebSocket), with a persona library - named personas, per-robot selection, built-ins that cannot be overwritten. Dashboard: records search + voice filter, log panel falls back to polling, sign-in history with CSV/JSON export, and JS errors now show on the page instead of silently blanking a tab.
94 lines
3.1 KiB
Python
94 lines
3.1 KiB
Python
"""Median fundamental frequency of PCM audio, in pure Python.
|
|
|
|
Used by the typed-replay pitch gate to reject a take whose tone does not match
|
|
the voice it is supposed to be. The shared host has no numpy, so this is
|
|
written against the stdlib only and deliberately samples a handful of frames
|
|
rather than the whole clip — the median only needs to be stable, not exact,
|
|
and a full autocorrelation over a 17 s paragraph would cost seconds of CPU.
|
|
|
|
Accuracy target: within a few Hz of a numpy implementation, which is far
|
|
tighter than the ~25 % drift the gate exists to catch.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import array
|
|
|
|
# Human speech range. 60-350 Hz covers every prebuilt voice in use
|
|
# (male ~100-140, female ~160-215) with margin.
|
|
_MIN_HZ = 60.0
|
|
_MAX_HZ = 350.0
|
|
# Decimate to this rate before correlating: 8 kHz keeps pitch information
|
|
# (harmonics well above it are irrelevant) and cuts the work by 3x.
|
|
_TARGET_RATE = 8000
|
|
_FRAME_SEC = 0.040 # matches the reference implementation
|
|
_MAX_FRAMES = 48 # sampled evenly across the clip
|
|
_SILENCE_RMS = 500.0 # int16 amplitude; below this a frame is not speech
|
|
_PEAK_RATIO = 0.3 # autocorrelation peak must be this fraction of lag 0
|
|
|
|
|
|
def _rms(frame) -> float:
|
|
total = 0
|
|
for s in frame:
|
|
total += s * s
|
|
return (total / len(frame)) ** 0.5 if frame else 0.0
|
|
|
|
|
|
def median_f0(pcm: bytes, sample_rate: int = 24000) -> float:
|
|
"""Median F0 in Hz over voiced frames, or 0.0 if nothing voiced was found."""
|
|
if not pcm or len(pcm) < 4:
|
|
return 0.0
|
|
samples = array.array("h")
|
|
samples.frombytes(pcm[: len(pcm) // 2 * 2])
|
|
|
|
step = max(1, int(round(sample_rate / float(_TARGET_RATE))))
|
|
rate = sample_rate // step
|
|
if step > 1:
|
|
samples = samples[::step]
|
|
|
|
frame_len = int(rate * _FRAME_SEC)
|
|
if frame_len < 32 or len(samples) < frame_len * 2:
|
|
return 0.0
|
|
|
|
lo = max(2, int(rate / _MAX_HZ))
|
|
hi = min(frame_len - 1, int(rate / _MIN_HZ))
|
|
if hi <= lo:
|
|
return 0.0
|
|
|
|
starts = []
|
|
span = len(samples) - frame_len
|
|
count = min(_MAX_FRAMES, max(1, span // frame_len))
|
|
for i in range(count):
|
|
starts.append(int(span * i / float(count)) if count > 1 else 0)
|
|
|
|
found = []
|
|
for start in starts:
|
|
frame = samples[start:start + frame_len]
|
|
if _rms(frame) < _SILENCE_RMS:
|
|
continue
|
|
mean = sum(frame) / float(frame_len)
|
|
centred = [s - mean for s in frame]
|
|
|
|
energy = 0.0
|
|
for v in centred:
|
|
energy += v * v
|
|
if energy <= 0:
|
|
continue
|
|
|
|
best_lag, best_val = 0, 0.0
|
|
for lag in range(lo, hi + 1):
|
|
total = 0.0
|
|
for i in range(frame_len - lag):
|
|
total += centred[i] * centred[i + lag]
|
|
if total > best_val:
|
|
best_val, best_lag = total, lag
|
|
if best_lag and best_val > _PEAK_RATIO * energy:
|
|
found.append(rate / float(best_lag))
|
|
|
|
if not found:
|
|
return 0.0
|
|
found.sort()
|
|
mid = len(found) // 2
|
|
if len(found) % 2:
|
|
return found[mid]
|
|
return (found[mid - 1] + found[mid]) / 2.0
|