realtimevad 0.0.4__py3-none-any.whl

RealtimeVAD/__init__.py

@@ -0,0 +1,8 @@
+"""RealtimeVAD: fast speech-end detection for low-latency voice assistants."""
+
+from .iterator import VADIterator
+from .vad import VAD, Event, Segment
+
+__version__ = "0.0.4"
+
+__all__ = ["VAD", "VADIterator", "Event", "Segment", "__version__"]

RealtimeVAD/assets/LICENSE-silero-vad.txt

@@ -0,0 +1,23 @@
+silero_vad_v6.2.1.onnx comes from Silero VAD (https://github.com/snakers4/silero-vad).
+
+MIT License
+
+Copyright (c) 2020-present Silero Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

RealtimeVAD/iterator.py

@@ -0,0 +1,84 @@
+"""Drop-in for ``silero_vad.VADIterator``: same constructor, call and results."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import numpy as np
+
+from .silero import SAMPLE_RATE
+from .vad import FRAME_S, VAD
+
+
+class _SileroModel:
+    """Adapts a model from ``silero_vad.load_silero_vad()`` to a frame -> probability call."""
+
+    def __init__(self, model) -> None:
+        self._model = model
+        try:
+            import torch
+
+            self._tensor = torch.from_numpy
+        except ImportError:
+            self._tensor = lambda a: a
+
+    def __call__(self, frame: np.ndarray) -> float:
+        p = self._model(self._tensor(np.ascontiguousarray(frame)), SAMPLE_RATE)
+        return float(p.item() if hasattr(p, "item") else p)
+
+    def reset(self) -> None:
+        self._model.reset_states()
+
+
+class VADIterator:
+    """Like Silero's ``VADIterator``, but ``end`` comes as soon as the audio goes quiet.
+
+    ``model`` may be a Silero model (used as given) or None (the bundled one).
+    Extra keyword arguments go to :class:`VAD`. Chunks may have any size.
+    """
+
+    def __init__(
+        self,
+        model=None,
+        threshold: float = 0.5,
+        sampling_rate: int = 16000,
+        min_silence_duration_ms: int = 100,
+        speech_pad_ms: int = 30,
+        neg_threshold: Optional[float] = None,
+        **options,
+    ) -> None:
+        if sampling_rate != SAMPLE_RATE:
+            raise ValueError("RealtimeVAD supports 16000 Hz only")
+        if neg_threshold is None:
+            neg_threshold = max(threshold - 0.15, 0.01)
+        self.sampling_rate = sampling_rate
+        self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
+        self.vad = VAD(
+            start_threshold=threshold,
+            continue_threshold=neg_threshold,
+            start_s=FRAME_S,  # Silero starts on one frame
+            end_s=min_silence_duration_ms / 1000,
+            preroll_s=speech_pad_ms / 1000,
+            probability=None if model is None else _SileroModel(model),
+            **options,
+        )
+
+    @property
+    def triggered(self) -> bool:
+        return self.vad.in_speech
+
+    def reset_states(self) -> None:
+        self.vad.reset()
+
+    def __call__(self, x, return_seconds: bool = False, time_resolution: int = 1) -> Optional[dict]:
+        result = {}
+        for event in self.vad.feed(np.asarray(x)):
+            if event.kind == "start":  # Silero: start of the first speech frame, minus the pad
+                sample = (event.time - FRAME_S) * self.sampling_rate - self.speech_pad_samples
+            elif event.kind == "end":  # Silero: where speech stopped, plus the pad
+                sample = event.segment.speech_end * self.sampling_rate + self.speech_pad_samples
+            else:
+                continue
+            sample = max(0, int(round(sample)))
+            result[event.kind] = round(sample / self.sampling_rate, time_resolution) if return_seconds else sample
+        return result or None

RealtimeVAD/silero.py

@@ -0,0 +1,76 @@
+"""Stateful Silero v6 ONNX inference on 512-sample, 16 kHz frames."""
+
+from __future__ import annotations
+
+import math
+from pathlib import Path
+
+import numpy as np
+
+SAMPLE_RATE = 16_000
+FRAME = 512
+MODEL = Path(__file__).parent / "assets" / "silero_vad_v6.2.1.onnx"
+
+
+class Silero:
+    """Silero v6 with recovery from its "deaf" state.
+
+    On a long stream the recurrent state can drift into an attractor where the
+    probability stays near 0 even through loud speech (low babble, near-silent
+    pauses). After ``restore_s`` of p < 0.1 the state is put back to where the
+    model last heard confident speech (p >= 0.9), or zeroed before any. A zero
+    state is the worse restart point: it scores the next utterance lower and
+    later. Costs no extra inference; 0 disables it.
+    """
+
+    def __init__(self, model: str | Path = MODEL, threads: int = 1, restore_s: float = 1.0) -> None:
+        import onnxruntime as ort
+
+        options = ort.SessionOptions()
+        options.intra_op_num_threads = threads
+        options.inter_op_num_threads = 1
+        options.log_severity_level = 3
+        self._session = ort.InferenceSession(
+            str(model), sess_options=options, providers=["CPUExecutionProvider"]
+        )
+        self._sr = np.asarray(SAMPLE_RATE, dtype=np.int64)
+        self._restore_frames = math.ceil(restore_s * SAMPLE_RATE / FRAME - 1e-9) if restore_s > 0 else 0
+        self.restores = 0
+        self.reset()
+
+    def reset(self) -> None:
+        """Start a new stream: zero state and context, forget the speech snapshot."""
+        self._context = np.zeros((1, 64), dtype=np.float32)
+        self._state = np.zeros((2, 1, 128), dtype=np.float32)
+        self._snapshot = None
+        self._quiet = 0
+
+    def __call__(self, frame: np.ndarray) -> float:
+        audio = np.asarray(frame, dtype=np.float32).reshape(1, FRAME)
+        p, self._state = self._session.run(
+            None,
+            {"input": np.concatenate((self._context, audio), axis=1), "sr": self._sr, "state": self._state},
+        )
+        self._context = audio[:, -64:].copy()
+        p = float(np.asarray(p).reshape(-1)[0])
+        if self._restore_frames:  # standalone rule; inside VAD, VAD decides both (see vad.py)
+            if p >= 0.9:
+                self.snapshot()
+            self._quiet = self._quiet + 1 if p < 0.1 else 0
+            if self._quiet >= self._restore_frames:
+                self.restore()
+        return p
+
+    def snapshot(self) -> None:
+        """Remember the current state as the restore point (fresh arrays each frame)."""
+        self._snapshot = (self._state, self._context)
+
+    def restore(self) -> None:
+        """Put the state back to the last confident speech (zeros before any); affects the next frame."""
+        self._quiet = 0
+        self.restores += 1
+        if self._snapshot is None:
+            self._context = np.zeros((1, 64), dtype=np.float32)
+            self._state = np.zeros((2, 1, 128), dtype=np.float32)
+        else:
+            self._state, self._context = (a.copy() for a in self._snapshot)

RealtimeVAD/vad.py

@@ -0,0 +1,436 @@
+"""Speech start/end detection. Silero decides speech; the end races Silero's
+silence count against a 10 ms near-silence clock (see SPEC.md)."""
+
+from __future__ import annotations
+
+import math
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+import numpy as np
+
+from .silero import FRAME, SAMPLE_RATE
+
+BLOCK = 160  # 10 ms quiet-clock block
+FRAME_S = FRAME / SAMPLE_RATE
+EPS = 1e-9
+
+
+@dataclass(frozen=True)
+class Segment:
+    audio: np.ndarray  # float32, 16 kHz, from just before the start up to the end
+    start: float
+    end: float  # stream time the end was decided
+    speech_end: float  # when the speech actually stopped
+    voiced_s: float
+    peak: float
+
+
+@dataclass(frozen=True)
+class Frame:
+    """One evaluated 32 ms model frame (only with ``frame_events=True``)."""
+
+    probability: float
+    voiced: bool
+    in_speech: bool
+    silent_frames: int  # consecutive unvoiced frames inside speech
+    streak: int  # voiced frames toward a start
+    voiced_s: float  # of the running segment
+    peak: float  # of the running segment
+    infer_s: float  # model inference time
+
+
+@dataclass(frozen=True)
+class Event:
+    kind: str  # "onset" | "start" | "pause" | "end" | "frame"
+    time: float
+    segment: Optional[Segment] = None  # set on "end"
+    source: str = ""  # "pause": "quiet" (10 ms clock) or "silero" (first unvoiced frame)
+    frame: Optional[Frame] = None  # set on "frame"
+
+
+def _frames(seconds: float) -> int:
+    return math.ceil(seconds / FRAME_S - EPS)
+
+
+def _dbfs(x: np.ndarray) -> float:
+    power = float(np.mean(x * x))
+    return max(-100.0, 10.0 * math.log10(power)) if power > 0.0 else -100.0
+
+
+def _pcm(audio) -> np.ndarray:
+    if isinstance(audio, (bytes, bytearray, memoryview)):
+        return np.frombuffer(audio, dtype="<i2").astype(np.float32) / 32768.0
+    a = np.asarray(audio)
+    if a.dtype == np.int16:
+        return a.astype(np.float32).reshape(-1) / 32768.0
+    return a.astype(np.float32, copy=False).reshape(-1)
+
+
+class VAD:
+    """Feed 16 kHz mono PCM; ``feed`` returns start/pause/end events in order."""
+
+    def __init__(
+        self,
+        *,
+        start_threshold: float = 0.90,
+        continue_threshold: float = 0.45,
+        start_s: float = 0.064,
+        end_s: float = 0.24,  # Silero's silence count
+        quiet_end_s: Optional[float] = None,  # quiet clock; default end_s
+        noisy_extra_s: float = 0.04,  # added when room noise has raised the quiet threshold
+        quiet_rms: float = 0.002,
+        noise_margin_db: float = 15.0,
+        speech_margin_db: float = 15.0,
+        preroll_s: float = 0.30,
+        lookback_s: float = 1.0,
+        restore_s: float = 1.0,  # Silero recovery after this much p < 0.1 over real (non-zero) audio; 0 = off
+        deaf_s: float = 0.19,  # ...or after Silero ignored speech-loud audio this long
+        max_speech_s: float = 120.0,
+        extend: Optional[Callable[[float], Optional[float]]] = None,
+        frame_events: bool = False,
+        onset_events: bool = False,  # "onset" at the first confident frame, before "start" confirms it
+        confirm_threshold: Optional[float] = None,  # each frame of the start debounce; default continue_threshold
+        rearm_s: Optional[float] = None,  # sound that re-arms the quiet clock; default start_s
+        probability: Optional[Callable[[np.ndarray], float]] = None,
+    ) -> None:
+        if probability is None:
+            from .silero import Silero
+
+            probability = Silero(restore_s=0)  # restores are decided here, from audio evidence
+        self._p = probability
+        self._extend = extend
+        self._frame_events = frame_events
+        self._onset_events = onset_events
+        self.start_threshold, self.continue_threshold = start_threshold, continue_threshold
+        # After the first confident frame, every frame until start_s is reached must stay at
+        # or above this (a click fades out above continue_threshold for a few frames).
+        self.confirm_threshold = continue_threshold if confirm_threshold is None else confirm_threshold
+        # The quiet clock can only make an end earlier than Silero's count, never later. Over
+        # near-silence it waits end_s like Silero; when room noise has raised the threshold, a
+        # word's fading tail already counts as quiet, so it waits noisy_extra_s longer.
+        self.end_s = end_s
+        self.quiet_end_s = end_s if quiet_end_s is None else quiet_end_s
+        self.noisy_extra_s = noisy_extra_s
+        self.quiet_rms = quiet_rms
+        self.noise_margin_db, self.speech_margin_db = noise_margin_db, speech_margin_db
+        self.start_s = start_s
+        self._end_frames = max(1, _frames(end_s))
+        self._max_frames = max(1, _frames(max_speech_s))
+        self._preroll_n = _frames(preroll_s)
+        self._lookback_n = max(0, _frames(lookback_s))
+        self._rearm_samples = max(1, math.ceil((start_s if rearm_s is None else rearm_s) * SAMPLE_RATE))
+        self._restore_n = _frames(restore_s) if restore_s > 0 else 0
+        self._deaf_n = _frames(deaf_s) if deaf_s > 0 and restore_s > 0 else 0
+        self.reset()
+
+    def reset(self) -> None:
+        reset = getattr(self._p, "reset", None)
+        if reset is not None:
+            reset()
+        self._n = 0  # samples processed
+        self._block_buf = np.zeros(0, dtype=np.float32)
+        self._frame_buf = np.zeros(0, dtype=np.float32)
+        self._echo_until = -math.inf
+        self._history: deque = deque(maxlen=self._preroll_n + self._lookback_n)  # (frame, dBFS, echo)
+        self._floor: Optional[float] = None  # idle floor for the onset look-back
+        self._noise_db: Optional[float] = None  # room noise for the quiet threshold
+        self._speech_db: Optional[float] = None
+        self._quiet = self.quiet_rms if self.quiet_rms > 0 else -1.0  # current quiet threshold (RMS)
+        self._tail = False  # waiting for Silero to drop after an early end
+        self._low = self._deaf = 0  # frames of p < 0.1 over real audio / over speech-loud audio
+        self._quiet_restored = False  # the quiet rule already restored in this pause
+        self._in_speech = False
+        self._idle = 0
+        self._streak = 0
+        self._lookback_s = 0.0
+        self._clear_segment()
+
+    @property
+    def start_s(self) -> float:
+        """Speech needed to confirm a start (the debounce after "onset"); settable while running."""
+        return self._start_s
+
+    @start_s.setter
+    def start_s(self, seconds: float) -> None:
+        self._start_s = seconds
+        self._start_frames = max(1, _frames(seconds))
+
+    @property
+    def in_speech(self) -> bool:
+        return self._in_speech
+
+    @property
+    def onset_lookback_s(self) -> float:
+        """How far the latest start was moved back over a soft opening."""
+        return self._lookback_s
+
+    @property
+    def quiet_clock(self) -> tuple[bool, bool, int]:
+        """(armed, unbroken, quiet blocks since it armed) of the 10 ms quiet clock."""
+        return self._t_quiet is not None, self._unbroken, self._quiet_blocks
+
+    def audio(self) -> Optional[np.ndarray]:
+        """The running segment's audio up to the latest sample fed, or None outside speech."""
+        if not self._in_speech:
+            return None
+        return np.concatenate(self._audio + [self._frame_buf])
+
+    def segment(self) -> Optional[Segment]:
+        """The running segment so far (end = now), or None outside speech."""
+        if not self._in_speech:
+            return None
+        return Segment(self.audio(), self._start, self._n / SAMPLE_RATE, self._last_voiced,
+                       self._voiced * FRAME_S, self._peak)
+
+    def feed(self, audio, playback: bool = False) -> list[Event]:
+        """``playback=True`` marks the chunk as possibly containing your own output (echo)."""
+        events: list[Event] = []
+        x = _pcm(audio)
+        i = 0
+        while i < len(x):
+            # advance to the next block or frame boundary, whichever is first
+            step = min(BLOCK - len(self._block_buf), FRAME - len(self._frame_buf), len(x) - i)
+            part = x[i : i + step]
+            i += step
+            self._n += step
+            self._block_buf = np.concatenate((self._block_buf, part))
+            self._frame_buf = np.concatenate((self._frame_buf, part))
+            if playback:
+                self._echo_until = self._n / SAMPLE_RATE + 0.5
+            block = self._block_buf if len(self._block_buf) == BLOCK else None
+            quiet = block is not None and self._block_start(block)
+            if len(self._frame_buf) == FRAME:
+                frame, self._frame_buf = self._frame_buf, np.zeros(0, dtype=np.float32)
+                self._frame(frame, self._n / SAMPLE_RATE, events)
+            if block is not None:
+                self._block_buf = np.zeros(0, dtype=np.float32)
+                self._block_end(quiet, self._n / SAMPLE_RATE, events)
+        return events
+
+    # -- per 10 ms block (quiet clock) -------------------------------------
+    def _block_start(self, block: np.ndarray) -> bool:
+        quiet = math.sqrt(float(np.mean(block * block))) <= self._quiet
+        if self._t_quiet is not None:
+            if quiet:
+                self._loud = 0
+                self._quiet_blocks += self._unbroken
+            else:
+                self._unbroken = False
+                self._loud += BLOCK
+                if self._loud >= self._rearm_samples:
+                    self._t_quiet, self._loud, self._paused = None, 0, False
+        return quiet
+
+    def _block_end(self, quiet: bool, t: float, events: list[Event]) -> None:
+        if not self._in_speech:
+            return
+        if self._t_quiet is not None and self._unbroken and t - self._t_quiet + EPS >= self._target(t):
+            self._finish(t, events, speech_end=self._t_quiet)
+        elif quiet and self._t_quiet is None:
+            self._t_quiet, self._unbroken, self._loud, self._quiet_blocks = t, True, 0, 1
+            raised = self._quiet > self.quiet_rms * (1 + 1e-6)
+            self._quiet_wait = self.quiet_end_s + (self.noisy_extra_s if raised else 0.0)
+            self._pause(t, events, "quiet")
+
+    def _target(self, t: float) -> float:
+        """Quiet-clock wait (set when it armed), or longer if ``extend`` asks once that is reached."""
+        wait = self._quiet_wait
+        if t - self._t_quiet + EPS < wait or self._extend is None:
+            return wait
+        extra = self._extend(t)
+        return wait if extra is None else max(wait, extra)
+
+    # -- per 32 ms Silero frame --------------------------------------------
+    def _frame(self, frame: np.ndarray, t: float, events: list[Event]) -> None:
+        t0 = time.perf_counter()
+        p = self._p(frame)
+        infer_s = time.perf_counter() - t0
+        if self._in_speech:
+            voiced = p >= self.continue_threshold
+        else:  # idle: the first frame needs start_threshold, the rest of the debounce confirm_threshold
+            voiced = p >= (self.confirm_threshold if self._streak else self.start_threshold)
+        level = _dbfs(frame)
+        echo = t <= self._echo_until
+        self._history.append((frame, level, echo))
+        self._levels(p, level, echo)
+        if self._tail:  # after an early end, Silero's fading tail must not open a new segment...
+            self._tail = p >= self.continue_threshold
+            if level <= 20.0 * math.log10(max(self._quiet, 1e-9)):
+                voiced = False  # ...over quiet audio; speech that resumes at once is loud and may start
+        if not self._in_speech:
+            self._idle_frame(p, voiced, level, t, events)
+            self._report(t, p, voiced, infer_s, events)
+            return
+        self._audio.append(frame)
+        self._peak = max(self._peak, p)
+        if voiced:
+            if self._silent:
+                self._paused = False
+                if self._t_quiet is not None and t > self._t_quiet:
+                    self._unbroken = False
+            self._voiced += 1
+            self._silent = 0
+            self._last_voiced = t
+            self._end_target = self._end_frames
+        else:
+            self._silent += 1
+            if self._silent == 1:
+                self._pause(t, events, "silero")
+        self._report(t, p, voiced, infer_s, events)
+        if len(self._audio) >= self._max_frames:
+            self._finish(t, events)
+            return
+        if self._silent >= self._end_frames and self._extend is not None:
+            extra = self._extend(t)
+            self._end_target = self._end_frames if extra is None else max(self._end_frames, _frames(extra))
+        if self._silent >= self._end_target:
+            self._finish(t, events)
+
+    def _levels(self, p: float, level: float, echo: bool) -> None:
+        """Learn speech and room-noise levels and set the quiet threshold."""
+        if p >= self.start_threshold:
+            s = self._speech_db
+            self._speech_db = level if s is None else s + max(-1.0, min(1.0, level - s))
+        elif p < self.continue_threshold and level > -90.0 and not echo:  # skip gated digital silence
+            n = self._noise_db
+            if n is None:
+                self._noise_db = level
+            elif self._in_speech:  # pauses inside speech: running median, about 1 dB per frame
+                self._noise_db = n + max(-1.0, min(1.0, level - n))
+            else:  # idle: falls fast, rises 3 dB/s, so a soft opening barely lifts it
+                self._noise_db = level if level <= n else n + min(level - n, 3.0 * FRAME_S)
+        if self.quiet_rms <= 0:
+            self._quiet = -1.0  # quiet clock off: no block is ever quiet
+        else:
+            # Noise can raise the threshold above quiet_rms, but only up to speech level - 15 dB
+            # (soft speech is never quiet); it never drops below quiet_rms, so it is never stricter.
+            floor_db = 20.0 * math.log10(self.quiet_rms)
+            db = floor_db
+            if self._noise_db is not None:
+                db = max(db, self._noise_db + self.noise_margin_db)
+            if self._speech_db is not None:
+                db = max(floor_db, min(db, self._speech_db - self.speech_margin_db))
+            self._quiet = 10.0 ** (db / 20.0)
+        # Silero's recurrent state drifts in long noisy or near-silent stretches until it is
+        # "deaf" (p ~ 0 through loud speech). Restore the last confident-speech state after a
+        # second of p < 0.1 over real audio (gated digital silence doesn't drift it, and a
+        # restore there only disturbs the next onset), never on a frame that sounds like an
+        # onset; or at once when Silero has ignored speech-loud audio for deaf_s.
+        #
+        # The restore point is the state after the *user's* confident speech, never after
+        # the assistant's own voice: a state that last heard playback echo scores following
+        # room noise as speech (p ~0.96 decaying over 0.4 s), which opens phantom segments.
+        restore = getattr(self._p, "restore", None)
+        snapshot = getattr(self._p, "snapshot", None)
+        if p >= self.start_threshold and not echo and snapshot is not None:
+            snapshot()
+        if self._in_speech:
+            self._quiet_restored = False  # a new pause may restore once more
+        if not self._restore_n or restore is None or self._in_speech or echo:
+            self._low = self._deaf = 0
+            return
+        speech_like = self._speech_like(level)
+        # a healthy Silero passes continue_threshold within 1-3 frames of loud speech
+        self._deaf = 0 if p >= self.continue_threshold else self._deaf + speech_like
+        self._low = self._low + 1 if p < 0.1 and level > -70.0 else 0  # near-silence (gates, noise suppression) never drifts it
+        deaf = self._deaf_n and self._deaf >= self._deaf_n
+        quiet = self._low >= self._restore_n and not speech_like and not self._quiet_restored
+        if deaf or quiet:  # the quiet rule fires once per pause: one restore un-sticks Silero
+            restore()
+            self._quiet_restored = self._quiet_restored or bool(quiet)
+            self._low = self._deaf = 0
+
+    def _speech_like(self, level: float) -> bool:
+        """Loud like the speaker and clearly above the idle floor."""
+        if self._speech_db is None or self._floor is None:
+            return False
+        margin = max(3.0, min(10.0, 0.5 * (self._speech_db - self._floor)))
+        return level >= max(self._floor + margin, self._speech_db - 10.0)
+
+    def _idle_frame(self, p: float, voiced: bool, level: float, t: float, events: list[Event]) -> None:
+        floor = self._floor
+        if self._lookback_n:
+            if floor is None or level <= floor:
+                self._floor = level if floor is None else floor + (level - floor) * 0.5
+            else:
+                self._floor = floor + min(level - floor, 3.0 * FRAME_S)
+        self._idle += 1
+        self._streak = self._streak + 1 if voiced else 0
+        self._peak = max(self._peak, p) if voiced else 0.0
+        if self._streak == 1 and self._onset_events:
+            events.append(Event("onset", t))  # maybe speech; "start" follows only if it holds
+        if self._streak < self._start_frames:
+            return
+        lookback = self._lookback(floor)
+        keep = self._preroll_n if not lookback else max(self._preroll_n, min(self._preroll_n + lookback, self._idle))
+        self._in_speech = True
+        self._audio = [f for f, _, _ in list(self._history)[-keep:]]
+        self._start = t - (self._streak - 1 + lookback) * FRAME_S  # end of first streak frame, as in Miep
+        self._lookback_s = lookback * FRAME_S
+        self._voiced = self._streak
+        self._last_voiced = t
+        events.append(Event("start", self._start))
+
+    def _lookback(self, floor: Optional[float]) -> int:
+        """Frames before the start streak that still belong to a soft opening."""
+        h, s = self._history, self._streak
+        if not self._lookback_n or floor is None or len(h) < s or any(h[-i][2] for i in range(1, s + 1)):
+            return 0
+        margin = 12.0  # an opening must be clearly louder than the room...
+        if self._speech_db is not None:  # ...but in loud noise, half the speech-to-noise gap is enough
+            margin = max(3.0, min(margin, 0.5 * (self._speech_db - floor)))
+        threshold = max(floor + margin, -60.0)
+        count = gap = 0
+        for step in range(1, min(self._lookback_n, self._idle - s, len(h) - s) + 1):
+            _, level, echo = h[-(s + step)]
+            if echo:
+                break
+            if level >= threshold:
+                count, gap = step, 0
+            else:
+                gap += 1
+                if gap > 3:  # 0.1 s
+                    break
+        return count if count > self._preroll_n - s else 0
+
+    # -- segment bookkeeping -----------------------------------------------
+    def _report(self, t: float, p: float, voiced: bool, infer_s: float, events: list[Event]) -> None:
+        if self._frame_events:
+            events.append(Event("frame", t, frame=Frame(
+                p, voiced, self._in_speech, self._silent, self._streak,
+                self._voiced * FRAME_S, self._peak, infer_s)))
+
+    def _pause(self, t: float, events: list[Event], source: str) -> None:
+        if not self._paused:
+            self._paused = True
+            events.append(Event("pause", t, source=source))
+
+    def _finish(self, t: float, events: list[Event], speech_end: Optional[float] = None) -> None:
+        segment = Segment(
+            audio=self.audio(),
+            start=self._start,
+            end=t,
+            speech_end=self._last_voiced if speech_end is None else speech_end,
+            voiced_s=self._voiced * FRAME_S,
+            peak=self._peak,
+        )
+        self._in_speech = False
+        self._tail = speech_end is not None  # ended by the quiet clock, Silero may still be high
+        self._idle = self._streak = 0
+        self._clear_segment()
+        events.append(Event("end", t, segment))
+
+    def _clear_segment(self) -> None:
+        self._audio: list[np.ndarray] = []
+        self._start = self._last_voiced = self._peak = 0.0
+        self._voiced = self._silent = 0
+        self._end_target = self._end_frames
+        self._paused = False
+        self._t_quiet: Optional[float] = None
+        self._unbroken = False
+        self._loud = 0
+        self._quiet_blocks = 0
+        self._quiet_wait = self.quiet_end_s

realtimevad-0.0.4.dist-info/METADATA

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: realtimevad
+Version: 0.0.4
+Summary: Fast speech-end detection for low-latency voice assistants
+Author: Kolja Beigel
+License: MIT
+Project-URL: Homepage, https://github.com/KoljaB/RealtimeVAD
+Project-URL: Issues, https://github.com/KoljaB/RealtimeVAD/issues
+Keywords: vad,voice activity detection,endpointing,turn detection,voice assistant,realtime
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.21
+Requires-Dist: onnxruntime>=1.16
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# RealtimeVAD
+
+*Know that the speaker stopped, about 110 ms before Silero does.*
+
+[RealtimeSTT](https://github.com/KoljaB/RealtimeSTT) · [RealtimeTTS](https://github.com/KoljaB/RealtimeTTS) · [Live demo](https://github.com/KoljaB/RealtimeVAD/blob/main/demo/README.md) · [Spec](https://github.com/KoljaB/RealtimeVAD/blob/main/SPEC.md)
+
+A voice assistant can't do anything until it knows the user has stopped
+talking. Speech recognition, the LLM call and the voice output all wait for
+that one decision, and a normal voice activity detector takes a few hundred
+milliseconds to make it.
+
+RealtimeVAD makes it sooner. It keeps [Silero VAD](https://github.com/snakers4/silero-vad)
+to decide what is speech, and adds a second, faster check: every 10 ms it
+compares the audio level with the room's noise. The moment the audio goes
+quiet, you get an event, long before Silero's probability has fallen.
+
+- **Input:** 16 kHz mono audio in chunks of any size (float32, int16 or int16 bytes).
+- **Output:** `start`, `pause` and `end` events, plus the audio of the running segment at any time.
+- **Lead:** speech-off a median 110 ms before Silero's probability drops, and 214 ms before Silero's `VADIterator` reports the end, on 3,238 real turns.
+- **Price:** it also fires in short gaps between words, about ten times as often as Silero.
+- **Runs on:** one CPU thread, far faster than realtime. Needs only numpy and onnxruntime; the Silero model is bundled, no torch.
+- **Status:** alpha (0.0.4). The API may still change.
+
+## Live demo
+
+https://github.com/user-attachments/assets/d01900dd-9601-4f3d-8c88-db55e8bc83a1
+
+The video uses a synthetic voice and shows a median lead of 92 ms. The 110 ms above is the median on real voice-assistant turns.
+
+[demo/](https://github.com/KoljaB/RealtimeVAD/blob/main/demo/README.md) runs this page on your own microphone. It shows, turn by turn, how much sooner
+RealtimeVAD knew that you stopped than Silero alone, on the same audio.
+
+## Quick start
+
+```bash
+pip install realtimevad
+```
+
+```python
+from RealtimeVAD import VAD
+
+vad = VAD()
+for chunk in microphone_chunks():
+    for event in vad.feed(chunk):
+        if event.kind == "start":
+            stream_to_asr(vad.audio())       # includes the audio just before speech began
+        elif event.kind == "pause":
+            speculate(vad.audio())           # the speaker may be done; this isn't the end yet
+        elif event.kind == "end":
+            finalize(event.segment.audio)
+```
+
+## Events
+
+| Event | Meaning | When |
+|---|---|---|
+| `start` | Speech has begun | After about 64 ms of confident speech, so clicks don't trigger it |
+| `pause` | The speaker may have stopped (speech-off) | Within about 10 ms of the audio going quiet |
+| `end` | The segment is over | 0.24 s after speech-off, unless speech resumes. Never later than Silero alone |
+| `onset` | Speech may have begun | At the first confident frame, before `start` confirms it. Off by default: `VAD(onset_events=True)` |
+
+`vad.audio()` returns the running segment at any time, starting about 0.3 s
+before the speech. `end` carries the finished segment with its audio, start
+time, end time and the time the speech stopped.
+
+If your assistant talks while the microphone is open, pass
+`vad.feed(chunk, playback=True)` for those chunks so its own voice is handled
+as possible echo. Every threshold and duration can be configured;
+[SPEC.md](https://github.com/KoljaB/RealtimeVAD/blob/main/SPEC.md) lists what is guaranteed.
+
+## How to use `pause`
+
+`pause` is early and often wrong, on purpose. It fires on every breath, every
+short gap between words and every stop consonant, so most `pause` events are
+followed by more speech.
+
+- **Use it to start work you can cancel:** finalize the transcript, call the
+  LLM, prepare the voice output. Cancel when speech continues. A wrong `pause`
+  then costs almost nothing, and a right one saves hundreds of milliseconds.
+- **Never treat it as the end of the turn.** Only `end` closes a segment.
+- **Can't cancel quickly?** Ignore `pause` and wait for `end`. The extra
+  false alarms would cost you more than they save.
+
+## Already using Silero?
+
+Change the import:
+
+```python
+from silero_vad import load_silero_vad
+from RealtimeVAD import VADIterator          # was: from silero_vad import VADIterator
+
+vad_iterator = VADIterator(load_silero_vad(), sampling_rate=16000)
+speech_dict = vad_iterator(chunk, return_seconds=True)   # {'start': ...} / {'end': ...} / None
+```
+
+The constructor and the results stay the same, but `end` arrives as soon as the
+audio goes quiet. Without a model argument, it uses the bundled Silero model and
+doesn't need torch.
+
+## Results
+
+Measured on 3,238 turns from a live voice assistant, each with its trailing
+silence, with default settings. Silero's decisions are computed from the very
+same frame probabilities, so both see identical input. "Speech-off" is
+RealtimeVAD's last `pause` before the turn ended, against the first frame of
+Silero's final run below its threshold.
+
+| RealtimeVAD speech-off earlier than … | median | p25 – p75 | p90 | earlier in |
+|---|---|---|---|---|
+| Silero, probability below 0.5 (its default threshold) | 110 ms | 84 – 190 ms | 242 ms | 98 % |
+| Silero, probability below 0.35 (`VADIterator`'s end threshold) | 120 ms | 98 – 200 ms | 252 ms | 98 % |
+| Silero `VADIterator` end, default 100 ms minimum silence | 214 ms | 192 – 298 ms | 356 ms | 99 % |
+
+| Speech-off signals followed by more speech | per second of speech | turns without any |
+|---|---|---|
+| RealtimeVAD `pause` | 1.6 | 16 % |
+| Silero, probability below 0.5 | 0.15 | 85 % |
+
+Reproduce with your own recordings (16 kHz mono WAVs, one turn each, with
+trailing silence):
+
+```bash
+python benchmarks/speech_off_lead.py <wav dir> out.json
+```
+
+## Limits
+
+- The numbers come from one speaker and one capture chain: browser capture
+  with noise suppression and echo cancellation, mostly quiet rooms.
+- The lead needs audio that actually goes quiet after a word, and noise
+  suppression helps that. In the 147 noisier turns (room level above −70 dBFS)
+  the median lead was 92–132 ms. Without noise suppression in a loud room,
+  expect less.
+- Earlier is not the same as correct: whether a speech-off sometimes lands
+  inside a soft word ending was not checked against word timings.
+- The lead is for speech-off, not for the turn end. `end` waits 0.24 s after
+  speech-off in both cases; its own gain over Silero is smaller and depends on
+  the noise.
+- 16 kHz mono only.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## License and credits
+
+MIT. The bundled speech model is [Silero VAD](https://github.com/snakers4/silero-vad) (MIT).

realtimevad-0.0.4.dist-info/RECORD

@@ -0,0 +1,11 @@
+RealtimeVAD/__init__.py,sha256=rumCIYSRL6ELLpdTTCPLaNjfQ03QUk4_zKYGSlcj0to,243
+RealtimeVAD/iterator.py,sha256=DPgF8vDdQ13HiVM_-pXUiC3xyNvptBWcnpbk9mtfpQU,3062
+RealtimeVAD/silero.py,sha256=GD_0ok6egLwvtsZOqzxnHgjyd8c-PV8Np6B-zU0VlLE,3139
+RealtimeVAD/vad.py,sha256=qYNJV31gytEcGJNTL2vqwxal8WsE6smwHYJgT6bf4O0,20941
+RealtimeVAD/assets/LICENSE-silero-vad.txt,sha256=Gie4kQFq7VC_m0lX9YeicyZLZmsJBLca97NaFFMGyIA,1187
+RealtimeVAD/assets/silero_vad_v6.2.1.onnx,sha256=dnHNBLAE6QdtoNSnsaWuw2rfFhw5IwwcuUpP1dtrvSg,2845718
+realtimevad-0.0.4.dist-info/licenses/LICENSE,sha256=DxIW6H5paKzZtBq9q3uqtZ9n_yhNa2q4wQTwxTRBXts,1090
+realtimevad-0.0.4.dist-info/METADATA,sha256=l-6vXNka0-YoXi9vnOYZ0_o-uqfMo2OXZb9tJxBl15E,7746
+realtimevad-0.0.4.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+realtimevad-0.0.4.dist-info/top_level.txt,sha256=gnTlQhGcr7VYWybQ8jfX6h08sVKfNRD_bhNmu-RquG4,12
+realtimevad-0.0.4.dist-info/RECORD,,

realtimevad-0.0.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

realtimevad-0.0.4.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kolja Beigel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

realtimevad-0.0.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+RealtimeVAD