pygpt-net 2.6.30__py3-none-any.whl → 2.6.32__py3-none-any.whl

pygpt_net/core/audio/backend/pyaudio/realtime.py

@@ -0,0 +1,312 @@
+import threading
+from typing import Optional
+
+import numpy as np
+
+from PySide6.QtCore import QTimer, QObject, Qt
+
+
+class RealtimeSessionPyAudio(QObject):
+    """
+    Realtime PCM playback session using PyAudio in callback mode.
+    Consumes already-converted PCM frames, keeps GUI responsive and emits volume updates.
+    """
+    def __init__(
+            self,
+            device_index: int,
+            rate: int,
+            channels: int,
+            width_bytes: int = 2,
+            parent: Optional[QObject] = None,
+            volume_emitter: Optional[callable] = None,
+    ):
+        super().__init__(parent)
+        import pyaudio  # local import to keep backend import-safe
+        self._pa = pyaudio.PyAudio()
+        self.device_index = int(device_index)
+        self.rate = int(rate)
+        self.channels = int(channels)
+        self.width = int(width_bytes)
+        self.frame_bytes = max(1, self.channels * self.width)
+        self.bytes_per_ms = max(1, int(self.rate * self.frame_bytes / 1000))
+
+        # choose PyAudio format from width
+        self.pa_format = self._pa.get_format_from_width(
+            self.width,
+            unsigned=(self.width == 1)
+        )
+
+        # internal buffers/flags
+        self._buffer = bytearray()
+        self._buf_lock = threading.Lock()
+        self._final = False
+        self._tail_ms = 60  # add a small silence tail to avoid clicks
+
+        # one-shot guard to avoid double stop and duplicate callbacks
+        self._stopping = False
+
+        # volume metering
+        self._volume_emitter = volume_emitter
+        self._vol_buffer = bytearray()
+        self._vol_lock = threading.Lock()
+        self._vol_timer = QTimer(self)
+        self._vol_timer.setTimerType(Qt.PreciseTimer)
+        self._vol_timer.setInterval(33)  # ~30 Hz meter
+        self._vol_timer.timeout.connect(self._emit_volume_tick)
+        self._vol_timer.start()
+
+        # open callback-based output stream
+        self._stream = self._pa.open(
+            format=self.pa_format,
+            channels=self.channels,
+            rate=self.rate,
+            output=True,
+            output_device_index=self.device_index,
+            stream_callback=self._callback,
+            frames_per_buffer=max(256, int(self.rate / 100))  # ~10 ms
+        )
+        try:
+            self._stream.start_stream()
+        except Exception:
+            pass
+
+        # finished-state watchdog: guarantees stop()+on_stopped once playback is truly done
+        self._finish_timer = QTimer(self)
+        self._finish_timer.setTimerType(Qt.PreciseTimer)
+        self._finish_timer.setInterval(15)  # fast but lightweight watchdog
+        self._finish_timer.timeout.connect(self._check_finished)
+        self._finish_timer.start()
+
+        # stop callback (set by backend)
+        self.on_stopped = None
+
+    def is_active(self) -> bool:
+        """
+        Return True if PortAudio stream is active.
+
+        :return: True if active
+        """
+        try:
+            return self._stream is not None and self._stream.is_active()
+        except Exception:
+            return False
+
+    def is_finalized(self) -> bool:
+        """
+        Return True if session was marked final.
+
+        :return: True if final
+        """
+        return bool(self._final)
+
+    def feed(self, data: bytes) -> None:
+        """
+        Append PCM bytes (already in session/device format).
+
+        :param data: bytes to append
+        """
+        if not data:
+            return
+        with self._buf_lock:
+            self._buffer.extend(data)
+        # push to volume window from the same bytes
+        self._vol_push(data)
+
+    def mark_final(self) -> None:
+        """No more data will be supplied; add a small silence tail."""
+        if not self._final:
+            pad = self.bytes_per_ms * self._tail_ms
+            pad -= (pad % self.frame_bytes)
+            if pad > 0:
+                with self._buf_lock:
+                    self._buffer.extend(self._silence(pad))
+        self._final = True
+
+    def stop(self) -> None:
+        """Stop playback and free resources. Idempotent."""
+        # ensure this executes only once even if called from multiple paths
+        if self._stopping:
+            return
+        self._stopping = True
+
+        # stop timers first to prevent re-entry
+        try:
+            if self._finish_timer:
+                self._finish_timer.stop()
+        except Exception:
+            pass
+        try:
+            if self._vol_timer:
+                self._vol_timer.stop()
+        except Exception:
+            pass
+
+        # gracefully stop PortAudio stream and close/terminate
+        try:
+            if self._stream and self._stream.is_active():
+                self._stream.stop_stream()  # drains queued audio per PortAudio docs
+        except Exception:
+            pass
+        try:
+            if self._stream:
+                self._stream.close()
+        except Exception:
+            pass
+        try:
+            if self._pa:
+                self._pa.terminate()
+        except Exception:
+            pass
+
+        # zero the meter
+        try:
+            if self._volume_emitter:
+                self._volume_emitter(0)
+        except Exception:
+            pass
+
+        self._stream = None
+        self._pa = None
+
+        cb = self.on_stopped
+        self.on_stopped = None
+        if cb:
+            try:
+                cb()
+            except Exception:
+                pass
+
+        self.deleteLater()
+
+    # ---- internal ----
+
+    def _callback(self, in_data, frame_count, time_info, status):
+        """
+        PortAudio callback: deliver frames from buffer.
+
+        :param in_data: input data (ignored)
+        :param frame_count: number of frames requested
+        :param time_info: timing info (ignored)
+        :param status: status flags (ignored)
+        :return: (data bytes, flag)
+        """
+        import pyaudio
+        need = frame_count * self.frame_bytes
+        out = b""
+        with self._buf_lock:
+            if len(self._buffer) >= need:
+                out = bytes(self._buffer[:need])
+                del self._buffer[:need]
+            elif len(self._buffer) > 0:
+                out = bytes(self._buffer)
+                self._buffer.clear()
+
+        if len(out) < need:
+            out += self._silence(need - len(out))
+
+        # meter push from what is actually written
+        self._vol_push(out)
+
+        # auto-finish: when final and nothing more to play, complete and stop()
+        if self._final and self._buffer_empty():
+            # Return paComplete and request stop on the GUI thread.
+            # PaComplete deactivates the stream after the last callback buffer is played.
+            QTimer.singleShot(0, self.stop)
+            return out, pyaudio.paComplete
+
+        return out, pyaudio.paContinue
+
+    def _check_finished(self) -> None:
+        """
+        Watchdog that runs on the Qt thread to guarantee a single, reliable stop().
+        Triggers when PortAudio deactivates the stream, or when the buffer is fully
+        drained after mark_final().
+        """
+        if self._stopping:
+            return
+
+        # If underlying PA stream is no longer active, we are done.
+        try:
+            if self._stream is not None and not self._stream.is_active():
+                self.stop()
+                return
+        except Exception:
+            # If querying state fails, assume the stream is done and stop.
+            self.stop()
+            return
+
+        # If we've been marked final and our buffer is empty, finalize proactively.
+        if self._final and self._buffer_empty():
+            self.stop()
+
+    def _buffer_empty(self) -> bool:
+        """
+        Check if internal buffer is empty.
+
+        :return: True if empty
+        """
+        with self._buf_lock:
+            return len(self._buffer) == 0
+
+    def _silence(self, n: int) -> bytes:
+        """
+        Generate n bytes of silence.
+
+        :param n: number of bytes
+        :return: bytes of silence
+        """
+        if n <= 0:
+            return b""
+        if self.width == 1:
+            return bytes([128]) * n  # silence for unsigned 8-bit
+        return b"\x00" * n
+
+    def _vol_push(self, chunk: bytes) -> None:
+        """
+        Push chunk to volume buffer and trim if needed.
+
+        :param chunk: bytes to push to volume buffer
+        """
+        if not chunk:
+            return
+        with self._vol_lock:
+            self._vol_buffer.extend(chunk)
+            max_bytes = max(1, self.bytes_per_ms * 100)  # ~100 ms window
+            if len(self._vol_buffer) > max_bytes:
+                del self._vol_buffer[:len(self._vol_buffer) - max_bytes]
+
+    def _emit_volume_tick(self) -> None:
+        """Emit volume level based on current volume buffer."""
+        if self._volume_emitter is None:
+            return
+        with self._vol_lock:
+            buf = bytes(self._vol_buffer)
+        if not buf:
+            try:
+                self._volume_emitter(0)
+            except Exception:
+                pass
+            return
+        try:
+            # decode by sample width
+            if self.width == 1:
+                arr = np.frombuffer(buf, dtype=np.uint8).astype(np.int16)
+                arr = (arr - 128).astype(np.float32) / 128.0
+            elif self.width == 2:
+                arr = np.frombuffer(buf, dtype=np.int16).astype(np.float32) / 32768.0
+            elif self.width == 4:
+                arr = np.frombuffer(buf, dtype=np.int32).astype(np.float32) / 2147483648.0
+            else:
+                arr = np.frombuffer(buf, dtype=np.int16).astype(np.float32) / 32768.0
+
+            if arr.size == 0:
+                self._volume_emitter(0)
+                return
+
+            rms = float(np.sqrt(np.mean(arr.astype(np.float64) ** 2)))
+            db = -60.0 if rms <= 1e-9 else 20.0 * float(np.log10(min(1.0, rms)))
+            db = max(-60.0, min(0.0, db))
+            volume = int(((db + 60.0) / 60.0) * 100.0)
+            self._volume_emitter(volume)
+        except Exception:
+            pass

pygpt_net/core/audio/backend/pygame/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from .pygame import PygameBackend

pygpt_net/core/audio/backend/{pygame.py → pygame/pygame.py}

@@ -6,15 +6,20 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.27 07:00:00                  #
+# Updated Date: 2025.08.31 04:00:00                  #
 # ================================================== #
 
 import time
 import wave
+import numpy as np
 from typing import List, Tuple
+from collections import deque
+from threading import Lock
 
 from PySide6.QtCore import QTimer
 
+from ..shared import f32_to_s16le, build_rt_input_delta_event
+
 class PygameBackend:
     MIN_FRAMES = 25  # minimum frames to start transcription
 
@@ -22,6 +27,8 @@ class PygameBackend:
         """
         Audio input capture core using pygame's SDL2 audio capture backend.
         Captured devices are stored as device name strings.
+
+        :param window: Window instance
         """
         self.window = window
         self.path = None
@@ -55,10 +62,14 @@ class PygameBackend:
         self.initialized = False
         self.mode = "input"  # input|control
 
+        # --- REALTIME INPUT (mic -> dispatcher) ---
+        self._rt_signals = None           # set with set_rt_signals()
+        self._rt_queue = deque()          # queue of raw float32 chunks from SDL audio thread
+        self._rt_lock = Lock()            # protects _rt_queue
+        self._is_recording = False        # suppress updates after stop
+
     def init(self):
-        """
-        Initialize the pygame audio system if not already initialized.
-        """
+        """Initialize the pygame audio system if not already initialized."""
         if not self.initialized:
             import pygame
             from pygame._sdl2 import (
@@ -110,10 +121,20 @@ class PygameBackend:
         """
         self.path = path
 
+    def set_rt_signals(self, signals) -> None:
+        """
+        Set signals object for realtime events.
+
+        :param signals: Signals object
+        """
+        self._rt_signals = signals
+
     def start(self):
         """
         Start audio recording using pygame’s SDL2 audio capture.
         Returns True if started successfully.
+
+        :return: True if started
         """
         self.init()
         # Clear previously recorded frames.
@@ -136,15 +157,23 @@ class PygameBackend:
         self.timer.timeout.connect(self._update_level)
         self.timer.start(50)  # update every 50ms
 
+        # mark recording as active after setup
+        self._is_recording = True
         return True
 
     def stop(self):
         """
         Stop audio recording.
         Returns True if stopped and audio data was saved (if path is set).
+
+        :return: True if stopped and saved
         """
         self.init()
         result = False
+
+        # immediately mark as not recording
+        self._is_recording = False
+
         if self.audio_source is not None:
             if self.timer is not None:
                 self.timer.stop()
@@ -154,6 +183,12 @@ class PygameBackend:
             self.audio_source.pause(1)
             self.audio_source = None
 
+            # Emit final input chunk marker for realtime consumers
+            try:
+                self._emit_rt_input_delta(b"", final=True)
+            except Exception:
+                pass
+
             if self.frames:
                 if self.path:
                     self.save_audio_file(self.path)
@@ -162,35 +197,48 @@ class PygameBackend:
                     print("File path is not set.")
             else:
                 print("No audio data recorded")
+
+        # reset level indicator
+        try:
+            self.reset_audio_level()
+        except Exception:
+            pass
+
         return result
 
     def has_source(self) -> bool:
         """
         Check if the audio source is available.
+
+        :return: True if audio source is available
         """
         return self.audio_source is not None
 
     def has_frames(self) -> bool:
         """
         Check if any audio frames have been recorded.
+
+        :return: True if any frames recorded
         """
         return bool(self.frames)
 
     def has_min_frames(self) -> bool:
         """
         Check if at least MIN_FRAMES audio frames have been recorded.
+
+        :return: True if at least MIN_FRAMES recorded
         """
         return len(self.frames) >= self.MIN_FRAMES
 
     def reset_audio_level(self):
-        """
-        Reset the audio level bar (if available).
-        """
+        """Reset the audio level bar (if available)."""
         self.window.controller.audio.ui.on_input_volume_change(0, self.mode)
 
     def check_audio_input(self) -> bool:
         """
         Check if a default audio input device is available using pygame.
+
+        :return: True if an audio input device is available
         """
         from pygame._sdl2 import (
             get_audio_device_names,
@@ -226,6 +274,8 @@ class PygameBackend:
     def device_changed(self, index: int):
         """
         Change the selected audio input device by its index in the devices list.
+
+        :param index: Index of the device in the devices list.
         """
         self.init()
         if 0 <= index < len(self.devices):
@@ -234,9 +284,7 @@ class PygameBackend:
             self.selected_device = None
 
     def prepare_device(self):
-        """
-        Set the current audio input device based on configuration.
-        """
+        """Set the current audio input device based on configuration."""
         self.init()
         if self.window is not None and hasattr(self.window, "core"):
             device_index = int(self.window.core.config.get('audio.input.device', 0))
@@ -251,14 +299,26 @@ class PygameBackend:
         """
         Callback function called in the audio thread.
         It receives a memoryview of audio data which is converted to bytes and appended.
+
+        :param audiodevice: The audio device instance (not used here).
+        :param audiomemoryview: MemoryView of the captured audio data.
         """
+        if not self._is_recording:
+            return
+
         # Append captured audio bytes to the frames list.
-        self.frames.append(bytes(audiomemoryview))
+        chunk = bytes(audiomemoryview)
+        self.frames.append(chunk)
+
+        # Enqueue chunk for realtime emission (processed on the Qt thread).
+        try:
+            with self._rt_lock:
+                self._rt_queue.append(chunk)
+        except Exception:
+            pass
 
     def setup_audio_input(self):
-        """
-        Create an AudioDevice with the selected device name and start recording.
-        """
+        """Create an AudioDevice with the selected device name and start recording."""
         self.init()
         from pygame._sdl2 import (
             AudioDevice,
@@ -289,16 +349,18 @@ class PygameBackend:
         Periodically called (via QTimer) to compute RMS from the last captured audio chunk
         and update the audio level bar.
         """
+        # Drain realtime queue first to keep latency low.
+        self._drain_rt_queue()
+
         if not self.frames:
             return
 
-        import numpy as np
         # Use the last captured chunk.
         last_chunk = self.frames[-1]
         try:
             # Interpret the bytes as float32 samples.
             samples = np.frombuffer(last_chunk, dtype=np.float32)
-        except Exception as e:
+        except Exception:
             return
         if samples.size == 0:
             return
@@ -329,7 +391,6 @@ class PygameBackend:
 
         :param filename: The path to the output WAV file.
         """
-        import numpy as np
         full_data = b"".join(self.frames)
         try:
             data_array = np.frombuffer(full_data, dtype=np.float32)
@@ -337,7 +398,7 @@ class PygameBackend:
             print("Error converting audio data:", e)
             return
         # Convert float32 values in the range -1.0 ... 1.0 to PCM int16.
-        int_data = (data_array * 32767).astype(np.int16)
+        int_data = (np.clip(data_array, -1.0, 1.0) * 32767.0).astype(np.int16)
         new_data = int_data.tobytes()
         with wave.open(filename, 'wb') as wf:
             wf.setnchannels(self.channels)
@@ -490,11 +551,61 @@ class PygameBackend:
     def get_default_input_device(self) -> tuple:
         """
         Retrieve the default input device using PyAudio.
+
+        :return: (index, name)
         """
         return 0, "Default Input Device"
 
     def get_default_output_device(self) -> tuple:
         """
         Retrieve the default output device using PyAudio.
+
+        :return: (index, name)
         """
-        return 0, "Default Output Device"
+        return 0, "Default Output Device"
+
+    # --------------------
+    # REALTIME INPUT HELPERS
+    # --------------------
+    def _emit_rt_input_delta(self, data: bytes, final: bool) -> None:
+        """
+        Emit RT_INPUT_AUDIO_DELTA with a provider-agnostic payload.
+        Standardizes to PCM16, little-endian, and includes rate/channels.
+
+        :param data: PCM16LE audio bytes
+        :param final: True if this is the final chunk
+        """
+        if not self._rt_signals:
+            return
+        try:
+            event = build_rt_input_delta_event(
+                rate=int(self.rate),
+                channels=int(self.channels),
+                data=data or b"",
+                final=bool(final),
+            )
+            # Ensure emission on the Qt thread
+            QTimer.singleShot(0, lambda: self._rt_signals.response.emit(event))
+        except Exception:
+            pass
+
+    def _drain_rt_queue(self) -> None:
+        """
+        Drain queued float32 chunks from the audio thread, convert to PCM16,
+        and emit a single realtime delta event.
+        """
+        if not self._rt_signals:
+            # nothing to emit
+            with self._rt_lock:
+                self._rt_queue.clear()
+            return
+
+        with self._rt_lock:
+            if not self._rt_queue:
+                return
+            raw = b"".join(self._rt_queue)
+            self._rt_queue.clear()
+
+        s16 = f32_to_s16le(raw)
+        if s16:
+            self._emit_rt_input_delta(s16, final=False)

pygpt_net/core/audio/backend/shared/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+# Shared helpers for audio backends
+
+from .rt import (
+    build_rt_input_delta_event,
+    build_output_volume_event,
+)
+from .conversions import (
+    qaudio_dtype,
+    qaudio_norm_factor,
+    qaudio_to_s16le,
+    pyaudio_to_s16le,
+    f32_to_s16le,
+    convert_s16_pcm,
+)
+from .envelope import compute_envelope_from_file
+
+__all__ = [
+    "build_rt_input_delta_event",
+    "build_output_volume_event",
+    "qaudio_dtype",
+    "qaudio_norm_factor",
+    "qaudio_to_s16le",
+    "pyaudio_to_s16le",
+    "f32_to_s16le",
+    "convert_s16_pcm",
+    "compute_envelope_from_file",
+]