pygpt-net 2.6.30__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/core/audio/backend/shared/conversions.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import numpy as np
+import audioop
+
+def qaudio_dtype(sample_format):
+    """
+    Map QAudioFormat.SampleFormat to numpy dtype.
+
+    Raises ValueError if the format is unsupported.
+
+    :param sample_format: QAudioFormat.SampleFormat
+    :return: numpy dtype
+    """
+    try:
+        from PySide6.QtMultimedia import QAudioFormat
+    except Exception:
+        raise
+
+    if sample_format == QAudioFormat.SampleFormat.UInt8:
+        return np.uint8
+    elif sample_format == QAudioFormat.SampleFormat.Int16:
+        return np.int16
+    elif sample_format == QAudioFormat.SampleFormat.Int32:
+        return np.int32
+    elif sample_format == QAudioFormat.SampleFormat.Float:
+        return np.float32
+    raise ValueError("Unsupported sample format")
+
+def qaudio_norm_factor(sample_format):
+    """
+    Normalization factor for QAudioFormat.SampleFormat.
+
+    Raises ValueError if the format is unsupported.
+
+    :param sample_format: QAudioFormat.SampleFormat
+    :return: normalization factor (float)
+    """
+    try:
+        from PySide6.QtMultimedia import QAudioFormat
+    except Exception:
+        raise
+
+    if sample_format == QAudioFormat.SampleFormat.UInt8:
+        return 255.0
+    elif sample_format == QAudioFormat.SampleFormat.Int16:
+        return 32768.0
+    elif sample_format == QAudioFormat.SampleFormat.Int32:
+        return float(2 ** 31)
+    elif sample_format == QAudioFormat.SampleFormat.Float:
+        return 1.0
+    raise ValueError("Unsupported sample format")
+
+def qaudio_to_s16le(raw: bytes, sample_format) -> bytes:
+    """
+    Convert arbitrary QAudioFormat sample format to PCM16 little-endian.
+
+    :param raw: input byte buffer
+    :param sample_format: QAudioFormat.SampleFormat
+    :return: converted byte buffer in PCM16 little-endian
+    """
+    if not raw:
+        return b""
+    try:
+        from PySide6.QtMultimedia import QAudioFormat
+    except Exception:
+        return raw
+
+    if sample_format == QAudioFormat.SampleFormat.Int16:
+        return raw
+    elif sample_format == QAudioFormat.SampleFormat.UInt8:
+        arr = np.frombuffer(raw, dtype=np.uint8).astype(np.int16)
+        arr = (arr - 128) << 8
+        return arr.tobytes()
+    elif sample_format == QAudioFormat.SampleFormat.Int32:
+        arr = np.frombuffer(raw, dtype=np.int32)
+        arr = (arr >> 16).astype(np.int16)
+        return arr.tobytes()
+    elif sample_format == QAudioFormat.SampleFormat.Float:
+        arr = np.frombuffer(raw, dtype=np.float32)
+        arr = np.clip(arr, -1.0, 1.0)
+        arr = (arr * 32767.0).astype(np.int16)
+        return arr.tobytes()
+    return raw
+
+def pyaudio_to_s16le(raw: bytes, fmt, pa_instance=None) -> bytes:
+    """
+    Convert PyAudio input buffer to PCM16 little-endian without changing
+    sample rate or channel count.
+
+    :param raw: input byte buffer
+    :param fmt: PyAudio format (e.g., pyaudio.paInt16)
+    :param pa_instance: Optional PyAudio instance for sample size queries
+    :return: converted byte buffer in PCM16 little-endian
+    """
+    if not raw:
+        return b""
+    try:
+        import pyaudio
+    except Exception:
+        return raw
+
+    try:
+        if fmt == pyaudio.paInt16:
+            return raw
+        elif fmt == pyaudio.paUInt8:
+            arr = np.frombuffer(raw, dtype=np.uint8).astype(np.int16)
+            arr = (arr - 128) << 8
+            return arr.tobytes()
+        elif fmt == pyaudio.paInt8:
+            arr = np.frombuffer(raw, dtype=np.int8).astype(np.int16)
+            arr = (arr.astype(np.int16) << 8)
+            return arr.tobytes()
+        elif fmt == pyaudio.paFloat32:
+            arr = np.frombuffer(raw, dtype=np.float32)
+            arr = np.clip(arr, -1.0, 1.0)
+            arr = (arr * 32767.0).astype(np.int16)
+            return arr.tobytes()
+        else:
+            try:
+                sw = pa_instance.get_sample_size(fmt) if pa_instance is not None else 2
+                return audioop.lin2lin(raw, sw, 2)
+            except Exception:
+                return raw
+    except Exception:
+        return raw
+
+def f32_to_s16le(raw: bytes) -> bytes:
+    """
+    Convert float32 little-endian PCM to int16 little-endian PCM.
+
+    :param raw: input byte buffer in float32
+    :return: converted byte buffer in int16
+    """
+    if not raw:
+        return b""
+    try:
+        arr = np.frombuffer(raw, dtype=np.float32)
+        arr = np.clip(arr, -1.0, 1.0)
+        s16 = (arr * 32767.0).astype(np.int16)
+        return s16.tobytes()
+    except Exception:
+        return b""
+
+def convert_s16_pcm(
+    data: bytes,
+    in_rate: int,
+    in_channels: int,
+    out_rate: int,
+    out_channels: int,
+    out_width: int = 2,
+    out_format: str = "s16"  # "s16" | "u8" | "f32"
+) -> bytes:
+    """
+    Minimal PCM converter to target format:
+    - assumes input is S16LE,
+    - converts channels (mono<->stereo) and sample rate,
+    - converts width if needed,
+    - applies bias for u8 or float conversion if requested.
+
+    :param data: input byte buffer in S16LE
+    :param in_rate: input sample rate
+    :param in_channels: input channel count
+    :param out_rate: output sample rate
+    :param out_channels: output channel count
+    :param out_width: output sample width in bytes (1, 2, or 4)
+    :param out_format: output format ("s16", "u8", or "f32")
+    :return: converted byte buffer
+    """
+    if not data:
+        return b""
+    try:
+        src = data
+
+        # channels
+        if in_channels != out_channels:
+            if in_channels == 2 and out_channels == 1:
+                src = audioop.tomono(src, 2, 0.5, 0.5)
+            elif in_channels == 1 and out_channels == 2:
+                src = audioop.tostereo(src, 2, 1.0, 1.0)
+            else:
+                mid = audioop.tomono(src, 2, 0.5, 0.5) if in_channels > 1 else src
+                src = audioop.tostereo(mid, 2, 1.0, 1.0) if out_channels == 2 else mid
+
+        # sample rate
+        if in_rate != out_rate:
+            src, _ = audioop.ratecv(src, 2, out_channels, in_rate, out_rate, None)
+
+        # sample width (Int16 -> other widths if needed)
+        if out_width != 2:
+            src = audioop.lin2lin(src, 2, out_width)
+
+        # sample format nuances
+        if out_format == "u8" and out_width == 1:
+            src = audioop.bias(src, 1, 128)  # center at 0x80
+        elif out_format == "f32" and out_width == 4:
+            arr = np.frombuffer(src, dtype=np.int16).astype(np.float32) / 32768.0
+            src = arr.tobytes()
+
+        return src
+    except Exception:
+        return data

pygpt_net/core/audio/backend/shared/envelope.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 04:00:00                  #
+# ================================================== #
+
+import numpy as np
+from pydub import AudioSegment
+
+def compute_envelope_from_file(audio_file: str, chunk_ms: int = 100) -> list:
+    """
+    Calculate the volume envelope of an audio file (0-100 per chunk).
+
+    :param audio_file: Path to the audio file
+    :param chunk_ms: Chunk size in milliseconds
+    :return: List of volume levels (0-100) per chunk
+    """
+    audio = AudioSegment.from_file(audio_file)
+    max_amplitude = 32767.0
+    envelope = []
+
+    for ms in range(0, len(audio), chunk_ms):
+        chunk = audio[ms:ms + chunk_ms]
+        rms = float(chunk.rms) if chunk.rms else 0.0
+        if rms > 0.0:
+            db = 20.0 * np.log10(max(1e-12, rms / max_amplitude))
+        else:
+            db = -60.0
+        db = max(-60.0, min(0.0, db))
+        volume = ((db + 60.0) / 60.0) * 100.0
+        envelope.append(volume)
+
+    return envelope

pygpt_net/core/audio/backend/shared/player.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 04:00:00                  #
+# ================================================== #
+
+from typing import Optional, Callable
+
+import os
+from PySide6.QtCore import QObject, QTimer, QUrl
+from PySide6.QtMultimedia import QMediaPlayer, QAudioOutput
+
+from ..shared import compute_envelope_from_file
+
+class NativePlayer(QObject):
+    """
+    Thin wrapper around QtMultimedia audio playback with level metering.
+    """
+    def __init__(self, window=None, chunk_ms: int = 10):
+        super().__init__(window)
+        self.window = window
+        self.chunk_ms = int(chunk_ms)
+        self.audio_output: Optional[QAudioOutput] = None
+        self.player: Optional[QMediaPlayer] = None
+        self.playback_timer: Optional[QTimer] = None
+        self.volume_timer: Optional[QTimer] = None
+        self.envelope = []
+
+    def stop_timers(self):
+        """Stop playback timers."""
+        if self.playback_timer is not None:
+            self.playback_timer.stop()
+            self.playback_timer = None
+        if self.volume_timer is not None:
+            self.volume_timer.stop()
+            self.volume_timer = None
+
+    def stop(self, signals=None):
+        """
+        Stop playback and timers.
+
+        :param signals: Signals to emit on stop
+        """
+        if self.player is not None:
+            try:
+                self.player.stop()
+            except Exception:
+                pass
+        self.stop_timers()
+        if signals is not None:
+            try:
+                signals.volume_changed.emit(0)
+            except Exception:
+                pass
+
+    def update_volume(self, signals=None):
+        """
+        Update the volume based on the current position in the audio file.
+
+        :param signals: Signals to emit volume changes
+        """
+        if not self.player:
+            return
+        pos = self.player.position()
+        index = int(pos / self.chunk_ms)
+        volume = self.envelope[index] if index < len(self.envelope) else 0
+        if signals is not None:
+            signals.volume_changed.emit(volume)
+
+    def play_after(
+        self,
+        audio_file: str,
+        event_name: str,
+        stopped: Callable[[], bool],
+        signals=None,
+        auto_convert_to_wav: bool = False,
+        select_output_device: Optional[Callable[[], object]] = None,
+    ):
+        """
+        Start audio playback using QtMultimedia with periodic volume updates.
+
+        :param audio_file: Path to audio file
+        :param event_name: Event name to emit on playback start
+        :param stopped: Callable returning True when playback should stop
+        :param signals: Signals to emit on playback
+        :param auto_convert_to_wav: auto convert mp3 to wav if True
+        :param select_output_device: callable returning QAudioDevice for output
+        """
+        self.audio_output = QAudioOutput()
+        self.audio_output.setVolume(1.0)
+
+        if callable(select_output_device):
+            try:
+                self.audio_output.setDevice(select_output_device())
+            except Exception:
+                pass
+
+        if auto_convert_to_wav and audio_file.lower().endswith('.mp3'):
+            tmp_dir = self.window.core.audio.get_cache_dir()
+            base_name = os.path.splitext(os.path.basename(audio_file))[0]
+            dst_file = os.path.join(tmp_dir, "_" + base_name + ".wav")
+            wav_file = self.window.core.audio.mp3_to_wav(audio_file, dst_file)
+            if wav_file:
+                audio_file = wav_file
+
+        def check_stop():
+            if stopped():
+                self.stop(signals=signals)
+            else:
+                if self.player:
+                    if self.player.playbackState() == QMediaPlayer.StoppedState:
+                        self.stop(signals=signals)
+
+        self.envelope = compute_envelope_from_file(audio_file, chunk_ms=self.chunk_ms)
+        self.player = QMediaPlayer()
+        self.player.setAudioOutput(self.audio_output)
+        self.player.setSource(QUrl.fromLocalFile(audio_file))
+        self.player.play()
+
+        self.playback_timer = QTimer()
+        self.playback_timer.setInterval(100)
+        self.playback_timer.timeout.connect(check_stop)
+
+        self.volume_timer = QTimer(self)
+        self.volume_timer.setInterval(10)
+        self.volume_timer.timeout.connect(lambda: self.update_volume(signals))
+
+        self.playback_timer.start()
+        self.volume_timer.start()
+        if signals is not None:
+            signals.volume_changed.emit(0)
+            signals.playback.emit(event_name)

pygpt_net/core/audio/backend/shared/rt.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from pygpt_net.core.events import RealtimeEvent
+
+def build_rt_input_delta_event(
+        rate: int,
+        channels: int,
+        data: bytes,
+        final: bool
+) -> RealtimeEvent:
+    """
+    Build RT_INPUT_AUDIO_DELTA event with a provider-agnostic payload.
+
+    :param rate: Sample rate (e.g., 16000)
+    :param channels: Number of channels (1 for mono, 2 for stereo)
+    :param data: Audio data bytes
+    :param final: Whether this is the final chunk
+    :return: RealtimeEvent instance
+    """
+    return RealtimeEvent(
+        RealtimeEvent.RT_INPUT_AUDIO_DELTA,
+        {
+            "payload": {
+                "data": data or b"",
+                "mime": "audio/pcm",
+                "rate": int(rate),
+                "channels": int(channels),
+                "final": bool(final),
+            }
+        }
+    )
+
+def build_output_volume_event(value: int) -> RealtimeEvent:
+    """
+    Build RT_OUTPUT_AUDIO_VOLUME_CHANGED event.
+
+    :param value: Volume level (0-100)
+    :return: RealtimeEvent instance
+    """
+    return RealtimeEvent(
+        RealtimeEvent.RT_OUTPUT_AUDIO_VOLUME_CHANGED,
+        {"volume": int(value)}
+    )

pygpt_net/core/audio/capture.py

@@ -42,6 +42,11 @@ class Capture:
             backend = "native"
         return self.backends[backend]
 
+    def setup(self):
+        """Setup audio input backend"""
+        for b in self.backends.values():
+            b.set_rt_signals(self.window.controller.realtime.signals)
+
     def get_default_input_device(self) -> Tuple[int, str]:
         """
         Get default input device

pygpt_net/core/audio/output.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.07 03:00:00                  #
+# Updated Date: 2025.08.31 23:00:00                  #
 # ================================================== #
 
 from typing import List, Tuple
@@ -42,6 +42,10 @@ class Output:
             backend = "native"
         return self.backends[backend]
 
+    def setup(self):
+        """Setup audio output backend"""
+        pass
+
     def play(
             self,
             audio_file: str,
@@ -88,4 +92,11 @@ class Output:
 
         :return: (id, name)
         """
-        return self.get_backend().get_default_output_device()
+        return self.get_backend().get_default_output_device()
+
+    def handle_realtime(self, payload, signals):
+        """
+        Handle real-time audio playback
+        """
+        #self.get_backend().set_signals(signals)
+        self.get_backend().handle_realtime(payload)

pygpt_net/core/audio/whisper.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 18:00:00                  #
+# Updated Date: 2025.08.29 18:00:00                  #
 # ================================================== #
 
 from typing import List
@@ -21,10 +21,14 @@ class Whisper:
         self.window = window
         self.voices = [
             "alloy",
+            "ash",
+            "ballad",
+            "coral",
             "echo",
             "fable",
-            "onyx",
             "nova",
+            "onyx",
+            "sage",
             "shimmer",
         ]
 

pygpt_net/core/bridge/bridge.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.28 09:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 import time
@@ -268,6 +268,7 @@ class Bridge:
         worker = BridgeWorker()
         worker.window = self.window
         worker.signals.response.connect(self.window.controller.kernel.listener)
+        worker.rt_signals = self.window.controller.realtime.signals  # Realtime signals
         return worker
 
     def apply_rate_limit(self):

pygpt_net/core/bridge/worker.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.28 20:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import QObject, Signal, QRunnable, Slot
@@ -33,6 +33,7 @@ class BridgeWorker(QRunnable):
     def __init__(self, *args, **kwargs):
         super().__init__()
         self.signals = BridgeSignals()
+        self.rt_signals = None
         self.args = args
         self.kwargs = kwargs
         self.window = None
@@ -116,12 +117,14 @@ class BridgeWorker(QRunnable):
                     result = self.window.core.api.google.call(
                         context=self.context,
                         extra=self.extra,
+                        rt_signals=self.rt_signals,
                     )
                 elif sdk == "openai":
                     # print("Using OpenAI SDK")
                     result = self.window.core.api.openai.call(
                         context=self.context,
                         extra=self.extra,
+                        rt_signals=self.rt_signals,
                     )
         except Exception as e:
             if self.signals:

pygpt_net/core/dispatcher/dispatcher.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.23 15:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 from typing import List, Tuple
@@ -17,6 +17,7 @@ from pygpt_net.core.events import (
     ControlEvent,
     AppEvent,
     RenderEvent,
+    RealtimeEvent,
 )
 
 
@@ -71,6 +72,14 @@ class Dispatcher:
 
         handled = False
 
+        # realtime first, if it's a realtime event
+        if isinstance(event, RealtimeEvent):
+            controller.realtime.handle(event)
+            if log_event:
+                debug.info(f"[event] Dispatch end: {event.full_name} ({event.call_id})")
+            self.call_id += 1
+            return [], event
+
         # kernel
         if isinstance(event, KernelEvent):
             kernel_auto = (KernelEvent.INIT, KernelEvent.RESTART, KernelEvent.STOP, KernelEvent.TERMINATE)
@@ -96,20 +105,47 @@ class Dispatcher:
         if handled:
             return [], event
 
+        # realtime
+        controller.realtime.handle(event)
+        if event.stop:
+            if log_event:
+                debug.info(f"[event] Skipping... (stopped): {event.name}")
+            return [], event
+
         # agents
         controller.agent.handle(event)
+        if event.stop:
+            if log_event:
+                debug.info(f"[event] Skipping... (stopped): {event.name}")
+            return [], event
 
         # ctx
         controller.ctx.handle(event)
+        if event.stop:
+            if log_event:
+                debug.info(f"[event] Skipping... (stopped): {event.name}")
+            return [], event
 
         # model
         controller.model.handle(event)
+        if event.stop:
+            if log_event:
+                debug.info(f"[event] Skipping... (stopped): {event.name}")
+            return [], event
 
         # idx
         controller.idx.handle(event)
+        if event.stop:
+            if log_event:
+                debug.info(f"[event] Skipping... (stopped): {event.name}")
+            return [], event
 
         # ui
         controller.ui.handle(event)
+        if event.stop:
+            if log_event:
+                debug.info(f"[event] Skipping... (stopped): {event.name}")
+            return [], event
 
         # access
         if isinstance(event, (ControlEvent, AppEvent)):

pygpt_net/core/events/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 03:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 from .base import BaseEvent
@@ -14,4 +14,5 @@ from .app import AppEvent
 from .control import ControlEvent
 from .event import Event
 from .kernel import KernelEvent
+from .realtime import RealtimeEvent
 from .render import RenderEvent