pygpt-net 2.6.29__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/core/audio/backend/shared/conversions.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import numpy as np
+import audioop
+
+def qaudio_dtype(sample_format):
+    """
+    Map QAudioFormat.SampleFormat to numpy dtype.
+
+    Raises ValueError if the format is unsupported.
+
+    :param sample_format: QAudioFormat.SampleFormat
+    :return: numpy dtype
+    """
+    try:
+        from PySide6.QtMultimedia import QAudioFormat
+    except Exception:
+        raise
+
+    if sample_format == QAudioFormat.SampleFormat.UInt8:
+        return np.uint8
+    elif sample_format == QAudioFormat.SampleFormat.Int16:
+        return np.int16
+    elif sample_format == QAudioFormat.SampleFormat.Int32:
+        return np.int32
+    elif sample_format == QAudioFormat.SampleFormat.Float:
+        return np.float32
+    raise ValueError("Unsupported sample format")
+
+def qaudio_norm_factor(sample_format):
+    """
+    Normalization factor for QAudioFormat.SampleFormat.
+
+    Raises ValueError if the format is unsupported.
+
+    :param sample_format: QAudioFormat.SampleFormat
+    :return: normalization factor (float)
+    """
+    try:
+        from PySide6.QtMultimedia import QAudioFormat
+    except Exception:
+        raise
+
+    if sample_format == QAudioFormat.SampleFormat.UInt8:
+        return 255.0
+    elif sample_format == QAudioFormat.SampleFormat.Int16:
+        return 32768.0
+    elif sample_format == QAudioFormat.SampleFormat.Int32:
+        return float(2 ** 31)
+    elif sample_format == QAudioFormat.SampleFormat.Float:
+        return 1.0
+    raise ValueError("Unsupported sample format")
+
+def qaudio_to_s16le(raw: bytes, sample_format) -> bytes:
+    """
+    Convert arbitrary QAudioFormat sample format to PCM16 little-endian.
+
+    :param raw: input byte buffer
+    :param sample_format: QAudioFormat.SampleFormat
+    :return: converted byte buffer in PCM16 little-endian
+    """
+    if not raw:
+        return b""
+    try:
+        from PySide6.QtMultimedia import QAudioFormat
+    except Exception:
+        return raw
+
+    if sample_format == QAudioFormat.SampleFormat.Int16:
+        return raw
+    elif sample_format == QAudioFormat.SampleFormat.UInt8:
+        arr = np.frombuffer(raw, dtype=np.uint8).astype(np.int16)
+        arr = (arr - 128) << 8
+        return arr.tobytes()
+    elif sample_format == QAudioFormat.SampleFormat.Int32:
+        arr = np.frombuffer(raw, dtype=np.int32)
+        arr = (arr >> 16).astype(np.int16)
+        return arr.tobytes()
+    elif sample_format == QAudioFormat.SampleFormat.Float:
+        arr = np.frombuffer(raw, dtype=np.float32)
+        arr = np.clip(arr, -1.0, 1.0)
+        arr = (arr * 32767.0).astype(np.int16)
+        return arr.tobytes()
+    return raw
+
+def pyaudio_to_s16le(raw: bytes, fmt, pa_instance=None) -> bytes:
+    """
+    Convert PyAudio input buffer to PCM16 little-endian without changing
+    sample rate or channel count.
+
+    :param raw: input byte buffer
+    :param fmt: PyAudio format (e.g., pyaudio.paInt16)
+    :param pa_instance: Optional PyAudio instance for sample size queries
+    :return: converted byte buffer in PCM16 little-endian
+    """
+    if not raw:
+        return b""
+    try:
+        import pyaudio
+    except Exception:
+        return raw
+
+    try:
+        if fmt == pyaudio.paInt16:
+            return raw
+        elif fmt == pyaudio.paUInt8:
+            arr = np.frombuffer(raw, dtype=np.uint8).astype(np.int16)
+            arr = (arr - 128) << 8
+            return arr.tobytes()
+        elif fmt == pyaudio.paInt8:
+            arr = np.frombuffer(raw, dtype=np.int8).astype(np.int16)
+            arr = (arr.astype(np.int16) << 8)
+            return arr.tobytes()
+        elif fmt == pyaudio.paFloat32:
+            arr = np.frombuffer(raw, dtype=np.float32)
+            arr = np.clip(arr, -1.0, 1.0)
+            arr = (arr * 32767.0).astype(np.int16)
+            return arr.tobytes()
+        else:
+            try:
+                sw = pa_instance.get_sample_size(fmt) if pa_instance is not None else 2
+                return audioop.lin2lin(raw, sw, 2)
+            except Exception:
+                return raw
+    except Exception:
+        return raw
+
+def f32_to_s16le(raw: bytes) -> bytes:
+    """
+    Convert float32 little-endian PCM to int16 little-endian PCM.
+
+    :param raw: input byte buffer in float32
+    :return: converted byte buffer in int16
+    """
+    if not raw:
+        return b""
+    try:
+        arr = np.frombuffer(raw, dtype=np.float32)
+        arr = np.clip(arr, -1.0, 1.0)
+        s16 = (arr * 32767.0).astype(np.int16)
+        return s16.tobytes()
+    except Exception:
+        return b""
+
+def convert_s16_pcm(
+    data: bytes,
+    in_rate: int,
+    in_channels: int,
+    out_rate: int,
+    out_channels: int,
+    out_width: int = 2,
+    out_format: str = "s16"  # "s16" | "u8" | "f32"
+) -> bytes:
+    """
+    Minimal PCM converter to target format:
+    - assumes input is S16LE,
+    - converts channels (mono<->stereo) and sample rate,
+    - converts width if needed,
+    - applies bias for u8 or float conversion if requested.
+
+    :param data: input byte buffer in S16LE
+    :param in_rate: input sample rate
+    :param in_channels: input channel count
+    :param out_rate: output sample rate
+    :param out_channels: output channel count
+    :param out_width: output sample width in bytes (1, 2, or 4)
+    :param out_format: output format ("s16", "u8", or "f32")
+    :return: converted byte buffer
+    """
+    if not data:
+        return b""
+    try:
+        src = data
+
+        # channels
+        if in_channels != out_channels:
+            if in_channels == 2 and out_channels == 1:
+                src = audioop.tomono(src, 2, 0.5, 0.5)
+            elif in_channels == 1 and out_channels == 2:
+                src = audioop.tostereo(src, 2, 1.0, 1.0)
+            else:
+                mid = audioop.tomono(src, 2, 0.5, 0.5) if in_channels > 1 else src
+                src = audioop.tostereo(mid, 2, 1.0, 1.0) if out_channels == 2 else mid
+
+        # sample rate
+        if in_rate != out_rate:
+            src, _ = audioop.ratecv(src, 2, out_channels, in_rate, out_rate, None)
+
+        # sample width (Int16 -> other widths if needed)
+        if out_width != 2:
+            src = audioop.lin2lin(src, 2, out_width)
+
+        # sample format nuances
+        if out_format == "u8" and out_width == 1:
+            src = audioop.bias(src, 1, 128)  # center at 0x80
+        elif out_format == "f32" and out_width == 4:
+            arr = np.frombuffer(src, dtype=np.int16).astype(np.float32) / 32768.0
+            src = arr.tobytes()
+
+        return src
+    except Exception:
+        return data

pygpt_net/core/audio/backend/shared/envelope.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 04:00:00                  #
+# ================================================== #
+
+import numpy as np
+from pydub import AudioSegment
+
+def compute_envelope_from_file(audio_file: str, chunk_ms: int = 100) -> list:
+    """
+    Calculate the volume envelope of an audio file (0-100 per chunk).
+
+    :param audio_file: Path to the audio file
+    :param chunk_ms: Chunk size in milliseconds
+    :return: List of volume levels (0-100) per chunk
+    """
+    audio = AudioSegment.from_file(audio_file)
+    max_amplitude = 32767.0
+    envelope = []
+
+    for ms in range(0, len(audio), chunk_ms):
+        chunk = audio[ms:ms + chunk_ms]
+        rms = float(chunk.rms) if chunk.rms else 0.0
+        if rms > 0.0:
+            db = 20.0 * np.log10(max(1e-12, rms / max_amplitude))
+        else:
+            db = -60.0
+        db = max(-60.0, min(0.0, db))
+        volume = ((db + 60.0) / 60.0) * 100.0
+        envelope.append(volume)
+
+    return envelope

pygpt_net/core/audio/backend/shared/player.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 04:00:00                  #
+# ================================================== #
+
+from typing import Optional, Callable
+
+import os
+from PySide6.QtCore import QObject, QTimer, QUrl
+from PySide6.QtMultimedia import QMediaPlayer, QAudioOutput
+
+from ..shared import compute_envelope_from_file
+
+class NativePlayer(QObject):
+    """
+    Thin wrapper around QtMultimedia audio playback with level metering.
+    """
+    def __init__(self, window=None, chunk_ms: int = 10):
+        super().__init__(window)
+        self.window = window
+        self.chunk_ms = int(chunk_ms)
+        self.audio_output: Optional[QAudioOutput] = None
+        self.player: Optional[QMediaPlayer] = None
+        self.playback_timer: Optional[QTimer] = None
+        self.volume_timer: Optional[QTimer] = None
+        self.envelope = []
+
+    def stop_timers(self):
+        """Stop playback timers."""
+        if self.playback_timer is not None:
+            self.playback_timer.stop()
+            self.playback_timer = None
+        if self.volume_timer is not None:
+            self.volume_timer.stop()
+            self.volume_timer = None
+
+    def stop(self, signals=None):
+        """
+        Stop playback and timers.
+
+        :param signals: Signals to emit on stop
+        """
+        if self.player is not None:
+            try:
+                self.player.stop()
+            except Exception:
+                pass
+        self.stop_timers()
+        if signals is not None:
+            try:
+                signals.volume_changed.emit(0)
+            except Exception:
+                pass
+
+    def update_volume(self, signals=None):
+        """
+        Update the volume based on the current position in the audio file.
+
+        :param signals: Signals to emit volume changes
+        """
+        if not self.player:
+            return
+        pos = self.player.position()
+        index = int(pos / self.chunk_ms)
+        volume = self.envelope[index] if index < len(self.envelope) else 0
+        if signals is not None:
+            signals.volume_changed.emit(volume)
+
+    def play_after(
+        self,
+        audio_file: str,
+        event_name: str,
+        stopped: Callable[[], bool],
+        signals=None,
+        auto_convert_to_wav: bool = False,
+        select_output_device: Optional[Callable[[], object]] = None,
+    ):
+        """
+        Start audio playback using QtMultimedia with periodic volume updates.
+
+        :param audio_file: Path to audio file
+        :param event_name: Event name to emit on playback start
+        :param stopped: Callable returning True when playback should stop
+        :param signals: Signals to emit on playback
+        :param auto_convert_to_wav: auto convert mp3 to wav if True
+        :param select_output_device: callable returning QAudioDevice for output
+        """
+        self.audio_output = QAudioOutput()
+        self.audio_output.setVolume(1.0)
+
+        if callable(select_output_device):
+            try:
+                self.audio_output.setDevice(select_output_device())
+            except Exception:
+                pass
+
+        if auto_convert_to_wav and audio_file.lower().endswith('.mp3'):
+            tmp_dir = self.window.core.audio.get_cache_dir()
+            base_name = os.path.splitext(os.path.basename(audio_file))[0]
+            dst_file = os.path.join(tmp_dir, "_" + base_name + ".wav")
+            wav_file = self.window.core.audio.mp3_to_wav(audio_file, dst_file)
+            if wav_file:
+                audio_file = wav_file
+
+        def check_stop():
+            if stopped():
+                self.stop(signals=signals)
+            else:
+                if self.player:
+                    if self.player.playbackState() == QMediaPlayer.StoppedState:
+                        self.stop(signals=signals)
+
+        self.envelope = compute_envelope_from_file(audio_file, chunk_ms=self.chunk_ms)
+        self.player = QMediaPlayer()
+        self.player.setAudioOutput(self.audio_output)
+        self.player.setSource(QUrl.fromLocalFile(audio_file))
+        self.player.play()
+
+        self.playback_timer = QTimer()
+        self.playback_timer.setInterval(100)
+        self.playback_timer.timeout.connect(check_stop)
+
+        self.volume_timer = QTimer(self)
+        self.volume_timer.setInterval(10)
+        self.volume_timer.timeout.connect(lambda: self.update_volume(signals))
+
+        self.playback_timer.start()
+        self.volume_timer.start()
+        if signals is not None:
+            signals.volume_changed.emit(0)
+            signals.playback.emit(event_name)

pygpt_net/core/audio/backend/shared/rt.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from pygpt_net.core.events import RealtimeEvent
+
+def build_rt_input_delta_event(
+        rate: int,
+        channels: int,
+        data: bytes,
+        final: bool
+) -> RealtimeEvent:
+    """
+    Build RT_INPUT_AUDIO_DELTA event with a provider-agnostic payload.
+
+    :param rate: Sample rate (e.g., 16000)
+    :param channels: Number of channels (1 for mono, 2 for stereo)
+    :param data: Audio data bytes
+    :param final: Whether this is the final chunk
+    :return: RealtimeEvent instance
+    """
+    return RealtimeEvent(
+        RealtimeEvent.RT_INPUT_AUDIO_DELTA,
+        {
+            "payload": {
+                "data": data or b"",
+                "mime": "audio/pcm",
+                "rate": int(rate),
+                "channels": int(channels),
+                "final": bool(final),
+            }
+        }
+    )
+
+def build_output_volume_event(value: int) -> RealtimeEvent:
+    """
+    Build RT_OUTPUT_AUDIO_VOLUME_CHANGED event.
+
+    :param value: Volume level (0-100)
+    :return: RealtimeEvent instance
+    """
+    return RealtimeEvent(
+        RealtimeEvent.RT_OUTPUT_AUDIO_VOLUME_CHANGED,
+        {"volume": int(value)}
+    )

pygpt_net/core/audio/capture.py

@@ -42,6 +42,11 @@ class Capture:
             backend = "native"
         return self.backends[backend]
 
+    def setup(self):
+        """Setup audio input backend"""
+        for b in self.backends.values():
+            b.set_rt_signals(self.window.controller.realtime.signals)
+
     def get_default_input_device(self) -> Tuple[int, str]:
         """
         Get default input device

pygpt_net/core/audio/output.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.07 03:00:00                  #
+# Updated Date: 2025.08.31 23:00:00                  #
 # ================================================== #
 
 from typing import List, Tuple
@@ -42,6 +42,10 @@ class Output:
             backend = "native"
         return self.backends[backend]
 
+    def setup(self):
+        """Setup audio output backend"""
+        pass
+
     def play(
             self,
             audio_file: str,
@@ -88,4 +92,11 @@ class Output:
 
         :return: (id, name)
         """
-        return self.get_backend().get_default_output_device()
+        return self.get_backend().get_default_output_device()
+
+    def handle_realtime(self, payload, signals):
+        """
+        Handle real-time audio playback
+        """
+        #self.get_backend().set_signals(signals)
+        self.get_backend().handle_realtime(payload)

pygpt_net/core/audio/whisper.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 18:00:00                  #
+# Updated Date: 2025.08.29 18:00:00                  #
 # ================================================== #
 
 from typing import List
@@ -21,10 +21,14 @@ class Whisper:
         self.window = window
         self.voices = [
             "alloy",
+            "ash",
+            "ballad",
+            "coral",
             "echo",
             "fable",
-            "onyx",
             "nova",
+            "onyx",
+            "sage",
             "shimmer",
         ]
 

pygpt_net/core/bridge/bridge.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.15 23:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 import time
@@ -59,7 +59,7 @@ class Bridge:
         if self.window.controller.kernel.stopped():
             return False
 
-        allowed_model_change = MODE_VISION
+        allowed_model_change = [MODE_CHAT]
         is_virtual = False
         force_sync = False
 
@@ -254,7 +254,7 @@ class Bridge:
                         context.mode = MODE_RESEARCH
 
         # default: OpenAI API call
-        return self.window.core.gpt.quick_call(
+        return self.window.core.api.openai.quick_call(
             context=context,
             extra=extra,
         )
@@ -268,6 +268,7 @@ class Bridge:
         worker = BridgeWorker()
         worker.window = self.window
         worker.signals.response.connect(self.window.controller.kernel.listener)
+        worker.rt_signals = self.window.controller.realtime.signals  # Realtime signals
         return worker
 
     def apply_rate_limit(self):

pygpt_net/core/bridge/worker.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.11 14:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import QObject, Signal, QRunnable, Slot
@@ -17,6 +17,8 @@ from pygpt_net.core.types import (
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
     MODE_ASSISTANT,
+    MODE_VISION,
+    MODE_LOOP_NEXT,
 )
 from pygpt_net.core.events import KernelEvent, Event
 
@@ -29,8 +31,9 @@ class BridgeSignals(QObject):
 class BridgeWorker(QRunnable):
     """Bridge worker"""
     def __init__(self, *args, **kwargs):
-        QRunnable.__init__(self)
+        super().__init__()
         self.signals = BridgeSignals()
+        self.rt_signals = None
         self.args = args
         self.kwargs = kwargs
         self.window = None
@@ -64,6 +67,8 @@ class BridgeWorker(QRunnable):
                     extra=self.extra,
                 )
                 """
+            elif self.mode == MODE_VISION:
+                raise Exception("Vision mode is deprecated from v2.6.30 and integrated into Chat. ")
 
             # LlamaIndex: chat with files
             if self.mode == MODE_LLAMA_INDEX:
@@ -87,7 +92,7 @@ class BridgeWorker(QRunnable):
                     self.extra["error"] = str(self.window.core.agents.runner.get_error())
 
             # Loop: next step
-            elif self.mode == "loop_next":  # virtual mode
+            elif self.mode == MODE_LOOP_NEXT:  # virtual mode
                 result = self.window.core.agents.runner.loop.run_next(
                     context=self.context,
                     extra=self.extra,
@@ -98,12 +103,29 @@ class BridgeWorker(QRunnable):
                 else:
                     self.extra["error"] = str(self.window.core.agents.runner.get_error())
 
-            # API OpenAI: chat, completion, vision, image, assistants
+            # API SDK: chat, completion, vision, image, assistants
             else:
-                result = self.window.core.gpt.call(
-                    context=self.context,
-                    extra=self.extra,
-                )
+                sdk = "openai"
+                model = self.context.model
+                if model.provider == "google":
+                    if self.window.core.config.get("api_native_google", False):
+                        sdk = "google"
+
+                # call appropriate SDK
+                if sdk == "google":
+                    # print("Using Google SDK")
+                    result = self.window.core.api.google.call(
+                        context=self.context,
+                        extra=self.extra,
+                        rt_signals=self.rt_signals,
+                    )
+                elif sdk == "openai":
+                    # print("Using OpenAI SDK")
+                    result = self.window.core.api.openai.call(
+                        context=self.context,
+                        extra=self.extra,
+                        rt_signals=self.rt_signals,
+                    )
         except Exception as e:
             if self.signals:
                 self.extra["error"] = e
@@ -170,7 +192,7 @@ class BridgeWorker(QRunnable):
         ad_context = self.window.controller.chat.attachment.get_context(ctx, self.context.history)
         ad_mode = self.window.controller.chat.attachment.get_mode()
         if ad_context:
-            self.context.prompt += "\n\n" + ad_context  # append to input text
+            self.context.prompt += f"\n\n{ad_context}"  # append to input text
             if (ad_mode == self.window.controller.chat.attachment.MODE_QUERY_CONTEXT
                     or self.mode in [MODE_AGENT_LLAMA, MODE_AGENT_OPENAI]):
                 ctx.hidden_input = ad_context  # store for future use, only if query context

pygpt_net/core/debug/console/console.py

@@ -69,8 +69,8 @@ class Console:
         elif msg.lower() == "mpkfa":
             self.log("GOD MODE ACTIVATED ;)")
         elif msg == "oclr":
-            if self.window.core.gpt.client:
-                self.window.core.gpt.client.close()
+            if self.window.core.api.openai.client:
+                self.window.core.api.openai.client.close()
                 self.log("OpenAI client closed")
             else:
                 self.log("OpenAI client not initialized")

pygpt_net/core/debug/presets.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.07.30 00:00:00                  #
+# Updated Date: 2025.08.28 09:00:00                  #
 # ================================================== #
 
 import os
@@ -62,7 +62,7 @@ class PresetsDebug:
                 MODE_CHAT: preset.chat,
                 MODE_COMPLETION: preset.completion,
                 MODE_IMAGE: preset.img,
-                MODE_VISION: preset.vision,
+                # MODE_VISION: preset.vision,
                 # MODE_LANGCHAIN: preset.langchain,
                 MODE_ASSISTANT: preset.assistant,
                 MODE_LLAMA_INDEX: preset.llama_index,