pygpt-net 2.6.30__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/core/events/realtime.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.30 06:00:00                  #
+# ================================================== #
+
+from typing import Optional
+
+from .base import BaseEvent
+
+
+class RealtimeEvent(BaseEvent):
+    """
+    Realtime events
+
+    - RT_OUTPUT_AUDIO_DELTA - audio output chunk (with payload)
+    - RT_OUTPUT_READY - audio output is ready (STREAM_BEGIN)
+    - RT_OUTPUT_TEXT_DELTA - text chunk (delta)
+    - RT_OUTPUT_AUDIO_END - audio output ended (STREAM_END)
+    - RT_OUTPUT_TURN_END - audio output turn ended (TURN_END)
+    - RT_OUTPUT_AUDIO_ERROR - audio output error (STREAM_ERROR)
+    - RT_OUTPUT_AUDIO_VOLUME_CHANGED - audio output volume changed (volume level)
+    """
+
+    # realtime events
+    RT_OUTPUT_AUDIO_DELTA = "rt.output.audio.delta"
+    RT_OUTPUT_AUDIO_END = "rt.output.audio.end"
+    RT_OUTPUT_AUDIO_ERROR = "rt.output.audio.error"
+    RT_OUTPUT_AUDIO_VOLUME_CHANGED = "rt.output.audio.volume.changed"
+    RT_OUTPUT_AUDIO_COMMIT = "rt.output.audio.commit"
+    RT_OUTPUT_READY = "rt.output.audio.ready"
+    RT_OUTPUT_TEXT_DELTA = "rt.output.text.delta"
+    RT_OUTPUT_TURN_END = "rt.output.turn.end"
+    RT_INPUT_AUDIO_DELTA = "rt.input.audio.delta"
+    RT_INPUT_AUDIO_MANUAL_START = "rt.input.audio.manual.start"
+    RT_INPUT_AUDIO_MANUAL_STOP = "rt.input.audio.manual.stop"
+
+    def __init__(
+            self,
+            name: Optional[str] = None,
+            data: Optional[dict] = None,
+    ):
+        """
+        Event object class
+
+        :param name: event name
+        :param data: event data
+        """
+        super(RealtimeEvent, self).__init__(name, data)
+        self.id = "RealtimeEvent"

pygpt_net/core/image/image.py

@@ -12,10 +12,11 @@
 import os
 import uuid
 from time import strftime
-from typing import List
+from typing import List, Dict
 
 from PySide6.QtCore import Slot, QObject
 
+from pygpt_net.core.types import IMAGE_AVAILABLE_RESOLUTIONS
 from pygpt_net.item.ctx import CtxItem
 from pygpt_net.utils import trans
 
@@ -141,3 +142,52 @@ class Image(QObject):
         img_dir = self.window.core.config.get_user_dir("img")
         filename = f"{dt_prefix}_{img_id}.png"
         return os.path.join(img_dir, filename)
+
+    def get_resolution_option(self) -> dict:
+        """
+        Get image resolution option for UI
+
+        :return: dict
+        """
+        return {
+            "type": "combo",
+            "slider": True,
+            "label": "img_resolution",
+            "value": "1024x1024",
+            "keys": self.get_available_resolutions(),
+        }
+
+    def get_available_resolutions(self, model: str = None) -> Dict[str, str]:
+        """
+        Get available image resolutions
+
+        :param model: model name
+        :return: dict of available resolutions
+        """
+        available = IMAGE_AVAILABLE_RESOLUTIONS
+        model_keys = available.keys()
+        # find by model if specified
+        if model:
+            model = self._normalize_model_name(model)
+            for key in model_keys:
+                if model.startswith(key):
+                    return available[key]
+
+        # return all available resolutions, but unique only
+        resolutions = {}
+        for key in model_keys:
+            resolutions.update(available[key])
+        return resolutions
+
+
+    def _normalize_model_name(self, model: str) -> str:
+        """
+        Normalize model id (strip optional 'models/' prefix).
+
+        :param model: model id
+        """
+        try:
+            return model.split("/")[-1]
+        except Exception:
+            return model
+

pygpt_net/core/realtime/options.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+@dataclass
+class RealtimeOptions:
+    """
+    Options for starting a realtime session.
+
+    :param provider: Provider name ("google" or "openai")
+    :param model: Model name
+    :param system_prompt: System prompt text
+    :param prompt: User prompt text
+    :param voice: Voice name for TTS
+    :param audio_data: Optional input audio data (bytes)
+    :param audio_format: Format of the input audio (e.g., "pcm16", "wav")
+    :param audio_rate: Sample rate of the input audio (e.g., 16000)
+    :param vad: Voice Activity Detection mode (e.g., "server_vad" or None for manual)
+    :param extra: Free-form dictionary for extra parameters
+    :param rt_signals: Real-time signals for event handling
+    """
+    provider: str = "openai"  # "google" | "openai"
+    model: Optional[str] = None
+    system_prompt: Optional[str] = None
+    prompt: Optional[str] = None
+    voice: Optional[str] = None
+
+    # Optional input audio
+    audio_data: Optional[bytes] = None
+    audio_format: Optional[str] = None  # e.g., "pcm16", "wav"
+    audio_rate: Optional[int] = None    # e.g., 16000
+
+    # Provider-specific VAD flag (use None for manual mode)
+    vad: Optional[str] = None           # e.g., "server_vad"
+
+    vad_end_silence_ms: Optional[int] = 2000  # VAD end silence in ms
+    vad_prefix_padding_ms: Optional[int] = 300  # VAD prefix padding in ms
+
+    # Real-time signals
+    rt_signals: field() = None  # RT signals
+
+    # Tools and remote tools
+    tools: Optional[list] = None
+    remote_tools: Optional[list] = None
+
+    # Auto-turn enable/disable
+    auto_turn: Optional[bool] = False
+
+    # Transcript enable/disable
+    transcribe: Optional[bool] = True
+
+    # Last session ID
+    rt_session_id: Optional[str] = None
+
+    # Extra parameters
+    extra: dict = field(default_factory=dict)
+
+    def to_dict(self):
+        return {
+            "provider": self.provider,
+            "model": self.model,
+            "system_prompt": self.system_prompt,
+            "prompt": self.prompt,
+            "voice": self.voice,
+            "audio_data (len)": len(self.audio_data) if self.audio_data else 0,
+            "audio_format": self.audio_format,
+            "audio_rate": self.audio_rate,
+            "vad": self.vad,
+            "vad_end_silence_ms": self.vad_end_silence_ms,
+            "vad_prefix_padding_ms": self.vad_prefix_padding_ms,
+            "tools": self.tools,
+            "remote_tools": self.remote_tools,
+            "auto_turn": self.auto_turn,
+            "transcribe": self.transcribe,
+            "rt_session_id": self.rt_session_id,
+            "extra": self.extra,
+        }

pygpt_net/core/realtime/shared/audio.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import io
+import math
+import os
+import wave
+import audioop
+from array import array
+import struct
+from typing import Optional, Tuple, List
+
+DEFAULT_24K = 24000
+
+def coerce_to_pcm16_mono(data: bytes, fmt: Optional[str], rate_hint: Optional[int], fallback_rate: int = DEFAULT_24K) -> Tuple[int, int, bytes]:
+    """
+    Convert input audio (PCM16 raw or WAV) to PCM16 mono bytes. Float WAV is treated as raw (best effort).
+    Returns (sample_rate, channels=1, pcm16_bytes).
+    """
+    if not data:
+        return fallback_rate, 1, b""
+    fmt = (fmt or "").lower().strip()
+    if fmt in ("pcm16", "pcm", "raw"):
+        sr = int(rate_hint) if rate_hint else fallback_rate
+        return sr, 1, data
+
+    # WAV path
+    try:
+        with wave.open(io.BytesIO(data), "rb") as wf:
+            sr = wf.getframerate() or fallback_rate
+            ch = wf.getnchannels() or 1
+            sw = wf.getsampwidth() or 2
+            frames = wf.readframes(wf.getnframes())
+
+        if sw != 2:
+            frames = audioop.lin2lin(frames, sw, 2)
+        if ch == 2:
+            frames = audioop.tomono(frames, 2, 0.5, 0.5)
+        elif ch != 1:
+            frames = audioop.tomono(frames, 2, 1.0, 0.0)
+
+        return sr, 1, frames
+    except Exception:
+        sr = int(rate_hint) if rate_hint else fallback_rate
+        return sr, 1, data
+
+def float32_to_int16_bytes(b: bytes) -> bytes:
+    """Convert little-endian float32 PCM [-1.0, 1.0] to int16 PCM."""
+    if not b:
+        return b""
+    try:
+        arr = array("f")
+        arr.frombytes(b)
+        if struct.unpack('<I', struct.pack('=I', 1))[0] != 1:  # fallback if non-little
+            arr.byteswap()
+        out = array("h", (max(-32768, min(32767, int(round(x * 32767.0)))) for x in arr))
+        return out.tobytes()
+    except Exception:
+        try:
+            n = len(b) // 4
+            vals = struct.unpack("<" + "f" * n, b[: n * 4])
+            out = array("h", (max(-32768, min(32767, int(round(x * 32767.0)))) for x in vals))
+            return out.tobytes()
+        except Exception:
+            return b""
+
+def parse_wav_fmt(data: bytes) -> Optional[dict]:
+    """Minimal WAV fmt chunk parser to detect float/int format."""
+    try:
+        if len(data) < 12 or data[0:4] != b"RIFF" or data[8:12] != b"WAVE":
+            return None
+        p = 12
+        while p + 8 <= len(data):
+            cid = data[p:p+4]
+            sz = int.from_bytes(data[p+4:p+8], "little", signed=False)
+            p += 8
+            if cid == b"fmt ":
+                fmtb = data[p:p+sz]
+                if len(fmtb) < 16:
+                    return None
+                format_tag = int.from_bytes(fmtb[0:2], "little")
+                channels = int.from_bytes(fmtb[2:4], "little")
+                sample_rate = int.from_bytes(fmtb[4:8], "little")
+                bits_per_sample = int.from_bytes(fmtb[14:16], "little")
+                sub_tag = None
+                if format_tag == 65534 and sz >= 40:  # WAVE_FORMAT_EXTENSIBLE
+                    sub_tag = int.from_bytes(fmtb[24:26], "little", signed=False)
+                return {
+                    "format_tag": format_tag,
+                    "channels": channels,
+                    "sample_rate": sample_rate,
+                    "bits_per_sample": bits_per_sample,
+                    "subformat_tag": sub_tag,
+                }
+            p += (sz + 1) & ~1
+        return None
+    except Exception:
+        return None
+
+def to_pcm16_mono(data: bytes, fmt: Optional[str], rate_hint: Optional[int], target_rate: int) -> Tuple[bytes, int]:
+    """
+    Normalize any input audio (RAW/WAV, int/float) to PCM16 mono at target_rate.
+    Returns (pcm16_bytes, target_rate).
+    """
+    if not data:
+        return b"", target_rate
+
+    fmt = (fmt or "").lower().strip()
+    if fmt in ("pcm16", "pcm", "raw"):
+        src_rate = int(rate_hint) if rate_hint else target_rate
+        pcm16 = data
+        if src_rate != target_rate:
+            try:
+                pcm16, _ = audioop.ratecv(pcm16, 2, 1, src_rate, target_rate, None)
+            except Exception:
+                return b"", target_rate
+        return pcm16, target_rate
+
+    # WAV path with float support
+    try:
+        fmt_info = parse_wav_fmt(data)
+        with wave.open(io.BytesIO(data), "rb") as wf:
+            sr = wf.getframerate() or target_rate
+            ch = wf.getnchannels() or 1
+            sw = wf.getsampwidth() or 2
+            frames = wf.readframes(wf.getnframes())
+
+        format_tag = (fmt_info or {}).get("format_tag", 1)
+        bits_per_sample = (fmt_info or {}).get("bits_per_sample", sw * 8)
+
+        # float32 -> int16
+        if format_tag == 3 or ((format_tag == 65534) and (fmt_info or {}).get("subformat_tag") == 3):
+            frames16 = float32_to_int16_bytes(frames)
+        else:
+            if sw != 2:
+                frames16 = audioop.lin2lin(frames, sw, 2)
+            else:
+                frames16 = frames
+
+        # mixdown to mono
+        if ch == 2:
+            try:
+                frames16 = audioop.tomono(frames16, 2, 0.5, 0.5)
+            except Exception:
+                frames16 = frames16[0::2] + b""
+        elif ch != 1:
+            try:
+                frames16 = audioop.tomono(frames16, 2, 1.0, 0.0)
+            except Exception:
+                pass
+
+        # resample
+        if sr != target_rate:
+            try:
+                frames16, _ = audioop.ratecv(frames16, 2, 1, sr, target_rate, None)
+            except Exception:
+                return b"", target_rate
+
+        return frames16, target_rate
+    except Exception:
+        return b"", target_rate
+
+def resample_pcm16_mono(pcm: bytes, src_rate: int, dst_rate: int) -> bytes:
+    if src_rate == dst_rate or not pcm:
+        return pcm
+    try:
+        out, _ = audioop.ratecv(pcm, 2, 1, src_rate, dst_rate, None)
+        return out
+    except Exception:
+        return pcm
+
+def iter_pcm_chunks(pcm: bytes, sr: int, ms: int = 50) -> List[bytes]:
+    """Split PCM16 mono stream into ~ms byte chunks."""
+    b_per_ms = int(sr * 2 / 1000)
+    n = max(b_per_ms * ms, 1)
+    return [pcm[i:i + n] for i in range(0, len(pcm), n)]
+
+def dump_wav(path: str, sample_rate: int, pcm16_mono: bytes):
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+    except Exception:
+        pass
+    try:
+        with wave.open(path, "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)
+            wf.setframerate(int(sample_rate))
+            wf.writeframes(pcm16_mono)
+    except Exception:
+        pass
+
+def pcm16_stats(pcm16_mono: bytes, sample_rate: int) -> dict:
+    try:
+        n_samp = len(pcm16_mono) // 2
+        dur = n_samp / float(sample_rate or 1)
+        rms = audioop.rms(pcm16_mono, 2)
+        peak = audioop.max(pcm16_mono, 2) if pcm16_mono else 0
+        try:
+            avg = audioop.avg(pcm16_mono, 2)
+        except Exception:
+            avg = 0
+        dbfs = (-999.0 if rms == 0 else 20.0 * math.log10(rms / 32768.0))
+        return {"duration_s": dur, "samples": n_samp, "rms": rms, "peak": peak, "dc_offset": avg, "dbfs": dbfs}
+    except Exception:
+        return {}

pygpt_net/core/realtime/shared/loop.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import asyncio
+import threading
+from typing import Optional
+
+class BackgroundLoop:
+    """
+    Dedicated background asyncio loop running in its own thread.
+    Safe cross-thread scheduling and sync wrappers.
+    """
+    def __init__(self, name: str = "RT-Loop"):
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._thread: Optional[threading.Thread] = None
+        self._name = name
+
+    @property
+    def loop(self) -> Optional[asyncio.AbstractEventLoop]:
+        return self._loop
+
+    def ensure(self):
+        if self._loop and self._loop.is_running():
+            return
+        self._loop = asyncio.new_event_loop()
+
+        def _runner(loop: asyncio.AbstractEventLoop):
+            asyncio.set_event_loop(loop)
+            loop.run_forever()
+
+        self._thread = threading.Thread(target=_runner, args=(self._loop,), name=self._name, daemon=True)
+        self._thread.start()
+
+    async def run(self, coro):
+        if not self._loop:
+            raise RuntimeError("Owner loop is not running")
+        cfut = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        return await asyncio.wrap_future(cfut)
+
+    def run_sync(self, coro, timeout: float = 5.0):
+        if not self._loop or not self._loop.is_running():
+            return None
+        fut = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        try:
+            return fut.result(timeout=timeout)
+        except Exception:
+            return None
+
+    def stop(self, timeout: float = 2.0):
+        loop, thread = self._loop, self._thread
+        if loop and loop.is_running():
+            loop.call_soon_threadsafe(loop.stop)
+        if thread and thread.is_alive():
+            thread.join(timeout=timeout)
+        self._loop = None
+        self._thread = None

pygpt_net/core/realtime/shared/session.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from typing import Optional
+from pygpt_net.item.ctx import CtxItem
+
+def set_ctx_rt_handle(ctx: Optional[CtxItem], handle: Optional[str], window=None):
+    """Persist server session handle into ctx.extra['rt_session_id'] (best effort)."""
+    try:
+        if not ctx:
+            return
+        if not isinstance(ctx.extra, dict):
+            ctx.extra = {}
+        val = (handle or "").strip()
+        if val:
+            ctx.extra["rt_session_id"] = val
+            if window:
+                try:
+                    window.core.ctx.update_item(ctx)
+                except Exception:
+                    pass
+    except Exception:
+        pass
+
+def set_rt_session_expires_at(ctx: Optional[CtxItem], epoch_seconds: Optional[int], window=None):
+    """Persist optional session expiration timestamp into ctx.extra."""
+    if not ctx or epoch_seconds is None:
+        return
+    try:
+        if not isinstance(ctx.extra, dict):
+            ctx.extra = {}
+        ctx.extra["rt_session_expires_at"] = int(epoch_seconds)
+        if window:
+            try:
+                window.core.ctx.update_item(ctx)
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+def extract_last_session_id(items: list[CtxItem]) -> Optional[str]:
+    """Extract last known session ID from a list of CtxItems."""
+    if not items:
+        return None
+    for item in reversed(items):
+        if not item or not isinstance(item.extra, dict):
+            continue
+        val = item.extra.get("rt_session_id")
+        if isinstance(val, str) and val.strip():
+            return val.strip()
+    return None

pygpt_net/core/realtime/shared/text.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+import re
+
+def coalesce_text(parts):
+    """Merge text parts, preserving intentional newlines and fixing spaces."""
+    if not parts:
+        return ""
+    out = []
+    for piece in parts:
+        if not piece:
+            continue
+        s = str(piece)
+        s = re.sub(r"[ \t\f\v]+", " ", s)
+        s = re.sub(r"[ \t]*\n[ \t]*", "\n", s)
+        if not out:
+            out.append(s.strip())
+            continue
+        if out[-1].endswith("\n") or s.startswith("\n"):
+            out.append(s.lstrip())
+        else:
+            out.append(" " + s.strip())
+    text = "".join(out)
+    text = re.sub(r"[ \t]+([,.;:!?%])", r"\1", text)
+    text = re.sub(r"[ \t]+([\)\]\}])", r"\1", text)
+    text = re.sub(r"[ \t]+(['\"])", r"\1", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()