pygpt-net 2.6.30__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/controller/chat/text.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.23 15:00:00                  #
+# Updated Date: 2025.08.30 06:00:00                  #
 # ================================================== #
 
 from typing import Optional
@@ -111,6 +111,7 @@ class Text:
         # if prev ctx is not empty, then copy input name to current ctx
         if prev_ctx is not None and prev_ctx.sub_call is True:  # sub_call = sent from expert
             ctx.input_name = prev_ctx.input_name
+
         if reply:
             ctx.extra["sub_reply"] = True  # mark as sub reply in extra data
 
@@ -238,7 +239,7 @@ class Text:
         """
         core = self.window.core
         stream = core.config.get("stream")
-        if mode in (MODE_AGENT_LLAMA, MODE_AUDIO):
+        if mode in (MODE_AGENT_LLAMA):
             return False  # TODO: check if this is correct in agent
         elif mode == MODE_LLAMA_INDEX:
             if core.config.get("llama.idx.mode") == "retrieval":

pygpt_net/controller/kernel/kernel.py

@@ -6,13 +6,13 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.06 19:00:00                  #
+# Updated Date: 2025.08.31 23:00:00                  #
 # ================================================== #
 
 import threading
 from typing import Any, Dict, Optional, Union, List
 
-from PySide6.QtCore import QObject, Slot
+from PySide6.QtCore import Slot
 from PySide6.QtWidgets import QApplication
 
 from pygpt_net.core.types import (
@@ -23,7 +23,7 @@ from pygpt_net.core.types import (
     MODE_EXPERT,
     MODE_LLAMA_INDEX,
 )
-from pygpt_net.core.events import KernelEvent, RenderEvent, BaseEvent
+from pygpt_net.core.events import KernelEvent, RenderEvent, BaseEvent, RealtimeEvent, Event
 from pygpt_net.core.bridge.context import BridgeContext
 from pygpt_net.item.ctx import CtxItem
 from pygpt_net.utils import trans
@@ -95,6 +95,13 @@ class Kernel:
             KernelEvent.INPUT_USER,
             KernelEvent.FORCE_CALL,
             KernelEvent.STATUS,
+            Event.AUDIO_INPUT_RECORD_TOGGLE,
+            RealtimeEvent.RT_INPUT_AUDIO_DELTA,
+            RealtimeEvent.RT_INPUT_AUDIO_MANUAL_STOP,
+            RealtimeEvent.RT_INPUT_AUDIO_MANUAL_START,
+            RealtimeEvent.RT_OUTPUT_AUDIO_COMMIT,
+            RealtimeEvent.RT_OUTPUT_TURN_END,
+            RealtimeEvent.RT_OUTPUT_READY,
         ]
 
     def init(self):
@@ -281,6 +288,7 @@ class Kernel:
         self.window.dispatch(KernelEvent(KernelEvent.TERMINATE))
         self.stop(exit=True)
         self.window.controller.plugins.destroy()
+        self.window.controller.realtime.shutdown()
 
     def stop(self, exit: bool = False):
         """

pygpt_net/controller/kernel/reply.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.23 15:00:00                  #
+# Updated Date: 2025.08.31 23:00:00                  #
 # ================================================== #
 
 import json
@@ -109,6 +109,10 @@ class Reply:
         core.ctx.update_item(self.reply_ctx)  # update context in db
         self.window.update_status('...')
 
+        # append tool calls from previous context (used for tool results handling)
+        if self.reply_ctx.tool_calls:
+            prev_ctx.extra["prev_tool_calls"] = self.reply_ctx.tool_calls
+
         # tool output append
         dispatch(RenderEvent(RenderEvent.TOOL_UPDATE, {
             "meta": self.reply_ctx.meta,

pygpt_net/controller/realtime/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.30 06:00:00                  #
+# ================================================== #
+
+from .realtime import Realtime

pygpt_net/controller/realtime/manager.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from typing import Optional
+
+from pygpt_net.core.realtime.worker import RealtimeWorker, RealtimeOptions
+from pygpt_net.item.ctx import CtxItem
+
+class Manager:
+    """
+    Manager that mirrors chat.stream controller shape.
+
+    Starts a RealtimeWorker and routes text events and lifecycle to the UI.
+    Audio is forwarded by the main-thread via RT_OUTPUT_AUDIO_DELTA events.
+    """
+    def __init__(self, window=None):
+        self.window = window
+        self.worker: Optional[RealtimeWorker] = None
+        self.ctx: Optional[CtxItem] = None
+        self.provider: Optional[str] = None
+        self.opts: Optional[RealtimeOptions] = None
+
+    def start(
+            self,
+            ctx: CtxItem,
+            opts: RealtimeOptions
+    ):
+        """
+        Start realtime worker
+
+        :param ctx: CtxItem
+        :param opts: RealtimeOptions
+        """
+        self.ctx = ctx
+        self.opts = opts
+        self.provider = opts.provider
+
+        worker = RealtimeWorker(self.window, ctx, opts)
+        self.worker = worker
+        self.window.core.debug.info(f"[realtime] Begin: provider={opts.provider}, model={opts.model}")
+        self.window.threadpool.start(worker)
+
+    def shutdown(self):
+        """Shutdown realtime worker"""
+        self.worker = None

pygpt_net/controller/realtime/realtime.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from PySide6.QtCore import Slot, QTimer
+
+from pygpt_net.core.events import RealtimeEvent, RenderEvent, BaseEvent, AppEvent, KernelEvent
+from pygpt_net.core.realtime.worker import RealtimeSignals
+from pygpt_net.core.types import MODE_AUDIO
+from pygpt_net.utils import trans
+from pygpt_net.core.tabs import Tab
+
+from .manager import Manager
+
+class Realtime:
+    def __init__(self, window=None):
+        """
+        Realtime controller
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.manager = Manager(window)
+        self.signals = RealtimeSignals()
+        self.signals.response.connect(self.handle_response)
+        self.current_active = None # openai | google
+        self.allowed_modes = [MODE_AUDIO]
+        self.manual_commit_sent = False
+
+    def setup(self):
+        """Setup realtime core, signals, etc. in main thread"""
+        self.window.core.audio.setup()  # setup RT signals in audio input/output core
+
+    def is_enabled(self) -> bool:
+        """
+        Check if realtime is enabled in settings
+
+        :return: True if enabled, False otherwise
+        """
+        mode = self.window.core.config.get("mode")
+        if mode == MODE_AUDIO:
+            if self.window.controller.ui.tabs.get_current_type() != Tab.TAB_NOTEPAD:
+                return True
+        return False
+
+    @Slot(object)
+    def handle(self, event: BaseEvent):
+        """
+        Handle realtime event (returned from dispatcher)
+
+        :param event: RealtimeEvent instance
+        """
+        # check if mode is supported
+        if not self.is_supported() and isinstance(event, RealtimeEvent):
+            event.stop = True # stop further propagation
+            return # ignore if not in realtime mode
+
+        # ----------------------------------------------------
+
+        # audio output chunk: send to audio output handler
+        if event.name == RealtimeEvent.RT_OUTPUT_AUDIO_DELTA:
+            self.set_idle()
+            payload = event.data.get("payload", None)
+            if payload:
+                self.window.core.audio.output.handle_realtime(payload, self.signals)
+
+        # audio input chunk: send to the active realtime client
+        elif event.name == RealtimeEvent.RT_INPUT_AUDIO_DELTA:
+            self.set_idle()
+            if self.current_active == "google":
+                self.window.core.api.google.realtime.handle_audio_input(event)
+            elif self.current_active == "openai":
+                self.window.core.api.openai.realtime.handle_audio_input(event)
+
+        # begin: first text chunk or audio chunk received, start rendering
+        elif event.name == RealtimeEvent.RT_OUTPUT_READY:
+            ctx = event.data.get('ctx', None)
+            if ctx:
+                self.window.dispatch(RenderEvent(RenderEvent.STREAM_BEGIN, {
+                    "meta": ctx.meta,
+                    "ctx": ctx,
+                }))
+                self.set_busy()
+
+        # commit: audio buffer sent, stop audio input and finalize the response
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_COMMIT:
+            self.set_busy()
+            if self.manual_commit_sent:
+                self.manual_commit_sent = False
+                return # abort if manual commit was already sent
+            self.window.controller.audio.execute_input_stop()
+
+        elif event.name == RealtimeEvent.RT_INPUT_AUDIO_MANUAL_STOP:
+            self.manual_commit_sent = True
+            self.set_busy()
+            QTimer.singleShot(0, lambda: self.manual_commit())
+
+        elif event.name == RealtimeEvent.RT_INPUT_AUDIO_MANUAL_START:
+            self.set_idle()
+            self.window.controller.chat.input.execute("...", force=True)
+            self.window.dispatch(KernelEvent(KernelEvent.STATUS, {
+                'status': trans("speech.listening"),
+            }))
+
+        # text delta: append text chunk to the response
+        elif event.name == RealtimeEvent.RT_OUTPUT_TEXT_DELTA:
+            self.set_idle()
+            ctx = event.data.get('ctx', None)
+            chunk = event.data.get('chunk', "")
+            if chunk and ctx:
+                self.window.dispatch(RenderEvent(RenderEvent.STREAM_APPEND, {
+                    "meta": ctx.meta,
+                    "ctx": ctx,
+                    "chunk": chunk,
+                    "begin": False,
+                }))
+
+        # audio end: stop audio playback
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_END:
+            self.set_idle()
+            self.window.controller.chat.common.unlock_input()
+
+        # end of turn: finalize the response
+        elif event.name == RealtimeEvent.RT_OUTPUT_TURN_END:
+            self.set_idle()
+            ctx = event.data.get('ctx', None)
+            if ctx:
+                self.end_turn(ctx)
+            if self.window.controller.audio.is_recording():
+                self.window.update_status(trans("speech.listening"))
+            self.window.controller.chat.common.unlock_input()
+
+        # volume change: update volume in audio output handler
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_VOLUME_CHANGED:
+            volume = event.data.get("volume", 1.0)
+            self.window.controller.audio.ui.on_output_volume_change(volume)
+
+        # error: audio output error
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_ERROR:
+            self.set_idle()
+            error = event.data.get("error")
+            self.window.core.debug.log(error)
+            self.window.controller.chat.common.unlock_input()
+
+        # -----------------------------------
+
+        # app events, always handled
+        elif event.name == AppEvent.MODE_SELECTED:
+            mode = self.window.core.config.get("mode")
+            if mode != MODE_AUDIO:
+                QTimer.singleShot(0, lambda: self.reset())
+
+        elif event.name == AppEvent.CTX_CREATED:
+            QTimer.singleShot(0, lambda: self.reset())
+
+        elif event.name == AppEvent.CTX_SELECTED:
+            QTimer.singleShot(0, lambda: self.reset())
+
+    @Slot(object)
+    def handle_response(self, event: RealtimeEvent):
+        """
+        Handle response event (send to kernel -> dispatcher)
+
+        :param event: RealtimeEvent instance
+        """
+        self.window.controller.kernel.listener(event)
+
+    def is_auto_turn(self) -> bool:
+        """
+        Check if auto-turn is enabled
+
+        :return: True if auto-turn is enabled, False otherwise
+        """
+        return self.window.core.config.get("audio.input.auto_turn", True)
+
+    def manual_commit(self):
+        """Manually commit the response (end of turn)"""
+        if self.current_active == "google":
+            self.window.core.api.google.realtime.manual_commit()
+        elif self.current_active == "openai":
+            self.window.core.api.openai.realtime.manual_commit()
+
+    def end_turn(self, ctx):
+        """
+        End of realtime turn - finalize the response
+
+        :param ctx: Context instance
+        """
+        self.set_idle()
+        if not ctx:
+            return
+        self.window.controller.chat.output.handle_after(
+            ctx=ctx,
+            mode=MODE_AUDIO,
+            stream=True,
+        )
+        self.window.controller.chat.output.post_handle(
+            ctx=ctx,
+            mode=MODE_AUDIO,
+            stream=True,
+        )
+        self.window.controller.chat.output.handle_end(
+            ctx=ctx,
+            mode=MODE_AUDIO,
+        )
+        self.window.controller.chat.common.show_response_tokens(ctx)
+
+    def shutdown(self):
+        """Shutdown all realtime threads and async loops"""
+        try:
+            self.window.core.api.openai.realtime.shutdown()
+        except Exception as e:
+            self.window.core.debug.log(f"[openai] Realtime shutdown error: {e}")
+        try:
+            self.window.core.api.google.realtime.shutdown()
+        except Exception as e:
+            self.window.core.debug.log(f"[google] Realtime shutdown error: {e}")
+        try:
+            self.manager.shutdown()
+        except Exception as e:
+            self.window.core.debug.log(f"[manager] Realtime shutdown error: {e}")
+
+    def reset(self):
+        """Reset realtime session"""
+        try:
+            self.window.core.api.openai.realtime.reset()
+        except Exception as e:
+            self.window.core.debug.log(f"[openai] Realtime reset error: {e}")
+        try:
+            self.window.core.api.google.realtime.reset()
+        except Exception as e:
+            self.window.core.debug.log(f"[google] Realtime reset error: {e}")
+
+    def is_supported(self) -> bool:
+        """
+        Check if current mode supports realtime
+
+        :return: True if mode supports realtime, False otherwise
+        """
+        mode = self.window.core.config.get("mode")
+        return mode in self.allowed_modes
+
+    def set_current_active(self, provider: str):
+        """
+        Set the current active realtime provider
+
+        :param provider: Provider name (openai, google)
+        """
+        self.current_active = provider.lower() if provider else None
+
+    def set_idle(self):
+        """Set kernel state to IDLE"""
+        QTimer.singleShot(0, lambda: self.window.dispatch(KernelEvent(KernelEvent.STATE_IDLE, {
+            "id": "realtime",
+        })))
+
+    def set_busy(self):
+        """Set kernel state to BUSY"""
+        QTimer.singleShot(0, lambda: self.window.dispatch(KernelEvent(KernelEvent.STATE_BUSY, {
+            "id": "realtime",
+        })))

pygpt_net/controller/ui/mode.py

@@ -20,6 +20,7 @@ from pygpt_net.core.types import (
     MODE_COMPUTER,
     MODE_AGENT_OPENAI,
     MODE_COMPLETION,
+    MODE_AUDIO,
 )
 from pygpt_net.core.tabs.tab import Tab
 from pygpt_net.core.events import Event
@@ -55,6 +56,12 @@ class Mode:
         is_image = mode == MODE_IMAGE
         is_llama_index = mode == MODE_LLAMA_INDEX
         is_completion = mode == MODE_COMPLETION
+        is_audio = mode == MODE_AUDIO
+
+        if not is_audio:
+            self.window.ui.nodes['audio.auto_turn'].setVisible(False)
+        else:
+            self.window.ui.nodes['audio.auto_turn'].setVisible(True)
 
         if not is_assistant:
             ui_nodes['presets.widget'].setVisible(True)

pygpt_net/controller/ui/ui.py

@@ -13,6 +13,7 @@ from typing import Optional
 
 from PySide6.QtGui import QColor
 
+from pygpt_net.core.types import MODE_IMAGE
 from pygpt_net.core.events import BaseEvent, Event
 from pygpt_net.utils import trans
 
@@ -64,6 +65,7 @@ class UI:
         self.update_tokens()
         self.vision.update()
         self.window.controller.agent.legacy.update()
+        self.img_update_available_resolutions()
 
     def handle(self, event: BaseEvent):
         """
@@ -215,4 +217,20 @@ class UI:
     def on_global_stop(self):
         """Global stop button action"""
         if self.stop_action == "idx":
-            self.window.controller.idx.force_stop()
+            self.window.controller.idx.force_stop()
+
+    def img_update_available_resolutions(self):
+        """Update available resolutions for images"""
+        mode = self.window.core.config.get('mode')
+        if mode != MODE_IMAGE:
+            return
+        model = self.window.core.config.get('model')
+        keys = self.window.core.image.get_available_resolutions(model)
+        current = self.window.core.config.get('img_resolution', '1024x1024')
+        self.window.ui.config['global']['img_resolution'].set_keys(keys, lock=False)
+        self.window.controller.config.apply_value(
+            parent_id="global",
+            key="img_resolution",
+            option=self.window.core.image.get_resolution_option(),
+            value=current,
+        )

pygpt_net/core/audio/audio.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.07 22:00:00                  #
+# Updated Date: 2025.08.31 23:00:00                  #
 # ================================================== #
 
 import hashlib
@@ -42,6 +42,11 @@ class Audio:
         }
         self.last_error = None
 
+    def setup(self):
+        """Initialize audio core"""
+        self.capture.setup()
+        self.output.setup()
+
     def get_input_devices(self) -> List[Tuple[int, str]]:
         """
         Get input devices

pygpt_net/core/audio/backend/native/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from .native import NativeBackend