pygpt-net 2.6.29__py3-none-any.whl → 2.6.31__py3-none-any.whl

pygpt_net/controller/realtime/manager.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from typing import Optional
+
+from pygpt_net.core.realtime.worker import RealtimeWorker, RealtimeOptions
+from pygpt_net.item.ctx import CtxItem
+
+class Manager:
+    """
+    Manager that mirrors chat.stream controller shape.
+
+    Starts a RealtimeWorker and routes text events and lifecycle to the UI.
+    Audio is forwarded by the main-thread via RT_OUTPUT_AUDIO_DELTA events.
+    """
+    def __init__(self, window=None):
+        self.window = window
+        self.worker: Optional[RealtimeWorker] = None
+        self.ctx: Optional[CtxItem] = None
+        self.provider: Optional[str] = None
+        self.opts: Optional[RealtimeOptions] = None
+
+    def start(
+            self,
+            ctx: CtxItem,
+            opts: RealtimeOptions
+    ):
+        """
+        Start realtime worker
+
+        :param ctx: CtxItem
+        :param opts: RealtimeOptions
+        """
+        self.ctx = ctx
+        self.opts = opts
+        self.provider = opts.provider
+
+        worker = RealtimeWorker(self.window, ctx, opts)
+        self.worker = worker
+        self.window.core.debug.info(f"[realtime] Begin: provider={opts.provider}, model={opts.model}")
+        self.window.threadpool.start(worker)
+
+    def shutdown(self):
+        """Shutdown realtime worker"""
+        self.worker = None

pygpt_net/controller/realtime/realtime.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from PySide6.QtCore import Slot, QTimer
+
+from pygpt_net.core.events import RealtimeEvent, RenderEvent, BaseEvent, AppEvent, KernelEvent
+from pygpt_net.core.realtime.worker import RealtimeSignals
+from pygpt_net.core.types import MODE_AUDIO
+from pygpt_net.utils import trans
+from pygpt_net.core.tabs import Tab
+
+from .manager import Manager
+
+class Realtime:
+    def __init__(self, window=None):
+        """
+        Realtime controller
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.manager = Manager(window)
+        self.signals = RealtimeSignals()
+        self.signals.response.connect(self.handle_response)
+        self.current_active = None # openai | google
+        self.allowed_modes = [MODE_AUDIO]
+        self.manual_commit_sent = False
+
+    def setup(self):
+        """Setup realtime core, signals, etc. in main thread"""
+        self.window.core.audio.setup()  # setup RT signals in audio input/output core
+
+    def is_enabled(self) -> bool:
+        """
+        Check if realtime is enabled in settings
+
+        :return: True if enabled, False otherwise
+        """
+        mode = self.window.core.config.get("mode")
+        if mode == MODE_AUDIO:
+            if self.window.controller.ui.tabs.get_current_type() != Tab.TAB_NOTEPAD:
+                return True
+        return False
+
+    @Slot(object)
+    def handle(self, event: BaseEvent):
+        """
+        Handle realtime event (returned from dispatcher)
+
+        :param event: RealtimeEvent instance
+        """
+        # check if mode is supported
+        if not self.is_supported() and isinstance(event, RealtimeEvent):
+            event.stop = True # stop further propagation
+            return # ignore if not in realtime mode
+
+        # ----------------------------------------------------
+
+        # audio output chunk: send to audio output handler
+        if event.name == RealtimeEvent.RT_OUTPUT_AUDIO_DELTA:
+            self.set_idle()
+            payload = event.data.get("payload", None)
+            if payload:
+                self.window.core.audio.output.handle_realtime(payload, self.signals)
+
+        # audio input chunk: send to the active realtime client
+        elif event.name == RealtimeEvent.RT_INPUT_AUDIO_DELTA:
+            self.set_idle()
+            if self.current_active == "google":
+                self.window.core.api.google.realtime.handle_audio_input(event)
+            elif self.current_active == "openai":
+                self.window.core.api.openai.realtime.handle_audio_input(event)
+
+        # begin: first text chunk or audio chunk received, start rendering
+        elif event.name == RealtimeEvent.RT_OUTPUT_READY:
+            ctx = event.data.get('ctx', None)
+            if ctx:
+                self.window.dispatch(RenderEvent(RenderEvent.STREAM_BEGIN, {
+                    "meta": ctx.meta,
+                    "ctx": ctx,
+                }))
+                self.set_busy()
+
+        # commit: audio buffer sent, stop audio input and finalize the response
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_COMMIT:
+            self.set_busy()
+            if self.manual_commit_sent:
+                self.manual_commit_sent = False
+                return # abort if manual commit was already sent
+            self.window.controller.audio.execute_input_stop()
+
+        elif event.name == RealtimeEvent.RT_INPUT_AUDIO_MANUAL_STOP:
+            self.manual_commit_sent = True
+            self.set_busy()
+            QTimer.singleShot(0, lambda: self.manual_commit())
+
+        elif event.name == RealtimeEvent.RT_INPUT_AUDIO_MANUAL_START:
+            self.set_idle()
+            self.window.controller.chat.input.execute("...", force=True)
+            self.window.dispatch(KernelEvent(KernelEvent.STATUS, {
+                'status': trans("speech.listening"),
+            }))
+
+        # text delta: append text chunk to the response
+        elif event.name == RealtimeEvent.RT_OUTPUT_TEXT_DELTA:
+            self.set_idle()
+            ctx = event.data.get('ctx', None)
+            chunk = event.data.get('chunk', "")
+            if chunk and ctx:
+                self.window.dispatch(RenderEvent(RenderEvent.STREAM_APPEND, {
+                    "meta": ctx.meta,
+                    "ctx": ctx,
+                    "chunk": chunk,
+                    "begin": False,
+                }))
+
+        # audio end: stop audio playback
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_END:
+            self.set_idle()
+            self.window.controller.chat.common.unlock_input()
+
+        # end of turn: finalize the response
+        elif event.name == RealtimeEvent.RT_OUTPUT_TURN_END:
+            self.set_idle()
+            ctx = event.data.get('ctx', None)
+            if ctx:
+                self.end_turn(ctx)
+            if self.window.controller.audio.is_recording():
+                self.window.update_status(trans("speech.listening"))
+            self.window.controller.chat.common.unlock_input()
+
+        # volume change: update volume in audio output handler
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_VOLUME_CHANGED:
+            volume = event.data.get("volume", 1.0)
+            self.window.controller.audio.ui.on_output_volume_change(volume)
+
+        # error: audio output error
+        elif event.name == RealtimeEvent.RT_OUTPUT_AUDIO_ERROR:
+            self.set_idle()
+            error = event.data.get("error")
+            self.window.core.debug.log(error)
+            self.window.controller.chat.common.unlock_input()
+
+        # -----------------------------------
+
+        # app events, always handled
+        elif event.name == AppEvent.MODE_SELECTED:
+            mode = self.window.core.config.get("mode")
+            if mode != MODE_AUDIO:
+                QTimer.singleShot(0, lambda: self.reset())
+
+        elif event.name == AppEvent.CTX_CREATED:
+            QTimer.singleShot(0, lambda: self.reset())
+
+        elif event.name == AppEvent.CTX_SELECTED:
+            QTimer.singleShot(0, lambda: self.reset())
+
+    @Slot(object)
+    def handle_response(self, event: RealtimeEvent):
+        """
+        Handle response event (send to kernel -> dispatcher)
+
+        :param event: RealtimeEvent instance
+        """
+        self.window.controller.kernel.listener(event)
+
+    def is_auto_turn(self) -> bool:
+        """
+        Check if auto-turn is enabled
+
+        :return: True if auto-turn is enabled, False otherwise
+        """
+        return self.window.core.config.get("audio.input.auto_turn", True)
+
+    def manual_commit(self):
+        """Manually commit the response (end of turn)"""
+        if self.current_active == "google":
+            self.window.core.api.google.realtime.manual_commit()
+        elif self.current_active == "openai":
+            self.window.core.api.openai.realtime.manual_commit()
+
+    def end_turn(self, ctx):
+        """
+        End of realtime turn - finalize the response
+
+        :param ctx: Context instance
+        """
+        self.set_idle()
+        if not ctx:
+            return
+        self.window.controller.chat.output.handle_after(
+            ctx=ctx,
+            mode=MODE_AUDIO,
+            stream=True,
+        )
+        self.window.controller.chat.output.post_handle(
+            ctx=ctx,
+            mode=MODE_AUDIO,
+            stream=True,
+        )
+        self.window.controller.chat.output.handle_end(
+            ctx=ctx,
+            mode=MODE_AUDIO,
+        )
+        self.window.controller.chat.common.show_response_tokens(ctx)
+
+    def shutdown(self):
+        """Shutdown all realtime threads and async loops"""
+        try:
+            self.window.core.api.openai.realtime.shutdown()
+        except Exception as e:
+            self.window.core.debug.log(f"[openai] Realtime shutdown error: {e}")
+        try:
+            self.window.core.api.google.realtime.shutdown()
+        except Exception as e:
+            self.window.core.debug.log(f"[google] Realtime shutdown error: {e}")
+        try:
+            self.manager.shutdown()
+        except Exception as e:
+            self.window.core.debug.log(f"[manager] Realtime shutdown error: {e}")
+
+    def reset(self):
+        """Reset realtime session"""
+        try:
+            self.window.core.api.openai.realtime.reset()
+        except Exception as e:
+            self.window.core.debug.log(f"[openai] Realtime reset error: {e}")
+        try:
+            self.window.core.api.google.realtime.reset()
+        except Exception as e:
+            self.window.core.debug.log(f"[google] Realtime reset error: {e}")
+
+    def is_supported(self) -> bool:
+        """
+        Check if current mode supports realtime
+
+        :return: True if mode supports realtime, False otherwise
+        """
+        mode = self.window.core.config.get("mode")
+        return mode in self.allowed_modes
+
+    def set_current_active(self, provider: str):
+        """
+        Set the current active realtime provider
+
+        :param provider: Provider name (openai, google)
+        """
+        self.current_active = provider.lower() if provider else None
+
+    def set_idle(self):
+        """Set kernel state to IDLE"""
+        QTimer.singleShot(0, lambda: self.window.dispatch(KernelEvent(KernelEvent.STATE_IDLE, {
+            "id": "realtime",
+        })))
+
+    def set_busy(self):
+        """Set kernel state to BUSY"""
+        QTimer.singleShot(0, lambda: self.window.dispatch(KernelEvent(KernelEvent.STATE_BUSY, {
+            "id": "realtime",
+        })))

pygpt_net/controller/theme/theme.py

@@ -15,7 +15,7 @@ from typing import Any, Optional
 from PySide6.QtWidgets import QApplication
 
 from pygpt_net.core.events import RenderEvent
-from pygpt_net.utils import trans
+from pygpt_net.utils import trans, freeze_updates
 
 from .common import Common
 from .markdown import Markdown
@@ -55,7 +55,8 @@ class Theme:
             return
         self.window.update_status(trans("status.reloading"))
         QApplication.processEvents()
-        self.toggle(name, force=True)
+        with freeze_updates(self.window):
+            self.toggle(name, force=True)
         self.window.update_status("")
 
     def toggle_option_by_menu(self, name: str, value: Any = None):

pygpt_net/controller/ui/mode.py

@@ -20,6 +20,7 @@ from pygpt_net.core.types import (
     MODE_COMPUTER,
     MODE_AGENT_OPENAI,
     MODE_COMPLETION,
+    MODE_AUDIO,
 )
 from pygpt_net.core.tabs.tab import Tab
 from pygpt_net.core.events import Event
@@ -55,6 +56,12 @@ class Mode:
         is_image = mode == MODE_IMAGE
         is_llama_index = mode == MODE_LLAMA_INDEX
         is_completion = mode == MODE_COMPLETION
+        is_audio = mode == MODE_AUDIO
+
+        if not is_audio:
+            self.window.ui.nodes['audio.auto_turn'].setVisible(False)
+        else:
+            self.window.ui.nodes['audio.auto_turn'].setVisible(True)
 
         if not is_assistant:
             ui_nodes['presets.widget'].setVisible(True)

pygpt_net/controller/ui/ui.py

@@ -13,6 +13,7 @@ from typing import Optional
 
 from PySide6.QtGui import QColor
 
+from pygpt_net.core.types import MODE_IMAGE
 from pygpt_net.core.events import BaseEvent, Event
 from pygpt_net.utils import trans
 
@@ -64,6 +65,7 @@ class UI:
         self.update_tokens()
         self.vision.update()
         self.window.controller.agent.legacy.update()
+        self.img_update_available_resolutions()
 
     def handle(self, event: BaseEvent):
         """
@@ -215,4 +217,20 @@ class UI:
     def on_global_stop(self):
         """Global stop button action"""
         if self.stop_action == "idx":
-            self.window.controller.idx.force_stop()
+            self.window.controller.idx.force_stop()
+
+    def img_update_available_resolutions(self):
+        """Update available resolutions for images"""
+        mode = self.window.core.config.get('mode')
+        if mode != MODE_IMAGE:
+            return
+        model = self.window.core.config.get('model')
+        keys = self.window.core.image.get_available_resolutions(model)
+        current = self.window.core.config.get('img_resolution', '1024x1024')
+        self.window.ui.config['global']['img_resolution'].set_keys(keys, lock=False)
+        self.window.controller.config.apply_value(
+            parent_id="global",
+            key="img_resolution",
+            option=self.window.core.image.get_resolution_option(),
+            value=current,
+        )

pygpt_net/controller/ui/vision.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.15 03:00:00                  #
+# Updated Date: 2025.08.28 09:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
@@ -40,9 +40,9 @@ class Vision:
         mode = self.window.core.config.get("mode")
         if mode == MODE_IMAGE:
             return False
-        if mode == MODE_VISION:
-            return True
-        if self.window.controller.plugins.is_type_enabled('vision'):
+        # if mode == MODE_VISION:
+            # return True
+        if self.window.controller.chat.vision.allowed():
             return True
         if self.is_vision_model() and mode in (
             MODE_CHAT,

pygpt_net/core/agents/legacy.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.06.28 16:00:00                  #
+# Updated Date: 2025.08.28 09:00:00                  #
 # ================================================== #
 
 from typing import List
@@ -32,7 +32,7 @@ class Legacy:
         self.allowed_modes = [
             MODE_CHAT,
             MODE_COMPLETION,
-            MODE_VISION,
+            # MODE_VISION,
             # MODE_LANGCHAIN,
             MODE_LLAMA_INDEX,
             MODE_AUDIO,

pygpt_net/core/agents/runners/openai_workflow.py

@@ -68,8 +68,8 @@ class OpenAIWorkflow(BaseRunner):
         context = agent_kwargs.get("context", BridgeContext())
         attachments = context.attachments if context else []
         history, previous_response_id = self.window.core.agents.memory.prepare_openai(context)
-        msg = self.window.core.gpt.vision.build_agent_input(prompt, attachments)  # build content with attachments
-        self.window.core.gpt.vision.append_images(ctx)  # append images to ctx if provided
+        msg = self.window.core.api.openai.vision.build_agent_input(prompt, attachments)  # build content with attachments
+        self.window.core.api.openai.vision.append_images(ctx)  # append images to ctx if provided
         history = history + msg
 
         # ------------ callbacks ----------------

pygpt_net/core/assistants/files.py

@@ -245,12 +245,12 @@ class Files:
             if store_id is None or store_id == "":
                 continue  # skip if no store_id
             try:
-                self.window.core.gpt.store.delete_store_file(store_id, file_id)  # remove from vector store
+                self.window.core.api.openai.store.delete_store_file(store_id, file_id)  # remove from vector store
             except Exception as e:
                 self.window.core.debug.log("Failed to delete file from vector store: " + str(e))
         self.provider.delete_by_id(file.record_id)  # delete file in DB
         try:
-            self.window.core.gpt.store.delete_file(file.file_id)  # delete file in API
+            self.window.core.api.openai.store.delete_file(file.file_id)  # delete file in API
         except Exception as e:
             self.window.core.debug.log("Failed to delete remote file: " + str(e))
         if file.record_id in self.items:
@@ -290,9 +290,9 @@ class Files:
         :return: True if truncated
         """
         if store_id is not None:
-            self.window.core.gpt.store.remove_from_store(store_id)  # remove files from vector store
+            self.window.core.api.openai.store.remove_from_store(store_id)  # remove files from vector store
         else:
-            self.window.core.gpt.store.remove_from_stores()  # remove files from all vector stores
+            self.window.core.api.openai.store.remove_from_stores()  # remove files from all vector stores
         return self.truncate_local(store_id)  # truncate files in DB
 
     def truncate_local(self, store_id: Optional[str] = None) -> bool:
@@ -316,7 +316,7 @@ class Files:
         :param store_id: store ID
         :return: True if imported
         """
-        files = self.window.core.gpt.store.import_store_files(store_id)
+        files = self.window.core.api.openai.store.import_store_files(store_id)
         for file in files:
             self.create(file.assistant, file.thread_id, file.file_id, file.name, file.path, file.size)
         return True

pygpt_net/core/assistants/store.py

@@ -106,7 +106,7 @@ class Store:
         :return: store item
         """
         name = "New vector store"
-        vector_store = self.window.core.gpt.store.create_store(name, 0)
+        vector_store = self.window.core.api.openai.store.create_store(name, 0)
         if vector_store is None:
             return None
         store = AssistantStoreItem()
@@ -124,7 +124,7 @@ class Store:
         :param store: store
         :return: updated store or None if failed
         """
-        vector_store = self.window.core.gpt.store.update_store(store.id, store.name, store.expire_days)
+        vector_store = self.window.core.api.openai.store.update_store(store.id, store.name, store.expire_days)
         if vector_store is None:
             return None
         self.items[store.id] = store
@@ -139,7 +139,7 @@ class Store:
         :return: status data, store data
         """
         status = {}
-        data = self.window.core.gpt.store.get_store(id)
+        data = self.window.core.api.openai.store.get_store(id)
         if data is not None:
             status = self.parse_status(data)
         return status, data
@@ -236,7 +236,7 @@ class Store:
         if id in self.items:
             store = self.items[id]
             self.provider.delete_by_id(store.record_id)
-            self.window.core.gpt.store.remove_store(id)
+            self.window.core.api.openai.store.remove_store(id)
             del self.items[id]
             return True
         return False

pygpt_net/core/audio/audio.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2025.08.07 22:00:00                  #
+# Updated Date: 2025.08.31 23:00:00                  #
 # ================================================== #
 
 import hashlib
@@ -42,6 +42,11 @@ class Audio:
         }
         self.last_error = None
 
+    def setup(self):
+        """Initialize audio core"""
+        self.capture.setup()
+        self.output.setup()
+
     def get_input_devices(self) -> List[Tuple[int, str]]:
         """
         Get input devices

pygpt_net/core/audio/backend/native/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2025.08.31 23:00:00                  #
+# ================================================== #
+
+from .native import NativeBackend