pygpt-net 2.4.30__py3-none-any.whl → 2.4.34__py3-none-any.whl

pygpt_net/item/ctx.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import copy
@@ -58,6 +58,7 @@ class CtxItem:
         self.current = False
         self.internal = False
         self.is_vision = False
+        self.is_audio = False
         self.idx = 0
         self.first = False
         self.live = False  # True if is current flow (not loaded from DB)
@@ -72,6 +73,9 @@ class CtxItem:
         self.sub_reply = False  # sub call reply
         self.hidden = False  # hidden context
         self.pid = 0
+        self.audio_id = None
+        self.audio_output = None
+        self.audio_expires_ts = 0
 
 
     @property
@@ -193,6 +197,8 @@ class CtxItem:
             "thread": self.thread,
             "msg_id": self.msg_id,
             "run_id": self.run_id,
+            "audio_id": self.audio_id,
+            "audio_expires_ts": self.audio_expires_ts,
             "input_name": self.input_name,
             "output_name": self.output_name,
             "input_timestamp": self.input_timestamp,
@@ -246,6 +252,8 @@ class CtxItem:
         self.thread = data.get("thread", None)
         self.msg_id = data.get("msg_id", None)
         self.run_id = data.get("run_id", None)
+        self.audio_id = data.get("audio_id", None)
+        self.audio_expires_ts = data.get("audio_expires_ts", None)
         self.input_name = data.get("input_name", None)
         self.output_name = data.get("output_name", None)
         self.input_timestamp = data.get("input_timestamp", None)

pygpt_net/item/preset.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.15 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,7 @@ class PresetItem:
         self.agent = False
         self.agent_llama = False
         self.expert = False
+        self.audio = False
         self.temperature = 1.0
         self.filename = None
         self.model = None
@@ -63,6 +64,7 @@ class PresetItem:
             "agent": self.agent,
             "agent_llama": self.agent_llama,
             "expert": self.expert,
+            "audio": self.audio,
             "temperature": self.temperature,
             "filename": self.filename,
             "model": self.model,
@@ -105,6 +107,8 @@ class PresetItem:
             self.agent_llama = data["agent_llama"]
         if "expert" in data:
             self.expert = data["expert"]
+        if "audio" in data:
+            self.audio = data["audio"]
         if "temperature" in data:
             self.temperature = data["temperature"]
         if "filename" in data:

pygpt_net/launcher.py

@@ -13,6 +13,7 @@ import sys
 import argparse
 from logging import ERROR, WARNING, INFO, DEBUG
 
+from PySide6 import QtCore
 from PySide6.QtCore import QCoreApplication, Qt
 from PySide6.QtGui import QScreen
 from PySide6.QtWidgets import QApplication
@@ -100,6 +101,7 @@ class Launcher:
         Platforms.prepare()  # setup platform specific options
         QCoreApplication.setAttribute(Qt.AA_ShareOpenGLContexts)
         self.app = QApplication(sys.argv)
+        self.app.setAttribute(QtCore.Qt.AA_DontUseNativeMenuBar)
         self.window = MainWindow(self.app, args=args)
         self.shortcut_filter = GlobalShortcutFilter(self.window)
 
@@ -225,7 +227,7 @@ class Launcher:
 
     def add_agent(self, agent: BaseAgent):
         """
-        Register agent (Llama-index agent)
+        Register agent (LlamaIndex agent)
 
         :param agent: Agent instance
         """

pygpt_net/migrations/Version20241126170000.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.26 19:00:00                  #
+# ================================================== #
+
+from sqlalchemy import text
+
+from .base import BaseMigration
+
+
+class Version20241126170000(BaseMigration):
+    def __init__(self, window=None):
+        super(Version20241126170000, self).__init__(window)
+        self.window = window
+
+    def up(self, conn):
+        conn.execute(text("""
+        ALTER TABLE ctx_item ADD COLUMN audio_id TEXT;
+        """))
+        conn.execute(text("""
+        ALTER TABLE ctx_item ADD COLUMN audio_expires_ts INTEGER;
+        """))

pygpt_net/migrations/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from .Version20231227152900 import Version20231227152900  # 2.0.59
@@ -21,6 +21,7 @@ from .Version20240408180000 import Version20240408180000  # 2.1.41
 from .Version20240426050000 import Version20240426050000  # 2.1.79
 from .Version20240501030000 import Version20240501030000  # 2.2.7
 from .Version20241122130000 import Version20241122130000  # 2.4.21
+from .Version20241126170000 import Version20241126170000  # 2.4.34
 
 class Migrations:
     def __init__(self):
@@ -46,4 +47,5 @@ class Migrations:
             Version20240426050000(),  # 2.1.79
             Version20240501030000(),  # 2.2.7
             Version20241122130000(),  # 2.4.21
+            Version20241126170000(),  # 2.4.34
         ]

pygpt_net/plugin/audio_input/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import os
@@ -343,6 +343,7 @@ class Plugin(BasePlugin):
             worker.signals.destroyed.connect(self.handle_destroy)
             worker.signals.started.connect(self.handle_started)
             worker.signals.stopped.connect(self.handle_stop)
+            worker.signals.on_realtime.connect(self.handle_realtime)
 
             worker.run_async()
 
@@ -371,6 +372,15 @@ class Plugin(BasePlugin):
         """
         self.window.ui.plugin_addon['audio.input'].set_status(status)
 
+    @Slot(str)
+    def handle_realtime(self, path: str):
+        """
+        Handle realtime audio input
+
+        :param path: audio input file path
+        """
+        self.window.controller.chat.audio.handle_input(path)
+
     @Slot(object, object)
     def handle_input(self, text: str, ctx: CtxItem = None):
         """

pygpt_net/plugin/audio_input/worker.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.18 21:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import os.path
@@ -22,6 +22,7 @@ from pygpt_net.plugin.base.worker import BaseWorker, BaseSignals
 
 class WorkerSignals(BaseSignals):
     transcribed = Signal(str, str)
+    on_realtime = Signal(str)
 
 
 class Worker(BaseWorker):
@@ -73,6 +74,13 @@ class Worker(BaseWorker):
             if os.path.exists(self.path):
                 # set status
                 self.status(trans('audio.speak.wait'))
+
+                # if multimodal audio, then only return path to audio file and do not transcribe
+                if self.plugin.window.controller.chat.audio.enabled():
+                    self.signals.on_realtime.emit(self.path)
+                    self.status('')
+                    return
+
                 # transcribe audio
                 transcript = self.plugin.get_provider().transcribe(self.path)
                 self.status('')

pygpt_net/plugin/audio_output/__init__.py

@@ -6,11 +6,12 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 03:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import Slot
 
+from pygpt_net.core.types import MODE_AUDIO
 from pygpt_net.plugin.base.plugin import BasePlugin
 from pygpt_net.provider.audio_output.base import BaseProvider
 from pygpt_net.core.events import Event
@@ -118,15 +119,21 @@ class Plugin(BasePlugin):
         name = event.name
         data = event.data
         ctx = event.ctx
+        mode = self.window.core.config.get("mode")
 
         if name == Event.INPUT_BEFORE:
             self.on_input_before(data['value'])
 
-        elif name in [
-            Event.CTX_AFTER,
-            Event.AUDIO_READ_TEXT
-        ]:
-            self.on_ctx_after(ctx, event)
+        elif name == Event.CTX_AFTER:
+            if mode == MODE_AUDIO:
+                return  # skip if audio mode
+            self.on_generate(ctx, event)
+
+        elif name == Event.AUDIO_READ_TEXT:
+            self.on_generate(ctx, event)
+
+        elif name == Event.AUDIO_PLAYBACK:
+            self.on_playback(ctx, event)
 
         elif name == Event.AUDIO_OUTPUT_STOP:
             self.stop_audio()
@@ -139,7 +146,7 @@ class Plugin(BasePlugin):
         """
         self.input_text = text
 
-    def on_ctx_after(self, ctx: CtxItem, event: Event):
+    def on_generate(self, ctx: CtxItem, event: Event):
         """
         Events: CTX_AFTER, AUDIO_READ_TEXT
 
@@ -166,6 +173,7 @@ class Plugin(BasePlugin):
                 worker.event = name
                 worker.cache_file = cache_file
                 worker.text = self.window.core.audio.clean_text(text)
+                worker.mode = "generate"
 
                 # signals
                 worker.signals.playback.connect(self.handle_playback)
@@ -180,6 +188,28 @@ class Plugin(BasePlugin):
         except Exception as e:
             self.error(e)
 
+    def on_playback(self, ctx: CtxItem, event: Event):
+        """
+        Events: AUDIO_PLAYBACK
+
+        :param ctx: CtxItem
+        :param event: Event
+        """     
+        try:
+            worker = Worker()
+            worker.from_defaults(self)
+            worker.audio_file = event.data["audio_file"]
+            worker.mode = "playback"
+
+            # signals
+            worker.signals.playback.connect(self.handle_playback)
+            worker.signals.stop.connect(self.handle_stop)
+
+            worker.run_async()
+
+        except Exception as e:
+            self.error(e)
+
     def destroy(self):
         """Destroy thread"""
         pass

pygpt_net/plugin/audio_output/worker.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.18 21:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import time
@@ -30,28 +30,50 @@ class Worker(BaseWorker):
         self.text = None
         self.event = None
         self.cache_file = None  # path to cache file
+        self.mode = "generate"  # generate|playback
+        self.audio_file = None
 
     @Slot()
     def run(self):
-        from pygame import mixer
         try:
-            if self.text is None or self.text == "":
-                time.sleep(0.2)  # wait
-                return
-            path = self.plugin.get_provider().speech(self.text)
-            if path:
-                mixer.init()
-                playback = mixer.Sound(path)
-                self.stop_playback()  # stop previous playback
-                playback.play()
-                self.send(playback)  # send playback object to main thread
-
-                # store in cache if enabled
-                if self.cache_file:
-                    self.cache_audio_file(path, self.cache_file)
+            if self.mode == "generate":
+                self.generate()
+            elif self.mode == "playback":
+                self.play()
         except Exception as e:
             self.error(e)
 
+    def generate(self):
+        """
+        Generate and play audio file
+        """
+        if self.text is None or self.text == "":
+            time.sleep(0.2)  # wait
+            return
+        path = self.plugin.get_provider().speech(self.text)
+        if path:
+            from pygame import mixer
+            mixer.init()
+            playback = mixer.Sound(path)
+            self.stop_playback()  # stop previous playback
+            playback.play()
+            self.send(playback)  # send playback object to main thread to allow force stop
+
+            # store in cache if enabled
+            if self.cache_file:
+                self.cache_audio_file(path, self.cache_file)
+
+    def play(self):
+        """
+        Play audio file only
+        """
+        if self.audio_file:
+            from pygame import mixer
+            mixer.init()
+            playback = mixer.Sound(self.audio_file)
+            playback.play()
+            self.send(playback)  # send playback object to main thread to allow force stop
+
     def cache_audio_file(self, src: str, dst: str):
         """
         Store audio file in cache
@@ -81,28 +103,3 @@ class Worker(BaseWorker):
     def stop(self):
         """Send stop signal to main thread"""
         self.signals.stop.emit()
-
-
-class PlayWorker(BaseWorker):
-    def __init__(self, *args, **kwargs):
-        super(PlayWorker, self).__init__()
-        self.signals = WorkerSignals()
-        self.args = args
-        self.kwargs = kwargs
-        self.window = None
-        self.path = None
-
-    @Slot()
-    def run(self):
-        from pygame import mixer
-        try:
-            if self.path:
-                mixer.init()
-                playback = mixer.Sound(self.path)
-                playback.play()
-        except Exception as e:
-            self.error(e)
-
-
-
-

pygpt_net/plugin/cmd_code_interpreter/runner.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.25 02:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 import os.path
@@ -140,7 +140,7 @@ class Runner:
 
         # append to ctx
         ctx.files = paths
-        ctx.images = images_list
+        ctx.images = self.plugin.window.core.filesystem.make_local_list(list(images_list))
         return paths
 
     def handle_result_ipython(self, ctx: CtxItem, response) -> str:

pygpt_net/plugin/cmd_mouse_control/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.24 04:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import Slot, QTimer
@@ -141,7 +141,9 @@ class Plugin(BasePlugin):
             self.window.controller.attachment.clear_silent()
             path = self.window.controller.painter.capture.screenshot(attach_cursor=True,
                                                                      silent=True)  # attach screenshot
-            ctx.images.append(path)
+            
+            img_path = self.window.core.filesystem.make_local(path)
+            ctx.images.append(img_path)
             #ctx.images_before.append(path)
 
         context = BridgeContext()

pygpt_net/plugin/openai_dalle/__init__.py

@@ -6,13 +6,14 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
     MODE_ASSISTANT,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
@@ -44,6 +45,7 @@ class Plugin(BasePlugin):
             MODE_LLAMA_INDEX,
             MODE_ASSISTANT,
             MODE_AGENT,
+            MODE_AUDIO,
         ]
         self.allowed_cmds = [
             "image",

pygpt_net/plugin/openai_vision/__init__.py

@@ -6,12 +6,13 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.24 04:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
+    MODE_AUDIO,
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
     MODE_VISION,
@@ -59,6 +60,7 @@ class Plugin(BasePlugin):
             MODE_AGENT_LLAMA,
             MODE_LLAMA_INDEX,
             MODE_LANGCHAIN,
+            MODE_AUDIO,
         ]
         self.worker = None
         self.config = Config(self)

pygpt_net/provider/core/attachment/json_file.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2023.12.31 04:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 import json
@@ -143,6 +143,7 @@ class JsonFileProvider(BaseProvider):
             'remote': attachment.remote,
             'send': attachment.send,
             'vector_store_ids': attachment.vector_store_ids,
+            'type': attachment.type,
         }
 
     @staticmethod
@@ -165,6 +166,8 @@ class JsonFileProvider(BaseProvider):
             attachment.send = data['send']
         if 'vector_store_ids' in data:
             attachment.vector_store_ids = data['vector_store_ids']
+        if 'type' in data:
+            attachment.type = data['type']
 
     def dump(self, item: AttachmentItem) -> str:
         """

pygpt_net/provider/core/config/patch.py

@@ -1701,6 +1701,22 @@ class Patch:
                     del data['plugins']['cmd_code_interpreter']['ipython_dockerfile']
                 updated = True
 
+            # < 2.4.31
+            if old < parse_version("2.4.31"):
+                print("Migrating config from < 2.4.31...")
+                if 'attachments_auto_index' not in data:
+                    data["attachments_auto_index"] = self.window.core.config.get_base(
+                        'attachments_auto_index')
+                updated = True
+
+            # < 2.4.34
+            if old < parse_version("2.4.34"):
+                print("Migrating config from < 2.4.34...")
+                if 'ctx.attachment.query.model' not in data:
+                    data["ctx.attachment.query.model"] = self.window.core.config.get_base(
+                        'ctx.attachment.query.model')
+                updated = True
+
         # update file
         migrated = False
         if updated:

pygpt_net/provider/core/ctx/db_sqlite/storage.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from datetime import datetime
@@ -718,7 +718,9 @@ class Storage:
                 output_tokens,
                 total_tokens,
                 is_internal,
-                docs_json
+                docs_json,
+                audio_id,
+                audio_expires_ts
             )
             VALUES 
             (
@@ -749,7 +751,9 @@ class Storage:
                 :output_tokens,
                 :total_tokens,
                 :is_internal,
-                :docs_json
+                :docs_json,
+                :audio_id,
+                :audio_expires_ts
             )
         """).bindparams(
             meta_id=int(meta.id),
@@ -780,6 +784,8 @@ class Storage:
             total_tokens=int(item.total_tokens or 0),
             is_internal=int(item.internal),
             docs_json=pack_item_value(item.doc_ids),
+            audio_id=item.audio_id,
+            audio_expires_ts=int(item.audio_expires_ts or 0)
         )
         with db.begin() as conn:
             result = conn.execute(stmt)
@@ -822,7 +828,9 @@ class Storage:
                 output_tokens = :output_tokens,
                 total_tokens = :total_tokens,
                 is_internal = :is_internal,
-                docs_json = :docs_json
+                docs_json = :docs_json,
+                audio_id = :audio_id,
+                audio_expires_ts = :audio_expires_ts
             WHERE id = :id
         """).bindparams(
             id=item.id,
@@ -852,6 +860,8 @@ class Storage:
             total_tokens=int(item.total_tokens or 0),
             is_internal=int(item.internal or 0),
             docs_json=pack_item_value(item.doc_ids),
+            audio_id=item.audio_id,
+            audio_expires_ts=int(item.audio_expires_ts or 0)
         )
         with db.begin() as conn:
             conn.execute(stmt)

pygpt_net/provider/core/ctx/db_sqlite/utils.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import json
@@ -158,11 +158,28 @@ def unpack_item(item: CtxItem, row: dict) -> CtxItem:
     item.total_tokens = unpack_var(row['total_tokens'], 'int')
     item.internal = unpack_var(row['is_internal'], 'bool')
     item.doc_ids = unpack_item_value(row['docs_json'])
-
+    item.audio_id = row['audio_id']
+    item.audio_expires_ts = row['audio_expires_ts']
+
+    # set defaults
+    if item.cmds is None:
+        item.cmds = []
+    if item.results is None:
+        item.results = []
+    if item.urls is None:
+        item.urls = []
+    if item.images is None:
+        item.images = []
+    if item.files is None:
+        item.files = []
+    if item.attachments is None:
+        item.attachments = []
     if item.additional_ctx is None:
         item.additional_ctx = []
     if item.doc_ids is None:
         item.doc_ids = []
+    if item.extra is None:
+        item.extra = {}
     return item
 
 

pygpt_net/provider/core/model/patch.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.15 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from packaging.version import parse as parse_version, Version
@@ -308,6 +308,12 @@ class Patch:
                             model.mode.append("agent_llama")
                 updated = True
 
+            # < 2.4.34 <--- add gpt-4o-audio-preview, gpt-4o-2024-11-20
+            if old < parse_version("2.4.34"):
+                print("Migrating models from < 2.4.34...")
+                # add missing gpt-4o-audio-preview, gpt-4o-2024-11-20
+                updated = True
+
         # update file
         if updated:
             data = dict(sorted(data.items()))