pygpt-net 2.4.30__py3-none-any.whl → 2.4.35__py3-none-any.whl

pygpt_net/controller/chat/audio.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.26 19:00:00                  #
+# ================================================== #
+
+import base64
+import os
+
+from pygpt_net.core.types import (
+    MODE_AUDIO,
+)
+from pygpt_net.core.bridge.context import MultimodalContext, BridgeContext
+from pygpt_net.core.events import KernelEvent
+from pygpt_net.item.ctx import CtxItem
+
+
+class Audio:
+    def __init__(self, window=None):
+        """
+        Chat audio controller
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.audio_file = "chat_output.wav"
+        self.tmp_input = False
+        self.tmp_output = False
+
+    def setup(self):
+        """Set up UI"""
+        pass
+
+    def update(self):
+        """Update input/output audio"""
+        mode = self.window.core.config.get("mode")
+        if mode == MODE_AUDIO:
+            if not self.window.controller.audio.is_output_enabled():
+                self.window.controller.audio.enable_output()
+                self.tmp_output = True
+            else:
+                self.tmp_output = False
+            if not self.window.controller.audio.is_input_enabled():
+                self.window.controller.audio.enable_input()
+                self.tmp_input = True
+            else:
+                self.tmp_input = False
+        else:
+            if self.tmp_output:
+                self.window.controller.audio.disable_output()
+            if self.tmp_input:
+                self.window.controller.audio.disable_input()
+
+    def handle_output(self, ctx: CtxItem):
+        """
+        Handle output audio
+
+        :param ctx: Context item
+        """
+        wav_path = os.path.join(self.window.core.config.get_user_path(), self.audio_file)
+        if ctx.is_audio and ctx.audio_output:
+            wav_bytes = base64.b64decode(ctx.audio_output)
+            with open(wav_path, "wb") as f:
+                f.write(wav_bytes)
+            self.window.controller.audio.play_chat_audio(wav_path)
+
+    def handle_input(self, path: str):
+        """
+        Handle input audio
+
+        :param path: audio file path
+        """
+        multimodal_ctx = MultimodalContext()
+        with open(path, "rb") as f:
+            multimodal_ctx.audio_data = f.read()
+        multimodal_ctx.is_audio_input = True
+
+        bridge_ctx = BridgeContext()
+        bridge_ctx.prompt = self.window.ui.nodes['input'].toPlainText()  # attach text input
+        bridge_ctx.multimodal_ctx = multimodal_ctx
+        event = KernelEvent(KernelEvent.INPUT_USER, {
+            'context': bridge_ctx,
+            'extra': {},
+        })
+        self.window.dispatch(event)
+
+    def enabled(self) -> bool:
+        """
+        Check if audio mode is enabled
+
+        :return: bool True if enabled
+        """
+        return self.window.core.config.get("mode") == MODE_AUDIO
+

pygpt_net/controller/chat/input.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.bridge import BridgeContext
+from pygpt_net.core.bridge.context import MultimodalContext
 from pygpt_net.core.events import Event, AppEvent, KernelEvent, RenderEvent
 from pygpt_net.core.types import (
     MODE_AGENT,
@@ -134,6 +135,7 @@ class Input:
         reply = extra.get("reply", False)
         internal = extra.get("internal", False)
         parent_id = extra.get("parent_id", None)
+        multimodal_ctx = context.multimodal_ctx
         self.execute(
             text=text,
             force=force,
@@ -141,6 +143,7 @@ class Input:
             internal=internal,
             prev_ctx=prev_ctx,
             parent_id=parent_id,
+            multimodal_ctx=multimodal_ctx,
         )
 
     def execute(
@@ -151,6 +154,7 @@ class Input:
             internal: bool = False,
             prev_ctx: CtxItem = None,
             parent_id: int = None,
+            multimodal_ctx: MultimodalContext = None,
     ):
         """
         Execute send input text to API
@@ -161,6 +165,7 @@ class Input:
         :param internal: internal call
         :param prev_ctx: previous context (if reply)
         :param parent_id: parent id (if expert)
+        :param multimodal_ctx: multimodal context
         """
         self.window.dispatch(KernelEvent(KernelEvent.STATE_IDLE, {
             "id": "chat",
@@ -208,8 +213,9 @@ class Input:
         camera_captured = (self.window.controller.ui.vision.has_vision()
                            and self.window.controller.attachment.has(mode))
 
-        # allow empty input only for vision modes, otherwise abort
-        if len(text.strip()) == 0 and not camera_captured:
+        # allow empty text input only if multimodal data, otherwise abort
+        is_audio = multimodal_ctx is not None and multimodal_ctx.is_audio_input
+        if len(text.strip()) == 0 and (not camera_captured and not is_audio):
             self.generating = False  # unlock as not generating
             return
 
@@ -256,6 +262,7 @@ class Input:
                 internal=internal,
                 prev_ctx=prev_ctx,
                 parent_id=parent_id,
+                multimodal_ctx=multimodal_ctx,
             )  # text mode: OpenAI, Langchain, Llama, etc.
 
     def log(self, data: any):

pygpt_net/controller/chat/output.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
@@ -111,6 +111,9 @@ class Output:
         # update response tokens
         self.window.controller.chat.common.show_response_tokens(ctx)
 
+        # handle audio output
+        self.window.controller.chat.audio.handle_output(ctx)
+
         # store to history
         if self.window.core.config.get('store_history'):
             self.window.core.history.append(ctx, "output")

pygpt_net/controller/chat/text.py

@@ -6,17 +6,18 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.25 01:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
+    MODE_AUDIO,
     MODE_ASSISTANT,
     MODE_LLAMA_INDEX,
 )
 from pygpt_net.core.events import Event, AppEvent, KernelEvent, RenderEvent
-from pygpt_net.core.bridge.context import BridgeContext
+from pygpt_net.core.bridge.context import BridgeContext, MultimodalContext
 from pygpt_net.item.ctx import CtxItem
 from pygpt_net.utils import trans
 
@@ -38,6 +39,7 @@ class Text:
             internal: bool = False,
             prev_ctx: CtxItem = None,
             parent_id: str = None,
+            multimodal_ctx: MultimodalContext = None,
     ) -> CtxItem:
         """
         Send text message
@@ -47,6 +49,7 @@ class Text:
         :param internal: internal call
         :param prev_ctx: previous context item (if reply)
         :param parent_id: parent context id
+        :param multimodal_ctx: multimodal context
         :return: context item
         """
         self.window.update_status(trans('status.sending'))
@@ -80,7 +83,7 @@ class Text:
         tools_outputs = []  # tools outputs (assistant only)
 
         # o1 models: disable stream mode
-        if model.startswith("o1") or mode == MODE_AGENT_LLAMA:
+        if model.startswith("o1") or mode in [MODE_AGENT_LLAMA, MODE_AUDIO]:
             stream_mode = False
 
         # create ctx item
@@ -233,6 +236,7 @@ class Text:
                 external_functions=functions,  # external functions
                 tools_outputs=tools_outputs,  # if not empty then will submit outputs to assistant
                 max_tokens=max_tokens,  # max output tokens
+                multimodal_ctx=multimodal_ctx,  # multimodal context
             )
             extra = {
                 'mode': mode,

pygpt_net/controller/dialogs/confirm.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 class Confirm:
@@ -349,6 +349,18 @@ class Confirm:
         elif type == 'ctx.group':
             self.window.controller.ctx.create_group(name, id)
 
+    def accept_url(self, type: str, id: any, url: str):
+        """
+        Update URL provided
+
+        :param type: dialog type
+        :param id: dialog object id
+        :param url: URL
+        """
+        # add attachment
+        if type == 'attachment':
+            self.window.controller.attachment.add_url(url)
+
     def dismiss_rename(self):
         """Dismiss rename dialog"""
         self.window.ui.dialog['rename'].close()
@@ -356,3 +368,7 @@ class Confirm:
     def dismiss_create(self):
         """Dismiss create dialog"""
         self.window.ui.dialog['create'].close()
+
+    def dismiss_url(self):
+        """Dismiss url dialog"""
+        self.window.ui.dialog['url'].close()

pygpt_net/controller/lang/custom.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import Qt
@@ -14,6 +14,7 @@ from PySide6.QtCore import Qt
 from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_EXPERT,
@@ -63,6 +64,7 @@ class Custom:
         self.window.ui.config['preset'][MODE_AGENT].box.setText(trans("preset.agent"))
         self.window.ui.config['preset'][MODE_AGENT_LLAMA].box.setText(trans("preset.agent_llama"))
         self.window.ui.config['preset'][MODE_EXPERT].box.setText(trans("preset.expert"))
+        self.window.ui.config['preset'][MODE_AUDIO].box.setText(trans("preset.audio"))
 
         self.window.ui.config['global']['img_raw'].setText(trans("img.raw"))
 

pygpt_net/controller/mode.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.events import Event, AppEvent
@@ -41,6 +41,7 @@ class Mode:
         self.window.dispatch(event)
         self.window.controller.attachment.update()
         self.window.controller.chat.attachment.update()
+        self.window.controller.chat.audio.update()
         self.window.dispatch(AppEvent(AppEvent.MODE_SELECTED))  # app event
 
     def set(self, mode: str):

pygpt_net/controller/painter/capture.py

@@ -85,7 +85,7 @@ class Capture:
 
             # clear attachments before capture if needed
             if self.window.controller.attachment.is_capture_clear():
-                self.window.controller.attachment.clear(True, auto=True)
+                self.window.controller.attachment.clear(True, auto=True, force=True)
 
         try:
             # prepare filename
@@ -127,7 +127,7 @@ class Capture:
 
         # clear attachments before capture if needed
         if self.window.controller.attachment.is_capture_clear():
-            self.window.controller.attachment.clear(True, auto=True)
+            self.window.controller.attachment.clear(True, auto=True, force=True)
 
         try:
             # prepare filename

pygpt_net/controller/presets/editor.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import datetime
@@ -16,12 +16,14 @@ from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
     MODE_ASSISTANT,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_EXPERT,
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
-    MODE_VISION, MODE_IMAGE,
+    MODE_VISION, 
+    MODE_IMAGE,
 )
 from pygpt_net.item.preset import PresetItem
 from pygpt_net.utils import trans
@@ -87,6 +89,10 @@ class Editor:
                 "type": "bool",
                 "label": "preset.agent",
             },
+            MODE_AUDIO: {
+                "type": "bool",
+                "label": "preset.audio",
+            },
             # "assistant": {
             # "type": "bool",
             # "label": "preset.assistant",
@@ -287,6 +293,8 @@ class Editor:
                 data.agent = True
             elif mode == MODE_AGENT_LLAMA:
                 data.agent_llama = True
+            elif mode == MODE_AUDIO:
+                data.audio = True
 
         options = {}
         data_dict = data.to_dict()
@@ -322,6 +330,10 @@ class Editor:
             self.window.ui.config[self.id]['tool.function'].model.updateData([])
 
         # set focus to name field
+        current_model = self.window.core.config.get('model')
+        # set current model in combo box as selected
+        if id is None:
+            self.window.ui.config[self.id]['model'].set_value(current_model)
         self.window.ui.config[self.id]['name'].setFocus()
         self.show_hide_by_mode()
 
@@ -347,6 +359,7 @@ class Editor:
             MODE_LLAMA_INDEX,
             MODE_EXPERT,
             MODE_AGENT_LLAMA,
+            MODE_AUDIO,
         ]
 
         # disallow editing default preset

pygpt_net/controller/ui/__init__.py

@@ -102,6 +102,8 @@ class UI:
         prompt = str(self.window.ui.nodes['input'].toPlainText().strip())
         input_tokens, system_tokens, extra_tokens, ctx_tokens, ctx_len, ctx_len_all, \
             sum_tokens, max_current, threshold = self.window.core.tokens.get_current(prompt)
+        attachments_tokens = self.window.controller.chat.attachment.get_current_tokens()
+        sum_tokens += attachments_tokens
 
         # ctx tokens
         ctx_string = "{} / {} - {} {}".format(
@@ -119,11 +121,12 @@ class UI:
         parsed_max_current = str(int(max_current))
         parsed_max_current = parsed_max_current.replace("000000", "M").replace("000", "k")
 
-        input_string = "{} + {} + {} + {} = {} / {}".format(
+        input_string = "{} + {} + {} + {} + {} = {} / {}".format(
             input_tokens,
             system_tokens,
             ctx_tokens,
             extra_tokens,
+            attachments_tokens,
             parsed_sum,
             parsed_max_current
         )

pygpt_net/core/access/voice.py

@@ -57,8 +57,8 @@ class Voice:
             ControlEvent.CTX_SEARCH_CLEAR: "Clear the search results",
             ControlEvent.INPUT_SEND: "Send the message to input",
             ControlEvent.INPUT_APPEND: "Append message to current input without sending it",
-            ControlEvent.MODE_CHAT: "Switch to chat mode",
-            ControlEvent.MODE_LLAMA_INDEX: "Switch to chat with files (llama-index) mode",
+            ControlEvent.MODE_CHAT: "Switch to Chat mode",
+            ControlEvent.MODE_LLAMA_INDEX: "Switch to Chat with Files (LlamaIndex) mode",
             ControlEvent.MODE_NEXT: "Switch to the next mode",
             ControlEvent.MODE_PREV: "Switch to the previous mode",
             ControlEvent.MODEL_NEXT: "Switch to the next model",

pygpt_net/core/agents/legacy.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
@@ -15,6 +15,7 @@ from pygpt_net.core.types import (
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
     MODE_VISION,
+    MODE_AUDIO,
 )
 
 class Legacy:
@@ -31,6 +32,7 @@ class Legacy:
             MODE_VISION,
             MODE_LANGCHAIN,
             MODE_LLAMA_INDEX,
+            MODE_AUDIO,
         ]
 
     def get_allowed_modes(self) -> list:

pygpt_net/core/attachments/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 import copy
@@ -176,7 +176,8 @@ class Attachments:
             mode: str,
             name: str = None,
             path: str = None,
-            auto_save: bool = True
+            auto_save: bool = True,
+            type: str = AttachmentItem.TYPE_FILE,
     ) -> AttachmentItem:
         """
         Create new attachment
@@ -185,16 +186,19 @@ class Attachments:
         :param name: name
         :param path: path
         :param auto_save: auto_save
+        :param type: type
         :return: AttachmentItem
         """
         # make local copy of external attachment if enabled
-        if self.window.core.config.get("upload.store"):
-            if not self.window.core.filesystem.in_work_dir(path):
-                path = self.window.core.filesystem.store_upload(path)
+        if type == AttachmentItem.TYPE_FILE and path is not None:
+            if self.window.core.config.get("upload.store"):
+                if not self.window.core.filesystem.in_work_dir(path):
+                    path = self.window.core.filesystem.store_upload(path)
 
         attachment = self.create()
         attachment.name = name
         attachment.path = path
+        attachment.type = type
 
         if mode not in self.items:
             self.items[mode] = {}
@@ -276,19 +280,20 @@ class Attachments:
             del self.items[mode][id]
             self.save()
 
-    def delete_all(self, mode: str, remove_local: bool = False, auto: bool = False):
+    def delete_all(self, mode: str, remove_local: bool = False, auto: bool = False, force: bool = False):
         """
         Delete all attachments
 
         :param mode: mode
         :param remove_local: remove local copy
         :param auto: auto delete
+        :param force: force delete
         """
         if mode not in self.items:
             self.items[mode] = {}
 
         for id in list(self.items[mode].keys()):
-            if not self.items[mode][id].consumed and auto:
+            if (not self.items[mode][id].consumed and auto) and not force:
                 continue
             if remove_local:
                 self.window.core.filesystem.remove_upload(
@@ -410,7 +415,7 @@ class Attachments:
         for mode in self.items:
             for id in self.items[mode]:
                 attachment = self.items[mode][id]
-                if attachment.path is not None:
+                if attachment.path is not None and attachment.type == AttachmentItem.TYPE_FILE:
                     attachment.path = self.window.core.filesystem.to_workdir(
                         attachment.path,
                     )
@@ -422,7 +427,7 @@ class Attachments:
         for mode in data:
             for id in data[mode]:
                 attachment = data[mode][id]
-                if attachment.path is not None:
+                if attachment.path is not None and attachment.type == AttachmentItem.TYPE_FILE:
                     attachment.path = self.window.core.filesystem.make_local(
                         attachment.path,
                     )