pygpt-net 2.4.28__py3-none-any.whl → 2.4.34__py3-none-any.whl

pygpt_net/controller/chat/attachment.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 21:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 import os
@@ -114,52 +114,123 @@ class Attachment(QObject):
         :return: True if uploaded
         """
         self.uploaded = False
+        auto_index = self.window.core.config.get("attachments_auto_index", False)
         attachments = self.window.core.attachments.get_all(mode, only_files=True)
+
         if self.is_verbose() and len(attachments) > 0:
             print("\nUploading attachments...\nWork Mode: {}".format(mode))
+
         for uuid in attachments:
             attachment = attachments[uuid]
-            if not self.is_allowed(attachment.path):
-                continue  # skip not allowed files
-            if self.window.core.filesystem.packer.is_archive(attachment.path):
-                if self.is_verbose():
-                    print("Unpacking archive: {}".format(attachment.path))
-                tmp_path = self.window.core.filesystem.packer.unpack(attachment.path)
-                if tmp_path:
-                    for root, dirs, files in os.walk(tmp_path):
-                        for file in files:
-                            path = os.path.join(root, file)
-                            sub_attachment = AttachmentItem()
-                            sub_attachment.path = path
-                            sub_attachment.name = os.path.basename(path)
-                            sub_attachment.consumed = False
-                            path_relative = os.path.relpath(path, tmp_path)
-                            if self.is_allowed(str(path)):
-                                if self.is_verbose():
-                                    print("Uploading unpacked from archive: {}".format(path_relative))
-                                item = self.window.core.attachments.context.upload(meta, sub_attachment, prompt)
-                                if item:
-                                    item["path"] = os.path.basename(attachment.path) + "/" + path_relative
-                                    item["size"] = os.path.getsize(path)
-                                    if meta.additional_ctx is None:
-                                        meta.additional_ctx = []
-                                    meta.additional_ctx.append(item)
-                                    self.uploaded = True
-                                    sub_attachment.consumed = True
-                                    attachment.consumed = True
-                    self.window.core.filesystem.packer.remove_tmp(tmp_path)  # clean
-            else:
-                item = self.window.core.attachments.context.upload(meta, attachment, prompt)
-                if item:
-                    if meta.additional_ctx is None:
-                        meta.additional_ctx = []
-                    meta.additional_ctx.append(item)
-                    attachment.consumed = True  # allow for deletion
+            if attachment.type == AttachmentItem.TYPE_FILE:
+                result = self.upload_file(
+                    attachment=attachment,
+                    meta=meta,
+                    prompt=prompt,
+                    auto_index=auto_index,
+                )
+                if result:
+                    self.uploaded = True
+            elif attachment.type == AttachmentItem.TYPE_URL:
+                result = self.upload_web(
+                    attachment=attachment,
+                    meta=meta,
+                    prompt=prompt,
+                    auto_index=auto_index,
+                )
+                if result:
                     self.uploaded = True
         if self.uploaded:
             self.window.core.ctx.save(meta.id)  # save meta
         return self.uploaded
 
+    def upload_file(
+            self,
+            attachment: AttachmentItem,
+            meta: CtxMeta,
+            prompt: str,
+            auto_index: bool
+    ) -> bool:
+        """
+        Upload file attachment
+
+        :param attachment: AttachmentItem
+        :param meta: CtxMeta
+        :param prompt: User input prompt
+        :param auto_index: Auto index
+        :return: True if uploaded
+        """
+        uploaded = False
+        if not self.is_allowed(attachment.path):
+            return False
+        if self.window.core.filesystem.packer.is_archive(attachment.path):
+            if self.is_verbose():
+                print("Unpacking archive: {}".format(attachment.path))
+            tmp_path = self.window.core.filesystem.packer.unpack(attachment.path)
+            if tmp_path:
+                for root, dirs, files in os.walk(tmp_path):
+                    for file in files:
+                        path = os.path.join(root, file)
+                        sub_attachment = AttachmentItem()
+                        sub_attachment.path = path
+                        sub_attachment.name = os.path.basename(path)
+                        sub_attachment.consumed = False
+                        path_relative = os.path.relpath(path, tmp_path)
+                        if self.is_allowed(str(path)):
+                            if self.is_verbose():
+                                print("Uploading unpacked from archive: {}".format(path_relative))
+                            item = self.window.core.attachments.context.upload(
+                                meta=meta,
+                                attachment=sub_attachment,
+                                prompt=prompt,
+                                real_path=attachment.path,
+                                auto_index=auto_index,
+                            )
+                            if item:
+                                item["path"] = os.path.basename(attachment.path) + "/" + path_relative
+                                item["size"] = os.path.getsize(path)
+                                if meta.additional_ctx is None:
+                                    meta.additional_ctx = []
+                                meta.additional_ctx.append(item)
+                                uploaded = True
+                                sub_attachment.consumed = True
+                                attachment.consumed = True
+                self.window.core.filesystem.packer.remove_tmp(tmp_path)  # clean
+        else:
+            item = self.window.core.attachments.context.upload(
+                meta=meta,
+                attachment=attachment,
+                prompt=prompt,
+                real_path=attachment.path,
+                auto_index=auto_index,
+            )
+            if item:
+                if meta.additional_ctx is None:
+                    meta.additional_ctx = []
+                meta.additional_ctx.append(item)
+                attachment.consumed = True  # allow for deletion
+                uploaded = True
+
+        return uploaded
+
+    def upload_web(
+            self,
+            attachment: AttachmentItem,
+            meta: CtxMeta,
+            prompt: str,
+            auto_index: bool
+    ) -> bool:
+        """
+        Upload web attachment
+
+        :param attachment: AttachmentItem
+        :param meta: CtxMeta
+        :param prompt: User input prompt
+        :param auto_index: Auto index
+        :return: True if uploaded
+        """
+        return self.upload_file(attachment, meta, prompt, auto_index)
+
     def has_context(self, meta: CtxMeta) -> bool:
         """
         Check if has additional context for attachment
@@ -203,6 +274,7 @@ class Attachment(QObject):
             if self.is_verbose():
                 print("\nPreparing additional context...\nContext Mode: {}".format(self.mode))
 
+        self.window.core.attachments.context.reset()
         if self.mode == self.MODE_FULL_CONTEXT:
             content = self.get_full_context(ctx)
         elif self.mode == self.MODE_QUERY_CONTEXT:
@@ -210,6 +282,14 @@ class Attachment(QObject):
         elif self.mode == self.MODE_QUERY_CONTEXT_SUMMARY:
             content = self.get_context_summary(ctx)
 
+        # append used files and urls to context
+        files = self.window.core.attachments.context.get_used_files()
+        urls = self.window.core.attachments.context.get_used_urls()
+        if files:
+            ctx.files = files
+        if urls:
+            ctx.urls = urls
+
         if content:
             if self.is_verbose():
                 print("\n--- Using additional context ---\n\n{}".format(content))
@@ -362,6 +442,128 @@ class Attachment(QObject):
         self.window.core.attachments.context.clear(meta, delete_files=remove_local)
         self.update_list(meta)
 
+    def select(self, idx: int):
+        """
+        Select uploaded file
+
+        :param idx: index of file
+        """
+        pass
+
+    def open_by_idx(self, idx: int):
+        """
+        Open attachment by index
+
+        :param idx: Index on list
+        """
+        meta = self.window.core.ctx.get_current_meta()
+        if meta is None or meta.additional_ctx is None:
+            return
+        items = self.window.core.attachments.context.get_all(meta)
+        if idx < len(items):
+            item = items[idx]
+            path = item["path"]
+            if "real_path" in item:
+                path = item["real_path"]
+            if os.path.exists(path) and os.path.isfile(path):
+                print("Opening attachment: {}".format(path))
+                self.window.controller.files.open(path)
+
+    def open_dir_src_by_idx(self, idx: int):
+        """
+        Open source directory by index
+
+        :param idx: Index on list
+        """
+        meta = self.window.core.ctx.get_current_meta()
+        if meta is None or meta.additional_ctx is None:
+            return
+        items = self.window.core.attachments.context.get_all(meta)
+        if idx < len(items):
+            item = items[idx]
+            path = item["path"]
+            if "real_path" in item:
+                path = item["real_path"]
+            dir = os.path.dirname(path)
+            if os.path.exists(dir) and os.path.isdir(dir):
+                print("Opening source directory: {}".format(dir))
+                self.window.controller.files.open(dir)
+
+    def open_dir_dest_by_idx(self, idx: int):
+        """
+        Open destination directory by index
+
+        :param idx: Index on list
+        """
+        meta = self.window.core.ctx.get_current_meta()
+        if meta is None or meta.additional_ctx is None:
+            return
+        items = self.window.core.attachments.context.get_all(meta)
+        if idx < len(items):
+            item = items[idx]
+            root_dir = self.window.core.attachments.context.get_dir(meta)
+            dir = os.path.join(root_dir, item["uuid"])
+            if os.path.exists(dir) and os.path.isdir(dir):
+                self.window.controller.files.open(dir)
+                print("Opening destination directory: {}".format(dir))
+
+    def has_file_by_idx(self, idx: int) -> bool:
+        """
+        Check if has file by index
+
+        :param idx: Index on list
+        :return: True if has file
+        """
+        meta = self.window.core.ctx.get_current_meta()
+        if meta is None or meta.additional_ctx is None:
+            return False
+        items = self.window.core.attachments.context.get_all(meta)
+        if idx < len(items):
+            item = items[idx]
+            path = item["path"]
+            if "real_path" in item:
+                path = item["real_path"]
+            return os.path.exists(path) and os.path.isfile(path)
+        return False
+
+    def has_src_by_idx(self, idx: int) -> bool:
+        """
+        Check if has source directory by index
+
+        :param idx: Index on list
+        :return: True if has source directory
+        """
+        meta = self.window.core.ctx.get_current_meta()
+        if meta is None or meta.additional_ctx is None:
+            return False
+        items = self.window.core.attachments.context.get_all(meta)
+        if idx < len(items):
+            item = items[idx]
+            path = item["path"]
+            if "real_path" in item:
+                path = item["real_path"]
+            dir = os.path.dirname(path)
+            return os.path.exists(dir) and os.path.isdir(dir)
+        return False
+
+    def has_dest_by_idx(self, idx: int) -> bool:
+        """
+        Check if has destination directory by index
+
+        :param idx: Index on list
+        :return: True if has destination directory
+        """
+        meta = self.window.core.ctx.get_current_meta()
+        if meta is None or meta.additional_ctx is None:
+            return False
+        items = self.window.core.attachments.context.get_all(meta)
+        if idx < len(items):
+            item = items[idx]
+            root_dir = self.window.core.attachments.context.get_dir(meta)
+            dir = os.path.join(root_dir, item["uuid"])
+            return os.path.exists(dir) and os.path.isdir(dir)
+        return False
+
     @Slot(object)
     def handle_upload_error(self, error: Exception):
         """

pygpt_net/controller/chat/audio.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.26 19:00:00                  #
+# ================================================== #
+
+import base64
+import os
+
+from pygpt_net.core.types import (
+    MODE_AUDIO,
+)
+from pygpt_net.core.bridge.context import MultimodalContext, BridgeContext
+from pygpt_net.core.events import KernelEvent
+from pygpt_net.item.ctx import CtxItem
+
+
+class Audio:
+    def __init__(self, window=None):
+        """
+        Chat audio controller
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.audio_file = "chat_output.wav"
+        self.tmp_input = False
+        self.tmp_output = False
+
+    def setup(self):
+        """Set up UI"""
+        pass
+
+    def update(self):
+        """Update input/output audio"""
+        mode = self.window.core.config.get("mode")
+        if mode == MODE_AUDIO:
+            if not self.window.controller.audio.is_output_enabled():
+                self.window.controller.audio.enable_output()
+                self.tmp_output = True
+            else:
+                self.tmp_output = False
+            if not self.window.controller.audio.is_input_enabled():
+                self.window.controller.audio.enable_input()
+                self.tmp_input = True
+            else:
+                self.tmp_input = False
+        else:
+            if self.tmp_output:
+                self.window.controller.audio.disable_output()
+            if self.tmp_input:
+                self.window.controller.audio.disable_input()
+
+    def handle_output(self, ctx: CtxItem):
+        """
+        Handle output audio
+
+        :param ctx: Context item
+        """
+        wav_path = os.path.join(self.window.core.config.get_user_path(), self.audio_file)
+        if ctx.is_audio and ctx.audio_output:
+            wav_bytes = base64.b64decode(ctx.audio_output)
+            with open(wav_path, "wb") as f:
+                f.write(wav_bytes)
+            self.window.controller.audio.play_chat_audio(wav_path)
+
+    def handle_input(self, path: str):
+        """
+        Handle input audio
+
+        :param path: audio file path
+        """
+        multimodal_ctx = MultimodalContext()
+        with open(path, "rb") as f:
+            multimodal_ctx.audio_data = f.read()
+        multimodal_ctx.is_audio_input = True
+
+        bridge_ctx = BridgeContext()
+        bridge_ctx.prompt = self.window.ui.nodes['input'].toPlainText()  # attach text input
+        bridge_ctx.multimodal_ctx = multimodal_ctx
+        event = KernelEvent(KernelEvent.INPUT_USER, {
+            'context': bridge_ctx,
+            'extra': {},
+        })
+        self.window.dispatch(event)
+
+    def enabled(self) -> bool:
+        """
+        Check if audio mode is enabled
+
+        :return: bool True if enabled
+        """
+        return self.window.core.config.get("mode") == MODE_AUDIO
+

pygpt_net/controller/chat/input.py

@@ -6,10 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.bridge import BridgeContext
+from pygpt_net.core.bridge.context import MultimodalContext
 from pygpt_net.core.events import Event, AppEvent, KernelEvent, RenderEvent
 from pygpt_net.core.types import (
     MODE_AGENT,
@@ -134,6 +135,7 @@ class Input:
         reply = extra.get("reply", False)
         internal = extra.get("internal", False)
         parent_id = extra.get("parent_id", None)
+        multimodal_ctx = context.multimodal_ctx
         self.execute(
             text=text,
             force=force,
@@ -141,6 +143,7 @@ class Input:
             internal=internal,
             prev_ctx=prev_ctx,
             parent_id=parent_id,
+            multimodal_ctx=multimodal_ctx,
         )
 
     def execute(
@@ -151,6 +154,7 @@ class Input:
             internal: bool = False,
             prev_ctx: CtxItem = None,
             parent_id: int = None,
+            multimodal_ctx: MultimodalContext = None,
     ):
         """
         Execute send input text to API
@@ -161,6 +165,7 @@ class Input:
         :param internal: internal call
         :param prev_ctx: previous context (if reply)
         :param parent_id: parent id (if expert)
+        :param multimodal_ctx: multimodal context
         """
         self.window.dispatch(KernelEvent(KernelEvent.STATE_IDLE, {
             "id": "chat",
@@ -208,8 +213,9 @@ class Input:
         camera_captured = (self.window.controller.ui.vision.has_vision()
                            and self.window.controller.attachment.has(mode))
 
-        # allow empty input only for vision modes, otherwise abort
-        if len(text.strip()) == 0 and not camera_captured:
+        # allow empty text input only if multimodal data, otherwise abort
+        is_audio = multimodal_ctx is not None and multimodal_ctx.is_audio_input
+        if len(text.strip()) == 0 and (not camera_captured and not is_audio):
             self.generating = False  # unlock as not generating
             return
 
@@ -256,6 +262,7 @@ class Input:
                 internal=internal,
                 prev_ctx=prev_ctx,
                 parent_id=parent_id,
+                multimodal_ctx=multimodal_ctx,
             )  # text mode: OpenAI, Langchain, Llama, etc.
 
     def log(self, data: any):

pygpt_net/controller/chat/output.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
@@ -111,6 +111,9 @@ class Output:
         # update response tokens
         self.window.controller.chat.common.show_response_tokens(ctx)
 
+        # handle audio output
+        self.window.controller.chat.audio.handle_output(ctx)
+
         # store to history
         if self.window.core.config.get('store_history'):
             self.window.core.history.append(ctx, "output")

pygpt_net/controller/chat/text.py

@@ -6,17 +6,18 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
+    MODE_AUDIO,
     MODE_ASSISTANT,
     MODE_LLAMA_INDEX,
 )
 from pygpt_net.core.events import Event, AppEvent, KernelEvent, RenderEvent
-from pygpt_net.core.bridge.context import BridgeContext
+from pygpt_net.core.bridge.context import BridgeContext, MultimodalContext
 from pygpt_net.item.ctx import CtxItem
 from pygpt_net.utils import trans
 
@@ -38,6 +39,7 @@ class Text:
             internal: bool = False,
             prev_ctx: CtxItem = None,
             parent_id: str = None,
+            multimodal_ctx: MultimodalContext = None,
     ) -> CtxItem:
         """
         Send text message
@@ -47,6 +49,7 @@ class Text:
         :param internal: internal call
         :param prev_ctx: previous context item (if reply)
         :param parent_id: parent context id
+        :param multimodal_ctx: multimodal context
         :return: context item
         """
         self.window.update_status(trans('status.sending'))
@@ -80,7 +83,7 @@ class Text:
         tools_outputs = []  # tools outputs (assistant only)
 
         # o1 models: disable stream mode
-        if model.startswith("o1") or mode == MODE_AGENT_LLAMA:
+        if model.startswith("o1") or mode in [MODE_AGENT_LLAMA, MODE_AUDIO]:
             stream_mode = False
 
         # create ctx item
@@ -103,7 +106,7 @@ class Text:
 
         # if reply from expert command
         if parent_id is not None:  # parent_id = reply from expert
-            ctx.meta = self.window.core.ctx.get_meta_by_id(parent_id)  # append to current ctx
+            # At this point, ctx.meta ID = slave META ID (parent_id is given from slave, not from master)
             ctx.sub_reply = True  # mark as sub reply
             ctx.input_name = prev_ctx.input_name
             ctx.output_name = prev_ctx.output_name
@@ -147,7 +150,8 @@ class Text:
         if mode == MODE_LLAMA_INDEX:
             # check if index is selected
             if self.window.controller.idx.index_selected():
-                disable_native_func_calls = True  # native func calls allowed only for LLM call, not the query engine
+                disable_native_func_calls = True
+                # ^^^ native func calls allowed only for LLM call, not for the query engine
 
         # build final prompt (+plugins)
         sys_prompt = self.window.core.prompt.prepare_sys_prompt(
@@ -232,6 +236,7 @@ class Text:
                 external_functions=functions,  # external functions
                 tools_outputs=tools_outputs,  # if not empty then will submit outputs to assistant
                 max_tokens=max_tokens,  # max output tokens
+                multimodal_ctx=multimodal_ctx,  # multimodal context
             )
             extra = {
                 'mode': mode,

pygpt_net/controller/dialogs/confirm.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 class Confirm:
@@ -349,6 +349,18 @@ class Confirm:
         elif type == 'ctx.group':
             self.window.controller.ctx.create_group(name, id)
 
+    def accept_url(self, type: str, id: any, url: str):
+        """
+        Update URL provided
+
+        :param type: dialog type
+        :param id: dialog object id
+        :param url: URL
+        """
+        # add attachment
+        if type == 'attachment':
+            self.window.controller.attachment.add_url(url)
+
     def dismiss_rename(self):
         """Dismiss rename dialog"""
         self.window.ui.dialog['rename'].close()
@@ -356,3 +368,7 @@ class Confirm:
     def dismiss_create(self):
         """Dismiss create dialog"""
         self.window.ui.dialog['create'].close()
+
+    def dismiss_url(self):
+        """Dismiss url dialog"""
+        self.window.ui.dialog['url'].close()

pygpt_net/controller/kernel/reply.py

@@ -6,14 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.25 01:00:00                  #
 # ================================================== #
 
 import json
 
-from pygpt_net.core.types import (
-    MODE_AGENT_LLAMA,
-)
 from pygpt_net.core.events import KernelEvent, RenderEvent
 from pygpt_net.core.bridge import BridgeContext
 from pygpt_net.item.ctx import CtxItem
@@ -31,13 +28,12 @@ class Reply:
         self.last_result = None
         self.reply_idx = -1
 
-    def add(self, context, extra, flush: bool = False) -> list:
+    def add(self, context, extra) -> list:
         """
         Send reply from plugins to model
 
         :param context: bridge context
         :param extra: extra data
-        :param flush: flush reply stack
         :return: list of results
         """
         flush = False
@@ -54,8 +50,9 @@ class Reply:
                 self.window.core.debug.debug("CTX REPLY: " + str(ctx))
             if ctx.reply:
                 if self.reply_idx >= ctx.pid:  # skip if reply already sent for this context
+                    # >>> this prevents multiple replies from the same ctx item <<<
                     return []
-                self.reply_idx = ctx.pid
+                self.reply_idx = ctx.pid 
                 self.append(ctx)
             if flush or self.window.controller.kernel.async_allowed(ctx):
                 self.flush()
@@ -70,7 +67,6 @@ class Reply:
         """
         self.window.core.debug.info("Reply stack (add)...")
         self.reply_stack.append(ctx.results)
-        # ctx.cmds = []  # clear commands  (disables expand output in render)
         ctx.results = []  # clear results
         self.reply_ctx = ctx
 
@@ -102,6 +98,7 @@ class Reply:
         self.window.core.ctx.update_item(self.reply_ctx)  # update context in db
         self.window.update_status('...')
 
+        # if response from sub call, from experts
         parent_id = None
         if self.reply_ctx.sub_call:
             if self.reply_ctx.meta is not None: