pygpt-net 2.4.28__py3-none-any.whl → 2.4.34__py3-none-any.whl

pygpt_net/controller/lang/custom.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import Qt
@@ -14,6 +14,7 @@ from PySide6.QtCore import Qt
 from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_EXPERT,
@@ -63,6 +64,7 @@ class Custom:
         self.window.ui.config['preset'][MODE_AGENT].box.setText(trans("preset.agent"))
         self.window.ui.config['preset'][MODE_AGENT_LLAMA].box.setText(trans("preset.agent_llama"))
         self.window.ui.config['preset'][MODE_EXPERT].box.setText(trans("preset.expert"))
+        self.window.ui.config['preset'][MODE_AUDIO].box.setText(trans("preset.audio"))
 
         self.window.ui.config['global']['img_raw'].setText(trans("img.raw"))
 

pygpt_net/controller/mode.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.events import Event, AppEvent
@@ -41,6 +41,7 @@ class Mode:
         self.window.dispatch(event)
         self.window.controller.attachment.update()
         self.window.controller.chat.attachment.update()
+        self.window.controller.chat.audio.update()
         self.window.dispatch(AppEvent(AppEvent.MODE_SELECTED))  # app event
 
     def set(self, mode: str):

pygpt_net/controller/presets/editor.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import datetime
@@ -16,12 +16,14 @@ from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
     MODE_ASSISTANT,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_EXPERT,
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
-    MODE_VISION, MODE_IMAGE,
+    MODE_VISION, 
+    MODE_IMAGE,
 )
 from pygpt_net.item.preset import PresetItem
 from pygpt_net.utils import trans
@@ -87,6 +89,10 @@ class Editor:
                 "type": "bool",
                 "label": "preset.agent",
             },
+            MODE_AUDIO: {
+                "type": "bool",
+                "label": "preset.audio",
+            },
             # "assistant": {
             # "type": "bool",
             # "label": "preset.assistant",
@@ -287,6 +293,8 @@ class Editor:
                 data.agent = True
             elif mode == MODE_AGENT_LLAMA:
                 data.agent_llama = True
+            elif mode == MODE_AUDIO:
+                data.audio = True
 
         options = {}
         data_dict = data.to_dict()
@@ -347,6 +355,7 @@ class Editor:
             MODE_LLAMA_INDEX,
             MODE_EXPERT,
             MODE_AGENT_LLAMA,
+            MODE_AUDIO,
         ]
 
         # disallow editing default preset

pygpt_net/core/access/voice.py

@@ -57,8 +57,8 @@ class Voice:
             ControlEvent.CTX_SEARCH_CLEAR: "Clear the search results",
             ControlEvent.INPUT_SEND: "Send the message to input",
             ControlEvent.INPUT_APPEND: "Append message to current input without sending it",
-            ControlEvent.MODE_CHAT: "Switch to chat mode",
-            ControlEvent.MODE_LLAMA_INDEX: "Switch to chat with files (llama-index) mode",
+            ControlEvent.MODE_CHAT: "Switch to Chat mode",
+            ControlEvent.MODE_LLAMA_INDEX: "Switch to Chat with Files (LlamaIndex) mode",
             ControlEvent.MODE_NEXT: "Switch to the next mode",
             ControlEvent.MODE_PREV: "Switch to the previous mode",
             ControlEvent.MODEL_NEXT: "Switch to the next model",

pygpt_net/core/agents/legacy.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.core.types import (
@@ -15,6 +15,7 @@ from pygpt_net.core.types import (
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
     MODE_VISION,
+    MODE_AUDIO,
 )
 
 class Legacy:
@@ -31,6 +32,7 @@ class Legacy:
             MODE_VISION,
             MODE_LANGCHAIN,
             MODE_LLAMA_INDEX,
+            MODE_AUDIO,
         ]
 
     def get_allowed_modes(self) -> list:

pygpt_net/core/attachments/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 import copy
@@ -176,7 +176,8 @@ class Attachments:
             mode: str,
             name: str = None,
             path: str = None,
-            auto_save: bool = True
+            auto_save: bool = True,
+            type: str = AttachmentItem.TYPE_FILE,
     ) -> AttachmentItem:
         """
         Create new attachment
@@ -185,16 +186,19 @@ class Attachments:
         :param name: name
         :param path: path
         :param auto_save: auto_save
+        :param type: type
         :return: AttachmentItem
         """
         # make local copy of external attachment if enabled
-        if self.window.core.config.get("upload.store"):
-            if not self.window.core.filesystem.in_work_dir(path):
-                path = self.window.core.filesystem.store_upload(path)
+        if type == AttachmentItem.TYPE_FILE and path is not None:
+            if self.window.core.config.get("upload.store"):
+                if not self.window.core.filesystem.in_work_dir(path):
+                    path = self.window.core.filesystem.store_upload(path)
 
         attachment = self.create()
         attachment.name = name
         attachment.path = path
+        attachment.type = type
 
         if mode not in self.items:
             self.items[mode] = {}
@@ -410,7 +414,7 @@ class Attachments:
         for mode in self.items:
             for id in self.items[mode]:
                 attachment = self.items[mode][id]
-                if attachment.path is not None:
+                if attachment.path is not None and attachment.type == AttachmentItem.TYPE_FILE:
                     attachment.path = self.window.core.filesystem.to_workdir(
                         attachment.path,
                     )
@@ -422,7 +426,7 @@ class Attachments:
         for mode in data:
             for id in data[mode]:
                 attachment = data[mode][id]
-                if attachment.path is not None:
+                if attachment.path is not None and attachment.type == AttachmentItem.TYPE_FILE:
                     attachment.path = self.window.core.filesystem.make_local(
                         attachment.path,
                     )

pygpt_net/core/attachments/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 21:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 import copy
@@ -31,6 +31,11 @@ class Context:
         """
         self.window = window
         self.dir_index = "index"
+        self.last_used_item = None
+        self.last_used_content = None
+        self.last_used_context = None
+        self.last_files = []
+        self.last_urls = []
         self.summary_prompt = """
         Summarize the text below by extracting the most important information, 
         especially those that may help answer the question: 
@@ -87,16 +92,38 @@ class Context:
         context = ""
         if os.path.exists(meta_path) and os.path.isdir(meta_path):
             for file in meta.additional_ctx:
-                if "type" not in file or file["type"] != "local_file":
+                if ("type" not in file
+                        or file["type"] not in ["local_file", "url"]):
                     continue
                 file_id = file["uuid"]
                 file_idx_path = os.path.join(meta_path, file_id)
                 text_path = os.path.join(file_idx_path, file_id + ".txt")
+                store_path = file["path"]
+                if "real_path" in file:
+                    store_path = file["real_path"]
                 if filename:
-                    context += "Filename: {}\n".format(file["name"]) + "\n"
+                    if file["type"] == "url":
+                        context += "URL: {}\n".format(file["path"]) + "\n"
+                    else:
+                        context += "Filename: {}\n".format(file["name"]) + "\n"
+
+                # store used files and URLs in ctx
+                if file["type"] == "url":
+                    if store_path not in self.last_urls:
+                        self.last_urls.append(store_path)
+                else:
+                    if store_path not in self.last_files:
+                        self.last_files.append(store_path)
+
                 if os.path.exists(text_path):
-                    with open(text_path, "r") as f:
-                        context += f.read() + "\n\n"
+                    try:
+                        with open(text_path, "r", encoding="utf-8") as f:
+                            context += f.read() + "\n\n"
+                    except Exception as e:
+                        print("Attachments: read error: {}".format(e))
+
+        self.last_used_content = context
+        self.last_used_context = context
         return context
 
     def query_context(self, meta: CtxMeta, query: str) -> str:
@@ -111,14 +138,55 @@ class Context:
         if not os.path.exists(meta_path) or not os.path.isdir(meta_path):
             return ""
         idx_path = os.path.join(self.get_dir(meta), self.dir_index)
-        model = None
-        result = self.window.core.idx.chat.query_attachment(query, idx_path, model)
+
+        indexed = False
+        # index files if not indexed by auto_index
+        for i, file in enumerate(meta.additional_ctx):
+            if "indexed" not in file or not file["indexed"]:
+                file_id = file["uuid"]
+                file_idx_path = os.path.join(meta_path, file_id)
+                file_path = os.path.join(file_idx_path, file["name"])
+                type = AttachmentItem.TYPE_FILE
+                source = file_path
+                if "type" in file:
+                    if file["type"] == "url":
+                        type = AttachmentItem.TYPE_URL
+                        source = file["path"] # URL
+                doc_ids = self.index_attachment(type, source, idx_path)
+                file["indexed"] = True
+                file["doc_ids"] = doc_ids
+                indexed = True
+
+        if indexed:
+            # update ctx in DB
+            self.window.core.ctx.replace(meta)
+            self.window.core.ctx.save(meta.id)
+
+        model, model_item = self.get_selected_model("query")
+        result = self.window.core.idx.chat.query_attachment(query, idx_path, model_item)
+        self.last_used_context = result
 
         if self.is_verbose():
             print("Attachments: query result: {}".format(result))
 
         return result
 
+    def get_selected_model(self, mode: str = "summary"):
+        """
+        Get selected model for attachments
+
+        :return: model name, model item
+        """
+        model_item = None
+        model = None
+        if mode == "summary":
+            model = self.window.core.config.get("ctx.attachment.summary.model", "gpt-4o-mini")
+        elif mode == "query":
+            model = self.window.core.config.get("ctx.attachment.query.model", "gpt-4o-mini")
+        if model:
+            model_item = self.window.core.models.get(model)
+        return model, model_item
+
     def summary_context(self, ctx: CtxItem, query: str) -> str:
         """
         Get summary of the context
@@ -127,20 +195,17 @@ class Context:
         :param query: query string
         :return: query result
         """
-        model_item = None
-        model = self.window.core.config.get("ctx.attachment.summary.model", "gpt-4o-mini")
-        if model:
-            model_item = self.window.core.models.get(model)
-
+        model, model_item = self.get_selected_model("summary")
         if model_item is None:
             raise Exception("Attachments: summary model not found: {}".format(model))
 
         if self.is_verbose():
             print("Attachments: using summary model: {}".format(model))
 
+        content = self.get_context_text(ctx, filename=True)
         prompt = self.summary_prompt.format(
             query=str(query).strip(),
-            content=str(self.get_context_text(ctx, filename=True)).strip(),
+            content=str(content).strip(),
         )
         if self.is_verbose():
             print("Attachments: summary prompt: {}".format(prompt))
@@ -158,17 +223,27 @@ class Context:
         })
         self.window.dispatch(event)
         response = event.data.get("response")
+        self.last_used_context = response
         if self.is_verbose():
             print("Attachments: summary received: {}".format(response))
         return response
 
-    def upload(self, meta: CtxMeta, attachment: AttachmentItem, prompt: str) -> dict:
+    def upload(
+            self,
+            meta: CtxMeta,
+            attachment: AttachmentItem,
+            prompt: str,
+            auto_index: bool = False,
+            real_path: str = None
+    ) -> dict:
         """
         Upload attachment for context
 
         :param meta: CtxMeta instance
         :param attachment: AttachmentItem instance
         :param prompt: user input prompt
+        :param auto_index: auto index
+        :param real_path: real path
         :return: Dict with attachment data
         """
         if self.is_verbose():
@@ -180,64 +255,147 @@ class Context:
         meta_path = self.get_dir(meta)
         file_idx_path = os.path.join(meta_path, file_id)
         index_path = os.path.join(meta_path, self.dir_index)
+
         os.makedirs(meta_path, exist_ok=True)
         os.makedirs(file_idx_path, exist_ok=True)
 
         if self.is_verbose():
             print("Attachments: created path: {}".format(meta_path))
-            print("Attachments: vector index path: {}".format(index_path))
+            if auto_index:
+                print("Attachments: vector index path: {}".format(index_path))
 
-        # copy raw file
-        raw_path = os.path.join(file_idx_path, name)
-        if os.path.exists(raw_path):
-            os.remove(raw_path)
-        copyfile(attachment.path, raw_path)
+        # store content to read
+        src_file = self.store_content(attachment, file_idx_path)
 
         # extract text content using data loader
-        loader_kwargs = {
-            "prompt": prompt,
-        }  # extra loader kwargs
-        text = self.window.core.idx.indexing.read_text_content(
-            path=raw_path,
-            loader_kwargs=loader_kwargs,
-        )
-        if text:
+        content = self.read_content(attachment, src_file, prompt)
+        if content:
             text_path = os.path.join(file_idx_path, file_id + ".txt")
-            with open(text_path, "w") as f:
-                f.write(text)
-
+            with open(text_path, "w", encoding="utf-8") as f:
+                f.write(content)
             if self.is_verbose():
-                print("Attachments: read text content: {}".format(text))
+                print("Attachments: read text content: {}".format(content))
 
         tokens = 0
-        if text:
-            tokens = self.window.core.tokens.from_str(text)
+        if content:
+            tokens = self.window.core.tokens.from_str(content)
 
-        # index file to ctx index
-        model = None
-        doc_ids = self.window.core.idx.indexing.index_attachment(attachment.path, index_path, model)
+        type = "local_file"
+        size = 0
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            size = os.path.getsize(attachment.path)
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            size = os.path.getsize(src_file)
+            type = "url"  # extra ctx type
 
-        if self.is_verbose():
-            print("Attachments: indexed. Doc IDs: {}".format(doc_ids))
+        # index file to ctx index
+        doc_ids = []
+        if auto_index:
+            source = src_file
+            if attachment.type == AttachmentItem.TYPE_URL:
+                source = attachment.path  # URL
+            doc_ids = self.index_attachment(attachment.type, source, index_path)
 
         result = {
             "name": name,
             "path": attachment.path,
-            "type": "local_file",
+            "type": type,
             "uuid": str(file_id),
-            "doc_ids": doc_ids,
-            "indexed": True,
             "content_type": "text",
-            "size": os.path.getsize(attachment.path),
-            "length": len(text),
+            "size": size,
+            "length": len(content),
             "tokens": tokens,
+            "indexed": False,
         }
+        if auto_index:
+            result["indexed"] = True
+            result["doc_ids"] = doc_ids
+
+        if real_path:
+            result["real_path"] = real_path
 
         if self.is_verbose():
             print("Attachments: uploaded: {}".format(result))
 
         return result
 
+    def read_content(self, attachment: AttachmentItem, path: str, prompt: str) -> str:
+        """
+        Read content from attachment
+
+        :param attachment: AttachmentItem instance
+        :param path: source file path
+        :param prompt: user input prompt
+        :return: content
+        """
+        content = ""
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            loader_kwargs = {
+                "prompt": prompt,
+            }  # extra loader kwargs
+            content = self.window.core.idx.indexing.read_text_content(
+                path=path,
+                loader_kwargs=loader_kwargs,
+            )
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            # directly from path
+            with open(path, "r", encoding="utf-8") as f:
+                content = f.read()  # already crawled
+
+        return content
+
+    def store_content(self, attachment: AttachmentItem, dir: str) -> str:
+        """
+        Prepare content for attachment
+
+        :param attachment: AttachmentItem instance
+        :param dir: directory to save content
+        :return: content
+        """
+        path = None
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            # copy raw file
+            name = os.path.basename(attachment.path)
+            path = os.path.join(dir, name)
+            if os.path.exists(path):
+                os.remove(path)
+            copyfile(attachment.path, path)
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            web_type = self.window.core.idx.indexing.get_webtype(attachment.path)
+            content = self.window.core.idx.indexing.read_web_content(
+                url=attachment.path,
+                type=web_type,  # webpage, default, TODO: add more types
+                extra_args={},
+            )
+            # src file save
+            name = "url.txt"
+            path = os.path.join(dir, name)
+            if os.path.exists(path):
+                os.remove(path)
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(content)
+        return path
+
+    def index_attachment(self, type: str, source: str, idx_path: str, documents: list = None) -> list:
+        """
+        Index attachment
+
+        :param type: attachment type
+        :param source: source file or URL
+        :param idx_path: index path
+        :param documents: list of documents (optional)
+        :return: list of doc IDs
+        """
+        model = None
+        doc_ids = []
+        if type == AttachmentItem.TYPE_FILE:
+            doc_ids = self.window.core.idx.indexing.index_attachment(source, idx_path, model, documents)
+        elif type == AttachmentItem.TYPE_URL:
+            doc_ids = self.window.core.idx.indexing.index_attachment_web(source, idx_path, model, documents)
+        if self.is_verbose():
+            print("Attachments: indexed. Doc IDs: {}".format(doc_ids))
+        return doc_ids
+
     def duplicate(self, from_meta_id: int, to_meta_id: int) -> bool:
         """
         Duplicate attachments from one meta to another
@@ -390,6 +548,30 @@ class Context:
         except Exception as e:
             self.window.core.debug.error("Attachment.truncate", e)
 
+    def reset(self):
+        """Reset context info"""
+        self.last_used_item = None
+        self.last_used_content = None
+        self.last_used_context = None
+        self.last_files = []
+        self.last_urls = []
+
+    def get_used_files(self) -> list:
+        """
+        Get last used files
+
+        :return: list of files
+        """
+        return self.last_files
+
+    def get_used_urls(self) -> list:
+        """
+        Get last used URLs
+
+        :return: list of URLs
+        """
+        return self.last_urls
+
     def is_verbose(self) -> bool:
         """
         Check if verbose mode is enabled

pygpt_net/core/{audio.py → audio/__init__.py}

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.02.24 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import re

pygpt_net/core/audio/context.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.21 20:00:00                  #
+# ================================================== #
+
+class AudioContext:
+    def __init__(self, **kwargs):
+        """
+        Audio context
+
+        :param kwargs: keyword arguments
+        """
+        self.data = kwargs.get("ctx", None)
+        self.prev_id = kwargs.get("prev_id", None)
+
+    def to_dict(self) -> dict:
+        """
+        Return as dictionary
+
+        :return: dictionary
+        """
+        data = {
+            "data": self.data,
+            "prev_id": self.prev_id
+        }
+        # sort by keys
+        data = dict(sorted(data.items(), key=lambda item: item[0]))
+        return data

pygpt_net/core/bridge/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.item.ctx import CtxItem
@@ -39,6 +39,7 @@ class BridgeContext:
         self.attachments = kwargs.get("attachments", [])
         self.file_ids = kwargs.get("file_ids", [])
         self.reply_context = kwargs.get("reply_ctx", None)  # ReplyContext
+        self.multimodal_ctx = kwargs.get("multimodal_ctx", MultimodalContext())  # AudioContext
 
         # check types
         if self.ctx is not None and not isinstance(self.ctx, CtxItem):
@@ -78,6 +79,33 @@ class BridgeContext:
         if self.model is not None:
             data["model"] = self.model.to_dict()
 
+        # sort by keys
+        data = dict(sorted(data.items(), key=lambda item: item[0]))
+        return data
+
+class MultimodalContext:
+    def __init__(self, **kwargs):
+        """
+        Multimodal context
+
+        :param kwargs: keyword arguments
+        """
+        self.is_audio_input = kwargs.get("is_audio_input", False)
+        self.is_audio_output = kwargs.get("is_audio_output", False)
+        self.audio_data = kwargs.get("audio_data", None)
+        self.audio_format = kwargs.get("audio_format", "wav")
+
+    def to_dict(self) -> dict:
+        """
+        Return as dictionary
+
+        :return: dictionary
+        """
+        data = {
+            "is_audio_input": self.is_audio_input,
+            "is_audio_output": self.is_audio_output,
+            "audio_format": self.audio_format,
+        }
         # sort by keys
         data = dict(sorted(data.items(), key=lambda item: item[0]))
         return data