pygpt-net 2.4.30__py3-none-any.whl → 2.4.35__py3-none-any.whl

pygpt_net/core/attachments/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 21:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 import copy
@@ -31,6 +31,11 @@ class Context:
         """
         self.window = window
         self.dir_index = "index"
+        self.last_used_item = None
+        self.last_used_content = None
+        self.last_used_context = None
+        self.last_files = []
+        self.last_urls = []
         self.summary_prompt = """
         Summarize the text below by extracting the most important information, 
         especially those that may help answer the question: 
@@ -87,16 +92,38 @@ class Context:
         context = ""
         if os.path.exists(meta_path) and os.path.isdir(meta_path):
             for file in meta.additional_ctx:
-                if "type" not in file or file["type"] != "local_file":
+                if ("type" not in file
+                        or file["type"] not in ["local_file", "url"]):
                     continue
                 file_id = file["uuid"]
                 file_idx_path = os.path.join(meta_path, file_id)
                 text_path = os.path.join(file_idx_path, file_id + ".txt")
+                store_path = file["path"]
+                if "real_path" in file:
+                    store_path = file["real_path"]
                 if filename:
-                    context += "Filename: {}\n".format(file["name"]) + "\n"
+                    if file["type"] == "url":
+                        context += "URL: {}\n".format(file["path"]) + "\n"
+                    else:
+                        context += "Filename: {}\n".format(file["name"]) + "\n"
+
+                # store used files and URLs in ctx
+                if file["type"] == "url":
+                    if store_path not in self.last_urls:
+                        self.last_urls.append(store_path)
+                else:
+                    if store_path not in self.last_files:
+                        self.last_files.append(store_path)
+
                 if os.path.exists(text_path):
-                    with open(text_path, "r") as f:
-                        context += f.read() + "\n\n"
+                    try:
+                        with open(text_path, "r", encoding="utf-8") as f:
+                            context += f.read() + "\n\n"
+                    except Exception as e:
+                        print("Attachments: read error: {}".format(e))
+
+        self.last_used_content = context
+        self.last_used_context = context
         return context
 
     def query_context(self, meta: CtxMeta, query: str) -> str:
@@ -111,14 +138,55 @@ class Context:
         if not os.path.exists(meta_path) or not os.path.isdir(meta_path):
             return ""
         idx_path = os.path.join(self.get_dir(meta), self.dir_index)
-        model = None
-        result = self.window.core.idx.chat.query_attachment(query, idx_path, model)
+
+        indexed = False
+        # index files if not indexed by auto_index
+        for i, file in enumerate(meta.additional_ctx):
+            if "indexed" not in file or not file["indexed"]:
+                file_id = file["uuid"]
+                file_idx_path = os.path.join(meta_path, file_id)
+                file_path = os.path.join(file_idx_path, file["name"])
+                type = AttachmentItem.TYPE_FILE
+                source = file_path
+                if "type" in file:
+                    if file["type"] == "url":
+                        type = AttachmentItem.TYPE_URL
+                        source = file["path"] # URL
+                doc_ids = self.index_attachment(type, source, idx_path)
+                file["indexed"] = True
+                file["doc_ids"] = doc_ids
+                indexed = True
+
+        if indexed:
+            # update ctx in DB
+            self.window.core.ctx.replace(meta)
+            self.window.core.ctx.save(meta.id)
+
+        model, model_item = self.get_selected_model("query")
+        result = self.window.core.idx.chat.query_attachment(query, idx_path, model_item)
+        self.last_used_context = result
 
         if self.is_verbose():
             print("Attachments: query result: {}".format(result))
 
         return result
 
+    def get_selected_model(self, mode: str = "summary"):
+        """
+        Get selected model for attachments
+
+        :return: model name, model item
+        """
+        model_item = None
+        model = None
+        if mode == "summary":
+            model = self.window.core.config.get("ctx.attachment.summary.model", "gpt-4o-mini")
+        elif mode == "query":
+            model = self.window.core.config.get("ctx.attachment.query.model", "gpt-4o-mini")
+        if model:
+            model_item = self.window.core.models.get(model)
+        return model, model_item
+
     def summary_context(self, ctx: CtxItem, query: str) -> str:
         """
         Get summary of the context
@@ -127,20 +195,17 @@ class Context:
         :param query: query string
         :return: query result
         """
-        model_item = None
-        model = self.window.core.config.get("ctx.attachment.summary.model", "gpt-4o-mini")
-        if model:
-            model_item = self.window.core.models.get(model)
-
+        model, model_item = self.get_selected_model("summary")
         if model_item is None:
             raise Exception("Attachments: summary model not found: {}".format(model))
 
         if self.is_verbose():
             print("Attachments: using summary model: {}".format(model))
 
+        content = self.get_context_text(ctx, filename=True)
         prompt = self.summary_prompt.format(
             query=str(query).strip(),
-            content=str(self.get_context_text(ctx, filename=True)).strip(),
+            content=str(content).strip(),
         )
         if self.is_verbose():
             print("Attachments: summary prompt: {}".format(prompt))
@@ -158,17 +223,27 @@ class Context:
         })
         self.window.dispatch(event)
         response = event.data.get("response")
+        self.last_used_context = response
         if self.is_verbose():
             print("Attachments: summary received: {}".format(response))
         return response
 
-    def upload(self, meta: CtxMeta, attachment: AttachmentItem, prompt: str) -> dict:
+    def upload(
+            self,
+            meta: CtxMeta,
+            attachment: AttachmentItem,
+            prompt: str,
+            auto_index: bool = False,
+            real_path: str = None
+    ) -> dict:
         """
         Upload attachment for context
 
         :param meta: CtxMeta instance
         :param attachment: AttachmentItem instance
         :param prompt: user input prompt
+        :param auto_index: auto index
+        :param real_path: real path
         :return: Dict with attachment data
         """
         if self.is_verbose():
@@ -180,64 +255,147 @@ class Context:
         meta_path = self.get_dir(meta)
         file_idx_path = os.path.join(meta_path, file_id)
         index_path = os.path.join(meta_path, self.dir_index)
+
         os.makedirs(meta_path, exist_ok=True)
         os.makedirs(file_idx_path, exist_ok=True)
 
         if self.is_verbose():
             print("Attachments: created path: {}".format(meta_path))
-            print("Attachments: vector index path: {}".format(index_path))
+            if auto_index:
+                print("Attachments: vector index path: {}".format(index_path))
 
-        # copy raw file
-        raw_path = os.path.join(file_idx_path, name)
-        if os.path.exists(raw_path):
-            os.remove(raw_path)
-        copyfile(attachment.path, raw_path)
+        # store content to read
+        src_file = self.store_content(attachment, file_idx_path)
 
         # extract text content using data loader
-        loader_kwargs = {
-            "prompt": prompt,
-        }  # extra loader kwargs
-        text = self.window.core.idx.indexing.read_text_content(
-            path=raw_path,
-            loader_kwargs=loader_kwargs,
-        )
-        if text:
+        content = self.read_content(attachment, src_file, prompt)
+        if content:
             text_path = os.path.join(file_idx_path, file_id + ".txt")
-            with open(text_path, "w") as f:
-                f.write(text)
-
+            with open(text_path, "w", encoding="utf-8") as f:
+                f.write(content)
             if self.is_verbose():
-                print("Attachments: read text content: {}".format(text))
+                print("Attachments: read text content: {}".format(content))
 
         tokens = 0
-        if text:
-            tokens = self.window.core.tokens.from_str(text)
+        if content:
+            tokens = self.window.core.tokens.from_str(content)
 
-        # index file to ctx index
-        model = None
-        doc_ids = self.window.core.idx.indexing.index_attachment(attachment.path, index_path, model)
+        type = "local_file"
+        size = 0
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            size = os.path.getsize(attachment.path)
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            size = os.path.getsize(src_file)
+            type = "url"  # extra ctx type
 
-        if self.is_verbose():
-            print("Attachments: indexed. Doc IDs: {}".format(doc_ids))
+        # index file to ctx index
+        doc_ids = []
+        if auto_index:
+            source = src_file
+            if attachment.type == AttachmentItem.TYPE_URL:
+                source = attachment.path  # URL
+            doc_ids = self.index_attachment(attachment.type, source, index_path)
 
         result = {
             "name": name,
             "path": attachment.path,
-            "type": "local_file",
+            "type": type,
             "uuid": str(file_id),
-            "doc_ids": doc_ids,
-            "indexed": True,
             "content_type": "text",
-            "size": os.path.getsize(attachment.path),
-            "length": len(text),
+            "size": size,
+            "length": len(content),
             "tokens": tokens,
+            "indexed": False,
         }
+        if auto_index:
+            result["indexed"] = True
+            result["doc_ids"] = doc_ids
+
+        if real_path:
+            result["real_path"] = real_path
 
         if self.is_verbose():
             print("Attachments: uploaded: {}".format(result))
 
         return result
 
+    def read_content(self, attachment: AttachmentItem, path: str, prompt: str) -> str:
+        """
+        Read content from attachment
+
+        :param attachment: AttachmentItem instance
+        :param path: source file path
+        :param prompt: user input prompt
+        :return: content
+        """
+        content = ""
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            loader_kwargs = {
+                "prompt": prompt,
+            }  # extra loader kwargs
+            content = self.window.core.idx.indexing.read_text_content(
+                path=path,
+                loader_kwargs=loader_kwargs,
+            )
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            # directly from path
+            with open(path, "r", encoding="utf-8") as f:
+                content = f.read()  # already crawled
+
+        return content
+
+    def store_content(self, attachment: AttachmentItem, dir: str) -> str:
+        """
+        Prepare content for attachment
+
+        :param attachment: AttachmentItem instance
+        :param dir: directory to save content
+        :return: content
+        """
+        path = None
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            # copy raw file
+            name = os.path.basename(attachment.path)
+            path = os.path.join(dir, name)
+            if os.path.exists(path):
+                os.remove(path)
+            copyfile(attachment.path, path)
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            web_type = self.window.core.idx.indexing.get_webtype(attachment.path)
+            content = self.window.core.idx.indexing.read_web_content(
+                url=attachment.path,
+                type=web_type,  # webpage, default, TODO: add more types
+                extra_args={},
+            )
+            # src file save
+            name = "url.txt"
+            path = os.path.join(dir, name)
+            if os.path.exists(path):
+                os.remove(path)
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(content)
+        return path
+
+    def index_attachment(self, type: str, source: str, idx_path: str, documents: list = None) -> list:
+        """
+        Index attachment
+
+        :param type: attachment type
+        :param source: source file or URL
+        :param idx_path: index path
+        :param documents: list of documents (optional)
+        :return: list of doc IDs
+        """
+        model = None
+        doc_ids = []
+        if type == AttachmentItem.TYPE_FILE:
+            doc_ids = self.window.core.idx.indexing.index_attachment(source, idx_path, model, documents)
+        elif type == AttachmentItem.TYPE_URL:
+            doc_ids = self.window.core.idx.indexing.index_attachment_web(source, idx_path, model, documents)
+        if self.is_verbose():
+            print("Attachments: indexed. Doc IDs: {}".format(doc_ids))
+        return doc_ids
+
     def duplicate(self, from_meta_id: int, to_meta_id: int) -> bool:
         """
         Duplicate attachments from one meta to another
@@ -390,6 +548,30 @@ class Context:
         except Exception as e:
             self.window.core.debug.error("Attachment.truncate", e)
 
+    def reset(self):
+        """Reset context info"""
+        self.last_used_item = None
+        self.last_used_content = None
+        self.last_used_context = None
+        self.last_files = []
+        self.last_urls = []
+
+    def get_used_files(self) -> list:
+        """
+        Get last used files
+
+        :return: list of files
+        """
+        return self.last_files
+
+    def get_used_urls(self) -> list:
+        """
+        Get last used URLs
+
+        :return: list of URLs
+        """
+        return self.last_urls
+
     def is_verbose(self) -> bool:
         """
         Check if verbose mode is enabled

pygpt_net/core/{audio.py → audio/__init__.py}

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.02.24 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import re

pygpt_net/core/audio/context.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.21 20:00:00                  #
+# ================================================== #
+
+class AudioContext:
+    def __init__(self, **kwargs):
+        """
+        Audio context
+
+        :param kwargs: keyword arguments
+        """
+        self.data = kwargs.get("ctx", None)
+        self.prev_id = kwargs.get("prev_id", None)
+
+    def to_dict(self) -> dict:
+        """
+        Return as dictionary
+
+        :return: dictionary
+        """
+        data = {
+            "data": self.data,
+            "prev_id": self.prev_id
+        }
+        # sort by keys
+        data = dict(sorted(data.items(), key=lambda item: item[0]))
+        return data

pygpt_net/core/bridge/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 from pygpt_net.item.ctx import CtxItem
@@ -39,6 +39,7 @@ class BridgeContext:
         self.attachments = kwargs.get("attachments", [])
         self.file_ids = kwargs.get("file_ids", [])
         self.reply_context = kwargs.get("reply_ctx", None)  # ReplyContext
+        self.multimodal_ctx = kwargs.get("multimodal_ctx", MultimodalContext())  # AudioContext
 
         # check types
         if self.ctx is not None and not isinstance(self.ctx, CtxItem):
@@ -78,6 +79,33 @@ class BridgeContext:
         if self.model is not None:
             data["model"] = self.model.to_dict()
 
+        # sort by keys
+        data = dict(sorted(data.items(), key=lambda item: item[0]))
+        return data
+
+class MultimodalContext:
+    def __init__(self, **kwargs):
+        """
+        Multimodal context
+
+        :param kwargs: keyword arguments
+        """
+        self.is_audio_input = kwargs.get("is_audio_input", False)
+        self.is_audio_output = kwargs.get("is_audio_output", False)
+        self.audio_data = kwargs.get("audio_data", None)
+        self.audio_format = kwargs.get("audio_format", "wav")
+
+    def to_dict(self) -> dict:
+        """
+        Return as dictionary
+
+        :return: dictionary
+        """
+        data = {
+            "is_audio_input": self.is_audio_input,
+            "is_audio_output": self.is_audio_output,
+            "audio_format": self.audio_format,
+        }
         # sort by keys
         data = dict(sorted(data.items(), key=lambda item: item[0]))
         return data

pygpt_net/core/ctx/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import copy
@@ -19,6 +19,7 @@ from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
     MODE_ASSISTANT,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_EXPERT,
@@ -78,6 +79,7 @@ class Ctx:
             MODE_LLAMA_INDEX,
             MODE_AGENT,
             MODE_EXPERT,
+            MODE_AUDIO,
         ]
         self.allowed_modes = {
             MODE_CHAT: self.all_modes,
@@ -89,6 +91,7 @@ class Ctx:
             MODE_LLAMA_INDEX: self.all_modes,
             MODE_AGENT: self.all_modes,
             MODE_EXPERT: self.all_modes,
+            MODE_AUDIO: self.all_modes,
             MODE_AGENT_LLAMA: [MODE_AGENT_LLAMA],
         }
         self.current_sys_prompt = ""

pygpt_net/core/db/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 00:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import os
@@ -82,6 +82,8 @@ class Database:
             'is_internal',
             'docs_json',
             'external_id',
+            'audio_id',
+            'audio_expires_ts',
         ]
         columns["ctx_meta"] = [
             'id',
@@ -218,7 +220,7 @@ class Database:
                 'columns': columns["ctx_item"],
                 'sort_by': columns["ctx_item"],
                 'search_fields': ['id', 'input', 'output', 'input_name', 'output_name', 'meta_id', 'hidden_input', 'hidden_output'],
-                'timestamp_columns': ['input_ts', 'output_ts'],
+                'timestamp_columns': ['input_ts', 'output_ts', 'audio_expires_ts'],
                 'json_columns': ['cmds_json', 'results_json', 'urls_json', 'images_json', 'files_json', 'attachments_json', 'docs_json', 'additional_ctx_json'],
                 'default_sort': 'id',
                 'default_order': 'DESC',

pygpt_net/core/debug/attachments.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 class AttachmentsDebug:
@@ -36,6 +36,8 @@ class AttachmentsDebug:
                     'send': attachment.send,
                     'key': key,
                     'mode': mode,
+                    'type': attachment.type,
+                    'consumed': attachment.consumed,
                 }
                 self.window.core.debug.add(self.id, attachment.name, str(data))
 

pygpt_net/core/debug/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 03:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 import json
 
@@ -45,6 +45,10 @@ class ContextDebug:
         self.window.core.debug.add(self.id, 'CMD (current)', str(self.window.core.ctx.current_cmd))
         self.window.core.debug.add(self.id, 'CMD schema (current)', str(self.window.core.ctx.current_cmd_schema))
         self.window.core.debug.add(self.id, 'FUNCTIONS (current)', str(self.get_functions()))
+        self.window.core.debug.add(self.id, 'Attachments: last used content',
+                                   str(self.window.core.attachments.context.last_used_content))
+        self.window.core.debug.add(self.id, 'Attachments: last used context',
+                                  str(self.window.core.attachments.context.last_used_context))
 
         current = None
         if self.window.core.ctx.get_current() is not None:

pygpt_net/core/debug/presets.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import os
@@ -15,6 +15,7 @@ from pygpt_net.core.types import (
     MODE_AGENT,
     MODE_AGENT_LLAMA,
     MODE_ASSISTANT,
+    MODE_AUDIO,
     MODE_CHAT,
     MODE_COMPLETION,
     MODE_EXPERT,
@@ -64,6 +65,7 @@ class PresetsDebug:
                 MODE_AGENT: preset.agent,
                 MODE_AGENT_LLAMA: preset.agent_llama,
                 MODE_EXPERT: preset.expert,
+                MODE_AUDIO: preset.audio,
                 'temperature': preset.temperature,
                 'version': preset.version,
             }

pygpt_net/core/events/event.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 03:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import json
@@ -27,6 +27,7 @@ class Event(BaseEvent):
     AUDIO_INPUT_TOGGLE = "audio.input.toggle"
     AUDIO_OUTPUT_STOP = "audio.output.stop"
     AUDIO_OUTPUT_TOGGLE = "audio.output.toggle"
+    AUDIO_PLAYBACK = "audio.playback"
     AUDIO_READ_TEXT = "audio.read_text"
     CMD_EXECUTE = "cmd.execute"
     CMD_INLINE = "cmd.inline"

pygpt_net/core/experts/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 
@@ -18,6 +18,7 @@ from pygpt_net.core.types import (
     MODE_LANGCHAIN,
     MODE_LLAMA_INDEX,
     MODE_VISION,
+    MODE_AUDIO,
 )
 from pygpt_net.core.bridge.context import BridgeContext
 from pygpt_net.core.events import Event, KernelEvent, RenderEvent
@@ -39,6 +40,7 @@ class Experts:
             MODE_VISION,
             MODE_LANGCHAIN,
             MODE_LLAMA_INDEX,
+            MODE_AUDIO,
         ]
         self.allowed_cmds = ["expert_call"]