pygpt-net 2.4.31__py3-none-any.whl → 2.4.33__py3-none-any.whl

pygpt_net/core/attachments/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 21:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 import copy
@@ -31,6 +31,11 @@ class Context:
         """
         self.window = window
         self.dir_index = "index"
+        self.last_used_item = None
+        self.last_used_content = None
+        self.last_used_context = None
+        self.last_files = []
+        self.last_urls = []
         self.summary_prompt = """
         Summarize the text below by extracting the most important information, 
         especially those that may help answer the question: 
@@ -87,16 +92,38 @@ class Context:
         context = ""
         if os.path.exists(meta_path) and os.path.isdir(meta_path):
             for file in meta.additional_ctx:
-                if "type" not in file or file["type"] != "local_file":
+                if ("type" not in file
+                        or file["type"] not in ["local_file", "url"]):
                     continue
                 file_id = file["uuid"]
                 file_idx_path = os.path.join(meta_path, file_id)
                 text_path = os.path.join(file_idx_path, file_id + ".txt")
+                store_path = file["path"]
+                if "real_path" in file:
+                    store_path = file["real_path"]
                 if filename:
-                    context += "Filename: {}\n".format(file["name"]) + "\n"
+                    if file["type"] == "url":
+                        context += "URL: {}\n".format(file["path"]) + "\n"
+                    else:
+                        context += "Filename: {}\n".format(file["name"]) + "\n"
+
+                # store used files and URLs in ctx
+                if file["type"] == "url":
+                    if store_path not in self.last_urls:
+                        self.last_urls.append(store_path)
+                else:
+                    if store_path not in self.last_files:
+                        self.last_files.append(store_path)
+
                 if os.path.exists(text_path):
-                    with open(text_path, "r") as f:
-                        context += f.read() + "\n\n"
+                    try:
+                        with open(text_path, "r", encoding="utf-8") as f:
+                            context += f.read() + "\n\n"
+                    except Exception as e:
+                        print("Attachments: read error: {}".format(e))
+
+        self.last_used_content = context
+        self.last_used_context = context
         return context
 
     def query_context(self, meta: CtxMeta, query: str) -> str:
@@ -119,13 +146,17 @@ class Context:
                 file_id = file["uuid"]
                 file_idx_path = os.path.join(meta_path, file_id)
                 file_path = os.path.join(file_idx_path, file["name"])
-                model = None
-                doc_ids = self.window.core.idx.indexing.index_attachment(file_path, idx_path, model)
+                type = AttachmentItem.TYPE_FILE
+                source = file_path
+                if "type" in file:
+                    if file["type"] == "url":
+                        type = AttachmentItem.TYPE_URL
+                        source = file["path"] # URL
+                doc_ids = self.index_attachment(type, source, idx_path)
                 if self.is_verbose():
                     print("Attachments: indexed. Doc IDs: {}".format(doc_ids))
                 file["indexed"] = True
                 file["doc_ids"] = doc_ids
-                #meta.additional_ctx[i] = file  # update meta
                 indexed = True
 
         if indexed:
@@ -133,8 +164,9 @@ class Context:
             self.window.core.ctx.replace(meta)
             self.window.core.ctx.save(meta.id)
 
-        model = None
+        model = None  # no model, retrieval is used
         result = self.window.core.idx.chat.query_attachment(query, idx_path, model)
+        self.last_used_context = result
 
         if self.is_verbose():
             print("Attachments: query result: {}".format(result))
@@ -160,9 +192,10 @@ class Context:
         if self.is_verbose():
             print("Attachments: using summary model: {}".format(model))
 
+        content = self.get_context_text(ctx, filename=True)
         prompt = self.summary_prompt.format(
             query=str(query).strip(),
-            content=str(self.get_context_text(ctx, filename=True)).strip(),
+            content=str(content).strip(),
         )
         if self.is_verbose():
             print("Attachments: summary prompt: {}".format(prompt))
@@ -180,6 +213,7 @@ class Context:
         })
         self.window.dispatch(event)
         response = event.data.get("response")
+        self.last_used_context = response
         if self.is_verbose():
             print("Attachments: summary received: {}".format(response))
         return response
@@ -211,6 +245,7 @@ class Context:
         meta_path = self.get_dir(meta)
         file_idx_path = os.path.join(meta_path, file_id)
         index_path = os.path.join(meta_path, self.dir_index)
+
         os.makedirs(meta_path, exist_ok=True)
         os.makedirs(file_idx_path, exist_ok=True)
 
@@ -219,54 +254,55 @@ class Context:
             if auto_index:
                 print("Attachments: vector index path: {}".format(index_path))
 
-        # copy raw file
-        raw_path = os.path.join(file_idx_path, name)
-        if os.path.exists(raw_path):
-            os.remove(raw_path)
-        copyfile(attachment.path, raw_path)
+        # store content to read
+        src_file = self.store_content(attachment, file_idx_path)
 
         # extract text content using data loader
-        loader_kwargs = {
-            "prompt": prompt,
-        }  # extra loader kwargs
-        text = self.window.core.idx.indexing.read_text_content(
-            path=raw_path,
-            loader_kwargs=loader_kwargs,
-        )
-        if text:
+        content = self.read_content(attachment, src_file, prompt)
+        if content:
             text_path = os.path.join(file_idx_path, file_id + ".txt")
-            with open(text_path, "w") as f:
-                f.write(text)
-
+            with open(text_path, "w", encoding="utf-8") as f:
+                f.write(content)
             if self.is_verbose():
-                print("Attachments: read text content: {}".format(text))
+                print("Attachments: read text content: {}".format(content))
 
         tokens = 0
-        if text:
-            tokens = self.window.core.tokens.from_str(text)
+        if content:
+            tokens = self.window.core.tokens.from_str(content)
+
+        type = "local_file"
+        size = 0
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            size = os.path.getsize(attachment.path)
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            size = os.path.getsize(src_file)
+            type = "url"  # extra ctx type
 
         # index file to ctx index
         doc_ids = []
         if auto_index:
-            model = None
-            doc_ids = self.window.core.idx.indexing.index_attachment(attachment.path, index_path, model)
+            source = src_file
+            if attachment.type == AttachmentItem.TYPE_URL:
+                source = attachment.path  # URL
+            doc_ids = self.index_attachment(attachment.type, source, index_path)
             if self.is_verbose():
                 print("Attachments: indexed. Doc IDs: {}".format(doc_ids))
 
         result = {
             "name": name,
             "path": attachment.path,
-            "type": "local_file",
+            "type": type,
             "uuid": str(file_id),
             "content_type": "text",
-            "size": os.path.getsize(attachment.path),
-            "length": len(text),
+            "size": size,
+            "length": len(content),
             "tokens": tokens,
             "indexed": False,
         }
         if auto_index:
             result["indexed"] = True
             result["doc_ids"] = doc_ids
+
         if real_path:
             result["real_path"] = real_path
 
@@ -275,6 +311,83 @@ class Context:
 
         return result
 
+    def read_content(self, attachment: AttachmentItem, path: str, prompt: str) -> str:
+        """
+        Read content from attachment
+
+        :param attachment: AttachmentItem instance
+        :param path: source file path
+        :param prompt: user input prompt
+        :return: content
+        """
+        content = ""
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            loader_kwargs = {
+                "prompt": prompt,
+            }  # extra loader kwargs
+            content = self.window.core.idx.indexing.read_text_content(
+                path=path,
+                loader_kwargs=loader_kwargs,
+            )
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            # directly from path
+            with open(path, "r", encoding="utf-8") as f:
+                content = f.read()  # already crawled
+
+        return content
+
+    def store_content(self, attachment: AttachmentItem, dir: str) -> str:
+        """
+        Prepare content for attachment
+
+        :param attachment: AttachmentItem instance
+        :param dir: directory to save content
+        :return: content
+        """
+        path = None
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            # copy raw file
+            name = os.path.basename(attachment.path)
+            path = os.path.join(dir, name)
+            if os.path.exists(path):
+                os.remove(path)
+            copyfile(attachment.path, path)
+        elif attachment.type == AttachmentItem.TYPE_URL:
+            web_type = self.window.core.idx.indexing.get_webtype(attachment.path)
+            content = self.window.core.idx.indexing.read_web_content(
+                url=attachment.path,
+                type=web_type,  # webpage, default, TODO: add more types
+                extra_args={},
+            )
+            # src file save
+            name = "url.txt"
+            path = os.path.join(dir, name)
+            if os.path.exists(path):
+                os.remove(path)
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(content)
+        return path
+
+    def index_attachment(self, type: str, source: str, idx_path: str, documents: list = None) -> list:
+        """
+        Index attachment
+
+        :param type: attachment type
+        :param source: source file or URL
+        :param idx_path: index path
+        :param documents: list of documents (optional)
+        :return: list of doc IDs
+        """
+        model = None
+        doc_ids = []
+        if type == AttachmentItem.TYPE_FILE:
+            doc_ids = self.window.core.idx.indexing.index_attachment(source, idx_path, model, documents)
+        elif type == AttachmentItem.TYPE_URL:
+            doc_ids = self.window.core.idx.indexing.index_attachment_web(source, idx_path, model, documents)
+        if self.is_verbose():
+            print("Attachments: indexed. Doc IDs: {}".format(doc_ids))
+        return doc_ids
+
     def duplicate(self, from_meta_id: int, to_meta_id: int) -> bool:
         """
         Duplicate attachments from one meta to another
@@ -427,6 +540,30 @@ class Context:
         except Exception as e:
             self.window.core.debug.error("Attachment.truncate", e)
 
+    def reset(self):
+        """Reset context info"""
+        self.last_used_item = None
+        self.last_used_content = None
+        self.last_used_context = None
+        self.last_files = []
+        self.last_urls = []
+
+    def get_used_files(self) -> list:
+        """
+        Get last used files
+
+        :return: list of files
+        """
+        return self.last_files
+
+    def get_used_urls(self) -> list:
+        """
+        Get last used URLs
+
+        :return: list of URLs
+        """
+        return self.last_urls
+
     def is_verbose(self) -> bool:
         """
         Check if verbose mode is enabled

pygpt_net/core/debug/attachments.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.21 20:00:00                  #
+# Updated Date: 2024.11.26 02:00:00                  #
 # ================================================== #
 
 class AttachmentsDebug:
@@ -36,6 +36,8 @@ class AttachmentsDebug:
                     'send': attachment.send,
                     'key': key,
                     'mode': mode,
+                    'type': attachment.type,
+                    'consumed': attachment.consumed,
                 }
                 self.window.core.debug.add(self.id, attachment.name, str(data))
 

pygpt_net/core/debug/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 03:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 import json
 
@@ -45,6 +45,10 @@ class ContextDebug:
         self.window.core.debug.add(self.id, 'CMD (current)', str(self.window.core.ctx.current_cmd))
         self.window.core.debug.add(self.id, 'CMD schema (current)', str(self.window.core.ctx.current_cmd_schema))
         self.window.core.debug.add(self.id, 'FUNCTIONS (current)', str(self.get_functions()))
+        self.window.core.debug.add(self.id, 'Attachments: last used content',
+                                   str(self.window.core.attachments.context.last_used_content))
+        self.window.core.debug.add(self.id, 'Attachments: last used context',
+                                  str(self.window.core.attachments.context.last_used_context))
 
         current = None
         if self.window.core.ctx.get_current() is not None:

pygpt_net/core/idx/indexing.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.23 21:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 import datetime
@@ -349,6 +349,65 @@ class Indexing:
             data.append(doc.text)
         return "\n".join(data)
 
+    def read_web_content(
+            self,
+            url: str,
+            type: str = "webpage",
+            extra_args: dict = None
+    ) -> str:
+        """
+        Get content from external resource
+
+        :param url: external url to index
+        :param type: type of URL (webpage, feed, etc.)
+        :param extra_args: extra arguments for loader
+        :return: file content
+        """
+        docs = self.read_web(url, type, extra_args)
+        data = []
+        for doc in docs:
+            data.append(doc.text)
+        return "\n".join(data)
+
+    def read_web(
+            self,
+            url: str,
+            type: str = "webpage",
+            extra_args: dict = None,
+    ) -> list[Document]:
+        """
+        Read data from external resource
+
+        :param url: external url to index
+        :param type: type of URL (webpage, feed, etc.)
+        :param extra_args: extra arguments for loader
+        :return: list of documents
+        """
+        documents = []
+
+        # check if web loader for defined type exists
+        if type not in self.loaders["web"]:
+            raise ValueError("No web loader for type: {}".format(type))
+
+        try:
+            if "url" not in extra_args:
+                extra_args["url"] = url
+
+            # get unique external content identifier
+            unique_id = self.data_providers[type].get_external_id(extra_args)
+            self.window.core.idx.log("Loading web documents from: {}".format(unique_id))
+            self.window.core.idx.log("Using web loader for type: {}".format(type))
+
+            args = self.data_providers[type].prepare_args(**extra_args)
+
+            # get documents from external resource
+            documents = self.loaders["web"][type].load_data(
+                **args
+            )
+        except Exception as e:
+            self.window.core.debug.log(e)
+        return documents
+
     def prepare_document(self, doc: Document):
         """
         Prepare document to store
@@ -924,7 +983,8 @@ class Indexing:
             self,
             file_path: str,
             index_path: str,
-            model: ModelItem = None
+            model: ModelItem = None,
+            documents: list = None,
     ) -> list:
         """
         Index context attachment
@@ -932,6 +992,45 @@ class Indexing:
         :param file_path: path to file to index
         :param index_path: index path
         :param model: model
+        :param documents: list of documents (optional)
+        :return: response
+        """
+        if model is None:
+            model = self.window.core.models.from_defaults()
+
+        service_context = self.window.core.idx.llm.get_service_context(model=model)
+        index = self.window.core.idx.storage.get_ctx_idx(index_path, service_context=service_context)  # get or create ctx index
+
+        idx = "tmp:{}".format(index_path)  # tmp index id
+        self.window.core.idx.log("Indexing to context attachment index: {}...".format(idx))
+
+        doc_ids = []
+        if documents is None:
+            documents = self.get_documents(file_path)
+        for d in documents:
+            if self.is_stopped():  # force stop
+                break
+            self.prepare_document(d)
+            self.index_document(index, d)
+            doc_ids.append(d.id_)  # add to index
+
+        self.window.core.idx.storage.store_ctx_idx(index_path, index)
+        return doc_ids
+
+    def index_attachment_web(
+            self,
+            url: str,
+            index_path: str,
+            model: ModelItem = None,
+            documents: list = None,
+    ) -> list:
+        """
+        Index context attachment
+
+        :param url: URL to index
+        :param index_path: index path
+        :param model: model
+        :param documents: list of documents (optional)
         :return: response
         """
         if model is None:
@@ -943,8 +1042,14 @@ class Indexing:
         idx = "tmp:{}".format(index_path)  # tmp index id
         self.window.core.idx.log("Indexing to context attachment index: {}...".format(idx))
 
+        web_type = self.get_webtype(url)
         doc_ids = []
-        documents = self.get_documents(file_path)
+        if documents is None:
+            documents = self.read_web(
+                url=url,
+                type=web_type,
+                extra_args={},
+            )
         for d in documents:
             if self.is_stopped():  # force stop
                 break
@@ -955,20 +1060,23 @@ class Indexing:
         self.window.core.idx.storage.store_ctx_idx(index_path, index)
         return doc_ids
 
+    def get_webtype(self, url: str) -> str:
         """
-        # query tmp index
-        output = None
-        if len(files) > 0:
-            self.log("Querying temporary in-memory index: {}...".format(idx))
-            response = index.as_query_engine(
-                llm=llm,
-                streaming=False,
-            ).query(query)  # query with default prompt
-            if response:
-                ctx.add_doc_meta(self.get_metadata(response.source_nodes))  # store metadata
-                output = response.
-        return output
+        Get web loader type by URL
+
+        :param url: URL
+        :return: web loader type
         """
+        type = "webpage"  # default
+        for id in self.data_providers:
+            loader = self.data_providers[id]
+            if hasattr(loader, "is_supported_attachment"):
+                if loader.is_supported_attachment(url):
+                    type = id
+                    break
+        print("Selected web data loader: {}".format(type))
+        return type
+
     def remove_attachment(self, index_path: str, doc_id: str) -> bool:
         """
         Remove document from index

pygpt_net/core/render/markdown/pid.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.05 23:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 class PidData():
@@ -17,6 +17,7 @@ class PidData():
         self.meta = meta
         self.images_appended = []
         self.urls_appended = []
+        self.files_appended = []
         self.buffer = ""
         self.prev_position = None  # previous cursor position (for chunk append)
         self.is_cmd = False

pygpt_net/core/render/plain/pid.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.05 23:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 class PidData():
@@ -17,5 +17,6 @@ class PidData():
         self.meta = meta
         self.images_appended = []
         self.urls_appended = []
+        self.files_appended = []
         self.buffer = ""
         self.is_cmd = False

pygpt_net/core/render/web/body.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 21:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 import os
@@ -167,10 +167,12 @@ class Body:
         if num is not None and num_all is not None and num_all > 1:
             num_str = " [{}]".format(num)
         url, path = self.window.core.filesystem.extract_local_url(url)
-        return """<a href="{url}"><img src="{path}" class="image"></a>
-        <p><b>{prefix}{num}:</b> <a href="{url}">{path}</a></p>""". \
+        basename = os.path.basename(path)
+        return """<div class="extra-src-img-box" title="{url}"><div class="img-outer"><div class="img-wrapper"><a href="{url}"><img src="{path}" class="image"></a></div>
+        <a href="{url}" class="title">{title}</a></div></div>""". \
             format(prefix=trans('chat.prefix.img'),
                    url=url,
+                   title=basename,
                    path=path,
                    num=num_str)
 
@@ -183,13 +185,19 @@ class Body:
         :param num_all: number of all URLs
         :return: HTML code
         """
+        icon_path = os.path.join(
+            self.window.core.config.get_app_path(),
+            "data", "icons", "public_filled.svg"
+        )
+        icon = '<img src="file://{}" width="25" height="25" valign="middle" class="extra-src-icon">'.format(icon_path)
         num_str = ""
         if num is not None and num_all is not None and num_all > 1:
             num_str = " [{}]".format(num)
-        return """<b>{prefix}{num}:</b> <a href="{url}" title="{url}">{url}</a>""". \
-            format(prefix=trans('chat.prefix.url'),
-                   url=url,
-                   num=num_str)
+        return """{icon}<b>{num}</b> <a href="{url}" title="{url}">{url}</a>""". \
+            format(url=url,
+                   num=num_str,
+                   icon=icon,
+            )
 
     def get_docs_html(self, docs: list) -> str:
         """
@@ -223,8 +231,16 @@ class Body:
         except Exception as e:
             pass
 
+        icon_path = os.path.join(
+            self.window.core.config.get_app_path(),
+            "data", "icons", "db.svg"
+        )
+        icon = '<img src="file://{}" width="25" height="25" valign="middle" class="extra-src-icon">'.format(icon_path)
         if html_sources != "":
-            html += "<p><small><b>{prefix}:</b></small></p>".format(prefix=trans('chat.prefix.doc'))
+            html += "<p>{icon}<small><b>{prefix}:</b></small></p>".format(
+                prefix=trans('chat.prefix.doc'),
+                icon=icon,
+            )
             html += "<div class=\"cmd\">"
             html += "<p>" + html_sources + "</p>"
             html += "</div> "
@@ -239,15 +255,21 @@ class Body:
         :param num_all: number of all files
         :return: HTML code
         """
+        icon_path = os.path.join(
+            self.window.core.config.get_app_path(),
+            "data", "icons", "attachments.svg"
+        )
+        icon = '<img src="file://{}" width="25" height="25" valign="middle" class="extra-src-icon">'.format(icon_path)
         num_str = ""
         if num is not None and num_all is not None and num_all > 1:
             num_str = " [{}]".format(num)
         url, path = self.window.core.filesystem.extract_local_url(url)
-        return """<div><b>{prefix}{num}:</b> <a href="{url}">{path}</a></div>""". \
-            format(prefix=trans('chat.prefix.file'),
-                   url=url,
+        return """{icon} <b>{num}</b> <a href="{url}">{path}</a>""". \
+            format(url=url,
                    path=path,
-                   num=num_str)
+                   num=num_str,
+                   icon=icon,
+            )
 
     def prepare_tool_extra(self, ctx: CtxItem) -> str:
         """

pygpt_net/core/render/web/pid.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.05 23:00:00                  #
+# Updated Date: 2024.11.26 04:00:00                  #
 # ================================================== #
 
 from pygpt_net.utils import trans
@@ -20,6 +20,7 @@ class PidData():
         self.meta = meta
         self.images_appended = []
         self.urls_appended = []
+        self.files_appended = []
         self.buffer = ""  # stream buffer
         self.is_cmd = False
         self.html = ""  # html buffer