pygpt-net 2.4.42__py3-none-any.whl → 2.4.45__py3-none-any.whl

pygpt_net/core/attachments/context.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 08:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import copy
@@ -15,7 +15,9 @@ import shutil
 import uuid
 
 from shutil import copyfile
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Dict, Any, Tuple
+
+from llama_index.core import Document
 
 from pygpt_net.core.bridge import BridgeContext
 from pygpt_net.core.events import KernelEvent
@@ -125,7 +127,7 @@ class Context:
         meta_path = self.get_dir(meta)
         context = ""
         if os.path.exists(meta_path) and os.path.isdir(meta_path):
-            for file in meta.additional_ctx:
+            for file in meta.get_additional_ctx():
                 if ("type" not in file
                         or file["type"] not in ["local_file", "url"]):
                     continue
@@ -183,7 +185,7 @@ class Context:
 
         indexed = False
         # index files if not indexed by auto_index
-        for i, file in enumerate(meta.additional_ctx):
+        for i, file in enumerate(meta.get_additional_ctx()):
             if "indexed" not in file or not file["indexed"]:
                 file_id = file["uuid"]
                 file_idx_path = os.path.join(meta_path, file_id)
@@ -313,7 +315,7 @@ class Context:
             prompt: str,
             auto_index: bool = False,
             real_path: Optional[str] = None
-    ) -> dict:
+    ) -> Dict[str, Any]:
         """
         Upload attachment for context
 
@@ -325,7 +327,10 @@ class Context:
         :return: Dict with attachment data
         """
         if self.is_verbose():
-            print("Uploading for meta ID: {}".format(meta.id))
+            if meta.group:
+                print("Uploading for meta group ID: {}".format(meta.group.id))
+            else:
+                print("Uploading for meta ID: {}".format(meta.id))
 
         # prepare idx dir
         name = os.path.basename(attachment.path)
@@ -342,11 +347,18 @@ class Context:
             if auto_index:
                 print("Attachments: vector index path: {}".format(index_path))
 
-        # store content to read
-        src_file = self.store_content(attachment, file_idx_path)
+        documents = None
+        
+        # store content to read, and get docs if type == web
+        src_file, docs = self.store_content(attachment, file_idx_path)
+        if attachment.type == AttachmentItem.TYPE_URL:
+            documents = docs
+
+        # extract text content using data loader, and get docs if type == file
+        content, docs = self.read_content(attachment, src_file, prompt)
+        if attachment.type == AttachmentItem.TYPE_FILE:
+            documents = docs
 
-        # extract text content using data loader
-        content = self.read_content(attachment, src_file, prompt)
         if content:
             text_path = os.path.join(file_idx_path, file_id + ".txt")
             with open(text_path, "w", encoding="utf-8") as f:
@@ -372,7 +384,7 @@ class Context:
             source = src_file
             if attachment.type == AttachmentItem.TYPE_URL:
                 source = attachment.path  # URL
-            doc_ids = self.index_attachment(attachment.type, source, index_path)
+            doc_ids = self.index_attachment(attachment.type, source, index_path, documents=documents)
 
         result = {
             "name": name,
@@ -402,57 +414,61 @@ class Context:
             attachment: AttachmentItem,
             path: str,
             prompt: str
-    ) -> str:
+    ) -> Tuple[str, List[Document]]:
         """
         Read content from attachment
 
         :param attachment: AttachmentItem instance
         :param path: source file path
         :param prompt: user input prompt
-        :return: content
+        :return: text content, list of documents
         """
         content = ""
+        docs = []
         if attachment.type == AttachmentItem.TYPE_FILE:
             loader_kwargs = {
                 "prompt": prompt,
             }  # extra loader kwargs
-            content = self.window.core.idx.indexing.read_text_content(
+            content, docs = self.window.core.idx.indexing.read_text_content(
                 path=path,
                 loader_kwargs=loader_kwargs,
             )
         elif attachment.type == AttachmentItem.TYPE_URL:
-            # directly from path
+            # directly from before stored path
             with open(path, "r", encoding="utf-8") as f:
-                content = f.read()  # already crawled
-
-        return content
+                content = f.read()  # data is already crawled in `store_content`
+        return content, docs
 
     def store_content(
             self,
             attachment: AttachmentItem,
             dir: str
-    ) -> str:
+    ) -> Tuple[str, List[Document]]:
         """
         Prepare content for attachment
 
         :param attachment: AttachmentItem instance
         :param dir: directory to save content
-        :return: content
+        :return: path, list of documents
         """
         path = None
+        docs = []
         if attachment.type == AttachmentItem.TYPE_FILE:
-            # copy raw file
+            # copy raw file only
             name = os.path.basename(attachment.path)
             path = os.path.join(dir, name)
             if os.path.exists(path):
                 os.remove(path)
             copyfile(attachment.path, path)
         elif attachment.type == AttachmentItem.TYPE_URL:
-            web_type = self.window.core.idx.indexing.get_webtype(attachment.path)
-            content = self.window.core.idx.indexing.read_web_content(
-                url=attachment.path,
-                type=web_type,  # webpage, default, TODO: add more types
-                extra_args={},
+            loader = attachment.extra.get("loader")
+            input_params = attachment.extra.get("input_params")
+            input_config = attachment.extra.get("input_config")
+            self.window.core.idx.indexing.update_loader_args(loader, input_config)  # update config
+            content, docs = self.window.core.idx.indexing.read_web_content(
+                url="",
+                type=loader,
+                extra_args=input_params,
             )
             # src file save
             name = "url.txt"
@@ -461,14 +477,14 @@ class Context:
                 os.remove(path)
             with open(path, "w", encoding="utf-8") as f:
                 f.write(content)
-        return path
+        return path, docs
 
     def index_attachment(
             self,
             type: str,
             source: str,
             idx_path: str,
-            documents: Optional[list] = None
+            documents: Optional[List[Document]] = None
     ) -> list:
         """
         Index attachment
@@ -496,7 +512,7 @@ class Context:
         :param meta: CtxMeta instance
         :return: list of attachments
         """
-        return meta.additional_ctx
+        return meta.get_additional_ctx()
 
     def get_dir(self, meta: CtxMeta) -> str:
         """
@@ -506,6 +522,8 @@ class Context:
         :return: directory path
         """
         meta_uuid = str(meta.uuid)
+        if meta.group:
+            meta_uuid = str(meta.group.uuid)
         return os.path.join(self.window.core.config.get_user_dir("ctx_idx"), meta_uuid)
 
     def get_selected_model(self, mode: str = "summary"):
@@ -562,7 +580,7 @@ class Context:
         :param meta: CtxMeta instance
         :return: number of attachments
         """
-        return len(meta.additional_ctx)
+        return len(meta.get_additional_ctx())
 
     def delete(
             self,
@@ -577,11 +595,11 @@ class Context:
         :param item: Attachment item dict
         :param delete_files: delete files
         """
-        meta.additional_ctx.remove(item)
+        meta.remove_additional_ctx(item)
         self.window.core.ctx.save(meta.id)
         if delete_files:
             self.delete_local(meta, item)
-        if len(meta.additional_ctx) == 0:
+        if len(meta.get_additional_ctx()) == 0:
             self.delete_index(meta)
 
     def delete_by_meta(self, meta: CtxMeta):
@@ -613,7 +631,7 @@ class Context:
         :param meta: CtxMeta instance
         :param delete_files: delete files
         """
-        meta.additional_ctx = []
+        meta.reset_additional_ctx()
         self.window.core.ctx.save(meta.id)
         if delete_files:
             self.delete_index(meta)
@@ -644,7 +662,7 @@ class Context:
         :param meta: CtxMeta instance
         :param delete_files: delete files
         """
-        meta.additional_ctx = []
+        meta.reset_additional_ctx()
         self.window.core.ctx.save(meta.id)
         if delete_files:
             self.delete_index(meta)

pygpt_net/core/db/__init__.py

@@ -122,6 +122,7 @@ class Database:
             'updated_ts',
             'created_ts',
             'uuid',
+            'additional_ctx_json',
         ]
         columns["idx_ctx"] = [
             'id',
@@ -242,7 +243,7 @@ class Database:
                 'sort_by': columns["ctx_group"],
                 'search_fields': ['id', 'name'],
                 'timestamp_columns': ['created_ts', 'updated_ts'],
-                'json_columns': [],
+                'json_columns': ['additional_ctx_json'],
                 'default_sort': 'id',
                 'default_order': 'DESC',
                 'primary_key': 'id',

pygpt_net/core/debug/attachments.py

@@ -38,6 +38,7 @@ class AttachmentsDebug:
                     'mode': mode,
                     'type': attachment.type,
                     'consumed': attachment.consumed,
+                    'extra': attachment.extra,
                 }
                 self.window.core.debug.add(self.id, attachment.name, str(data))
 

pygpt_net/core/idx/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 22:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import datetime
@@ -24,6 +24,7 @@ from .indexing import Indexing
 from .llm import Llm
 from .chat import Chat
 from .metadata import Metadata
+from .ui import UI
 
 from .types.ctx import Ctx
 from .types.external import External
@@ -43,6 +44,7 @@ class Idx:
         self.storage = Storage(window)
         self.chat = Chat(window, self.storage)
         self.metadata = Metadata(window)
+        self.ui = UI(window)
 
         self.providers = {
             "json_file": JsonFileProvider(window),  # only for patching
@@ -529,7 +531,7 @@ class Idx:
                     self.items[store_id][idx].id = idx
                     self.items[store_id][idx].name = idx
 
-    def get_idx_ids(self) -> List[str]:
+    def get_idx_ids(self) -> List[Dict[str, str]]:
         """
         Get list of indexes
 
@@ -539,7 +541,10 @@ class Idx:
         data =  self.window.core.config.get('llama.idx.list')
         if data is not None:
             for item in data:
-                ids.append(item['id'])
+                name = item['name']
+                if name is None or name == "":
+                    name = item['id']
+                ids.append({item['id']: name})
         return ids
 
     def clear(self, idx: str):

pygpt_net/core/idx/indexing.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 22:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import datetime
@@ -89,18 +89,35 @@ class Indexing:
                             "key": key,
                             "value": loader.init_args[key],
                             "type": "str",  # default = str
+                            "label": key,
+                            "description": None,
                         }
                         # from config
                         if key in loader.args:
                             self.external_config[loader.id][key]["value"] = loader.args[key]
                         if key in loader.init_args_types:
                             self.external_config[loader.id][key]["type"] = loader.init_args_types[key]
+                        if key in loader.init_args_labels:
+                            self.external_config[loader.id][key]["label"] = loader.init_args_labels[key]
+                        if key in loader.init_args_desc:
+                            self.external_config[loader.id][key]["description"] = loader.init_args_desc[key]
 
             except ImportError as e:
                 msg = "Error while registering data loader: " + loader.id + " - " + str(e)
                 self.window.core.debug.log(msg)
                 self.window.core.debug.log(e)
 
+    def get_loader(self, loader: str) -> Optional[BaseLoader]:
+        """
+        Get data loader by id
+
+        :param loader: loader id
+        :return: data loader instance
+        """
+        if loader in self.data_providers:
+            return self.data_providers[loader]
+        return None
+
     def update_loader_args(
             self,
             loader: str,
@@ -344,13 +361,13 @@ class Indexing:
             self,
             path: str,
             loader_kwargs: Optional[Dict[str, Any]] = None
-    ) -> str:
+    ) -> Tuple[str, List[Document]]:
         """
         Get content from file using loaders
 
         :param path: path to file
         :param loader_kwargs: additional keyword arguments for data loader
-        :return: file content
+        :return: text content, list of documents
         """
         docs = self.get_documents(
             path,
@@ -361,27 +378,27 @@ class Indexing:
         data = []
         for doc in docs:
             data.append(doc.text)
-        return "\n".join(data)
+        return "\n".join(data), docs
 
     def read_web_content(
             self,
             url: str,
             type: str = "webpage",
             extra_args: Optional[Dict[str, Any]] = None
-    ) -> str:
+    ) -> Tuple[str, List[Document]]:
         """
         Get content from external resource
 
         :param url: external url to index
         :param type: type of URL (webpage, feed, etc.)
         :param extra_args: extra arguments for loader
-        :return: file content
+        :return: text content, list of documents
         """
         docs = self.read_web(url, type, extra_args)
         data = []
         for doc in docs:
             data.append(doc.text)
-        return "\n".join(data)
+        return "\n".join(data), docs
 
     def read_web(
             self,

pygpt_net/core/idx/ui/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.12.16 01:00:00                  #
+# ================================================== #
+
+from .loaders import Loaders
+
+class UI:
+    def __init__(self, window=None):
+        """
+        UI components
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.loaders = Loaders(window)

pygpt_net/core/idx/ui/loaders.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.12.16 01:00:00                  #
+# ================================================== #
+
+import json
+from typing import Dict, Tuple, Any, Optional
+
+from PySide6.QtCore import Qt
+from PySide6.QtWidgets import QVBoxLayout, QLabel, QHBoxLayout, QWidget
+
+from pygpt_net.ui.widget.element.labels import HelpLabel
+from pygpt_net.ui.widget.option.input import OptionInput
+
+
+class Loaders:
+    def __init__(self, window=None):
+        """
+        UI - loaders components
+
+        :param window: Window instance
+        """
+        self.window = window
+
+    def handle_options(
+            self,
+            select_loader,
+            prefix_options,
+            prefix_config
+    ) -> Tuple[bool, Optional[str], Dict[str, Any], Dict[str, Any]]:
+        """
+        Handle options
+
+        :param select_loader: loader selection
+        :param prefix_options: prefix for options
+        :param prefix_config: prefix for config
+        :return: bool, loader name, input_params, input_config
+        """
+        input_params = {}
+        input_config = {}
+        loader = select_loader.get_value()
+        if not loader:
+            return False, loader, input_params, input_config
+        loaders = self.window.core.idx.indexing.get_external_instructions()
+        if loader in loaders:
+            params = loaders[loader]
+            for k in params["args"]:
+                key_path = prefix_options + "." + loader + "." + k
+                if key_path in self.window.ui.nodes:
+                    tmp_value = self.window.ui.nodes[key_path].text()
+                    type = params["args"][k]["type"]
+                    try:
+                        if tmp_value:
+                            if type == "int":
+                                tmp_value = int(tmp_value)
+                            elif type == "float":
+                                tmp_value = float(tmp_value)
+                            elif type == "bool":
+                                if tmp_value.lower() in ["true", "1"]:
+                                    tmp_value = True
+                                else:
+                                    tmp_value = False
+                            elif type == "list":
+                                tmp_value = tmp_value.split(",")
+                            elif type == "dict":
+                                tmp_value = json.loads(tmp_value)
+                            input_params[k] = tmp_value
+                    except Exception as e:
+                        self.window.core.debug.log(e)
+                        self.window.ui.dialogs.alert(e)
+
+        loaders = self.window.core.idx.indexing.get_external_config()
+        if loader in loaders:
+            params = loaders[loader]
+            for k in params:
+                key_path = prefix_config + "." + loader + "." + k
+                type = params[k]["type"]
+                if key_path in self.window.ui.nodes:
+                    tmp_value = self.window.ui.nodes[key_path].text()
+                    try:
+                        if tmp_value:
+                            if type == "int":
+                                tmp_value = int(tmp_value)
+                            elif type == "float":
+                                tmp_value = float(tmp_value)
+                            elif type == "bool":
+                                if tmp_value.lower() in ["true", "1"]:
+                                    tmp_value = True
+                                else:
+                                    tmp_value = False
+                            elif type == "list":
+                                tmp_value = tmp_value.split(",")
+                            elif type == "dict":
+                                tmp_value = json.loads(tmp_value)
+                            input_config[k] = tmp_value
+                    except Exception as e:
+                        self.window.core.debug.log(e)
+                        self.window.ui.dialogs.alert(e)
+
+        return True, loader, input_params, input_config
+
+    def setup_loader_options(self):
+        """Setup loader options"""
+        inputs = {}
+        groups = {}
+        loaders = self.window.core.idx.indexing.get_external_instructions()
+        for loader in loaders:
+            params = loaders[loader]
+            inputs[loader] = {}
+            group = QVBoxLayout()
+            for k in params["args"]:
+                label = k
+                description = None
+                is_label = False
+                if "label" in params["args"][k]:
+                    label = params["args"][k]["label"]
+                    is_label = True
+                if "description" in params["args"][k]:
+                    description = params["args"][k]["description"]
+                option_id = "web.loader." + loader + ".option." + k
+                option_widget = OptionInput(self.window, "tool.indexer", option_id, {
+                    "label": label,
+                    "value": "",
+                })
+                option_widget.setPlaceholderText(params["args"][k]["type"])
+                inputs[loader][k] = option_widget
+
+                option_label = QLabel(label)
+                option_label.setToolTip(k)
+
+                row = QHBoxLayout()  # cols
+                row.addWidget(option_label)
+                row.addWidget(option_widget)
+                row.setContentsMargins(5, 0, 5, 0)
+
+                option_layout = QVBoxLayout()
+                option_layout.addLayout(row)
+                if description:
+                    option_layout.addWidget(HelpLabel(description))
+
+                option_layout.setContentsMargins(5, 0, 0, 0)
+
+                group.addLayout(option_layout)
+                group.setContentsMargins(0, 0, 0, 0)
+
+            group_widget = QWidget()
+            group_widget.setLayout(group)
+            groups[loader] = group_widget
+
+        return inputs, groups
+
+    def setup_loader_config(self):
+        """Setup loader config"""
+        inputs = {}
+        groups = {}
+        loaders = self.window.core.idx.indexing.get_external_config()
+        for loader in loaders:
+            params = loaders[loader]
+            inputs[loader] = {}
+            group = QVBoxLayout()
+            for k in params:
+                label = k
+                description = None
+                is_label = False
+                if "label" in params[k]:
+                    label = params[k]["label"]
+                    is_label = True
+                if "description" in params[k]:
+                    description = params[k]["description"]
+                option_id = "web.loader." + loader + ".config." + k
+                option_widget = OptionInput(self.window, "tool.indexer", option_id, {
+                    "label": label,
+                    "value": params[k]["value"],
+                })
+                try:
+                    if params[k]["value"] is not None:
+                        if params[k]["type"] == "list" and isinstance(params[k]["value"], list):
+                            option_widget.setText(", ".join(params[k]["value"]))
+                        elif params[k]["type"] == "dict" and isinstance(params[k]["value"], dict):
+                            option_widget.setText(json.dumps(params[k]["value"]))
+                        else:
+                            option_widget.setText(str(params[k]["value"]))
+                except Exception as e:
+                    self.window.core.debug.log(e)
+
+                option_widget.setPlaceholderText(params[k]["type"])
+                inputs[loader][k] = option_widget
+
+                option_label = QLabel(label)
+                option_label.setToolTip(k)
+
+                row = QHBoxLayout()  # cols
+                row.addWidget(option_label)
+                row.addWidget(option_widget)
+                row.setContentsMargins(5, 0, 5, 0)
+
+                option_layout = QVBoxLayout()
+                option_layout.addLayout(row)
+                if description:
+                    option_layout.addWidget(HelpLabel(description))
+
+                option_layout.setContentsMargins(5, 0, 0, 0)
+
+                group.addLayout(option_layout)
+                group.setContentsMargins(0, 0, 0, 0)
+
+            group_widget = QWidget()
+            group_widget.setLayout(group)
+            groups[loader] = group_widget
+
+        return inputs, groups

pygpt_net/data/config/config.json

@@ -1,8 +1,8 @@
 {
   "__meta__": {
-    "version": "2.4.42",
-    "app.version": "2.4.42",
-    "updated_at": "2024-12-15T00:00:00"
+    "version": "2.4.45",
+    "app.version": "2.4.45",
+    "updated_at": "2024-12-16T00:00:00"
   },
   "access.audio.event.speech": false,
   "access.audio.event.speech.disabled": [],
@@ -100,7 +100,7 @@
       6,
       7
   ],
-  "ctx.records.folders.top": false,
+  "ctx.records.folders.top": true,
   "ctx.records.limit": 0,
   "ctx.records.separators": true,
   "ctx.records.groups.separators": true,

pygpt_net/data/config/models.json

@@ -1,8 +1,8 @@
 {
     "__meta__": {
-        "version": "2.4.42",
-        "app.version": "2.4.42",
-        "updated_at": "2024-12-15T00:00:00"
+        "version": "2.4.45",
+        "app.version": "2.4.45",
+        "updated_at": "2024-12-16T00:00:00"
     },
     "items": {
         "claude-3-5-sonnet-20240620": {