pygpt-net 2.4.41__py3-none-any.whl → 2.4.44__py3-none-any.whl

pygpt_net/core/idx/indexing.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 08:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import datetime
@@ -89,18 +89,35 @@ class Indexing:
                             "key": key,
                             "value": loader.init_args[key],
                             "type": "str",  # default = str
+                            "label": key,
+                            "description": None,
                         }
                         # from config
                         if key in loader.args:
                             self.external_config[loader.id][key]["value"] = loader.args[key]
                         if key in loader.init_args_types:
                             self.external_config[loader.id][key]["type"] = loader.init_args_types[key]
+                        if key in loader.init_args_labels:
+                            self.external_config[loader.id][key]["label"] = loader.init_args_labels[key]
+                        if key in loader.init_args_desc:
+                            self.external_config[loader.id][key]["description"] = loader.init_args_desc[key]
 
             except ImportError as e:
                 msg = "Error while registering data loader: " + loader.id + " - " + str(e)
                 self.window.core.debug.log(msg)
                 self.window.core.debug.log(e)
 
+    def get_loader(self, loader: str) -> Optional[BaseLoader]:
+        """
+        Get data loader by id
+
+        :param loader: loader id
+        :return: data loader instance
+        """
+        if loader in self.data_providers:
+            return self.data_providers[loader]
+        return None
+
     def update_loader_args(
             self,
             loader: str,
@@ -344,13 +361,13 @@ class Indexing:
             self,
             path: str,
             loader_kwargs: Optional[Dict[str, Any]] = None
-    ) -> str:
+    ) -> Tuple[str, List[Document]]:
         """
         Get content from file using loaders
 
         :param path: path to file
         :param loader_kwargs: additional keyword arguments for data loader
-        :return: file content
+        :return: text content, list of documents
         """
         docs = self.get_documents(
             path,
@@ -361,27 +378,27 @@ class Indexing:
         data = []
         for doc in docs:
             data.append(doc.text)
-        return "\n".join(data)
+        return "\n".join(data), docs
 
     def read_web_content(
             self,
             url: str,
             type: str = "webpage",
             extra_args: Optional[Dict[str, Any]] = None
-    ) -> str:
+    ) -> Tuple[str, List[Document]]:
         """
         Get content from external resource
 
         :param url: external url to index
         :param type: type of URL (webpage, feed, etc.)
         :param extra_args: extra arguments for loader
-        :return: file content
+        :return: text content, list of documents
         """
         docs = self.read_web(url, type, extra_args)
         data = []
         for doc in docs:
             data.append(doc.text)
-        return "\n".join(data)
+        return "\n".join(data), docs
 
     def read_web(
             self,
@@ -600,7 +617,10 @@ class Indexing:
 
         return indexed, errors
 
-    def get_db_data_from_ts(self, updated_ts: int = 0) -> list:
+    def get_db_data_from_ts(
+            self,
+            updated_ts: int = 0
+    ) -> List[Document]:
         """
         Get data from database from timestamp
 
@@ -640,7 +660,10 @@ class Indexing:
                 documents.append(doc)
         return documents
 
-    def get_db_meta_ids_from_ts(self, updated_ts: int = 0) -> list:
+    def get_db_meta_ids_from_ts(
+            self,
+            updated_ts: int = 0
+    ) -> List[int]:
         """
         Get IDs of meta from database from timestamp
 
@@ -664,7 +687,11 @@ class Indexing:
                 ids.append(data["id"])
         return ids
 
-    def get_db_data_by_id(self, id: int = 0, updated_ts: int = 0) -> list:
+    def get_db_data_by_id(
+            self,
+            id: int = 0,
+            updated_ts: int = 0
+    ) -> List[Document]:
         """
         Get data from database by meta id
 
@@ -726,7 +753,8 @@ class Indexing:
                 self.window.core.idx.log("Indexing documents from database by meta id: {}".format(id))
                 self.remove_old_meta_id(idx, id)
             elif from_ts > 0:
-                self.window.core.idx.log("Indexing documents from database by meta id: {} from timestamp: {}".format(id, from_ts))
+                self.window.core.idx.log("Indexing documents from database by meta id: {} from timestamp: {}".
+                                         format(id, from_ts))
 
             # get items from database
             documents = self.get_db_data_by_id(id, from_ts)
@@ -736,7 +764,8 @@ class Indexing:
 
                 self.index_document(index, d)
                 doc_id = d.id_
-                self.window.core.idx.log("Inserted ctx DB document: {} / {}, id: {}, metadata: {}".format(n+1, len(documents), d.id_, d.metadata))
+                self.window.core.idx.log("Inserted ctx DB document: {} / {}, id: {}, metadata: {}".
+                                         format(n+1, len(documents), d.id_, d.metadata))
                 self.window.core.ctx.idx.set_meta_as_indexed(id, idx, doc_id)  # update ctx
                 n += 1
         except Exception as e:
@@ -850,7 +879,8 @@ class Indexing:
                         idx=idx,
                         doc_id=doc_id,
                     )  # update external index
-                self.window.core.idx.log("Inserted web document: {} / {}, id: {}, metadata: {}".format(n+1, len(documents), d.id_, d.metadata))
+                self.window.core.idx.log("Inserted web document: {} / {}, id: {}, metadata: {}".
+                                         format(n+1, len(documents), d.id_, d.metadata))
                 n += 1
         except Exception as e:
             errors.append(str(e))
@@ -1044,7 +1074,10 @@ class Indexing:
             model = self.window.core.models.from_defaults()
 
         service_context = self.window.core.idx.llm.get_service_context(model=model)
-        index = self.window.core.idx.storage.get_ctx_idx(index_path, service_context=service_context)  # get or create ctx index
+        index = self.window.core.idx.storage.get_ctx_idx(
+            index_path,
+            service_context=service_context
+        )  # get or create ctx index
 
         idx = "tmp:{}".format(index_path)  # tmp index id
         self.window.core.idx.log("Indexing to context attachment index: {}...".format(idx))

pygpt_net/core/idx/ui/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.12.16 01:00:00                  #
+# ================================================== #
+
+from .loaders import Loaders
+
+class UI:
+    def __init__(self, window=None):
+        """
+        UI components
+
+        :param window: Window instance
+        """
+        self.window = window
+        self.loaders = Loaders(window)

pygpt_net/core/idx/ui/loaders.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.12.16 01:00:00                  #
+# ================================================== #
+
+import json
+from typing import Dict, Tuple, Any, Optional
+
+from PySide6.QtCore import Qt
+from PySide6.QtWidgets import QVBoxLayout, QLabel, QHBoxLayout, QWidget
+
+from pygpt_net.ui.widget.element.labels import HelpLabel
+from pygpt_net.ui.widget.option.input import OptionInput
+
+
+class Loaders:
+    def __init__(self, window=None):
+        """
+        UI - loaders components
+
+        :param window: Window instance
+        """
+        self.window = window
+
+    def handle_options(
+            self,
+            select_loader,
+            prefix_options,
+            prefix_config
+    ) -> Tuple[bool, Optional[str], Dict[str, Any], Dict[str, Any]]:
+        """
+        Handle options
+
+        :param select_loader: loader selection
+        :param prefix_options: prefix for options
+        :param prefix_config: prefix for config
+        :return: bool, loader name, input_params, input_config
+        """
+        input_params = {}
+        input_config = {}
+        loader = select_loader.get_value()
+        if not loader:
+            return False, loader, input_params, input_config
+        loaders = self.window.core.idx.indexing.get_external_instructions()
+        if loader in loaders:
+            params = loaders[loader]
+            for k in params["args"]:
+                key_path = prefix_options + "." + loader + "." + k
+                if key_path in self.window.ui.nodes:
+                    tmp_value = self.window.ui.nodes[key_path].text()
+                    type = params["args"][k]["type"]
+                    try:
+                        if tmp_value:
+                            if type == "int":
+                                tmp_value = int(tmp_value)
+                            elif type == "float":
+                                tmp_value = float(tmp_value)
+                            elif type == "bool":
+                                if tmp_value.lower() in ["true", "1"]:
+                                    tmp_value = True
+                                else:
+                                    tmp_value = False
+                            elif type == "list":
+                                tmp_value = tmp_value.split(",")
+                            elif type == "dict":
+                                tmp_value = json.loads(tmp_value)
+                            input_params[k] = tmp_value
+                    except Exception as e:
+                        self.window.core.debug.log(e)
+                        self.window.ui.dialogs.alert(e)
+
+        loaders = self.window.core.idx.indexing.get_external_config()
+        if loader in loaders:
+            params = loaders[loader]
+            for k in params:
+                key_path = prefix_config + "." + loader + "." + k
+                type = params[k]["type"]
+                if key_path in self.window.ui.nodes:
+                    tmp_value = self.window.ui.nodes[key_path].text()
+                    try:
+                        if tmp_value:
+                            if type == "int":
+                                tmp_value = int(tmp_value)
+                            elif type == "float":
+                                tmp_value = float(tmp_value)
+                            elif type == "bool":
+                                if tmp_value.lower() in ["true", "1"]:
+                                    tmp_value = True
+                                else:
+                                    tmp_value = False
+                            elif type == "list":
+                                tmp_value = tmp_value.split(",")
+                            elif type == "dict":
+                                tmp_value = json.loads(tmp_value)
+                            input_config[k] = tmp_value
+                    except Exception as e:
+                        self.window.core.debug.log(e)
+                        self.window.ui.dialogs.alert(e)
+
+        return True, loader, input_params, input_config
+
+    def setup_loader_options(self):
+        """Setup loader options"""
+        inputs = {}
+        groups = {}
+        loaders = self.window.core.idx.indexing.get_external_instructions()
+        for loader in loaders:
+            params = loaders[loader]
+            inputs[loader] = {}
+            group = QVBoxLayout()
+            for k in params["args"]:
+                label = k
+                description = None
+                is_label = False
+                if "label" in params["args"][k]:
+                    label = params["args"][k]["label"]
+                    is_label = True
+                if "description" in params["args"][k]:
+                    description = params["args"][k]["description"]
+                option_id = "web.loader." + loader + ".option." + k
+                option_widget = OptionInput(self.window, "tool.indexer", option_id, {
+                    "label": label,
+                    "value": "",
+                })
+                option_widget.setPlaceholderText(params["args"][k]["type"])
+                inputs[loader][k] = option_widget
+
+                option_label = QLabel(label)
+                option_label.setToolTip(k)
+
+                row = QHBoxLayout()  # cols
+                row.addWidget(option_label)
+                row.addWidget(option_widget)
+                row.setContentsMargins(5, 0, 5, 0)
+
+                option_layout = QVBoxLayout()
+                option_layout.addLayout(row)
+                if description:
+                    option_layout.addWidget(HelpLabel(description))
+
+                option_layout.setContentsMargins(5, 0, 0, 0)
+
+                group.addLayout(option_layout)
+                group.setContentsMargins(0, 0, 0, 0)
+
+            group_widget = QWidget()
+            group_widget.setLayout(group)
+            groups[loader] = group_widget
+
+        return inputs, groups
+
+    def setup_loader_config(self):
+        """Setup loader config"""
+        inputs = {}
+        groups = {}
+        loaders = self.window.core.idx.indexing.get_external_config()
+        for loader in loaders:
+            params = loaders[loader]
+            inputs[loader] = {}
+            group = QVBoxLayout()
+            for k in params:
+                label = k
+                description = None
+                is_label = False
+                if "label" in params[k]:
+                    label = params[k]["label"]
+                    is_label = True
+                if "description" in params[k]:
+                    description = params[k]["description"]
+                option_id = "web.loader." + loader + ".config." + k
+                option_widget = OptionInput(self.window, "tool.indexer", option_id, {
+                    "label": label,
+                    "value": params[k]["value"],
+                })
+                try:
+                    if params[k]["value"] is not None:
+                        if params[k]["type"] == "list" and isinstance(params[k]["value"], list):
+                            option_widget.setText(", ".join(params[k]["value"]))
+                        elif params[k]["type"] == "dict" and isinstance(params[k]["value"], dict):
+                            option_widget.setText(json.dumps(params[k]["value"]))
+                        else:
+                            option_widget.setText(str(params[k]["value"]))
+                except Exception as e:
+                    self.window.core.debug.log(e)
+
+                option_widget.setPlaceholderText(params[k]["type"])
+                inputs[loader][k] = option_widget
+
+                option_label = QLabel(label)
+                option_label.setToolTip(k)
+
+                row = QHBoxLayout()  # cols
+                row.addWidget(option_label)
+                row.addWidget(option_widget)
+                row.setContentsMargins(5, 0, 5, 0)
+
+                option_layout = QVBoxLayout()
+                option_layout.addLayout(row)
+                if description:
+                    option_layout.addWidget(HelpLabel(description))
+
+                option_layout.setContentsMargins(5, 0, 0, 0)
+
+                group.addLayout(option_layout)
+                group.setContentsMargins(0, 0, 0, 0)
+
+            group_widget = QWidget()
+            group_widget.setLayout(group)
+            groups[loader] = group_widget
+
+        return inputs, groups

pygpt_net/core/installer.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2023.12.31 04:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 from pathlib import Path
@@ -23,9 +23,7 @@ class Installer:
         self.window = window
 
     def install(self):
-        """
-        Install db, config data and directories
-        """
+        """Install db, config data and directories"""
         try:
             # create user config directory
             path = Path(self.window.core.config.path)

pygpt_net/core/models.py

@@ -6,11 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 00:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import copy
-from typing import Optional
+from typing import Optional, List, Dict
 
 from packaging.version import Version
 
@@ -92,7 +92,7 @@ class Models:
         if key in self.items:
             return self.items[key]
 
-    def get_ids(self) -> list:
+    def get_ids(self) -> List[str]:
         """
         Return models ids
 
@@ -118,7 +118,11 @@ class Models:
         """
         return model in self.items
 
-    def is_allowed(self, model: str, mode: str) -> bool:
+    def is_allowed(
+            self,
+            model: str,
+            mode: str
+    ) -> bool:
         """
         Check if model is allowed for mode
 
@@ -130,7 +134,10 @@ class Models:
             return mode in self.items[model].mode
         return False
 
-    def get_id(self, key: str) -> str:
+    def get_id(
+            self,
+            key: str
+    ) -> str:
         """
         Return model internal ID
 
@@ -140,7 +147,11 @@ class Models:
         if key in self.items:
             return self.items[key].id
 
-    def get_by_idx(self, idx: int, mode: str) -> str:
+    def get_by_idx(
+            self,
+            idx: int,
+            mode: str
+    ) -> str:
         """
         Return model by index
 
@@ -151,7 +162,10 @@ class Models:
         items = self.get_by_mode(mode)
         return list(items.keys())[idx]
 
-    def get_by_mode(self, mode: str) -> dict:
+    def get_by_mode(
+            self,
+            mode: str
+    ) -> Dict[str, ModelItem]:
         """
         Return models for mode
 
@@ -164,7 +178,11 @@ class Models:
                 items[key] = self.items[key]
         return items
 
-    def get_next(self, model: str, mode: str) -> str:
+    def get_next(
+            self,
+            model: str,
+            mode: str
+    ) -> str:
         """
         Return next model
 
@@ -179,7 +197,11 @@ class Models:
             return keys[idx + 1]
         return keys[0]
 
-    def get_prev(self, model: str, mode: str) -> str:
+    def get_prev(
+            self,
+            model: str,
+            mode: str
+    ) -> str:
         """
         Return previous model
 
@@ -218,7 +240,7 @@ class Models:
         self.items[id] = model
         return model
 
-    def get_all(self) -> dict:
+    def get_all(self) -> Dict[str, ModelItem]:
         """
         Return all models
 
@@ -248,7 +270,11 @@ class Models:
         if model in self.items:
             del self.items[model]
 
-    def has_model(self, mode: str, model: str) -> bool:
+    def has_model(
+            self,
+            mode: str,
+            model: str
+    ) -> bool:
         """
         Check if model exists for mode
 
@@ -296,8 +322,15 @@ class Models:
             return self.items[model].ctx
         return 4096
 
-    def restore_default(self, model: Optional[str] = None):
-        """Restore default models"""
+    def restore_default(
+            self,
+            model: Optional[str] = None
+    ):
+        """
+        Restore default models
+
+        :param model: model name
+        """
         # restore all models
         if model is None:
             self.load_base()
@@ -308,8 +341,12 @@ class Models:
         if model in items:
             self.items[model] = items[model]
 
-    def get_base(self) -> dict:
-        """Get base models"""
+    def get_base(self) -> Dict[str, ModelItem]:
+        """
+        Get base models
+
+        :return: base models
+        """
         return self.provider.load_base()
 
     def load_base(self):
@@ -330,7 +367,11 @@ class Models:
         """Save models"""
         self.provider.save(self.items)
 
-    def get_supported_mode(self, model: ModelItem, mode: str) -> str:
+    def get_supported_mode(
+            self,
+            model: ModelItem,
+            mode: str
+    ) -> str:
         """
         Get supported mode
 
@@ -353,5 +394,9 @@ class Models:
         return mode
 
     def get_version(self) -> str:
-        """Get config version"""
+        """
+        Get config version
+
+        :return: config version
+        """
         return self.provider.get_version()

pygpt_net/core/modes.py

@@ -6,9 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.26 19:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
+from typing import Dict, List
+
 from pygpt_net.provider.core.mode.json_file import JsonFileProvider
 from pygpt_net.core.types import (
     MODE_AGENT,
@@ -51,7 +53,7 @@ class Modes:
         ]
         self.items = {}
 
-    def get_by_idx(self, idx):
+    def get_by_idx(self, idx) -> str:
         """
         Return mode by index
 
@@ -61,7 +63,7 @@ class Modes:
         modes = self.get_all()
         return list(modes.keys())[idx]
 
-    def get_idx_by_name(self, name):
+    def get_idx_by_name(self, name) -> int:
         """
         Return mode index by name
 
@@ -71,7 +73,7 @@ class Modes:
         modes = self.get_all()
         return list(modes.keys()).index(name)
 
-    def get_all(self):
+    def get_all(self) -> Dict[str, List[str]]:
         """
         Return modes
 
@@ -79,7 +81,7 @@ class Modes:
         """
         return self.items
 
-    def get_default(self):
+    def get_default(self) -> str:
         """
         Return default mode name
 
@@ -89,7 +91,7 @@ class Modes:
             if self.items[id].default:
                 return id
 
-    def get_next(self, mode: str):
+    def get_next(self, mode: str) -> str:
         """
         Return next mode
 
@@ -103,7 +105,7 @@ class Modes:
             return keys[idx + 1]
         return keys[0]
 
-    def get_prev(self, mode: str):
+    def get_prev(self, mode: str) -> str:
         """
         Return previous mode
 
@@ -118,13 +120,9 @@ class Modes:
         return keys[-1]
 
     def load(self):
-        """
-        Load modes
-        """
+        """Load modes"""
         self.items = self.provider.load()
 
     def save(self):
-        """
-        Save modes
-        """
+        """Save modes"""
         self.provider.save(self.items)