pygpt-net 2.4.42__py3-none-any.whl → 2.4.45__py3-none-any.whl

pygpt_net/plugin/cmd_files/__init__.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.20 03:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import os
@@ -141,7 +141,8 @@ class Plugin(BasePlugin):
         """
         # use_loaders = False
         if use_loaders:
-            return str(self.window.core.idx.indexing.read_text_content(path))
+            content, docs = str(self.window.core.idx.indexing.read_text_content(path))
+            return content
         else:
             data = ""
             if os.path.isfile(path):

pygpt_net/provider/core/attachment/json_file.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.14 22:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -145,6 +145,7 @@ class JsonFileProvider(BaseProvider):
             'send': attachment.send,
             'vector_store_ids': attachment.vector_store_ids,
             'type': attachment.type,
+            'extra': attachment.extra,
         }
 
     @staticmethod
@@ -169,6 +170,8 @@ class JsonFileProvider(BaseProvider):
             attachment.vector_store_ids = data['vector_store_ids']
         if 'type' in data:
             attachment.type = data['type']
+        if 'extra' in data:
+            attachment.extra = data['extra']
 
     def dump(self, item: AttachmentItem) -> str:
         """

pygpt_net/provider/core/config/patch.py

@@ -1776,6 +1776,18 @@ class Patch:
                 self.window.core.updater.patch_css('web-chatgpt_wide.css', True)  # force update
                 updated = True
 
+            # < 2.4.44
+            if old < parse_version("2.4.44"):
+                print("Migrating config from < 2.4.44...")
+                data["ctx.records.folders.top"] = True
+                updated = True
+
+            # < 2.4.45
+            if old < parse_version("2.4.45"):
+                print("Migrating config from < 2.4.45...")
+                self.window.core.updater.patch_css('style.css', True)  # force update
+                updated = True
+
         # update file
         migrated = False
         if updated:

pygpt_net/provider/core/ctx/db_sqlite/storage.py

@@ -68,6 +68,9 @@ class Storage:
         # only base by default
         where_clauses.append("(m.root_id IS NULL OR m.root_id = 0)")
 
+        # join group
+        join_clauses.append("LEFT JOIN ctx_group g ON m.group_id = g.id")
+
         # search_string
         if search_string:
             date_ranges = search_by_date_string(search_string)
@@ -106,15 +109,16 @@ class Storage:
                     continue
                 mode = filter.get('mode', '=')
                 value = filter.get('value', '')
+                key_name = 'm.' + key
                 if isinstance(value, int):
-                    where_clauses.append(f"{key} {mode} :{key}")
+                    where_clauses.append(f"{key_name} {mode} :{key}")
                     bind_params[key] = value
                 elif isinstance(value, str):
-                    where_clauses.append(f"{key} {mode} :{key}")
+                    where_clauses.append(f"{key_name} {mode} :{key}")
                     bind_params[key] = f"%{value}%"
                 elif isinstance(value, list):
                     values = "(" + ",".join([str(x) for x in value]) + ")"
-                    where_clauses.append(f"{key} {mode} {values}")
+                    where_clauses.append(f"{key_name} {mode} {values}")
 
         where_statement = " AND ".join(where_clauses) if where_clauses else "1"
         join_statement = " ".join(join_clauses) if join_clauses else ""
@@ -154,8 +158,18 @@ class Storage:
             append_date_ranges=True,
         )
         stmt_text = f"""
-            SELECT m.* FROM ctx_meta m {join_statement} WHERE {where_statement}
-            ORDER BY m.updated_ts DESC {limit_suffix}
+            SELECT 
+                m.*,
+                g.name as group_name,
+                g.uuid as group_uuid,
+                g.additional_ctx_json as group_additional_ctx_json
+            FROM 
+                ctx_meta m 
+                {join_statement} 
+            WHERE 
+                {where_statement}
+            ORDER BY 
+                m.updated_ts DESC {limit_suffix}
         """
         stmt = text(stmt_text).bindparams(**bind_params)
 
@@ -177,7 +191,17 @@ class Storage:
         :return: dict of CtxMeta
         """
         stmt_text = f"""
-            SELECT * FROM ctx_meta WHERE indexed_ts > 0
+            SELECT 
+                m.*, 
+                g.name as group_name,
+                g.uuid as group_uuid,
+                g.additional_ctx_json as group_additional_ctx_json 
+            FROM 
+                ctx_meta m 
+            LEFT JOIN 
+                ctx_group g ON m.group_id = g.id
+            WHERE 
+                indexed_ts > 0
         """
         stmt = text(stmt_text)
         items = {}
@@ -431,7 +455,26 @@ class Storage:
         )
         with db.begin() as conn:
             conn.execute(stmt)
-            return True
+
+        # update group
+        if meta.group:
+            stmt = text("""
+                UPDATE ctx_group
+                SET
+                    name = :name,
+                    additional_ctx_json = :additional_ctx_json,
+                    updated_ts = :updated_ts
+                WHERE id = :id
+            """).bindparams(
+                id=meta.group.id,
+                name=meta.group.name,
+                additional_ctx_json=pack_item_value(meta.group.additional_ctx),
+                updated_ts=int(time.time()),
+            )
+            with db.begin() as conn:
+                conn.execute(stmt)
+
+        return True
 
     def update_meta_all(
             self,

pygpt_net/provider/core/ctx/db_sqlite/utils.py

@@ -125,7 +125,10 @@ def unpack_item_value(value: Any) -> Any:
         return value
 
 
-def unpack_item(item: CtxItem, row: Dict[str, Any]) -> CtxItem:
+def unpack_item(
+        item: CtxItem,
+        row: Dict[str, Any]
+) -> CtxItem:
     """
     Unpack context item from DB row
 
@@ -185,9 +188,12 @@ def unpack_item(item: CtxItem, row: Dict[str, Any]) -> CtxItem:
     return item
 
 
-def unpack_meta(meta: CtxMeta, row: Dict[str, Any]) -> CtxMeta:
+def unpack_meta(
+        meta: CtxMeta,
+        row: Dict[str, Any]
+) -> CtxMeta:
     """
-    Unpack context meta data from DB row
+    Unpack context meta-data from DB row
 
     :param meta: Context meta (CtxMeta)
     :param row: DB row
@@ -221,20 +227,38 @@ def unpack_meta(meta: CtxMeta, row: Dict[str, Any]) -> CtxMeta:
 
     if meta.additional_ctx is None:
         meta.additional_ctx = []
+
+    # add group if exists
+    if meta.group_id:
+        group = CtxGroup()
+        group.id = meta.group_id
+        group.uuid = row['group_uuid']
+        group.name = row['group_name']
+        group.additional_ctx = unpack_item_value(row['group_additional_ctx_json'])
+        if group.additional_ctx is None:
+            group.additional_ctx = []
+        meta.group = group
+
     return meta
 
 
-def unpack_group(group: CtxGroup, row: Dict[str, Any]) -> CtxGroup:
+def unpack_group(
+        group: CtxGroup,
+        row: Dict[str, Any]
+) -> CtxGroup:
     """
     Unpack context group data from DB row
 
     :param group: Context group (CtxGroup)
     :param row: DB row
-    :return: context meta
+    :return: context group
     """
     group.id = unpack_var(row['id'], 'int')
     group.uuid = row['uuid']
     group.created = unpack_var(row['created_ts'], 'int')
     group.updated = unpack_var(row['updated_ts'], 'int')
     group.name = row['name']
+    group.additional_ctx = unpack_item_value(row['additional_ctx_json'])
+    if group.additional_ctx is None:
+        group.additional_ctx = []
     return group

pygpt_net/provider/loaders/base.py

@@ -22,7 +22,9 @@ class BaseLoader:
         self.instructions = []  # list of instructions for 'web_index' command for how to handle this type
         self.args = {}  # custom keyword arguments
         self.init_args = {}  # initial keyword arguments
+        self.init_args_labels = {}
         self.init_args_types = {}
+        self.init_args_desc = {}
         self.allow_compiled = True  # allow in compiled and Snap versions
         # This is required due to some readers may require Python environment to install additional packages
 
@@ -42,6 +44,18 @@ class BaseLoader:
         """
         self.args = args
 
+    def explode(self, value: str) -> list:
+        """
+        Explode list from string
+
+        :param value: value string
+        :return: list
+        """
+        if value:
+            items = value.split(",")
+            return [item.strip() for item in items]
+        return []
+
     def get_args(self):
         """
         Prepare keyword arguments for reader init method

pygpt_net/provider/loaders/hub/google/gmail.py

@@ -21,7 +21,7 @@ class GmailReader(BaseReader, BaseModel):
     Reads emails
 
     Args:
-        max_results (int): Defaults to 10.
+        max_results (int): Defaults to 10.``
         query (str): Gmail query. Defaults to None.
         service (Any): Gmail service. Defaults to None.
         results_per_page (Optional[int]): Max number of results per page. Defaults to 10.
@@ -32,7 +32,7 @@ class GmailReader(BaseReader, BaseModel):
     token_path: str = "token.json"
     use_iterative_parser: bool = False
     max_results: int = 10
-    service: Any
+    service: Any = None
     results_per_page: Optional[int]
 
     def load_data(self, query: str = None) -> List[Document]:

pygpt_net/provider/loaders/hub/yt/base.py

@@ -15,6 +15,7 @@ class YoutubeTranscriptReader(BasePydanticReader):
     """Youtube Transcript reader."""
 
     is_remote: bool = True
+    languages: List[str] = ["en"]
 
     @classmethod
     def class_name(cls) -> str:
@@ -34,6 +35,10 @@ class YoutubeTranscriptReader(BasePydanticReader):
                 for which transcripts are to be read.
 
         """
+        languages = self.languages
+        if not languages:
+            languages = ["en"]
+
         results = []
         for link in ytlinks:
             video_id = self._extract_video_id(link)

pygpt_net/provider/loaders/web_database.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,14 +30,16 @@ class Loader(BaseLoader):
                     "args": {
                         "query": {
                             "type": "str",
+                            "label": "SQL query",
+                            "description": "SQL query to read data from database, e.g. SELECT * FROM table",
                         },
                     },
                 }
             }
         ]
         self.init_args = {
-            "sql_database": None,
-            "engine": None,
+            # "sql_database": None,
+            # "engine": None,
             "uri": None,
             "scheme": None,
             "host": None,
@@ -47,8 +49,8 @@ class Loader(BaseLoader):
             "dbname": None,
         }
         self.init_args_types = {
-            "sql_database": "str",
-            "engine": "str",
+            # "sql_database": "str",
+            # "engine": "str",
             "uri": "str",
             "scheme": "str",
             "host": "str",
@@ -57,6 +59,12 @@ class Loader(BaseLoader):
             "password": "str",
             "dbname": "str",
         }
+        self.init_args_desc = {
+            # "sql_database": "str",
+            # "engine": "str",
+            "uri": "You can provide a single URI in the form of: {scheme}://{user}:{password}@{host}:{port}/{dbname}, "
+                   "or you can provide each field manually:",
+        }
 
     def get(self) -> BaseReader:
         """

pygpt_net/provider/loaders/web_github_issues.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,19 +30,27 @@ class Loader(BaseLoader):
                     "args": {
                         "owner": {
                             "type": "str",
+                            "label": "Owner",
                         },
                         "repository": {
                             "type": "str",
+                            "label": "Repository",
                         },
                         "state": {
                             "type": "enum",
                             "options": ["open", "closed", "all"],
+                            "label": "State",
+                            "description": "Issue state, default: open, available options: open, closed, all",
                         },
                         "label_filters_include": {
                             "type": "list",
+                            "label": "Label filters include",
+                            "description": "List of labels to include, separated by comma (,)",
                         },
                         "label_filters_exclude": {
                             "type": "list",
+                            "label": "Label filters exclude",
+                            "description": "List of labels to exclude, separated by comma (,)",
                         },
                     },
                 }
@@ -98,13 +106,22 @@ class Loader(BaseLoader):
         if "repository" in kwargs and kwargs.get("repository"):
             if isinstance(kwargs.get("repository"), str):
                 args["repository"] = kwargs.get("repository")  # repo name
+        if "state" in kwargs and kwargs.get("state"):
+            if isinstance(kwargs.get("state"), str):
+                args["state"] = kwargs.get("state")
+                if args["state"] not in ["open", "closed", "all"]:
+                    args["state"] = "open"
 
         # filters
         if "label_filters_include" in kwargs and kwargs.get("label_filters_include"):
             if isinstance(kwargs.get("label_filters_include"), list):
                 args["label_filters_include"] = kwargs.get("label_filters_include")
+            elif isinstance(kwargs.get("label_filters_include"), str):
+                args["label_filters_include"] = self.explode(kwargs.get("label_filters_include"))
         if "label_filters_exclude" in kwargs and kwargs.get("label_filters_exclude"):
             if isinstance(kwargs.get("label_filters_exclude"), list):
                 args["label_filters_exclude"] = kwargs.get("label_filters_exclude")
+            elif isinstance(kwargs.get("label_filters_exclude"), str):
+                args["label_filters_exclude"] = self.explode(kwargs.get("label_filters_exclude"))
 
         return args

pygpt_net/provider/loaders/web_github_repo.py

@@ -30,15 +30,19 @@ class Loader(BaseLoader):
                     "args": {
                         "commit_sha": {
                             "type": "str",
+                            "label": "Commit SHA",
                         },
                         "branch": {
                             "type": "str",
+                            "label": "Branch",
                         },
                         "owner": {
                             "type": "str",
+                            "label": "Owner",
                         },
                         "repository": {
                             "type": "str",
+                            "label": "Repository",
                         }
                     },
                 },
@@ -68,6 +72,12 @@ class Loader(BaseLoader):
             "filter_file_ext_include": "list",
             "filter_file_ext_exclude": "list",
         }
+        self.init_args_desc = {
+            "filter_dirs_include": "List of directories to include, separated by comma (,)",
+            "filter_dirs_exclude": "List of directories to exclude, separated by comma (,)",
+            "filter_file_ext_include": "List of file extensions to include, separated by comma (,)",
+            "filter_file_ext_exclude": "list of file extensions to exclude, separated by comma (,)",
+        }
 
     def get(self) -> BaseReader:
         """

pygpt_net/provider/loaders/web_google_calendar.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,9 +30,13 @@ class Loader(BaseLoader):
                     "args": {
                         "number_of_results": {
                             "type": "int",
+                            "label": "Number of results",
+                            "description": "Number of results to fetch, default: 100",
                         },
                         "start_date": {
                             "type": "str",
+                            "label": "Start date",
+                            "description": "Start date for events, ISO format: YYYY-MM-DD, default: today",
                         },
                     },
                 }
@@ -79,8 +83,12 @@ class Loader(BaseLoader):
         if "number_of_results" in kwargs and kwargs.get("number_of_results"):
             if isinstance(kwargs.get("number_of_results"), int):
                 args["number_of_results"] = kwargs.get("number_of_results")  # number of results
+                if args["number_of_results"] == 0:
+                    args["number_of_results"] = None
 
         if "start_date" in kwargs and kwargs.get("start_date"):
             if isinstance(kwargs.get("start_date"), str):
                 args["start_date"] = kwargs.get("start_date")  # start date
+                if args["start_date"] == "today" or args["start_date"] == "":
+                    args["start_date"] = None
         return args

pygpt_net/provider/loaders/web_google_docs.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,9 @@ class Loader(BaseLoader):
                     "args": {
                         "document_ids": {
                             "type": "list",
+                            "label": "Document IDs",
+                            "description": "List of document IDs to index, separated by comma (,)",
+                            "required": True,
                         },
                     },
                 }
@@ -76,4 +79,6 @@ class Loader(BaseLoader):
         if "document_ids" in kwargs and kwargs.get("document_ids"):
             if isinstance(kwargs.get("document_ids"), list):
                 args["document_ids"] = kwargs.get("document_ids")  # list of document ids
+            elif isinstance(kwargs.get("document_ids"), str):
+                args["document_ids"] = self.explode(kwargs.get("document_ids"))
         return args

pygpt_net/provider/loaders/web_google_drive.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.08.19 20:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,12 +30,17 @@ class Loader(BaseLoader):
                     "args": {
                         "folder_id": {
                             "type": "str",
+                            "label": "Folder ID",
                         },
                         "file_ids": {
                             "type": "list",
+                            "label": "File IDs",
+                            "description": "List of file ids, separated by comma (,)",
                         },
                         "mime_types": {
                             "type": "list",
+                            "label": "Mime Types",
+                            "description": "List of mime types, separated by comma (,)",
                         },
                     },
                 }
@@ -98,8 +103,12 @@ class Loader(BaseLoader):
         if "file_ids" in kwargs and kwargs.get("file_ids"):
             if isinstance(kwargs.get("file_ids"), list):
                 args["file_ids"] = kwargs.get("file_ids")  # list of file ids
+            elif isinstance(kwargs.get("file_ids"), str):
+                args["file_ids"] = self.explode(kwargs.get("file_ids"))
 
         if "mime_types" in kwargs and kwargs.get("mime_types"):
             if isinstance(kwargs.get("mime_types"), list):
                 args["mime_types"] = kwargs.get("mime_types")  # list of mime types
+            elif isinstance(kwargs.get("mime_types"), str):
+                args["mime_types"] = self.explode(kwargs.get("mime_types"))
         return args

pygpt_net/provider/loaders/web_google_gmail.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,8 @@ class Loader(BaseLoader):
                     "args": {
                         "query": {
                             "type": "str",
+                            "label": "Query",
+                            "description": "Query to search emails, e.g. from:me after:2023-01-01",
                         },
                     },
                 }
@@ -40,7 +42,7 @@ class Loader(BaseLoader):
             "token_path": "token.json",
             "use_iterative_parser": False,
             "max_results": 10,
-            "service": None,
+            # "service": None,
             "results_per_page": None,
         }
         self.init_args_types = {
@@ -48,7 +50,7 @@ class Loader(BaseLoader):
             "token_path": "str",
             "use_iterative_parser": "bool",
             "max_results": "int",
-            "service": "str",
+            # "service": "str",
             "results_per_page": "int",
         }
 

pygpt_net/provider/loaders/web_google_keep.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,8 @@ class Loader(BaseLoader):
                     "args": {
                         "document_ids": {
                             "type": "list",
+                            "label": "Document IDs",
+                            "description": "List of document ids, separated by comma (,)",
                         },
                     },
                 }
@@ -74,4 +76,6 @@ class Loader(BaseLoader):
         if "document_ids" in kwargs and kwargs.get("document_ids"):
             if isinstance(kwargs.get("document_ids"), list):
                 args["document_ids"] = kwargs.get("document_ids")  # list of document ids
+            elif isinstance(kwargs.get("document_ids"), str):
+                args["document_ids"] = self.explode(kwargs.get("document_ids"))
         return args

pygpt_net/provider/loaders/web_google_sheets.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,8 @@ class Loader(BaseLoader):
                     "args": {
                         "spreadsheet_ids": {
                             "type": "list",
+                            "label": "Spreadsheet IDs",
+                            "description": "List of spreadsheet ids, separated by comma (,)",
                         },
                     },
                 }
@@ -78,4 +80,6 @@ class Loader(BaseLoader):
         if "spreadsheet_ids" in kwargs and kwargs.get("spreadsheet_ids"):
             if isinstance(kwargs.get("spreadsheet_ids"), list):
                 args["spreadsheet_ids"] = kwargs.get("spreadsheet_ids")  # spreadsheet ids
+            elif isinstance(kwargs.get("spreadsheet_ids"), str):
+                args["spreadsheet_ids"] = self.explode(kwargs.get("spreadsheet_ids"))
         return args