pygpt-net 2.4.42__py3-none-any.whl → 2.4.44__py3-none-any.whl

pygpt_net/provider/core/ctx/db_sqlite/storage.py

@@ -68,6 +68,9 @@ class Storage:
         # only base by default
         where_clauses.append("(m.root_id IS NULL OR m.root_id = 0)")
 
+        # join group
+        join_clauses.append("LEFT JOIN ctx_group g ON m.group_id = g.id")
+
         # search_string
         if search_string:
             date_ranges = search_by_date_string(search_string)
@@ -106,15 +109,16 @@ class Storage:
                     continue
                 mode = filter.get('mode', '=')
                 value = filter.get('value', '')
+                key_name = 'm.' + key
                 if isinstance(value, int):
-                    where_clauses.append(f"{key} {mode} :{key}")
+                    where_clauses.append(f"{key_name} {mode} :{key}")
                     bind_params[key] = value
                 elif isinstance(value, str):
-                    where_clauses.append(f"{key} {mode} :{key}")
+                    where_clauses.append(f"{key_name} {mode} :{key}")
                     bind_params[key] = f"%{value}%"
                 elif isinstance(value, list):
                     values = "(" + ",".join([str(x) for x in value]) + ")"
-                    where_clauses.append(f"{key} {mode} {values}")
+                    where_clauses.append(f"{key_name} {mode} {values}")
 
         where_statement = " AND ".join(where_clauses) if where_clauses else "1"
         join_statement = " ".join(join_clauses) if join_clauses else ""
@@ -154,8 +158,18 @@ class Storage:
             append_date_ranges=True,
         )
         stmt_text = f"""
-            SELECT m.* FROM ctx_meta m {join_statement} WHERE {where_statement}
-            ORDER BY m.updated_ts DESC {limit_suffix}
+            SELECT 
+                m.*,
+                g.name as group_name,
+                g.uuid as group_uuid,
+                g.additional_ctx_json as group_additional_ctx_json
+            FROM 
+                ctx_meta m 
+                {join_statement} 
+            WHERE 
+                {where_statement}
+            ORDER BY 
+                m.updated_ts DESC {limit_suffix}
         """
         stmt = text(stmt_text).bindparams(**bind_params)
 
@@ -177,7 +191,17 @@ class Storage:
         :return: dict of CtxMeta
         """
         stmt_text = f"""
-            SELECT * FROM ctx_meta WHERE indexed_ts > 0
+            SELECT 
+                m.*, 
+                g.name as group_name,
+                g.uuid as group_uuid,
+                g.additional_ctx_json as group_additional_ctx_json 
+            FROM 
+                ctx_meta m 
+            LEFT JOIN 
+                ctx_group g ON m.group_id = g.id
+            WHERE 
+                indexed_ts > 0
         """
         stmt = text(stmt_text)
         items = {}
@@ -431,7 +455,26 @@ class Storage:
         )
         with db.begin() as conn:
             conn.execute(stmt)
-            return True
+
+        # update group
+        if meta.group:
+            stmt = text("""
+                UPDATE ctx_group
+                SET
+                    name = :name,
+                    additional_ctx_json = :additional_ctx_json,
+                    updated_ts = :updated_ts
+                WHERE id = :id
+            """).bindparams(
+                id=meta.group.id,
+                name=meta.group.name,
+                additional_ctx_json=pack_item_value(meta.group.additional_ctx),
+                updated_ts=int(time.time()),
+            )
+            with db.begin() as conn:
+                conn.execute(stmt)
+
+        return True
 
     def update_meta_all(
             self,

pygpt_net/provider/core/ctx/db_sqlite/utils.py

@@ -125,7 +125,10 @@ def unpack_item_value(value: Any) -> Any:
         return value
 
 
-def unpack_item(item: CtxItem, row: Dict[str, Any]) -> CtxItem:
+def unpack_item(
+        item: CtxItem,
+        row: Dict[str, Any]
+) -> CtxItem:
     """
     Unpack context item from DB row
 
@@ -185,9 +188,12 @@ def unpack_item(item: CtxItem, row: Dict[str, Any]) -> CtxItem:
     return item
 
 
-def unpack_meta(meta: CtxMeta, row: Dict[str, Any]) -> CtxMeta:
+def unpack_meta(
+        meta: CtxMeta,
+        row: Dict[str, Any]
+) -> CtxMeta:
     """
-    Unpack context meta data from DB row
+    Unpack context meta-data from DB row
 
     :param meta: Context meta (CtxMeta)
     :param row: DB row
@@ -221,20 +227,38 @@ def unpack_meta(meta: CtxMeta, row: Dict[str, Any]) -> CtxMeta:
 
     if meta.additional_ctx is None:
         meta.additional_ctx = []
+
+    # add group if exists
+    if meta.group_id:
+        group = CtxGroup()
+        group.id = meta.group_id
+        group.uuid = row['group_uuid']
+        group.name = row['group_name']
+        group.additional_ctx = unpack_item_value(row['group_additional_ctx_json'])
+        if group.additional_ctx is None:
+            group.additional_ctx = []
+        meta.group = group
+
     return meta
 
 
-def unpack_group(group: CtxGroup, row: Dict[str, Any]) -> CtxGroup:
+def unpack_group(
+        group: CtxGroup,
+        row: Dict[str, Any]
+) -> CtxGroup:
     """
     Unpack context group data from DB row
 
     :param group: Context group (CtxGroup)
     :param row: DB row
-    :return: context meta
+    :return: context group
     """
     group.id = unpack_var(row['id'], 'int')
     group.uuid = row['uuid']
     group.created = unpack_var(row['created_ts'], 'int')
     group.updated = unpack_var(row['updated_ts'], 'int')
     group.name = row['name']
+    group.additional_ctx = unpack_item_value(row['additional_ctx_json'])
+    if group.additional_ctx is None:
+        group.additional_ctx = []
     return group

pygpt_net/provider/loaders/base.py

@@ -22,7 +22,9 @@ class BaseLoader:
         self.instructions = []  # list of instructions for 'web_index' command for how to handle this type
         self.args = {}  # custom keyword arguments
         self.init_args = {}  # initial keyword arguments
+        self.init_args_labels = {}
         self.init_args_types = {}
+        self.init_args_desc = {}
         self.allow_compiled = True  # allow in compiled and Snap versions
         # This is required due to some readers may require Python environment to install additional packages
 
@@ -42,6 +44,18 @@ class BaseLoader:
         """
         self.args = args
 
+    def explode(self, value: str) -> list:
+        """
+        Explode list from string
+
+        :param value: value string
+        :return: list
+        """
+        if value:
+            items = value.split(",")
+            return [item.strip() for item in items]
+        return []
+
     def get_args(self):
         """
         Prepare keyword arguments for reader init method

pygpt_net/provider/loaders/hub/yt/base.py

@@ -15,6 +15,7 @@ class YoutubeTranscriptReader(BasePydanticReader):
     """Youtube Transcript reader."""
 
     is_remote: bool = True
+    languages: List[str] = ["en"]
 
     @classmethod
     def class_name(cls) -> str:
@@ -34,6 +35,10 @@ class YoutubeTranscriptReader(BasePydanticReader):
                 for which transcripts are to be read.
 
         """
+        languages = self.languages
+        if not languages:
+            languages = ["en"]
+
         results = []
         for link in ytlinks:
             video_id = self._extract_video_id(link)

pygpt_net/provider/loaders/web_database.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,14 +30,16 @@ class Loader(BaseLoader):
                     "args": {
                         "query": {
                             "type": "str",
+                            "label": "SQL query",
+                            "description": "SQL query to read data from database, e.g. SELECT * FROM table",
                         },
                     },
                 }
             }
         ]
         self.init_args = {
-            "sql_database": None,
-            "engine": None,
+            # "sql_database": None,
+            # "engine": None,
             "uri": None,
             "scheme": None,
             "host": None,
@@ -47,8 +49,8 @@ class Loader(BaseLoader):
             "dbname": None,
         }
         self.init_args_types = {
-            "sql_database": "str",
-            "engine": "str",
+            # "sql_database": "str",
+            # "engine": "str",
             "uri": "str",
             "scheme": "str",
             "host": "str",
@@ -57,6 +59,12 @@ class Loader(BaseLoader):
             "password": "str",
             "dbname": "str",
         }
+        self.init_args_desc = {
+            # "sql_database": "str",
+            # "engine": "str",
+            "uri": "You can provide a single URI in the form of: {scheme}://{user}:{password}@{host}:{port}/{dbname}, "
+                   "or you can provide each field manually:",
+        }
 
     def get(self) -> BaseReader:
         """

pygpt_net/provider/loaders/web_github_issues.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -103,8 +103,12 @@ class Loader(BaseLoader):
         if "label_filters_include" in kwargs and kwargs.get("label_filters_include"):
             if isinstance(kwargs.get("label_filters_include"), list):
                 args["label_filters_include"] = kwargs.get("label_filters_include")
+            elif isinstance(kwargs.get("label_filters_include"), str):
+                args["label_filters_include"] = self.explode(kwargs.get("label_filters_include"))
         if "label_filters_exclude" in kwargs and kwargs.get("label_filters_exclude"):
             if isinstance(kwargs.get("label_filters_exclude"), list):
                 args["label_filters_exclude"] = kwargs.get("label_filters_exclude")
+            elif isinstance(kwargs.get("label_filters_exclude"), str):
+                args["label_filters_exclude"] = self.explode(kwargs.get("label_filters_exclude"))
 
         return args

pygpt_net/provider/loaders/web_google_calendar.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,9 +30,13 @@ class Loader(BaseLoader):
                     "args": {
                         "number_of_results": {
                             "type": "int",
+                            "label": "Number of results",
+                            "description": "Number of results to fetch, default: 100",
                         },
                         "start_date": {
                             "type": "str",
+                            "label": "Start date",
+                            "description": "Start date for events, ISO format: YYYY-MM-DD, default: today",
                         },
                     },
                 }
@@ -79,8 +83,12 @@ class Loader(BaseLoader):
         if "number_of_results" in kwargs and kwargs.get("number_of_results"):
             if isinstance(kwargs.get("number_of_results"), int):
                 args["number_of_results"] = kwargs.get("number_of_results")  # number of results
+                if args["number_of_results"] == 0:
+                    args["number_of_results"] = None
 
         if "start_date" in kwargs and kwargs.get("start_date"):
             if isinstance(kwargs.get("start_date"), str):
                 args["start_date"] = kwargs.get("start_date")  # start date
+                if args["start_date"] == "today" or args["start_date"] == "":
+                    args["start_date"] = None
         return args

pygpt_net/provider/loaders/web_google_docs.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,9 @@ class Loader(BaseLoader):
                     "args": {
                         "document_ids": {
                             "type": "list",
+                            "label": "Document IDs",
+                            "description": "List of document IDs to index, separated by comma (,)",
+                            "required": True,
                         },
                     },
                 }
@@ -76,4 +79,6 @@ class Loader(BaseLoader):
         if "document_ids" in kwargs and kwargs.get("document_ids"):
             if isinstance(kwargs.get("document_ids"), list):
                 args["document_ids"] = kwargs.get("document_ids")  # list of document ids
+            elif isinstance(kwargs.get("document_ids"), str):
+                args["document_ids"] = self.explode(kwargs.get("document_ids"))
         return args

pygpt_net/provider/loaders/web_google_drive.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.08.19 20:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,12 +30,17 @@ class Loader(BaseLoader):
                     "args": {
                         "folder_id": {
                             "type": "str",
+                            "label": "Folder ID",
                         },
                         "file_ids": {
                             "type": "list",
+                            "label": "File IDs",
+                            "description": "List of file ids, separated by comma (,)",
                         },
                         "mime_types": {
                             "type": "list",
+                            "label": "Mime Types",
+                            "description": "List of mime types, separated by comma (,)",
                         },
                     },
                 }
@@ -98,8 +103,12 @@ class Loader(BaseLoader):
         if "file_ids" in kwargs and kwargs.get("file_ids"):
             if isinstance(kwargs.get("file_ids"), list):
                 args["file_ids"] = kwargs.get("file_ids")  # list of file ids
+            elif isinstance(kwargs.get("file_ids"), str):
+                args["file_ids"] = self.explode(kwargs.get("file_ids"))
 
         if "mime_types" in kwargs and kwargs.get("mime_types"):
             if isinstance(kwargs.get("mime_types"), list):
                 args["mime_types"] = kwargs.get("mime_types")  # list of mime types
+            elif isinstance(kwargs.get("mime_types"), str):
+                args["mime_types"] = self.explode(kwargs.get("mime_types"))
         return args

pygpt_net/provider/loaders/web_google_gmail.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,7 @@ class Loader(BaseLoader):
                     "args": {
                         "query": {
                             "type": "str",
+                            "label": "Query",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_google_keep.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,8 @@ class Loader(BaseLoader):
                     "args": {
                         "document_ids": {
                             "type": "list",
+                            "label": "Document IDs",
+                            "description": "List of document ids, separated by comma (,)",
                         },
                     },
                 }
@@ -74,4 +76,6 @@ class Loader(BaseLoader):
         if "document_ids" in kwargs and kwargs.get("document_ids"):
             if isinstance(kwargs.get("document_ids"), list):
                 args["document_ids"] = kwargs.get("document_ids")  # list of document ids
+            elif isinstance(kwargs.get("document_ids"), str):
+                args["document_ids"] = self.explode(kwargs.get("document_ids"))
         return args

pygpt_net/provider/loaders/web_google_sheets.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,6 +30,8 @@ class Loader(BaseLoader):
                     "args": {
                         "spreadsheet_ids": {
                             "type": "list",
+                            "label": "Spreadsheet IDs",
+                            "description": "List of spreadsheet ids, separated by comma (,)",
                         },
                     },
                 }
@@ -78,4 +80,6 @@ class Loader(BaseLoader):
         if "spreadsheet_ids" in kwargs and kwargs.get("spreadsheet_ids"):
             if isinstance(kwargs.get("spreadsheet_ids"), list):
                 args["spreadsheet_ids"] = kwargs.get("spreadsheet_ids")  # spreadsheet ids
+            elif isinstance(kwargs.get("spreadsheet_ids"), str):
+                args["spreadsheet_ids"] = self.explode(kwargs.get("spreadsheet_ids"))
         return args

pygpt_net/provider/loaders/web_microsoft_onedrive.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,18 +30,26 @@ class Loader(BaseLoader):
                     "args": {
                         "folder_id": {
                             "type": "str",
+                            "label": "Folder ID",
                         },
                         "file_ids": {
                             "type": "list",
+                            "label": "File IDs",
+                            "description": "List of file ids, separated by comma (,)",
                         },
                         "mime_types": {
                             "type": "list",
+                            "label": "Mime Types",
+                            "description": "List of mime types, separated by comma (,)",
                         },
                         "folder_path": {
                             "type": "str",
+                            "label": "Folder Path",
                         },
                         "file_paths": {
                             "type": "list",
+                            "label": "File Paths",
+                            "description": "List of file paths, separated by comma (,)",
                         },
                     },
                 }
@@ -98,10 +106,14 @@ class Loader(BaseLoader):
         if "file_ids" in kwargs and kwargs.get("file_ids"):
             if isinstance(kwargs.get("file_ids"), list):
                 args["file_ids"] = kwargs.get("file_ids")  # list of file ids
+            elif isinstance(kwargs.get("file_ids"), str):
+                args["file_ids"] = self.explode(kwargs.get("file_ids"))
 
         if "mime_types" in kwargs and kwargs.get("mime_types"):
             if isinstance(kwargs.get("mime_types"), list):
                 args["mime_types"] = kwargs.get("mime_types")  # list of mime types
+            elif isinstance(kwargs.get("mime_types"), str):
+                args["mime_types"] = self.explode(kwargs.get("mime_types"))
 
         if "folder_path" in kwargs and kwargs.get("folder_path"):
             if isinstance(kwargs.get("folder_path"), str):
@@ -110,4 +122,6 @@ class Loader(BaseLoader):
         if "file_paths" in kwargs and kwargs.get("file_paths"):
             if isinstance(kwargs.get("file_paths"), list):
                 args["file_paths"] = kwargs.get("file_paths")  # list of file paths
+            elif isinstance(kwargs.get("file_paths"), str):
+                args["file_paths"] = self.explode(kwargs.get("file_paths"))
         return args

pygpt_net/provider/loaders/web_page.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.12 06:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -19,7 +19,7 @@ class Loader(BaseLoader):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.id = "webpage"
-        self.name = "Webpages"
+        self.name = "Webpage"
         self.type = ["web"]
         self.instructions = [
             {
@@ -28,6 +28,8 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "URL",
+                            "description": "URL of the webpage to index, e.g. https://www.example.com",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_rss.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.12 06:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -28,6 +28,7 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "URL",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_sitemap.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -28,6 +28,7 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "URL",
                         },
                     },
                 }

pygpt_net/provider/loaders/web_twitter.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.04.17 01:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 import json
@@ -30,9 +30,11 @@ class Loader(BaseLoader):
                     "args": {
                         "users": {
                             "type": "list",
+                            "label": "Twitter/X usernames",
                         },
                         "max_tweets": {
                             "type": "int",
+                            "label": "Max tweets",
                         },
                     },
                 }
@@ -80,7 +82,7 @@ class Loader(BaseLoader):
             if isinstance(kwargs.get("users"), list):
                 args["twitterhandles"] = kwargs.get("users")  # usernames
             elif isinstance(kwargs.get("users"), str):
-                args["twitterhandles"] = [kwargs.get("users")]
+                args["twitterhandles"] = self.explode(kwargs.get("users"))
 
         if "max_tweets" in kwargs and kwargs.get("max_tweets"):
             if isinstance(kwargs.get("max_tweets"), int):

pygpt_net/provider/loaders/web_yt.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.26 04:00:00                  #
+# Updated Date: 2024.12.16 01:00:00                  #
 # ================================================== #
 
 from llama_index.core.readers.base import BaseReader
@@ -28,11 +28,25 @@ class Loader(BaseLoader):
                     "args": {
                         "url": {
                             "type": "str",
+                            "label": "Youtube URL",
+                            "description": "URL of the YouTube video, e.g. https://www.youtube.com/watch?v=CRRlbK5w8AE",
                         },
                     },
                 }
             }
         ]
+        self.init_args = {
+            "languages": ["en"],
+        }
+        self.init_args_types = {
+            "languages": "list",
+        }
+        self.init_args_labels = {
+            "languages": "Languages",
+        }
+        self.init_args_desc = {
+            "languages": "List of languages to extract from the video, separated by comma (,), e.g. 'en,de,fr'. Default is 'en'",
+        }
 
     def get(self) -> BaseReader:
         """
@@ -40,7 +54,8 @@ class Loader(BaseLoader):
 
         :return: Data reader instance
         """
-        return YoutubeTranscriptReader()
+        args = self.get_args()
+        return YoutubeTranscriptReader(**args)
 
     def prepare_args(self, **kwargs) -> dict:
         """

pygpt_net/provider/vector_stores/ctx_attachment.py

@@ -50,7 +50,7 @@ class CtxAttachmentProvider(BaseStore):
 
         :return: True if exists
         """
-        path = self.get_path()
+        path = self.get_path("")
         if os.path.exists(path):
             store = os.path.join(path, "docstore.json")
             if os.path.exists(store):