pygpt-net 2.4.41__py3-none-any.whl → 2.4.43__py3-none-any.whl

pygpt_net/provider/core/ctx/db_sqlite/storage.py

@@ -6,12 +6,13 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.12.09 00:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 from datetime import datetime
 import re
 import time
+from typing import Dict, Optional, Tuple, List
 
 from sqlalchemy import text
 
@@ -46,11 +47,11 @@ class Storage:
 
     def prepare_query(
             self,
-            search_string: str = None,
-            filters: dict = None,
+            search_string: Optional[str] = None,
+            filters: Optional[dict] = None,
             search_content: bool = False,
             append_date_ranges: bool = True,
-    ):
+    ) -> Tuple[str, str, dict]:
         """
         Prepare query for search_string and filters
 
@@ -67,6 +68,9 @@ class Storage:
         # only base by default
         where_clauses.append("(m.root_id IS NULL OR m.root_id = 0)")
 
+        # join group
+        join_clauses.append("LEFT JOIN ctx_group g ON m.group_id = g.id")
+
         # search_string
         if search_string:
             date_ranges = search_by_date_string(search_string)
@@ -105,15 +109,16 @@ class Storage:
                     continue
                 mode = filter.get('mode', '=')
                 value = filter.get('value', '')
+                key_name = 'm.' + key
                 if isinstance(value, int):
-                    where_clauses.append(f"{key} {mode} :{key}")
+                    where_clauses.append(f"{key_name} {mode} :{key}")
                     bind_params[key] = value
                 elif isinstance(value, str):
-                    where_clauses.append(f"{key} {mode} :{key}")
+                    where_clauses.append(f"{key_name} {mode} :{key}")
                     bind_params[key] = f"%{value}%"
                 elif isinstance(value, list):
                     values = "(" + ",".join([str(x) for x in value]) + ")"
-                    where_clauses.append(f"{key} {mode} {values}")
+                    where_clauses.append(f"{key_name} {mode} {values}")
 
         where_statement = " AND ".join(where_clauses) if where_clauses else "1"
         join_statement = " ".join(join_clauses) if join_clauses else ""
@@ -122,14 +127,14 @@ class Storage:
 
     def get_meta(
             self,
-            search_string: str = None,
-            order_by: str = None,
-            order_direction: str = None,
-            limit: int = None,
-            offset: int = None,
-            filters: dict = None,
+            search_string: Optional[str] = None,
+            order_by: Optional[str] = None,
+            order_direction: Optional[str] = None,
+            limit: Optional[int] = None,
+            offset: Optional[int] = None,
+            filters: Optional[dict] = None,
             search_content: bool = False,
-    ) -> dict:
+    ) -> Dict[int, CtxMeta]:
         """
         Return dict with CtxMeta objects, indexed by ID
 
@@ -153,8 +158,18 @@ class Storage:
             append_date_ranges=True,
         )
         stmt_text = f"""
-            SELECT m.* FROM ctx_meta m {join_statement} WHERE {where_statement}
-            ORDER BY m.updated_ts DESC {limit_suffix}
+            SELECT 
+                m.*,
+                g.name as group_name,
+                g.uuid as group_uuid,
+                g.additional_ctx_json as group_additional_ctx_json
+            FROM 
+                ctx_meta m 
+                {join_statement} 
+            WHERE 
+                {where_statement}
+            ORDER BY 
+                m.updated_ts DESC {limit_suffix}
         """
         stmt = text(stmt_text).bindparams(**bind_params)
 
@@ -169,14 +184,24 @@ class Storage:
 
         return items
 
-    def get_meta_indexed(self) -> dict:
+    def get_meta_indexed(self) -> Dict[int, CtxMeta]:
         """
         Return dict with indexed CtxMeta objects, indexed by ID
 
         :return: dict of CtxMeta
         """
         stmt_text = f"""
-            SELECT * FROM ctx_meta WHERE indexed_ts > 0
+            SELECT 
+                m.*, 
+                g.name as group_name,
+                g.uuid as group_uuid,
+                g.additional_ctx_json as group_additional_ctx_json 
+            FROM 
+                ctx_meta m 
+            LEFT JOIN 
+                ctx_group g ON m.group_id = g.id
+            WHERE 
+                indexed_ts > 0
         """
         stmt = text(stmt_text)
         items = {}
@@ -189,7 +214,7 @@ class Storage:
                 items[meta.id] = meta
         return items
 
-    def get_item_by_id(self, id: int) -> CtxItem:
+    def get_item_by_id(self, id: int) -> Optional[CtxItem]:
         """
         Return ctx item by ID
 
@@ -210,7 +235,11 @@ class Storage:
                 return item
         return None
 
-    def get_meta_by_root_id_and_preset_id(self, root_id: int, preset_id: str) -> dict:
+    def get_meta_by_root_id_and_preset_id(
+            self,
+            root_id: int,
+            preset_id: str
+    ) -> Dict[int, CtxMeta]:
         """
         Return dict with indexed CtxMeta objects, indexed by ID
 
@@ -233,7 +262,7 @@ class Storage:
                 items[meta.id] = meta
         return items
 
-    def get_meta_by_id(self, id: int) -> CtxMeta or None:
+    def get_meta_by_id(self, id: int) -> Optional[CtxMeta]:
         """
         Return ctx meta by ID
 
@@ -269,7 +298,7 @@ class Storage:
                 return int(row.id)
         return 0
 
-    def get_items(self, id: int) -> list:
+    def get_items(self, id: int) -> List[CtxItem]:
         """
         Return ctx items list by ctx meta ID
 
@@ -426,9 +455,32 @@ class Storage:
         )
         with db.begin() as conn:
             conn.execute(stmt)
-            return True
 
-    def update_meta_all(self, meta: CtxMeta, items: list) -> bool:
+        # update group
+        if meta.group:
+            stmt = text("""
+                UPDATE ctx_group
+                SET
+                    name = :name,
+                    additional_ctx_json = :additional_ctx_json,
+                    updated_ts = :updated_ts
+                WHERE id = :id
+            """).bindparams(
+                id=meta.group.id,
+                name=meta.group.name,
+                additional_ctx_json=pack_item_value(meta.group.additional_ctx),
+                updated_ts=int(time.time()),
+            )
+            with db.begin() as conn:
+                conn.execute(stmt)
+
+        return True
+
+    def update_meta_all(
+            self,
+            meta: CtxMeta,
+            items: List[CtxItem]
+    ) -> bool:
         """
         Update all, meta and items
 
@@ -891,12 +943,12 @@ class Storage:
     def get_ctx_count_by_day(
             self,
             year: int,
-            month: int = None,
-            day: int = None,
-            search_string: str = None,
-            filters: dict = None,
+            month: Optional[int] = None,
+            day: Optional[int] = None,
+            search_string: Optional[str] = None,
+            filters: Optional[dict] = None,
             search_content: bool = False,
-    ) -> dict:
+    ) -> Dict[str, int]:
         """
         Return ctx counters by day for given year and month
 
@@ -987,12 +1039,12 @@ class Storage:
     def get_ctx_labels_count_by_day(
             self,
             year: int,
-            month: int = None,
-            day: int = None,
-            search_string: str = None,
-            filters: dict = None,
+            month: Optional[int] = None,
+            day: Optional[int] = None,
+            search_string: Optional[str] = None,
+            filters: Optional[dict] = None,
             search_content: bool = False,
-    ) -> dict:
+    ) -> Dict[str, Dict[int, int]]:
         """
         Return ctx counters by day for given year and month
 
@@ -1116,7 +1168,7 @@ class Storage:
 
                 return result_dict
 
-    def get_groups(self) -> dict:
+    def get_groups(self) -> Dict[int, CtxGroup]:
         """
         Return dict with CtxGroup objects, indexed by ID
 
@@ -1189,16 +1241,17 @@ class Storage:
             conn.execute(text("DELETE FROM sqlite_sequence WHERE name='ctx_group'"))
         return True
 
-    def clear_meta(self, meta_id: int):
+    def clear_meta(self, meta_id: int) -> bool:
         """
         Delete all items with meta ID
 
         :param meta_id: meta ID
-        :return:
+        :return: True if cleared
         """
         db = self.window.core.db.get_db()
         with db.begin() as conn:
             conn.execute(text(f"DELETE FROM ctx_item WHERE meta_id = {meta_id}"))
+        return True
 
     def update_group(self, group: CtxGroup) -> bool:
         """
@@ -1258,12 +1311,13 @@ class Storage:
             group.id = result.lastrowid
             return group.id
 
-    def update_meta_group_id(self, meta_id: int, group_id: int = None):
+    def update_meta_group_id(self, meta_id: int, group_id: int = None) -> bool:
         """
         Update meta group ID
 
         :param meta_id: ctx meta ID
         :param group_id: ctx group ID
+        :return: True if updated
         """
         db = self.window.core.db.get_db()
         stmt = text("""

pygpt_net/provider/core/ctx/db_sqlite/utils.py

@@ -6,18 +6,20 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.11.26 19:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
 import json
 import re
+
 from datetime import datetime, timedelta
+from typing import List, Tuple, Any, Dict
 
 from pygpt_net.item.ctx import CtxMeta, CtxItem, CtxGroup
 from pygpt_net.utils import unpack_var
 
 
-def search_by_date_string(search_string: str) -> list:
+def search_by_date_string(search_string: str) -> List[Tuple[Any, Any]]:
     """
     Prepare date ranges from search string if @date() syntax is used
 
@@ -64,7 +66,7 @@ def search_by_date_string(search_string: str) -> list:
     return date_ranges
 
 
-def get_month_start_end_timestamps(year: int, month: int) -> (int, int):
+def get_month_start_end_timestamps(year: int, month: int) -> Tuple[int, int]:
     """
     Get start and end timestamps for given month
 
@@ -82,7 +84,7 @@ def get_month_start_end_timestamps(year: int, month: int) -> (int, int):
     return start_timestamp, end_timestamp
 
 
-def get_year_start_end_timestamps(year: int) -> (int, int):
+def get_year_start_end_timestamps(year: int) -> Tuple[int, int]:
     """
     Get start and end timestamps for given year
 
@@ -96,7 +98,7 @@ def get_year_start_end_timestamps(year: int) -> (int, int):
     return start_timestamp, end_timestamp
 
 
-def pack_item_value(value: any) -> str:
+def pack_item_value(value: Any) -> str:
     """
     Pack item value to JSON
 
@@ -108,7 +110,7 @@ def pack_item_value(value: any) -> str:
     return value
 
 
-def unpack_item_value(value: any) -> any:
+def unpack_item_value(value: Any) -> Any:
     """
     Unpack item value from JSON
 
@@ -123,7 +125,10 @@ def unpack_item_value(value: any) -> any:
         return value
 
 
-def unpack_item(item: CtxItem, row: dict) -> CtxItem:
+def unpack_item(
+        item: CtxItem,
+        row: Dict[str, Any]
+) -> CtxItem:
     """
     Unpack context item from DB row
 
@@ -183,9 +188,12 @@ def unpack_item(item: CtxItem, row: dict) -> CtxItem:
     return item
 
 
-def unpack_meta(meta: CtxMeta, row: dict) -> CtxMeta:
+def unpack_meta(
+        meta: CtxMeta,
+        row: Dict[str, Any]
+) -> CtxMeta:
     """
-    Unpack context meta data from DB row
+    Unpack context meta-data from DB row
 
     :param meta: Context meta (CtxMeta)
     :param row: DB row
@@ -219,20 +227,38 @@ def unpack_meta(meta: CtxMeta, row: dict) -> CtxMeta:
 
     if meta.additional_ctx is None:
         meta.additional_ctx = []
+
+    # add group if exists
+    if meta.group_id:
+        group = CtxGroup()
+        group.id = meta.group_id
+        group.uuid = row['group_uuid']
+        group.name = row['group_name']
+        group.additional_ctx = unpack_item_value(row['group_additional_ctx_json'])
+        if group.additional_ctx is None:
+            group.additional_ctx = []
+        meta.group = group
+
     return meta
 
 
-def unpack_group(group: CtxGroup, row: dict) -> CtxGroup:
+def unpack_group(
+        group: CtxGroup,
+        row: Dict[str, Any]
+) -> CtxGroup:
     """
     Unpack context group data from DB row
 
     :param group: Context group (CtxGroup)
     :param row: DB row
-    :return: context meta
+    :return: context group
     """
     group.id = unpack_var(row['id'], 'int')
     group.uuid = row['uuid']
     group.created = unpack_var(row['created_ts'], 'int')
     group.updated = unpack_var(row['updated_ts'], 'int')
     group.name = row['name']
+    group.additional_ctx = unpack_item_value(row['additional_ctx_json'])
+    if group.additional_ctx is None:
+        group.additional_ctx = []
     return group

pygpt_net/provider/core/index/base.py

@@ -6,9 +6,11 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.06 02:00:00                  #
+# Updated Date: 2024.12.14 22:00:00                  #
 # ================================================== #
 
+from typing import Dict, Any, Optional
+
 from packaging.version import Version
 
 from pygpt_net.item.index import IndexItem
@@ -29,70 +31,174 @@ class BaseProvider:
     def patch(self, version: Version) -> bool:
         pass
 
-    def load(self, store_id: str) -> dict:
+    def load(
+            self,
+            store_id: str
+    ) -> Dict[str, IndexItem]:
         pass
 
-    def append_file(self, store_id: str, idx: str, data: dict) -> int:
+    def append_file(
+            self,
+            store_id: str,
+            idx: str,
+            data: Dict[str, Any]
+    ) -> int:
         pass
 
-    def append_ctx_meta(self, store_id: str, idx: str, meta_id: int, doc_id: str) -> int:
+    def append_ctx_meta(
+            self,
+            store_id: str,
+            idx: str,
+            meta_id: int,
+            doc_id: str
+    ) -> int:
         pass
 
-    def append_external(self, store_id: str, idx: str, data: dict) -> int:
+    def append_external(
+            self,
+            store_id: str,
+            idx: str,
+            data: Dict[str, Any]
+    ) -> int:
         pass
 
-    def update_file(self, id: int, doc_id: str, ts: int) -> bool:
+    def update_file(
+            self,
+            id: int,
+            doc_id: str,
+            ts: int
+    ) -> bool:
         pass
 
-    def update_ctx_meta(self, id: int, doc_id: str) -> bool:
+    def update_ctx_meta(
+            self,
+            id: int,
+            doc_id: str
+    ) -> bool:
         pass
 
-    def update_external(self, content: str, type: str, doc_id: str, ts: int) -> bool:
+    def update_external(
+            self,
+            content: str,
+            type: str,
+            doc_id: str,
+            ts: int
+    ) -> bool:
         pass
 
-    def is_meta_indexed(self, store_id: str, idx: str, meta_id: int) -> bool:
+    def is_meta_indexed(
+            self,
+            store_id: str,
+            idx: str,
+            meta_id: int
+    ) -> bool:
         pass
 
-    def is_file_indexed(self, store_id: str, idx: str, file_id: str) -> bool:
+    def is_file_indexed(
+            self,
+            store_id: str,
+            idx: str,
+            file_id: str
+    ) -> bool:
         pass
 
-    def is_external_indexed(self, store_id: str, idx: str, content: str, type: str) -> bool:
+    def is_external_indexed(
+            self,
+            store_id: str,
+            idx: str,
+            content: str,
+            type: str
+    ) -> bool:
         pass
 
-    def get_meta_doc_id(self, store_id: str, idx: str, meta_id: int) -> str:
+    def get_meta_doc_id(
+            self,
+            store_id: str,
+            idx: str,
+            meta_id: int
+    ) -> str:
         pass
 
-    def get_file_doc_id(self, store_id: str, idx: str, file_id: str) -> str:
+    def get_file_doc_id(
+            self,
+            store_id: str,
+            idx: str,
+            file_id: str
+    ) -> str:
         pass
 
-    def get_external_doc_id(self, store_id: str, idx: str, content: str, type: str) -> str:
+    def get_external_doc_id(
+            self,
+            store_id: str,
+            idx: str,
+            content: str,
+            type: str
+    ) -> str:
         pass
 
-    def remove_file(self, store_id: str, idx: str, doc_id: str):
+    def remove_file(
+            self,
+            store_id: str,
+            idx: str,
+            doc_id: str
+    ):
         pass
 
-    def remove_external(self, store_id: str, idx: str, doc_id: str):
+    def remove_external(
+            self,
+            store_id: str,
+            idx: str,
+            doc_id: str
+    ):
         pass
 
-    def remove_ctx_meta(self, store_id: str, idx: str, meta_id: str):
+    def remove_ctx_meta(
+            self,
+            store_id: str,
+            idx: str,
+            meta_id: str
+    ):
         pass
 
-    def truncate(self, store_id: str, idx: str):
+    def truncate(
+            self,
+            store_id: str,
+            idx: str
+    ):
         pass
 
-    def truncate_all(self, store_id: str = None, idx: str = None) -> bool:
+    def truncate_all(
+            self,
+            store_id: Optional[str] = None,
+            idx: Optional[str] = None
+    ) -> bool:
         pass
 
-    def truncate_files(self, store_id: str = None, idx: str = None) -> bool:
+    def truncate_files(
+            self,
+            store_id: Optional[str] = None,
+            idx: Optional[str] = None
+    ) -> bool:
         pass
 
-    def truncate_ctx(self, store_id: str = None, idx: str = None) -> bool:
+    def truncate_ctx(
+            self,
+            store_id: Optional[str] = None,
+            idx: Optional[str] = None
+    ) -> bool:
         pass
 
-    def truncate_external(self, store_id: str = None, idx: str = None) -> bool:
+    def truncate_external(
+            self,
+            store_id: Optional[str] = None,
+            idx: Optional[str] = None
+    ) -> bool:
         pass
 
-    def get_counters(self, type: str) -> dict:
+    def get_counters(
+            self,
+            type: str
+    ) -> Dict[str, Dict[str, int]]:
         pass
 
     def dump(self, index: IndexItem) -> str: