pygpt-net 2.1.37__py3-none-any.whl → 2.4.48__py3-none-any.whl

pygpt_net/core/ctx/__init__.py

@@ -6,20 +6,37 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.15 10:00:00                  #
+# Updated Date: 2024.12.14 08:00:00                  #
 # ================================================== #
 
 import copy
 import datetime
+import uuid
+from typing import Optional, Tuple, List, Dict
 
 from packaging.version import Version
 
-from pygpt_net.item.ctx import CtxItem, CtxMeta
+from pygpt_net.core.types import (
+    MODE_AGENT,
+    MODE_AGENT_LLAMA,
+    MODE_ASSISTANT,
+    MODE_AUDIO,
+    MODE_CHAT,
+    MODE_COMPLETION,
+    MODE_EXPERT,
+    MODE_IMAGE,
+    MODE_LANGCHAIN,
+    MODE_LLAMA_INDEX,
+    MODE_VISION,
+)
+from pygpt_net.item.ctx import CtxItem, CtxMeta, CtxGroup
 from pygpt_net.provider.core.ctx.base import BaseProvider
 from pygpt_net.provider.core.ctx.db_sqlite import DbSqliteProvider
 from pygpt_net.utils import trans
 
 from .idx import Idx
+from .container import Container
+from .output import Output
 
 
 class Ctx:
@@ -31,9 +48,10 @@ class Ctx:
         """
         self.window = window
         self.provider = DbSqliteProvider(window)
+        self.container = Container(window)  # context container
+        self.output = Output(window)  # context render output
         self.idx = Idx(window)  # context indexing core
         self.meta = {}
-        self.items = []
         self.current = None
         self.last_item = None
         self.assistant = None
@@ -47,21 +65,284 @@ class Ctx:
         self.last_model = None
         self.tmp_meta = None
         self.search_string = None  # search string
+        self.groups = {}  # groups
         self.filters = {}  # search filters
         self.filters_labels = []  # search labels
         self.current_cmd = []  # current commands
         self.current_cmd_schema = "" # current commands schema
+        self.all_modes = [
+            MODE_CHAT,
+            MODE_COMPLETION,
+            MODE_IMAGE,
+            MODE_LANGCHAIN,
+            MODE_VISION,
+            MODE_ASSISTANT,
+            MODE_LLAMA_INDEX,
+            MODE_AGENT,
+            MODE_EXPERT,
+            MODE_AUDIO,
+        ]
         self.allowed_modes = {
-            'chat': ["chat", "completion", "img", "langchain", "vision", "assistant", "llama_index", "agent"],
-            'completion': ["chat", "completion", "img", "langchain", "vision", "assistant", "llama_index", "agent"],
-            'img': ["img"],
-            'langchain': ["chat", "completion", "img", "langchain", "vision", "assistant", "llama_index", "agent"],
-            'vision': ["chat", "completion", "img", "langchain", "vision", "assistant", "llama_index", "agent"],
-            'assistant': ["assistant"],
-            'llama_index': ["chat", "completion", "img", "langchain", "vision", "assistant", "llama_index", "agent"],
-            'agent': ["chat", "completion", "img", "langchain", "vision", "assistant", "llama_index", "agent"],
+            MODE_CHAT: self.all_modes,
+            MODE_COMPLETION: self.all_modes,
+            MODE_IMAGE: [MODE_IMAGE],
+            MODE_LANGCHAIN: self.all_modes,
+            MODE_VISION: self.all_modes,
+            MODE_ASSISTANT: [MODE_ASSISTANT],
+            MODE_LLAMA_INDEX: self.all_modes,
+            MODE_AGENT: self.all_modes,
+            MODE_EXPERT: self.all_modes,
+            MODE_AUDIO: self.all_modes,
+            MODE_AGENT_LLAMA: [MODE_AGENT_LLAMA],
         }
         self.current_sys_prompt = ""
+        self.groups_loaded = False
+
+    def get_items(self) -> List[CtxItem]:
+        """
+        Get context items
+
+        :return: context items
+        """
+        return self.container.get_items()
+
+    def set_items(self, items: List[CtxItem]):
+        """
+        Set context items
+
+        :param items: context items
+        """
+        self.container.set_items(items)
+
+    def clear_items(self):
+        """Clear context items"""
+        self.container.clear_items()
+
+    def count_items(self) -> int:
+        """
+        Count context items
+        :return: context items count
+        """
+        return self.container.count_items()
+
+    def get_current(self) -> int:
+        """
+        Get current context ID
+
+        :return: current context ID
+        """
+        return self.current
+
+    def set_current(self, current: int):
+        """
+        Set current context ID
+
+        :param current: current context ID
+        """
+        self.current = current
+
+    def clear_current(self):
+        """Clear current context ID"""
+        self.current = None
+
+    def get_last_item(self) -> Optional[CtxItem]:
+        """
+        Get last item
+
+        :return: last item
+        """
+        return self.last_item
+
+    def set_last_item(self, last_item: Optional[CtxItem]):
+        """
+        Set last item
+
+        :param last_item: last item
+        """
+        self.last_item = last_item
+
+    def get_assistant(self) -> str:
+        """
+        Get assistant name
+
+        :return: assistant name
+        """
+        return self.assistant
+
+    def set_assistant(self, assistant: str):
+        """
+        Set assistant name
+
+        :param assistant: assistant name
+        :return: assistant name
+        """
+        self.assistant = assistant
+
+    def get_mode(self) -> str:
+        """
+        Get mode
+
+        :return: mode
+        """
+        return self.mode
+
+    def set_mode(self, mode: str):
+        """
+        Set mode
+
+        :param mode: mode
+        """
+        self.mode = mode
+
+    def get_model(self) -> str:
+        """
+        Get model name
+        :return: model name
+        """
+        return self.model
+
+    def set_model(self, model: str):
+        """
+        Set model name
+
+        :param model: model name
+        """
+        self.model = model
+
+    def get_preset(self) -> str:
+        """
+        Get preset name
+        :return: preset
+        """
+        return self.preset
+
+    def set_preset(self, preset: str):
+        """
+        Set preset name
+
+        :param preset: preset name
+        """
+        self.preset = preset
+
+    def get_run(self) -> str:
+        """
+        Get run ID
+
+        :return: run ID
+        """
+        return self.run
+
+    def set_status(self, status: int):
+        """
+        Set status (label color)
+
+        :param status: status
+        """
+        self.status = status
+
+    def get_status(self) -> int:
+        """
+        Get status (label color)
+
+        :return: status
+        """
+        return self.status
+
+    def set_run(self, run: str):
+        """
+        Set run ID
+
+        :param run: run ID
+        """
+        self.preset = run
+
+    def get_thread(self) -> str:
+        """
+        Get thread ID
+
+        :return: thread ID
+        """
+        return self.thread
+
+    def set_thread(self, thread: str):
+        """
+        Set thread ID
+
+        :param thread: thread ID
+        """
+        self.thread = thread
+
+    def clear_thread(self):
+        """Clear thread ID"""
+        self.thread = None
+
+    def get_last_mode(self) -> str:
+        """
+        Get last mode
+
+        :return: last mode
+        """
+        return self.last_mode
+
+    def set_last_mode(self, last_mode: str):
+        """
+        Set last mode
+
+        :param last_mode: last mode
+        """
+        self.last_mode = last_mode
+
+    def get_last_model(self) -> str:
+        """
+        Get last model name
+
+        :return: last model
+        """
+        return self.last_model
+
+    def set_last_model(self, last_model: str):
+        """
+        Set last model name
+
+        :param last_model: last model name
+        """
+        self.last_model = last_model
+
+    def get_tmp_meta(self) -> Optional[CtxMeta]:
+        """
+        Get temporary meta
+
+        :return: temporary meta
+        """
+        return self.tmp_meta
+
+    def set_tmp_meta(self, tmp_meta: Optional[CtxMeta]):
+        """
+        Set temporary meta
+
+        :param tmp_meta: temporary meta
+        """
+        self.tmp_meta = tmp_meta
+
+    def get_search_string(self) -> Optional[str]:
+        """
+        Get search string
+
+        :return: search string
+        """
+        return self.search_string
+
+    def set_search_string(self, search_string: Optional[str]):
+        """
+        Set search string
+
+        :param search_string: search string
+        """
+        self.search_string = search_string
+
+    def clear_search_string(self):
+        """Clear search string"""
+        self.search_string = None
 
     def install(self):
         """Install provider data"""
@@ -84,11 +365,16 @@ class Ctx:
         """
         return self.provider
 
-    def select(self, id: int):
+    def select(
+            self,
+            id: int,
+            restore_model: bool = True
+    ):
         """
         Select ctx meta by ID and load ctx items
 
         :param id: context meta id
+        :param restore_model: restore model
         """
         if id not in self.meta:
             self.load_tmp_meta(id)
@@ -109,26 +395,30 @@ class Ctx:
             self.preset = ctx.preset
 
             # restore model if exists in current mode
-            if ctx.last_model is not None \
-                    and self.window.core.models.has_model(self.mode, ctx.last_model):
-                self.model = ctx.last_model
-            elif ctx.model is not None \
-                    and self.window.core.models.has_model(self.mode, ctx.model):
-                self.model = ctx.model
+            if restore_model:
+                if ctx.last_model is not None \
+                        and self.window.core.models.has_model(self.mode, ctx.last_model):
+                    self.model = ctx.last_model
+                elif ctx.model is not None \
+                        and self.window.core.models.has_model(self.mode, ctx.model):
+                    self.model = ctx.model
 
-            self.items = self.load(id)
+            self.set_items(self.load(id))
 
-    def new(self) -> CtxMeta or None:
+    def new(
+            self,
+            group_id: Optional[int] = None
+    ) -> Optional[CtxMeta]:
         """
         Create new ctx and set as current
 
+        :param group_id: group id
         :return: CtxMeta instance (new ctx meta)
         """
-        meta = self.create()  # create new ctx meta
+        meta = self.create(group_id)  # create new ctx meta
         if meta is None:
             self.window.core.debug.log("Error creating new ctx")
             return
-
         self.meta[meta.id] = meta
         self.tmp_meta = meta
         self.current = meta.id
@@ -137,7 +427,7 @@ class Ctx:
         self.mode = self.window.core.config.get('mode')
         self.model = self.window.core.config.get('model')
         self.preset = self.window.core.config.get('preset')
-        self.items = []
+        self.clear_items()
         self.save(meta.id)
 
         return meta
@@ -158,24 +448,41 @@ class Ctx:
         meta.initialized = False
         return meta
 
-    def create(self) -> CtxMeta:
+    def create(
+            self,
+            group_id: Optional[int] = None
+    ) -> CtxMeta:
         """
         Send created meta to provider and return new ID
 
+        :param group_id: group id
         :return: CtxMeta instance
         """
         meta = self.build()
+        if group_id is not None:
+            meta.group_id = group_id
         id = self.provider.create(meta)
         meta.id = id
         return meta
 
-    def add(self, item: CtxItem):
+    def add(
+            self,
+            item: CtxItem,
+            parent_id: Optional[int] = None
+    ):
         """
         Add CtxItem to contexts and saves context
 
         :param item: CtxItem to append
+        :param parent_id: parent id
         """
-        self.items.append(item)  # add CtxItem to context items
+        # custom parent
+        if parent_id is not None:
+            self.add_to_meta(item, parent_id)
+            return
+
+        # to current
+        self.get_items().append(item)  # add CtxItem to context items
 
         # append in provider
         if self.current is not None:
@@ -187,6 +494,24 @@ class Ctx:
                 if not result:
                     self.store()  # if not stored, e.g. in JSON file provider, then store whole ctx (save all)
 
+    def add_to_meta(
+            self,
+            item: CtxItem,
+            meta_id: Optional[int] = None
+    ):
+        """
+        Add CtxItem to custom meta
+
+        :param item: CtxItem to append
+        :param meta_id: meta id
+        """
+        if meta_id in self.meta:
+            meta = self.meta[meta_id]
+        else:
+            meta = self.provider.get_meta_by_id(meta_id)
+        if meta is not None:
+            self.provider.append_item(meta, item)
+
     def update_item(self, item: CtxItem):
         """
         Update CtxItem in context
@@ -214,7 +539,7 @@ class Ctx:
         if self.current is None:
             return True
         else:
-            if len(self.items) == 0:
+            if self.count_items() == 0:
                 return True
         return False
 
@@ -236,6 +561,14 @@ class Ctx:
         self.meta[self.current].mode = self.mode
         self.save(self.current)
 
+    def replace(self, meta: CtxMeta):
+        """
+        Replace meta
+
+        :param meta: CtxMeta
+        """
+        self.meta[meta.id] = meta
+
     def post_update(self, mode: str):
         """
         Update current (last) ctx data
@@ -309,8 +642,8 @@ class Ctx:
         :param idx: item index
         :return: context item
         """
-        if idx < len(self.items):
-            return self.items[idx]
+        if idx < self.count_items():
+            return self.get_items()[idx]
 
     def get_item_by_id(self, id: int) -> CtxItem:
         """
@@ -319,9 +652,19 @@ class Ctx:
         :param id: item id
         :return: context item
         """
-        for item in self.items:
+        for item in self.get_items():
             if item.id == id:
                 return item
+        return self.fetch_item_by_id(id)   # if no item found, try to fetch from DB
+
+    def fetch_item_by_id(self, id: int) -> CtxItem:
+        """
+        Fetch ctx item by id
+
+        :param id: item id
+        :return: context item
+        """
+        return self.provider.get_item_by_id(id)
 
     def get_meta(self, reload: bool = False) -> dict:
         """
@@ -334,6 +677,15 @@ class Ctx:
             self.load_meta()
         return self.meta
 
+    def get_current_meta(self) -> Optional[CtxMeta]:
+        """
+        Get current meta
+
+        :return: current meta
+        """
+        if self.current is not None:
+            return self.get_meta_by_id(self.current)
+
     def get_id_by_idx(self, idx: int) -> int:
         """
         Get ctx id (id) by index
@@ -378,14 +730,14 @@ class Ctx:
         if id in self.meta:
             return self.meta[id]
 
-    def get_last(self) -> CtxItem or None:
+    def get_last(self) -> Optional[CtxItem]:
         """
         Return last item from ctx
 
         :return: last ctx item
         """
-        if len(self.items) > 0:
-            return self.items[-1]
+        if self.count_items() > 0:
+            return self.get_items()[-1]
         return None
 
     def is_first_item(self, item_id: int) -> bool:
@@ -395,8 +747,8 @@ class Ctx:
         :param item_id: item id
         :return: True if first
         """
-        if len(self.items) > 0:
-            return self.items[0].id == item_id
+        if self.count_items() > 0:
+            return self.get_items()[0].id == item_id
         return False
 
     def is_last_item(self, item_id: int) -> bool:
@@ -406,21 +758,21 @@ class Ctx:
         :param item_id: item id
         :return: True if last
         """
-        if len(self.items) > 0:
-            return self.items[-1].id == item_id
+        if self.count_items() > 0:
+            return self.get_items()[-1].id == item_id
         return False
 
-    def get_previous_item(self, item_id: int) -> CtxItem or None:
+    def get_previous_item(self, item_id: int) -> Optional[CtxItem]:
         """
         Get previous item from ctx
 
         :param item_id: item id
         :return: ctx item
         """
-        for i in range(len(self.items)):
-            if self.items[i].id == item_id:
+        for i in range(self.count_items()):
+            if self.get_items()[i].id == item_id:
                 if i > 0:
-                    return self.items[i - 1]
+                    return self.get_items()[i - 1]
         return None
 
     def prepare(self):
@@ -435,7 +787,7 @@ class Ctx:
 
         :return: ctx items count
         """
-        return len(self.items)
+        return self.count_items()
 
     def count_meta(self) -> int:
         """
@@ -448,13 +800,28 @@ class Ctx:
             extra = 1  # prevent create new if tmp meta exists
         return len(self.meta) + extra
 
-    def all(self) -> list:
+    def count_found_meta(self) -> int:
         """
-        Return ctx items
+        Count ctx meta items
 
+        :return: ctx meta count
+        """
+        return len(self.meta)
+
+    def all(
+            self,
+            meta_id: Optional[int] = None
+    ) -> List[CtxItem]:
+        """
+        Return ctx items (current or by meta_id if provided)
+
+        :param meta_id: meta id
         :return: ctx items
         """
-        return self.items
+        if meta_id is None:
+            return self.get_items()
+        else:
+            return self.load(meta_id)
 
     def remove(self, id: int):
         """
@@ -472,13 +839,17 @@ class Ctx:
 
         :param id: ctx id
         """
-        for item in self.items:
+        for item in self.get_items():
             if item.id == id:
-                self.items.remove(item)
+                self.get_items().remove(item)
                 self.provider.remove_item(id)
                 break
 
-    def remove_items_from(self, meta_id: int, item_id: int):
+    def remove_items_from(
+            self,
+            meta_id: int,
+            item_id: int
+    ):
         """
         Remove ctx items from meta_id
 
@@ -497,7 +868,7 @@ class Ctx:
 
     def clear(self):
         """Clear ctx items"""
-        self.items = []
+        self.clear_items()
 
     def append_thread(self, thread: str):
         """
@@ -542,16 +913,83 @@ class Ctx:
             self.meta[self.current].status = self.status
             self.save(self.current)
 
-    def count_prompt_items(
+    def get_or_create_slave_meta(
+            self,
+            master_ctx: CtxItem,
+            preset_id: str
+    ) -> CtxMeta:
+        """
+        Get or create slave meta
+
+        :param master_ctx: master context
+        :param preset_id: preset ID
+        :return: slave meta
+        """
+        slaves = []
+        if master_ctx.meta is not None:
+            slaves = self.provider.get_meta_by_root_id_and_preset_id(
+                master_ctx.meta.id,
+                preset_id,
+            )
+        if len(slaves) > 0:
+            return list(slaves.values())[0]
+        slave = self.build()
+
+        if master_ctx.meta is not None:
+            slave.root_id = master_ctx.meta.id
+            slave.parent_id = master_ctx.meta.id
+
+        slave.preset = preset_id
+        id = self.provider.create(slave)
+        slave.id = id
+        return slave
+
+    def get_prev(self) -> Optional[int]:
+        """
+        Get previous context
+
+        :return: previous context
+        """
+        if self.current is None:
+            return None
+        idx = self.get_idx_by_id(self.current)
+        if idx is not None:
+            if idx > 0:
+                return self.get_id_by_idx(idx - 1)
+
+    def get_next(self) -> Optional[int]:
+        """
+        Get next context
+
+        :return: next context
+        """
+        if self.current is None:
+            return None
+        idx = self.get_idx_by_id(self.current)
+        if idx is not None:
+            if idx < self.count_meta() - 1:
+                return self.get_id_by_idx(idx + 1)
+
+    def get_last_meta(self) -> Optional[int]:
+        """
+        Get last context
+
+        :return: last meta
+        """
+        return self.provider.get_last_meta_id()
+
+    def count_history(
             self,
+            history_items: List[CtxItem],
             model: str,
             mode: str,
             used_tokens: int = 100,
             max_tokens: int = 1000
-    ) -> (int, int):
+    ) -> Tuple[int, int]:
         """
         Count ctx items to add to prompt
 
+        :param history_items: history items list
         :param model: model
         :param mode: mode
         :param used_tokens: used tokens
@@ -562,7 +1000,7 @@ class Ctx:
         # loop on items from end to start
         tokens = used_tokens
         context_tokens = 0
-        for item in reversed(self.items):
+        for item in reversed(history_items):
             num = self.window.core.tokens.from_ctx(item, mode, model)  # get num tokens for input and output
             tokens += num
             if tokens > max_tokens:
@@ -572,10 +1010,11 @@ class Ctx:
 
         return i, context_tokens
 
-    def get_prompt_items(
+    def get_history(
             self,
+            history_items: List[CtxItem],
             model: str,
-            mode: str = "chat",
+            mode: str = MODE_CHAT,
             used_tokens: int = 100,
             max_tokens: int = 1000,
             ignore_first: bool = True
@@ -583,6 +1022,7 @@ class Ctx:
         """
         Return ctx items to add to prompt
 
+        :param history_items: history items list
         :param model: model
         :param mode: mode
         :param used_tokens: used tokens
@@ -594,7 +1034,7 @@ class Ctx:
         # loop on items from end to start
         tokens = used_tokens
         is_first = True
-        for item in reversed(self.items):
+        for item in reversed(history_items):
             if is_first and ignore_first:
                 is_first = False
                 continue
@@ -607,6 +1047,57 @@ class Ctx:
         items.reverse()
         return items
 
+    def count_prompt_items(
+            self,
+            model: str,
+            mode: str,
+            used_tokens: int = 100,
+            max_tokens: int = 1000
+    ) -> Tuple[int, int]:
+        """
+        Count ctx items to add to prompt
+
+        :param model: model
+        :param mode: mode
+        :param used_tokens: used tokens
+        :param max_tokens: max tokens
+        :return: ctx items count, ctx tokens count
+        """
+        return self.count_history(
+            self.get_items(),
+            model,
+            mode,
+            used_tokens,
+            max_tokens,
+        )
+
+    def get_prompt_items(
+            self,
+            model: str,
+            mode: str = MODE_CHAT,
+            used_tokens: int = 100,
+            max_tokens: int = 1000,
+            ignore_first: bool = True
+    ) -> list:
+        """
+        Return ctx items to add to prompt
+
+        :param model: model
+        :param mode: mode
+        :param used_tokens: used tokens
+        :param max_tokens: max tokens
+        :param ignore_first: ignore current item (provided by user)
+        :return: ctx items list
+        """
+        return self.get_history(
+            self.get_items(),
+            model,
+            mode,
+            used_tokens,
+            max_tokens,
+            ignore_first,
+        )
+
     def get_all_items(self, ignore_first: bool = True) -> list:
         """
         Return all ctx items
@@ -616,7 +1107,7 @@ class Ctx:
         """
         items = []
         is_first = True
-        for item in reversed(self.items):
+        for item in reversed(self.get_items()):
             if is_first and ignore_first:
                 is_first = False
                 continue
@@ -669,8 +1160,8 @@ class Ctx:
 
     def remove_last(self):
         """Remove last item"""
-        if len(self.items) > 0:
-            self.items.pop()
+        if self.count_items() > 0:
+            self.get_items().pop()
 
     def duplicate(self, id: int) -> int:
         """
@@ -680,7 +1171,7 @@ class Ctx:
         :return: new ctx id
         """
         if id in self.meta:
-            meta = self.create()
+            meta = self.create(self.meta[id].group_id)
             new_id = meta.id
             old_data = self.meta[id].to_dict()
             meta.from_dict(old_data)
@@ -691,8 +1182,8 @@ class Ctx:
 
     def remove_first(self):
         """Remove first item"""
-        if len(self.items) > 0:
-            self.items.pop(0)
+        if self.count_items() > 0:
+            self.get_items().pop(0)
 
     def set_display_filters(self, filters: dict):
         """
@@ -702,7 +1193,11 @@ class Ctx:
         """
         self.filters = filters
 
-    def is_allowed_for_mode(self, mode: str, check_assistant: bool = True) -> bool:
+    def is_allowed_for_mode(
+            self,
+            mode: str,
+            check_assistant: bool = True
+    ) -> bool:
         """
         Check if ctx is allowed for this mode
 
@@ -854,18 +1349,127 @@ class Ctx:
             }
         return filters
 
-    def load(self, id: int) -> list:
+    def load(self, id: int) -> List[CtxItem]:
         """
-        Load ctx data from provider
+        Load ctx items from provider and append meta to each item
 
         :param id: ctx id
         :return: ctx items list
         """
-        return self.provider.load(id)
+        items = self.provider.load(id)
+        meta = self.get_meta_by_id(id)
+        for item in items:
+            item.meta = meta  # append meta to each item
+        return items
+
+    def load_groups(self):
+        """Load groups"""
+        self.groups = self.provider.get_groups()
+
+    def has_group(self, id: int) -> bool:
+        """
+        Check if group exists
+
+        :param id: group id
+        :return: True if exists
+        """
+        if id in self.groups:
+            return True
+        return False
+
+    def get_groups(self) -> Dict[int, CtxGroup]:
+        """
+        Get groups
+
+        :return: groups
+        """
+        if not self.groups_loaded:
+            self.load_groups()
+            self.groups_loaded = True
+        return self.groups
+
+    def get_group_by_id(self, id: int) -> Optional[CtxGroup]:
+        """
+        Get group by ID
+
+        :param id: group id
+        :return: group instance
+        """
+        if not self.groups_loaded:
+            self.load_groups()
+            self.groups_loaded = True
+        if id not in self.groups:
+            return None
+        return self.groups[id]
+
+    def remove_group(
+            self,
+            group: CtxGroup,
+            all: bool = False
+    ):
+        """
+        Remove group
+
+        :param group: group instance
+        :param all: remove all items
+        """
+        self.provider.remove_group(group.id, all=all)
+        self.load_groups()
+
+    def truncate_groups(self):
+        """Remove all groups"""
+        self.provider.truncate_groups()
+        self.load_groups()
+
+    def make_group(self, name: str) -> CtxGroup:
+        """
+        Make group
+
+        :param name: group name
+        :return: group instance
+        """
+        group = CtxGroup()
+        group.uuid = str(uuid.uuid4())
+        group.name = name
+        return group
+
+    def insert_group(self, group: CtxGroup) -> int:
+        """
+        Insert group
 
-    def get_items_by_id(self, id: int) -> list:
+        :param group: group instance
+        :return: group id
         """
-        Get ctx items by id
+        id = self.provider.insert_group(group)
+        self.load_groups()
+        return id
+
+    def update_group(self, group: CtxGroup):
+        """
+        Update group
+
+        :param group: group instance
+        """
+        self.provider.update_group(group)
+        self.load_groups()
+
+    def update_meta_group_id(self, id: int, group_id: int):
+        """
+        Update meta group ID
+
+        :param id: meta id
+        :param group_id: group id
+        """
+        meta = self.get_meta_by_id(id)
+        if meta is None:
+            return
+        meta.group_id = group_id
+        self.provider.update_meta_group_id(id, group_id)
+        self.load_groups()
+
+    def get_items_by_id(self, id: int) -> List[str]:
+        """
+        Get ctx items by id as string list
 
         :param id: ctx id
         :return: ctx items list
@@ -877,7 +1481,11 @@ class Ctx:
             data.append("Human: " + str(item.input) + "\n" + "Assistant: " + str(item.output) + "\n")
         return data
 
-    def get_list_in_date_range(self, search_string, limit: int = 0) -> list:
+    def get_list_in_date_range(
+            self,
+            search_string: Optional[str] = None,
+            limit: int = 0
+    ) -> List[Dict[str, Dict]]:
         """
         Get ctx list in date range
 
@@ -921,7 +1529,7 @@ class Ctx:
         if not self.window.core.config.get('store_history'):
             return
 
-        self.provider.save(id, self.meta[id], self.items)
+        self.provider.save(id, self.meta[id], self.get_items())
 
     def store(self):
         """Store current ctx"""
@@ -931,6 +1539,17 @@ class Ctx:
             if self.current in self.meta:
                 self.save(self.current)
 
+    def reset_meta(self, id: int):
+        """
+        Reset meta
+
+        :param id: meta id
+        """
+        if id in self.meta:
+            self.provider.clear_meta(id)
+            self.meta[id].initialized = False
+        self.load_meta()
+
     def as_previous(self, ctx: CtxItem) -> CtxItem:
         """
         Prepare previous context item and clear current reply results
@@ -940,15 +1559,22 @@ class Ctx:
         """
         prev_ctx = CtxItem()
         prev_ctx.urls = copy.deepcopy(ctx.urls)
+        prev_ctx.urls_before = copy.deepcopy(ctx.urls_before)
         prev_ctx.images = copy.deepcopy(ctx.images)
+        prev_ctx.images_before = copy.deepcopy(ctx.images_before)
         prev_ctx.files = copy.deepcopy(ctx.files)
+        prev_ctx.files_before = copy.deepcopy(ctx.files_before)
         prev_ctx.attachments = copy.deepcopy(ctx.attachments)
+        prev_ctx.attachments_before = copy.deepcopy(ctx.attachments_before)
         prev_ctx.results = copy.deepcopy(ctx.results)
         prev_ctx.index_meta = copy.deepcopy(ctx.index_meta)
         prev_ctx.doc_ids = copy.deepcopy(ctx.doc_ids)
+        prev_ctx.input_name = copy.deepcopy(ctx.input_name)
+        prev_ctx.output_name = copy.deepcopy(ctx.output_name)
 
         ctx.clear_reply()  # clear current reply result
-        ctx.from_previous()  # get result from previous if exists
+        if len(ctx.cmds) == 0:
+            ctx.from_previous()  # get result from previous if exists
         return prev_ctx
 
     def dump(self, ctx: CtxItem) -> str:
@@ -958,3 +1584,28 @@ class Ctx:
         :param ctx: CtxItem instance
         """
         return self.provider.dump(ctx)
+
+    def reset(self):
+        """Reset all data"""
+        self.meta = {}
+        self.clear_items()
+        self.current = None
+        self.last_item = None
+        self.assistant = None
+        self.mode = None
+        self.model = None
+        self.preset = None
+        self.run = None
+        self.status = None
+        self.thread = None
+        self.last_mode = None
+        self.last_model = None
+        self.tmp_meta = None
+        self.search_string = None  # search string
+        self.groups = {}  # groups
+        self.filters = {}  # search filters
+        self.filters_labels = []  # search labels
+        self.current_cmd = []  # current commands
+        self.current_cmd_schema = ""  # current commands schema
+        self.current_sys_prompt = ""
+        self.groups_loaded = False