swarms 7.8.3__py3-none-any.whl → 7.8.7__py3-none-any.whl

swarms/structs/conversation.py

@@ -1,18 +1,18 @@
 import concurrent.futures
 import datetime
-import hashlib
 import json
 import os
 import threading
 import uuid
 from typing import (
     TYPE_CHECKING,
-    Any,
+    Callable,
     Dict,
     List,
     Optional,
     Union,
     Literal,
+    Any,
 )
 
 import yaml
@@ -33,8 +33,115 @@ def generate_conversation_id():
     return str(uuid.uuid4())
 
 
+def get_conversation_dir():
+    """Get the directory for storing conversation logs."""
+    # Get the current working directory
+    conversation_dir = os.path.join(os.getcwd(), "conversations")
+    try:
+        os.makedirs(conversation_dir, mode=0o755, exist_ok=True)
+    except Exception as e:
+        logger.error(
+            f"Failed to create conversations directory: {str(e)}"
+        )
+        # Fallback to the same directory as the script
+        conversation_dir = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "conversations",
+        )
+        os.makedirs(conversation_dir, mode=0o755, exist_ok=True)
+    return conversation_dir
+
+
 # Define available providers
-providers = Literal["mem0", "in-memory"]
+providers = Literal[
+    "mem0",
+    "in-memory",
+    "supabase",
+    "redis",
+    "sqlite",
+    "duckdb",
+    "pulsar",
+]
+
+
+def _create_backend_conversation(backend: str, **kwargs):
+    """
+    Create a backend conversation instance based on the specified backend type.
+
+    This function uses lazy loading to import backend dependencies only when needed.
+    Each backend class handles its own dependency management and error messages.
+
+    Args:
+        backend (str): The backend type to create
+        **kwargs: Arguments to pass to the backend constructor
+
+    Returns:
+        Backend conversation instance
+
+    Raises:
+        ImportError: If required packages for the backend are not installed (raised by lazy loading)
+        ValueError: If backend is not supported
+    """
+    try:
+        if backend == "supabase":
+            from swarms.communication.supabase_wrap import (
+                SupabaseConversation,
+            )
+
+            return SupabaseConversation(**kwargs)
+        elif backend == "redis":
+            from swarms.communication.redis_wrap import (
+                RedisConversation,
+            )
+
+            return RedisConversation(**kwargs)
+        elif backend == "sqlite":
+            from swarms.communication.sqlite_wrap import (
+                SQLiteConversation,
+            )
+
+            return SQLiteConversation(**kwargs)
+        elif backend == "duckdb":
+            from swarms.communication.duckdb_wrap import (
+                DuckDBConversation,
+            )
+
+            return DuckDBConversation(**kwargs)
+        elif backend == "pulsar":
+            from swarms.communication.pulsar_struct import (
+                PulsarConversation,
+            )
+
+            return PulsarConversation(**kwargs)
+        else:
+            raise ValueError(
+                f"Unsupported backend: {backend}. "
+                f"Available backends: supabase, redis, sqlite, duckdb, pulsar"
+            )
+    except ImportError as e:
+        # Provide helpful error messages for missing dependencies
+        backend_deps = {
+            "supabase": "pip install supabase",
+            "redis": "pip install redis",
+            "sqlite": "Built-in to Python - check your installation",
+            "duckdb": "pip install duckdb",
+            "pulsar": "pip install pulsar-client",
+        }
+
+        install_cmd = backend_deps.get(
+            backend, f"Check documentation for {backend}"
+        )
+        logger.error(
+            f"Failed to initialize {backend} backend. "
+            f"Missing dependencies. Install with: {install_cmd}"
+        )
+        raise ImportError(
+            f"Backend '{backend}' dependencies not available. "
+            f"Install with: {install_cmd}. Original error: {e}"
+        )
+    except Exception as e:
+        logger.error(f"Failed to create {backend} backend: {e}")
+        raise
 
 
 class Conversation(BaseStructure):
@@ -43,6 +150,19 @@ class Conversation(BaseStructure):
     and retrieval of messages, as well as saving and loading the conversation
     history in various formats.
 
+    The Conversation class now supports multiple backends for persistent storage:
+    - "in-memory": Default memory-based storage (no persistence)
+    - "mem0": Memory-based storage with mem0 integration (requires: pip install mem0ai)
+    - "supabase": PostgreSQL-based storage using Supabase (requires: pip install supabase)
+    - "redis": Redis-based storage (requires: pip install redis)
+    - "sqlite": SQLite-based storage (built-in to Python)
+    - "duckdb": DuckDB-based storage (requires: pip install duckdb)
+    - "pulsar": Apache Pulsar messaging backend (requires: pip install pulsar-client)
+
+    All backends use lazy loading - database dependencies are only imported when the
+    specific backend is instantiated. Each backend class provides its own detailed
+    error messages if required packages are not installed.
+
     Attributes:
         system_prompt (Optional[str]): The system prompt for the conversation.
         time_enabled (bool): Flag to enable time tracking for messages.
@@ -58,10 +178,6 @@ class Conversation(BaseStructure):
         save_as_json_bool (bool): Flag to save conversation history as JSON.
         token_count (bool): Flag to enable token counting for messages.
         conversation_history (list): List to store the history of messages.
-        cache_enabled (bool): Flag to enable prompt caching.
-        cache_stats (dict): Statistics about cache usage.
-        cache_lock (threading.Lock): Lock for thread-safe cache operations.
-        conversations_dir (str): Directory to store cached conversations.
     """
 
     def __init__(
@@ -70,54 +186,228 @@ class Conversation(BaseStructure):
         name: str = None,
         system_prompt: Optional[str] = None,
         time_enabled: bool = False,
-        autosave: bool = False,
+        autosave: bool = False,  # Changed default to False
+        save_enabled: bool = False,  # New parameter to control if saving is enabled
         save_filepath: str = None,
-        tokenizer: Any = None,
+        load_filepath: str = None,  # New parameter to specify which file to load from
+        tokenizer: Callable = None,
         context_length: int = 8192,
         rules: str = None,
         custom_rules_prompt: str = None,
         user: str = "User:",
-        auto_save: bool = True,
-        save_as_yaml: bool = True,
+        save_as_yaml: bool = False,
         save_as_json_bool: bool = False,
         token_count: bool = True,
-        cache_enabled: bool = True,
-        conversations_dir: Optional[str] = None,
+        message_id_on: bool = False,
         provider: providers = "in-memory",
+        backend: Optional[str] = None,
+        # Backend-specific parameters
+        supabase_url: Optional[str] = None,
+        supabase_key: Optional[str] = None,
+        redis_host: str = "localhost",
+        redis_port: int = 6379,
+        redis_db: int = 0,
+        redis_password: Optional[str] = None,
+        db_path: Optional[str] = None,
+        table_name: str = "conversations",
+        # Additional backend parameters
+        use_embedded_redis: bool = True,
+        persist_redis: bool = True,
+        auto_persist: bool = True,
+        redis_data_dir: Optional[str] = None,
+        conversations_dir: Optional[str] = None,
         *args,
         **kwargs,
     ):
         super().__init__()
 
+        # Support both 'provider' and 'backend' parameters for backwards compatibility
+        # 'backend' takes precedence if both are provided
+        self.backend = backend or provider
+        self.backend_instance = None
+
+        # Validate backend
+        valid_backends = [
+            "in-memory",
+            "mem0",
+            "supabase",
+            "redis",
+            "sqlite",
+            "duckdb",
+            "pulsar",
+        ]
+        if self.backend not in valid_backends:
+            raise ValueError(
+                f"Invalid backend: '{self.backend}'. "
+                f"Valid backends are: {', '.join(valid_backends)}"
+            )
+
         # Initialize all attributes first
         self.id = id
         self.name = name or id
         self.system_prompt = system_prompt
         self.time_enabled = time_enabled
         self.autosave = autosave
-        self.save_filepath = save_filepath
+        self.save_enabled = save_enabled
+        self.conversations_dir = conversations_dir
+        self.message_id_on = message_id_on
+
+        # Handle save filepath
+        if save_enabled and save_filepath:
+            self.save_filepath = save_filepath
+        elif save_enabled and conversations_dir:
+            self.save_filepath = os.path.join(
+                conversations_dir, f"{self.id}.json"
+            )
+        else:
+            self.save_filepath = None
+
+        self.load_filepath = load_filepath
         self.conversation_history = []
         self.tokenizer = tokenizer
         self.context_length = context_length
         self.rules = rules
         self.custom_rules_prompt = custom_rules_prompt
         self.user = user
-        self.auto_save = auto_save
         self.save_as_yaml = save_as_yaml
         self.save_as_json_bool = save_as_json_bool
         self.token_count = token_count
-        self.cache_enabled = cache_enabled
-        self.provider = provider
-        self.cache_stats = {
-            "hits": 0,
-            "misses": 0,
-            "cached_tokens": 0,
-            "total_tokens": 0,
-        }
-        self.cache_lock = threading.Lock()
+        self.provider = provider  # Keep for backwards compatibility
         self.conversations_dir = conversations_dir
 
-        self.setup()
+        # Initialize backend if using persistent storage
+        if self.backend in [
+            "supabase",
+            "redis",
+            "sqlite",
+            "duckdb",
+            "pulsar",
+        ]:
+            try:
+                self._initialize_backend(
+                    supabase_url=supabase_url,
+                    supabase_key=supabase_key,
+                    redis_host=redis_host,
+                    redis_port=redis_port,
+                    redis_db=redis_db,
+                    redis_password=redis_password,
+                    db_path=db_path,
+                    table_name=table_name,
+                    use_embedded_redis=use_embedded_redis,
+                    persist_redis=persist_redis,
+                    auto_persist=auto_persist,
+                    redis_data_dir=redis_data_dir,
+                    **kwargs,
+                )
+            except Exception as e:
+                logger.warning(
+                    f"Failed to initialize {self.backend} backend: {e}. "
+                    f"Falling back to in-memory storage."
+                )
+                self.backend = "in-memory"
+                self.backend_instance = None
+                self.setup()
+        else:
+            # For in-memory and mem0 backends, use the original setup
+            self.setup()
+
+    def _initialize_backend(self, **kwargs):
+        """
+        Initialize the persistent storage backend.
+
+        Args:
+            **kwargs: Backend-specific configuration parameters
+        """
+        # Prepare common backend arguments
+        backend_kwargs = {
+            "system_prompt": self.system_prompt,
+            "time_enabled": self.time_enabled,
+            "autosave": self.autosave,
+            "save_filepath": self.save_filepath,
+            "tokenizer": self.tokenizer,
+            "context_length": self.context_length,
+            "rules": self.rules,
+            "custom_rules_prompt": self.custom_rules_prompt,
+            "user": self.user,
+            "save_as_yaml": self.save_as_yaml,
+            "save_as_json_bool": self.save_as_json_bool,
+            "token_count": self.token_count,
+        }
+
+        # Add backend-specific parameters
+        if self.backend == "supabase":
+            supabase_url = kwargs.get("supabase_url") or os.getenv(
+                "SUPABASE_URL"
+            )
+            supabase_key = kwargs.get("supabase_key") or os.getenv(
+                "SUPABASE_ANON_KEY"
+            )
+
+            if not supabase_url or not supabase_key:
+                raise ValueError(
+                    "Supabase backend requires 'supabase_url' and 'supabase_key' parameters "
+                    "or SUPABASE_URL and SUPABASE_ANON_KEY environment variables"
+                )
+            backend_kwargs.update(
+                {
+                    "supabase_url": supabase_url,
+                    "supabase_key": supabase_key,
+                    "table_name": kwargs.get(
+                        "table_name", "conversations"
+                    ),
+                }
+            )
+
+        elif self.backend == "redis":
+            backend_kwargs.update(
+                {
+                    "redis_host": kwargs.get(
+                        "redis_host", "localhost"
+                    ),
+                    "redis_port": kwargs.get("redis_port", 6379),
+                    "redis_db": kwargs.get("redis_db", 0),
+                    "redis_password": kwargs.get("redis_password"),
+                    "use_embedded_redis": kwargs.get(
+                        "use_embedded_redis", True
+                    ),
+                    "persist_redis": kwargs.get(
+                        "persist_redis", True
+                    ),
+                    "auto_persist": kwargs.get("auto_persist", True),
+                    "redis_data_dir": kwargs.get("redis_data_dir"),
+                    "conversation_id": self.id,
+                    "name": self.name,
+                }
+            )
+
+        elif self.backend in ["sqlite", "duckdb"]:
+            db_path = kwargs.get("db_path")
+            if db_path:
+                backend_kwargs["db_path"] = db_path
+
+        elif self.backend == "pulsar":
+            # Add pulsar-specific parameters
+            backend_kwargs.update(
+                {
+                    "pulsar_url": kwargs.get(
+                        "pulsar_url", "pulsar://localhost:6650"
+                    ),
+                    "topic": kwargs.get(
+                        "topic", f"conversation-{self.id}"
+                    ),
+                }
+            )
+
+        # Create the backend instance
+        logger.info(f"Initializing {self.backend} backend...")
+        self.backend_instance = _create_backend_conversation(
+            self.backend, **backend_kwargs
+        )
+
+        # Log successful initialization
+        logger.info(
+            f"Successfully initialized {self.backend} backend for conversation '{self.name}'"
+        )
 
     def setup(self):
         # Set up conversations directory
@@ -146,22 +436,32 @@ class Conversation(BaseStructure):
                     "history", []
                 )
         else:
-            # If system prompt is not None, add it to the conversation history
-            if self.system_prompt is not None:
-                self.add("System", self.system_prompt)
-
-            if self.rules is not None:
-                self.add(self.user or "User", self.rules)
-
-            if self.custom_rules_prompt is not None:
-                self.add(
-                    self.user or "User", self.custom_rules_prompt
+            self._initialize_new_conversation()
+
+    def _initialize_new_conversation(self):
+        """Initialize a new conversation with system prompt and rules."""
+        if self.system_prompt is not None:
+            self.add("System", self.system_prompt)
+
+        if self.rules is not None:
+            self.add(self.user or "User", self.rules)
+
+        if self.custom_rules_prompt is not None:
+            self.add(self.user or "User", self.custom_rules_prompt)
+
+        if self.tokenizer is not None:
+            self.truncate_memory_with_tokenizer()
+
+    def _autosave(self):
+        """Automatically save the conversation if autosave is enabled."""
+        if self.autosave and self.save_filepath:
+            try:
+                self.save_as_json(self.save_filepath)
+            except Exception as e:
+                logger.error(
+                    f"Failed to autosave conversation: {str(e)}"
                 )
 
-            # If tokenizer then truncate
-            if self.tokenizer is not None:
-                self.truncate_memory_with_tokenizer()
-
     def mem0_provider(self):
         try:
             from mem0 import AsyncMemory
@@ -180,104 +480,10 @@ class Conversation(BaseStructure):
             )
             return None
 
-    def _generate_cache_key(
-        self, content: Union[str, dict, list]
-    ) -> str:
-        """Generate a cache key for the given content.
-
-        Args:
-            content (Union[str, dict, list]): The content to generate a cache key for.
-
-        Returns:
-            str: The cache key.
-        """
-        if isinstance(content, (dict, list)):
-            content = json.dumps(content, sort_keys=True)
-        return hashlib.md5(content.encode()).hexdigest()
-
-    def _get_cached_tokens(
-        self, content: Union[str, dict, list]
-    ) -> Optional[int]:
-        """Get the number of cached tokens for the given content.
-
-        Args:
-            content (Union[str, dict, list]): The content to check.
-
-        Returns:
-            Optional[int]: The number of cached tokens, or None if not cached.
-        """
-        if not self.cache_enabled:
-            return None
-
-        with self.cache_lock:
-            cache_key = self._generate_cache_key(content)
-            if cache_key in self.cache_stats:
-                self.cache_stats["hits"] += 1
-                return self.cache_stats[cache_key]
-            self.cache_stats["misses"] += 1
-            return None
-
-    def _update_cache_stats(
-        self, content: Union[str, dict, list], token_count: int
-    ):
-        """Update cache statistics for the given content.
-
-        Args:
-            content (Union[str, dict, list]): The content to update stats for.
-            token_count (int): The number of tokens in the content.
-        """
-        if not self.cache_enabled:
-            return
-
-        with self.cache_lock:
-            cache_key = self._generate_cache_key(content)
-            self.cache_stats[cache_key] = token_count
-            self.cache_stats["cached_tokens"] += token_count
-            self.cache_stats["total_tokens"] += token_count
-
-    def _save_to_cache(self):
-        """Save the current conversation state to the cache directory."""
-        if not self.conversations_dir:
-            return
-
-        conversation_file = os.path.join(
-            self.conversations_dir, f"{self.name}.json"
-        )
-
-        # Prepare metadata
-        metadata = {
-            "id": self.id,
-            "name": self.name,
-            "system_prompt": self.system_prompt,
-            "time_enabled": self.time_enabled,
-            "autosave": self.autosave,
-            "save_filepath": self.save_filepath,
-            "context_length": self.context_length,
-            "rules": self.rules,
-            "custom_rules_prompt": self.custom_rules_prompt,
-            "user": self.user,
-            "auto_save": self.auto_save,
-            "save_as_yaml": self.save_as_yaml,
-            "save_as_json_bool": self.save_as_json_bool,
-            "token_count": self.token_count,
-            "cache_enabled": self.cache_enabled,
-        }
-
-        # Prepare data to save
-        save_data = {
-            "metadata": metadata,
-            "history": self.conversation_history,
-            "cache_stats": self.cache_stats,
-        }
-
-        # Save to file
-        with open(conversation_file, "w") as f:
-            json.dump(save_data, f, indent=4)
-
     def add_in_memory(
         self,
         role: str,
-        content: Union[str, dict, list],
+        content: Union[str, dict, list, Any],
         *args,
         **kwargs,
     ):
@@ -287,39 +493,32 @@ class Conversation(BaseStructure):
             role (str): The role of the speaker (e.g., 'User', 'System').
             content (Union[str, dict, list]): The content of the message to be added.
         """
-        # Base message with role
+        # Base message with role and timestamp
         message = {
             "role": role,
+            "content": content,
         }
 
-        # Handle different content types
-        if isinstance(content, dict) or isinstance(content, list):
-            message["content"] = content
-        elif self.time_enabled:
-            message["content"] = (
-                f"Time: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} \n {content}"
-            )
-        else:
-            message["content"] = content
+        if self.time_enabled:
+            message["timestamp"] = datetime.datetime.now().isoformat()
 
-        # Check cache for token count
-        cached_tokens = self._get_cached_tokens(content)
-        if cached_tokens is not None:
-            message["token_count"] = cached_tokens
-            message["cached"] = True
-        else:
-            message["cached"] = False
+        if self.message_id_on:
+            message["message_id"] = str(uuid.uuid4())
 
-        # Add message to appropriate backend
+        # Add message to conversation history
         self.conversation_history.append(message)
 
-        if self.token_count is True and not message.get(
-            "cached", False
-        ):
+        if self.token_count is True:
             self._count_tokens(content, message)
 
-        # Save to cache after adding message
-        self._save_to_cache()
+        # Autosave after adding message, but only if saving is enabled
+        if self.autosave and self.save_enabled and self.save_filepath:
+            try:
+                self.save_as_json(self.save_filepath)
+            except Exception as e:
+                logger.error(
+                    f"Failed to autosave conversation: {str(e)}"
+                )
 
     def add_mem0(
         self,
@@ -330,12 +529,19 @@ class Conversation(BaseStructure):
         """Add a message to the conversation history using the Mem0 provider."""
         if self.provider == "mem0":
             memory = self.mem0_provider()
-            memory.add(
-                messages=content,
-                agent_id=role,
-                run_id=self.id,
-                metadata=metadata,
-            )
+            if memory is not None:
+                memory.add(
+                    messages=content,
+                    agent_id=role,
+                    run_id=self.id,
+                    metadata=metadata,
+                )
+            else:
+                # Fallback to in-memory if mem0 is not available
+                logger.warning(
+                    "Mem0 provider not available, falling back to in-memory storage"
+                )
+                self.add_in_memory(role, content)
 
     def add(
         self,
@@ -344,10 +550,21 @@ class Conversation(BaseStructure):
         metadata: Optional[dict] = None,
     ):
         """Add a message to the conversation history."""
-        if self.provider == "in-memory":
-            self.add_in_memory(role, content)
+        # If using a persistent backend, delegate to it
+        if self.backend_instance:
+            try:
+                return self.backend_instance.add(
+                    role=role, content=content, metadata=metadata
+                )
+            except Exception as e:
+                logger.error(
+                    f"Backend add failed: {e}. Falling back to in-memory."
+                )
+                return self.add_in_memory(role, content)
+        elif self.provider == "in-memory":
+            return self.add_in_memory(role, content)
         elif self.provider == "mem0":
-            self.add_mem0(
+            return self.add_mem0(
                 role=role, content=content, metadata=metadata
             )
         else:
@@ -367,8 +584,6 @@ class Conversation(BaseStructure):
                 tokens = count_tokens(any_to_str(content))
                 # Update the message that's already in the conversation history
                 message["token_count"] = int(tokens)
-                # Update cache stats
-                self._update_cache_stats(content, int(tokens))
 
                 # If autosave is enabled, save after token count is updated
                 if self.autosave:
@@ -407,52 +622,77 @@ class Conversation(BaseStructure):
             concurrent.futures.wait(futures)
 
     def delete(self, index: str):
-        """Delete a message from the conversation history.
-
-        Args:
-            index (str): Index of the message to delete.
-        """
-        self.conversation_history.pop(index)
-        self._save_to_cache()
+        """Delete a message from the conversation history."""
+        if self.backend_instance:
+            try:
+                return self.backend_instance.delete(index)
+            except Exception as e:
+                logger.error(f"Backend delete failed: {e}")
+                raise
+        self.conversation_history.pop(int(index))
 
     def update(self, index: str, role, content):
         """Update a message in the conversation history.
 
         Args:
-            index (str): Index of the message to update.
-            role (str): Role of the speaker.
-            content (Union[str, dict]): New content of the message.
+            index (int): The index of the message to update.
+            role (str): The role of the speaker.
+            content: The new content of the message.
         """
-        self.conversation_history[index] = {
-            "role": role,
-            "content": content,
-        }
-        self._save_to_cache()
+        if self.backend_instance:
+            try:
+                return self.backend_instance.update(
+                    index, role, content
+                )
+            except Exception as e:
+                logger.error(f"Backend update failed: {e}")
+                raise
+        if 0 <= int(index) < len(self.conversation_history):
+            self.conversation_history[int(index)]["role"] = role
+            self.conversation_history[int(index)]["content"] = content
+        else:
+            logger.warning(f"Invalid index: {index}")
 
     def query(self, index: str):
-        """Query a message in the conversation history.
+        """Query a message from the conversation history.
 
         Args:
-            index (str): Index of the message to query.
+            index (int): The index of the message to query.
 
         Returns:
-            dict: The message with its role and content.
+            dict: The message at the specified index.
         """
-        return self.conversation_history[index]
+        if self.backend_instance:
+            try:
+                return self.backend_instance.query(index)
+            except Exception as e:
+                logger.error(f"Backend query failed: {e}")
+                raise
+        if 0 <= int(index) < len(self.conversation_history):
+            return self.conversation_history[int(index)]
+        return None
 
     def search(self, keyword: str):
-        """Search for a message in the conversation history.
+        """Search for messages containing a keyword.
 
         Args:
-            keyword (str): Keyword to search for.
+            keyword (str): The keyword to search for.
 
         Returns:
-            list: List of messages containing the keyword.
+            list: A list of messages containing the keyword.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.search(keyword)
+            except Exception as e:
+                logger.error(f"Backend search failed: {e}")
+                # Fallback to in-memory search
+                pass
+
         return [
-            msg
-            for msg in self.conversation_history
-            if keyword in msg["content"]
+            message
+            for message in self.conversation_history
+            if keyword in str(message["content"])
         ]
 
     def display_conversation(self, detailed: bool = False):
@@ -461,10 +701,38 @@ class Conversation(BaseStructure):
         Args:
             detailed (bool, optional): Flag to display detailed information. Defaults to False.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.display_conversation(
+                    detailed
+                )
+            except Exception as e:
+                logger.error(f"Backend display failed: {e}")
+                # Fallback to in-memory display
+                pass
+
+        # In-memory display implementation with proper formatting
         for message in self.conversation_history:
-            formatter.print_panel(
-                f"{message['role']}: {message['content']}\n\n"
-            )
+            content = message.get("content", "")
+            role = message.get("role", "Unknown")
+
+            # Format the message content
+            if isinstance(content, (dict, list)):
+                content = json.dumps(content, indent=2)
+
+            # Create the display string
+            display_str = f"{role}: {content}"
+
+            # Add details if requested
+            if detailed:
+                display_str += f"\nTimestamp: {message.get('timestamp', 'Unknown')}"
+                display_str += f"\nMessage ID: {message.get('message_id', 'Unknown')}"
+                if "token_count" in message:
+                    display_str += (
+                        f"\nTokens: {message['token_count']}"
+                    )
+
+            formatter.print_panel(display_str)
 
     def export_conversation(self, filename: str, *args, **kwargs):
         """Export the conversation history to a file.
@@ -472,9 +740,28 @@ class Conversation(BaseStructure):
         Args:
             filename (str): Filename to export to.
         """
-        with open(filename, "w") as f:
-            for message in self.conversation_history:
-                f.write(f"{message['role']}: {message['content']}\n")
+
+        if self.backend_instance:
+            try:
+                return self.backend_instance.export_conversation(
+                    filename, *args, **kwargs
+                )
+            except Exception as e:
+                logger.error(f"Backend export failed: {e}")
+                # Fallback to in-memory export
+                pass
+
+        # In-memory export implementation
+        # If the filename ends with .json, use save_as_json
+        if filename.endswith(".json"):
+            self.save_as_json(filename)
+        else:
+            # Simple text export for non-JSON files
+            with open(filename, "w", encoding="utf-8") as f:
+                for message in self.conversation_history:
+                    f.write(
+                        f"{message['role']}: {message['content']}\n"
+                    )
 
     def import_conversation(self, filename: str):
         """Import a conversation history from a file.
@@ -482,10 +769,16 @@ class Conversation(BaseStructure):
         Args:
             filename (str): Filename to import from.
         """
-        with open(filename) as f:
-            for line in f:
-                role, content = line.split(": ", 1)
-                self.add(role, content.strip())
+        if self.backend_instance:
+            try:
+                return self.backend_instance.import_conversation(
+                    filename
+                )
+            except Exception as e:
+                logger.error(f"Backend import failed: {e}")
+                # Fallback to in-memory import
+                pass
+        self.load_from_json(filename)
 
     def count_messages_by_role(self):
         """Count the number of messages by role.
@@ -493,14 +786,33 @@ class Conversation(BaseStructure):
         Returns:
             dict: A dictionary with counts of messages by role.
         """
+        # Check backend instance first
+        if self.backend_instance:
+            try:
+                return self.backend_instance.count_messages_by_role()
+            except Exception as e:
+                logger.error(
+                    f"Backend count_messages_by_role failed: {e}"
+                )
+                # Fallback to local implementation below
+                pass
+        # Initialize counts with expected roles
         counts = {
             "system": 0,
             "user": 0,
             "assistant": 0,
             "function": 0,
         }
+
+        # Count messages by role
         for message in self.conversation_history:
-            counts[message["role"]] += 1
+            role = message["role"]
+            if role in counts:
+                counts[role] += 1
+            else:
+                # Handle unexpected roles dynamically
+                counts[role] = counts.get(role, 0) + 1
+
         return counts
 
     def return_history_as_string(self):
@@ -509,6 +821,18 @@ class Conversation(BaseStructure):
         Returns:
             str: The conversation history formatted as a string.
         """
+        if self.backend_instance:
+            try:
+                return (
+                    self.backend_instance.return_history_as_string()
+                )
+            except Exception as e:
+                logger.error(
+                    f"Backend return_history_as_string failed: {e}"
+                )
+                # Fallback to in-memory implementation
+                pass
+
         formatted_messages = []
         for message in self.conversation_history:
             formatted_messages.append(
@@ -523,6 +847,13 @@ class Conversation(BaseStructure):
         Returns:
             str: The conversation history.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.get_str()
+            except Exception as e:
+                logger.error(f"Backend get_str failed: {e}")
+                # Fallback to in-memory implementation
+                pass
         return self.return_history_as_string()
 
     def save_as_json(self, filename: str = None):
@@ -531,9 +862,55 @@ class Conversation(BaseStructure):
         Args:
             filename (str): Filename to save the conversation history.
         """
-        if filename is not None:
-            with open(filename, "w") as f:
-                json.dump(self.conversation_history, f)
+        # Check backend instance first
+        if self.backend_instance:
+            try:
+                return self.backend_instance.save_as_json(filename)
+            except Exception as e:
+                logger.error(f"Backend save_as_json failed: {e}")
+                # Fallback to local save implementation below
+
+        # Don't save if saving is disabled
+        if not self.save_enabled:
+            return
+
+        save_path = filename or self.save_filepath
+        if save_path is not None:
+            try:
+                # Prepare metadata
+                metadata = {
+                    "id": self.id,
+                    "name": self.name,
+                    "created_at": datetime.datetime.now().isoformat(),
+                    "system_prompt": self.system_prompt,
+                    "rules": self.rules,
+                    "custom_rules_prompt": self.custom_rules_prompt,
+                }
+
+                # Prepare save data
+                save_data = {
+                    "metadata": metadata,
+                    "history": self.conversation_history,
+                }
+
+                # Create directory if it doesn't exist
+                os.makedirs(
+                    os.path.dirname(save_path),
+                    mode=0o755,
+                    exist_ok=True,
+                )
+
+                # Write directly to file
+                with open(save_path, "w") as f:
+                    json.dump(save_data, f, indent=2)
+
+                # Only log explicit saves, not autosaves
+                if not self.autosave:
+                    logger.info(
+                        f"Successfully saved conversation to {save_path}"
+                    )
+            except Exception as e:
+                logger.error(f"Failed to save conversation: {str(e)}")
 
     def load_from_json(self, filename: str):
         """Load the conversation history from a JSON file.
@@ -541,9 +918,32 @@ class Conversation(BaseStructure):
         Args:
             filename (str): Filename to load from.
         """
-        if filename is not None:
-            with open(filename) as f:
-                self.conversation_history = json.load(f)
+        if filename is not None and os.path.exists(filename):
+            try:
+                with open(filename) as f:
+                    data = json.load(f)
+
+                # Load metadata
+                metadata = data.get("metadata", {})
+                self.id = metadata.get("id", self.id)
+                self.name = metadata.get("name", self.name)
+                self.system_prompt = metadata.get(
+                    "system_prompt", self.system_prompt
+                )
+                self.rules = metadata.get("rules", self.rules)
+                self.custom_rules_prompt = metadata.get(
+                    "custom_rules_prompt", self.custom_rules_prompt
+                )
+
+                # Load conversation history
+                self.conversation_history = data.get("history", [])
+
+                logger.info(
+                    f"Successfully loaded conversation from {filename}"
+                )
+            except Exception as e:
+                logger.error(f"Failed to load conversation: {str(e)}")
+                raise
 
     def search_keyword_in_conversation(self, keyword: str):
         """Search for a keyword in the conversation history.
@@ -599,8 +999,14 @@ class Conversation(BaseStructure):
 
     def clear(self):
         """Clear the conversation history."""
+        if self.backend_instance:
+            try:
+                return self.backend_instance.clear()
+            except Exception as e:
+                logger.error(f"Backend clear failed: {e}")
+                # Fallback to in-memory clear
+                pass
         self.conversation_history = []
-        self._save_to_cache()
 
     def to_json(self):
         """Convert the conversation history to a JSON string.
@@ -608,6 +1014,13 @@ class Conversation(BaseStructure):
         Returns:
             str: The conversation history as a JSON string.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.to_json()
+            except Exception as e:
+                logger.error(f"Backend to_json failed: {e}")
+                # Fallback to in-memory implementation
+                pass
         return json.dumps(self.conversation_history)
 
     def to_dict(self):
@@ -616,6 +1029,13 @@ class Conversation(BaseStructure):
         Returns:
             list: The conversation history as a list of dictionaries.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.to_dict()
+            except Exception as e:
+                logger.error(f"Backend to_dict failed: {e}")
+                # Fallback to in-memory implementation
+                pass
         return self.conversation_history
 
     def to_yaml(self):
@@ -624,6 +1044,13 @@ class Conversation(BaseStructure):
         Returns:
             str: The conversation history as a YAML string.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.to_yaml()
+            except Exception as e:
+                logger.error(f"Backend to_yaml failed: {e}")
+                # Fallback to in-memory implementation
+                pass
         return yaml.dump(self.conversation_history)
 
     def get_visible_messages(self, agent: "Agent", turn: int):
@@ -659,11 +1086,24 @@ class Conversation(BaseStructure):
         Returns:
             str: The last message formatted as 'role: content'.
         """
-        if self.provider == "mem0":
+        if self.backend_instance:
+            try:
+                return (
+                    self.backend_instance.get_last_message_as_string()
+                )
+            except Exception as e:
+                logger.error(
+                    f"Backend get_last_message_as_string failed: {e}"
+                )
+                # Fallback to in-memory implementation
+                pass
+        elif self.provider == "mem0":
             memory = self.mem0_provider()
             return memory.get_all(run_id=self.id)
         elif self.provider == "in-memory":
-            return f"{self.conversation_history[-1]['role']}: {self.conversation_history[-1]['content']}"
+            if self.conversation_history:
+                return f"{self.conversation_history[-1]['role']}: {self.conversation_history[-1]['content']}"
+            return ""
         else:
             raise ValueError(f"Invalid provider: {self.provider}")
 
@@ -673,6 +1113,15 @@ class Conversation(BaseStructure):
         Returns:
             list: List of messages formatted as 'role: content'.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.return_messages_as_list()
+            except Exception as e:
+                logger.error(
+                    f"Backend return_messages_as_list failed: {e}"
+                )
+                # Fallback to in-memory implementation
+                pass
         return [
             f"{message['role']}: {message['content']}"
             for message in self.conversation_history
@@ -684,6 +1133,17 @@ class Conversation(BaseStructure):
         Returns:
             list: List of dictionaries containing role and content of each message.
         """
+        if self.backend_instance:
+            try:
+                return (
+                    self.backend_instance.return_messages_as_dictionary()
+                )
+            except Exception as e:
+                logger.error(
+                    f"Backend return_messages_as_dictionary failed: {e}"
+                )
+                # Fallback to in-memory implementation
+                pass
         return [
             {
                 "role": message["role"],
@@ -718,7 +1178,16 @@ class Conversation(BaseStructure):
         Returns:
             str: The final message formatted as 'role: content'.
         """
-        return f"{self.conversation_history[-1]['role']}: {self.conversation_history[-1]['content']}"
+        if self.backend_instance:
+            try:
+                return self.backend_instance.get_final_message()
+            except Exception as e:
+                logger.error(f"Backend get_final_message failed: {e}")
+                # Fallback to in-memory implementation
+                pass
+        if self.conversation_history:
+            return f"{self.conversation_history[-1]['role']}: {self.conversation_history[-1]['content']}"
+        return ""
 
     def get_final_message_content(self):
         """Return the content of the final message from the conversation history.
@@ -726,9 +1195,21 @@ class Conversation(BaseStructure):
         Returns:
             str: The content of the final message.
         """
-        output = self.conversation_history[-1]["content"]
-        # print(output)
-        return output
+        if self.backend_instance:
+            try:
+                return (
+                    self.backend_instance.get_final_message_content()
+                )
+            except Exception as e:
+                logger.error(
+                    f"Backend get_final_message_content failed: {e}"
+                )
+                # Fallback to in-memory implementation
+                pass
+        if self.conversation_history:
+            output = self.conversation_history[-1]["content"]
+            return output
+        return ""
 
     def return_all_except_first(self):
         """Return all messages except the first one.
@@ -736,6 +1217,15 @@ class Conversation(BaseStructure):
         Returns:
             list: List of messages except the first one.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.return_all_except_first()
+            except Exception as e:
+                logger.error(
+                    f"Backend return_all_except_first failed: {e}"
+                )
+                # Fallback to in-memory implementation
+                pass
         return self.conversation_history[2:]
 
     def return_all_except_first_string(self):
@@ -744,6 +1234,17 @@ class Conversation(BaseStructure):
         Returns:
             str: All messages except the first one as a string.
         """
+        if self.backend_instance:
+            try:
+                return (
+                    self.backend_instance.return_all_except_first_string()
+                )
+            except Exception as e:
+                logger.error(
+                    f"Backend return_all_except_first_string failed: {e}"
+                )
+                # Fallback to in-memory implementation
+                pass
         return "\n".join(
             [
                 f"{msg['content']}"
@@ -757,80 +1258,147 @@ class Conversation(BaseStructure):
         Args:
             messages (List[dict]): List of messages to add.
         """
+        if self.backend_instance:
+            try:
+                return self.backend_instance.batch_add(messages)
+            except Exception as e:
+                logger.error(f"Backend batch_add failed: {e}")
+                # Fallback to in-memory implementation
+                pass
         self.conversation_history.extend(messages)
 
-    def get_cache_stats(self) -> Dict[str, int]:
-        """Get statistics about cache usage.
-
-        Returns:
-            Dict[str, int]: Statistics about cache usage.
-        """
-        with self.cache_lock:
-            return {
-                "hits": self.cache_stats["hits"],
-                "misses": self.cache_stats["misses"],
-                "cached_tokens": self.cache_stats["cached_tokens"],
-                "total_tokens": self.cache_stats["total_tokens"],
-                "hit_rate": (
-                    self.cache_stats["hits"]
-                    / (
-                        self.cache_stats["hits"]
-                        + self.cache_stats["misses"]
-                    )
-                    if (
-                        self.cache_stats["hits"]
-                        + self.cache_stats["misses"]
-                    )
-                    > 0
-                    else 0
-                ),
-            }
+    def clear_memory(self):
+        """Clear the memory of the conversation."""
+        self.conversation_history = []
 
     @classmethod
     def load_conversation(
-        cls, name: str, conversations_dir: Optional[str] = None
+        cls,
+        name: str,
+        conversations_dir: Optional[str] = None,
+        load_filepath: Optional[str] = None,
     ) -> "Conversation":
-        """Load a conversation from the cache by name.
+        """Load a conversation from saved file by name or specific file.
 
         Args:
             name (str): Name of the conversation to load
-            conversations_dir (Optional[str]): Directory containing cached conversations
+            conversations_dir (Optional[str]): Directory containing conversations
+            load_filepath (Optional[str]): Specific file to load from
 
         Returns:
             Conversation: The loaded conversation object
         """
-        return cls(name=name, conversations_dir=conversations_dir)
+        if load_filepath:
+            return cls(
+                name=name,
+                load_filepath=load_filepath,
+                save_enabled=False,  # Don't enable saving when loading specific file
+            )
+
+        conv_dir = conversations_dir or get_conversation_dir()
+        # Try loading by name first
+        filepath = os.path.join(conv_dir, f"{name}.json")
+
+        # If not found by name, try loading by ID
+        if not os.path.exists(filepath):
+            filepath = os.path.join(conv_dir, f"{name}")
+            if not os.path.exists(filepath):
+                logger.warning(
+                    f"No conversation found with name or ID: {name}"
+                )
+                return cls(
+                    name=name,
+                    conversations_dir=conv_dir,
+                    save_enabled=True,
+                )
+
+        return cls(
+            name=name,
+            conversations_dir=conv_dir,
+            load_filepath=filepath,
+            save_enabled=True,
+        )
+
+    def return_dict_final(self):
+        """Return the final message as a dictionary."""
+        return (
+            self.conversation_history[-1]["content"],
+            self.conversation_history[-1]["content"],
+        )
 
     @classmethod
-    def list_cached_conversations(
+    def list_conversations(
         cls, conversations_dir: Optional[str] = None
-    ) -> List[str]:
-        """List all cached conversations.
+    ) -> List[Dict[str, str]]:
+        """List all saved conversations.
 
         Args:
-            conversations_dir (Optional[str]): Directory containing cached conversations
+            conversations_dir (Optional[str]): Directory containing conversations
 
         Returns:
-            List[str]: List of conversation names (without .json extension)
+            List[Dict[str, str]]: List of conversation metadata
         """
-        if conversations_dir is None:
-            conversations_dir = os.path.join(
-                os.path.expanduser("~"), ".swarms", "conversations"
-            )
-
-        if not os.path.exists(conversations_dir):
+        conv_dir = conversations_dir or get_conversation_dir()
+        if not os.path.exists(conv_dir):
             return []
 
         conversations = []
-        for file in os.listdir(conversations_dir):
-            if file.endswith(".json"):
-                conversations.append(
-                    file[:-5]
-                )  # Remove .json extension
-        return conversations
+        seen_ids = (
+            set()
+        )  # Track seen conversation IDs to avoid duplicates
+
+        for filename in os.listdir(conv_dir):
+            if filename.endswith(".json"):
+                try:
+                    filepath = os.path.join(conv_dir, filename)
+                    with open(filepath) as f:
+                        data = json.load(f)
+                        metadata = data.get("metadata", {})
+                        conv_id = metadata.get("id")
+                        name = metadata.get("name")
+                        created_at = metadata.get("created_at")
+
+                        # Skip if we've already seen this ID or if required fields are missing
+                        if (
+                            not all([conv_id, name, created_at])
+                            or conv_id in seen_ids
+                        ):
+                            continue
+
+                        seen_ids.add(conv_id)
+                        conversations.append(
+                            {
+                                "id": conv_id,
+                                "name": name,
+                                "created_at": created_at,
+                                "filepath": filepath,
+                            }
+                        )
+                except json.JSONDecodeError:
+                    logger.warning(
+                        f"Skipping corrupted conversation file: {filename}"
+                    )
+                    continue
+                except Exception as e:
+                    logger.error(
+                        f"Failed to read conversation {filename}: {str(e)}"
+                    )
+                    continue
+
+        # Sort by creation date, newest first
+        return sorted(
+            conversations, key=lambda x: x["created_at"], reverse=True
+        )
 
     def clear_memory(self):
         """Clear the memory of the conversation."""
+        if self.backend_instance:
+            try:
+                return self.backend_instance.clear()
+            except Exception as e:
+                logger.error(f"Backend clear_memory failed: {e}")
+                # Fallback to in-memory implementation
+                pass
         self.conversation_history = []