memorisdk 1.0.2__py3-none-any.whl → 2.0.1__py3-none-any.whl

memori/tools/memory_tool.py

@@ -73,11 +73,24 @@ class MemoryTool:
 
         # Use retrieval agent for intelligent search
         try:
+            logger.debug(
+                f"Attempting to import MemorySearchEngine for query: '{query}'"
+            )
             from ..agents.retrieval_agent import MemorySearchEngine
 
+            logger.debug("Successfully imported MemorySearchEngine")
+
             # Create search engine if not already initialized
             if not hasattr(self, "_search_engine"):
-                self._search_engine = MemorySearchEngine()
+                if (
+                    hasattr(self.memori, "provider_config")
+                    and self.memori.provider_config
+                ):
+                    self._search_engine = MemorySearchEngine(
+                        provider_config=self.memori.provider_config
+                    )
+                else:
+                    self._search_engine = MemorySearchEngine()
 
             # Execute search using retrieval agent
             results = self._search_engine.execute_search(
@@ -88,18 +101,62 @@ class MemoryTool:
             )
 
             if not results:
+                logger.debug(
+                    f"Primary search returned no results for query: '{query}', trying fallback search"
+                )
+                # Try fallback direct database search
+                try:
+                    fallback_results = self.memori.db_manager.search_memories(
+                        query=query, namespace=self.memori.namespace, limit=5
+                    )
+
+                    if fallback_results:
+                        logger.debug(
+                            f"Fallback search found {len(fallback_results)} results"
+                        )
+                        results = fallback_results
+                    else:
+                        logger.warning(
+                            f"Both primary and fallback search returned no results for query: '{query}'"
+                        )
+                        return f"No relevant memories found for query: '{query}'"
+
+                except Exception as fallback_e:
+                    logger.error(
+                        f"Fallback search also failed for query '{query}': {fallback_e}"
+                    )
+                    return f"No relevant memories found for query: '{query}'"
+
+            # Ensure we have results to format
+            if not results:
+                logger.warning(
+                    f"No results available for formatting for query: '{query}'"
+                )
                 return f"No relevant memories found for query: '{query}'"
 
             # Format results as a readable string
+            logger.debug(
+                f"Starting to format {len(results)} results for query: '{query}'"
+            )
             formatted_output = f"🔍 Memory Search Results for: '{query}'\n\n"
 
             for i, result in enumerate(results, 1):
                 try:
+                    logger.debug(
+                        f"Formatting result {i}: type={type(result)}, keys={list(result.keys()) if isinstance(result, dict) else 'not-dict'}"
+                    )
+
                     # Try to parse processed data for better formatting
                     if "processed_data" in result:
                         import json
 
-                        processed_data = json.loads(result["processed_data"])
+                        if isinstance(result["processed_data"], dict):
+                            processed_data = result["processed_data"]
+                        elif isinstance(result["processed_data"], str):
+                            processed_data = json.loads(result["processed_data"])
+                        else:
+                            raise ValueError("Error, wrong 'processed_data' format")
+
                         summary = processed_data.get("summary", "")
                         category = processed_data.get("category", {}).get(
                             "primary_category", ""
@@ -124,34 +181,63 @@ class MemoryTool:
 
                     formatted_output += "\n"
 
-                except Exception:
+                except Exception as format_e:
+                    logger.warning(f"Error formatting result {i}: {format_e}")
                     # Fallback formatting
                     content = result.get(
                         "searchable_content", "Memory content available"
                     )[:100]
                     formatted_output += f"{i}. {content}...\n\n"
 
+            logger.debug(
+                f"Successfully formatted results, output length: {len(formatted_output)}"
+            )
             return formatted_output.strip()
 
-        except ImportError:
+        except ImportError as import_e:
+            logger.warning(
+                f"Failed to import MemorySearchEngine for query '{query}': {import_e}"
+            )
             # Fallback to original search methods if retrieval agent is not available
+            logger.debug(
+                f"Using ImportError fallback search methods for query: '{query}'"
+            )
+
             # Try different search strategies based on query content
             if any(word in query.lower() for word in ["name", "who am i", "about me"]):
+                logger.debug(
+                    f"Trying essential conversations for personal query: '{query}'"
+                )
                 # Personal information query - try essential conversations first
                 essential_result = self._get_essential_conversations()
                 if essential_result.get("count", 0) > 0:
+                    logger.debug(
+                        f"Essential conversations found {essential_result.get('count', 0)} results"
+                    )
                     return self._format_dict_to_string(essential_result)
 
             # General search
+            logger.debug(f"Trying general search for query: '{query}'")
             search_result = self._search_memories(query=query, limit=10)
+            logger.debug(
+                f"General search returned results_count: {search_result.get('results_count', 0)}"
+            )
             if search_result.get("results_count", 0) > 0:
                 return self._format_dict_to_string(search_result)
 
             # Fallback to context retrieval
+            logger.debug(f"Trying context retrieval fallback for query: '{query}'")
             context_result = self._retrieve_context(query=query, limit=5)
+            logger.debug(
+                f"Context retrieval returned context_count: {context_result.get('context_count', 0)}"
+            )
             return self._format_dict_to_string(context_result)
 
         except Exception as e:
+            logger.error(
+                f"Unexpected error in memory tool execute for query '{query}': {e}",
+                exc_info=True,
+            )
             return f"Error searching memories: {str(e)}"
 
     def _format_dict_to_string(self, result_dict: Dict[str, Any]) -> str:
@@ -293,6 +379,8 @@ class MemoryTool:
 
     def _get_stats(self, **kwargs) -> Dict[str, Any]:
         """Get memory and integration statistics"""
+        # kwargs can be used for future filtering options
+        _ = kwargs  # Mark as intentionally unused
         try:
             memory_stats = self.memori.get_memory_stats()
             integration_stats = self.memori.get_integration_stats()
@@ -348,6 +436,8 @@ class MemoryTool:
 
     def _trigger_analysis(self, **kwargs) -> Dict[str, Any]:
         """Trigger conscious agent analysis"""
+        # kwargs can be used for future analysis options
+        _ = kwargs  # Mark as intentionally unused
         try:
             if hasattr(self.memori, "trigger_conscious_analysis"):
                 self.memori.trigger_conscious_analysis()

memori/utils/input_validator.py

@@ -0,0 +1,395 @@
+"""
+Input validation and sanitization utilities for Memori
+Provides security-focused validation for all database inputs
+"""
+
+import html
+import json
+import re
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+from loguru import logger
+
+from .exceptions import ValidationError
+
+
+class InputValidator:
+    """Comprehensive input validation and sanitization"""
+
+    # SQL injection patterns to detect and block
+    SQL_INJECTION_PATTERNS = [
+        r"(\b(SELECT|INSERT|UPDATE|DELETE|DROP|CREATE|ALTER|EXEC|UNION)\b)",
+        r"(\b(OR|AND)\s+[\w\s]*=[\w\s]*)",
+        r"(;|\|\||&&)",
+        r"(\-\-|\#|/\*|\*/)",
+        r"(\bxp_cmdshell\b|\bsp_executesql\b)",
+        r"(\bINTO\s+OUTFILE\b|\bINTO\s+DUMPFILE\b)",
+    ]
+
+    # XSS patterns to detect and sanitize
+    XSS_PATTERNS = [
+        r"<\s*script[^>]*>.*?</\s*script\s*>",
+        r"<\s*iframe[^>]*>.*?</\s*iframe\s*>",
+        r"<\s*object[^>]*>.*?</\s*object\s*>",
+        r"<\s*embed[^>]*>",
+        r"javascript\s*:",
+        r"on\w+\s*=",
+    ]
+
+    @classmethod
+    def validate_and_sanitize_query(cls, query: str, max_length: int = 10000) -> str:
+        """Validate and sanitize search query input"""
+        if not isinstance(query, (str, type(None))):
+            raise ValidationError("Query must be a string or None")
+
+        if query is None:
+            return ""
+
+        # Length validation
+        if len(query) > max_length:
+            raise ValidationError(f"Query too long (max {max_length} characters)")
+
+        # Check for SQL injection patterns
+        query_lower = query.lower()
+        for pattern in cls.SQL_INJECTION_PATTERNS:
+            if re.search(pattern, query_lower, re.IGNORECASE):
+                logger.warning(f"Potential SQL injection attempt blocked: {pattern}")
+                raise ValidationError(
+                    "Invalid query: contains potentially dangerous content"
+                )
+
+        # Check for XSS patterns
+        for pattern in cls.XSS_PATTERNS:
+            if re.search(pattern, query, re.IGNORECASE):
+                logger.warning(f"Potential XSS attempt blocked: {pattern}")
+                # Sanitize instead of blocking for XSS
+                query = re.sub(pattern, "", query, flags=re.IGNORECASE)
+
+        # HTML escape for additional safety
+        sanitized_query = html.escape(query.strip())
+
+        return sanitized_query
+
+    @classmethod
+    def validate_namespace(cls, namespace: str) -> str:
+        """Validate and sanitize namespace"""
+        if not isinstance(namespace, str):
+            raise ValidationError("Namespace must be a string")
+
+        # Namespace validation rules
+        sanitized_namespace = namespace.strip()
+
+        if not sanitized_namespace:
+            sanitized_namespace = "default"
+
+        # Only allow alphanumeric, underscore, hyphen
+        if not re.match(r"^[a-zA-Z0-9_\-]+$", sanitized_namespace):
+            raise ValidationError(
+                "Namespace contains invalid characters (only alphanumeric, underscore, hyphen allowed)"
+            )
+
+        if len(sanitized_namespace) > 100:
+            raise ValidationError("Namespace too long (max 100 characters)")
+
+        return sanitized_namespace
+
+    @classmethod
+    def validate_category_filter(
+        cls, category_filter: Optional[List[str]]
+    ) -> List[str]:
+        """Validate and sanitize category filter list"""
+        if category_filter is None:
+            return []
+
+        if not isinstance(category_filter, list):
+            raise ValidationError("Category filter must be a list or None")
+
+        if len(category_filter) > 50:  # Reasonable limit
+            raise ValidationError("Too many categories in filter (max 50)")
+
+        sanitized_categories = []
+        for category in category_filter:
+            if not isinstance(category, str):
+                continue  # Skip non-string categories
+
+            sanitized_category = category.strip()
+            if not sanitized_category:
+                continue  # Skip empty categories
+
+            # Validate category format
+            if not re.match(r"^[a-zA-Z0-9_\-\s]+$", sanitized_category):
+                logger.warning(f"Invalid category format: {sanitized_category}")
+                continue  # Skip invalid categories
+
+            if len(sanitized_category) > 100:
+                sanitized_category = sanitized_category[:100]  # Truncate if too long
+
+            sanitized_categories.append(sanitized_category)
+
+        return sanitized_categories
+
+    @classmethod
+    def validate_limit(cls, limit: Union[int, str]) -> int:
+        """Validate and sanitize limit parameter"""
+        try:
+            int_limit = int(limit)
+        except (ValueError, TypeError):
+            raise ValidationError("Limit must be a valid integer")
+
+        # Enforce reasonable bounds
+        if int_limit < 1:
+            return 1
+        elif int_limit > 1000:  # Maximum reasonable limit
+            return 1000
+
+        return int_limit
+
+    @classmethod
+    def validate_memory_id(cls, memory_id: str) -> str:
+        """Validate memory ID format"""
+        if not isinstance(memory_id, str):
+            raise ValidationError("Memory ID must be a string")
+
+        sanitized_id = memory_id.strip()
+
+        if not sanitized_id:
+            raise ValidationError("Memory ID cannot be empty")
+
+        # UUID-like format validation
+        if not re.match(r"^[a-fA-F0-9\-]{36}$", sanitized_id):
+            # Also allow shorter alphanumeric IDs for flexibility
+            if not re.match(r"^[a-zA-Z0-9_\-]+$", sanitized_id):
+                raise ValidationError("Invalid memory ID format")
+
+        if len(sanitized_id) > 100:
+            raise ValidationError("Memory ID too long")
+
+        return sanitized_id
+
+    @classmethod
+    def validate_json_field(cls, json_data: Any, field_name: str = "data") -> str:
+        """Validate and sanitize JSON data"""
+        if json_data is None:
+            return "{}"
+
+        try:
+            if isinstance(json_data, str):
+                # Validate it's proper JSON
+                parsed_data = json.loads(json_data)
+                # Re-serialize to ensure clean format
+                clean_json = json.dumps(
+                    parsed_data, ensure_ascii=True, separators=(",", ":")
+                )
+            else:
+                # Serialize Python object to JSON
+                clean_json = json.dumps(
+                    json_data, ensure_ascii=True, separators=(",", ":")
+                )
+
+            # Size limit check (1MB for JSON data)
+            if len(clean_json) > 1024 * 1024:
+                raise ValidationError(f"{field_name} JSON too large (max 1MB)")
+
+            return clean_json
+
+        except (json.JSONDecodeError, TypeError) as e:
+            raise ValidationError(f"Invalid JSON in {field_name}: {e}")
+
+    @classmethod
+    def validate_text_content(
+        cls, content: str, field_name: str = "content", max_length: int = 100000
+    ) -> str:
+        """Validate and sanitize text content"""
+        if not isinstance(content, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        # Length check
+        if len(content) > max_length:
+            raise ValidationError(
+                f"{field_name} too long (max {max_length} characters)"
+            )
+
+        # XSS sanitization
+        sanitized_content = content
+        for pattern in cls.XSS_PATTERNS:
+            sanitized_content = re.sub(
+                pattern, "", sanitized_content, flags=re.IGNORECASE
+            )
+
+        # Basic HTML escaping for storage
+        sanitized_content = html.escape(sanitized_content)
+
+        return sanitized_content.strip()
+
+    @classmethod
+    def validate_timestamp(cls, timestamp: Union[datetime, str, None]) -> datetime:
+        """Validate and normalize timestamp"""
+        if timestamp is None:
+            return datetime.now()
+
+        if isinstance(timestamp, datetime):
+            # Make timezone-naive for SQLite compatibility
+            return timestamp.replace(tzinfo=None)
+
+        if isinstance(timestamp, str):
+            try:
+                # Try to parse ISO format
+                parsed_timestamp = datetime.fromisoformat(
+                    timestamp.replace("Z", "+00:00")
+                )
+                return parsed_timestamp.replace(tzinfo=None)
+            except ValueError:
+                raise ValidationError("Invalid timestamp format (use ISO format)")
+
+        raise ValidationError("Timestamp must be datetime object, ISO string, or None")
+
+    @classmethod
+    def validate_score(
+        cls, score: Union[float, int, str], field_name: str = "score"
+    ) -> float:
+        """Validate and normalize score values (0.0 to 1.0)"""
+        try:
+            float_score = float(score)
+        except (ValueError, TypeError):
+            raise ValidationError(f"{field_name} must be a valid number")
+
+        # Clamp to valid range
+        if float_score < 0.0:
+            return 0.0
+        elif float_score > 1.0:
+            return 1.0
+
+        return float_score
+
+    @classmethod
+    def validate_boolean_field(cls, value: Any, field_name: str = "field") -> bool:
+        """Validate and convert boolean field"""
+        if isinstance(value, bool):
+            return value
+
+        if isinstance(value, int):
+            return bool(value)
+
+        if isinstance(value, str):
+            return value.lower() in ("true", "1", "yes", "on")
+
+        return False  # Default to False for safety
+
+    @classmethod
+    def sanitize_sql_identifier(cls, identifier: str) -> str:
+        """Sanitize SQL identifiers (table names, column names)"""
+        if not isinstance(identifier, str):
+            raise ValidationError("SQL identifier must be a string")
+
+        # Remove dangerous characters and validate format
+        sanitized = re.sub(r"[^a-zA-Z0-9_]", "", identifier)
+
+        if not sanitized or not re.match(r"^[a-zA-Z][a-zA-Z0-9_]*$", sanitized):
+            raise ValidationError("Invalid SQL identifier format")
+
+        if len(sanitized) > 64:  # SQL standard limit
+            raise ValidationError("SQL identifier too long")
+
+        # Block reserved words (basic list)
+        reserved_words = {
+            "SELECT",
+            "INSERT",
+            "UPDATE",
+            "DELETE",
+            "DROP",
+            "CREATE",
+            "ALTER",
+            "TABLE",
+            "DATABASE",
+            "INDEX",
+            "VIEW",
+            "TRIGGER",
+            "PROCEDURE",
+            "FUNCTION",
+            "EXEC",
+            "EXECUTE",
+            "UNION",
+            "WHERE",
+            "FROM",
+            "JOIN",
+        }
+
+        if sanitized.upper() in reserved_words:
+            raise ValidationError(
+                f"Cannot use reserved word as identifier: {sanitized}"
+            )
+
+        return sanitized
+
+
+class DatabaseInputValidator:
+    """Database-specific input validation"""
+
+    @classmethod
+    def validate_insert_params(
+        cls, table: str, params: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Validate parameters for database insert operations"""
+        sanitized_params = {}
+
+        # Validate table name
+        InputValidator.sanitize_sql_identifier(table)
+
+        for key, value in params.items():
+            # Validate column names
+            sanitized_key = InputValidator.sanitize_sql_identifier(key)
+
+            # Type-specific validation
+            if key.endswith("_id"):
+                if value is not None:
+                    sanitized_params[sanitized_key] = InputValidator.validate_memory_id(
+                        str(value)
+                    )
+                else:
+                    sanitized_params[sanitized_key] = None
+            elif key == "namespace":
+                sanitized_params[sanitized_key] = InputValidator.validate_namespace(
+                    str(value)
+                )
+            elif key.endswith("_score"):
+                sanitized_params[sanitized_key] = InputValidator.validate_score(
+                    value, key
+                )
+            elif key.endswith("_at") or key == "timestamp":
+                sanitized_params[sanitized_key] = InputValidator.validate_timestamp(
+                    value
+                )
+            elif key.endswith("_json") or key == "metadata":
+                sanitized_params[sanitized_key] = InputValidator.validate_json_field(
+                    value, key
+                )
+            elif isinstance(value, bool) or key.startswith("is_"):
+                sanitized_params[sanitized_key] = InputValidator.validate_boolean_field(
+                    value, key
+                )
+            elif isinstance(value, str):
+                sanitized_params[sanitized_key] = InputValidator.validate_text_content(
+                    value, key, max_length=50000
+                )
+            else:
+                # Pass through numeric and other safe types
+                sanitized_params[sanitized_key] = value
+
+        return sanitized_params
+
+    @classmethod
+    def validate_search_params(
+        cls,
+        query: str,
+        namespace: str,
+        category_filter: Optional[List[str]],
+        limit: int,
+    ) -> Dict[str, Any]:
+        """Validate all search parameters together"""
+        return {
+            "query": InputValidator.validate_and_sanitize_query(query),
+            "namespace": InputValidator.validate_namespace(namespace),
+            "category_filter": InputValidator.validate_category_filter(category_filter),
+            "limit": InputValidator.validate_limit(limit),
+        }