memorisdk 1.0.0__py3-none-any.whl

memoriai/utils/helpers.py

@@ -0,0 +1,433 @@
+"""
+Utility helper functions for Memoriai
+"""
+
+import asyncio
+import functools
+import hashlib
+import json
+import uuid
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union
+
+from .exceptions import MemoriError
+
+T = TypeVar("T")
+
+
+class StringUtils:
+    """String manipulation utilities"""
+
+    @staticmethod
+    def generate_id(prefix: str = "") -> str:
+        """Generate a unique ID"""
+        unique_id = str(uuid.uuid4())
+        return f"{prefix}{unique_id}" if prefix else unique_id
+
+    @staticmethod
+    def truncate_text(text: str, max_length: int = 100, suffix: str = "...") -> str:
+        """Truncate text to specified length"""
+        if len(text) <= max_length:
+            return text
+        return text[: max_length - len(suffix)] + suffix
+
+    @staticmethod
+    def sanitize_filename(filename: str) -> str:
+        """Sanitize filename for safe file system usage"""
+        import re
+
+        # Remove or replace invalid characters
+        sanitized = re.sub(r'[<>:"/\\|?*]', "_", filename)
+        # Remove leading/trailing spaces and dots
+        sanitized = sanitized.strip(" .")
+        # Ensure it's not empty
+        return sanitized or "unnamed"
+
+    @staticmethod
+    def hash_text(text: str, algorithm: str = "sha256") -> str:
+        """Generate hash of text"""
+        hash_obj = hashlib.new(algorithm)
+        hash_obj.update(text.encode("utf-8"))
+        return hash_obj.hexdigest()
+
+    @staticmethod
+    def extract_keywords(text: str, max_keywords: int = 10) -> List[str]:
+        """Extract keywords from text (simple implementation)"""
+        import re
+
+        # Simple keyword extraction - remove common stopwords
+        stopwords = {
+            "the",
+            "a",
+            "an",
+            "and",
+            "or",
+            "but",
+            "in",
+            "on",
+            "at",
+            "to",
+            "for",
+            "of",
+            "with",
+            "by",
+            "from",
+            "up",
+            "about",
+            "into",
+            "through",
+            "during",
+            "before",
+            "after",
+            "above",
+            "below",
+            "between",
+            "among",
+            "is",
+            "are",
+            "was",
+            "were",
+            "be",
+            "been",
+            "being",
+            "have",
+            "has",
+            "had",
+            "do",
+            "does",
+            "did",
+            "will",
+            "would",
+            "could",
+            "should",
+            "may",
+            "might",
+            "must",
+            "can",
+            "this",
+            "that",
+            "these",
+            "those",
+            "i",
+            "you",
+            "he",
+            "she",
+            "it",
+            "we",
+            "they",
+        }
+
+        # Extract words (alphanumeric sequences)
+        words = re.findall(r"\b[a-zA-Z]{3,}\b", text.lower())
+
+        # Filter out stopwords and get unique words
+        keywords = list({word for word in words if word not in stopwords})
+
+        # Sort by length (longer words often more meaningful)
+        keywords.sort(key=len, reverse=True)
+
+        return keywords[:max_keywords]
+
+
+class DateTimeUtils:
+    """Date and time utilities"""
+
+    @staticmethod
+    def now() -> datetime:
+        """Get current datetime"""
+        return datetime.now()
+
+    @staticmethod
+    def format_datetime(dt: datetime, format_str: str = "%Y-%m-%d %H:%M:%S") -> str:
+        """Format datetime to string"""
+        return dt.strftime(format_str)
+
+    @staticmethod
+    def parse_datetime(dt_str: str, format_str: str = "%Y-%m-%d %H:%M:%S") -> datetime:
+        """Parse datetime from string"""
+        return datetime.strptime(dt_str, format_str)
+
+    @staticmethod
+    def add_days(dt: datetime, days: int) -> datetime:
+        """Add days to datetime"""
+        return dt + timedelta(days=days)
+
+    @staticmethod
+    def subtract_days(dt: datetime, days: int) -> datetime:
+        """Subtract days from datetime"""
+        return dt - timedelta(days=days)
+
+    @staticmethod
+    def is_expired(dt: datetime, expiry_hours: int = 24) -> bool:
+        """Check if datetime is expired"""
+        return datetime.now() > dt + timedelta(hours=expiry_hours)
+
+    @staticmethod
+    def time_ago_string(dt: datetime) -> str:
+        """Generate human-readable time ago string"""
+        now = datetime.now()
+        diff = now - dt
+
+        if diff.days > 0:
+            return f"{diff.days} day{'s' if diff.days != 1 else ''} ago"
+        elif diff.seconds > 3600:
+            hours = diff.seconds // 3600
+            return f"{hours} hour{'s' if hours != 1 else ''} ago"
+        elif diff.seconds > 60:
+            minutes = diff.seconds // 60
+            return f"{minutes} minute{'s' if minutes != 1 else ''} ago"
+        else:
+            return "just now"
+
+
+class JsonUtils:
+    """JSON handling utilities"""
+
+    @staticmethod
+    def safe_loads(json_str: str, default: Any = None) -> Any:
+        """Safely load JSON string"""
+        try:
+            return json.loads(json_str) if json_str else default
+        except (json.JSONDecodeError, TypeError):
+            return default
+
+    @staticmethod
+    def safe_dumps(obj: Any, default: Any = None, indent: int = 2) -> str:
+        """Safely dump object to JSON string"""
+        try:
+            return json.dumps(obj, default=str, indent=indent, ensure_ascii=False)
+        except (TypeError, ValueError):
+            return json.dumps(default or {}, indent=indent)
+
+    @staticmethod
+    def merge_dicts(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
+        """Deep merge two dictionaries"""
+        result = base.copy()
+
+        for key, value in override.items():
+            if (
+                key in result
+                and isinstance(result[key], dict)
+                and isinstance(value, dict)
+            ):
+                result[key] = JsonUtils.merge_dicts(result[key], value)
+            else:
+                result[key] = value
+
+        return result
+
+
+class FileUtils:
+    """File handling utilities"""
+
+    @staticmethod
+    def ensure_directory(path: Union[str, Path]) -> Path:
+        """Ensure directory exists, create if not"""
+        path = Path(path)
+        path.mkdir(parents=True, exist_ok=True)
+        return path
+
+    @staticmethod
+    def safe_read_text(
+        file_path: Union[str, Path], encoding: str = "utf-8"
+    ) -> Optional[str]:
+        """Safely read text file"""
+        try:
+            return Path(file_path).read_text(encoding=encoding)
+        except (FileNotFoundError, PermissionError, UnicodeDecodeError):
+            return None
+
+    @staticmethod
+    def safe_write_text(
+        file_path: Union[str, Path], content: str, encoding: str = "utf-8"
+    ) -> bool:
+        """Safely write text file"""
+        try:
+            path = Path(file_path)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(content, encoding=encoding)
+            return True
+        except (PermissionError, UnicodeEncodeError):
+            return False
+
+    @staticmethod
+    def get_file_size(file_path: Union[str, Path]) -> int:
+        """Get file size in bytes"""
+        try:
+            return Path(file_path).stat().st_size
+        except (FileNotFoundError, PermissionError):
+            return 0
+
+    @staticmethod
+    def is_file_recent(file_path: Union[str, Path], hours: int = 24) -> bool:
+        """Check if file was modified recently"""
+        try:
+            mtime = datetime.fromtimestamp(Path(file_path).stat().st_mtime)
+            return not DateTimeUtils.is_expired(mtime, hours)
+        except (FileNotFoundError, PermissionError):
+            return False
+
+
+class RetryUtils:
+    """Retry and error handling utilities"""
+
+    @staticmethod
+    def retry_on_exception(
+        max_attempts: int = 3,
+        delay: float = 1.0,
+        backoff: float = 2.0,
+        exceptions: tuple = (Exception,),
+    ) -> Callable:
+        """Decorator for retrying function calls on exceptions"""
+
+        def decorator(func: Callable[..., T]) -> Callable[..., T]:
+            @functools.wraps(func)
+            def wrapper(*args, **kwargs) -> T:
+                last_exception = None
+
+                for attempt in range(max_attempts):
+                    try:
+                        return func(*args, **kwargs)
+                    except exceptions as e:
+                        last_exception = e
+                        if attempt < max_attempts - 1:
+                            sleep_time = delay * (backoff**attempt)
+                            import time
+
+                            time.sleep(sleep_time)
+                        continue
+
+                # If all attempts failed, raise the last exception
+                if last_exception:
+                    raise last_exception
+
+                # This shouldn't happen, but just in case
+                raise MemoriError("Retry attempts exhausted")
+
+            return wrapper
+
+        return decorator
+
+    @staticmethod
+    async def async_retry_on_exception(
+        max_attempts: int = 3,
+        delay: float = 1.0,
+        backoff: float = 2.0,
+        exceptions: tuple = (Exception,),
+    ) -> Callable:
+        """Async decorator for retrying function calls on exceptions"""
+
+        def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
+            @functools.wraps(func)
+            async def wrapper(*args, **kwargs) -> T:
+                last_exception = None
+
+                for attempt in range(max_attempts):
+                    try:
+                        return await func(*args, **kwargs)
+                    except exceptions as e:
+                        last_exception = e
+                        if attempt < max_attempts - 1:
+                            sleep_time = delay * (backoff**attempt)
+                            await asyncio.sleep(sleep_time)
+                        continue
+
+                if last_exception:
+                    raise last_exception
+
+                raise MemoriError("Async retry attempts exhausted")
+
+            return wrapper
+
+        return decorator
+
+
+class PerformanceUtils:
+    """Performance monitoring utilities"""
+
+    @staticmethod
+    def time_function(func: Callable[..., T]) -> Callable[..., T]:
+        """Decorator to time function execution"""
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> T:
+            import time
+
+            start_time = time.time()
+
+            try:
+                result = func(*args, **kwargs)
+                end_time = time.time()
+                execution_time = end_time - start_time
+
+                # Log execution time (you can customize this)
+                from .logging import get_logger
+
+                logger = get_logger("performance")
+                logger.debug(
+                    f"{func.__name__} executed in {execution_time:.4f} seconds"
+                )
+
+                return result
+            except Exception as e:
+                end_time = time.time()
+                execution_time = end_time - start_time
+
+                from .logging import get_logger
+
+                logger = get_logger("performance")
+                logger.error(
+                    f"{func.__name__} failed after {execution_time:.4f} seconds: {e}"
+                )
+
+                raise
+
+        return wrapper
+
+    @staticmethod
+    def memory_usage() -> Dict[str, float]:
+        """Get current memory usage"""
+        try:
+            import psutil
+
+            process = psutil.Process()
+            memory_info = process.memory_info()
+
+            return {
+                "rss_mb": memory_info.rss / 1024 / 1024,  # Resident Set Size
+                "vms_mb": memory_info.vms / 1024 / 1024,  # Virtual Memory Size
+                "percent": process.memory_percent(),
+            }
+        except ImportError:
+            return {"error": "psutil not available"}
+        except Exception as e:
+            return {"error": str(e)}
+
+
+class AsyncUtils:
+    """Async utilities"""
+
+    @staticmethod
+    async def run_in_executor(func: Callable[..., T], *args, **kwargs) -> T:
+        """Run synchronous function in executor"""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            None, functools.partial(func, **kwargs), *args
+        )
+
+    @staticmethod
+    async def gather_with_concurrency(limit: int, *tasks) -> List[Any]:
+        """Run tasks with concurrency limit"""
+        semaphore = asyncio.Semaphore(limit)
+
+        async def sem_task(task):
+            async with semaphore:
+                return await task
+
+        return await asyncio.gather(*[sem_task(task) for task in tasks])
+
+    @staticmethod
+    def create_task_with_timeout(coro: Awaitable[T], timeout: float) -> asyncio.Task:
+        """Create task with timeout"""
+        return asyncio.create_task(asyncio.wait_for(coro, timeout=timeout))

memoriai/utils/logging.py

@@ -0,0 +1,204 @@
+"""
+Centralized logging configuration for Memoriai
+"""
+
+import sys
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from loguru import logger
+
+from ..config.settings import LoggingSettings, LogLevel
+from .exceptions import ConfigurationError
+
+
+class LoggingManager:
+    """Centralized logging management"""
+
+    _initialized = False
+    _current_config: Optional[LoggingSettings] = None
+
+    @classmethod
+    def setup_logging(cls, settings: LoggingSettings, verbose: bool = False) -> None:
+        """Setup logging configuration"""
+        try:
+            # Remove default handler if it exists
+            if not cls._initialized:
+                logger.remove()
+
+            if verbose:
+                # When verbose mode is enabled, disable all other loggers and show only loguru
+                cls._disable_other_loggers()
+
+                # Configure console logging with DEBUG level and full formatting
+                logger.add(
+                    sys.stderr,
+                    level="DEBUG",
+                    format=settings.format,
+                    colorize=True,
+                    backtrace=True,
+                    diagnose=True,
+                )
+            else:
+                # When verbose is False, minimize loguru output to essential logs only
+                logger.add(
+                    sys.stderr,
+                    level="WARNING",  # Only show warnings and errors
+                    format="<level>{level}</level>: {message}",  # Simplified format
+                    colorize=False,
+                    backtrace=False,
+                    diagnose=False,
+                )
+
+            # Configure file logging if enabled
+            if settings.log_to_file:
+                log_path = Path(settings.log_file_path)
+                log_path.parent.mkdir(parents=True, exist_ok=True)
+
+                if settings.structured_logging:
+                    # JSON structured logging
+                    logger.add(
+                        log_path,
+                        level=settings.level.value,
+                        format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level} | {name}:{function}:{line} | {message}",
+                        rotation=settings.log_rotation,
+                        retention=settings.log_retention,
+                        compression=settings.log_compression,
+                        serialize=True,
+                    )
+                else:
+                    # Regular text logging
+                    logger.add(
+                        log_path,
+                        level=settings.level.value,
+                        format=settings.format,
+                        rotation=settings.log_rotation,
+                        retention=settings.log_retention,
+                        compression=settings.log_compression,
+                    )
+
+            cls._initialized = True
+            cls._current_config = settings
+            logger.info("Logging configuration initialized")
+
+        except Exception as e:
+            raise ConfigurationError(f"Failed to setup logging: {e}") from e
+
+    @classmethod
+    def get_logger(cls, name: str) -> "logger":
+        """Get a logger instance with the given name"""
+        return logger.bind(name=name)
+
+    @classmethod
+    def update_log_level(cls, level: LogLevel) -> None:
+        """Update the logging level"""
+        if not cls._initialized:
+            raise ConfigurationError("Logging not initialized")
+
+        try:
+            # Remove existing handlers and recreate with new level
+            logger.remove()
+
+            if cls._current_config:
+                cls._current_config.level = level
+                cls.setup_logging(cls._current_config)
+
+        except Exception as e:
+            logger.error(f"Failed to update log level: {e}")
+
+    @classmethod
+    def add_custom_handler(cls, handler_config: Dict[str, Any]) -> None:
+        """Add a custom logging handler"""
+        try:
+            logger.add(**handler_config)
+            logger.debug(f"Added custom logging handler: {handler_config}")
+        except Exception as e:
+            logger.error(f"Failed to add custom handler: {e}")
+
+    @classmethod
+    def is_initialized(cls) -> bool:
+        """Check if logging is initialized"""
+        return cls._initialized
+
+    @classmethod
+    def get_current_config(cls) -> Optional[LoggingSettings]:
+        """Get current logging configuration"""
+        return cls._current_config
+
+    @classmethod
+    def _disable_other_loggers(cls) -> None:
+        """Disable all other loggers when verbose mode is enabled"""
+        import logging
+
+        # Set the root logger to CRITICAL and disable it
+        root_logger = logging.getLogger()
+        root_logger.setLevel(logging.CRITICAL)
+        root_logger.disabled = True
+
+        # Remove all handlers from the root logger
+        for handler in root_logger.handlers[:]:
+            root_logger.removeHandler(handler)
+
+        # Disable common third-party library loggers
+        third_party_loggers = [
+            "urllib3",
+            "requests",
+            "httpx",
+            "httpcore",
+            "openai",
+            "anthropic",
+            "litellm",
+            "sqlalchemy",
+            "alembic",
+            "asyncio",
+            "concurrent.futures",
+            "charset_normalizer",
+            "certifi",
+            "idna",
+        ]
+
+        for logger_name in third_party_loggers:
+            lib_logger = logging.getLogger(logger_name)
+            lib_logger.disabled = True
+            lib_logger.setLevel(logging.CRITICAL)
+            # Remove all handlers
+            for handler in lib_logger.handlers[:]:
+                lib_logger.removeHandler(handler)
+
+        # Set all existing loggers to CRITICAL level and disable them
+        for name in list(logging.Logger.manager.loggerDict.keys()):
+            existing_logger = logging.getLogger(name)
+            existing_logger.setLevel(logging.CRITICAL)
+            existing_logger.disabled = True
+            # Remove all handlers
+            for handler in existing_logger.handlers[:]:
+                existing_logger.removeHandler(handler)
+
+        # Also disable warnings from the warnings module
+        import warnings
+
+        warnings.filterwarnings("ignore")
+
+        # Override the logging module's basicConfig to prevent new loggers
+        def disabled_basicConfig(*args, **kwargs):
+            pass
+
+        logging.basicConfig = disabled_basicConfig
+
+        # Override the getLogger function to disable new loggers immediately
+        original_getLogger = logging.getLogger
+
+        def disabled_getLogger(name=None):
+            logger_instance = original_getLogger(name)
+            logger_instance.disabled = True
+            logger_instance.setLevel(logging.CRITICAL)
+            for handler in logger_instance.handlers[:]:
+                logger_instance.removeHandler(handler)
+            return logger_instance
+
+        logging.getLogger = disabled_getLogger
+
+
+def get_logger(name: str = "memori") -> "logger":
+    """Convenience function to get a logger"""
+    return LoggingManager.get_logger(name)