pythonLogs 5.0.2__cp313-cp313-macosx_15_0_arm64.whl

pythonLogs/log_utils.py

@@ -0,0 +1,359 @@
+import errno
+import gzip
+import logging.handlers
+import os
+import shutil
+import sys
+import threading
+import time
+from datetime import datetime, timedelta, timezone as dttz
+from functools import lru_cache
+from pathlib import Path
+from typing import Callable, Optional, Set
+from zoneinfo import ZoneInfo
+from pythonLogs.constants import DEFAULT_FILE_MODE, LEVEL_MAP
+
+
+# Global cache for checked directories with thread safety and size limits
+_checked_directories: Set[str] = set()
+_directory_lock = threading.Lock()
+_max_cached_directories = 500  # Limit cache size to prevent unbounded growth
+
+
+def get_stream_handler(
+    level: int,
+    formatter: logging.Formatter,
+) -> logging.StreamHandler:
+    stream_hdlr = logging.StreamHandler()
+    stream_hdlr.setFormatter(formatter)
+    stream_hdlr.setLevel(level)
+    return stream_hdlr
+
+
+def get_logger_and_formatter(
+    name: str,
+    datefmt: str,
+    show_location: bool,
+    timezone_: str,
+) -> tuple[logging.Logger, logging.Formatter]:
+    logger = logging.getLogger(name)
+
+    # More efficient handler cleanup with context manager-like pattern
+    handlers_to_remove = list(logger.handlers)
+    for handler in handlers_to_remove:
+        try:
+            handler.close()
+        except (OSError, ValueError):
+            pass  # Ignore expected errors during cleanup
+        finally:
+            logger.removeHandler(handler)
+
+    formatt = get_format(show_location, name, timezone_)
+    formatter = logging.Formatter(formatt, datefmt=datefmt)
+    formatter.converter = get_timezone_function(timezone_)
+    return logger, formatter
+
+
+def check_filename_instance(filenames: list | tuple) -> None:
+    if not isinstance(filenames, (list, tuple)):
+        err_msg = f"Unable to parse filenames. Filename instance is not list or tuple. | {filenames}"
+        write_stderr(err_msg)
+        raise TypeError(err_msg)
+
+
+def check_directory_permissions(directory_path: str) -> None:
+    # Thread-safe check with double-checked locking pattern
+    if directory_path in _checked_directories:
+        return
+
+    with _directory_lock:
+        # Check again inside the lock to avoid race conditions
+        if directory_path in _checked_directories:
+            return
+
+        path_obj = Path(directory_path)
+
+        if path_obj.exists():
+            if not os.access(directory_path, os.W_OK | os.X_OK):
+                err_msg = f"Unable to access directory | {directory_path}"
+                write_stderr(err_msg)
+                raise PermissionError(err_msg)
+        else:
+            try:
+                path_obj.mkdir(mode=DEFAULT_FILE_MODE, parents=True, exist_ok=True)
+            except PermissionError as e:
+                err_msg = f"Unable to create directory | {directory_path}"
+                write_stderr(f"{err_msg} | {repr(e)}")
+                raise PermissionError(err_msg)
+
+        # Add to cache with size limit enforcement
+        if len(_checked_directories) >= _max_cached_directories:
+            # Remove a random entry to make space (simple eviction strategy)
+            _checked_directories.pop()
+        _checked_directories.add(directory_path)
+
+
+def remove_old_logs(logs_dir: str, days_to_keep: int) -> None:
+    if days_to_keep <= 0:
+        return
+
+    cutoff_time = datetime.now() - timedelta(days=days_to_keep)
+
+    try:
+        for file_path in Path(logs_dir).glob("*.gz"):
+            try:
+                if file_path.stat().st_mtime < cutoff_time.timestamp():
+                    file_path.unlink()
+            except (OSError, IOError) as e:
+                write_stderr(f"Unable to delete old log | {file_path} | {repr(e)}")
+    except OSError as e:
+        write_stderr(f"Unable to scan directory for old logs | {logs_dir} | {repr(e)}")
+
+
+def delete_file(path: str) -> bool:
+    """Remove the given file and returns True if the file was successfully removed"""
+    path_obj = Path(path)
+
+    try:
+        if path_obj.is_file():
+            path_obj.unlink()
+        elif path_obj.is_dir():
+            shutil.rmtree(path_obj)
+        elif path_obj.exists():
+            # Handle special files
+            path_obj.unlink()
+        else:
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
+    except OSError as e:
+        write_stderr(repr(e))
+        raise e
+    return True
+
+
+def is_older_than_x_days(path: str, days: int) -> bool:
+    """Check if a file or directory is older than the specified number of days"""
+    path_obj = Path(path)
+
+    if not path_obj.exists():
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
+
+    try:
+        if int(days) == 0:
+            cutoff_time = datetime.now()
+        else:
+            cutoff_time = datetime.now() - timedelta(days=int(days))
+    except ValueError as e:
+        write_stderr(repr(e))
+        raise e
+
+    file_time = datetime.fromtimestamp(path_obj.stat().st_mtime)
+    return file_time < cutoff_time
+
+
+# Cache stderr timezone for better performance
+@lru_cache(maxsize=1)
+def get_stderr_timezone():
+    timezone_name = os.getenv("LOG_TIMEZONE", "UTC")
+    if timezone_name.lower() == "localtime":
+        return None  # Use system local timezone
+    try:
+        return ZoneInfo(timezone_name)
+    except Exception:
+        # Fallback to local timezone if requested timezone is not available
+        return None
+
+
+def write_stderr(msg: str) -> None:
+    """Write msg to stderr with optimized timezone handling"""
+    try:
+        tz = get_stderr_timezone()
+        if tz is None:
+            # Use local timezone
+            dt = datetime.now()
+        else:
+            dt = datetime.now(dttz.utc).astimezone(tz)
+        dt_timezone = dt.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
+        sys.stderr.write(f"[{dt_timezone}]:[ERROR]:{msg}\n")
+    except (OSError, ValueError, KeyError):
+        # Fallback to simple timestamp if timezone fails
+        sys.stderr.write(f"[{datetime.now().isoformat()}]:[ERROR]:{msg}\n")
+
+
+def get_level(level: str) -> int:
+    """Get logging level using enum values"""
+    if not isinstance(level, str):
+        write_stderr(f"Unable to get log level. Setting default level to: 'INFO' ({logging.INFO})")
+        return logging.INFO
+
+    return LEVEL_MAP.get(level.lower(), logging.INFO)
+
+
+def get_log_path(directory: str, filename: str) -> str:
+    """Get log file path with optimized validation"""
+    log_file_path = str(Path(directory) / filename)
+
+    # Check directory permissions (cached)
+    check_directory_permissions(directory)
+
+    # Only validate write access to directory, not create the file
+    if not os.access(directory, os.W_OK):
+        err_message = f"Unable to write to log directory | {directory}"
+        write_stderr(err_message)
+        raise PermissionError(err_message)
+
+    return log_file_path
+
+
+@lru_cache(maxsize=32)
+def get_timezone_offset(timezone_: str) -> str:
+    """Cache timezone offset calculation with fallback for missing timezone data"""
+    if timezone_.lower() == "localtime":
+        return time.strftime("%z")
+    else:
+        try:
+            return datetime.now(ZoneInfo(timezone_)).strftime("%z")
+        except Exception:
+            # Fallback to localtime if the requested timezone is not available,
+            # This is common on Windows systems without full timezone data
+            return time.strftime("%z")
+
+
+def get_format(show_location: bool, name: str, timezone_: str) -> str:
+    """Get log format string with cached timezone offset"""
+    _debug_fmt = ""
+    _logger_name = ""
+
+    if name:
+        _logger_name = f"[{name}]:"
+
+    if show_location:
+        _debug_fmt = "[%(filename)s:%(funcName)s:%(lineno)d]:"
+
+    utc_offset = get_timezone_offset(timezone_)
+    return f"[%(asctime)s.%(msecs)03d{utc_offset}]:[%(levelname)s]:{_logger_name}{_debug_fmt}%(message)s"
+
+
+def gzip_file_with_sufix(file_path: str, sufix: str) -> str | None:
+    """gzip file with improved error handling and performance"""
+    path_obj = Path(file_path)
+
+    if not path_obj.is_file():
+        return None
+
+    # Use pathlib for cleaner path operations
+    renamed_dst = path_obj.with_name(f"{path_obj.stem}_{sufix}{path_obj.suffix}.gz")
+
+    # Windows-specific retry mechanism for file locking issues
+    max_retries = 3 if sys.platform == "win32" else 1
+    retry_delay = 0.1  # 100ms delay between retries
+
+    for attempt in range(max_retries):
+        try:
+            with open(file_path, "rb") as fin:
+                with gzip.open(renamed_dst, "wb", compresslevel=6) as fout:  # Balanced compression
+                    shutil.copyfileobj(fin, fout, length=64 * 1024)  # type: ignore # 64KB chunks for better performance
+            break  # Success, exit retry loop
+        except PermissionError as e:
+            # Windows file locking issue - retry with delay
+            if attempt < max_retries - 1 and sys.platform == "win32":
+                time.sleep(retry_delay)
+                continue
+            # Final attempt failed or not Windows - treat as regular error
+            write_stderr(f"Unable to gzip log file | {file_path} | {repr(e)}")
+            raise e
+        except (OSError, IOError) as e:
+            write_stderr(f"Unable to gzip log file | {file_path} | {repr(e)}")
+            raise e
+
+    try:
+        path_obj.unlink()  # Use pathlib for deletion
+    except OSError as e:
+        write_stderr(f"Unable to delete source log file | {file_path} | {repr(e)}")
+        raise e
+
+    return str(renamed_dst)
+
+
+@lru_cache(maxsize=32)
+def get_timezone_function(time_zone: str) -> Callable:
+    """Get timezone function with caching and fallback for missing timezone data"""
+    match time_zone.lower():
+        case "utc":
+            try:
+                # Try to create UTC timezone to verify it's available
+                ZoneInfo("UTC")
+                return time.gmtime
+            except Exception:
+                # Fallback to localtime if UTC timezone data is missing
+                return time.localtime
+        case "localtime":
+            return time.localtime
+        case _:
+            try:
+                # Cache the timezone object
+                tz = ZoneInfo(time_zone)
+                return lambda *args: datetime.now(tz=tz).timetuple()
+            except Exception:
+                # Fallback to localtime if the requested timezone is not available
+                return time.localtime
+
+
+# Shared handler cleanup utility
+def cleanup_logger_handlers(logger: Optional[logging.Logger]) -> None:
+    """Clean up logger resources by closing all handlers.
+
+    This is a centralized utility to ensure consistent cleanup behavior
+    across all logger types and prevent code duplication.
+
+    Args:
+        logger: The logger to clean up (can be None)
+    """
+    if logger is None:
+        return
+
+    # Create a snapshot of handlers to avoid modification during iteration
+    handlers_to_remove = list(logger.handlers)
+    for handler in handlers_to_remove:
+        try:
+            handler.close()
+        except (OSError, ValueError):
+            # Ignore errors during cleanup to prevent cascading failures
+            pass
+        finally:
+            logger.removeHandler(handler)
+
+
+# Public API for directory cache management
+def set_directory_cache_limit(max_directories: int) -> None:
+    """Set the maximum number of directories to cache.
+
+    Args:
+        max_directories: Maximum number of directories to keep in cache
+    """
+    global _max_cached_directories
+    
+    with _directory_lock:
+        _max_cached_directories = max_directories
+        # Trim cache if it exceeds new limit
+        while len(_checked_directories) > max_directories:
+            _checked_directories.pop()
+
+
+def clear_directory_cache() -> None:
+    """Clear the directory cache to free memory."""
+    with _directory_lock:
+        _checked_directories.clear()
+
+
+def get_directory_cache_stats() -> dict:
+    """Get statistics about the directory cache.
+    
+    Returns:
+        Dict with cache statistics including size and limit
+    """
+    with _directory_lock:
+        return {
+            "cached_directories": len(_checked_directories),
+            "max_directories": _max_cached_directories,
+            "directories": list(_checked_directories)
+        }

pythonLogs/memory_utils.py

@@ -0,0 +1,182 @@
+import logging
+import threading
+import weakref
+from functools import lru_cache
+from typing import Any, Dict, Optional, Set
+
+from . import log_utils
+from .log_utils import cleanup_logger_handlers
+
+
+
+# Formatter cache to reduce memory usage for identical formatters
+_formatter_cache: Dict[str, logging.Formatter] = {}
+_formatter_cache_lock = threading.Lock()
+_max_formatters = 50  # Limit formatter cache size
+
+
+def get_cached_formatter(format_string: str, datefmt: Optional[str] = None) -> logging.Formatter:
+    """Get a cached formatter or create and cache a new one.
+
+    This reduces memory usage by reusing formatter instances with
+    identical configuration instead of creating new ones each time.
+
+    Args:
+        format_string: The format string for the formatter
+        datefmt: Optional date format string
+
+    Returns:
+        Cached or newly created formatter instance
+    """
+    # Create cache key from configuration
+    cache_key = f"{format_string}|{datefmt or ''}"
+
+    with _formatter_cache_lock:
+        # Return existing formatter if cached
+        if cache_key in _formatter_cache:
+            return _formatter_cache[cache_key]
+
+        # Enforce cache size limit
+        if len(_formatter_cache) >= _max_formatters:
+            # Remove the oldest entry (FIFO eviction)
+            oldest_key = next(iter(_formatter_cache))
+            _formatter_cache.pop(oldest_key)
+
+        # Create and cache new formatter
+        formatter = logging.Formatter(fmt=format_string, datefmt=datefmt)
+        _formatter_cache[cache_key] = formatter
+        return formatter
+
+
+def clear_formatter_cache() -> None:
+    """Clear the formatter cache to free memory."""
+    with _formatter_cache_lock:
+        _formatter_cache.clear()
+
+
+# Directory cache utilities with memory management
+def set_directory_cache_limit(max_directories: int) -> None:
+    """Set the maximum number of directories to cache.
+
+    Args:
+        max_directories: Maximum number of directories to keep in cache
+    """
+    log_utils.set_directory_cache_limit(max_directories)
+
+
+def clear_directory_cache() -> None:
+    """Clear the directory cache to free memory."""
+    log_utils.clear_directory_cache()
+
+
+# Weak reference registry for tracking active loggers without preventing GC
+_active_loggers: Set[weakref.ReferenceType] = set()
+_weak_ref_lock = threading.Lock()
+
+
+def register_logger_weakref(logger: logging.Logger) -> None:
+    """Register a weak reference to a logger for memory tracking.
+
+    This allows monitoring active loggers without preventing garbage collection.
+
+    Args:
+        logger: Logger to track
+    """
+    global _active_loggers
+
+    def cleanup_callback(ref):
+        with _weak_ref_lock:
+            _active_loggers.discard(ref)
+
+    with _weak_ref_lock:
+        weak_ref = weakref.ref(logger, cleanup_callback)
+        _active_loggers.add(weak_ref)
+
+
+def get_active_logger_count() -> int:
+    """Get the count of currently active loggers.
+
+    Returns:
+        Number of active logger instances
+    """
+    global _active_loggers
+    with _weak_ref_lock:
+        # Clean up dead references
+        dead_refs = {ref for ref in _active_loggers if ref() is None}
+        _active_loggers -= dead_refs
+        return len(_active_loggers)
+
+
+def get_memory_stats() -> Dict[str, Any]:
+    """Get memory usage statistics for the logging system.
+
+    Returns:
+        Dictionary containing memory usage statistics
+    """
+    from . import factory
+    
+    # Get registry stats using public API
+    registered_loggers = factory.LoggerFactory.get_registered_loggers()
+    registry_size = len(registered_loggers)
+    
+    # Get memory limits using public API
+    factory_limits = factory.LoggerFactory.get_memory_limits()
+
+    with _formatter_cache_lock:
+        formatter_cache_size = len(_formatter_cache)
+
+    # Get directory cache stats using public API
+    directory_stats = log_utils.get_directory_cache_stats()
+
+    return {
+        'registry_size': registry_size,
+        'formatter_cache_size': formatter_cache_size,
+        'directory_cache_size': directory_stats['cached_directories'],
+        'active_logger_count': get_active_logger_count(),
+        'max_registry_size': factory_limits['max_loggers'],
+        'max_formatter_cache': _max_formatters,
+        'max_directory_cache': directory_stats['max_directories'],
+    }
+
+
+# LRU cache size optimization
+def optimize_lru_cache_sizes() -> None:
+    """Optimize LRU cache sizes based on typical usage patterns."""
+    # Clear existing caches and reduce their sizes
+    
+    # Clear and recreate timezone function cache with smaller size
+    log_utils.get_timezone_function.cache_clear()
+    log_utils.get_timezone_function = lru_cache(maxsize=8)(log_utils.get_timezone_function.__wrapped__)
+
+    # Clear and recreate timezone offset cache with smaller size
+    log_utils.get_timezone_offset.cache_clear()
+    log_utils.get_timezone_offset = lru_cache(maxsize=8)(log_utils.get_timezone_offset.__wrapped__)
+
+    # Clear and recreate stderr timezone cache with smaller size
+    log_utils.get_stderr_timezone.cache_clear()
+    log_utils.get_stderr_timezone = lru_cache(maxsize=4)(log_utils.get_stderr_timezone.__wrapped__)
+
+
+def force_garbage_collection() -> Dict[str, int]:
+    """Force garbage collection and return collection statistics.
+
+    This can be useful for testing memory leaks or forcing cleanup
+    in long-running applications.
+
+    Returns:
+        Dictionary with garbage collection statistics
+    """
+    import gc
+
+    # Clear all our caches first using public APIs
+    clear_formatter_cache()
+    clear_directory_cache()
+
+    # Force garbage collection
+    collected = gc.collect()
+
+    return {
+        'objects_collected': collected,
+        'garbage_count': len(gc.garbage),
+        'reference_cycles': gc.get_count(),
+    }

pythonLogs/settings.py

@@ -0,0 +1,56 @@
+from functools import lru_cache
+from typing import Optional
+from dotenv import load_dotenv
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pythonLogs.constants import (
+    DEFAULT_BACKUP_COUNT,
+    DEFAULT_DATE_FORMAT,
+    DEFAULT_ENCODING,
+    DEFAULT_ROTATE_SUFFIX,
+    DEFAULT_TIMEZONE,
+    LogLevel,
+    RotateWhen,
+)
+
+
+# Lazy loading flag for dotenv
+_dotenv_loaded = False
+
+
+class LogSettings(BaseSettings):
+    """If any ENV variable is omitted, it falls back to default values here"""
+
+    level: Optional[LogLevel] = Field(default=LogLevel.INFO)
+    timezone: Optional[str] = Field(default=DEFAULT_TIMEZONE)
+    encoding: Optional[str] = Field(default=DEFAULT_ENCODING)
+    appname: Optional[str] = Field(default="app")
+    filename: Optional[str] = Field(default="app.log")
+    directory: Optional[str] = Field(default="./logs")
+    days_to_keep: Optional[int] = Field(default=DEFAULT_BACKUP_COUNT)
+    date_format: Optional[str] = Field(default=DEFAULT_DATE_FORMAT)
+    stream_handler: Optional[bool] = Field(default=True)
+    show_location: Optional[bool] = Field(default=False)
+    # Memory management
+    max_loggers: Optional[int] = Field(default=100)
+    logger_ttl_seconds: Optional[int] = Field(default=3600)
+
+    # SizeRotatingLog
+    max_file_size_mb: Optional[int] = Field(default=10)
+
+    # TimedRotatingLog
+    rotate_when: Optional[RotateWhen] = Field(default=RotateWhen.MIDNIGHT)
+    rotate_at_utc: Optional[bool] = Field(default=True)
+    rotate_file_sufix: Optional[str] = Field(default=DEFAULT_ROTATE_SUFFIX)
+
+    model_config = SettingsConfigDict(env_prefix="LOG_", env_file=".env", extra="allow")
+
+
+@lru_cache(maxsize=1)
+def get_log_settings() -> LogSettings:
+    """Get cached log settings instance to avoid repeated instantiation"""
+    global _dotenv_loaded
+    if not _dotenv_loaded:
+        load_dotenv()
+        _dotenv_loaded = True
+    return LogSettings()