aegis-stack 0.2.0rc2__py3-none-any.whl

aegis/templates/cookiecutter-aegis-project/{{cookiecutter.project_slug}}/app/services/system/health.py.j2

@@ -0,0 +1,1333 @@
+"""
+System health monitoring functions.
+
+Pure functions for system health checking, monitoring, and status reporting.
+All functions use Pydantic models for type safety and validation.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from datetime import UTC, datetime
+import os
+import sqlite3
+import sys
+from typing import Any, cast
+
+import psutil
+
+from app.core.config import settings
+from app.core.log import logger
+
+from .alerts import send_critical_alert, send_health_alert
+from .models import ComponentStatus, ComponentStatusType, SystemStatus
+
+# Global registry for custom health checks
+_health_checks: dict[str, Callable[[], Awaitable[ComponentStatus]]] = {}
+
+# Global registry for service health checks
+_service_health_checks: dict[str, Callable[[], Awaitable[ComponentStatus]]] = {}
+
+
+def format_bytes(size: int) -> str:
+    """Format bytes into human-readable string."""
+    if size == 0:
+        return "0 B"
+
+    size_float = float(size)
+    for unit in ['B', 'KB', 'MB', 'GB']:
+        if size_float < 1024.0:
+            if unit == 'B':
+                return f"{int(size_float)} {unit}"
+            else:
+                return f"{size_float:.1f} {unit}"
+        size_float /= 1024.0
+    return f"{size_float:.1f} TB"
+
+
+def propagate_status(child_statuses: list[ComponentStatusType]) -> ComponentStatusType:
+    """
+    Determine parent status from child statuses using standard hierarchy.
+
+    Status priority (highest to lowest):
+    1. UNHEALTHY - Any unhealthy child makes parent unhealthy
+    2. WARNING - Any warning child makes parent warning (if no unhealthy)
+    3. INFO - Any info child makes parent info (if no unhealthy/warning)
+    4. HEALTHY - All children healthy makes parent healthy
+
+    Args:
+        child_statuses: List of ComponentStatusType from child components
+
+    Returns:
+        ComponentStatusType for the parent component
+    """
+    if not child_statuses:
+        return ComponentStatusType.HEALTHY
+
+    if any(status == ComponentStatusType.UNHEALTHY for status in child_statuses):
+        return ComponentStatusType.UNHEALTHY
+    elif any(status == ComponentStatusType.WARNING for status in child_statuses):
+        return ComponentStatusType.WARNING
+    elif any(status == ComponentStatusType.INFO for status in child_statuses):
+        return ComponentStatusType.INFO
+    elif all(status == ComponentStatusType.HEALTHY for status in child_statuses):
+        return ComponentStatusType.HEALTHY
+    else:
+        return ComponentStatusType.HEALTHY  # Default for edge cases
+
+# Cache for system metrics to improve performance
+_system_metrics_cache: dict[str, tuple[ComponentStatus, datetime]] = {}
+
+
+def register_health_check(
+    name: str, check_fn: Callable[[], Awaitable[ComponentStatus]]
+) -> None:
+    """
+    Register a custom health check function.
+
+    Args:
+        name: Unique name for the health check
+        check_fn: Async function that returns ComponentStatus or bool
+    """
+    _health_checks[name] = check_fn
+    logger.info(f"Registered custom health check: {name}")
+
+
+def register_service_health_check(
+    name: str, check_fn: Callable[[], Awaitable[ComponentStatus]]
+) -> None:
+    """
+    Register a service health check function.
+
+    Args:
+        name: Unique name for the service health check
+        check_fn: Async function that returns ComponentStatus
+    """
+    _service_health_checks[name] = check_fn
+    logger.info(f"Registered service health check: {name}")
+
+
+async def get_system_status() -> SystemStatus:
+    """
+    Get comprehensive system status.
+
+    Returns:
+        SystemStatus with all component health information organized as Aegis tree
+    """
+    start_time = datetime.now(UTC)
+
+    # Run custom component checks (these are top-level components)
+    component_results = {}
+    component_tasks = []
+    for name, check_fn in _health_checks.items():
+        task = asyncio.create_task(_run_health_check(name, check_fn))
+        component_tasks.append((name, task))
+
+    # Collect component results
+    for name, task in component_tasks:
+        try:
+            component_results[name] = await task
+        except Exception as e:
+            logger.error(f"Component check failed for {name}: {e}")
+            component_results[name] = ComponentStatus(
+                name=name,
+                status=ComponentStatusType.UNHEALTHY,
+                message=f"Health check failed: {str(e)}",
+                response_time_ms=None,
+            )
+
+    # Run service health checks
+    service_results = {}
+    service_tasks = []
+    for name, check_fn in _service_health_checks.items():
+        task = asyncio.create_task(_run_health_check(name, check_fn))
+        service_tasks.append((name, task))
+
+    # Collect service results
+    for name, task in service_tasks:
+        try:
+            service_results[name] = await task
+        except Exception as e:
+            logger.error(f"Service check failed for {name}: {e}")
+            service_results[name] = ComponentStatus(
+                name=name,
+                status=ComponentStatusType.UNHEALTHY,
+                message=f"Service health check failed: {str(e)}",
+                response_time_ms=None,
+            )
+
+    # Get system metrics (with caching for performance)
+    system_metrics = await _get_cached_system_metrics(start_time)
+
+    # Group system metrics under backend component if it exists
+    if "backend" in component_results:
+        # Backend exists - recreate with system metrics as sub-components
+        backend_component = component_results["backend"]
+
+        # Propagate status from system metrics and original backend status
+        system_metrics_statuses = [
+            getattr(metric, 'status', ComponentStatusType.HEALTHY)
+            for metric in system_metrics.values()
+        ]
+        original_backend_status = getattr(
+            backend_component, 'status', ComponentStatusType.HEALTHY
+        )
+        all_backend_statuses = system_metrics_statuses + [original_backend_status]
+
+        backend_status = propagate_status(all_backend_statuses)
+
+        component_results["backend"] = ComponentStatus(
+            name=backend_component.name,
+            status=backend_status,
+            message=backend_component.message,
+            response_time_ms=backend_component.response_time_ms,
+            metadata=backend_component.metadata,
+            sub_components=system_metrics,
+        )
+    else:
+        # Backend doesn't exist - create a virtual backend component to hold
+        # system metrics
+        backend_healthy = all(metric.healthy for metric in system_metrics.values())
+
+        # Propagate status from system metrics only
+        system_metrics_statuses = [
+            getattr(metric, 'status', ComponentStatusType.HEALTHY)
+            for metric in system_metrics.values()
+        ]
+        backend_status = propagate_status(system_metrics_statuses)
+
+        backend_message = (
+            "System container metrics"
+            if backend_healthy
+            else "System container has issues"
+        )
+
+        component_results["backend"] = ComponentStatus(
+            name="backend",
+            status=backend_status,
+            message=backend_message,
+            response_time_ms=None,
+            metadata={"type": "system_container", "virtual": True},
+            sub_components=system_metrics,
+        )
+
+    # Calculate overall health (including sub-components and services)
+    all_statuses = list(component_results.values()) + list(service_results.values())
+    for component in component_results.values():
+        all_statuses.extend(component.sub_components.values())
+    overall_healthy = all(status.healthy for status in all_statuses)
+
+    # Create Aegis root structure with components underneath
+    aegis_healthy = all(status.healthy for status in all_statuses)
+
+    # Propagate status from all top-level components and services
+    component_statuses = [
+        getattr(component, 'status', ComponentStatusType.HEALTHY)
+        for component in component_results.values()
+    ]
+    service_statuses = [
+        getattr(service, 'status', ComponentStatusType.HEALTHY)
+        for service in service_results.values()
+    ]
+    all_top_level_statuses = component_statuses + service_statuses
+    aegis_status = propagate_status(all_top_level_statuses)
+
+    aegis_message = (
+        "Aegis Stack application" if aegis_healthy else "Aegis Stack has issues"
+    )
+
+    # Create aegis sub-components structure with components and services grouped
+    aegis_sub_components = {}
+
+    # Group all components under "components" if any exist
+    if component_results:
+        components_status = propagate_status(component_statuses)
+        components_healthy = all(
+            component.healthy for component in component_results.values()
+        )
+        components_message = (
+            f"{len(component_results)} components available"
+            if components_healthy
+            else "Some components have issues"
+        )
+
+        aegis_sub_components["components"] = ComponentStatus(
+            name="components",
+            status=components_status,
+            message=components_message,
+            response_time_ms=None,
+            metadata={
+                "type": "components_group",
+                "total_components": len(component_results),
+                "component_names": list(component_results.keys()),
+            },
+            sub_components=component_results,
+        )
+
+    # Add services as a sub-component if any services are registered
+    if service_results:
+        # Determine services component status
+        services_status = propagate_status(service_statuses)
+        services_healthy = all(service.healthy for service in service_results.values())
+        services_message = (
+            f"{len(service_results)} services available"
+            if services_healthy
+            else "Some services have issues"
+        )
+
+        aegis_sub_components["services"] = ComponentStatus(
+            name="services",
+            status=services_status,
+            message=services_message,
+            response_time_ms=None,
+            metadata={
+                "type": "services_group",
+                "total_services": len(service_results),
+                "service_names": list(service_results.keys()),
+            },
+            sub_components=service_results,
+        )
+
+    root_components = {
+        "aegis": ComponentStatus(
+            name="aegis",
+            status=aegis_status,
+            message=aegis_message,
+            response_time_ms=None,
+            metadata={"type": "application_root", "version": "1.0"},
+            sub_components=aegis_sub_components,
+        )
+    }
+
+    # Get system information
+    system_info = _get_system_info()
+
+    status = SystemStatus(
+        components=root_components,
+        overall_healthy=overall_healthy,
+        timestamp=start_time,
+        system_info=system_info,
+    )
+
+    # Log unhealthy components
+    if not overall_healthy:
+        logger.warning(
+            f"System unhealthy: {status.unhealthy_components}",
+            extra={"unhealthy_components": status.unhealthy_components},
+        )
+
+    return status
+
+
+async def is_system_healthy() -> bool:
+    """Quick check if system is overall healthy."""
+    status = await get_system_status()
+    return status.overall_healthy
+
+
+async def check_system_status() -> None:
+    """
+    Scheduled health check function for use in APScheduler jobs.
+
+    This function gets the system status and logs any issues.
+    Can be extended to send alerts to Slack, email, etc.
+    """
+    logger.info("🩺 Running scheduled system health check")
+
+    try:
+        status = await get_system_status()
+
+        if status.overall_healthy:
+            log_msg = (
+                f"✅ System healthy: {len(status.healthy_top_level_components)}/"
+                f"{status.total_components} components OK"
+            )
+            logger.info(log_msg)
+        else:
+            logger.warning(
+                f"⚠️ System issues detected: "
+                f"{len(status.unhealthy_components)} unhealthy components",
+                extra={
+                    "unhealthy_components": status.unhealthy_components,
+                    "health_percentage": status.health_percentage,
+                },
+            )
+
+            # Log details for each unhealthy component
+            for component_name in status.unhealthy_components:
+                component = status.components[component_name]
+                logger.error(
+                    f"❌ {component_name}: {component.message}",
+                    extra={"component": component.name, "metadata": component.metadata},
+                )
+
+            # Send health alerts
+            await send_health_alert(status)
+
+    except Exception as e:
+        logger.error(f"💥 System health check failed: {e}")
+        # Send critical alert about monitoring failure
+        await send_critical_alert(f"Health monitoring failed: {e}", str(e))
+
+
+async def _get_cached_system_metrics(
+    current_time: datetime,
+) -> dict[str, ComponentStatus]:
+    """Get system metrics with caching for better performance."""
+    cache_duration = settings.SYSTEM_METRICS_CACHE_SECONDS
+    system_metric_checks = {
+        "memory": _check_memory,
+        "disk": _check_disk_space,
+        "cpu": _check_cpu_usage,
+    }
+
+    system_metrics = {}
+    tasks = []
+
+    for name, check_fn in system_metric_checks.items():
+        # Check if we have a valid cached result
+        if name in _system_metrics_cache:
+            cached_result, cached_time = _system_metrics_cache[name]
+            age_seconds = (current_time - cached_time).total_seconds()
+
+            if age_seconds < cache_duration:
+                # Use cached result
+                system_metrics[name] = cached_result
+                continue
+
+        # Need to run the check
+        task = asyncio.create_task(
+            _run_health_check_with_cache(name, check_fn, current_time)
+        )
+        tasks.append((name, task))
+
+    # Collect results from non-cached checks
+    for name, task in tasks:
+        try:
+            system_metrics[name] = await task
+        except Exception as e:
+            logger.error(f"System metric check failed for {name}: {e}")
+            system_metrics[name] = ComponentStatus(
+                name=name,
+                status=ComponentStatusType.UNHEALTHY,
+                message=f"Health check failed: {str(e)}",
+                response_time_ms=None,
+            )
+
+    return system_metrics
+
+
+async def _run_health_check_with_cache(
+    name: str, check_fn: Callable[[], Awaitable[ComponentStatus]], timestamp: datetime
+) -> ComponentStatus:
+    """Run health check and cache the result."""
+    result = await _run_health_check(name, check_fn)
+    _system_metrics_cache[name] = (result, timestamp)
+    return result
+
+
+async def _run_health_check(
+    name: str, check_fn: Callable[[], Awaitable[ComponentStatus]]
+) -> ComponentStatus:
+    """Run a single health check with timing."""
+    start_time = datetime.now(UTC)
+    try:
+        result = await check_fn()
+        end_time = datetime.now(UTC)
+        response_time = (end_time - start_time).total_seconds() * 1000
+
+        if isinstance(result, ComponentStatus):
+            result.response_time_ms = response_time
+            return result
+        else:
+            return ComponentStatus(
+                name=name,
+                status=(
+                    ComponentStatusType.HEALTHY if bool(result)
+                    else ComponentStatusType.UNHEALTHY
+                ),
+                message="OK" if result else "Failed",
+                response_time_ms=response_time,
+            )
+    except Exception as e:
+        end_time = datetime.now(UTC)
+        response_time = (end_time - start_time).total_seconds() * 1000
+        return ComponentStatus(
+            name=name,
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Error: {str(e)}",
+            response_time_ms=response_time,
+        )
+
+
+def _get_system_info() -> dict[str, Any]:
+    """Get general system information."""
+    try:
+        return {
+            "python_version": (
+                f"{sys.version_info.major}."
+                f"{sys.version_info.minor}."
+                f"{sys.version_info.micro}"
+            ),
+            "platform": psutil.WINDOWS if psutil.WINDOWS else "unix",
+            "containerized": "docker" if os.path.exists("/.dockerenv") else "false",
+        }
+    except Exception as e:
+        logger.warning(f"Failed to get system info: {e}")
+        return {"error": str(e)}
+
+
+async def _check_memory() -> ComponentStatus:
+    """Check system memory usage."""
+    try:
+        # Run in thread to avoid blocking
+        memory = await asyncio.to_thread(psutil.virtual_memory)
+        memory_percent = memory.percent
+
+        # Determine status based on memory usage thresholds
+        if memory_percent >= settings.MEMORY_THRESHOLD_PERCENT:
+            status = ComponentStatusType.UNHEALTHY
+        elif memory_percent >= settings.MEMORY_THRESHOLD_PERCENT * 0.8:
+            status = ComponentStatusType.WARNING
+        else:
+            status = ComponentStatusType.HEALTHY
+
+        return ComponentStatus(
+            name="memory",
+            status=status,
+            message=f"Memory usage: {memory_percent:.1f}%",
+            response_time_ms=None,
+            metadata={
+                "percent_used": memory_percent,
+                "total_gb": round(memory.total / (1024**3), 2),
+                "available_gb": round(memory.available / (1024**3), 2),
+                "threshold_percent": settings.MEMORY_THRESHOLD_PERCENT,
+            },
+        )
+    except Exception as e:
+        return ComponentStatus(
+            name="memory",
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Failed to check memory: {e}",
+            response_time_ms=None,
+        )
+
+
+async def _check_disk_space() -> ComponentStatus:
+    """Check disk space usage."""
+    try:
+        # Run in thread to avoid blocking
+        disk = await asyncio.to_thread(psutil.disk_usage, "/")
+        disk_percent = (disk.used / disk.total) * 100
+
+        # Determine status based on disk usage thresholds
+        if disk_percent >= settings.DISK_THRESHOLD_PERCENT:
+            status = ComponentStatusType.UNHEALTHY
+        elif disk_percent >= settings.DISK_THRESHOLD_PERCENT * 0.8:
+            status = ComponentStatusType.WARNING
+        else:
+            status = ComponentStatusType.HEALTHY
+
+        return ComponentStatus(
+            name="disk",
+            status=status,
+            message=f"Disk usage: {disk_percent:.1f}%",
+            response_time_ms=None,
+            metadata={
+                "percent_used": disk_percent,
+                "total_gb": round(disk.total / (1024**3), 2),
+                "free_gb": round(disk.free / (1024**3), 2),
+                "threshold_percent": settings.DISK_THRESHOLD_PERCENT,
+            },
+        )
+    except Exception as e:
+        return ComponentStatus(
+            name="disk",
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Failed to check disk space: {e}",
+            response_time_ms=None,
+        )
+
+
+async def _check_cpu_usage() -> ComponentStatus:
+    """Check CPU usage (instant sampling)."""
+    try:
+        # Get instant CPU usage (non-blocking, immediate reading)
+        cpu_percent = await asyncio.to_thread(psutil.cpu_percent, None)
+
+        # Determine status based on CPU usage thresholds
+        if cpu_percent >= settings.CPU_THRESHOLD_PERCENT:
+            status = ComponentStatusType.UNHEALTHY
+        elif cpu_percent >= settings.CPU_THRESHOLD_PERCENT * 0.8:
+            status = ComponentStatusType.WARNING
+        else:
+            status = ComponentStatusType.HEALTHY
+
+        return ComponentStatus(
+            name="cpu",
+            status=status,
+            message=f"CPU usage: {cpu_percent:.1f}%",
+            response_time_ms=None,
+            metadata={
+                "percent_used": cpu_percent,
+                "cpu_count": psutil.cpu_count(),
+                "threshold_percent": settings.CPU_THRESHOLD_PERCENT,
+            },
+        )
+    except Exception as e:
+        return ComponentStatus(
+            name="cpu",
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Failed to check CPU usage: {e}",
+            response_time_ms=None,
+        )
+
+
+{% if cookiecutter.include_redis == "yes" %}
+async def check_cache_health() -> ComponentStatus:
+    """
+    Check cache connectivity and basic functionality.
+
+    Returns:
+        ComponentStatus indicating cache health
+    """
+    try:
+        import redis.asyncio as aioredis
+
+        # Create Redis connection with timeout
+        redis_url = (
+            settings.redis_url_effective 
+            if hasattr(settings, 'redis_url_effective') 
+            else settings.REDIS_URL
+        )
+        redis_connection = aioredis.from_url(  # type: ignore[no-untyped-call]
+            redis_url,
+            db=settings.REDIS_DB,
+            socket_timeout=settings.HEALTH_CHECK_TIMEOUT_SECONDS,
+            socket_connect_timeout=settings.HEALTH_CHECK_TIMEOUT_SECONDS,
+        )
+        redis_client: aioredis.Redis = cast(aioredis.Redis, redis_connection)
+
+        start_time = datetime.now(UTC)
+
+        # Test basic connectivity with ping
+        await redis_client.ping()
+
+        # Test basic set/get functionality
+        test_key = "health_check:test"
+        test_value = f"test_{start_time.timestamp()}"
+        await redis_client.set(test_key, test_value, ex=10)  # Expire in 10 seconds
+        retrieved_value = await redis_client.get(test_key)
+
+        # Cleanup test key
+        await redis_client.delete(test_key)
+        await redis_client.aclose()
+
+        # Verify test worked
+        if retrieved_value.decode() != test_value:
+            raise Exception("Redis set/get test failed")
+
+        # Get comprehensive Redis info for detailed monitoring
+        redis_info_connection = aioredis.from_url(  # type: ignore[no-untyped-call]
+            redis_url, db=settings.REDIS_DB
+        )
+        redis_info_client: aioredis.Redis = cast(aioredis.Redis, redis_info_connection)
+        
+        # Get multiple INFO sections for comprehensive metrics
+        info = await redis_info_client.info()
+        stats_info = await redis_info_client.info('stats')
+        memory_info = await redis_info_client.info('memory')
+        clients_info = await redis_info_client.info('clients')
+        keyspace_info = await redis_info_client.info('keyspace')
+        
+        await redis_info_client.aclose()
+        
+        # Calculate derived metrics
+        keyspace_hits = stats_info.get('keyspace_hits', 0)
+        keyspace_misses = stats_info.get('keyspace_misses', 0)
+        total_keyspace_ops = keyspace_hits + keyspace_misses
+        hit_rate = (keyspace_hits / max(total_keyspace_ops, 1)) * 100
+        
+        # Extract total keys from all databases
+        total_keys = 0
+        keys_with_expiry = 0
+        for key, value in keyspace_info.items():
+            if key.startswith('db'):
+                # Redis info('keyspace') returns nested dict format
+                if isinstance(value, dict):
+                    total_keys += value.get('keys', 0)
+                    keys_with_expiry += value.get('expires', 0)
+        
+        # Memory usage calculations
+        used_memory = memory_info.get('used_memory', 0)
+        used_memory_peak = memory_info.get('used_memory_peak', 0)
+        mem_fragmentation_ratio = memory_info.get('mem_fragmentation_ratio', 1.0)
+        
+        return ComponentStatus(
+            name="cache",
+            status=ComponentStatusType.HEALTHY,
+            message="Redis cache connection and operations successful",
+            response_time_ms=None,  # Will be set by caller
+            metadata={
+                "implementation": "redis",
+                "version": info.get("redis_version", "unknown"),
+                "url": redis_url,
+                "db": settings.REDIS_DB,
+                
+                # Connection and client metrics
+                "connected_clients": clients_info.get("connected_clients", 0),
+                "blocked_clients": clients_info.get("blocked_clients", 0),
+                "client_longest_output_list": clients_info.get(
+                    "client_longest_output_list", 0
+                ),
+                
+                # Server uptime
+                "uptime_in_seconds": info.get("uptime_in_seconds", 0),
+                
+                # Memory metrics
+                "used_memory": used_memory,
+                "used_memory_human": memory_info.get("used_memory_human", "unknown"),
+                "used_memory_peak": used_memory_peak,
+                "used_memory_peak_human": memory_info.get(
+                    "used_memory_peak_human", "unknown"
+                ),
+                "mem_fragmentation_ratio": mem_fragmentation_ratio,
+                "maxmemory": memory_info.get("maxmemory", 0),
+                "maxmemory_human": memory_info.get("maxmemory_human", "0B"),
+                
+                # Performance and cache metrics
+                "instantaneous_ops_per_sec": stats_info.get(
+                    "instantaneous_ops_per_sec", 0
+                ),
+                "keyspace_hits": keyspace_hits,
+                "keyspace_misses": keyspace_misses,
+                "hit_rate_percent": hit_rate,
+                "evicted_keys": stats_info.get("evicted_keys", 0),
+                "expired_keys": stats_info.get("expired_keys", 0),
+                
+                # Keyspace statistics
+                "total_keys": total_keys,
+                "keys_with_expiry": keys_with_expiry,
+                
+                # Additional useful stats
+                "total_commands_processed": stats_info.get(
+                    "total_commands_processed", 0
+                ),
+                "total_connections_received": stats_info.get(
+                    "total_connections_received", 0
+                ),
+                "rejected_connections": stats_info.get("rejected_connections", 0),
+            },
+        )
+
+    except ImportError:
+        return ComponentStatus(
+            name="cache",
+            status=ComponentStatusType.UNHEALTHY,
+            message="Cache library not installed",
+            response_time_ms=None,
+            metadata={
+                "implementation": "redis",
+                "error": "Redis library not available",
+            },
+        )
+    except Exception as e:
+        return ComponentStatus(
+            name="cache",
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Cache health check failed: {str(e)}",
+            response_time_ms=None,
+            metadata={
+                "implementation": "redis",
+                "url": (
+                    settings.redis_url_effective 
+                    if hasattr(settings, 'redis_url_effective') 
+                    else settings.REDIS_URL
+                ),
+                "db": settings.REDIS_DB,
+                "error": str(e),
+                # Provide fallback values for card display
+                "connected_clients": 0,
+                "used_memory_human": "unknown",
+                "uptime_in_seconds": 0,
+                "instantaneous_ops_per_sec": 0,
+                "hit_rate_percent": 0,
+                "total_keys": 0,
+                "evicted_keys": 0,
+                "expired_keys": 0,
+            },
+        )
+{% endif %}
+
+
+{% if cookiecutter.include_database == "yes" %}
+async def check_database_health() -> ComponentStatus:
+    """
+    Check database connectivity and basic functionality.
+
+    Returns:
+        ComponentStatus indicating database health
+    """
+    try:
+        # Import db_session from generated project
+        from app.core.db import db_session
+        from pathlib import Path
+
+        # Check if database file exists for SQLite
+        db_url = settings.DATABASE_URL
+        if db_url.startswith("sqlite:///"):
+            # Extract path from SQLite URL
+            db_path = db_url.replace("sqlite:///", "").lstrip("./")
+
+            # Check if database file exists
+            if not Path(db_path).exists():
+                return ComponentStatus(
+                    name="database",
+                    status=ComponentStatusType.WARNING,
+                    message="Database not initialized - file does not exist",
+                    response_time_ms=None,
+                    metadata={
+                        "implementation": "sqlite",
+                        "database_exists": False,
+                        "expected_path": db_path,
+                        "url": settings.DATABASE_URL,
+                        "recommendation": (
+                            "Run database migrations or create database file"
+                        ),
+                    },
+                )
+
+        # Test database connection with simple query and collect enhanced metadata
+        enhanced_metadata = {
+            "implementation": "sqlite",
+            "url": settings.DATABASE_URL,
+            "database_exists": True,
+            "engine_echo": settings.DATABASE_ENGINE_ECHO,
+        }
+
+        # Collect additional metadata for SQLite databases
+        if db_url.startswith("sqlite:///"):
+            try:
+                # Add SQLite version
+                enhanced_metadata["version"] = sqlite3.sqlite_version
+
+                # Extract and add file size information
+                db_path = db_url.replace("sqlite:///", "").lstrip("./")
+                if Path(db_path).exists():
+                    file_size = Path(db_path).stat().st_size
+                    enhanced_metadata["file_size_bytes"] = file_size
+                    enhanced_metadata["file_size_human"] = format_bytes(file_size)
+
+                # Get engine and connection pool information
+                from app.core.db import engine
+                if hasattr(engine.pool, 'size'):
+                    enhanced_metadata["connection_pool_size"] = engine.pool.size()
+                else:
+                    # SQLite typically uses NullPool or StaticPool with size 1
+                    enhanced_metadata["connection_pool_size"] = 1
+
+            except Exception as e:
+                # If any enhanced metadata collection fails, log but don't break
+                # health check
+                logger.debug(
+                    "Failed to collect enhanced database metadata", exc_info=True
+                )
+
+        # Test database connection and collect PRAGMA settings
+        with db_session(autocommit=False) as session:
+            # Execute a simple query to test connectivity
+            from sqlalchemy import text
+            session.execute(text("SELECT 1"))
+
+            # Collect table information for SQLite databases
+            table_info = []
+            if db_url.startswith("sqlite:///"):
+                try:
+                    # Get all table names and row counts
+                    from sqlalchemy import inspect
+                    inspector = inspect(session.bind)
+                    if inspector is None:
+                        # Skip table introspection if inspector is None
+                        table_names = []
+                    else:
+                        table_names = inspector.get_table_names()
+                    
+                    for table_name in table_names:
+                        try:
+                            # Get row count for each table using quoted identifier
+                            from sqlalchemy import quoted_name
+                            quoted_table = quoted_name(table_name, quote=True)
+                            count_result = session.execute(
+                                text(f"SELECT COUNT(*) FROM {quoted_table}")
+                            ).fetchone()
+                            row_count = count_result[0] if count_result else 0
+                            
+                            table_info.append({
+                                "name": table_name,
+                                "rows": row_count
+                            })
+                        except Exception as e:
+                            # If we can't count rows for a table, still include it
+                            logger.debug(
+                                f"Failed to count rows for table {table_name}: {e}"
+                            )
+                            table_info.append({
+                                "name": table_name,
+                                "rows": 0
+                            })
+                    
+                    enhanced_metadata["tables"] = table_info
+                    enhanced_metadata["table_count"] = len(table_names)
+                    
+                except Exception as e:
+                    logger.debug("Failed to collect table information", exc_info=True)
+                    enhanced_metadata["tables"] = []
+                    enhanced_metadata["table_count"] = 0
+
+            # Collect SQLite PRAGMA settings for additional metadata
+            if db_url.startswith("sqlite:///"):
+                try:
+                    pragma_settings = {}
+
+                    # Get foreign keys setting
+                    result = session.execute(text("PRAGMA foreign_keys")).fetchone()
+                    if result:
+                        pragma_settings["foreign_keys"] = bool(result[0])
+
+                    # Get journal mode
+                    result = session.execute(text("PRAGMA journal_mode")).fetchone()
+                    if result:
+                        journal_mode = result[0].lower()
+                        pragma_settings["journal_mode"] = journal_mode
+                        enhanced_metadata["wal_enabled"] = journal_mode == "wal"
+
+                    # Add cache size if available
+                    result = session.execute(text("PRAGMA cache_size")).fetchone()
+                    if result:
+                        pragma_settings["cache_size"] = result[0]
+
+                    enhanced_metadata["pragma_settings"] = pragma_settings
+
+                except Exception as e:
+                    # PRAGMA queries can fail in some SQLite configurations
+                    logger.debug(
+                        "Failed to collect SQLite PRAGMA settings", exc_info=True
+                    )
+
+            # No need to commit since we're just testing connectivity
+
+        return ComponentStatus(
+            name="database",
+            status=ComponentStatusType.HEALTHY,
+            message="Database connection successful",
+            response_time_ms=None,  # Will be set by caller
+            metadata=enhanced_metadata,
+        )
+
+    except ImportError:
+        return ComponentStatus(
+            name="database",
+            status=ComponentStatusType.UNHEALTHY,
+            message="Database module not available",
+            response_time_ms=None,
+            metadata={
+                "implementation": "sqlite",
+                "error": "Database module not imported or configured",
+            },
+        )
+    except Exception as e:
+        # Check if it's a file not found error
+        error_str = str(e).lower()
+        if "unable to open database file" in error_str or "no such file" in error_str:
+            return ComponentStatus(
+                name="database",
+                status=ComponentStatusType.WARNING,
+                message="Database file not accessible",
+                response_time_ms=None,
+                metadata={
+                    "implementation": "sqlite",
+                    "url": settings.DATABASE_URL,
+                    "error": str(e),
+                    "recommendation": "Check database file path and permissions",
+                },
+            )
+
+        return ComponentStatus(
+            name="database",
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Database connection failed: {str(e)}",
+            response_time_ms=None,
+            metadata={
+                "implementation": "sqlite",
+                "url": settings.DATABASE_URL,
+                "error": str(e),
+            },
+        )
+{% endif %}
+
+
+{% if cookiecutter.include_worker == "yes" %}
+async def check_worker_health() -> ComponentStatus:
+    """
+    Check arq worker status using arq's native health checks and queue configuration.
+
+    Returns:
+        ComponentStatus indicating worker infrastructure health with queue
+        sub-components
+    """
+    try:
+        import re
+
+        import redis.asyncio as aioredis
+
+        # Create Redis connection with auto-detection for local vs Docker
+        # Step 1: Call untyped function with explicit ignore
+        redis_connection = aioredis.from_url(  # type: ignore[no-untyped-call]
+            settings.redis_url_effective,
+            db=settings.REDIS_DB
+        )
+        # Step 2: Cast the result to proper type
+        redis_client: aioredis.Redis = cast(aioredis.Redis, redis_connection)
+
+        # Get queue metadata from WorkerSettings classes via dynamic discovery
+        from app.components.worker.registry import get_all_queue_metadata
+        functional_queues = get_all_queue_metadata()
+
+        # Check each queue and create sub-components
+        queue_sub_components = {}
+        total_queued = 0
+        total_completed = 0
+        total_failed = 0
+        total_retried = 0
+        total_ongoing = 0
+        overall_healthy = True
+        active_workers = 0
+
+        for queue_type, queue_config in functional_queues.items():
+            queue_name = queue_config["queue_name"]
+
+            try:
+                # Get queue length (actual queued jobs) - arq uses sorted sets
+                queue_length_result = redis_client.zcard(queue_name)
+                if hasattr(queue_length_result, '__await__'):
+                    queue_length = await queue_length_result
+                else:
+                    queue_length = queue_length_result
+                total_queued += queue_length
+
+                # Look for arq health check key for this queue
+                # arq health check key format: {queue_name}:health-check
+                health_check_key = f"{queue_name}:health-check"
+                health_check_data = await redis_client.get(health_check_key)
+
+                # Parse arq health check data if available
+                j_complete = j_failed = j_retried = j_ongoing = 0
+                worker_alive = False
+                last_health_check = None
+
+                if health_check_data:
+                    health_string = health_check_data.decode()
+                    # Parse format: "Mar-01 17:41:22 j_complete=0 j_failed=0 ..."
+
+                    # Extract timestamp (first part before job stats)
+                    timestamp_match = re.match(r"^(\w+-\d+ \d+:\d+:\d+)", health_string)
+                    if timestamp_match:
+                        last_health_check = timestamp_match.group(1)
+
+                    # Extract job statistics using regex
+                    j_complete_match = re.search(r"j_complete=(\d+)", health_string)
+                    j_failed_match = re.search(r"j_failed=(\d+)", health_string)
+                    j_retried_match = re.search(r"j_retried=(\d+)", health_string)
+                    j_ongoing_match = re.search(r"j_ongoing=(\d+)", health_string)
+
+                    if j_complete_match:
+                        j_complete = int(j_complete_match.group(1))
+                        total_completed += j_complete
+                    if j_failed_match:
+                        j_failed = int(j_failed_match.group(1))
+                        total_failed += j_failed
+                    if j_retried_match:
+                        j_retried = int(j_retried_match.group(1))
+                        total_retried += j_retried
+                    if j_ongoing_match:
+                        j_ongoing = int(j_ongoing_match.group(1))
+                        total_ongoing += j_ongoing
+
+                    # Worker is alive if we have health check data
+                    # arq health checks expire automatically, having data means recent
+                    worker_alive = True
+                    
+                    if worker_alive:
+                        active_workers += 1
+
+                # Create queue status message
+                status_parts = []
+                if not worker_alive:
+                    status_parts.append("worker offline - no health check data")
+                elif j_ongoing > 0:
+                    status_parts.append(f"{j_ongoing} processing")
+                elif queue_length > 0:
+                    status_parts.append(f"{queue_length} queued")
+                else:
+                    status_parts.append("idle")
+
+                # Add job statistics to status if worker is alive
+                if worker_alive and (j_complete > 0 or j_failed > 0):
+                    if j_failed > 0:
+                        failure_rate = (j_failed / max(j_complete + j_failed, 1)) * 100
+                        status_parts.append(f"{j_failed} failed ({failure_rate:.1f}%)")
+                    if j_complete > 0:
+                        status_parts.append(f"{j_complete} completed")
+
+                # Check if queue has no functions configured (empty functions list)
+                queue_functions = queue_config.get("functions", [])
+                has_functions = len(queue_functions) > 0
+
+                # Determine queue status based on worker health and failure rate
+                failure_rate = (
+                    (j_failed / max(j_complete + j_failed, 1)) * 100
+                    if worker_alive
+                    else 100
+                )
+
+                if not worker_alive and not has_functions:
+                    # Queue configured but no functions - show as INFO
+                    queue_status = ComponentStatusType.INFO
+                    status_parts = ["configured - no functions defined"]
+                elif not worker_alive:
+                    queue_status = ComponentStatusType.UNHEALTHY
+                elif failure_rate > 25:  # Unhealthy threshold at 25%
+                    queue_status = ComponentStatusType.UNHEALTHY
+                elif failure_rate > 10:  # Warning threshold at 10%
+                    queue_status = ComponentStatusType.WARNING
+                else:
+                    queue_status = ComponentStatusType.HEALTHY
+
+                queue_message = (
+                    f"{queue_config['description']}: {', '.join(status_parts)}"
+                )
+
+                # Update overall health based on this queue
+                if queue_status == ComponentStatusType.UNHEALTHY:
+                    overall_healthy = False
+
+                queue_metadata = {
+                    "queue_type": queue_type,
+                    "queue_name": queue_name,
+                    "queued_jobs": queue_length,
+                    "max_concurrency": queue_config["max_jobs"],
+                    "timeout_seconds": queue_config["timeout"],
+                    "description": queue_config["description"],
+                    "worker_alive": worker_alive,
+                    "health_check_key": health_check_key,
+                }
+
+                # Add arq health check statistics if available
+                if worker_alive:
+                    queue_metadata.update(
+                        {
+                            "jobs_completed": j_complete,
+                            "jobs_failed": j_failed,
+                            "jobs_retried": j_retried,
+                            "jobs_ongoing": j_ongoing,
+                            "failure_rate_percent": round(failure_rate, 1),
+                            "last_health_check": last_health_check,
+                        }
+                    )
+                else:
+                    queue_metadata["offline_reason"] = "Health check key not found"
+
+                queue_sub_components[queue_type] = ComponentStatus(
+                    name=queue_type,
+                    status=queue_status,
+                    message=queue_message,
+                    response_time_ms=None,
+                    metadata=queue_metadata,
+                    sub_components={},
+                )
+
+            except aioredis.ConnectionError as e:
+                logger.error(f"Redis connection failed for {queue_type}: {e}")
+                overall_healthy = False
+
+                # Extract more specific connection error details
+                error_details = str(e).lower()
+                if "connection refused" in error_details:
+                    connection_issue = "Redis server not running"
+                elif (
+                    "name or service not known" in error_details
+                    or "nodename nor servname" in error_details
+                ):
+                    connection_issue = "Redis server DNS resolution failed"
+                elif "timeout" in error_details:
+                    connection_issue = "Redis server connection timeout"
+                else:
+                    connection_issue = "Redis server unreachable"
+
+                queue_sub_components[queue_type] = ComponentStatus(
+                    name=queue_type,
+                    status=ComponentStatusType.UNHEALTHY,
+                    message=f"{connection_issue} - worker offline",
+                    response_time_ms=None,
+                    metadata={
+                        "queue_type": queue_type,
+                        "queue_name": queue_name,
+                        "error_type": "redis_connection_error",
+                        "error": str(e),
+                        "connection_issue": connection_issue,
+                        "recommendation": (
+                            "Check Redis server status and network connectivity"
+                        ),
+                    },
+                    sub_components={},
+                )
+            except aioredis.ResponseError as e:
+                if "WRONGTYPE" in str(e):
+                    logger.error(f"Redis data corruption for {queue_type}: {e}")
+                    message = f"Redis data corruption detected"
+                    recommendation = "Clear Redis cache to fix data type conflicts"
+                    error_type = "redis_key_type_error"
+                else:
+                    logger.error(f"Redis operation failed for {queue_type}: {e}")
+                    message = f"Redis operation failed"
+                    recommendation = "Check Redis configuration and permissions"
+                    error_type = "redis_response_error"
+
+                overall_healthy = False
+                queue_sub_components[queue_type] = ComponentStatus(
+                    name=queue_type,
+                    status=ComponentStatusType.UNHEALTHY,
+                    message=message,
+                    response_time_ms=None,
+                    metadata={
+                        "queue_type": queue_type,
+                        "queue_name": queue_name,
+                        "error_type": error_type,
+                        "error": str(e),
+                        "recommendation": recommendation,
+                    },
+                    sub_components={},
+                )
+            except Exception as e:
+                logger.error(
+                    f"Unexpected error checking {queue_type} queue health: {e}"
+                )
+                overall_healthy = False
+                queue_sub_components[queue_type] = ComponentStatus(
+                    name=queue_type,
+                    status=ComponentStatusType.UNHEALTHY,
+                    message=f"Health check failed: {type(e).__name__}",
+                    response_time_ms=None,
+                    metadata={
+                        "queue_type": queue_type,
+                        "queue_name": queue_name,
+                        "error_type": "unexpected_error",
+                        "error": str(e),
+                        "exception_class": type(e).__name__,
+                    },
+                    sub_components={},
+                )
+
+        await redis_client.aclose()
+
+        # Create main worker status message
+        message_parts = []
+        if active_workers == 0:
+            message_parts.append("No active workers")
+            overall_healthy = False
+        else:
+            message_parts.append(
+                f"{active_workers}/{len(functional_queues)} workers active"
+            )
+
+        if total_queued > 0:
+            message_parts.append(f"{total_queued} queued")
+        if total_ongoing > 0:
+            message_parts.append(f"{total_ongoing} processing")
+        if total_failed > 0:
+            failure_rate = (total_failed / max(total_completed + total_failed, 1)) * 100
+            message_parts.append(f"{total_failed} failed ({failure_rate:.1f}%)")
+
+        main_message = f"arq worker infrastructure: {', '.join(message_parts)}"
+
+        # Create a "queues" intermediate component that contains all queue
+        # sub-components - determine status from child statuses
+        queue_statuses = [queue.status for queue in queue_sub_components.values()]
+        queues_status = propagate_status(queue_statuses)
+
+
+        queues_message = f"{len(functional_queues)} functional queues configured"
+        if active_workers < len(functional_queues):
+            queues_message += f" ({active_workers} active)"
+
+        queues_component = ComponentStatus(
+            name="queues",
+            status=queues_status,
+            message=queues_message,
+            response_time_ms=None,
+            metadata={
+                "configured_queues": len(functional_queues),
+                "active_workers": active_workers,
+                "queue_types": list(functional_queues.keys()),
+            },
+            sub_components=queue_sub_components,
+        )
+
+        # Determine worker status based on overall health and queues status
+        if not overall_healthy:
+            worker_status = ComponentStatusType.UNHEALTHY
+        else:
+            worker_status = propagate_status([queues_status])
+
+        return ComponentStatus(
+            name="worker",
+            status=worker_status,
+            message=main_message,
+            response_time_ms=None,
+            metadata={
+                "total_queued": total_queued,
+                "total_completed": total_completed,
+                "total_failed": total_failed,
+                "total_retried": total_retried,
+                "total_ongoing": total_ongoing,
+                "overall_failure_rate_percent": (
+                    round(
+                        (total_failed / max(total_completed + total_failed, 1)) * 100, 1
+                    )
+                    if total_completed + total_failed > 0
+                    else 0
+                ),
+                "redis_url": settings.REDIS_URL,
+                "queue_configuration": {
+                    queue_type: {
+                        "description": config["description"],
+                        "max_jobs": config["max_jobs"],
+                        "timeout_seconds": config["timeout"],
+                    }
+                    for queue_type, config in functional_queues.items()
+                },
+            },
+            sub_components={"queues": queues_component},
+        )
+
+    except ImportError:
+        return ComponentStatus(
+            name="worker",
+            status=ComponentStatusType.UNHEALTHY,
+            message="Redis library not available for worker health check",
+            response_time_ms=None,
+            sub_components={},
+        )
+    except Exception as e:
+        logger.error(f"Worker health check failed: {e}")
+        return ComponentStatus(
+            name="worker",
+            status=ComponentStatusType.UNHEALTHY,
+            message=f"Worker health check failed: {str(e)}",
+            response_time_ms=None,
+            metadata={
+                "error": str(e),
+                "redis_url": settings.REDIS_URL,
+            },
+            sub_components={},
+        )
+{% endif %}