aiecs 1.5.1__py3-none-any.whl

aiecs/infrastructure/monitoring/__init__.py

@@ -0,0 +1,34 @@
+"""Infrastructure monitoring module
+
+Contains monitoring, metrics, and observability infrastructure.
+"""
+
+from .executor_metrics import ExecutorMetrics
+from .tracing_manager import TracingManager
+from .global_metrics_manager import (
+    initialize_global_metrics,
+    get_global_metrics,
+    close_global_metrics,
+    is_metrics_initialized,
+    get_metrics_summary,
+    record_operation,
+    record_duration,
+    record_operation_success,
+    record_operation_failure,
+    record_retry,
+)
+
+__all__ = [
+    "ExecutorMetrics",
+    "TracingManager",
+    "initialize_global_metrics",
+    "get_global_metrics",
+    "close_global_metrics",
+    "is_metrics_initialized",
+    "get_metrics_summary",
+    "record_operation",
+    "record_duration",
+    "record_operation_success",
+    "record_operation_failure",
+    "record_retry",
+]

aiecs/infrastructure/monitoring/executor_metrics.py

@@ -0,0 +1,174 @@
+import logging
+import functools
+from typing import Dict, Optional, Any
+from prometheus_client import Counter, Histogram, start_http_server
+
+logger = logging.getLogger(__name__)
+
+
+class ExecutorMetrics:
+    """
+    Specialized handler for executor performance monitoring and metrics collection
+    """
+
+    def __init__(self, enable_metrics: bool = True, metrics_port: int = 8001):
+        self.enable_metrics = enable_metrics
+        self.metrics_port = metrics_port
+        self.metrics: Dict[str, Any] = {}
+
+        if self.enable_metrics:
+            self._init_prometheus_metrics()
+
+    def _init_prometheus_metrics(self):
+        """Initialize Prometheus metrics"""
+        try:
+            start_http_server(self.metrics_port)
+            self.metrics = {
+                "intent_latency": Histogram("intent_latency_seconds", "Latency of intent parsing"),
+                "intent_success": Counter(
+                    "intent_success_total",
+                    "Number of successful intent parsings",
+                ),
+                "intent_retries": Counter(
+                    "intent_retries_total", "Number of intent parsing retries"
+                ),
+                "plan_latency": Histogram("plan_latency_seconds", "Latency of task planning"),
+                "plan_success": Counter("plan_success_total", "Number of successful plans"),
+                "plan_retries": Counter("plan_retries_total", "Number of plan retries"),
+                "execute_latency": Histogram(
+                    "execute_latency_seconds",
+                    "Latency of task execution",
+                    ["task_type"],
+                ),
+                "execute_success": Counter(
+                    "execute_success_total",
+                    "Number of successful executions",
+                    ["task_type"],
+                ),
+                "execute_retries": Counter(
+                    "execute_retries_total",
+                    "Number of execution retries",
+                    ["task_type"],
+                ),
+            }
+            logger.info(f"Prometheus metrics server started on port {self.metrics_port}")
+        except Exception as e:
+            logger.warning(f"Failed to start metrics server: {e}")
+            self.metrics = {}
+
+    def record_operation_latency(self, operation: str, duration: float):
+        """Record operation latency"""
+        if not self.enable_metrics or f"{operation}_latency" not in self.metrics:
+            return
+        self.metrics[f"{operation}_latency"].observe(duration)
+
+    def record_operation_success(self, operation: str, labels: Optional[Dict[str, str]] = None):
+        """Record operation success"""
+        if not self.enable_metrics or f"{operation}_success" not in self.metrics:
+            return
+        metric = self.metrics[f"{operation}_success"]
+        if labels:
+            metric = metric.labels(**labels)
+        metric.inc()
+
+    def record_operation_failure(
+        self,
+        operation: str,
+        error_type: str,
+        labels: Optional[Dict[str, str]] = None,
+    ):
+        """Record operation failure"""
+        if not self.enable_metrics:
+            return
+        # Failure metrics can be added
+        logger.error(f"Operation {operation} failed with error type: {error_type}")
+
+    def record_retry(self, operation: str, attempt_number: int):
+        """Record retry"""
+        if not self.enable_metrics or f"{operation}_retries" not in self.metrics:
+            return
+        if attempt_number > 1:
+            self.metrics[f"{operation}_retries"].inc()
+
+    def with_metrics(self, metric_name: str, labels: Optional[Dict[str, str]] = None):
+        """Monitoring decorator"""
+
+        def decorator(func):
+            @functools.wraps(func)
+            async def wrapper(*args, **kwargs):
+                if not self.metrics or f"{metric_name}_latency" not in self.metrics:
+                    return await func(*args, **kwargs)
+
+                labels_dict = labels or {}
+                metric = self.metrics[f"{metric_name}_latency"]
+                if labels:
+                    metric = metric.labels(**labels_dict)
+
+                with metric.time():
+                    try:
+                        result = await func(*args, **kwargs)
+                        if f"{metric_name}_success" in self.metrics:
+                            success_metric = self.metrics[f"{metric_name}_success"]
+                            if labels:
+                                success_metric = success_metric.labels(**labels_dict)
+                            success_metric.inc()
+                        return result
+                    except Exception as e:
+                        logger.error(f"Error in {func.__name__}: {e}")
+                        raise
+
+            return wrapper
+
+        return decorator
+
+    def get_metrics_summary(self) -> Dict[str, Any]:
+        """Get metrics summary"""
+        if not self.enable_metrics:
+            return {"metrics_enabled": False}
+
+        return {
+            "metrics_enabled": True,
+            "metrics_port": self.metrics_port,
+            "available_metrics": list(self.metrics.keys()),
+        }
+
+    def record_operation(
+        self,
+        operation_type: str,
+        success: bool = True,
+        duration: Optional[float] = None,
+        **kwargs,
+    ):
+        """Record a general operation for metrics tracking"""
+        if not self.enable_metrics:
+            return
+
+        try:
+            # Record operation success/failure
+            if success:
+                self.record_operation_success(operation_type, kwargs.get("labels"))
+            else:
+                error_type = kwargs.get("error_type", "unknown")
+                self.record_operation_failure(operation_type, error_type, kwargs.get("labels"))
+
+            # Record operation latency if provided
+            if duration is not None:
+                self.record_operation_latency(operation_type, duration)
+
+        except Exception as e:
+            logger.warning(f"Failed to record operation metrics: {e}")
+
+    def record_duration(
+        self,
+        operation: str,
+        duration: float,
+        labels: Optional[Dict[str, str]] = None,
+    ):
+        """Record operation duration for metrics tracking"""
+        if not self.enable_metrics:
+            return
+
+        try:
+            self.record_operation_latency(operation, duration)
+        except Exception as e:
+            logger.warning(f"Failed to record duration metrics: {e}")

aiecs/infrastructure/monitoring/global_metrics_manager.py

@@ -0,0 +1,213 @@
+"""
+Global Metrics Manager
+
+This module provides a singleton ExecutorMetrics instance that can be shared
+across all components in the application. It follows the same pattern as
+other global managers in the infrastructure layer.
+
+Usage:
+    # In main.py startup:
+    await initialize_global_metrics()
+
+    # In any component:
+    from aiecs.infrastructure.monitoring.global_metrics_manager import get_global_metrics
+    metrics = get_global_metrics()
+"""
+
+import logging
+import asyncio
+import os
+from typing import Optional, Dict, Any
+from .executor_metrics import ExecutorMetrics
+
+logger = logging.getLogger(__name__)
+
+# Global singleton instance
+_global_metrics: Optional[ExecutorMetrics] = None
+_initialization_lock = asyncio.Lock()
+_initialized = False
+
+
+async def initialize_global_metrics(
+    enable_metrics: bool = True,
+    metrics_port: Optional[int] = None,
+    config: Optional[Dict[str, Any]] = None,
+) -> Optional[ExecutorMetrics]:
+    """
+    Initialize the global ExecutorMetrics instance.
+
+    This should be called once during application startup (in main.py lifespan).
+
+    Args:
+        enable_metrics: Whether to enable metrics collection (default: True)
+        metrics_port: Port for metrics server (default: from env or 8001)
+        config: Additional configuration options
+
+    Returns:
+        The initialized ExecutorMetrics instance or None if initialization fails
+
+    Example:
+        @asynccontextmanager
+        async def lifespan(app: FastAPI):
+            # Startup
+            await initialize_global_metrics()
+            yield
+            # Shutdown
+            await close_global_metrics()
+    """
+    global _global_metrics, _initialized
+
+    if _initialized and _global_metrics:
+        logger.info("Global metrics already initialized")
+        return _global_metrics
+
+    async with _initialization_lock:
+        # Double-check after acquiring lock
+        if _initialized and _global_metrics:
+            return _global_metrics
+
+        try:
+            # Determine metrics port
+            if metrics_port is None:
+                metrics_port = int(os.environ.get("METRICS_PORT", "8001"))
+
+            # Check if metrics should be enabled
+            if not enable_metrics:
+                enable_metrics = os.environ.get("ENABLE_METRICS", "true").lower() == "true"
+
+            logger.info(
+                f"Initializing global metrics (port: {metrics_port}, enabled: {enable_metrics})..."
+            )
+
+            # Create metrics instance
+            _global_metrics = ExecutorMetrics(
+                enable_metrics=enable_metrics, metrics_port=metrics_port
+            )
+
+            _initialized = True
+            logger.info("✅ Global metrics initialized successfully")
+            return _global_metrics
+
+        except Exception as e:
+            logger.error(f"❌ Failed to initialize global metrics: {e}")
+            logger.warning("Application will continue without metrics (degraded mode)")
+            _global_metrics = None
+            _initialized = False
+            return None
+
+
+def get_global_metrics() -> Optional[ExecutorMetrics]:
+    """
+    Get the global ExecutorMetrics instance.
+
+    Returns:
+        The global ExecutorMetrics instance or None if not initialized
+
+    Raises:
+        RuntimeError: If metrics are requested but not initialized
+
+    Example:
+        metrics = get_global_metrics()
+        if metrics:
+            metrics.record_operation('my_operation', 1)
+    """
+    if _global_metrics is None:
+        logger.warning("Global metrics not initialized - call initialize_global_metrics() first")
+        return None
+
+    return _global_metrics
+
+
+async def close_global_metrics():
+    """
+    Close the global metrics instance.
+
+    This should be called during application shutdown.
+    """
+    global _global_metrics, _initialized
+
+    if _global_metrics:
+        try:
+            # ExecutorMetrics doesn't have a close method, but we can clean up
+            logger.info("Closing global metrics...")
+            _global_metrics = None
+            _initialized = False
+            logger.info("✅ Global metrics closed successfully")
+        except Exception as e:
+            logger.error(f"❌ Error closing global metrics: {e}")
+
+
+def is_metrics_initialized() -> bool:
+    """
+    Check if global metrics are initialized.
+
+    Returns:
+        True if metrics are initialized, False otherwise
+    """
+    return _initialized and _global_metrics is not None
+
+
+def get_metrics_summary() -> Dict[str, Any]:
+    """
+    Get a summary of the global metrics status.
+
+    Returns:
+        Dictionary containing metrics status information
+    """
+    if not is_metrics_initialized():
+        return {
+            "initialized": False,
+            "message": "Global metrics not initialized",
+        }
+
+    try:
+        return _global_metrics.get_metrics_summary()
+    except Exception as e:
+        return {
+            "initialized": True,
+            "error": str(e),
+            "message": "Failed to get metrics summary",
+        }
+
+
+# Convenience functions for common operations
+def record_operation(
+    operation_type: str,
+    success: bool = True,
+    duration: Optional[float] = None,
+    **kwargs,
+):
+    """Record an operation using global metrics."""
+    metrics = get_global_metrics()
+    if metrics:
+        metrics.record_operation(operation_type, success, duration, **kwargs)
+
+
+def record_duration(operation: str, duration: float, labels: Optional[Dict[str, str]] = None):
+    """Record operation duration using global metrics."""
+    metrics = get_global_metrics()
+    if metrics:
+        metrics.record_duration(operation, duration, labels)
+
+
+def record_operation_success(operation: str, labels: Optional[Dict[str, str]] = None):
+    """Record operation success using global metrics."""
+    metrics = get_global_metrics()
+    if metrics:
+        metrics.record_operation_success(operation, labels)
+
+
+def record_operation_failure(
+    operation: str, error_type: str, labels: Optional[Dict[str, str]] = None
+):
+    """Record operation failure using global metrics."""
+    metrics = get_global_metrics()
+    if metrics:
+        metrics.record_operation_failure(operation, error_type, labels)
+
+
+def record_retry(operation: str, attempt_number: int):
+    """Record retry using global metrics."""
+    metrics = get_global_metrics()
+    if metrics:
+        metrics.record_retry(operation, attempt_number)

aiecs/infrastructure/monitoring/structured_logger.py

@@ -0,0 +1,48 @@
+"""Structured logging setup for aiecs."""
+
+import logging
+import sys
+
+
+def setup_structured_logging(level: str = "INFO", format_type: str = "json") -> None:
+    """
+    Setup structured logging for the application.
+
+    Args:
+        level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        format_type: Format type (json, text)
+    """
+    # Convert string level to logging level
+    numeric_level = getattr(logging, level.upper(), logging.INFO)
+
+    # Create formatter
+    if format_type.lower() == "json":
+        # Simple JSON-like format for now
+        formatter = logging.Formatter(
+            '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "module": "%(name)s", "message": "%(message)s"}'
+        )
+    else:
+        # Standard text format
+        formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+    # Setup root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(numeric_level)
+
+    # Remove existing handlers
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+
+    # Add console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(numeric_level)
+    console_handler.setFormatter(formatter)
+    root_logger.addHandler(console_handler)
+
+    # Set specific logger levels
+    logging.getLogger("aiecs").setLevel(numeric_level)
+
+    # Suppress noisy third-party loggers
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("httpx").setLevel(logging.WARNING)