runbooks 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

runbooks/sre/production_monitoring_framework.py

@@ -50,7 +50,7 @@ from runbooks.common.rich_utils import (
 
 class AlertSeverity(Enum):
     """Alert severity levels for monitoring framework."""
-    
+
     INFO = "INFO"
     WARNING = "WARNING"
     CRITICAL = "CRITICAL"
@@ -59,7 +59,7 @@ class AlertSeverity(Enum):
 
 class OperationStatus(Enum):
     """Operation status for monitoring."""
-    
+
     HEALTHY = "HEALTHY"
     DEGRADED = "DEGRADED"
     UNHEALTHY = "UNHEALTHY"
@@ -69,7 +69,7 @@ class OperationStatus(Enum):
 @dataclass
 class SLATarget:
     """SLA target definition with thresholds."""
-    
+
     name: str
     target_value: float
     warning_threshold: float
@@ -81,7 +81,7 @@ class SLATarget:
 @dataclass
 class MonitoringMetric:
     """Individual monitoring metric result."""
-    
+
     metric_name: str
     current_value: float
     target_value: float
@@ -93,7 +93,7 @@ class MonitoringMetric:
 @dataclass
 class AlertEvent:
     """Alert event structure."""
-    
+
     alert_id: str
     severity: AlertSeverity
     metric_name: str
@@ -107,201 +107,195 @@ class AlertEvent:
 class ProductionMonitoringFramework:
     """
     Enterprise production monitoring framework for CloudOps operations.
-    
+
     Monitors SLA compliance, performance metrics, and operational health
     across 61-account enterprise environment.
     """
-    
+
     def __init__(self, console_instance: Optional[Console] = None):
         """
         Initialize production monitoring framework.
-        
+
         Args:
             console_instance: Rich console for output
         """
         self.console = console_instance or console
         self.start_time = time.time()
-        
+
         # SLA targets for enterprise operations
         self.sla_targets = {
-            'availability': SLATarget(
-                name='availability',
+            "availability": SLATarget(
+                name="availability",
                 target_value=99.9,
                 warning_threshold=99.5,
                 critical_threshold=99.0,
-                unit='%',
-                description='System availability percentage'
+                unit="%",
+                description="System availability percentage",
             ),
-            'latency_p95': SLATarget(
-                name='latency_p95',
+            "latency_p95": SLATarget(
+                name="latency_p95",
                 target_value=30.0,
                 warning_threshold=45.0,
                 critical_threshold=60.0,
-                unit='seconds',
-                description='95th percentile operation latency'
+                unit="seconds",
+                description="95th percentile operation latency",
             ),
-            'success_rate': SLATarget(
-                name='success_rate',
+            "success_rate": SLATarget(
+                name="success_rate",
                 target_value=95.0,
                 warning_threshold=90.0,
                 critical_threshold=85.0,
-                unit='%',
-                description='Operation success rate'
+                unit="%",
+                description="Operation success rate",
             ),
-            'error_budget': SLATarget(
-                name='error_budget',
+            "error_budget": SLATarget(
+                name="error_budget",
                 target_value=0.1,
                 warning_threshold=0.05,
                 critical_threshold=0.01,
-                unit='%',
-                description='Monthly error budget remaining'
-            )
+                unit="%",
+                description="Monthly error budget remaining",
+            ),
         }
-        
+
         # Monitoring state
         self.active_alerts = []
         self.metrics_history = []
         self.circuit_breaker_state = {}
         self.monitoring_active = False
-        
+
         # Performance tracking
         self.operation_metrics = {
-            'total_operations': 0,
-            'successful_operations': 0,
-            'failed_operations': 0,
-            'average_latency': 0.0,
-            'p95_latency': 0.0
+            "total_operations": 0,
+            "successful_operations": 0,
+            "failed_operations": 0,
+            "average_latency": 0.0,
+            "p95_latency": 0.0,
         }
-        
+
     async def start_monitoring(self, interval_seconds: int = 60) -> None:
         """
         Start continuous monitoring loop.
-        
+
         Args:
             interval_seconds: Monitoring interval in seconds
         """
         self.monitoring_active = True
-        
+
         print_success("🚀 Production monitoring framework started")
-        
+
         with Live(self._create_monitoring_dashboard(), refresh_per_second=1, console=self.console) as live:
             while self.monitoring_active:
                 try:
                     # Collect current metrics
                     current_metrics = await self._collect_current_metrics()
-                    
+
                     # Evaluate SLA compliance
                     sla_violations = self._evaluate_sla_compliance(current_metrics)
-                    
+
                     # Process alerts
                     await self._process_alerts(sla_violations)
-                    
+
                     # Update circuit breaker states
                     self._update_circuit_breakers(current_metrics)
-                    
+
                     # Update dashboard
                     live.update(self._create_monitoring_dashboard())
-                    
+
                     # Store metrics history
-                    self.metrics_history.append({
-                        'timestamp': datetime.now(),
-                        'metrics': current_metrics
-                    })
-                    
+                    self.metrics_history.append({"timestamp": datetime.now(), "metrics": current_metrics})
+
                     # Clean old history (keep 24 hours)
                     self._cleanup_metrics_history()
-                    
+
                     await asyncio.sleep(interval_seconds)
-                    
+
                 except Exception as e:
                     print_error(f"Monitoring loop error: {str(e)}")
                     await asyncio.sleep(5)  # Short retry interval
-    
+
     async def stop_monitoring(self) -> None:
         """Stop the monitoring framework gracefully."""
         self.monitoring_active = False
         print_info("📊 Production monitoring framework stopped")
-    
+
     async def _collect_current_metrics(self) -> Dict[str, MonitoringMetric]:
         """
         Collect current operational metrics.
-        
+
         Returns:
             Dictionary of current metrics
         """
         current_metrics = {}
-        
+
         # Calculate availability (based on successful operations)
-        total_ops = max(self.operation_metrics['total_operations'], 1)
-        success_ops = self.operation_metrics['successful_operations']
+        total_ops = max(self.operation_metrics["total_operations"], 1)
+        success_ops = self.operation_metrics["successful_operations"]
         availability = (success_ops / total_ops) * 100
-        
-        current_metrics['availability'] = MonitoringMetric(
-            metric_name='availability',
+
+        current_metrics["availability"] = MonitoringMetric(
+            metric_name="availability",
             current_value=availability,
-            target_value=self.sla_targets['availability'].target_value,
-            status=self._determine_status('availability', availability),
+            target_value=self.sla_targets["availability"].target_value,
+            status=self._determine_status("availability", availability),
             timestamp=datetime.now(),
             details={
-                'total_operations': total_ops,
-                'successful_operations': success_ops,
-                'failed_operations': self.operation_metrics['failed_operations']
-            }
+                "total_operations": total_ops,
+                "successful_operations": success_ops,
+                "failed_operations": self.operation_metrics["failed_operations"],
+            },
         )
-        
+
         # P95 latency monitoring
-        p95_latency = self.operation_metrics['p95_latency']
-        current_metrics['latency_p95'] = MonitoringMetric(
-            metric_name='latency_p95',
+        p95_latency = self.operation_metrics["p95_latency"]
+        current_metrics["latency_p95"] = MonitoringMetric(
+            metric_name="latency_p95",
             current_value=p95_latency,
-            target_value=self.sla_targets['latency_p95'].target_value,
-            status=self._determine_status('latency_p95', p95_latency),
+            target_value=self.sla_targets["latency_p95"].target_value,
+            status=self._determine_status("latency_p95", p95_latency),
             timestamp=datetime.now(),
-            details={
-                'average_latency': self.operation_metrics['average_latency'],
-                'p95_latency': p95_latency
-            }
+            details={"average_latency": self.operation_metrics["average_latency"], "p95_latency": p95_latency},
         )
-        
+
         # Success rate monitoring
         success_rate = (success_ops / total_ops) * 100
-        current_metrics['success_rate'] = MonitoringMetric(
-            metric_name='success_rate',
+        current_metrics["success_rate"] = MonitoringMetric(
+            metric_name="success_rate",
             current_value=success_rate,
-            target_value=self.sla_targets['success_rate'].target_value,
-            status=self._determine_status('success_rate', success_rate),
+            target_value=self.sla_targets["success_rate"].target_value,
+            status=self._determine_status("success_rate", success_rate),
             timestamp=datetime.now(),
-            details={'success_percentage': success_rate}
+            details={"success_percentage": success_rate},
         )
-        
+
         # Error budget monitoring (simplified calculation)
-        error_budget = max(0.0, 1.0 - (self.operation_metrics['failed_operations'] / total_ops)) * 100
-        current_metrics['error_budget'] = MonitoringMetric(
-            metric_name='error_budget',
+        error_budget = max(0.0, 1.0 - (self.operation_metrics["failed_operations"] / total_ops)) * 100
+        current_metrics["error_budget"] = MonitoringMetric(
+            metric_name="error_budget",
             current_value=error_budget,
-            target_value=self.sla_targets['error_budget'].target_value,
-            status=self._determine_status('error_budget', error_budget),
+            target_value=self.sla_targets["error_budget"].target_value,
+            status=self._determine_status("error_budget", error_budget),
             timestamp=datetime.now(),
-            details={'error_budget_remaining': error_budget}
+            details={"error_budget_remaining": error_budget},
         )
-        
+
         return current_metrics
-    
+
     def _determine_status(self, metric_name: str, current_value: float) -> OperationStatus:
         """
         Determine operation status based on current value and thresholds.
-        
+
         Args:
             metric_name: Name of the metric
             current_value: Current metric value
-            
+
         Returns:
             OperationStatus enum value
         """
         sla = self.sla_targets[metric_name]
-        
+
         # For latency, higher is worse
-        if metric_name == 'latency_p95':
+        if metric_name == "latency_p95":
             if current_value <= sla.target_value:
                 return OperationStatus.HEALTHY
             elif current_value <= sla.warning_threshold:
@@ -310,7 +304,7 @@ class ProductionMonitoringFramework:
                 return OperationStatus.UNHEALTHY
             else:
                 return OperationStatus.CRITICAL
-        
+
         # For other metrics, lower is worse
         else:
             if current_value >= sla.target_value:
@@ -321,29 +315,29 @@ class ProductionMonitoringFramework:
                 return OperationStatus.UNHEALTHY
             else:
                 return OperationStatus.CRITICAL
-    
+
     def _evaluate_sla_compliance(self, current_metrics: Dict[str, MonitoringMetric]) -> List[MonitoringMetric]:
         """
         Evaluate SLA compliance and identify violations.
-        
+
         Args:
             current_metrics: Current metric values
-            
+
         Returns:
             List of metrics that violate SLA thresholds
         """
         violations = []
-        
+
         for metric in current_metrics.values():
             if metric.status in [OperationStatus.UNHEALTHY, OperationStatus.CRITICAL]:
                 violations.append(metric)
-        
+
         return violations
-    
+
     async def _process_alerts(self, violations: List[MonitoringMetric]) -> None:
         """
         Process SLA violations and generate alerts.
-        
+
         Args:
             violations: List of metric violations
         """
@@ -351,23 +345,25 @@ class ProductionMonitoringFramework:
             # Create alert event
             alert = AlertEvent(
                 alert_id=f"SLA-{violation.metric_name}-{int(time.time())}",
-                severity=AlertSeverity.CRITICAL if violation.status == OperationStatus.CRITICAL else AlertSeverity.WARNING,
+                severity=AlertSeverity.CRITICAL
+                if violation.status == OperationStatus.CRITICAL
+                else AlertSeverity.WARNING,
                 metric_name=violation.metric_name,
                 current_value=violation.current_value,
                 threshold_value=self.sla_targets[violation.metric_name].critical_threshold,
                 message=f"SLA violation detected for {violation.metric_name}: {violation.current_value:.2f}{self.sla_targets[violation.metric_name].unit}",
-                timestamp=datetime.now()
+                timestamp=datetime.now(),
             )
-            
+
             # Add to active alerts if not already present
             if not any(a.metric_name == alert.metric_name and not a.resolved for a in self.active_alerts):
                 self.active_alerts.append(alert)
                 await self._send_alert(alert)
-    
+
     async def _send_alert(self, alert: AlertEvent) -> None:
         """
         Send alert notification (placeholder for integration with alerting systems).
-        
+
         Args:
             alert: Alert event to send
         """
@@ -376,32 +372,32 @@ class ProductionMonitoringFramework:
         # - PagerDuty/OpsGenie
         # - Email notifications
         # - ServiceNow incidents
-        
+
         if alert.severity == AlertSeverity.CRITICAL:
             print_error(f"🚨 CRITICAL ALERT: {alert.message}")
         else:
             print_warning(f"⚠️  WARNING ALERT: {alert.message}")
-    
+
     def _update_circuit_breakers(self, current_metrics: Dict[str, MonitoringMetric]) -> None:
         """
         Update circuit breaker states based on current metrics.
-        
+
         Args:
             current_metrics: Current metric values
         """
         for metric_name, metric in current_metrics.items():
             if metric.status == OperationStatus.CRITICAL:
-                self.circuit_breaker_state[metric_name] = 'OPEN'
+                self.circuit_breaker_state[metric_name] = "OPEN"
             elif metric.status == OperationStatus.HEALTHY:
-                self.circuit_breaker_state[metric_name] = 'CLOSED'
+                self.circuit_breaker_state[metric_name] = "CLOSED"
             else:
                 # Keep current state for degraded/unhealthy
                 pass
-    
+
     def _create_monitoring_dashboard(self) -> Panel:
         """
         Create Rich dashboard for monitoring display.
-        
+
         Returns:
             Rich Panel with monitoring dashboard
         """
@@ -411,136 +407,130 @@ class ProductionMonitoringFramework:
         metrics_table.add_column("Current", style="yellow")
         metrics_table.add_column("Target", style="green")
         metrics_table.add_column("Status", style="blue")
-        
+
         for sla_name, sla in self.sla_targets.items():
             # Get current value from operation metrics
-            if sla_name == 'availability':
-                total = max(self.operation_metrics['total_operations'], 1)
-                current = (self.operation_metrics['successful_operations'] / total) * 100
-            elif sla_name == 'latency_p95':
-                current = self.operation_metrics['p95_latency']
-            elif sla_name == 'success_rate':
-                total = max(self.operation_metrics['total_operations'], 1)
-                current = (self.operation_metrics['successful_operations'] / total) * 100
+            if sla_name == "availability":
+                total = max(self.operation_metrics["total_operations"], 1)
+                current = (self.operation_metrics["successful_operations"] / total) * 100
+            elif sla_name == "latency_p95":
+                current = self.operation_metrics["p95_latency"]
+            elif sla_name == "success_rate":
+                total = max(self.operation_metrics["total_operations"], 1)
+                current = (self.operation_metrics["successful_operations"] / total) * 100
             else:  # error_budget
                 current = 0.1  # Placeholder calculation
-            
+
             status = self._determine_status(sla_name, current)
             status_color = {
                 OperationStatus.HEALTHY: "[green]HEALTHY[/green]",
                 OperationStatus.DEGRADED: "[yellow]DEGRADED[/yellow]",
                 OperationStatus.UNHEALTHY: "[red]UNHEALTHY[/red]",
-                OperationStatus.CRITICAL: "[red bold]CRITICAL[/red bold]"
+                OperationStatus.CRITICAL: "[red bold]CRITICAL[/red bold]",
             }[status]
-            
+
             metrics_table.add_row(
-                sla.description,
-                f"{current:.2f}{sla.unit}",
-                f"{sla.target_value:.2f}{sla.unit}",
-                status_color
+                sla.description, f"{current:.2f}{sla.unit}", f"{sla.target_value:.2f}{sla.unit}", status_color
             )
-        
+
         # Active alerts table
         alerts_table = Table(title="🚨 Active Alerts")
         alerts_table.add_column("Severity", style="red")
         alerts_table.add_column("Metric", style="cyan")
         alerts_table.add_column("Message", style="yellow")
         alerts_table.add_column("Time", style="blue")
-        
+
         active_alerts = [a for a in self.active_alerts if not a.resolved][-5:]  # Show last 5
         for alert in active_alerts:
             alerts_table.add_row(
                 alert.severity.value,
                 alert.metric_name,
                 alert.message[:50] + "..." if len(alert.message) > 50 else alert.message,
-                alert.timestamp.strftime("%H:%M:%S")
+                alert.timestamp.strftime("%H:%M:%S"),
             )
-        
+
         if not active_alerts:
             alerts_table.add_row("None", "All systems operational", "No active alerts", "")
-        
+
         # Create dashboard layout
         dashboard_content = f"""
 [bold blue]CloudOps Production Monitoring Dashboard[/bold blue]
 
-📊 Operations: {self.operation_metrics['total_operations']} total
-✅ Success: {self.operation_metrics['successful_operations']} 
-❌ Failed: {self.operation_metrics['failed_operations']}
-⏱️  Avg Latency: {self.operation_metrics['average_latency']:.2f}s
+📊 Operations: {self.operation_metrics["total_operations"]} total
+✅ Success: {self.operation_metrics["successful_operations"]} 
+❌ Failed: {self.operation_metrics["failed_operations"]}
+⏱️  Avg Latency: {self.operation_metrics["average_latency"]:.2f}s
 
 {metrics_table}
 
 {alerts_table}
 
-🔧 Circuit Breakers: {len([k for k, v in self.circuit_breaker_state.items() if v == 'OPEN'])} OPEN
+🔧 Circuit Breakers: {len([k for k, v in self.circuit_breaker_state.items() if v == "OPEN"])} OPEN
 ⚡ Uptime: {time.time() - self.start_time:.0f}s
 """
-        
+
         return create_panel(dashboard_content, title="Enterprise SRE Monitoring")
-    
+
     def _cleanup_metrics_history(self) -> None:
         """Clean up old metrics history to prevent memory leaks."""
         cutoff_time = datetime.now() - timedelta(hours=24)
-        self.metrics_history = [
-            entry for entry in self.metrics_history
-            if entry['timestamp'] > cutoff_time
-        ]
-    
+        self.metrics_history = [entry for entry in self.metrics_history if entry["timestamp"] > cutoff_time]
+
     # Public interface for recording operations
     def record_operation_start(self, operation_name: str) -> str:
         """
         Record the start of an operation for monitoring.
-        
+
         Args:
             operation_name: Name of the operation
-            
+
         Returns:
             Operation tracking ID
         """
         operation_id = f"{operation_name}-{int(time.time())}"
-        self.operation_metrics['total_operations'] += 1
+        self.operation_metrics["total_operations"] += 1
         return operation_id
-    
+
     def record_operation_success(self, operation_id: str, latency: float) -> None:
         """
         Record successful operation completion.
-        
+
         Args:
             operation_id: Operation tracking ID
             latency: Operation latency in seconds
         """
-        self.operation_metrics['successful_operations'] += 1
-        
+        self.operation_metrics["successful_operations"] += 1
+
         # Update latency metrics (simplified calculation)
-        total_ops = self.operation_metrics['total_operations']
-        current_avg = self.operation_metrics['average_latency']
+        total_ops = self.operation_metrics["total_operations"]
+        current_avg = self.operation_metrics["average_latency"]
         new_avg = ((current_avg * (total_ops - 1)) + latency) / total_ops
-        self.operation_metrics['average_latency'] = new_avg
-        
+        self.operation_metrics["average_latency"] = new_avg
+
         # Simplified P95 calculation (use 95% of max latency seen)
-        self.operation_metrics['p95_latency'] = max(self.operation_metrics['p95_latency'], latency * 0.95)
-    
+        self.operation_metrics["p95_latency"] = max(self.operation_metrics["p95_latency"], latency * 0.95)
+
     def record_operation_failure(self, operation_id: str, error: str) -> None:
         """
         Record failed operation.
-        
+
         Args:
             operation_id: Operation tracking ID
             error: Error message
         """
-        self.operation_metrics['failed_operations'] += 1
-    
+        self.operation_metrics["failed_operations"] += 1
+
     def is_circuit_breaker_open(self, metric_name: str) -> bool:
         """
         Check if circuit breaker is open for a specific metric.
-        
+
         Args:
             metric_name: Name of the metric to check
-            
+
         Returns:
             True if circuit breaker is open
         """
-        return self.circuit_breaker_state.get(metric_name) == 'OPEN'
+        return self.circuit_breaker_state.get(metric_name) == "OPEN"
 
 
 # Export public interface
@@ -557,28 +547,28 @@ __all__ = [
 # CLI interface for running monitoring
 if __name__ == "__main__":
     import argparse
-    
+
     parser = argparse.ArgumentParser(description="CloudOps Production Monitoring Framework")
     parser.add_argument("--interval", type=int, default=60, help="Monitoring interval in seconds")
     parser.add_argument("--demo", action="store_true", help="Run in demo mode with simulated metrics")
-    
+
     args = parser.parse_args()
-    
+
     async def main():
         monitoring = ProductionMonitoringFramework()
-        
+
         if args.demo:
             # Simulate some operations for demo
-            monitoring.operation_metrics['total_operations'] = 1000
-            monitoring.operation_metrics['successful_operations'] = 950
-            monitoring.operation_metrics['failed_operations'] = 50
-            monitoring.operation_metrics['average_latency'] = 15.5
-            monitoring.operation_metrics['p95_latency'] = 28.2
-            
+            monitoring.operation_metrics["total_operations"] = 1000
+            monitoring.operation_metrics["successful_operations"] = 950
+            monitoring.operation_metrics["failed_operations"] = 50
+            monitoring.operation_metrics["average_latency"] = 15.5
+            monitoring.operation_metrics["p95_latency"] = 28.2
+
         await monitoring.start_monitoring(args.interval)
-    
+
     # Run the monitoring framework
     try:
         asyncio.run(main())
     except KeyboardInterrupt:
-        console.print("\n[yellow]Monitoring framework stopped by user[/yellow]")
+        console.print("\n[yellow]Monitoring framework stopped by user[/yellow]")