ipulse-shared-core-ftredge 20.0.1__py3-none-any.whl → 22.1.1__py3-none-any.whl

ipulse_shared_core_ftredge/monitoring/microservmon.py

@@ -1,526 +0,0 @@
-"""
-Microservmon - Lightweight logging collector for microservice requests and functions and API endpoints
-"""
-import uuid
-import time
-from datetime import datetime, timezone
-from typing import Dict, Any, Optional
-from contextlib import contextmanager
-from collections import defaultdict
-from ipulse_shared_base_ftredge import (LogLevel, AbstractResource,
-                                      ProgressStatus, Action,
-                                      Alert, StructLog)
-from ipulse_shared_base_ftredge.status import StatusCounts, map_progress_status_to_log_level, eval_statuses
-
-class Microservmon:
-    """
-    Microservmon is a lightweight version of Pipelinemon designed specifically for monitoring
-    microservice events such as HTTP Requests, PubSub Triggers etc within execution environment like
-    Cloud Functions, Cloud Run etc.
-
-    It provides:
-    1. Structured logging with context tracking
-    2. Performance metrics capture per trace
-    3. Microservice health monitoring
-    4. Integration with FastAPI request/response cycle
-    5. Memory-efficient trace lifecycle management
-    """
-
-    def __init__(self, logger,
-                 base_context: str,
-                 microservice_name: str,
-                 max_log_field_len: Optional[int] = 8000,
-                 max_log_dict_byte_size: Optional[float] = 256 * 1024 * 0.80,
-                 exclude_none_from_logs: bool = True):
-        """
-        Initialize Microservmon with basic configuration.
-
-        Args:
-            logger: The logger instance to use for logging
-            base_context: Base context information for all logs
-            microservice_name: Name of the microservice being monitored
-            max_log_field_len: Maximum length for any string field in logs
-            max_log_dict_byte_size: Maximum byte size for log dictionary
-            exclude_none_from_logs: Whether to exclude None values from log output (default: True)
-        """
-        # Set up microservice tracking details
-        self._microservice_name = microservice_name
-        self._microservice_start_time = time.time()
-
-        timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
-        uuid_suffix = str(uuid.uuid4())[:8]  # Take first 8 chars of UUID
-        self._id = f"{timestamp}_{uuid_suffix}"
-
-        # Set up context handling
-        self._base_context = base_context
-        self._context_stack = []
-
-        # Configure logging
-        self._logger = logger
-        self._max_log_field_len = max_log_field_len
-        self._max_log_dict_byte_size = max_log_dict_byte_size
-        self._exclude_none_from_logs = exclude_none_from_logs
-
-        # Trace-based tracking - key change from original
-        self._active_traces: Dict[str, Dict[str, Any]] = {}
-
-        # Microservice-wide metrics (aggregated across all traces)
-        self._microservice_metrics = {
-            "total_traces": 0,
-            "active_traces": 0,
-            "completed_traces": 0,
-            "failed_traces": 0,
-            "by_event_count": defaultdict(int),
-            "by_level_code_count": defaultdict(int),
-            "status_counts": StatusCounts()  # Add StatusCounts for better status tracking
-        }
-
-        # Log microservice startup
-        self._log_microservice_start()
-
-    def _log_microservice_start(self):
-        """Log the microservice instance startup."""
-        startup_log = StructLog(
-            level=LogLevel.INFO,
-            resource=AbstractResource.MICROSERVMON,
-            action=Action.EXECUTE,
-            progress_status=ProgressStatus.STARTED,
-            description=f"Microservice {self.microservice_name} instance started",
-            collector_id=self.id,
-            base_context=self.base_context,
-            context="microservice_startup"
-        )
-        self._write_log_to_logger(startup_log)
-
-    @property
-    def id(self) -> str:
-        """Get the unique ID for this microservice instance."""
-        return self._id
-
-    @property
-    def base_context(self) -> str:
-        """Get the base context for this microservice execution."""
-        return self._base_context
-
-    @property
-    def microservice_name(self) -> str:
-        """Get the microservice name being monitored."""
-        return self._microservice_name
-
-    @property
-    def microservice_metrics(self) -> Dict[str, Any]:
-        """Get the current microservice-wide metrics."""
-        metrics = self._microservice_metrics.copy()
-        metrics["by_event_count"] = dict(metrics["by_event_count"])
-        metrics["by_level_code_count"] = dict(metrics["by_level_code_count"])
-        return metrics
-
-    @property
-    def active_trace_count(self) -> int:
-        """Get count of currently active traces."""
-        return len(self._active_traces)
-
-    @property
-    def current_context(self) -> str:
-        """Get the current context stack as a string."""
-        return " >> ".join(self._context_stack) if self._context_stack else "root"
-
-    @property
-    def current_trace_id(self) -> Optional[str]:
-        """Get the current trace_id from context stack (first element if present)."""
-        return self._context_stack[0] if self._context_stack else None
-
-    @contextmanager
-    def context(self, context_name: str):
-        """
-        Context manager for tracking execution context.
-        Note: This is for microservice-wide context, not trace-specific.
-
-        Args:
-            context_name: The name of the current execution context
-        """
-        self.push_context(context_name)
-        try:
-            yield
-        finally:
-            self.pop_context()
-
-    @contextmanager
-    def trace(self, description: Optional[str] = None):
-        """
-        Context manager for trace lifecycle management.
-        Automatically starts and ends a trace, managing the context stack.
-
-        Args:
-            description: Optional description of the trace
-
-        Yields:
-            str: The trace_id for the current trace
-        """
-        trace_id = self.start_trace(description)
-        try:
-            yield trace_id
-        finally:
-            self.end_trace(trace_id)
-
-    def push_context(self, context: str):
-        """Add a context level to the stack."""
-        self._context_stack.append(context)
-
-    def pop_context(self):
-        """Remove the most recent context from the stack."""
-        if self._context_stack:
-            return self._context_stack.pop()
-
-    def start_trace(self, description: Optional[str] = None) -> str:
-        """
-        Start monitoring a new trace (request/event) with auto-generated trace ID.
-
-        Args:
-            description: Optional description of the trace
-
-        Returns:
-            str: Auto-generated trace_id
-        """
-        # Auto-generate trace ID
-        timestamp = datetime.now(timezone.utc).strftime('%H%M%S_%f')[:-3]  # Include milliseconds
-        trace_id = f"trace_{timestamp}_{str(uuid.uuid4())[:6]}"
-
-        start_time = time.time()
-
-        # Initialize trace metrics with StatusCounts
-        self._active_traces[trace_id] = {
-            "start_time": start_time,
-            "status": ProgressStatus.IN_PROGRESS.name,
-            "by_event_count": defaultdict(int),
-            "by_level_code_count": defaultdict(int),
-            "status_counts": StatusCounts()  # Track status progression within trace
-        }
-
-        # Add initial status to trace
-        self._active_traces[trace_id]["status_counts"].add_status(ProgressStatus.IN_PROGRESS)
-
-        # Update microservice metrics
-        self._microservice_metrics["total_traces"] += 1
-        self._microservice_metrics["active_traces"] = len(self._active_traces)
-        self._microservice_metrics["status_counts"].add_status(ProgressStatus.IN_PROGRESS)
-
-        # Push trace_id as first element in context stack
-        self._context_stack.insert(0, trace_id)
-
-        # Log the trace start using StructLog
-        msg = description if description else f"Starting trace {trace_id}"
-        start_log = StructLog(
-            level=LogLevel.INFO,
-            description=msg,
-            resource=AbstractResource.MICROSERVICE_TRACE,
-            action=Action.EXECUTE,
-            progress_status=ProgressStatus.IN_PROGRESS
-        )
-        self.log(start_log, trace_id)
-
-        return trace_id
-
-    def end_trace(self, trace_id: str, force_status: Optional[ProgressStatus] = None) -> None:
-        """
-        End monitoring for a trace and record final metrics.
-        Status is automatically calculated based on trace metrics unless forced.
-
-        Args:
-            trace_id: The trace identifier to end
-            force_status: Optional status to force, overriding automatic calculation
-        """
-        if trace_id not in self._active_traces:
-            # Log warning about unknown trace using StructLog
-            warning_log = StructLog(
-                level=LogLevel.WARNING,
-                description=f"Attempted to end unknown trace: {trace_id}",
-                resource=AbstractResource.MICROSERVICE_TRACE,
-                action=Action.EXECUTE,
-                progress_status=ProgressStatus.UNFINISHED,
-                alert=Alert.NOT_FOUND
-            )
-            self.log(warning_log)
-            return
-
-        trace_metrics = self._active_traces[trace_id]
-
-        # Calculate duration
-        end_time = time.time()
-        duration_ms = int((end_time - trace_metrics["start_time"]) * 1000)
-        trace_metrics["duration_ms"] = duration_ms
-        trace_metrics["end_time"] = end_time
-
-        # Get readable error/warning counts using existing helper functions
-        error_count = self._get_error_count_for_trace(trace_id)
-        warning_count = self._get_warning_count_for_trace(trace_id)
-
-        # Use status helpers for intelligent status calculation
-        if force_status is not None:
-            final_status = force_status
-            level = map_progress_status_to_log_level(final_status)
-        else:
-            # Build status list based on log levels for evaluation
-            status_list = []
-            if error_count > 0:
-                status_list.append(ProgressStatus.FINISHED_WITH_ISSUES)
-            elif warning_count > 0:
-                status_list.append(ProgressStatus.DONE_WITH_WARNINGS)
-            else:
-                status_list.append(ProgressStatus.DONE)
-
-            # Use eval_statuses for consistent status evaluation
-            final_status = eval_statuses(
-                status_list,
-                fail_or_unfinish_if_any_pending=True,  # Since this is end of trace
-                issues_allowed=True
-            )
-            level = map_progress_status_to_log_level(final_status)
-
-        # Update trace status
-        trace_metrics["status"] = final_status.name
-        trace_metrics["status_counts"].add_status(final_status)
-
-        # Prepare summary message
-        status_source = "FORCED" if force_status is not None else "AUTO"
-        summary_msg = (
-            f"Trace {trace_id} completed with status {final_status.name} ({status_source}). "
-            f"Duration: {duration_ms}ms. "
-            f"Errors: {error_count}, Warnings: {warning_count}"
-        )
-
-        # Log the completion using StructLog
-        completion_log = StructLog(
-            level=level,
-            description=summary_msg,
-            resource=AbstractResource.MICROSERVMON,
-            action=Action.EXECUTE,
-            progress_status=final_status
-        )
-        self.log(completion_log, trace_id)
-
-        # Update microservice-wide metrics using StatusCounts
-        self._microservice_metrics["completed_traces"] += 1
-        if final_status in ProgressStatus.failure_statuses():
-            self._microservice_metrics["failed_traces"] += 1
-
-        # Add final status to microservice status counts
-        self._microservice_metrics["status_counts"].add_status(final_status)
-
-        # Aggregate trace metrics to microservice level
-        for event, count in trace_metrics["by_event_count"].items():
-            self._microservice_metrics["by_event_count"][event] += count
-        for level_code, count in trace_metrics["by_level_code_count"].items():
-            self._microservice_metrics["by_level_code_count"][level_code] += count
-
-        # Clean up trace data from memory
-        del self._active_traces[trace_id]
-        self._microservice_metrics["active_traces"] = len(self._active_traces)
-
-        # Remove trace_id from context stack if it's the first element
-        if self._context_stack and self._context_stack[0] == trace_id:
-            self._context_stack.pop(0)
-
-    def log(self, log: StructLog, trace_id: Optional[str] = None) -> None:
-        """
-        Log a StructLog message with trace context.
-
-        Args:
-            log: StructLog instance to log
-            trace_id: Optional trace ID for trace-specific logging. If not provided,
-                     will use current_trace_id from context stack
-        """
-        # Use provided trace_id or infer from context stack
-        effective_trace_id = trace_id or self.current_trace_id
-
-        # Calculate elapsed time
-        elapsed_ms = None
-        if effective_trace_id and effective_trace_id in self._active_traces:
-            start_time = self._active_traces[effective_trace_id]["start_time"]
-            elapsed_ms = int((time.time() - start_time) * 1000)
-        else:
-            # Use microservice start time if no trace
-            elapsed_ms = int((time.time() - self._microservice_start_time) * 1000)
-
-        # Set microservice-specific context on the log
-        log.collector_id = self.id
-        log.base_context = self.base_context
-
-        # Context with trace_id first, then other context elements
-        if effective_trace_id:
-            context_parts = [effective_trace_id]
-            # Add non-trace_id context elements (skip first element if it's trace_id)
-            other_contexts = [ctx for ctx in self._context_stack if ctx != effective_trace_id]
-            context_parts.extend(other_contexts)
-            log.context = " >> ".join(context_parts)
-        else:
-            log.context = self.current_context
-
-        log.trace_id = effective_trace_id
-
-        # Append elapsed time to existing notes
-        existing_note = log.note or ""
-        elapsed_note = f"elapsed_ms: {elapsed_ms}"
-        log.note = f"{existing_note}; {elapsed_note}" if existing_note else elapsed_note
-
-        # Update metrics for the trace or microservice
-        self._update_counts(log, effective_trace_id)
-
-        # Write to logger
-        self._write_log_to_logger(log)
-
-    def _update_counts(self, log: StructLog, trace_id: Optional[str] = None):
-        """Update counts for event tracking."""
-        event_tuple = log.getEvent()
-        level = log.level
-
-        # Update trace-specific metrics if trace_id provided
-        if trace_id and trace_id in self._active_traces:
-            trace_metrics = self._active_traces[trace_id]
-            trace_metrics["by_event_count"][event_tuple] += 1
-            trace_metrics["by_level_code_count"][level.value] += 1
-
-    def _get_error_count_for_trace(self, trace_id: str) -> int:
-        """Get total error count (ERROR + CRITICAL) for a specific trace."""
-        if trace_id not in self._active_traces:
-            return 0
-
-        trace_metrics = self._active_traces[trace_id]
-        return (trace_metrics["by_level_code_count"].get(LogLevel.ERROR.value, 0) +
-                trace_metrics["by_level_code_count"].get(LogLevel.CRITICAL.value, 0))
-
-    def _get_warning_count_for_trace(self, trace_id: str) -> int:
-        """Get warning count for a specific trace."""
-        if trace_id not in self._active_traces:
-            return 0
-
-        trace_metrics = self._active_traces[trace_id]
-        return trace_metrics["by_level_code_count"].get(LogLevel.WARNING.value, 0)
-
-    def get_readable_level_counts_for_trace(self, trace_id: str) -> Dict[str, int]:
-        """Get readable level counts for a specific trace."""
-        if trace_id not in self._active_traces:
-            return {}
-
-        trace_metrics = self._active_traces[trace_id]
-        readable_counts = {}
-
-        for level_code, count in trace_metrics["by_level_code_count"].items():
-            if count > 0:
-                # Convert level code back to LogLevel enum and get name
-                for level in LogLevel:
-                    if level.value == level_code:
-                        readable_counts[level.name] = count
-                        break
-
-        return readable_counts
-
-    def get_readable_microservice_level_counts(self) -> Dict[str, int]:
-        """Get readable level counts for the entire microservice."""
-        readable_counts = {}
-
-        for level_code, count in self._microservice_metrics["by_level_code_count"].items():
-            if count > 0:
-                # Convert level code back to LogLevel enum and get name
-                for level in LogLevel:
-                    if level.value == level_code:
-                        readable_counts[level.name] = count
-                        break
-
-        return readable_counts
-
-    def get_microservice_status_summary(self) -> Dict[str, Any]:
-        """Get comprehensive status summary using StatusCounts"""
-        status_counts = self._microservice_metrics["status_counts"]
-
-        return {
-            "microservice_name": self.microservice_name,
-            "microservice_id": self.id,
-            "active_traces": self.active_trace_count,
-            "total_traces": self._microservice_metrics["total_traces"],
-            "completed_traces": self._microservice_metrics["completed_traces"],
-            "failed_traces": self._microservice_metrics["failed_traces"],
-            "status_breakdown": status_counts.get_count_breakdown(),
-            "completion_rate": status_counts.completion_rate,
-            "success_rate": status_counts.success_rate,
-            "has_issues": status_counts.has_issues,
-            "has_failures": status_counts.has_failures,
-            "readable_level_counts": self.get_readable_microservice_level_counts()
-        }
-
-    def evaluate_microservice_health(self) -> Dict[str, Any]:
-        """Evaluate overall microservice health using status helpers"""
-        status_counts = self._microservice_metrics["status_counts"]
-
-        # Determine overall health status
-        if status_counts.has_failures:
-            health_status = "UNHEALTHY"
-            health_level = LogLevel.ERROR
-        elif status_counts.has_issues:
-            health_status = "DEGRADED"
-            health_level = LogLevel.WARNING
-        elif status_counts.has_warnings:
-            health_status = "WARNING"
-            health_level = LogLevel.WARNING
-        else:
-            health_status = "HEALTHY"
-            health_level = LogLevel.INFO
-
-        return {
-            "health_status": health_status,
-            "health_level": health_level.name,
-            "summary": status_counts.get_summary(),
-            "metrics": self.get_microservice_status_summary()
-        }
-
-    def log_health_check(self) -> None:
-        """Log a health check using the status evaluation"""
-        health = self.evaluate_microservice_health()
-
-        health_log = StructLog(
-            level=LogLevel[health["health_level"]],
-            description=f"Microservice health check: {health['health_status']}",
-            resource=AbstractResource.MICROSERVMON,
-            action=Action.VALIDATE,
-            progress_status=ProgressStatus.DONE,
-            note=health["summary"]
-        )
-        self.log(health_log)
-
-    # Legacy compatibility methods
-    def start(self, description: Optional[str] = None) -> None:
-        """Legacy method for backward compatibility."""
-        trace_id = self.start_trace(description)
-
-    def end(self, force_status: Optional[ProgressStatus] = None) -> Dict[str, Any]:
-        """Legacy method for backward compatibility."""
-        # Find the most recent trace or create a dummy one
-        if self._active_traces:
-            trace_id = max(self._active_traces.keys())
-            self.end_trace(trace_id, force_status)
-            # Return microservice metrics for legacy compatibility
-            return self.microservice_metrics
-        return {}
-
-    @property
-    def metrics(self) -> Dict[str, Any]:
-        """Legacy property for backward compatibility."""
-        return self.microservice_metrics
-
-    def _write_log_to_logger(self, log: StructLog):
-        """Write structured log to the logger."""
-        log_dict = log.to_dict(
-            max_field_len=self._max_log_field_len,
-            byte_size_limit=self._max_log_dict_byte_size,
-            exclude_none=self._exclude_none_from_logs
-        )
-
-        # Write to logger based on level
-        if log.level.value >= LogLevel.ERROR.value:
-            self._logger.error(log_dict)
-        elif log.level.value >= LogLevel.WARNING.value:
-            self._logger.warning(log_dict)
-        elif log.level.value >= LogLevel.INFO.value:
-            self._logger.info(log_dict)
-        else:
-            self._logger.debug(log_dict)