kubiya-control-plane-api 0.9.15__py3-none-any.whl

control_plane_api/worker/utils/logging_config.py

@@ -0,0 +1,335 @@
+"""
+Centralized logging configuration for the worker process.
+
+This module provides:
+- Dynamic log level configuration via KUBIYA_CLI_LOG_LEVEL environment variable
+- Multiple log format support (pretty, json, text) via KUBIYA_LOG_FORMAT
+- Backward compatibility with CLAUDE_CODE_DEBUG
+- Sensitive data sanitization helpers
+- Unified structlog configuration
+"""
+
+import os
+import logging
+import time
+import structlog
+from typing import Any, Dict
+
+
+def get_log_level() -> int:
+    """
+    Get log level from environment variables with fallback hierarchy.
+
+    Priority:
+    1. KUBIYA_CLI_LOG_LEVEL (new, preferred)
+    2. LOG_LEVEL (generic fallback)
+    3. CLAUDE_CODE_DEBUG=true → DEBUG (deprecated, with warning)
+    4. Default: INFO
+
+    Returns:
+        int: Python logging level constant (logging.DEBUG, logging.INFO, etc.)
+    """
+    logger = structlog.get_logger(__name__)
+
+    # Check KUBIYA_CLI_LOG_LEVEL first (preferred)
+    kubiya_log_level = os.getenv("KUBIYA_CLI_LOG_LEVEL", "").upper()
+    if kubiya_log_level:
+        level_map = {
+            "DEBUG": logging.DEBUG,
+            "INFO": logging.INFO,
+            "WARNING": logging.WARNING,
+            "ERROR": logging.ERROR,
+            "CRITICAL": logging.CRITICAL,
+        }
+        if kubiya_log_level in level_map:
+            return level_map[kubiya_log_level]
+
+    # Check generic LOG_LEVEL
+    log_level = os.getenv("LOG_LEVEL", "").upper()
+    if log_level:
+        level_map = {
+            "DEBUG": logging.DEBUG,
+            "INFO": logging.INFO,
+            "WARNING": logging.WARNING,
+            "ERROR": logging.ERROR,
+            "CRITICAL": logging.CRITICAL,
+        }
+        if log_level in level_map:
+            return level_map[log_level]
+
+    # Check deprecated CLAUDE_CODE_DEBUG
+    if os.getenv("CLAUDE_CODE_DEBUG", "false").lower() == "true":
+        # Show deprecation warning (but only once, using a module-level flag)
+        if not hasattr(get_log_level, "_warned"):
+            logger.warning(
+                "deprecated_env_var",
+                old_var="CLAUDE_CODE_DEBUG",
+                new_var="KUBIYA_CLI_LOG_LEVEL",
+                message="CLAUDE_CODE_DEBUG is deprecated. Please use KUBIYA_CLI_LOG_LEVEL=DEBUG instead."
+            )
+            get_log_level._warned = True
+        return logging.DEBUG
+
+    # Default to INFO
+    return logging.INFO
+
+
+def get_log_format() -> str:
+    """
+    Get log format from environment variable.
+
+    Priority:
+    1. KUBIYA_LOG_FORMAT
+    2. Default: "pretty"
+
+    Supported formats:
+    - pretty: Human-readable colored output with emojis (default)
+    - json: JSON-formatted for log aggregation
+    - text: Simple text without colors
+
+    Returns:
+        str: Log format ("pretty", "json", or "text")
+    """
+    log_format = os.getenv("KUBIYA_LOG_FORMAT", "pretty").lower()
+    if log_format not in ["pretty", "json", "text"]:
+        # Invalid format, default to pretty
+        return "pretty"
+    return log_format
+
+
+def sanitize_value(key: str, value: Any) -> str:
+    """
+    Sanitize sensitive values based on key name.
+
+    Keys containing TOKEN, SECRET, PASSWORD, KEY, or CREDENTIAL will be masked.
+    Shows first 10 and last 5 characters for values longer than 15 chars,
+    otherwise shows "***".
+
+    Args:
+        key: The key/variable name
+        value: The value to potentially sanitize
+
+    Returns:
+        str: Sanitized value if key is sensitive, otherwise original value as string
+    """
+    sensitive_keywords = ["TOKEN", "SECRET", "PASSWORD", "KEY", "CREDENTIAL"]
+
+    # Check if key contains any sensitive keyword
+    is_sensitive = any(keyword in key.upper() for keyword in sensitive_keywords)
+
+    if not is_sensitive:
+        return str(value)
+
+    # Sanitize sensitive value
+    value_str = str(value)
+    if len(value_str) > 15:
+        return f"{value_str[:10]}...{value_str[-5:]}"
+    else:
+        return "***"
+
+
+def pretty_console_renderer(logger, name, event_dict):
+    """
+    Render logs in a pretty, human-readable format instead of JSON.
+    Uses colors and emojis for better readability.
+
+    Args:
+        logger: The logger instance
+        name: The logger name
+        event_dict: Dictionary containing log event data
+
+    Returns:
+        str: Formatted log message
+    """
+    level = event_dict.get("level", "info").upper()
+    event = event_dict.get("event", "")
+    timestamp = event_dict.get("timestamp", "")
+
+    # Extract timestamp (just time part)
+    if timestamp:
+        try:
+            time_part = timestamp.split("T")[1].split(".")[0]  # HH:MM:SS
+        except:
+            time_part = timestamp
+    else:
+        time_part = time.strftime("%H:%M:%S")
+
+    # Color codes
+    RESET = "\033[0m"
+    GRAY = "\033[90m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    RED = "\033[91m"
+    CYAN = "\033[96m"
+    BOLD = "\033[1m"
+
+    # Level icons and colors
+    level_config = {
+        "INFO": ("ℹ️", CYAN),
+        "WARNING": ("⚠️", YELLOW),
+        "ERROR": ("❌", RED),
+        "DEBUG": ("🔍", GRAY),
+    }
+
+    icon, color = level_config.get(level, ("•", RESET))
+
+    # Format the main message
+    message = f"{GRAY}[{time_part}]{RESET} {icon}  {event}"
+
+    # Add relevant context (skip internal keys)
+    skip_keys = {"level", "event", "timestamp", "logger"}
+    context_parts = []
+
+    for key, value in event_dict.items():
+        if key in skip_keys:
+            continue
+        # Format value nicely
+        if isinstance(value, bool):
+            value_str = "✓" if value else "✗"
+        elif isinstance(value, str) and len(value) > 60:
+            value_str = value[:57] + "..."
+        else:
+            value_str = str(value)
+
+        context_parts.append(f"{GRAY}{key}={RESET}{value_str}")
+
+    if context_parts:
+        message += f" {GRAY}({', '.join(context_parts)}){RESET}"
+
+    return message
+
+
+def json_renderer(logger, name, event_dict):
+    """
+    Render logs in JSON format for log aggregation systems.
+
+    Args:
+        logger: The logger instance
+        name: The logger name
+        event_dict: Dictionary containing log event data
+
+    Returns:
+        str: JSON-formatted log message
+    """
+    import json
+    return json.dumps(event_dict)
+
+
+def text_renderer(logger, name, event_dict):
+    """
+    Render logs in simple text format without colors.
+
+    Args:
+        logger: The logger instance
+        name: The logger name
+        event_dict: Dictionary containing log event data
+
+    Returns:
+        str: Simple text log message
+    """
+    level = event_dict.get("level", "info").upper()
+    event = event_dict.get("event", "")
+    timestamp = event_dict.get("timestamp", "")
+
+    # Extract timestamp (just time part)
+    if timestamp:
+        try:
+            time_part = timestamp.split("T")[1].split(".")[0]  # HH:MM:SS
+        except:
+            time_part = timestamp
+    else:
+        time_part = time.strftime("%H:%M:%S")
+
+    # Format the main message
+    message = f"[{time_part}] {level:8} {event}"
+
+    # Add relevant context (skip internal keys)
+    skip_keys = {"level", "event", "timestamp", "logger"}
+    context_parts = []
+
+    for key, value in event_dict.items():
+        if key in skip_keys:
+            continue
+        context_parts.append(f"{key}={value}")
+
+    if context_parts:
+        message += f" ({', '.join(context_parts)})"
+
+    return message
+
+
+def add_trace_context(logger, method_name, event_dict):
+    """
+    Processor that automatically adds OpenTelemetry trace context to all logs.
+
+    This enables trace-log correlation by including:
+    - trace_id: The current trace ID (32-character hex)
+    - span_id: The current span ID (16-character hex)
+
+    These IDs allow you to:
+    1. Copy trace_id from logs → Search in Jaeger UI
+    2. See which logs belong to which trace
+    3. Correlate application logs with distributed traces
+
+    Args:
+        logger: The logger instance
+        method_name: The method being called (info, warning, etc.)
+        event_dict: Dictionary containing log event data
+
+    Returns:
+        dict: Updated event_dict with trace context
+    """
+    try:
+        from control_plane_api.app.observability.optional import get_current_span
+
+        span = get_current_span()
+        if span and span.is_recording():
+            span_context = span.get_span_context()
+            if span_context and span_context.is_valid:
+                # Add trace context to every log message
+                event_dict["trace_id"] = format(span_context.trace_id, '032x')
+                event_dict["span_id"] = format(span_context.span_id, '016x')
+    except Exception:
+        # If OTEL not available or error, silently continue
+        # (graceful degradation - logs still work without tracing)
+        pass
+
+    return event_dict
+
+
+def configure_logging() -> None:
+    """
+    Configure structlog with dynamic settings from environment variables.
+
+    This function sets up structlog with:
+    - Dynamic log level from KUBIYA_CLI_LOG_LEVEL (or fallback)
+    - Dynamic log format from KUBIYA_LOG_FORMAT
+    - Appropriate renderer based on format
+    - Standard processors for context and timestamps
+    - Automatic trace context injection (trace_id, span_id)
+
+    Should be called once at application startup before any logging occurs.
+    """
+    log_level = get_log_level()
+    log_format = get_log_format()
+
+    # Select renderer based on format
+    if log_format == "json":
+        renderer = json_renderer
+    elif log_format == "text":
+        renderer = text_renderer
+    else:  # "pretty" (default)
+        renderer = pretty_console_renderer
+
+    # Configure structlog with trace context processor
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.processors.add_log_level,
+            add_trace_context,  # ← Automatically adds trace_id and span_id to all logs
+            structlog.processors.TimeStamper(fmt="iso"),
+            renderer,
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(log_level),
+        logger_factory=structlog.PrintLoggerFactory(),
+    )

control_plane_api/worker/utils/logging_helper.py

@@ -0,0 +1,326 @@
+"""Unified logging helper for clear and structured logs across the worker.
+
+This module provides consistent logging utilities to make logs easy to read and understand.
+"""
+
+import structlog
+from typing import Optional, Dict, Any
+from datetime import datetime
+
+
+class ExecutionLogger:
+    """Unified logger for execution workflows with clear, human-readable output."""
+
+    def __init__(self, logger_name: str = "worker"):
+        self.logger = structlog.get_logger(logger_name)
+
+    @staticmethod
+    def _format_execution_id(execution_id: str) -> str:
+        """Format execution ID for display (show first 8 chars)."""
+        if not execution_id:
+            return "unknown"
+        return execution_id[:8] if len(execution_id) > 8 else execution_id
+
+    @staticmethod
+    def _get_emoji(status: str) -> str:
+        """Get emoji for status."""
+        emoji_map = {
+            "started": "🚀",
+            "running": "⚙️",
+            "streaming": "📡",
+            "waiting": "⏳",
+            "completed": "✅",
+            "failed": "❌",
+            "cancelled": "🛑",
+            "retry": "🔄",
+            "warning": "⚠️",
+            "info": "ℹ️",
+        }
+        return emoji_map.get(status.lower(), "•")
+
+    def execution_started(
+        self,
+        execution_id: str,
+        agent_id: Optional[str] = None,
+        model: Optional[str] = None,
+        runtime: Optional[str] = None,
+    ):
+        """Log execution start."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"🚀 Execution Started: {exec_short}"
+
+        details = []
+        if agent_id:
+            details.append(f"agent={agent_id[:8]}")
+        if model:
+            details.append(f"model={model}")
+        if runtime:
+            details.append(f"runtime={runtime}")
+
+        if details:
+            msg += f" ({', '.join(details)})"
+
+        self.logger.info(msg, execution_id=execution_id)
+
+        # Print clear separator to distinguish between executions
+        print(f"\n{'─' * 80}")
+        print(f"🚀 Execution Started: {exec_short}")
+        if details:
+            print(f"   {' | '.join(details)}")
+        print(f"{'─' * 80}")
+
+    def execution_completed(self, execution_id: str, duration_ms: Optional[int] = None):
+        """Log execution completion."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"✅ Execution Completed: {exec_short}"
+
+        if duration_ms:
+            duration_sec = duration_ms / 1000
+            msg += f" (took {duration_sec:.2f}s)"
+
+        self.logger.info(msg, execution_id=execution_id)
+
+        # Print clear end separator
+        print(f"\n{'─' * 80}")
+        print(msg)
+        print(f"{'─' * 80}\n")
+
+    def execution_failed(
+        self,
+        execution_id: str,
+        error: str,
+        error_type: Optional[str] = None,
+        recoverable: bool = False,
+    ):
+        """Log execution failure."""
+        exec_short = self._format_execution_id(execution_id)
+
+        if recoverable:
+            msg = f"⚠️  Execution Error (Recoverable): {exec_short}"
+        else:
+            msg = f"❌ Execution Failed: {exec_short}"
+
+        self.logger.error(
+            msg,
+            execution_id=execution_id,
+            error=error,
+            error_type=error_type,
+        )
+
+        # Print clear failure separator
+        print(f"\n{'─' * 80}")
+        print(msg)
+        if error_type:
+            print(f"   Error Type: {error_type}")
+        print(f"   Error: {error}")
+        print(f"{'─' * 80}\n")
+
+    def activity_started(
+        self,
+        activity_name: str,
+        execution_id: str,
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        """Log activity start."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"⚙️  {activity_name}: {exec_short}"
+
+        if details:
+            detail_str = ", ".join([f"{k}={v}" for k, v in details.items()])
+            msg += f" ({detail_str})"
+
+        self.logger.info(msg, execution_id=execution_id, activity=activity_name)
+        print(f"  → {msg}")
+
+    def activity_completed(
+        self,
+        activity_name: str,
+        execution_id: str,
+        result: Optional[str] = None,
+    ):
+        """Log activity completion."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"✅ {activity_name}: {exec_short}"
+
+        if result:
+            msg += f" - {result}"
+
+        self.logger.info(msg, execution_id=execution_id, activity=activity_name)
+        print(f"  ✓ {msg}")
+
+    def activity_failed(
+        self,
+        activity_name: str,
+        execution_id: str,
+        error: str,
+        will_retry: bool = False,
+    ):
+        """Log activity failure."""
+        exec_short = self._format_execution_id(execution_id)
+
+        if will_retry:
+            msg = f"🔄 {activity_name} Failed (Retrying): {exec_short}"
+        else:
+            msg = f"❌ {activity_name} Failed: {exec_short}"
+
+        self.logger.error(
+            msg,
+            execution_id=execution_id,
+            activity=activity_name,
+            error=error,
+        )
+        print(f"  ✗ {msg}")
+        print(f"    Error: {error}")
+
+    def streaming_started(self, execution_id: str):
+        """Log streaming start."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"📡 Streaming Response: {exec_short}"
+        self.logger.info(msg, execution_id=execution_id)
+        print(f"  → {msg}")
+
+    def streaming_progress(
+        self,
+        execution_id: str,
+        chunks_received: int,
+        response_length: int,
+    ):
+        """Log streaming progress (only at milestones)."""
+        exec_short = self._format_execution_id(execution_id)
+
+        # Only log at milestones to avoid spam
+        if chunks_received % 50 == 0 or chunks_received < 5:
+            msg = f"📡 Streaming: {exec_short} - {chunks_received} chunks, {response_length} chars"
+            self.logger.debug(msg, execution_id=execution_id)
+            # Only print for first few chunks or milestones
+            if chunks_received < 5:
+                print(f"    {msg}")
+
+    def status_update(
+        self,
+        execution_id: str,
+        old_status: Optional[str],
+        new_status: str,
+        reason: Optional[str] = None,
+    ):
+        """Log status update."""
+        exec_short = self._format_execution_id(execution_id)
+        emoji = self._get_emoji(new_status)
+
+        if old_status:
+            msg = f"{emoji} Status Change: {exec_short} ({old_status} → {new_status})"
+        else:
+            msg = f"{emoji} Status: {exec_short} ({new_status})"
+
+        if reason:
+            msg += f" - {reason}"
+
+        self.logger.info(msg, execution_id=execution_id, status=new_status)
+        print(f"  {msg}")
+
+    def workflow_signal(
+        self,
+        execution_id: str,
+        signal_name: str,
+        details: Optional[str] = None,
+    ):
+        """Log workflow signal received."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"📨 Signal Received: {exec_short} ({signal_name})"
+
+        if details:
+            msg += f" - {details}"
+
+        self.logger.info(msg, execution_id=execution_id, signal=signal_name)
+        print(f"  {msg}")
+
+    def turn_started(self, execution_id: str, turn_number: int):
+        """Log conversation turn start."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"💬 Turn {turn_number} Started: {exec_short}"
+        self.logger.info(msg, execution_id=execution_id, turn=turn_number)
+        print(f"\n  {msg}")
+
+    def turn_completed(
+        self,
+        execution_id: str,
+        turn_number: int,
+        tokens_used: Optional[int] = None,
+    ):
+        """Log conversation turn completion."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"💬 Turn {turn_number} Completed: {exec_short}"
+
+        if tokens_used:
+            msg += f" ({tokens_used} tokens)"
+
+        self.logger.info(msg, execution_id=execution_id, turn=turn_number)
+        print(f"  {msg}")
+
+    def tool_call_started(
+        self,
+        execution_id: str,
+        tool_name: str,
+        tool_id: Optional[str] = None,
+    ):
+        """Log tool call start."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"🔧 Tool Called: {tool_name}"
+
+        if tool_id:
+            msg += f" (id: {tool_id[:8]})"
+
+        self.logger.info(
+            msg,
+            execution_id=execution_id,
+            tool_name=tool_name,
+            tool_id=tool_id,
+        )
+        print(f"    → {msg}")
+
+    def tool_call_completed(
+        self,
+        execution_id: str,
+        tool_name: str,
+        success: bool,
+        duration_ms: Optional[int] = None,
+    ):
+        """Log tool call completion."""
+        exec_short = self._format_execution_id(execution_id)
+
+        if success:
+            emoji = "✅"
+            status = "completed"
+        else:
+            emoji = "❌"
+            status = "failed"
+
+        msg = f"{emoji} Tool {status}: {tool_name}"
+
+        if duration_ms:
+            msg += f" (took {duration_ms}ms)"
+
+        self.logger.info(
+            msg,
+            execution_id=execution_id,
+            tool_name=tool_name,
+            success=success,
+        )
+        print(f"    {msg}")
+
+    def warning(self, execution_id: str, message: str, details: Optional[Dict] = None):
+        """Log warning."""
+        exec_short = self._format_execution_id(execution_id)
+        msg = f"⚠️  Warning: {exec_short} - {message}"
+
+        self.logger.warning(msg, execution_id=execution_id, **(details or {}))
+        print(f"  {msg}")
+
+    def debug(self, execution_id: str, message: str):
+        """Log debug message (only in debug mode)."""
+        exec_short = self._format_execution_id(execution_id)
+        self.logger.debug(message, execution_id=execution_id)
+
+
+# Global logger instance
+execution_logger = ExecutionLogger()