kubiya-control-plane-api 0.9.15__py3-none-any.whl

control_plane_api/worker/control_plane_client.py

@@ -0,0 +1,1491 @@
+"""
+Control Plane Client - Clean API for worker to communicate with Control Plane.
+
+This centralizes all HTTP and WebSocket communication between worker and Control Plane,
+providing a clean interface for:
+- Event streaming (real-time UI updates via WebSocket or HTTP fallback)
+- Session persistence (history storage)
+- Metadata caching (execution types)
+- Skill resolution
+- Bi-directional control messages
+
+Usage:
+    from control_plane_client import get_control_plane_client
+
+    client = get_control_plane_client()
+    await client.start_websocket()  # If WebSocket enabled
+    await client.publish_event_async(execution_id, "message_chunk", {...})
+    client.persist_session(execution_id, session_id, user_id, messages)
+"""
+
+import os
+import httpx
+import asyncio
+import threading
+from datetime import datetime, timezone
+from typing import Optional, Dict, List, Any
+import structlog
+
+logger = structlog.get_logger()
+
+
+class ControlPlaneClient:
+    """Client for communicating with the Control Plane API from workers."""
+
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        websocket_enabled: bool = False,
+        websocket_url: Optional[str] = None,
+        worker_id: Optional[str] = None,
+        event_bus_config: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Initialize Control Plane client.
+
+        Args:
+            base_url: Control Plane URL (e.g., http://localhost:8000)
+            api_key: Kubiya API key for authentication
+            websocket_enabled: Whether WebSocket is enabled
+            websocket_url: WebSocket URL if enabled
+            worker_id: Worker ID for WebSocket connection
+            event_bus_config: Optional event bus configuration dict
+        """
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.headers = {"Authorization": f"UserKey {api_key}"}
+        self.worker_id = worker_id
+
+        # Event bus manager for multi-provider support
+        self.event_bus_manager = None
+        if event_bus_config:
+            try:
+                from control_plane_api.app.lib.event_bus.manager import (
+                    EventBusManager,
+                    EventBusManagerConfig,
+                )
+                from control_plane_api.app.lib.event_bus.providers.http_provider import HTTPConfig
+                from control_plane_api.app.lib.event_bus.providers.redis_provider import RedisConfig
+                from control_plane_api.app.lib.event_bus.providers.websocket_provider import WebSocketConfig
+
+                # Parse config dicts into config objects
+                parsed_config = {}
+
+                if "http" in event_bus_config and event_bus_config["http"]:
+                    parsed_config["http"] = HTTPConfig(**event_bus_config["http"])
+
+                if "redis" in event_bus_config and event_bus_config["redis"]:
+                    parsed_config["redis"] = RedisConfig(**event_bus_config["redis"])
+
+                if "websocket" in event_bus_config and event_bus_config["websocket"]:
+                    parsed_config["websocket"] = WebSocketConfig(**event_bus_config["websocket"])
+
+                if "nats" in event_bus_config and event_bus_config["nats"]:
+                    try:
+                        from control_plane_api.app.lib.event_bus.providers.nats_provider import NATSConfig
+                        parsed_config["nats"] = NATSConfig(**event_bus_config["nats"])
+                    except ImportError:
+                        logger.warning("nats_provider_not_installed", message="Install with: pip install kubiya-control-plane-api[nats]")
+
+                manager_config = EventBusManagerConfig(**parsed_config)
+                self.event_bus_manager = EventBusManager(manager_config)
+                logger.info(
+                    "worker_event_bus_initialized",
+                    worker_id=worker_id[:8] if worker_id else "unknown",
+                    providers=list(parsed_config.keys())
+                )
+            except ImportError as e:
+                logger.warning(
+                    "event_bus_dependencies_missing",
+                    error=str(e),
+                    message="Install event bus dependencies with: pip install kubiya-control-plane-api[event-bus]"
+                )
+            except Exception as e:
+                logger.error(
+                    "worker_event_bus_init_failed",
+                    error=str(e),
+                    worker_id=worker_id[:8] if worker_id else "unknown"
+                )
+
+        # Thread-local storage for event loop reuse in sync context
+        # This prevents creating a new event loop per publish_event() call
+        self._thread_local = threading.local()
+
+        # Use BOTH sync and async clients for different use cases
+        # Sync client for backwards compatibility with non-async code
+        self._client = httpx.Client(
+            timeout=httpx.Timeout(30.0, connect=5.0, read=30.0, write=10.0),
+            limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
+        )
+
+        # Async client for streaming/real-time operations
+        # Longer read timeout to handle streaming scenarios
+        self._async_client = httpx.AsyncClient(
+            timeout=httpx.Timeout(60.0, connect=5.0, read=60.0, write=10.0),
+            limits=httpx.Limits(max_connections=20, max_keepalive_connections=10),
+        )
+
+        # WebSocket client for persistent connection
+        self.websocket_client: Optional[Any] = None
+
+        # Initialize WebSocket client if enabled and environment supports it
+        if websocket_enabled and websocket_url and worker_id:
+            from control_plane_api.worker.utils.environment import should_use_websocket
+
+            if should_use_websocket():
+                from control_plane_api.worker.websocket_client import WorkerWebSocketClient
+
+                self.websocket_client = WorkerWebSocketClient(
+                    worker_id=worker_id,
+                    websocket_url=websocket_url,
+                    api_key=api_key,
+                    on_control_message=self._handle_control_message
+                )
+                logger.info("websocket_client_initialized", worker_id=worker_id[:8])
+            else:
+                logger.info("websocket_skipped_serverless_environment")
+
+        # SSE stream completion tracking for single execution mode
+        # This allows the worker to wait for SSE streaming to complete before shutdown
+        self._sse_stream_completed: Dict[str, asyncio.Event] = {}
+        self._sse_completion_lock = asyncio.Lock()
+
+    def __del__(self):
+        """Close the HTTP clients on cleanup."""
+        try:
+            self._client.close()
+        except:
+            pass
+        # Async client cleanup happens via context manager or explicit close
+
+    def _get_thread_event_loop(self) -> asyncio.AbstractEventLoop:
+        """
+        Get or create a persistent event loop for the current thread.
+
+        This reuses the same event loop for all publish_event() calls within
+        a thread, preventing resource leaks and "await wasn't used with future"
+        errors that occur when creating a new loop per call.
+
+        Returns:
+            The thread-local event loop
+        """
+        if not hasattr(self._thread_local, 'loop') or self._thread_local.loop is None or self._thread_local.loop.is_closed():
+            self._thread_local.loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._thread_local.loop)
+            logger.debug(
+                "created_thread_local_event_loop",
+                thread_id=threading.current_thread().ident,
+                thread_name=threading.current_thread().name,
+            )
+        return self._thread_local.loop
+
+    def close_thread_event_loop(self):
+        """
+        Close the thread-local event loop if it exists.
+
+        Call this when the thread is done publishing events (e.g., at end of
+        Agno streaming execution) to properly clean up resources.
+        """
+        if hasattr(self._thread_local, 'loop') and self._thread_local.loop is not None:
+            loop = self._thread_local.loop
+            if not loop.is_closed():
+                try:
+                    # Cancel any pending tasks
+                    pending = asyncio.all_tasks(loop)
+                    for task in pending:
+                        task.cancel()
+
+                    # Run loop until all tasks are cancelled
+                    if pending:
+                        loop.run_until_complete(
+                            asyncio.gather(*pending, return_exceptions=True)
+                        )
+
+                    loop.close()
+                    logger.debug(
+                        "closed_thread_local_event_loop",
+                        thread_id=threading.current_thread().ident,
+                        pending_tasks_cancelled=len(pending) if pending else 0,
+                    )
+                except Exception as e:
+                    logger.warning(
+                        "thread_event_loop_close_error",
+                        error=str(e),
+                        thread_id=threading.current_thread().ident,
+                    )
+            self._thread_local.loop = None
+
+    # =========================================================================
+    # SSE Stream Completion Tracking (for single execution mode)
+    # =========================================================================
+
+    def register_execution_for_sse_tracking(self, execution_id: str):
+        """
+        Register an execution for SSE completion tracking.
+
+        Call this when an execution starts so the worker can later wait for
+        the SSE stream to complete before shutting down.
+
+        Args:
+            execution_id: The execution ID to track
+        """
+        if execution_id not in self._sse_stream_completed:
+            self._sse_stream_completed[execution_id] = asyncio.Event()
+            logger.debug(
+                "sse_tracking_registered",
+                execution_id=execution_id[:8] if execution_id else None
+            )
+
+    def mark_sse_stream_completed(self, execution_id: str):
+        """
+        Signal that SSE streaming has completed for an execution.
+
+        Call this from the SSE streamer after sending the 'done' event.
+
+        Args:
+            execution_id: The execution ID whose SSE stream completed
+        """
+        if execution_id in self._sse_stream_completed:
+            self._sse_stream_completed[execution_id].set()
+            logger.info(
+                "sse_stream_marked_completed",
+                execution_id=execution_id[:8] if execution_id else None
+            )
+        else:
+            # Auto-register and mark complete if not pre-registered
+            self._sse_stream_completed[execution_id] = asyncio.Event()
+            self._sse_stream_completed[execution_id].set()
+            logger.debug(
+                "sse_stream_marked_completed_auto_registered",
+                execution_id=execution_id[:8] if execution_id else None
+            )
+
+    async def wait_for_sse_stream_completion(
+        self,
+        execution_id: str,
+        timeout: float = 30.0
+    ) -> bool:
+        """
+        Wait for SSE stream to complete, with timeout.
+
+        Call this from the single execution monitor before shutting down
+        to ensure all SSE events have been sent to the client.
+
+        Args:
+            execution_id: The execution ID to wait for
+            timeout: Maximum seconds to wait (default: 30s)
+
+        Returns:
+            True if SSE stream completed, False if timeout reached
+        """
+        # Auto-register if not already tracked
+        if execution_id not in self._sse_stream_completed:
+            self._sse_stream_completed[execution_id] = asyncio.Event()
+
+        try:
+            await asyncio.wait_for(
+                self._sse_stream_completed[execution_id].wait(),
+                timeout=timeout
+            )
+            logger.info(
+                "sse_stream_wait_completed",
+                execution_id=execution_id[:8] if execution_id else None
+            )
+            return True
+        except asyncio.TimeoutError:
+            logger.warning(
+                "sse_stream_wait_timeout",
+                execution_id=execution_id[:8] if execution_id else None,
+                timeout_seconds=timeout
+            )
+            return False
+
+    def cleanup_sse_tracking(self, execution_id: str):
+        """
+        Clean up SSE tracking for an execution.
+
+        Call this after the execution is fully complete and the worker
+        has confirmed SSE streaming is done.
+
+        Args:
+            execution_id: The execution ID to clean up
+        """
+        if execution_id in self._sse_stream_completed:
+            del self._sse_stream_completed[execution_id]
+            logger.debug(
+                "sse_tracking_cleaned_up",
+                execution_id=execution_id[:8] if execution_id else None
+            )
+
+    async def initialize_event_bus(self):
+        """Initialize event bus manager asynchronously with connection testing."""
+        if self.event_bus_manager and not self.event_bus_manager.is_initialized():
+            try:
+                await self.event_bus_manager.initialize()
+
+                # Test provider connectivity (especially Redis)
+                provider_health = {}
+                for provider_name, provider in self.event_bus_manager.providers.items():
+                    try:
+                        health = await provider.health_check()
+                        provider_health[provider_name] = health.get("healthy", False)
+                    except Exception as e:
+                        logger.warning(
+                            "provider_health_check_failed",
+                            provider=provider_name,
+                            error=str(e),
+                            worker_id=self.worker_id[:8] if self.worker_id else "unknown"
+                        )
+                        provider_health[provider_name] = False
+
+                # Log provider status
+                healthy_providers = [name for name, healthy in provider_health.items() if healthy]
+                unhealthy_providers = [name for name, healthy in provider_health.items() if not healthy]
+
+                if healthy_providers:
+                    logger.info(
+                        "worker_event_bus_ready",
+                        worker_id=self.worker_id[:8] if self.worker_id else "unknown",
+                        providers=self.event_bus_manager.get_provider_names(),
+                        healthy_providers=healthy_providers,
+                        unhealthy_providers=unhealthy_providers if unhealthy_providers else None
+                    )
+
+                    # If Redis failed but was configured, log warning about falling back to HTTP
+                    if "redis" in unhealthy_providers:
+                        logger.warning(
+                            "redis_connection_failed_will_fallback",
+                            worker_id=self.worker_id[:8] if self.worker_id else "unknown",
+                            message="Redis unavailable, will fallback to HTTP endpoint for event streaming"
+                        )
+                else:
+                    logger.warning(
+                        "all_event_bus_providers_unhealthy",
+                        worker_id=self.worker_id[:8] if self.worker_id else "unknown",
+                        providers=list(provider_health.keys()),
+                        message="Will fallback to HTTP endpoint for event streaming"
+                    )
+
+            except Exception as e:
+                logger.error(
+                    "worker_event_bus_init_failed",
+                    error=str(e),
+                    worker_id=self.worker_id[:8] if self.worker_id else "unknown"
+                )
+                # Don't fail initialization - just won't use event bus
+                self.event_bus_manager = None
+
+    async def aclose(self):
+        """Async cleanup for async client and event bus."""
+        try:
+            # Shutdown event bus first
+            if self.event_bus_manager:
+                await self.event_bus_manager.shutdown()
+                logger.info("worker_event_bus_shutdown", worker_id=self.worker_id[:8] if self.worker_id else "unknown")
+
+            # Then close async client
+            await self._async_client.aclose()
+        except:
+            pass
+
+    async def start_websocket(self):
+        """Start WebSocket client if enabled."""
+        if self.websocket_client:
+            await self.websocket_client.start()
+            logger.info("websocket_started")
+
+    async def stop_websocket(self):
+        """Stop WebSocket client if running."""
+        if self.websocket_client:
+            await self.websocket_client.stop()
+            logger.info("websocket_stopped")
+
+    def _get_running_loop_safe(self) -> Optional[asyncio.AbstractEventLoop]:
+        """
+        Safely get the running event loop if one exists.
+
+        Returns:
+            The running event loop, or None if not in an async context
+        """
+        try:
+            return asyncio.get_running_loop()
+        except RuntimeError:
+            # No running loop in this thread
+            return None
+
+    def publish_event(
+        self,
+        execution_id: str,
+        event_type: str,
+        data: Dict[str, Any],
+    ) -> bool:
+        """
+        Publish a streaming event for real-time UI updates (SYNC version).
+
+        NOTE: This is the BLOCKING version. For real-time streaming,
+        use publish_event_async() instead to avoid blocking the event loop.
+
+        IMPORTANT: This method now auto-detects if it's being called from within
+        an async context (like Claude Code SDK hooks) and schedules tasks
+        appropriately to avoid "Cannot run the event loop while another loop
+        is running" errors.
+
+        Strategy (in order):
+        1. Try Event Bus (multi-provider) if configured [DEFAULT: includes Redis for fast path]
+        2. Try WebSocket if connected
+        3. Fallback to HTTP endpoint
+
+        Args:
+            execution_id: Execution ID
+            event_type: Event type (message_chunk, tool_started, etc.)
+            data: Event payload
+
+        Returns:
+            True if successful, False otherwise
+        """
+        # Check if we're in an async context
+        running_loop = self._get_running_loop_safe()
+        in_async_context = running_loop is not None
+
+        # Strategy 1: Try Event Bus first (Redis is auto-configured by default)
+        if self.event_bus_manager and self.event_bus_manager.is_initialized():
+            metadata = {}
+            if self.worker_id:
+                metadata["worker_id"] = self.worker_id
+
+            if in_async_context:
+                # We're in an async context - schedule task directly without creating coroutine first
+                try:
+                    # Create and schedule the task in one go
+                    coro = self.event_bus_manager.publish_event(
+                        execution_id=execution_id,
+                        event_type=event_type,
+                        data=data,
+                        metadata=metadata
+                    )
+                    task = running_loop.create_task(coro)
+
+                    # Add error callback
+                    def handle_task_error(t):
+                        try:
+                            exc = t.exception()
+                            if exc:
+                                logger.warning(
+                                    "background_event_bus_task_error",
+                                    error=str(exc),
+                                    execution_id=execution_id[:8],
+                                    event_type=event_type,
+                                )
+                        except asyncio.CancelledError:
+                            pass
+                        except Exception:
+                            pass
+
+                    task.add_done_callback(handle_task_error)
+                    logger.debug(
+                        "worker_event_scheduled_via_event_bus_async",
+                        execution_id=execution_id[:8],
+                        event_type=event_type,
+                        note="Task scheduled in running event loop"
+                    )
+                    return True
+                except Exception as e:
+                    logger.warning(
+                        "failed_to_schedule_event_bus_task",
+                        error=str(e),
+                        execution_id=execution_id[:8],
+                        event_type=event_type,
+                    )
+                    # Fall through to fallback strategies
+            else:
+                # Not in async context - use blocking execution
+                try:
+                    loop = self._get_thread_event_loop()
+                    coro = self.event_bus_manager.publish_event(
+                        execution_id=execution_id,
+                        event_type=event_type,
+                        data=data,
+                        metadata=metadata
+                    )
+
+                    try:
+                        results = loop.run_until_complete(coro)
+                    except RuntimeError as re:
+                        # Handle nested event loop case
+                        if "Cannot run the event loop while another loop is running" in str(re):
+                            logger.warning(
+                                "nested_event_loop_detected",
+                                execution_id=execution_id[:8],
+                                event_type=event_type,
+                                note="Skipping event bus publish due to nested loop"
+                            )
+                            coro.close()
+                            # Fall through to fallback strategies
+                            results = None
+                        else:
+                            coro.close()
+                            raise
+
+                    if results is not None:
+                        # Success if any provider succeeded
+                        success_count = sum(1 for success in results.values() if success)
+                        if success_count > 0:
+                            logger.debug(
+                                "worker_event_published_via_event_bus_sync",
+                                execution_id=execution_id[:8],
+                                event_type=event_type,
+                                success_count=success_count,
+                                total_providers=len(results)
+                            )
+                            return True
+                        else:
+                            logger.warning(
+                                "worker_event_bus_all_providers_failed_fallback_sync",
+                                execution_id=execution_id[:8],
+                                event_type=event_type,
+                                results=results
+                            )
+                            # Fall through to WebSocket/HTTP fallback
+                except Exception as e:
+                    logger.error(
+                        "worker_event_bus_publish_error_sync",
+                        error=str(e),
+                        execution_id=execution_id[:8],
+                        event_type=event_type
+                    )
+                    # Fall through to WebSocket/HTTP fallback
+
+        # Strategy 2: Try WebSocket if available and connected
+        if self.websocket_client and self.websocket_client.is_connected():
+            if in_async_context:
+                # Schedule WebSocket send as a task
+                try:
+                    coro = self.websocket_client.send_event(execution_id, event_type, data)
+                    task = running_loop.create_task(coro)
+                    logger.debug(
+                        "worker_event_scheduled_via_websocket_async",
+                        execution_id=execution_id[:8],
+                        event_type=event_type
+                    )
+                    return True
+                except Exception as e:
+                    logger.warning(
+                        "failed_to_schedule_websocket_task",
+                        error=str(e),
+                        execution_id=execution_id[:8],
+                    )
+                    # Fall through to HTTP fallback
+            else:
+                # WebSocket send_event is async, need to run it in event loop
+                try:
+                    loop = self._get_thread_event_loop()
+                    coro = self.websocket_client.send_event(execution_id, event_type, data)
+
+                    try:
+                        success = loop.run_until_complete(coro)
+                    except RuntimeError as re:
+                        if "Cannot run the event loop while another loop is running" in str(re):
+                            coro.close()
+                            success = None
+                        else:
+                            coro.close()
+                            raise
+
+                    if success:
+                        logger.debug(
+                            "worker_event_published_via_websocket_sync",
+                            execution_id=execution_id[:8],
+                            event_type=event_type
+                        )
+                        return True
+
+                    # Queue full - fallback to HTTP immediately
+                    if success is not None:
+                        logger.warning("websocket_queue_full_fallback_http_sync", execution_id=execution_id[:8])
+                except Exception as e:
+                    logger.error(
+                        "websocket_publish_error_sync",
+                        error=str(e),
+                        execution_id=execution_id[:8]
+                    )
+                    # Fall through to HTTP fallback
+
+        # Strategy 3: Fallback to HTTP
+        logger.debug(
+            "worker_event_publishing_via_http_fallback_sync",
+            execution_id=execution_id[:8],
+            event_type=event_type
+        )
+
+        try:
+            # Sanitize data to remove non-JSON-serializable objects
+            import json
+            import asyncio
+
+            def sanitize_value(val):
+                """Remove non-JSON-serializable objects"""
+                try:
+                    # Fast path for JSON primitives
+                    if val is None or isinstance(val, (bool, int, float, str)):
+                        return val
+
+                    # Check type name to avoid event loop issues
+                    type_name = type(val).__name__
+                    type_module = str(type(val).__module__)
+
+                    # Remove asyncio objects by checking module and type name
+                    if 'asyncio' in type_module or any(x in type_name for x in ['Event', 'Lock', 'Queue', 'Semaphore', 'Condition']):
+                        return f"<{type_name}>"
+                    elif isinstance(val, dict):
+                        return {k: sanitize_value(v) for k, v in val.items()}
+                    elif isinstance(val, (list, tuple)):
+                        return [sanitize_value(v) for v in val]
+                    else:
+                        try:
+                            json.dumps(val)
+                            return val
+                        except (TypeError, ValueError, RuntimeError):
+                            # RuntimeError catches "bound to different event loop" errors
+                            return f"<non-serializable: {type_name}>"
+                except Exception as e:
+                    # Catch-all for ANY errors during sanitization itself
+                    # Do NOT attempt to inspect the value here - it may cause event loop errors
+                    return "<sanitization-error>"
+
+            sanitized_data = sanitize_value(data)
+
+            url = f"{self.base_url}/api/v1/executions/{execution_id}/events"
+            payload = {
+                "event_type": event_type,
+                "data": sanitized_data,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+            }
+
+            # Double-check: Try to serialize the payload before sending
+            try:
+                json.dumps(payload)
+            except Exception as serialize_err:
+                logger.error(
+                    "payload_serialization_test_failed",
+                    execution_id=execution_id[:8],
+                    event_type=event_type,
+                    error=str(serialize_err)[:200],
+                )
+                # If we can't serialize it, don't even try to send
+                return False
+
+            response = self._client.post(url, json=payload, headers=self.headers)
+
+            if response.status_code not in (200, 202):
+                logger.warning(
+                    "event_publish_failed",
+                    status=response.status_code,
+                    execution_id=execution_id[:8],
+                    event_type=event_type,
+                )
+                return False
+
+            return True
+
+        except Exception as e:
+            # Sanitize error message to avoid serialization issues
+            import re
+            error_str = str(e) or "(empty)"
+            error_type = type(e).__name__
+            # Remove asyncio object references that cause serialization errors
+            error_str = re.sub(r'<asyncio\.\w+\.\w+ object at 0x[0-9a-f]+ \[[\w\s]+\]>', '[asyncio-object]', error_str)
+
+            logger.warning(
+                "event_publish_error",
+                error=error_str[:500],  # Truncate to prevent huge error messages
+                error_type=error_type,
+                execution_id=execution_id[:8],
+                event_type=event_type,
+            )
+            return False
+
+    async def publish_event_async(
+        self,
+        execution_id: str,
+        event_type: str,
+        data: Dict[str, Any],
+    ) -> bool:
+        """
+        Publish a streaming event for real-time UI updates (ASYNC version).
+
+        Strategy (in order):
+        1. Try Event Bus (multi-provider) if configured [DEFAULT: includes Redis for fast path]
+        2. Try WebSocket if connected
+        3. Fallback to HTTP endpoint (control plane handles Redis internally)
+
+        By default, workers receive Redis credentials during registration and
+        publish directly to Redis (fast path). If Redis is unavailable, falls
+        back to HTTP endpoint.
+
+        Args:
+            execution_id: Execution ID
+            event_type: Event type (message_chunk, tool_started, etc.)
+            data: Event payload
+
+        Returns:
+            True if at least one provider succeeded, False otherwise
+        """
+        # Strategy 1: Try Event Bus first (Redis is auto-configured by default)
+        if self.event_bus_manager and self.event_bus_manager.is_initialized():
+            try:
+                metadata = {}
+                if self.worker_id:
+                    metadata["worker_id"] = self.worker_id
+
+                results = await self.event_bus_manager.publish_event(
+                    execution_id=execution_id,
+                    event_type=event_type,
+                    data=data,
+                    metadata=metadata
+                )
+
+                # Success if any provider succeeded
+                success_count = sum(1 for success in results.values() if success)
+                if success_count > 0:
+                    logger.debug(
+                        "worker_event_published_via_event_bus",
+                        execution_id=execution_id[:8],
+                        event_type=event_type,
+                        success_count=success_count,
+                        total_providers=len(results)
+                    )
+                    return True
+                else:
+                    logger.warning(
+                        "worker_event_bus_all_providers_failed_fallback",
+                        execution_id=execution_id[:8],
+                        event_type=event_type,
+                        results=results
+                    )
+                    # Fall through to WebSocket/HTTP fallback
+            except Exception as e:
+                logger.error(
+                    "worker_event_bus_publish_error",
+                    error=str(e),
+                    execution_id=execution_id[:8],
+                    event_type=event_type
+                )
+                # Fall through to WebSocket/HTTP fallback
+
+        # Strategy 2: Try WebSocket if available and connected
+        if self.websocket_client and self.websocket_client.is_connected():
+            success = await self.websocket_client.send_event(execution_id, event_type, data)
+            if success:
+                logger.debug(
+                    "worker_event_published_via_websocket",
+                    execution_id=execution_id[:8],
+                    event_type=event_type
+                )
+                return True
+
+            # Queue full - fallback to HTTP immediately
+            logger.warning("websocket_queue_full_fallback_http", execution_id=execution_id[:8])
+
+        # Strategy 3: Fallback to HTTP
+        logger.debug(
+            "worker_event_publishing_via_http_fallback",
+            execution_id=execution_id[:8],
+            event_type=event_type
+        )
+        return await self._publish_event_http(execution_id, event_type, data)
+
+    async def _publish_event_http(
+        self,
+        execution_id: str,
+        event_type: str,
+        data: Dict[str, Any],
+    ) -> bool:
+        """
+        Publish event via HTTP (internal method for fallback).
+
+        Args:
+            execution_id: Execution ID
+            event_type: Event type
+            data: Event payload
+
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Sanitize data to remove non-JSON-serializable objects
+            import json
+            import asyncio
+
+            def sanitize_value(val):
+                """Remove non-JSON-serializable objects"""
+                try:
+                    # Fast path for JSON primitives
+                    if val is None or isinstance(val, (bool, int, float, str)):
+                        return val
+
+                    # Check type name to avoid event loop issues
+                    type_name = type(val).__name__
+                    type_module = str(type(val).__module__)
+
+                    # Remove asyncio objects by checking module and type name
+                    if 'asyncio' in type_module or any(x in type_name for x in ['Event', 'Lock', 'Queue', 'Semaphore', 'Condition']):
+                        return f"<{type_name}>"
+                    elif isinstance(val, dict):
+                        return {k: sanitize_value(v) for k, v in val.items()}
+                    elif isinstance(val, (list, tuple)):
+                        return [sanitize_value(v) for v in val]
+                    else:
+                        try:
+                            json.dumps(val)
+                            return val
+                        except (TypeError, ValueError, RuntimeError):
+                            # RuntimeError catches "bound to different event loop" errors
+                            return f"<non-serializable: {type_name}>"
+                except Exception as e:
+                    # Catch-all for ANY errors during sanitization itself
+                    # Do NOT attempt to inspect the value here - it may cause event loop errors
+                    return "<sanitization-error>"
+
+            sanitized_data = sanitize_value(data)
+
+            url = f"{self.base_url}/api/v1/executions/{execution_id}/events"
+            payload = {
+                "event_type": event_type,
+                "data": sanitized_data,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+            }
+
+            # Double-check: Try to serialize the payload before sending
+            try:
+                json.dumps(payload)
+            except Exception as serialize_err:
+                logger.error(
+                    "payload_serialization_test_failed",
+                    execution_id=execution_id[:8],
+                    event_type=event_type,
+                    error=str(serialize_err)[:200],
+                )
+                # If we can't serialize it, don't even try to send
+                return False
+
+            response = await self._async_client.post(url, json=payload, headers=self.headers)
+
+            if response.status_code not in (200, 202):
+                logger.warning(
+                    "event_publish_failed",
+                    status=response.status_code,
+                    execution_id=execution_id[:8],
+                    event_type=event_type,
+                )
+                return False
+
+            return True
+
+        except Exception as e:
+            # Sanitize error message to avoid serialization issues
+            import re
+            error_str = str(e) or "(empty)"
+            error_type = type(e).__name__
+            # Remove asyncio object references that cause serialization errors
+            error_str = re.sub(r'<asyncio\.\w+\.\w+ object at 0x[0-9a-f]+ \[[\w\s]+\]>', '[asyncio-object]', error_str)
+
+            logger.warning(
+                "event_publish_error",
+                error=error_str[:500],  # Truncate to prevent huge error messages
+                error_type=error_type,
+                execution_id=execution_id[:8],
+                event_type=event_type,
+            )
+            return False
+
+    def cache_metadata(
+        self,
+        execution_id: str,
+        execution_type: str,
+    ) -> bool:
+        """
+        Cache execution metadata in Redis for fast SSE lookups.
+
+        This eliminates the need for database queries on every SSE connection.
+
+        Args:
+            execution_id: Execution ID
+            execution_type: "AGENT" or "TEAM"
+
+        Returns:
+            True if successful, False otherwise
+        """
+        return self.publish_event(
+            execution_id=execution_id,
+            event_type="metadata",
+            data={"execution_type": execution_type},
+        )
+
+    def get_session(
+        self,
+        execution_id: str,
+        session_id: Optional[str] = None,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Retrieve session history from Control Plane database.
+
+        This loads conversation history so workers can restore context
+        across multiple execution turns.
+
+        Args:
+            execution_id: Execution ID
+            session_id: Session ID (defaults to execution_id if not provided)
+
+        Returns:
+            Dict with session data including messages, or None if not found
+        """
+        try:
+            session_id = session_id or execution_id
+            url = f"{self.base_url}/api/v1/executions/{execution_id}/session"
+
+            response = self._client.get(url, headers=self.headers)
+
+            if response.status_code == 200:
+                session_data = response.json()
+                logger.info(
+                    "session_loaded",
+                    execution_id=execution_id[:8],
+                    message_count=len(session_data.get("messages", [])),
+                )
+                return session_data
+            elif response.status_code == 404:
+                logger.info(
+                    "session_not_found",
+                    execution_id=execution_id[:8],
+                )
+                return None
+            else:
+                logger.warning(
+                    "session_load_failed",
+                    status=response.status_code,
+                    execution_id=execution_id[:8],
+                )
+                return None
+
+        except Exception as e:
+            logger.warning(
+                "session_load_error",
+                error=str(e),
+                execution_id=execution_id[:8],
+            )
+            return None
+
+    def persist_session(
+        self,
+        execution_id: str,
+        session_id: str,
+        user_id: Optional[str],
+        messages: List[Dict[str, Any]],
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> bool:
+        """
+        Persist session history to Control Plane database.
+
+        This ensures history is available even when worker is offline.
+
+        Args:
+            execution_id: Execution ID
+            session_id: Session ID
+            user_id: User ID
+            messages: List of session messages
+            metadata: Optional metadata
+
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            url = f"{self.base_url}/api/v1/executions/{execution_id}/session"
+            payload = {
+                "session_id": session_id,
+                "user_id": user_id,
+                "messages": messages,
+                "metadata": metadata or {},
+            }
+
+            response = self._client.post(url, json=payload, headers=self.headers)
+
+            if response.status_code in (200, 201):
+                logger.info(
+                    "session_persisted",
+                    execution_id=execution_id[:8],
+                    message_count=len(messages),
+                )
+                return True
+            else:
+                logger.warning(
+                    "session_persistence_failed",
+                    status=response.status_code,
+                    execution_id=execution_id[:8],
+                )
+                return False
+
+        except Exception as e:
+            logger.warning(
+                "session_persistence_error",
+                error=str(e),
+                execution_id=execution_id[:8],
+            )
+            return False
+
+    def get_skills(
+        self,
+        agent_id: str,
+    ) -> List[Dict[str, Any]]:
+        """
+        Fetch resolved skills for an agent from Control Plane.
+
+        This endpoint returns skills merged from all layers:
+        - All agent environments (many-to-many)
+        - Team skills (if agent has team)
+        - All team environments (many-to-many)
+        - Agent's own skills
+
+        Args:
+            agent_id: Agent ID
+
+        Returns:
+            List of skill configurations with source and inheritance info
+        """
+        try:
+            url = f"{self.base_url}/api/v1/skills/associations/agents/{agent_id}/skills/resolved"
+            response = self._client.get(url, headers=self.headers)
+
+            if response.status_code == 200:
+                skills = response.json()
+                logger.info(
+                    "skills_fetched",
+                    agent_id=agent_id[:8],
+                    skill_count=len(skills),
+                )
+                return skills
+            else:
+                logger.warning(
+                    "skills_fetch_failed",
+                    status=response.status_code,
+                    agent_id=agent_id[:8],
+                )
+                return []
+
+        except Exception as e:
+            logger.warning(
+                "skills_fetch_error",
+                error=str(e),
+                agent_id=agent_id[:8],
+            )
+            return []
+
+    def get_team_skills(
+        self,
+        team_id: str,
+    ) -> List[Dict[str, Any]]:
+        """
+        Fetch resolved skills for a team from Control Plane.
+
+        This endpoint returns skills merged from all layers:
+        - All team environments (many-to-many)
+        - Team's own skills
+
+        Args:
+            team_id: Team ID
+
+        Returns:
+            List of skill configurations with source and inheritance info
+        """
+        try:
+            url = f"{self.base_url}/api/v1/skills/associations/teams/{team_id}/skills/resolved"
+            response = self._client.get(url, headers=self.headers)
+
+            if response.status_code == 200:
+                skills = response.json()
+                logger.info(
+                    "team_skills_fetched",
+                    team_id=team_id[:8],
+                    skill_count=len(skills),
+                )
+                return skills
+            else:
+                logger.warning(
+                    "team_skills_fetch_failed",
+                    status=response.status_code,
+                    team_id=team_id[:8],
+                )
+                return []
+
+        except Exception as e:
+            logger.warning(
+                "team_skills_fetch_error",
+                error=str(e),
+                team_id=team_id[:8],
+            )
+            return []
+
+    def get_agent_execution_environment(
+        self,
+        agent_id: str,
+    ) -> Dict[str, str]:
+        """
+        Fetch resolved execution environment for an agent from Control Plane.
+
+        This endpoint returns a fully resolved environment variable dict with:
+        - Custom env vars from agent configuration
+        - Secret values (resolved from Kubiya vault)
+        - Integration tokens (resolved and mapped to env var names like GH_TOKEN, JIRA_TOKEN)
+
+        Args:
+            agent_id: Agent ID
+
+        Returns:
+            Dict of environment variables ready to inject into agent execution
+        """
+        try:
+            url = f"{self.base_url}/api/v1/execution-environment/agents/{agent_id}/resolved"
+            response = self._client.get(url, headers=self.headers)
+
+            if response.status_code == 200:
+                env_vars = response.json()
+                logger.info(
+                    "agent_execution_environment_fetched",
+                    agent_id=agent_id[:8],
+                    env_var_count=len(env_vars),
+                    env_var_keys=list(env_vars.keys()),
+                )
+                return env_vars
+            else:
+                logger.warning(
+                    "agent_execution_environment_fetch_failed",
+                    status=response.status_code,
+                    agent_id=agent_id[:8],
+                )
+                return {}
+
+        except Exception as e:
+            logger.warning(
+                "agent_execution_environment_fetch_error",
+                error=str(e),
+                agent_id=agent_id[:8],
+            )
+            return {}
+
+    def get_team_execution_environment(
+        self,
+        team_id: str,
+    ) -> Dict[str, str]:
+        """
+        Fetch resolved execution environment for a team from Control Plane.
+
+        This endpoint returns a fully resolved environment variable dict with:
+        - Custom env vars from team configuration
+        - Secret values (resolved from Kubiya vault)
+        - Integration tokens (resolved and mapped to env var names like GH_TOKEN, JIRA_TOKEN)
+
+        Args:
+            team_id: Team ID
+
+        Returns:
+            Dict of environment variables ready to inject into team execution
+        """
+        try:
+            url = f"{self.base_url}/api/v1/execution-environment/teams/{team_id}/resolved"
+            response = self._client.get(url, headers=self.headers)
+
+            if response.status_code == 200:
+                env_vars = response.json()
+                logger.info(
+                    "team_execution_environment_fetched",
+                    team_id=team_id[:8],
+                    env_var_count=len(env_vars),
+                    env_var_keys=list(env_vars.keys()),
+                )
+                return env_vars
+            else:
+                logger.warning(
+                    "team_execution_environment_fetch_failed",
+                    status=response.status_code,
+                    team_id=team_id[:8],
+                )
+                return {}
+
+        except Exception as e:
+            logger.warning(
+                "team_execution_environment_fetch_error",
+                error=str(e),
+                team_id=team_id[:8],
+            )
+            return {}
+
+    async def create_job_execution_record(
+        self,
+        execution_id: str,
+        job_id: Optional[str],
+        organization_id: str,
+        entity_type: str,
+        entity_id: Optional[str],
+        prompt: str,
+        trigger_type: str,
+        trigger_metadata: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """
+        Create execution and job_executions records for a scheduled job.
+
+        This calls the Control Plane API to create execution records
+        instead of directly accessing Supabase.
+
+        Args:
+            execution_id: Execution ID
+            job_id: Job ID (optional)
+            organization_id: Organization ID
+            entity_type: "agent" or "team"
+            entity_id: Agent or team ID
+            prompt: Prompt text
+            trigger_type: "cron", "webhook", or "manual"
+            trigger_metadata: Additional trigger metadata
+
+        Returns:
+            Dict with execution_id, status, and created_at
+        """
+        try:
+            url = f"{self.base_url}/api/v1/executions/create"
+            payload = {
+                "execution_id": execution_id,
+                "job_id": job_id,
+                "organization_id": organization_id,
+                "entity_type": entity_type,
+                "entity_id": entity_id,
+                "prompt": prompt,
+                "trigger_type": trigger_type,
+                "trigger_metadata": trigger_metadata,
+            }
+
+            response = await self._async_client.post(url, json=payload, headers=self.headers)
+
+            if response.status_code == 201:
+                result = response.json()
+                logger.info(
+                    "job_execution_record_created",
+                    execution_id=execution_id[:8],
+                    job_id=job_id[:8] if job_id else None,
+                )
+                return result
+            else:
+                logger.error(
+                    "job_execution_record_creation_failed",
+                    status=response.status_code,
+                    execution_id=execution_id[:8],
+                    response=response.text,
+                )
+                raise Exception(f"Failed to create execution record: HTTP {response.status_code}")
+
+        except Exception as e:
+            logger.error(
+                "job_execution_record_creation_error",
+                error=str(e),
+                execution_id=execution_id[:8],
+            )
+            raise
+
+    async def update_job_execution_status(
+        self,
+        execution_id: str,
+        job_id: str,
+        status: str,
+        duration_ms: Optional[int] = None,
+        error_message: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Update job_executions record with execution results.
+
+        This calls the Control Plane API to update job execution status
+        instead of directly accessing Supabase.
+
+        Args:
+            execution_id: Execution ID
+            job_id: Job ID
+            status: Final status ("completed" or "failed")
+            duration_ms: Execution duration in milliseconds
+            error_message: Error message if failed
+
+        Returns:
+            Dict with job_id, execution_id, and status
+        """
+        try:
+            url = f"{self.base_url}/api/v1/executions/{execution_id}/job/{job_id}/status"
+            payload = {
+                "status": status,
+                "duration_ms": duration_ms,
+                "error_message": error_message,
+            }
+
+            response = await self._async_client.post(url, json=payload, headers=self.headers)
+
+            if response.status_code == 200:
+                result = response.json()
+                logger.info(
+                    "job_execution_status_updated",
+                    execution_id=execution_id[:8],
+                    job_id=job_id[:8],
+                    status=status,
+                )
+                return result
+            else:
+                logger.error(
+                    "job_execution_status_update_failed",
+                    status_code=response.status_code,
+                    execution_id=execution_id[:8],
+                    job_id=job_id[:8],
+                    response=response.text,
+                )
+                raise Exception(f"Failed to update job execution status: HTTP {response.status_code}")
+
+        except Exception as e:
+            logger.error(
+                "job_execution_status_update_error",
+                error=str(e),
+                execution_id=execution_id[:8],
+                job_id=job_id[:8],
+            )
+            raise
+
+    async def _handle_control_message(self, message: Dict[str, Any]):
+        """
+        Handle control messages from control plane via WebSocket.
+
+        This method is called when the worker receives a control message
+        from the control plane (pause, resume, cancel, reload_config).
+
+        Args:
+            message: Control message with command, execution_id, and data
+        """
+        command = message.get("command")
+        execution_id = message.get("execution_id")
+
+        try:
+            # Import Temporal client here to avoid circular import
+            from control_plane_api.app.lib.temporal_client import get_temporal_client
+
+            temporal_client = get_temporal_client()
+            workflow_handle = temporal_client.get_workflow_handle(execution_id)
+
+            if command == "pause":
+                await workflow_handle.signal("pause_execution")
+                logger.info("control_command_executed", command="pause", execution_id=execution_id[:8])
+
+            elif command == "resume":
+                await workflow_handle.signal("resume_execution")
+                logger.info("control_command_executed", command="resume", execution_id=execution_id[:8])
+
+            elif command == "cancel":
+                await workflow_handle.cancel()
+                logger.info("control_command_executed", command="cancel", execution_id=execution_id[:8])
+
+            elif command == "reload_config":
+                # Future: Reload config without restart
+                logger.info("control_command_not_implemented", command="reload_config", execution_id=execution_id[:8])
+
+            else:
+                logger.warning("unknown_control_command", command=command, execution_id=execution_id[:8])
+
+        except Exception as e:
+            logger.error(
+                "control_command_error",
+                error=str(e),
+                command=command,
+                execution_id=execution_id[:8] if execution_id else None
+            )
+
+
+# Singleton instance
+_control_plane_client: Optional[ControlPlaneClient] = None
+
+
+def get_control_plane_client() -> ControlPlaneClient:
+    """
+    Get or create the Control Plane client singleton.
+
+    Reads configuration from environment variables:
+    - CONTROL_PLANE_URL: Control Plane URL
+    - KUBIYA_API_KEY: API key for authentication
+    - REDIS_URL: Redis URL for direct event streaming (from registration)
+    - REDIS_PASSWORD: Redis password if needed (from registration)
+    - REDIS_ENABLED: Whether Redis is enabled (from registration)
+    - WEBSOCKET_ENABLED: Whether WebSocket is enabled (from registration)
+    - WEBSOCKET_URL: WebSocket URL (from registration)
+    - WORKER_ID: Worker ID (from registration)
+    - EVENT_BUS_CONFIG: JSON string with event bus configuration (from registration, optional)
+
+    Returns:
+        ControlPlaneClient instance
+
+    Raises:
+        ValueError: If required environment variables are not set
+    """
+    global _control_plane_client
+
+    if _control_plane_client is None:
+        base_url = os.environ.get("CONTROL_PLANE_URL")
+        api_key = os.environ.get("KUBIYA_API_KEY")
+
+        # WebSocket config from environment (set by worker.py after registration)
+        websocket_enabled = os.environ.get("WEBSOCKET_ENABLED", "false").lower() == "true"
+        websocket_url = os.environ.get("WEBSOCKET_URL")
+        worker_id = os.environ.get("WORKER_ID")
+
+        # Redis config from environment (set by worker.py after registration)
+        # This is the DEFAULT fast path for event streaming
+        redis_url = os.environ.get("REDIS_URL")
+        redis_password = os.environ.get("REDIS_PASSWORD")
+        redis_enabled = os.environ.get("REDIS_ENABLED", "false").lower() == "true"
+
+        # Event bus config from environment (set by worker.py after registration)
+        event_bus_config = None
+        event_bus_config_str = os.environ.get("EVENT_BUS_CONFIG")
+        if event_bus_config_str:
+            try:
+                import json
+                event_bus_config = json.loads(event_bus_config_str)
+                logger.info("event_bus_config_loaded_from_env", providers=list(event_bus_config.keys()))
+            except Exception as e:
+                logger.warning("event_bus_config_parse_failed", error=str(e))
+
+        # AUTO-CONFIGURE: If Redis credentials provided, auto-enable Redis provider
+        # This makes Redis the default fast path without explicit event_bus_config
+        if redis_enabled and redis_url and not event_bus_config:
+            event_bus_config = {
+                "redis": {
+                    "enabled": True,
+                    "redis_url": redis_url,
+                }
+            }
+            logger.info(
+                "redis_auto_configured_as_default",
+                worker_id=worker_id[:8] if worker_id else "unknown",
+                redis_url=redis_url.split("@")[-1] if "@" in redis_url else redis_url,  # Log without password
+            )
+
+        if not base_url:
+            raise ValueError("CONTROL_PLANE_URL environment variable not set")
+        if not api_key:
+            raise ValueError("KUBIYA_API_KEY environment variable not set")
+
+        _control_plane_client = ControlPlaneClient(
+            base_url=base_url,
+            api_key=api_key,
+            websocket_enabled=websocket_enabled,
+            websocket_url=websocket_url,
+            worker_id=worker_id,
+            event_bus_config=event_bus_config
+        )
+
+        logger.info(
+            "control_plane_client_initialized",
+            base_url=base_url,
+            websocket_enabled=websocket_enabled,
+            event_bus_configured=event_bus_config is not None
+        )
+
+    return _control_plane_client