jarviscore-framework 0.1.0__py3-none-any.whl

jarviscore/p2p/keepalive.py

@@ -0,0 +1,361 @@
+"""
+P2P Keepalive Manager for maintaining active ZMQ connections in agent mesh.
+
+Prevents idle connection closure by sending periodic keepalive messages
+while intelligently suppressing when real workflow traffic exists.
+"""
+
+import asyncio
+import time
+import logging
+from typing import Optional, Dict, Any, Callable
+from dataclasses import dataclass
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+
+class CircuitState(Enum):
+    """Circuit breaker states for connection health."""
+    CLOSED = "CLOSED"
+    HALF_OPEN = "HALF_OPEN"
+    OPEN = "OPEN"
+    UNKNOWN = "UNKNOWN"
+
+
+@dataclass
+class KeepaliveMetrics:
+    """Metrics for P2P keepalive health monitoring."""
+    keepalives_sent: int = 0
+    keepalives_received: int = 0
+    acks_received: int = 0
+    timeouts: int = 0
+    suppressed_count: int = 0
+    last_successful_keepalive: float = 0.0
+    last_keepalive_latency: float = 0.0
+    circuit_breaker_events: int = 0
+
+
+class P2PKeepaliveManager:
+    """
+    Manages P2P keepalive messages to prevent ZMQ connection idle closure.
+    
+    Features:
+    - Periodic keepalive with configurable interval
+    - Smart suppression when recent workflow traffic exists
+    - Circuit breaker integration for adaptive behavior
+    - Health metrics and observability
+    - Bidirectional keepalive + ACK pattern
+    """
+    
+    def __init__(
+        self,
+        agent_id: str,
+        send_p2p_callback: Callable[[str, str, Dict[str, Any]], bool],
+        broadcast_p2p_callback: Optional[Callable[[str, Dict[str, Any]], int]] = None,
+        config: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Initialize P2P Keepalive Manager.
+        
+        Args:
+            agent_id: Unique identifier for this agent
+            send_p2p_callback: Function to send P2P message to specific peer
+            broadcast_p2p_callback: Optional function to broadcast to all peers
+            config: Configuration dictionary
+        """
+        self.agent_id = agent_id
+        self.send_p2p_message = send_p2p_callback
+        self.broadcast_p2p_message = broadcast_p2p_callback
+        
+        # Configuration with production defaults
+        config = config or {}
+        self.enabled = config.get('P2P_KEEPALIVE_ENABLED', True)
+        self.interval = config.get('P2P_KEEPALIVE_INTERVAL', 90)  # 90s default
+        self.timeout = config.get('P2P_KEEPALIVE_TIMEOUT', 10)  # 10s timeout
+        self.activity_suppress_window = config.get('P2P_ACTIVITY_SUPPRESS_WINDOW', 60)  # 60s
+        self.circuit_half_open_interval = config.get('P2P_CIRCUIT_HALF_OPEN_INTERVAL', 30)  # 30s aggressive
+        
+        # State tracking
+        self.last_p2p_activity = time.time()  # Track any P2P activity
+        self.last_keepalive_sent = 0.0
+        self.pending_keepalives: Dict[str, float] = {}  # peer_id -> sent_time
+        self.circuit_state = CircuitState.UNKNOWN
+        
+        # Metrics
+        self.metrics = KeepaliveMetrics()
+        
+        # Control
+        self._running = False
+        self._keepalive_task: Optional[asyncio.Task] = None
+        
+        logger.info(f"P2P_KEEPALIVE ({self.agent_id}): Initialized with interval={self.interval}s, "
+                   f"suppress_window={self.activity_suppress_window}s, enabled={self.enabled}")
+    
+    async def start(self):
+        """Start the keepalive loop."""
+        if not self.enabled:
+            logger.info(f"P2P_KEEPALIVE ({self.agent_id}): Disabled by configuration")
+            return
+        
+        if self._running:
+            logger.warning(f"P2P_KEEPALIVE ({self.agent_id}): Already running")
+            return
+        
+        self._running = True
+        self._keepalive_task = asyncio.create_task(self._keepalive_loop())
+        logger.info(f"P2P_KEEPALIVE ({self.agent_id}): Started keepalive loop")
+    
+    async def stop(self):
+        """Stop the keepalive loop."""
+        self._running = False
+        
+        if self._keepalive_task:
+            self._keepalive_task.cancel()
+            try:
+                await self._keepalive_task
+            except asyncio.CancelledError:
+                pass
+        
+        logger.info(f"P2P_KEEPALIVE ({self.agent_id}): Stopped keepalive loop")
+    
+    def record_p2p_activity(self):
+        """
+        Record that P2P activity occurred (workflow message, nudge, broadcast).
+        Used for smart suppression of keepalives.
+        """
+        self.last_p2p_activity = time.time()
+    
+    def update_circuit_state(self, state: CircuitState):
+        """
+        Update circuit breaker state for adaptive keepalive behavior.
+        
+        Args:
+            state: Current circuit breaker state
+        """
+        if state != self.circuit_state:
+            logger.info(f"P2P_KEEPALIVE ({self.agent_id}): Circuit state changed: "
+                       f"{self.circuit_state.value} -> {state.value}")
+            self.circuit_state = state
+            self.metrics.circuit_breaker_events += 1
+    
+    def _should_send_keepalive(self) -> bool:
+        """
+        Determine if keepalive should be sent based on smart suppression logic.
+        
+        Returns:
+            True if keepalive should be sent, False if suppressed
+        """
+        current_time = time.time()
+        
+        # Check if recent P2P activity exists
+        time_since_activity = current_time - self.last_p2p_activity
+        if time_since_activity < self.activity_suppress_window:
+            logger.debug(f"P2P_KEEPALIVE ({self.agent_id}): Suppressed - recent activity "
+                        f"{time_since_activity:.1f}s ago")
+            self.metrics.suppressed_count += 1
+            return False
+        
+        # Check interval based on circuit state
+        interval = self._get_adaptive_interval()
+        time_since_last_keepalive = current_time - self.last_keepalive_sent
+        
+        if time_since_last_keepalive < interval:
+            return False
+        
+        return True
+    
+    def _get_adaptive_interval(self) -> float:
+        """
+        Get adaptive keepalive interval based on circuit breaker state.
+        
+        Returns:
+            Keepalive interval in seconds
+        """
+        if self.circuit_state == CircuitState.HALF_OPEN:
+            # Aggressive keepalives to help circuit recovery
+            return self.circuit_half_open_interval
+        elif self.circuit_state == CircuitState.OPEN:
+            # Try to trigger recovery probes
+            return self.circuit_half_open_interval
+        else:
+            # Normal operation
+            return self.interval
+    
+    async def _keepalive_loop(self):
+        """Main keepalive loop with adaptive timing."""
+        logger.info(f"P2P_KEEPALIVE ({self.agent_id}): Keepalive loop started")
+        
+        # Initial delay to allow P2P mesh to stabilize
+        await asyncio.sleep(30)
+        
+        while self._running:
+            try:
+                if self._should_send_keepalive():
+                    await self._send_keepalive()
+                
+                # Check for keepalive timeouts
+                await self._check_timeouts()
+                
+                # Log metrics periodically
+                await self._log_metrics()
+                
+                # Sleep with adaptive interval
+                await asyncio.sleep(10)  # Check every 10s, send based on interval
+                
+            except Exception as e:
+                logger.error(f"P2P_KEEPALIVE ({self.agent_id}): Error in keepalive loop: {e}", 
+                           exc_info=True)
+                await asyncio.sleep(30)  # Back off on error
+        
+        logger.info(f"P2P_KEEPALIVE ({self.agent_id}): Keepalive loop stopped")
+    
+    async def _send_keepalive(self):
+        """Send keepalive message to all peers."""
+        try:
+            current_time = time.time()
+            
+            payload = {
+                'agent_id': self.agent_id,
+                'timestamp': current_time,
+                'circuit_state': self.circuit_state.value,
+                'metrics': {
+                    'sent': self.metrics.keepalives_sent,
+                    'received': self.metrics.keepalives_received,
+                    'acks': self.metrics.acks_received
+                }
+            }
+            
+            # Broadcast keepalive to all peers
+            if self.broadcast_p2p_message:
+                success_count = await self.broadcast_p2p_message('P2P_KEEPALIVE', payload)
+                
+                if success_count > 0:
+                    self.last_keepalive_sent = current_time
+                    self.metrics.keepalives_sent += 1
+                    logger.debug(f"P2P_KEEPALIVE ({self.agent_id}): Sent keepalive to {success_count} peers")
+                else:
+                    logger.warning(f"P2P_KEEPALIVE ({self.agent_id}): Failed to send keepalive to any peer")
+            else:
+                logger.warning(f"P2P_KEEPALIVE ({self.agent_id}): No broadcast callback available")
+                
+        except Exception as e:
+            logger.error(f"P2P_KEEPALIVE ({self.agent_id}): Error sending keepalive: {e}")
+    
+    async def handle_keepalive_received(self, sender_id: str, payload: Dict[str, Any]):
+        """
+        Handle incoming keepalive message from peer.
+        
+        Args:
+            sender_id: ID of the peer that sent keepalive
+            payload: Keepalive message payload
+        """
+        try:
+            self.metrics.keepalives_received += 1
+            logger.debug(f"P2P_KEEPALIVE ({self.agent_id}): Received keepalive from {sender_id}")
+            
+            # Send ACK back to sender
+            ack_payload = {
+                'agent_id': self.agent_id,
+                'timestamp': time.time(),
+                'original_timestamp': payload.get('timestamp')
+            }
+            
+            # Send ACK using direct message (not broadcast)
+            if self.send_p2p_message:
+                success = await self.send_p2p_message(sender_id, 'P2P_KEEPALIVE_ACK', ack_payload)
+                if success:
+                    logger.debug(f"P2P_KEEPALIVE ({self.agent_id}): Sent ACK to {sender_id}")
+                else:
+                    logger.warning(f"P2P_KEEPALIVE ({self.agent_id}): Failed to send ACK to {sender_id}")
+            
+        except Exception as e:
+            logger.error(f"P2P_KEEPALIVE ({self.agent_id}): Error handling keepalive: {e}")
+    
+    async def handle_keepalive_ack(self, sender_id: str, payload: Dict[str, Any]):
+        """
+        Handle incoming keepalive ACK from peer.
+        
+        Args:
+            sender_id: ID of the peer that sent ACK
+            payload: ACK message payload
+        """
+        try:
+            self.metrics.acks_received += 1
+            current_time = time.time()
+            
+            # Calculate latency if original timestamp available
+            original_timestamp = payload.get('original_timestamp')
+            if original_timestamp:
+                latency = current_time - original_timestamp
+                self.metrics.last_keepalive_latency = latency
+                logger.debug(f"P2P_KEEPALIVE ({self.agent_id}): ACK from {sender_id}, "
+                           f"latency={latency*1000:.1f}ms")
+            
+            self.metrics.last_successful_keepalive = current_time
+            
+            # Remove from pending if tracked
+            if sender_id in self.pending_keepalives:
+                del self.pending_keepalives[sender_id]
+            
+        except Exception as e:
+            logger.error(f"P2P_KEEPALIVE ({self.agent_id}): Error handling ACK: {e}")
+    
+    async def _check_timeouts(self):
+        """Check for keepalive timeouts and clean up pending requests."""
+        current_time = time.time()
+        timed_out = []
+        
+        for peer_id, sent_time in self.pending_keepalives.items():
+            if current_time - sent_time > self.timeout:
+                timed_out.append(peer_id)
+        
+        for peer_id in timed_out:
+            del self.pending_keepalives[peer_id]
+            self.metrics.timeouts += 1
+            logger.warning(f"P2P_KEEPALIVE ({self.agent_id}): Keepalive timeout for peer {peer_id}")
+    
+    async def _log_metrics(self):
+        """Periodically log keepalive metrics."""
+        current_time = time.time()
+        
+        # Log every 5 minutes
+        if current_time % 300 < 10:
+            logger.info(
+                f"P2P_KEEPALIVE_METRICS ({self.agent_id}): "
+                f"Sent={self.metrics.keepalives_sent}, "
+                f"Received={self.metrics.keepalives_received}, "
+                f"ACKs={self.metrics.acks_received}, "
+                f"Timeouts={self.metrics.timeouts}, "
+                f"Suppressed={self.metrics.suppressed_count}, "
+                f"Circuit_Events={self.metrics.circuit_breaker_events}, "
+                f"Last_Latency={self.metrics.last_keepalive_latency*1000:.1f}ms"
+            )
+    
+    def get_health_status(self) -> Dict[str, Any]:
+        """
+        Get current health status and metrics.
+        
+        Returns:
+            Dictionary with health status and metrics
+        """
+        current_time = time.time()
+        time_since_last_success = current_time - self.metrics.last_successful_keepalive
+        
+        return {
+            'enabled': self.enabled,
+            'running': self._running,
+            'circuit_state': self.circuit_state.value,
+            'last_activity': current_time - self.last_p2p_activity,
+            'last_keepalive': current_time - self.last_keepalive_sent,
+            'last_success': time_since_last_success,
+            'metrics': {
+                'sent': self.metrics.keepalives_sent,
+                'received': self.metrics.keepalives_received,
+                'acks': self.metrics.acks_received,
+                'timeouts': self.metrics.timeouts,
+                'suppressed': self.metrics.suppressed_count,
+                'circuit_events': self.metrics.circuit_breaker_events,
+                'latency_ms': self.metrics.last_keepalive_latency * 1000
+            }
+        }

jarviscore/p2p/swim_manager.py

@@ -0,0 +1,290 @@
+"""
+SWIM Thread Manager for JarvisCore Framework
+Runs SWIM protocol in dedicated thread to prevent GIL blocking from CPU-bound workflow operations.
+
+Adapted from integration-agent/src/swim_thread_manager.py
+- Updated imports to use jarviscore.config
+- Kept core functionality identical
+"""
+import asyncio
+import logging
+import threading
+import time
+from typing import Optional
+
+from swim.transport.hybrid import HybridTransport
+from swim.protocol.node import Node
+from swim.config import get_config as get_swim_config, validate_config as validate_swim_config
+from swim.events.dispatcher import EventDispatcher
+from swim.integration.agent import ZMQAgentIntegration
+from swim.main import SWIMZMQBridge, parse_address as swim_parse_address
+
+logger = logging.getLogger(__name__)
+
+
+class SWIMThreadManager:
+    """Manages SWIM node in a dedicated thread with its own event loop."""
+
+    def __init__(self, config: Optional[dict] = None):
+        """
+        Initialize SWIM Thread Manager.
+
+        Args:
+            config: Configuration dictionary (uses defaults if not provided)
+        """
+        self.config = config or {}
+        self.swim_loop: Optional[asyncio.AbstractEventLoop] = None
+        self.swim_thread: Optional[threading.Thread] = None
+        self.swim_node = None
+        self.zmq_agent = None
+        self.swim_zmq_bridge = None
+        self.event_dispatcher = None
+        self.bind_addr = None  # Store bind address for node_id access
+        self._started = False
+        self._initialized = threading.Event()
+        self._shutdown_event = threading.Event()
+        self._init_error: Optional[str] = None
+
+    def start_swim_in_thread_simple(self):
+        """
+        Start SWIM in dedicated thread using configuration.
+        """
+        if self._started:
+            logger.warning("SWIM thread already started")
+            return
+
+        logger.info("Starting SWIM in dedicated thread...")
+        self.swim_thread = threading.Thread(
+            target=self._run_swim_loop,
+            daemon=True,
+            name="SWIM-Protocol-Thread"
+        )
+        self.swim_thread.start()
+        self._started = True
+        logger.info("SWIM thread started")
+
+    def _run_swim_loop(self):
+        """Run SWIM in dedicated event loop (runs in thread)."""
+        try:
+            # Create new event loop for this thread
+            self.swim_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self.swim_loop)
+
+            logger.info("SWIM thread event loop created")
+
+            # Initialize SWIM
+            self.swim_loop.run_until_complete(self._init_swim())
+
+            if not self.swim_node:
+                logger.error("SWIM initialization failed in thread")
+                self._init_error = "Failed to create SWIM node"
+                self._initialized.set()
+                return
+
+            logger.info("✅ SWIM initialized successfully in dedicated thread")
+            logger.info("   SWIM will never be blocked by workflow execution!")
+
+            # Signal that initialization is complete
+            self._initialized.set()
+
+            # Run event loop until shutdown
+            self._run_until_shutdown()
+
+        except Exception as e:
+            logger.error(f"Error in SWIM thread: {e}", exc_info=True)
+            self._init_error = str(e)
+            self._initialized.set()
+        finally:
+            if self.swim_loop:
+                try:
+                    self.swim_loop.close()
+                except Exception as e:
+                    logger.error(f"Error closing SWIM loop: {e}")
+
+    async def _init_swim(self):
+        """Initialize SWIM components."""
+        try:
+            # Get configuration with defaults
+            bind_host = self.config.get('bind_host', '127.0.0.1')
+            bind_port = self.config.get('bind_port', 7946)
+            node_name = self.config.get('node_name', 'jarviscore-node')
+            seed_nodes = self.config.get('seed_nodes', '')
+            transport_type = self.config.get('transport_type', 'hybrid')
+            zmq_port_offset = self.config.get('zmq_port_offset', 1000)
+
+            # Parse bind address
+            self.bind_addr = swim_parse_address(f"{bind_host}:{bind_port}")
+            logger.info(f"SWIM bind address: {self.bind_addr}")
+
+            # Parse seed nodes
+            seed_addrs = []
+            if seed_nodes:
+                for seed in seed_nodes.split(','):
+                    if seed.strip():
+                        seed_addrs.append(swim_parse_address(seed.strip()))
+            logger.info(f"SWIM seed nodes: {seed_addrs}")
+
+            # Get SWIM config
+            swim_config = get_swim_config()
+            swim_config.update({
+                "NODE_NAME": node_name,
+                "ZMQ_ENABLED": True,
+                "SEND_ON_JOIN": True,
+                "ZMQ_PORT_OFFSET": zmq_port_offset,
+                "TRANSPORT_TYPE": transport_type,
+                "STABILITY_TIMEOUT_SECONDS": 3.0
+            })
+
+            # Validate config
+            errors = validate_swim_config(swim_config)
+            if errors:
+                logger.error(f"SWIM config validation errors: {errors}")
+                return
+
+            # Create transport
+            transport = HybridTransport(
+                udp_max_size=swim_config.get("UDP_MAX_SIZE", 1400),
+                tcp_buffer_size=swim_config.get("TCP_BUFFER_SIZE", 65536),
+                tcp_max_connections=swim_config.get("TCP_MAX_CONNECTIONS", 128)
+            )
+
+            # Create event dispatcher
+            self.event_dispatcher = EventDispatcher(
+                max_history_size=swim_config.get("EVENT_HISTORY", 1000),
+                enable_history=swim_config.get("EVENTS_ENABLED", True)
+            )
+
+            # Create SWIM node
+            logger.info("Creating SWIM node in dedicated thread...")
+            self.swim_node = await Node.create(
+                bind_addr=self.bind_addr,
+                transport=transport,
+                seed_addrs=seed_addrs,
+                config=swim_config,
+                event_dispatcher=self.event_dispatcher,
+                validate_ports=True
+            )
+
+            if not self.swim_node:
+                logger.error("Failed to create SWIM node")
+                return
+
+            logger.info(f"SWIM node created at {self.bind_addr}")
+
+            # Setup ZMQ integration
+            zmq_port = self.bind_addr[1] + swim_config.get("ZMQ_PORT_OFFSET", zmq_port_offset)
+            zmq_addr = f"{self.bind_addr[0]}:{zmq_port}"
+            node_id = f"{self.bind_addr[0]}:{self.bind_addr[1]}"
+            logger.info(f"Setting up ZMQ integration at {zmq_addr}")
+
+            self.zmq_agent = ZMQAgentIntegration(
+                node_id=node_id,
+                bind_address=zmq_addr,
+                event_dispatcher=self.event_dispatcher,
+                config=swim_config
+            )
+
+            # Start ZMQ agent
+            logger.info("Starting ZMQ agent...")
+            await self.zmq_agent.start()
+            logger.info("ZMQ agent started successfully")
+
+            # Setup SWIM-ZMQ Bridge
+            logger.info("Setting up SWIM-ZMQ Bridge...")
+            self.swim_zmq_bridge = SWIMZMQBridge(self.swim_node, self.zmq_agent, swim_config)
+            await self.swim_zmq_bridge.start()
+            logger.info("SWIM-ZMQ Bridge started successfully")
+
+            # Start the SWIM protocol
+            logger.info("Starting SWIM protocol...")
+            await self.swim_node.start()
+            logger.info("SWIM node started successfully")
+
+        except Exception as e:
+            logger.error(f"Error initializing SWIM: {e}", exc_info=True)
+            raise
+
+    def _run_until_shutdown(self):
+        """Keep SWIM event loop running until shutdown requested."""
+        while not self._shutdown_event.is_set():
+            try:
+                # Process events with timeout so we can check shutdown flag
+                self.swim_loop.run_until_complete(asyncio.sleep(0.5))
+            except Exception as e:
+                logger.error(f"Error in SWIM event loop: {e}")
+
+        logger.info("SWIM thread shutdown requested")
+
+        # Cleanup SWIM components
+        try:
+            if self.swim_zmq_bridge and hasattr(self.swim_zmq_bridge, 'stop'):
+                self.swim_loop.run_until_complete(self.swim_zmq_bridge.stop())
+            if self.zmq_agent:
+                self.swim_loop.run_until_complete(self.zmq_agent.stop())
+            if self.swim_node:
+                self.swim_loop.run_until_complete(self.swim_node.stop())
+        except Exception as e:
+            logger.error(f"Error during SWIM shutdown: {e}")
+
+    def wait_for_init(self, timeout: float = 20.0) -> bool:
+        """
+        Wait for SWIM to initialize.
+
+        Args:
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            True if initialized successfully, False if timeout or error
+        """
+        logger.info(f"Waiting for SWIM initialization (timeout: {timeout}s)...")
+
+        if self._initialized.wait(timeout=timeout):
+            if self._init_error:
+                logger.error(f"SWIM initialization failed: {self._init_error}")
+                return False
+            if self.swim_node and self.zmq_agent:
+                logger.info("SWIM initialization confirmed")
+                return True
+            else:
+                logger.error("SWIM initialization incomplete")
+                return False
+        else:
+            logger.error(f"SWIM initialization timeout after {timeout}s")
+            return False
+
+    def is_healthy(self) -> bool:
+        """Check if SWIM thread is healthy."""
+        return (
+            self._started and
+            self.swim_thread is not None and
+            self.swim_thread.is_alive() and
+            self.swim_node is not None and
+            self.zmq_agent is not None
+        )
+
+    def get_status(self) -> dict:
+        """Get SWIM thread status."""
+        return {
+            'started': self._started,
+            'thread_alive': self.swim_thread.is_alive() if self.swim_thread else False,
+            'swim_node': self.swim_node is not None,
+            'zmq_agent': self.zmq_agent is not None,
+            'bridge': self.swim_zmq_bridge is not None,
+            'healthy': self.is_healthy()
+        }
+
+    def shutdown(self, timeout: float = 10.0):
+        """Shutdown SWIM thread gracefully."""
+        if not self._started:
+            return
+
+        logger.info("Shutting down SWIM thread...")
+        self._shutdown_event.set()
+
+        if self.swim_thread:
+            self.swim_thread.join(timeout=timeout)
+            if self.swim_thread.is_alive():
+                logger.warning("SWIM thread did not exit cleanly")
+
+        self._started = False
+        logger.info("SWIM thread shutdown complete")

jarviscore/profiles/__init__.py

@@ -0,0 +1,6 @@
+"""Execution profiles for agents."""
+
+from .autoagent import AutoAgent
+from .customagent import CustomAgent
+
+__all__ = ["AutoAgent", "CustomAgent"]