kailash 0.8.3__py3-none-any.whl → 0.8.5__py3-none-any.whl

kailash/edge/coordination/global_ordering.py

@@ -0,0 +1,355 @@
+"""Global ordering service for distributed events using hybrid logical clocks."""
+
+import asyncio
+import hashlib
+import json
+from collections import defaultdict
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class HybridLogicalClock:
+    """Hybrid Logical Clock (HLC) implementation for global ordering.
+
+    Combines physical time with logical counters to provide:
+    - Causally consistent timestamps
+    - Tolerance for clock skew
+    - Total ordering of events
+    """
+
+    def __init__(self, node_id: str):
+        """Initialize HLC.
+
+        Args:
+            node_id: Unique identifier for this node
+        """
+        self.node_id = node_id
+        self.logical_time = 0
+        self.logical_counter = 0
+        self._lock = asyncio.Lock()
+
+    async def now(self) -> Tuple[int, int, str]:
+        """Get current HLC timestamp.
+
+        Returns:
+            Tuple of (logical_time, logical_counter, node_id)
+        """
+        async with self._lock:
+            physical_time = int(datetime.now().timestamp() * 1000)  # Milliseconds
+
+            if physical_time > self.logical_time:
+                self.logical_time = physical_time
+                self.logical_counter = 0
+            else:
+                self.logical_counter += 1
+
+            return (self.logical_time, self.logical_counter, self.node_id)
+
+    async def update(self, remote_time: int, remote_counter: int):
+        """Update clock with remote timestamp.
+
+        Args:
+            remote_time: Remote logical time
+            remote_counter: Remote logical counter
+        """
+        async with self._lock:
+            physical_time = int(datetime.now().timestamp() * 1000)
+
+            if physical_time > max(self.logical_time, remote_time):
+                self.logical_time = physical_time
+                self.logical_counter = 0
+            elif self.logical_time == remote_time:
+                self.logical_counter = max(self.logical_counter, remote_counter) + 1
+            elif self.logical_time < remote_time:
+                self.logical_time = remote_time
+                self.logical_counter = remote_counter + 1
+            else:
+                self.logical_counter += 1
+
+    def compare(self, ts1: Tuple[int, int, str], ts2: Tuple[int, int, str]) -> int:
+        """Compare two HLC timestamps.
+
+        Args:
+            ts1: First timestamp
+            ts2: Second timestamp
+
+        Returns:
+            -1 if ts1 < ts2, 0 if equal, 1 if ts1 > ts2
+        """
+        if ts1[0] != ts2[0]:
+            return -1 if ts1[0] < ts2[0] else 1
+        if ts1[1] != ts2[1]:
+            return -1 if ts1[1] < ts2[1] else 1
+        if ts1[2] != ts2[2]:
+            return -1 if ts1[2] < ts2[2] else 1
+        return 0
+
+
+class GlobalOrderingService:
+    """Global ordering service for distributed events.
+
+    Provides:
+    - Total ordering of events across edge nodes
+    - Causal dependency tracking
+    - Conflict detection and resolution
+    - Event deduplication
+    """
+
+    def __init__(self, node_id: str):
+        """Initialize global ordering service.
+
+        Args:
+            node_id: Unique identifier for this node
+        """
+        self.node_id = node_id
+        self.clock = HybridLogicalClock(node_id)
+        self.event_history: List[Dict[str, Any]] = []
+        self.causal_graph: Dict[str, List[str]] = defaultdict(list)
+        self.seen_events: set = set()
+        self._lock = asyncio.Lock()
+
+    async def order_events(self, events: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Order a list of events globally.
+
+        Args:
+            events: List of events to order
+
+        Returns:
+            Dict with ordered events and metadata
+        """
+        async with self._lock:
+            ordered_events = []
+
+            for event in events:
+                # Generate event ID if not present
+                if "id" not in event:
+                    event["id"] = self._generate_event_id(event)
+
+                # Skip duplicates
+                if event["id"] in self.seen_events:
+                    continue
+
+                # Assign HLC timestamp
+                timestamp = await self.clock.now()
+                event["hlc_timestamp"] = timestamp
+                event["hlc_time"] = timestamp[0]
+                event["hlc_counter"] = timestamp[1]
+                event["hlc_node"] = timestamp[2]
+
+                # Track causal dependencies
+                if "depends_on" in event:
+                    for dep in event["depends_on"]:
+                        self.causal_graph[event["id"]].append(dep)
+
+                ordered_events.append(event)
+                self.seen_events.add(event["id"])
+
+            # Sort by HLC timestamp
+            ordered_events.sort(
+                key=lambda e: (e["hlc_time"], e["hlc_counter"], e["hlc_node"])
+            )
+
+            # Add to history
+            self.event_history.extend(ordered_events)
+
+            return {
+                "ordered_events": ordered_events,
+                "logical_clock": self.clock.logical_time,
+                "causal_dependencies": dict(self.causal_graph),
+                "total_events": len(self.event_history),
+            }
+
+    async def merge_histories(
+        self, remote_history: List[Dict[str, Any]]
+    ) -> Dict[str, Any]:
+        """Merge remote event history with local history.
+
+        Args:
+            remote_history: Event history from remote node
+
+        Returns:
+            Dict with merged history and conflict information
+        """
+        async with self._lock:
+            conflicts = []
+            merged_events = []
+
+            # Update clock with remote timestamps
+            for event in remote_history:
+                if "hlc_time" in event and "hlc_counter" in event:
+                    await self.clock.update(event["hlc_time"], event["hlc_counter"])
+
+            # Merge histories
+            local_by_id = {e["id"]: e for e in self.event_history if "id" in e}
+
+            for remote_event in remote_history:
+                event_id = remote_event.get("id")
+                if not event_id:
+                    continue
+
+                if event_id in local_by_id:
+                    # Check for conflicts
+                    local_event = local_by_id[event_id]
+                    if self._events_conflict(local_event, remote_event):
+                        conflicts.append(
+                            {
+                                "event_id": event_id,
+                                "local": local_event,
+                                "remote": remote_event,
+                            }
+                        )
+                    # Keep event with higher timestamp
+                    if self._compare_event_timestamps(remote_event, local_event) > 0:
+                        local_by_id[event_id] = remote_event
+                else:
+                    # New event
+                    local_by_id[event_id] = remote_event
+                    self.seen_events.add(event_id)
+
+            # Rebuild ordered history
+            self.event_history = list(local_by_id.values())
+            self.event_history.sort(
+                key=lambda e: (
+                    e.get("hlc_time", 0),
+                    e.get("hlc_counter", 0),
+                    e.get("hlc_node", ""),
+                )
+            )
+
+            return {
+                "merged_events": len(self.event_history),
+                "conflicts": conflicts,
+                "conflict_count": len(conflicts),
+                "logical_clock": self.clock.logical_time,
+            }
+
+    def get_causal_order(self, event_id: str) -> List[str]:
+        """Get causal ordering for an event.
+
+        Args:
+            event_id: Event ID to get dependencies for
+
+        Returns:
+            List of event IDs that must precede this event
+        """
+        visited = set()
+        order = []
+
+        def dfs(eid: str):
+            if eid in visited:
+                return
+            visited.add(eid)
+
+            for dep in self.causal_graph.get(eid, []):
+                dfs(dep)
+
+            order.append(eid)
+
+        dfs(event_id)
+        return order[:-1]  # Exclude the event itself
+
+    def detect_causal_violations(self) -> List[Dict[str, Any]]:
+        """Detect violations of causal ordering.
+
+        Returns:
+            List of violations found
+        """
+        violations = []
+        event_positions = {
+            e["id"]: i for i, e in enumerate(self.event_history) if "id" in e
+        }
+
+        for event_id, deps in self.causal_graph.items():
+            event_pos = event_positions.get(event_id)
+            if event_pos is None:
+                continue
+
+            for dep in deps:
+                dep_pos = event_positions.get(dep)
+                if dep_pos is None:
+                    violations.append(
+                        {
+                            "type": "missing_dependency",
+                            "event": event_id,
+                            "missing": dep,
+                        }
+                    )
+                elif dep_pos > event_pos:
+                    violations.append(
+                        {
+                            "type": "causal_violation",
+                            "event": event_id,
+                            "dependency": dep,
+                            "event_position": event_pos,
+                            "dependency_position": dep_pos,
+                        }
+                    )
+
+        return violations
+
+    def _generate_event_id(self, event: Dict[str, Any]) -> str:
+        """Generate unique event ID.
+
+        Args:
+            event: Event data
+
+        Returns:
+            Unique event ID
+        """
+        # Create deterministic ID from event content
+        content = json.dumps(event, sort_keys=True)
+        hash_obj = hashlib.sha256(content.encode())
+        return f"event_{hash_obj.hexdigest()[:16]}_{self.node_id}"
+
+    def _events_conflict(self, event1: Dict[str, Any], event2: Dict[str, Any]) -> bool:
+        """Check if two events conflict.
+
+        Args:
+            event1: First event
+            event2: Second event
+
+        Returns:
+            True if events conflict
+        """
+        # Events conflict if they have same ID but different content
+        if event1.get("id") != event2.get("id"):
+            return False
+
+        # Compare non-timestamp fields
+        e1_copy = {
+            k: v
+            for k, v in event1.items()
+            if not k.startswith("hlc_") and k != "timestamp"
+        }
+        e2_copy = {
+            k: v
+            for k, v in event2.items()
+            if not k.startswith("hlc_") and k != "timestamp"
+        }
+
+        return e1_copy != e2_copy
+
+    def _compare_event_timestamps(
+        self, event1: Dict[str, Any], event2: Dict[str, Any]
+    ) -> int:
+        """Compare event timestamps.
+
+        Args:
+            event1: First event
+            event2: Second event
+
+        Returns:
+            -1 if event1 < event2, 0 if equal, 1 if event1 > event2
+        """
+        ts1 = (
+            event1.get("hlc_time", 0),
+            event1.get("hlc_counter", 0),
+            event1.get("hlc_node", ""),
+        )
+        ts2 = (
+            event2.get("hlc_time", 0),
+            event2.get("hlc_counter", 0),
+            event2.get("hlc_node", ""),
+        )
+
+        return self.clock.compare(ts1, ts2)

kailash/edge/coordination/leader_election.py

@@ -0,0 +1,217 @@
+"""Edge leader election service using Raft consensus."""
+
+import asyncio
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
+
+from .raft import RaftNode, RaftState
+
+
+class EdgeLeaderElection:
+    """Leader election service for edge nodes using Raft consensus.
+
+    This service manages leader election across edge nodes, providing:
+    - Automatic leader election on startup
+    - Leader failure detection and re-election
+    - Stable leader information for coordination
+    - Network partition handling
+    """
+
+    def __init__(self, raft_nodes: Dict[str, RaftNode]):
+        """Initialize leader election service.
+
+        Args:
+            raft_nodes: Dictionary of node_id -> RaftNode instances
+        """
+        self.raft_nodes = raft_nodes
+        self.current_leader: Optional[str] = None
+        self.current_term: int = 0
+        self.last_leader_change = datetime.now()
+        self.stability_threshold = timedelta(seconds=5)
+        self.logger = logging.getLogger("EdgeLeaderElection")
+
+        # Election monitoring
+        self._monitor_task: Optional[asyncio.Task] = None
+        self._running = False
+
+    async def start(self):
+        """Start leader election monitoring."""
+        self._running = True
+        self._monitor_task = asyncio.create_task(self._monitor_leadership())
+        self.logger.info("Leader election service started")
+
+    async def stop(self):
+        """Stop leader election monitoring."""
+        self._running = False
+        if self._monitor_task:
+            self._monitor_task.cancel()
+            try:
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+        self.logger.info("Leader election service stopped")
+
+    async def start_election(self) -> Dict[str, Any]:
+        """Start a new leader election.
+
+        Returns:
+            Dict with election results including leader and term
+        """
+        self.logger.info("Starting new leader election")
+
+        # Find a candidate node to trigger election
+        candidate_nodes = [
+            node for node in self.raft_nodes.values() if node.state != RaftState.LEADER
+        ]
+
+        if not candidate_nodes:
+            # Current leader still active
+            return self.get_current_leader()
+
+        # Trigger election on first non-leader node
+        candidate = candidate_nodes[0]
+        candidate._become_candidate()
+        await candidate._collect_votes()
+
+        # Wait briefly for election to complete
+        await asyncio.sleep(0.1)
+
+        # Update and return leader info
+        self._update_leader_info()
+        return self.get_current_leader()
+
+    def get_current_leader(self) -> Dict[str, Any]:
+        """Get current leader information.
+
+        Returns:
+            Dict with leader ID, term, and stability status
+        """
+        self._update_leader_info()
+
+        stable = False
+        if self.current_leader:
+            time_since_change = datetime.now() - self.last_leader_change
+            stable = time_since_change > self.stability_threshold
+
+        return {
+            "leader": self.current_leader,
+            "term": self.current_term,
+            "stable": stable,
+            "time_since_change": (
+                datetime.now() - self.last_leader_change
+            ).total_seconds(),
+        }
+
+    def force_election(self) -> None:
+        """Force a new election by demoting current leader."""
+        for node_id, node in self.raft_nodes.items():
+            if node.state == RaftState.LEADER:
+                node._become_follower()
+                self.logger.info(f"Forced leader {node_id} to step down")
+                break
+
+    async def wait_for_stable_leader(self, timeout: float = 10.0) -> Dict[str, Any]:
+        """Wait for a stable leader to be elected.
+
+        Args:
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            Leader information once stable
+
+        Raises:
+            TimeoutError: If no stable leader within timeout
+        """
+        start_time = datetime.now()
+
+        while (datetime.now() - start_time).total_seconds() < timeout:
+            leader_info = self.get_current_leader()
+
+            if leader_info["leader"] and leader_info["stable"]:
+                return leader_info
+
+            await asyncio.sleep(0.1)
+
+        raise TimeoutError(f"No stable leader elected within {timeout} seconds")
+
+    def _update_leader_info(self):
+        """Update current leader information from Raft nodes."""
+        new_leader = None
+        new_term = 0
+
+        for node_id, node in self.raft_nodes.items():
+            if node.state == RaftState.LEADER:
+                new_leader = node_id
+                new_term = node.current_term
+                break
+
+        # Check if leader changed
+        if new_leader != self.current_leader or new_term != self.current_term:
+            self.current_leader = new_leader
+            self.current_term = new_term
+            self.last_leader_change = datetime.now()
+
+            if new_leader:
+                self.logger.info(f"New leader elected: {new_leader} (term {new_term})")
+            else:
+                self.logger.warning("No leader - cluster in election")
+
+    async def _monitor_leadership(self):
+        """Background task to monitor leadership stability."""
+        while self._running:
+            try:
+                self._update_leader_info()
+
+                # Check if we need to trigger election
+                leader_info = self.get_current_leader()
+                if not leader_info["leader"]:
+                    # No leader for too long
+                    time_without_leader = (
+                        datetime.now() - self.last_leader_change
+                    ).total_seconds()
+                    if time_without_leader > 2.0:  # 2 seconds without leader
+                        self.logger.warning(
+                            "No leader for 2 seconds, triggering election"
+                        )
+                        await self.start_election()
+
+                await asyncio.sleep(0.5)  # Check every 500ms
+
+            except Exception as e:
+                self.logger.error(f"Leadership monitor error: {e}")
+
+    def get_cluster_health(self) -> Dict[str, Any]:
+        """Get health information about the cluster.
+
+        Returns:
+            Dict with cluster health metrics
+        """
+        total_nodes = len(self.raft_nodes)
+        leader_count = sum(
+            1 for node in self.raft_nodes.values() if node.state == RaftState.LEADER
+        )
+        follower_count = sum(
+            1 for node in self.raft_nodes.values() if node.state == RaftState.FOLLOWER
+        )
+        candidate_count = sum(
+            1 for node in self.raft_nodes.values() if node.state == RaftState.CANDIDATE
+        )
+
+        # Check for split brain
+        split_brain = leader_count > 1
+
+        # Check for partitions
+        has_quorum = (follower_count + leader_count) > total_nodes // 2
+
+        return {
+            "total_nodes": total_nodes,
+            "leader_count": leader_count,
+            "follower_count": follower_count,
+            "candidate_count": candidate_count,
+            "split_brain": split_brain,
+            "has_quorum": has_quorum,
+            "current_leader": self.current_leader,
+            "current_term": self.current_term,
+            "healthy": leader_count == 1 and has_quorum and not split_brain,
+        }