qyro 2.0.0__py3-none-any.whl

qyro/common/health.py

@@ -0,0 +1,358 @@
+"""
+Nexus Health Check System
+Provides health check endpoints and metrics for monitoring.
+"""
+
+import time
+import os
+import psutil
+from typing import Dict, Any, List, Optional
+from dataclasses import dataclass, field
+from enum import Enum
+
+from .logging import get_logger
+from .constants import PROTOCOL_VERSION, MEM_FILE
+
+logger = get_logger("nexus.health")
+
+
+class HealthStatus(Enum):
+    """Health status levels."""
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class ComponentHealth:
+    """Health status of a single component."""
+    name: str
+    status: HealthStatus
+    message: str = ""
+    latency_ms: float = 0.0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class SystemHealth:
+    """Overall system health status."""
+    status: HealthStatus
+    version: str
+    uptime_seconds: float
+    components: List[ComponentHealth]
+    timestamp: float = field(default_factory=time.time)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "status": self.status.value,
+            "version": self.version,
+            "uptime_seconds": round(self.uptime_seconds, 2),
+            "timestamp": self.timestamp,
+            "components": [
+                {
+                    "name": c.name,
+                    "status": c.status.value,
+                    "message": c.message,
+                    "latency_ms": round(c.latency_ms, 2),
+                    "metadata": c.metadata
+                }
+                for c in self.components
+            ]
+        }
+
+
+class HealthChecker:
+    """
+    Health check system for Nexus runtime.
+    
+    Checks:
+    - Memory availability and integrity
+    - Lock file accessibility
+    - Process status
+    - Resource usage
+    """
+    
+    def __init__(self, memory=None, orchestrator=None):
+        self._memory = memory
+        self._orchestrator = orchestrator
+        self._start_time = time.time()
+        self._version = f"2.0.0-nbp{PROTOCOL_VERSION}"
+    
+    def check(self) -> SystemHealth:
+        """Perform full health check."""
+        components = []
+        
+        # Check memory
+        components.append(self._check_memory())
+        
+        # Check lock file
+        components.append(self._check_lock())
+        
+        # Check processes
+        if self._orchestrator:
+            components.append(self._check_processes())
+        
+        # Check system resources
+        components.append(self._check_resources())
+        
+        # Determine overall status
+        statuses = [c.status for c in components]
+        if all(s == HealthStatus.HEALTHY for s in statuses):
+            overall = HealthStatus.HEALTHY
+        elif any(s == HealthStatus.UNHEALTHY for s in statuses):
+            overall = HealthStatus.UNHEALTHY
+        else:
+            overall = HealthStatus.DEGRADED
+        
+        return SystemHealth(
+            status=overall,
+            version=self._version,
+            uptime_seconds=time.time() - self._start_time,
+            components=components
+        )
+    
+    def _check_memory(self) -> ComponentHealth:
+        """Check shared memory health."""
+        start = time.time()
+        
+        try:
+            if not os.path.exists(MEM_FILE):
+                return ComponentHealth(
+                    name="memory",
+                    status=HealthStatus.UNHEALTHY,
+                    message="Memory file not found"
+                )
+            
+            # Check file size
+            size = os.path.getsize(MEM_FILE)
+            if size < 1000:
+                return ComponentHealth(
+                    name="memory",
+                    status=HealthStatus.DEGRADED,
+                    message=f"Memory file too small: {size} bytes"
+                )
+            
+            # Try to read if memory object available
+            if self._memory:
+                try:
+                    stats = self._memory.get_stats()
+                    latency = (time.time() - start) * 1000
+                    
+                    return ComponentHealth(
+                        name="memory",
+                        status=HealthStatus.HEALTHY,
+                        message="Memory accessible",
+                        latency_ms=latency,
+                        metadata={
+                            "size_mb": round(stats.get("total_size", 0) / 1024 / 1024, 2),
+                            "utilization": round(stats.get("utilization", 0) * 100, 1),
+                            "sequence": stats.get("sequence", 0),
+                            "encrypted": stats.get("encrypted", False),
+                            "protocol": stats.get("protocol_version", 0)
+                        }
+                    )
+                except Exception as e:
+                    return ComponentHealth(
+                        name="memory",
+                        status=HealthStatus.DEGRADED,
+                        message=f"Memory read error: {str(e)}"
+                    )
+            
+            return ComponentHealth(
+                name="memory",
+                status=HealthStatus.HEALTHY,
+                message=f"Memory file exists ({size} bytes)",
+                latency_ms=(time.time() - start) * 1000
+            )
+            
+        except Exception as e:
+            return ComponentHealth(
+                name="memory",
+                status=HealthStatus.UNHEALTHY,
+                message=str(e)
+            )
+    
+    def _check_lock(self) -> ComponentHealth:
+        """Check lock file accessibility."""
+        lock_file = ".nexus_global.lock"
+        start = time.time()
+        
+        try:
+            # Try to open lock file
+            with open(lock_file, 'a') as f:
+                pass
+            
+            return ComponentHealth(
+                name="lock",
+                status=HealthStatus.HEALTHY,
+                message="Lock file accessible",
+                latency_ms=(time.time() - start) * 1000
+            )
+        except Exception as e:
+            return ComponentHealth(
+                name="lock",
+                status=HealthStatus.DEGRADED,
+                message=f"Lock file issue: {str(e)}"
+            )
+    
+    def _check_processes(self) -> ComponentHealth:
+        """Check supervised process health."""
+        try:
+            if not self._orchestrator:
+                return ComponentHealth(
+                    name="processes",
+                    status=HealthStatus.UNKNOWN,
+                    message="No orchestrator"
+                )
+            
+            status = self._orchestrator.get_status()
+            processes = status.get("processes", [])
+            
+            running = sum(1 for p in processes if p.get("status") == "running")
+            total = len(processes)
+            
+            if running == total:
+                return ComponentHealth(
+                    name="processes",
+                    status=HealthStatus.HEALTHY,
+                    message=f"All {total} processes running",
+                    metadata={"running": running, "total": total}
+                )
+            elif running > 0:
+                return ComponentHealth(
+                    name="processes",
+                    status=HealthStatus.DEGRADED,
+                    message=f"{running}/{total} processes running",
+                    metadata={"running": running, "total": total}
+                )
+            else:
+                return ComponentHealth(
+                    name="processes",
+                    status=HealthStatus.UNHEALTHY,
+                    message="No processes running",
+                    metadata={"running": 0, "total": total}
+                )
+                
+        except Exception as e:
+            return ComponentHealth(
+                name="processes",
+                status=HealthStatus.UNKNOWN,
+                message=str(e)
+            )
+    
+    def _check_resources(self) -> ComponentHealth:
+        """Check system resource usage."""
+        try:
+            cpu_percent = psutil.cpu_percent(interval=0.1)
+            memory = psutil.virtual_memory()
+            disk = psutil.disk_usage('.')
+            
+            # Determine status based on usage
+            if memory.percent > 90 or disk.percent > 90:
+                status = HealthStatus.UNHEALTHY
+                message = "Critical resource usage"
+            elif memory.percent > 75 or disk.percent > 80:
+                status = HealthStatus.DEGRADED
+                message = "High resource usage"
+            else:
+                status = HealthStatus.HEALTHY
+                message = "Resources OK"
+            
+            return ComponentHealth(
+                name="resources",
+                status=status,
+                message=message,
+                metadata={
+                    "cpu_percent": round(cpu_percent, 1),
+                    "memory_percent": round(memory.percent, 1),
+                    "memory_available_gb": round(memory.available / 1024 / 1024 / 1024, 2),
+                    "disk_percent": round(disk.percent, 1),
+                    "disk_free_gb": round(disk.free / 1024 / 1024 / 1024, 2)
+                }
+            )
+            
+        except Exception as e:
+            return ComponentHealth(
+                name="resources",
+                status=HealthStatus.UNKNOWN,
+                message=str(e)
+            )
+    
+    def liveness(self) -> bool:
+        """Simple liveness check (is the process alive)."""
+        return True
+    
+    def readiness(self) -> bool:
+        """
+        Readiness check (is the service ready to handle requests).
+        Returns True if memory is accessible.
+        """
+        try:
+            return os.path.exists(MEM_FILE)
+        except:
+            return False
+
+
+# HTTP Health Endpoint Handler (for FastAPI/Flask integration)
+def create_health_routes(app, health_checker: HealthChecker):
+    """
+    Add health check routes to a FastAPI or Flask app.
+    
+    Usage (FastAPI):
+        from nexus_core.health import HealthChecker, create_health_routes
+        health = HealthChecker(memory)
+        create_health_routes(app, health)
+    """
+    try:
+        # Try FastAPI
+        from fastapi import APIRouter
+        router = APIRouter()
+        
+        @router.get("/health")
+        def health():
+            return health_checker.check().to_dict()
+        
+        @router.get("/health/live")
+        def liveness():
+            return {"status": "alive" if health_checker.liveness() else "dead"}
+        
+        @router.get("/health/ready")
+        def readiness():
+            return {"status": "ready" if health_checker.readiness() else "not_ready"}
+        
+        app.include_router(router)
+        return router
+        
+    except ImportError:
+        pass
+    
+    try:
+        # Try Flask
+        from flask import jsonify
+        
+        @app.route("/health")
+        def health():
+            return jsonify(health_checker.check().to_dict())
+        
+        @app.route("/health/live")
+        def liveness():
+            return jsonify({"status": "alive" if health_checker.liveness() else "dead"})
+        
+        @app.route("/health/ready")
+        def readiness():
+            return jsonify({"status": "ready" if health_checker.readiness() else "not_ready"})
+            
+    except ImportError:
+        logger.warning("no_web_framework", msg="Neither FastAPI nor Flask found")
+
+
+# Global health checker instance
+_health_checker: Optional[HealthChecker] = None
+
+def get_health_checker(memory=None, orchestrator=None) -> HealthChecker:
+    """Get or create the global health checker."""
+    global _health_checker
+    if _health_checker is None:
+        _health_checker = HealthChecker(memory, orchestrator)
+    return _health_checker

qyro/common/kafka_manager.py

@@ -0,0 +1,192 @@
+"""
+Kafka Manager for Nexus
+Handles Kafka integration for inter-module communication.
+"""
+
+import asyncio
+import json
+from typing import Dict, Any, Callable, Optional
+from dataclasses import dataclass
+from aiokafka import AIOKafkaProducer, AIOKafkaConsumer
+import logging
+
+from .config import QyroConfig
+from .logging import get_logger
+
+
+logger = get_logger("nexus.kafka_manager")
+
+
+@dataclass
+class KafkaConfig:
+    """Configuration for Kafka integration."""
+    bootstrap_servers: str = "localhost:9092"
+    topic_prefix: str = "nexus_"
+    consumer_group: str = "nexus_group"
+    enable_auto_commit: bool = True
+    auto_offset_reset: str = "earliest"
+
+
+class KafkaManager:
+    """Manages Kafka producers and consumers for Nexus modules."""
+    
+    def __init__(self, config: QyroConfig):
+        self.config = config
+        self.producer: Optional[AIOKafkaProducer] = None
+        self.consumer: Optional[AIOKafkaConsumer] = None
+        self.running = False
+        
+        # Initialize Kafka configuration
+        self.kafka_config = KafkaConfig(
+            bootstrap_servers=config.kafka_bootstrap_servers
+        )
+        
+        logger.info(f"Kafka manager initialized with servers: {self.kafka_config.bootstrap_servers}")
+
+    async def start_producer(self):
+        """Start the Kafka producer."""
+        try:
+            self.producer = AIOKafkaProducer(
+                bootstrap_servers=self.kafka_config.bootstrap_servers.split(','),
+                value_serializer=lambda x: json.dumps(x).encode('utf-8'),
+                acks='all'
+            )
+            await self.producer.start()
+            logger.info("Kafka producer started successfully")
+        except Exception as e:
+            logger.error(f"Failed to start Kafka producer: {e}")
+            raise
+
+    async def start_consumer(self, topics: list):
+        """Start the Kafka consumer."""
+        try:
+            self.consumer = AIOKafkaConsumer(
+                *topics,
+                bootstrap_servers=self.kafka_config.bootstrap_servers.split(','),
+                value_deserializer=lambda x: json.loads(x.decode('utf-8')),
+                group_id=self.kafka_config.consumer_group,
+                enable_auto_commit=self.kafka_config.enable_auto_commit,
+                auto_offset_reset=self.kafka_config.auto_offset_reset
+            )
+            await self.consumer.start()
+            logger.info(f"Kafka consumer started successfully for topics: {topics}")
+        except Exception as e:
+            logger.error(f"Failed to start Kafka consumer: {e}")
+            raise
+
+    async def stop_producer(self):
+        """Stop the Kafka producer."""
+        if self.producer:
+            await self.producer.stop()
+            logger.info("Kafka producer stopped")
+
+    async def stop_consumer(self):
+        """Stop the Kafka consumer."""
+        if self.consumer:
+            await self.consumer.stop()
+            logger.info("Kafka consumer stopped")
+
+    async def send_message(self, topic: str, message: Dict[str, Any], key: Optional[str] = None):
+        """Send a message to a Kafka topic."""
+        if not self.producer:
+            await self.start_producer()
+            
+        try:
+            await self.producer.send_and_wait(topic, message, key=key.encode('utf-8') if key else None)
+            logger.debug(f"Message sent to topic '{topic}': {message}")
+        except Exception as e:
+            logger.error(f"Failed to send message to topic '{topic}': {e}")
+            raise
+
+    async def consume_messages(self, topic: str, callback: Callable[[Dict[str, Any]], None]):
+        """Consume messages from a Kafka topic."""
+        if not self.consumer:
+            await self.start_consumer([topic])
+            
+        try:
+            async for msg in self.consumer:
+                if msg.topic == topic:
+                    logger.debug(f"Received message from topic '{topic}': {msg.value}")
+                    callback(msg.value)
+        except Exception as e:
+            logger.error(f"Error consuming messages from topic '{topic}': {e}")
+            raise
+
+    def start(self):
+        """Start the Kafka manager in a background task."""
+        self.running = True
+        # Start producer in background
+        asyncio.create_task(self._run_producer())
+
+    def stop(self):
+        """Stop the Kafka manager."""
+        self.running = False
+
+    async def _run_producer(self):
+        """Internal method to run the producer."""
+        await self.start_producer()
+        while self.running:
+            await asyncio.sleep(0.1)  # Keep the task alive
+
+    async def publish_state_change(self, state_diff: Dict[str, Any], module_name: str):
+        """Publish a state change to the state change topic."""
+        topic = f"{self.kafka_config.topic_prefix}state_change"
+        message = {
+            "module": module_name,
+            "timestamp": asyncio.get_event_loop().time(),
+            "state_diff": state_diff
+        }
+        await self.send_message(topic, message)
+
+    async def subscribe_to_state_changes(self, callback: Callable[[Dict[str, Any]], None]):
+        """Subscribe to state change events."""
+        topic = f"{self.kafka_config.topic_prefix}state_change"
+        await self.consume_messages(topic, callback)
+
+    async def publish_module_event(self, event_type: str, module_name: str, data: Dict[str, Any]):
+        """Publish a module event to the events topic."""
+        topic = f"{self.kafka_config.topic_prefix}module_events"
+        message = {
+            "event_type": event_type,
+            "module": module_name,
+            "timestamp": asyncio.get_event_loop().time(),
+            "data": data
+        }
+        await self.send_message(topic, message)
+
+    async def publish_rpc_request(self, func_name: str, args: Dict[str, Any], request_id: str):
+        """Publish an RPC request to the appropriate topic."""
+        topic = f"{self.kafka_config.topic_prefix}rpc_requests"
+        message = {
+            "func_name": func_name,
+            "args": args,
+            "request_id": request_id,
+            "timestamp": asyncio.get_event_loop().time()
+        }
+        await self.send_message(topic, message)
+
+    async def publish_rpc_response(self, request_id: str, result: Any, error: Optional[str] = None):
+        """Publish an RPC response to the appropriate topic."""
+        topic = f"{self.kafka_config.topic_prefix}rpc_responses"
+        message = {
+            "request_id": request_id,
+            "result": result,
+            "error": error,
+            "timestamp": asyncio.get_event_loop().time()
+        }
+        await self.send_message(topic, message)
+
+    async def broadcast_message(self, message: Dict[str, Any], broadcast_id: str):
+        """Broadcast a message to all modules."""
+        topic = f"{self.kafka_config.topic_prefix}broadcast"
+        broadcast_msg = {
+            "broadcast_id": broadcast_id,
+            "message": message,
+            "timestamp": asyncio.get_event_loop().time()
+        }
+        await self.send_message(topic, broadcast_msg)
+
+    async def subscribe_to_broadcasts(self, callback: Callable[[Dict[str, Any]], None]):
+        """Subscribe to broadcast messages."""
+        topic = f"{self.kafka_config.topic_prefix}broadcast"
+        await self.consume_messages(topic, callback)

qyro/common/logging.py

@@ -0,0 +1,149 @@
+"""
+Qyro Structured Logging
+JSON-based logging with severity levels for cross-language consistency.
+"""
+
+import json
+import sys
+import time
+from datetime import datetime
+from enum import IntEnum
+from typing import Any, Dict, Optional
+from pathlib import Path
+
+
+class LogLevel(IntEnum):
+    DEBUG = 10
+    INFO = 20
+    WARNING = 30
+    ERROR = 40
+    CRITICAL = 50
+
+
+class QyroLogger:
+    """
+    Structured JSON logger for Qyro system.
+    Outputs machine-readable logs that can be aggregated across languages.
+    """
+
+    _instances: Dict[str, 'QyroLogger'] = {}
+
+    def __init__(
+        self,
+        name: str = "qyro",
+        level: LogLevel = LogLevel.INFO,
+        output_file: Optional[str] = None,
+        json_output: bool = True
+    ):
+        self.name = name
+        self.level = level
+        self.json_output = json_output
+        self._file = None
+        
+        if output_file:
+            Path(output_file).parent.mkdir(parents=True, exist_ok=True)
+            self._file = open(output_file, 'a', encoding='utf-8')
+    
+    @classmethod
+    def get_logger(cls, name: str = "qyro") -> 'QyroLogger':
+        """Get or create a logger instance."""
+        if name not in cls._instances:
+            cls._instances[name] = cls(name)
+        return cls._instances[name]
+    
+    def _format_message(
+        self,
+        level: LogLevel,
+        message: str,
+        **context
+    ) -> str:
+        """Format log message as JSON or plain text."""
+        timestamp = datetime.utcnow().isoformat() + "Z"
+        
+        if self.json_output:
+            log_entry = {
+                "timestamp": timestamp,
+                "level": level.name,
+                "logger": self.name,
+                "message": message,
+                **context
+            }
+            return json.dumps(log_entry, default=str)
+        else:
+            ctx_str = " ".join(f"{k}={v}" for k, v in context.items())
+            return f"[{timestamp}] [{level.name}] [{self.name}] {message} {ctx_str}".strip()
+    
+    def _log(self, level: LogLevel, message: str, **context):
+        """Internal log method."""
+        if level < self.level:
+            return
+        
+        formatted = self._format_message(level, message, **context)
+        
+        # Output to stderr for errors, stdout for others
+        output = sys.stderr if level >= LogLevel.ERROR else sys.stdout
+        try:
+            print(formatted, file=output, flush=True)
+        except UnicodeEncodeError:
+            # Fallback for Windows consoles: use the stream's own encoding with replacements
+            encoding = getattr(output, 'encoding', 'utf-8') or 'utf-8'
+            safe_msg = formatted.encode(encoding, errors='replace').decode(encoding)
+            print(safe_msg, file=output, flush=True)
+        
+        # Also write to file if configured
+        if self._file:
+            print(formatted, file=self._file, flush=True)
+    
+    def debug(self, message: str, **context):
+        self._log(LogLevel.DEBUG, message, **context)
+    
+    def info(self, message: str, **context):
+        self._log(LogLevel.INFO, message, **context)
+    
+    def warning(self, message: str, **context):
+        self._log(LogLevel.WARNING, message, **context)
+    
+    def error(self, message: str, **context):
+        self._log(LogLevel.ERROR, message, **context)
+    
+    def critical(self, message: str, **context):
+        self._log(LogLevel.CRITICAL, message, **context)
+    
+    def exception(self, message: str, exc: Exception, **context):
+        """Log exception with traceback info."""
+        import traceback
+        context['exception_type'] = type(exc).__name__
+        context['exception_message'] = str(exc)
+        context['traceback'] = traceback.format_exc()
+        self._log(LogLevel.ERROR, message, **context)
+    
+    def close(self):
+        if self._file:
+            self._file.close()
+
+
+def get_logger(name: str = "qyro") -> QyroLogger:
+    """Get a named logger instance."""
+    return QyroLogger.get_logger(name)
+
+
+def set_log_level(level: LogLevel):
+    """Set the global log level."""
+    get_logger().level = level
+
+
+# Convenience functions
+def debug(message: str, **context):
+    get_logger().debug(message, **context)
+
+def info(message: str, **context):
+    get_logger().info(message, **context)
+
+def warning(message: str, **context):
+    get_logger().warning(message, **context)
+
+def error(message: str, **context):
+    get_logger().error(message, **context)
+
+def critical(message: str, **context):
+    get_logger().critical(message, **context)