isa-model 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

isa_model/deployment/services/model_service.py

@@ -0,0 +1,332 @@
+"""
+ModelService - Core abstraction for deployed model services in the MaaS platform
+
+This represents a deployed service instance that can be discovered, monitored, and invoked.
+It's the bridge between the high-level AIFactory interface and the underlying platform services.
+"""
+
+from typing import Dict, List, Optional, Any, Union
+from enum import Enum
+from dataclasses import dataclass, field
+from datetime import datetime
+import logging
+
+logger = logging.getLogger(__name__)
+
+class ServiceStatus(str, Enum):
+    """Service deployment and health status"""
+    PENDING = "pending"
+    DEPLOYING = "deploying"
+    HEALTHY = "healthy"
+    UNHEALTHY = "unhealthy"
+    STOPPED = "stopped"
+
+class ServiceType(str, Enum):
+    """Types of services available in the platform"""
+    LLM = "llm"
+    EMBEDDING = "embedding"
+    VISION = "vision"
+    AUDIO = "audio"
+    IMAGE_GEN = "image_gen"
+
+class DeploymentPlatform(str, Enum):
+    """Supported deployment platforms for self-owned services only"""
+    MODAL = "modal"
+    KUBERNETES = "kubernetes"
+    RUNPOD = "runpod"
+    YYDS = "yyds"
+    OLLAMA = "ollama"  # Local deployment
+
+@dataclass
+class HealthMetrics:
+    """Service health metrics"""
+    is_healthy: bool
+    response_time_ms: Optional[int] = None
+    status_code: Optional[int] = None
+    cpu_usage_percent: Optional[float] = None
+    memory_usage_mb: Optional[int] = None
+    gpu_usage_percent: Optional[float] = None
+    error_message: Optional[str] = None
+    checked_at: Optional[datetime] = None
+
+@dataclass
+class UsageMetrics:
+    """Service usage and cost metrics"""
+    request_count: int = 0
+    total_processing_time_ms: int = 0
+    error_count: int = 0
+    total_cost_usd: float = 0.0
+    window_start: Optional[datetime] = None
+    window_end: Optional[datetime] = None
+
+@dataclass
+class ResourceRequirements:
+    """Service resource requirements"""
+    gpu_type: Optional[str] = None
+    memory_mb: Optional[int] = None
+    cpu_cores: Optional[int] = None
+    storage_gb: Optional[int] = None
+    min_replicas: int = 0
+    max_replicas: int = 1
+
+class ModelService:
+    """
+    Core abstraction for a deployed model service in the MaaS platform
+    
+    This class represents a self-owned deployed service instance that:
+    - Has been deployed to a platform (Modal, Kubernetes, RunPod, etc.)
+    - Can be discovered through the ServiceRegistry
+    - Can be health-checked and monitored
+    - Provides inference capabilities through specific endpoints
+    
+    Note: This is only for self-owned deployments. Third-party services 
+    (OpenAI, Replicate, etc.) are managed by ThirdPartyServiceManager.
+    """
+    
+    def __init__(
+        self,
+        service_id: str,
+        service_name: str,
+        model_id: Optional[str],
+        deployment_platform: DeploymentPlatform,
+        service_type: ServiceType,
+        inference_endpoint: Optional[str] = None,
+        health_endpoint: Optional[str] = None,
+        capabilities: Optional[List[str]] = None,
+        config: Optional[Dict[str, Any]] = None,
+        resource_requirements: Optional[ResourceRequirements] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        status: ServiceStatus = ServiceStatus.PENDING
+    ):
+        self.service_id = service_id
+        self.service_name = service_name
+        self.model_id = model_id
+        self.deployment_platform = deployment_platform
+        self.service_type = service_type
+        self.status = status
+        
+        # Endpoints
+        self.inference_endpoint = inference_endpoint
+        self.health_endpoint = health_endpoint
+        
+        # Capabilities and configuration
+        self.capabilities = capabilities or []
+        self.config = config or {}
+        self.resource_requirements = resource_requirements or ResourceRequirements()
+        self.metadata = metadata or {}
+        
+        # Metrics (populated by monitoring systems)
+        self.health_metrics: Optional[HealthMetrics] = None
+        self.usage_metrics: Optional[UsageMetrics] = None
+        
+        # Timestamps
+        self.created_at: Optional[datetime] = None
+        self.updated_at: Optional[datetime] = None
+        
+        logger.debug(f"Created ModelService: {service_id} ({service_name})")
+    
+    def is_healthy(self) -> bool:
+        """Check if the service is currently healthy"""
+        if self.status != ServiceStatus.HEALTHY:
+            return False
+        
+        if self.health_metrics:
+            return self.health_metrics.is_healthy
+        
+        # If no health metrics, assume healthy if status is healthy
+        return True
+    
+    def is_available(self) -> bool:
+        """Check if the service is available for inference requests"""
+        return (
+            self.status == ServiceStatus.HEALTHY and
+            self.inference_endpoint is not None and
+            self.is_healthy()
+        )
+    
+    def has_capability(self, capability: str) -> bool:
+        """Check if this service provides a specific capability"""
+        return capability in self.capabilities
+    
+    def get_endpoint_url(self, endpoint_type: str = "inference") -> Optional[str]:
+        """Get endpoint URL for the service"""
+        if endpoint_type == "inference":
+            return self.inference_endpoint
+        elif endpoint_type == "health":
+            return self.health_endpoint
+        else:
+            # Check if it's in metadata
+            endpoints = self.metadata.get("endpoints", {})
+            return endpoints.get(endpoint_type)
+    
+    def update_health_metrics(self, metrics: HealthMetrics) -> None:
+        """Update health metrics for this service"""
+        self.health_metrics = metrics
+        
+        # Update service status based on health
+        if metrics.is_healthy:
+            if self.status != ServiceStatus.HEALTHY:
+                self.status = ServiceStatus.HEALTHY
+                logger.info(f"Service {self.service_id} is now healthy")
+        else:
+            if self.status == ServiceStatus.HEALTHY:
+                self.status = ServiceStatus.UNHEALTHY
+                logger.warning(f"Service {self.service_id} is now unhealthy: {metrics.error_message}")
+    
+    def update_usage_metrics(self, metrics: UsageMetrics) -> None:
+        """Update usage metrics for this service"""
+        self.usage_metrics = metrics
+        logger.debug(f"Updated usage metrics for {self.service_id}: {metrics.request_count} requests")
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert service to dictionary representation"""
+        return {
+            "service_id": self.service_id,
+            "service_name": self.service_name,
+            "model_id": self.model_id,
+            "deployment_platform": self.deployment_platform.value,
+            "service_type": self.service_type.value,
+            "status": self.status.value,
+            "inference_endpoint": self.inference_endpoint,
+            "health_endpoint": self.health_endpoint,
+            "capabilities": self.capabilities,
+            "config": self.config,
+            "resource_requirements": {
+                "gpu_type": self.resource_requirements.gpu_type,
+                "memory_mb": self.resource_requirements.memory_mb,
+                "cpu_cores": self.resource_requirements.cpu_cores,
+                "storage_gb": self.resource_requirements.storage_gb,
+                "min_replicas": self.resource_requirements.min_replicas,
+                "max_replicas": self.resource_requirements.max_replicas,
+            },
+            "metadata": self.metadata,
+            "health_metrics": {
+                "is_healthy": self.health_metrics.is_healthy if self.health_metrics else None,
+                "response_time_ms": self.health_metrics.response_time_ms if self.health_metrics else None,
+                "status_code": self.health_metrics.status_code if self.health_metrics else None,
+                "error_message": self.health_metrics.error_message if self.health_metrics else None,
+                "checked_at": self.health_metrics.checked_at.isoformat() if self.health_metrics and self.health_metrics.checked_at else None,
+            } if self.health_metrics else None,
+            "usage_metrics": {
+                "request_count": self.usage_metrics.request_count if self.usage_metrics else 0,
+                "total_processing_time_ms": self.usage_metrics.total_processing_time_ms if self.usage_metrics else 0,
+                "error_count": self.usage_metrics.error_count if self.usage_metrics else 0,
+                "total_cost_usd": self.usage_metrics.total_cost_usd if self.usage_metrics else 0.0,
+            } if self.usage_metrics else None,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'ModelService':
+        """Create ModelService from dictionary representation"""
+        # Create resource requirements
+        resource_data = data.get("resource_requirements", {})
+        resources = ResourceRequirements(
+            gpu_type=resource_data.get("gpu_type"),
+            memory_mb=resource_data.get("memory_mb"),
+            cpu_cores=resource_data.get("cpu_cores"),
+            storage_gb=resource_data.get("storage_gb"),
+            min_replicas=resource_data.get("min_replicas", 0),
+            max_replicas=resource_data.get("max_replicas", 1),
+        )
+        
+        # Create service
+        service = cls(
+            service_id=data["service_id"],
+            service_name=data["service_name"],
+            model_id=data.get("model_id"),
+            deployment_platform=DeploymentPlatform(data["deployment_platform"]),
+            service_type=ServiceType(data["service_type"]),
+            status=ServiceStatus(data.get("status", "pending")),
+            inference_endpoint=data.get("inference_endpoint"),
+            health_endpoint=data.get("health_endpoint"),
+            capabilities=data.get("capabilities", []),
+            config=data.get("config", {}),
+            resource_requirements=resources,
+            metadata=data.get("metadata", {}),
+        )
+        
+        # Set timestamps
+        if data.get("created_at"):
+            service.created_at = datetime.fromisoformat(data["created_at"].replace('Z', '+00:00'))
+        if data.get("updated_at"):
+            service.updated_at = datetime.fromisoformat(data["updated_at"].replace('Z', '+00:00'))
+        
+        # Set health metrics
+        health_data = data.get("health_metrics")
+        if health_data and health_data.get("is_healthy") is not None:
+            checked_at = None
+            if health_data.get("checked_at"):
+                checked_at = datetime.fromisoformat(health_data["checked_at"].replace('Z', '+00:00'))
+            
+            service.health_metrics = HealthMetrics(
+                is_healthy=health_data["is_healthy"],
+                response_time_ms=health_data.get("response_time_ms"),
+                status_code=health_data.get("status_code"),
+                error_message=health_data.get("error_message"),
+                checked_at=checked_at,
+            )
+        
+        # Set usage metrics
+        usage_data = data.get("usage_metrics")
+        if usage_data:
+            service.usage_metrics = UsageMetrics(
+                request_count=usage_data.get("request_count", 0),
+                total_processing_time_ms=usage_data.get("total_processing_time_ms", 0),
+                error_count=usage_data.get("error_count", 0),
+                total_cost_usd=usage_data.get("total_cost_usd", 0.0),
+            )
+        
+        return service
+    
+    def __repr__(self) -> str:
+        return f"ModelService(id={self.service_id}, name={self.service_name}, platform={self.deployment_platform.value}, status={self.status.value})"
+    
+    def __str__(self) -> str:
+        return f"{self.service_name} ({self.service_id}) on {self.deployment_platform.value} - {self.status.value}"
+
+# Factory functions for common service types
+
+def create_modal_service(
+    service_name: str,
+    model_id: str,
+    inference_endpoint: str,
+    health_endpoint: Optional[str] = None,
+    capabilities: Optional[List[str]] = None,
+    gpu_type: str = "T4",
+    memory_mb: int = 16384,
+    **kwargs
+) -> ModelService:
+    """Factory function for Modal-deployed services"""
+    service_id = f"{service_name}-modal-{datetime.now().strftime('%Y%m%d%H%M%S')}"
+    
+    resources = ResourceRequirements(
+        gpu_type=gpu_type,
+        memory_mb=memory_mb,
+        min_replicas=0,  # Modal can scale to zero
+        max_replicas=10,  # Reasonable default
+    )
+    
+    return ModelService(
+        service_id=service_id,
+        service_name=service_name,
+        model_id=model_id,
+        deployment_platform=DeploymentPlatform.MODAL,
+        service_type=ServiceType.VISION,  # Most Modal services are vision
+        inference_endpoint=inference_endpoint,
+        health_endpoint=health_endpoint,
+        capabilities=capabilities or [],
+        resource_requirements=resources,
+        metadata={
+            "platform": "modal",
+            "auto_scaling": True,
+            "scale_to_zero": True,
+            **kwargs
+        },
+        status=ServiceStatus.HEALTHY,  # Assume healthy when creating
+    )
+
+# REMOVED: create_openai_service function
+# OpenAI is a third-party service provider, not a deployment platform.
+# Use ThirdPartyServiceManager in the inference module instead.

isa_model/deployment/services/service_monitor.py

@@ -0,0 +1,356 @@
+"""
+Service Health Monitor - Automated health checking and service discovery for MaaS platform
+
+This module provides automated health monitoring and service discovery capabilities
+for the ISA Model MaaS platform.
+"""
+
+import asyncio
+import logging
+import time
+from typing import Dict, List, Optional, Any
+from datetime import datetime, timezone
+import httpx
+import json
+
+from .service_registry import ServiceRegistry
+from .model_service import ModelService, HealthMetrics, ServiceStatus
+
+logger = logging.getLogger(__name__)
+
+class ServiceMonitor:
+    """
+    Service health monitor that automatically checks service health and updates registry
+    
+    Features:
+    - Periodic health checks for all registered services
+    - Automatic service discovery from endpoints
+    - Health metrics collection and storage
+    - Service status updates based on health
+    """
+    
+    def __init__(self, service_registry: ServiceRegistry, check_interval: int = 300):
+        """
+        Initialize the service monitor
+        
+        Args:
+            service_registry: ServiceRegistry instance to monitor
+            check_interval: Health check interval in seconds (default: 5 minutes)
+        """
+        self.service_registry = service_registry
+        self.check_interval = check_interval
+        self._monitoring = False
+        self._monitor_task: Optional[asyncio.Task] = None
+        
+        logger.info(f"ServiceMonitor initialized with {check_interval}s check interval")
+    
+    async def start_monitoring(self):
+        """Start the health monitoring background task"""
+        if self._monitoring:
+            logger.warning("Service monitoring is already running")
+            return
+        
+        self._monitoring = True
+        self._monitor_task = asyncio.create_task(self._monitor_loop())
+        logger.info("Service health monitoring started")
+    
+    async def stop_monitoring(self):
+        """Stop the health monitoring background task"""
+        if not self._monitoring:
+            return
+        
+        self._monitoring = False
+        if self._monitor_task:
+            self._monitor_task.cancel()
+            try:
+                await self._monitor_task
+            except asyncio.CancelledError:
+                pass
+        
+        logger.info("Service health monitoring stopped")
+    
+    async def _monitor_loop(self):
+        """Main monitoring loop that runs health checks periodically"""
+        while self._monitoring:
+            try:
+                await self.check_all_services()
+                await asyncio.sleep(self.check_interval)
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Error in monitoring loop: {e}")
+                await asyncio.sleep(min(self.check_interval, 60))  # Don't wait too long on error
+    
+    async def check_all_services(self) -> Dict[str, bool]:
+        """
+        Check health of all registered services
+        
+        Returns:
+            Dictionary mapping service_id to health status (True = healthy)
+        """
+        # Get all services from registry - this is a simplified implementation
+        # In practice, you'd want a method to get all services from ServiceRegistry
+        results = {}
+        
+        try:
+            # For now, we'll check known service names
+            known_services = ["isa_vision_table", "isa_vision_ui", "isa_vision_doc"]
+            
+            for service_name in known_services:
+                try:
+                    services = await self.service_registry.get_services_by_name(service_name)
+                    for service in services:
+                        health_result = await self.check_service_health(service)
+                        results[service.service_id] = health_result
+                except Exception as e:
+                    logger.error(f"Failed to check services for {service_name}: {e}")
+            
+            logger.info(f"Health check completed for {len(results)} services")
+            return results
+            
+        except Exception as e:
+            logger.error(f"Failed to check all services: {e}")
+            return results
+    
+    async def check_service_health(self, service: ModelService) -> bool:
+        """
+        Check health of a specific service
+        
+        Args:
+            service: ModelService instance to check
+            
+        Returns:
+            True if service is healthy, False otherwise
+        """
+        start_time = time.time()
+        
+        try:
+            # Check if service has a health endpoint
+            health_endpoint = service.health_endpoint or service.get_endpoint_url("health")
+            
+            if not health_endpoint:
+                # No health endpoint, try to ping inference endpoint
+                health_endpoint = service.inference_endpoint
+            
+            if not health_endpoint:
+                logger.warning(f"No endpoint available for service {service.service_id}")
+                return False
+            
+            # Perform health check
+            async with httpx.AsyncClient(timeout=30.0) as client:
+                try:
+                    response = await client.get(health_endpoint)
+                    response_time_ms = int((time.time() - start_time) * 1000)
+                    
+                    # Create health metrics
+                    health_metrics = HealthMetrics(
+                        is_healthy=response.status_code == 200,
+                        response_time_ms=response_time_ms,
+                        status_code=response.status_code,
+                        checked_at=datetime.now(timezone.utc)
+                    )
+                    
+                    # Try to extract additional metrics from response
+                    if response.status_code == 200:
+                        try:
+                            health_data = response.json()
+                            if isinstance(health_data, dict):
+                                # Extract metrics if available
+                                health_metrics.cpu_usage_percent = health_data.get("cpu_usage")
+                                health_metrics.memory_usage_mb = health_data.get("memory_usage_mb")
+                                health_metrics.gpu_usage_percent = health_data.get("gpu_usage")
+                        except json.JSONDecodeError:
+                            pass  # Health endpoint might not return JSON
+                    else:
+                        health_metrics.error_message = f"HTTP {response.status_code}: {response.text[:200]}"
+                    
+                    # Update service health in registry
+                    await self.service_registry.update_service_health(service.service_id, health_metrics)
+                    
+                    logger.debug(f"Health check for {service.service_id}: {health_metrics.is_healthy} ({response_time_ms}ms)")
+                    return health_metrics.is_healthy
+                    
+                except httpx.TimeoutException:
+                    # Service is not responding
+                    health_metrics = HealthMetrics(
+                        is_healthy=False,
+                        response_time_ms=int((time.time() - start_time) * 1000),
+                        error_message="Service timeout",
+                        checked_at=datetime.now(timezone.utc)
+                    )
+                    
+                    await self.service_registry.update_service_health(service.service_id, health_metrics)
+                    logger.warning(f"Service {service.service_id} health check timed out")
+                    return False
+                    
+                except httpx.RequestError as e:
+                    # Network or connection error
+                    health_metrics = HealthMetrics(
+                        is_healthy=False,
+                        response_time_ms=int((time.time() - start_time) * 1000),
+                        error_message=f"Request error: {str(e)}",
+                        checked_at=datetime.now(timezone.utc)
+                    )
+                    
+                    await self.service_registry.update_service_health(service.service_id, health_metrics)
+                    logger.warning(f"Service {service.service_id} health check failed: {e}")
+                    return False
+                    
+        except Exception as e:
+            # Unexpected error
+            health_metrics = HealthMetrics(
+                is_healthy=False,
+                response_time_ms=int((time.time() - start_time) * 1000),
+                error_message=f"Health check error: {str(e)}",
+                checked_at=datetime.now(timezone.utc)
+            )
+            
+            try:
+                await self.service_registry.update_service_health(service.service_id, health_metrics)
+            except Exception as update_error:
+                logger.error(f"Failed to update health metrics: {update_error}")
+            
+            logger.error(f"Unexpected error checking service {service.service_id}: {e}")
+            return False
+    
+    async def discover_services(self) -> List[ModelService]:
+        """
+        Discover services from known endpoints and register them if not already registered
+        
+        Returns:
+            List of discovered ModelService instances
+        """
+        discovered_services = []
+        
+        # Known service endpoints to check for discovery
+        known_endpoints = [
+            {
+                "name": "isa_vision_table",
+                "base_url": "https://qwen-vision-table.modal.run",
+                "service_type": "vision",
+                "capabilities": ["table_detection", "table_structure_recognition"]
+            },
+            {
+                "name": "isa_vision_doc", 
+                "base_url": "https://isa-vision-doc.modal.run",
+                "service_type": "vision",
+                "capabilities": ["table_detection", "ocr", "image_analysis"]
+            },
+            {
+                "name": "isa_vision_ui",
+                "base_url": "https://isa-vision-ui.modal.run", 
+                "service_type": "vision",
+                "capabilities": ["ui_detection", "element_detection"]
+            }
+        ]
+        
+        for endpoint_info in known_endpoints:
+            try:
+                service = await self._discover_service_from_endpoint(endpoint_info)
+                if service:
+                    discovered_services.append(service)
+            except Exception as e:
+                logger.warning(f"Failed to discover service from {endpoint_info['name']}: {e}")
+        
+        logger.info(f"Discovered {len(discovered_services)} services")
+        return discovered_services
+    
+    async def _discover_service_from_endpoint(self, endpoint_info: Dict[str, Any]) -> Optional[ModelService]:
+        """
+        Discover a service from an endpoint by checking its health/info endpoint
+        
+        Args:
+            endpoint_info: Dictionary with service endpoint information
+            
+        Returns:
+            ModelService instance if discovered successfully, None otherwise
+        """
+        try:
+            base_url = endpoint_info["base_url"]
+            health_url = f"{base_url}/health_check"
+            
+            async with httpx.AsyncClient(timeout=10.0) as client:
+                response = await client.get(health_url)
+                
+                if response.status_code == 200:
+                    # Service is responding, check if it's already registered
+                    existing_services = await self.service_registry.get_services_by_name(endpoint_info["name"])
+                    
+                    if existing_services:
+                        logger.debug(f"Service {endpoint_info['name']} already registered")
+                        return existing_services[0]  # Return existing service
+                    
+                    # Service is not registered, create and register it
+                    from .model_service import ServiceType, DeploymentPlatform, ServiceStatus, ResourceRequirements
+                    
+                    service = ModelService(
+                        service_id=f"{endpoint_info['name']}-discovered-{int(time.time())}",
+                        service_name=endpoint_info["name"],
+                        model_id=f"{endpoint_info['name']}-model",
+                        deployment_platform=DeploymentPlatform.MODAL,
+                        service_type=ServiceType.VISION,
+                        status=ServiceStatus.HEALTHY,
+                        inference_endpoint=f"{base_url}/",
+                        health_endpoint=health_url,
+                        capabilities=endpoint_info.get("capabilities", []),
+                        resource_requirements=ResourceRequirements(),
+                        metadata={
+                            "discovered": True,
+                            "discovery_time": datetime.now(timezone.utc).isoformat(),
+                            "base_url": base_url
+                        }
+                    )
+                    
+                    # Register the discovered service
+                    success = await self.service_registry.register_service(service)
+                    
+                    if success:
+                        logger.info(f"Successfully registered discovered service: {endpoint_info['name']}")
+                        return service
+                    else:
+                        logger.warning(f"Failed to register discovered service: {endpoint_info['name']}")
+                        return None
+                        
+                else:
+                    logger.debug(f"Service at {base_url} not responding (HTTP {response.status_code})")
+                    return None
+                    
+        except Exception as e:
+            logger.warning(f"Failed to discover service from {endpoint_info['base_url']}: {e}")
+            return None
+    
+    async def get_service_statistics(self) -> Dict[str, Any]:
+        """
+        Get comprehensive service statistics including health metrics
+        
+        Returns:
+            Dictionary with service statistics
+        """
+        try:
+            # Get basic statistics from registry
+            stats = await self.service_registry.get_service_statistics()
+            
+            # Add monitoring-specific statistics
+            stats.update({
+                "monitoring_enabled": self._monitoring,
+                "check_interval_seconds": self.check_interval,
+                "last_check": datetime.now(timezone.utc).isoformat()
+            })
+            
+            return stats
+            
+        except Exception as e:
+            logger.error(f"Failed to get service statistics: {e}")
+            return {
+                "error": str(e),
+                "monitoring_enabled": self._monitoring,
+                "check_interval_seconds": self.check_interval
+            }
+    
+    def __del__(self):
+        """Cleanup when monitor is destroyed"""
+        if self._monitoring and self._monitor_task:
+            try:
+                self._monitor_task.cancel()
+            except:
+                pass