golem-vm-provider 0.1.0__py3-none-any.whl

provider/api/routes.py

@@ -0,0 +1,159 @@
+import json
+from typing import List
+from pathlib import Path
+from fastapi import APIRouter, HTTPException
+
+from ..config import settings
+from ..utils.logging import setup_logger, PROCESS, SUCCESS
+from ..utils.ascii_art import vm_creation_animation, vm_status_change
+from ..vm.models import VMInfo, VMStatus, VMAccessInfo, VMConfig, VMResources
+from .models import CreateVMRequest
+from ..vm.multipass import MultipassProvider, MultipassError
+from ..discovery.resource_tracker import ResourceTracker
+from ..vm.port_manager import PortManager
+
+logger = setup_logger(__name__)
+router = APIRouter()
+
+# Initialize components
+resource_tracker = ResourceTracker()
+port_manager = PortManager()
+provider = MultipassProvider(resource_tracker, port_manager)
+
+@router.post("/vms", response_model=VMInfo)
+async def create_vm(request: CreateVMRequest) -> VMInfo:
+    """Create a new VM."""
+    try:
+        logger.info(f"📥 Received VM creation request for '{request.name}'")
+        
+        # Determine resources based on request
+        resources = request.resources
+        if resources is None:
+            # This shouldn't happen due to validator, but just in case
+            resources = VMResources(cpu=1, memory=1, storage=10)
+            
+        logger.info(f"📥 Using resources: {resources.cpu} CPU, {resources.memory}GB RAM, {resources.storage}GB storage")
+        
+        # Validate against minimum requirements
+        if resources.cpu < settings.MIN_CPU_CORES:
+            logger.error(f"❌ CPU cores {resources.cpu} below minimum {settings.MIN_CPU_CORES}")
+            raise HTTPException(400, f"Minimum CPU cores required: {settings.MIN_CPU_CORES}")
+        if resources.memory < settings.MIN_MEMORY_GB:
+            logger.error(f"❌ Memory {resources.memory}GB below minimum {settings.MIN_MEMORY_GB}GB")
+            raise HTTPException(400, f"Minimum memory required: {settings.MIN_MEMORY_GB}GB")
+        if resources.storage < settings.MIN_STORAGE_GB:
+            logger.error(f"❌ Storage {resources.storage}GB below minimum {settings.MIN_STORAGE_GB}GB")
+            raise HTTPException(400, f"Minimum storage required: {settings.MIN_STORAGE_GB}GB")
+
+        # Check and allocate resources
+        logger.process("🔄 Allocating resources")
+        if not await resource_tracker.allocate(resources):
+            logger.error("❌ Insufficient resources available")
+            raise HTTPException(400, "Insufficient resources available on provider")
+        
+        try:
+            # Create VM config
+            config = VMConfig(
+                name=request.name,
+                image=request.image or settings.DEFAULT_VM_IMAGE,
+                resources=resources,
+                ssh_key=request.ssh_key
+            )
+            
+            # Create VM
+            logger.process(f"🔄 Creating VM with config: {config}")
+            vm_info = await provider.create_vm(config)
+
+            # Show success message
+            await vm_creation_animation(request.name)
+            return vm_info
+        except Exception as e:
+            # If VM creation fails, deallocate resources
+            logger.warning("⚠️ VM creation failed, deallocating resources")
+            await resource_tracker.deallocate(resources)
+            raise
+        
+    except MultipassError as e:
+        logger.error(f"Failed to create VM: {e}")
+        raise HTTPException(500, str(e))
+
+@router.get("/vms", response_model=List[VMInfo])
+async def list_vms() -> List[VMInfo]:
+    """List all VMs."""
+    try:
+        logger.info("📋 Listing all VMs")
+        vms = []
+        for vm_id in resource_tracker.get_allocated_vms():
+            vm_info = await provider.get_vm_status(vm_id)
+            vms.append(vm_info)
+        return vms
+    except MultipassError as e:
+        logger.error(f"Failed to list VMs: {e}")
+        raise HTTPException(500, str(e))
+
+@router.get("/vms/{requestor_name}", response_model=VMInfo)
+async def get_vm_status(requestor_name: str) -> VMInfo:
+    """Get VM status."""
+    try:
+        logger.info(f"🔍 Getting status for VM '{requestor_name}'")
+        status = await provider.get_vm_status(requestor_name)
+        vm_status_change(requestor_name, status.status.value)
+        return status
+    except MultipassError as e:
+        logger.error(f"Failed to get VM status: {e}")
+        raise HTTPException(500, str(e))
+
+@router.get("/vms/{requestor_name}/access", response_model=VMAccessInfo)
+async def get_vm_access(requestor_name: str) -> VMAccessInfo:
+    """Get VM access information."""
+    try:
+        # Get VM info
+        vm = await provider.get_vm_status(requestor_name)
+        if not vm:
+            raise HTTPException(404, "VM not found")
+        
+        # Get multipass name from mapper
+        multipass_name = await provider.name_mapper.get_multipass_name(requestor_name)
+        if not multipass_name:
+            raise HTTPException(404, "VM mapping not found")
+        
+        # Return access info with both names
+        return VMAccessInfo(
+            ssh_host=settings.PUBLIC_IP or "localhost",
+            ssh_port=vm.ssh_port or 22,
+            vm_id=requestor_name,
+            multipass_name=multipass_name
+        )
+        
+    except MultipassError as e:
+        logger.error(f"Failed to get VM access info: {e}")
+        raise HTTPException(500, str(e))
+
+@router.delete("/vms/{requestor_name}")
+async def delete_vm(requestor_name: str) -> None:
+    """Delete a VM.
+    
+    Args:
+        requestor_name: Name of the VM as provided by requestor
+    """
+    try:
+        logger.process(f"🗑️  Deleting VM '{requestor_name}'")
+        
+        # Get multipass name from mapper
+        multipass_name = await provider.name_mapper.get_multipass_name(requestor_name)
+        if not multipass_name:
+            logger.warning(f"No multipass name found for VM '{requestor_name}' (may have been already deleted)")
+            return
+            
+        try:
+            vm_status_change(requestor_name, "STOPPING", "Cleanup in progress")
+            await provider.delete_vm(requestor_name)
+            vm_status_change(requestor_name, "TERMINATED", "Cleanup complete")
+            logger.success(f"✨ Successfully deleted VM '{requestor_name}'")
+        except MultipassError as e:
+            logger.error(f"Failed to delete VM: {e}")
+            raise HTTPException(500, str(e))
+            
+    except Exception as e:
+        logger.error(f"Failed to delete VM: {e}")
+        raise HTTPException(500, str(e))

provider/config.py

@@ -0,0 +1,160 @@
+import os
+from pathlib import Path
+from typing import Optional
+import uuid
+
+from pydantic import BaseSettings, validator
+
+
+class Settings(BaseSettings):
+    """Provider configuration settings."""
+
+    # API Settings
+    DEBUG: bool = True
+    HOST: str = "0.0.0.0"
+    PORT: int = 7466
+
+    # Provider Settings
+    PROVIDER_ID: str = ""  # Will be set from Ethereum identity
+    PROVIDER_NAME: str = "golem-provider"
+    PROVIDER_COUNTRY: str = "SE"
+    ETHEREUM_KEY_DIR: str = ""
+
+    @validator("ETHEREUM_KEY_DIR", pre=True)
+    def resolve_key_dir(cls, v: str) -> str:
+        """Resolve Ethereum key directory path."""
+        if not v:
+            return str(Path.home() / ".golem" / "provider" / "keys")
+        path = Path(v)
+        if not path.is_absolute():
+            path = Path.home() / path
+        return str(path)
+
+    @validator("PROVIDER_ID", always=True)
+    def get_or_create_provider_id(cls, v: str, values: dict) -> str:
+        """Get or create provider ID from Ethereum identity."""
+        from provider.security.ethereum import EthereumIdentity
+
+        # If ID provided in env, use it
+        if v:
+            return v
+
+        # Get ID from Ethereum identity
+        key_dir = values.get("ETHEREUM_KEY_DIR")
+        identity = EthereumIdentity(key_dir)
+        return identity.get_or_create_identity()
+
+    # Discovery Service Settings
+    DISCOVERY_URL: str = "http://localhost:7465"
+    ADVERTISEMENT_INTERVAL: int = 240  # seconds
+
+    # VM Settings
+    MAX_VMS: int = 10
+    DEFAULT_VM_IMAGE: str = "ubuntu:24.04"
+    VM_DATA_DIR: str = ""
+    SSH_KEY_DIR: str = ""
+
+    @validator("VM_DATA_DIR", pre=True)
+    def resolve_vm_data_dir(cls, v: str) -> str:
+        """Resolve VM data directory path."""
+        if not v:
+            return str(Path.home() / ".golem" / "provider" / "vms")
+        path = Path(v)
+        if not path.is_absolute():
+            path = Path.home() / path
+        return str(path)
+
+    @validator("SSH_KEY_DIR", pre=True)
+    def resolve_ssh_key_dir(cls, v: str) -> str:
+        """Resolve SSH key directory path."""
+        if not v:
+            return str(Path.home() / ".golem" / "provider" / "ssh")
+        path = Path(v)
+        if not path.is_absolute():
+            path = Path.home() / path
+        return str(path)
+
+    # Resource Settings
+    MIN_MEMORY_GB: int = 1
+    MIN_STORAGE_GB: int = 10
+    MIN_CPU_CORES: int = 1
+
+    # Resource Thresholds (%)
+    CPU_THRESHOLD: int = 90
+    MEMORY_THRESHOLD: int = 85
+    STORAGE_THRESHOLD: int = 90
+
+    # Rate Limiting
+    RATE_LIMIT_PER_MINUTE: int = 100
+
+    # Multipass Settings
+    MULTIPASS_BINARY_PATH: str = ""
+
+    @validator("MULTIPASS_BINARY_PATH", pre=True)
+    def detect_multipass_path(cls, v: str) -> str:
+        """Detect and validate Multipass binary path."""
+        if v:
+            path = v
+        else:
+            # Common Multipass binary locations
+            binary_name = "multipass"
+            search_paths = [
+                "/usr/local/bin",          # Common Unix/Linux
+                "/usr/bin",                # Linux
+                "/opt/homebrew/bin",       # macOS M1 (Homebrew)
+                "/snap/bin",               # Linux (Snap)
+            ]
+
+            # Search for multipass binary
+            for directory in search_paths:
+                path = os.path.join(directory, binary_name)
+                if os.path.isfile(path) and os.access(path, os.X_OK):
+                    return path
+
+            raise ValueError(
+                "Multipass binary not found. Please install Multipass or set "
+                "GOLEM_PROVIDER_MULTIPASS_BINARY_PATH to your Multipass binary path."
+            )
+
+        # Validate the path
+        if not os.path.isfile(path):
+            raise ValueError(f"Multipass binary not found at: {path}")
+        if not os.access(path, os.X_OK):
+            raise ValueError(f"Multipass binary at {path} is not executable")
+        return path
+
+    # Proxy Settings
+    PORT_RANGE_START: int = 50800
+    PORT_RANGE_END: int = 50900
+    PROXY_STATE_DIR: str = ""
+    PUBLIC_IP: Optional[str] = None
+
+    @validator("PROXY_STATE_DIR", pre=True)
+    def resolve_proxy_state_dir(cls, v: str) -> str:
+        """Resolve proxy state directory path."""
+        if not v:
+            return str(Path.home() / ".golem" / "provider" / "proxy")
+        path = Path(v)
+        if not path.is_absolute():
+            path = Path.home() / path
+        return str(path)
+
+    @validator("PUBLIC_IP", pre=True)
+    def get_public_ip(cls, v: Optional[str]) -> Optional[str]:
+        """Get public IP if set to 'auto'."""
+        if v == "auto":
+            try:
+                import requests
+                response = requests.get("https://api.ipify.org")
+                return response.text.strip()
+            except Exception:
+                return None
+        return v
+
+    class Config:
+        env_prefix = "GOLEM_PROVIDER_"
+        case_sensitive = True
+
+
+# Global settings instance
+settings = Settings()

provider/discovery/__init__.py

@@ -0,0 +1,6 @@
+from .advertiser import ResourceMonitor, ResourceAdvertiser
+
+__all__ = [
+    "ResourceMonitor",
+    "ResourceAdvertiser"
+]

provider/discovery/advertiser.py

@@ -0,0 +1,179 @@
+import aiohttp
+import asyncio
+import logging
+import psutil
+from datetime import datetime
+from typing import Dict, Optional
+
+from ..config import settings
+from ..utils.retry import async_retry
+
+logger = logging.getLogger(__name__)
+
+class ResourceMonitor:
+    """Monitor system resources."""
+    
+    @staticmethod
+    def get_cpu_count() -> int:
+        """Get number of CPU cores."""
+        return psutil.cpu_count()
+
+    @staticmethod
+    def get_memory_gb() -> int:
+        """Get available memory in GB."""
+        return psutil.virtual_memory().available // (1024 ** 3)
+
+    @staticmethod
+    def get_storage_gb() -> int:
+        """Get available storage in GB."""
+        return psutil.disk_usage("/").free // (1024 ** 3)
+
+    @staticmethod
+    def get_cpu_percent() -> float:
+        """Get CPU usage percentage."""
+        return psutil.cpu_percent(interval=1)
+
+    @staticmethod
+    def get_memory_percent() -> float:
+        """Get memory usage percentage."""
+        return psutil.virtual_memory().percent
+
+    @staticmethod
+    def get_storage_percent() -> float:
+        """Get storage usage percentage."""
+        return psutil.disk_usage("/").percent
+
+class ResourceAdvertiser:
+    """Advertise available resources to discovery service."""
+    
+    def __init__(
+        self,
+        resource_tracker: 'ResourceTracker',
+        discovery_url: Optional[str] = None,
+        provider_id: Optional[str] = None,
+        update_interval: Optional[int] = None
+    ):
+        self.resource_tracker = resource_tracker
+        self.discovery_url = discovery_url or settings.DISCOVERY_URL
+        self.provider_id = provider_id or settings.PROVIDER_ID
+        self.update_interval = update_interval or settings.ADVERTISEMENT_INTERVAL
+        self.session: Optional[aiohttp.ClientSession] = None
+        self._stop_event = asyncio.Event()
+
+    async def start(self):
+        """Start advertising resources."""
+        self.session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10))
+        # Register for resource updates
+        self.resource_tracker.on_update(self._post_advertisement)
+        
+        # Test discovery service connection with retries
+        try:
+            await self._check_discovery_health()
+        except Exception as e:
+            logger.warning(f"Could not connect to discovery service after retries, continuing without advertising: {e}")
+            return
+            
+        try:
+            while not self._stop_event.is_set():
+                try:
+                    await self._post_advertisement()
+                except aiohttp.ClientError as e:
+                    logger.error(f"Network error posting advertisement: {e}")
+                    await asyncio.sleep(min(60, self.update_interval))
+                except Exception as e:
+                    logger.error(f"Failed to post advertisement: {e}")
+                    await asyncio.sleep(min(60, self.update_interval))
+                else:
+                    await asyncio.sleep(self.update_interval)
+        finally:
+            await self.stop()
+
+    async def stop(self):
+        """Stop advertising resources."""
+        self._stop_event.set()
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    @async_retry(retries=5, delay=1.0, backoff=2.0, exceptions=(aiohttp.ClientError, asyncio.TimeoutError))
+    async def _check_discovery_health(self):
+        """Check discovery service health with retries."""
+        if not self.session:
+            raise RuntimeError("Session not initialized")
+            
+        async with self.session.get(f"{self.discovery_url}/health") as response:
+            if not response.ok:
+                raise Exception(f"Discovery service health check failed: {response.status}")
+
+    @async_retry(retries=3, delay=1.0, backoff=2.0, exceptions=(aiohttp.ClientError, asyncio.TimeoutError))
+    async def _post_advertisement(self):
+        """Post resource advertisement to discovery service."""
+        if not self.session:
+            raise RuntimeError("Session not initialized")
+
+        resources = self.resource_tracker.get_available_resources()
+        
+        # Don't advertise if resources are too low
+        if not self.resource_tracker._meets_minimum_requirements(resources):
+            logger.warning("Resources too low, skipping advertisement")
+            return
+
+        # Get public IP with retries
+        try:
+            ip_address = await self._get_public_ip()
+        except Exception as e:
+            logger.error(f"Could not get public IP after retries: {e}")
+            return
+
+        try:
+            async with self.session.post(
+                f"{self.discovery_url}/api/v1/advertisements",
+                headers={
+                    "X-Provider-ID": self.provider_id,
+                    "X-Provider-Signature": "signature",  # TODO: Implement signing
+                    "Content-Type": "application/json"
+                },
+                json={
+                    "ip_address": ip_address,
+                    "country": settings.PROVIDER_COUNTRY,
+                    "resources": resources
+                },
+                timeout=aiohttp.ClientTimeout(total=5)  # 5 second timeout for advertisement
+            ) as response:
+                if not response.ok:
+                    error_text = await response.text()
+                    raise Exception(
+                        f"Failed to post advertisement: {response.status} - {error_text}"
+                    )
+                logger.info(
+                    f"Posted advertisement with resources: CPU={resources['cpu']}, "
+                    f"Memory={resources['memory']}GB, Storage={resources['storage']}GB"
+                )
+        except asyncio.TimeoutError:
+            logger.error("Advertisement request timed out")
+            raise
+
+    @async_retry(retries=3, delay=1.0, backoff=2.0, exceptions=(aiohttp.ClientError, asyncio.TimeoutError))
+    async def _get_public_ip(self) -> str:
+        """Get public IP address with retries."""
+        if not self.session:
+            raise RuntimeError("Session not initialized")
+
+        # Try multiple IP services in case one fails
+        services = [
+            "https://api.ipify.org",
+            "https://ifconfig.me/ip",
+            "https://api.my-ip.io/ip"
+        ]
+
+        errors = []
+        for service in services:
+            try:
+                async with self.session.get(service) as response:
+                    if response.ok:
+                        return (await response.text()).strip()
+            except Exception as e:
+                errors.append(f"{service}: {str(e)}")
+                continue
+
+        raise Exception(f"Failed to get public IP address from all services: {'; '.join(errors)}")

provider/discovery/resource_tracker.py

@@ -0,0 +1,152 @@
+import asyncio
+import logging
+from typing import Dict, List, Callable, Optional
+from ..vm.models import VMResources
+from ..config import settings
+
+logger = logging.getLogger(__name__)
+
+class ResourceTracker:
+    """Track and manage provider resources."""
+
+    def __init__(self):
+        """Initialize resource tracker."""
+        from .advertiser import ResourceMonitor
+        self.total_resources = {
+            "cpu": ResourceMonitor.get_cpu_count(),
+            "memory": ResourceMonitor.get_memory_gb(),
+            "storage": ResourceMonitor.get_storage_gb()
+        }
+        self.allocated_resources = {
+            "cpu": 0,
+            "memory": 0,
+            "storage": 0
+        }
+        self._lock = asyncio.Lock()
+        self._update_callbacks: List[Callable] = []
+        self._allocated_vms: Dict[str, VMResources] = {}
+
+    def _can_allocate(self, resources: VMResources) -> bool:
+        """Check if resources can be allocated."""
+        available = self.get_available_resources()
+        return (
+            resources.cpu <= available["cpu"] and
+            resources.memory <= available["memory"] and
+            resources.storage <= available["storage"]
+        )
+
+    def _meets_minimum_requirements(self, resources: Dict[str, int]) -> bool:
+        """Check if available resources meet minimum requirements."""
+        return (
+            resources["cpu"] >= settings.MIN_CPU_CORES and
+            resources["memory"] >= settings.MIN_MEMORY_GB and
+            resources["storage"] >= settings.MIN_STORAGE_GB
+        )
+
+    async def allocate(self, resources: VMResources, vm_id: Optional[str] = None) -> bool:
+        """Allocate resources for a VM."""
+        async with self._lock:
+            if not self._can_allocate(resources):
+                return False
+            
+            self.allocated_resources["cpu"] += resources.cpu
+            self.allocated_resources["memory"] += resources.memory
+            self.allocated_resources["storage"] += resources.storage
+            
+            if vm_id:
+                self._allocated_vms[vm_id] = resources
+            
+            logger.info(
+                f"Allocated resources: CPU={resources.cpu}, "
+                f"Memory={resources.memory}GB, Storage={resources.storage}GB"
+            )
+            
+            await self._notify_update()
+            return True
+
+    async def deallocate(self, resources: VMResources, vm_id: Optional[str] = None) -> None:
+        """Deallocate resources from a VM."""
+        async with self._lock:
+            self.allocated_resources["cpu"] = max(
+                0, self.allocated_resources["cpu"] - resources.cpu
+            )
+            self.allocated_resources["memory"] = max(
+                0, self.allocated_resources["memory"] - resources.memory
+            )
+            self.allocated_resources["storage"] = max(
+                0, self.allocated_resources["storage"] - resources.storage
+            )
+            
+            if vm_id and vm_id in self._allocated_vms:
+                del self._allocated_vms[vm_id]
+            
+            logger.info(
+                f"Deallocated resources: CPU={resources.cpu}, "
+                f"Memory={resources.memory}GB, Storage={resources.storage}GB"
+            )
+            
+            await self._notify_update()
+
+    def get_allocated_vms(self) -> List[str]:
+        """Get list of allocated VM IDs."""
+        return list(self._allocated_vms.keys())
+
+    def get_available_resources(self) -> Dict[str, int]:
+        """Get currently available resources."""
+        return {
+            "cpu": max(0, self.total_resources["cpu"] - self.allocated_resources["cpu"]),
+            "memory": max(0, self.total_resources["memory"] - self.allocated_resources["memory"]),
+            "storage": max(0, self.total_resources["storage"] - self.allocated_resources["storage"])
+        }
+
+    def can_accept_resources(self, resources: VMResources) -> bool:
+        """Check if resources can be accepted."""
+        available = self.get_available_resources()
+        return (
+            resources.cpu <= available["cpu"] and
+            resources.memory <= available["memory"] and
+            resources.storage <= available["storage"] and
+            self._meets_minimum_requirements(available)
+        )
+
+    def on_update(self, callback: Callable) -> None:
+        """Register callback for resource updates."""
+        self._update_callbacks.append(callback)
+
+    async def _notify_update(self) -> None:
+        """Notify all registered callbacks of resource update."""
+        for callback in self._update_callbacks:
+            try:
+                await callback()
+            except Exception as e:
+                logger.error(f"Error in resource update callback: {e}")
+
+    async def sync_with_multipass(self, vm_resources: Dict[str, VMResources]) -> None:
+        """Sync resource tracker state with actual multipass VM states.
+        
+        Args:
+            vm_resources: Dictionary mapping VM names to their resources
+        """
+        async with self._lock:
+            # Reset allocated resources
+            self.allocated_resources = {
+                "cpu": 0,
+                "memory": 0,
+                "storage": 0
+            }
+            self._allocated_vms.clear()
+            
+            # Add resources for each running VM
+            for vm_name, resources in vm_resources.items():
+                self.allocated_resources["cpu"] += resources.cpu
+                self.allocated_resources["memory"] += resources.memory
+                self.allocated_resources["storage"] += resources.storage
+                self._allocated_vms[vm_name] = resources
+                
+            logger.info(
+                f"Synced allocated resources: CPU={self.allocated_resources['cpu']}, "
+                f"Memory={self.allocated_resources['memory']}GB, "
+                f"Storage={self.allocated_resources['storage']}GB"
+            )
+            
+            await self._notify_update()