golem-vm-provider 0.1.26__py3-none-any.whl → 0.1.28__py3-none-any.whl

provider/vm/multipass_adapter.py

@@ -0,0 +1,221 @@
+import json
+import uuid
+import subprocess
+from pathlib import Path
+import asyncio
+from typing import Dict, List, Optional
+from ..utils.retry import async_retry_unless_not_found
+
+from ..config import settings
+from ..utils.logging import setup_logger
+from .models import VMConfig, VMInfo, VMResources, VMStatus, VMError, VMNotFoundError
+from .provider import VMProvider
+
+logger = setup_logger(__name__)
+
+
+class MultipassError(VMError):
+    """Raised when multipass operations fail."""
+    pass
+
+
+class MultipassAdapter(VMProvider):
+    """Manages VMs using Multipass."""
+
+    def __init__(self, proxy_manager, name_mapper):
+        self.multipass_path = settings.MULTIPASS_BINARY_PATH
+        self.proxy_manager = proxy_manager
+        self.name_mapper = name_mapper
+
+    async def _run_multipass(self, args: List[str], check: bool = True) -> subprocess.CompletedProcess:
+        """Run a multipass command."""
+        # Commands that produce JSON or version info that we need to parse.
+        commands_to_capture = ['info', 'version']
+        should_capture = args[0] in commands_to_capture
+
+        # We add a timeout to the launch command to prevent it from hanging indefinitely
+        # e.g. during image download. 300 seconds = 5 minutes.
+        timeout = 300 if args[0] == 'launch' else None
+
+        try:
+            return await asyncio.to_thread(
+                subprocess.run,
+                [self.multipass_path, *args],
+                capture_output=should_capture,
+                text=True,
+                check=check,
+                timeout=timeout
+            )
+        except subprocess.CalledProcessError as e:
+            stderr = e.stderr if should_capture and e.stderr else "No stderr captured. See provider logs for command output."
+            raise MultipassError(f"Multipass command failed: {stderr}")
+        except subprocess.TimeoutExpired as e:
+            stderr = e.stderr if should_capture and e.stderr else "No stderr captured. See provider logs for command output."
+            raise MultipassError(f"Multipass command '{' '.join(args)}' timed out after {timeout} seconds. Stderr: {stderr}")
+
+    @async_retry_unless_not_found(retries=5, delay=2.0)
+    async def _get_vm_info(self, vm_id: str) -> Dict:
+        """Get detailed information about a VM."""
+        try:
+            result = await self._run_multipass(["info", vm_id, "--format", "json"])
+            logger.info(f"Raw multipass info for {vm_id}: {result.stdout}")
+            info = json.loads(result.stdout)
+            vm_info = info["info"][vm_id]
+            essential_fields = ["state", "ipv4", "cpu_count", "memory", "disks"]
+            if not all(field in vm_info for field in essential_fields):
+                raise KeyError(f"Essential fields missing from VM info. Got: {list(vm_info.keys())}")
+            return vm_info
+        except MultipassError as e:
+            if "does not exist" in str(e):
+                raise VMNotFoundError(f"VM {vm_id} not found in multipass") from e
+            raise
+        except (json.JSONDecodeError, KeyError) as e:
+            raise MultipassError(f"Failed to parse VM info or essential fields are missing: {e}")
+
+    async def initialize(self) -> None:
+        """Initialize the VM provider."""
+        try:
+            result = await self._run_multipass(["version"])
+            logger.info(f"🔧 Using Multipass version: {result.stdout.strip()}")
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
+            raise MultipassError(f"Failed to verify multipass installation: {e}")
+
+    async def create_vm(self, config: VMConfig) -> VMInfo:
+        """Create a new VM."""
+        multipass_name = f"golem-{uuid.uuid4()}"
+        await self.name_mapper.add_mapping(config.name, multipass_name)
+
+        launch_cmd = [
+            "launch",
+            config.image,
+            "--name", multipass_name,
+            "--cloud-init", config.cloud_init_path,
+            "--cpus", str(config.resources.cpu),
+            "--memory", f"{config.resources.memory}G",
+            "--disk", f"{config.resources.storage}G"
+        ]
+        try:
+            logger.info(f"Running multipass command: {' '.join(launch_cmd)}")
+            await self._run_multipass(launch_cmd)
+            logger.info(f"VM {multipass_name} launched, waiting for it to be ready...")
+
+            ip_address = None
+            max_retries = 15
+            retry_delay = 5  # seconds
+            for attempt in range(max_retries):
+                try:
+                    info = await self._get_vm_info(multipass_name)
+                    if info.get("state", "").lower() == "running" and info.get("ipv4"):
+                        ip_address = info["ipv4"][0]
+                        break
+                    logger.debug(f"VM {config.name} status is {info.get('state')}, waiting...")
+                except (MultipassError, VMNotFoundError):
+                    logger.debug(f"VM {config.name} not found yet, retrying in {retry_delay}s...")
+                
+                await asyncio.sleep(retry_delay)
+            
+            if not ip_address:
+                raise MultipassError(f"VM {config.name} did not become ready or get an IP in time.")
+
+            # Configure proxy to allocate a port
+            if not await self.proxy_manager.add_vm(multipass_name, ip_address):
+                raise MultipassError(f"Failed to configure proxy for VM {multipass_name}")
+
+            # Now get the full status, which will include the allocated port
+            vm_info = await self.get_vm_status(multipass_name)
+            logger.info(f"Successfully created VM: {vm_info.dict()}")
+            return vm_info
+
+        except Exception as e:
+            logger.error(f"VM creation for {config.name} failed. Cleaning up.", exc_info=True)
+            await self._run_multipass(["delete", multipass_name, "--purge"], check=False)
+            await self.proxy_manager.remove_vm(multipass_name)
+            await self.name_mapper.remove_mapping(config.name)
+            raise MultipassError(f"Failed to create VM {config.name}: {e}") from e
+
+    async def delete_vm(self, multipass_name: str) -> None:
+        """Delete a VM."""
+        requestor_name = await self.name_mapper.get_requestor_name(multipass_name)
+        if not requestor_name:
+            logger.warning(f"No mapping found for {multipass_name}, cannot remove mapping.")
+        else:
+            await self.name_mapper.remove_mapping(requestor_name)
+        await self._run_multipass(["delete", multipass_name, "--purge"], check=False)
+
+    async def list_vms(self) -> List[VMInfo]:
+        """List all VMs."""
+        all_mappings = self.name_mapper.list_mappings()
+        vms = []
+        for requestor_name in all_mappings.keys():
+            try:
+                vm_info = await self.get_vm_status(requestor_name)
+                vms.append(vm_info)
+            except VMNotFoundError:
+                logger.warning(f"VM {requestor_name} not found, but a mapping exists. It may have been deleted externally.")
+        return vms
+
+    async def start_vm(self, multipass_name: str) -> VMInfo:
+        """Start a VM."""
+        await self._run_multipass(["start", multipass_name])
+        return await self.get_vm_status(multipass_name)
+
+    async def stop_vm(self, multipass_name: str) -> VMInfo:
+        """Stop a VM."""
+        await self._run_multipass(["stop", multipass_name])
+        return await self.get_vm_status(multipass_name)
+
+    async def get_vm_status(self, multipass_name: str) -> VMInfo:
+        """Get the status of a VM."""
+        try:
+            info = await self._get_vm_info(multipass_name)
+        except MultipassError:
+            raise VMNotFoundError(f"VM {multipass_name} not found in multipass")
+
+        requestor_name = await self.name_mapper.get_requestor_name(multipass_name)
+        if not requestor_name:
+            raise VMNotFoundError(f"Mapping for VM {multipass_name} not found")
+
+        ipv4 = info.get("ipv4")
+        ip_address = ipv4[0] if ipv4 else None
+        logger.debug(f"Parsed VM info for {requestor_name}: {info}")
+        
+        disks_info = info.get("disks", {})
+        total_storage = sum(int(disk.get("total", 0)) for disk in disks_info.values())
+        vm_info_obj = VMInfo(
+            id=requestor_name,
+            name=requestor_name,
+            status=VMStatus(info["state"].lower()),
+            resources=VMResources(
+                cpu=int(info.get("cpu_count", "1")),
+                memory=round(info.get("memory", {}).get("total", 1024**3) / (1024**3)),
+                storage=round(total_storage / (1024**3)) if total_storage > 0 else 10
+            ),
+            ip_address=ip_address,
+            ssh_port=self.proxy_manager.get_port(multipass_name)
+        )
+        logger.debug(f"Constructed VMInfo object: {vm_info_obj.dict()}")
+        return vm_info_obj
+
+    async def get_all_vms_resources(self) -> Dict[str, VMResources]:
+        """Get resources for all running VMs."""
+        all_mappings = self.name_mapper.list_mappings()
+        vm_resources = {}
+        for requestor_name, multipass_name in all_mappings.items():
+            try:
+                info = await self._get_vm_info(multipass_name)
+                disks_info = info.get("disks", {})
+                total_storage = sum(int(disk.get("total", 0)) for disk in disks_info.values())
+                vm_resources[requestor_name] = VMResources(
+                    cpu=int(info.get("cpu_count", "1")),
+                    memory=round(info.get("memory", {}).get("total", 1024**3) / (1024**3)),
+                    storage=round(total_storage / (1024**3)) if total_storage > 0 else 10
+                )
+            except (MultipassError, VMNotFoundError):
+                logger.warning(f"Could not retrieve resources for VM {requestor_name} ({multipass_name}). It may have been deleted.")
+            except Exception as e:
+                logger.error(f"Failed to get info for VM {requestor_name}: {e}")
+        return vm_resources
+
+    async def cleanup(self) -> None:
+        """Cleanup resources used by the provider."""
+        pass

provider/vm/name_mapper.py

@@ -9,21 +9,21 @@ logger = logging.getLogger(__name__)
 class VMNameMapper:
     """Maps between requestor VM names and multipass VM names."""
 
-    def __init__(self, storage_path: Optional[Path] = None):
+    def __init__(self, db_path: Optional[Path] = None):
         """Initialize name mapper.
         
         Args:
-            storage_path: Optional path to persist mappings
+            db_path: Optional path to persist mappings
         """
         self._name_map: Dict[str, str] = {}  # requestor_name -> multipass_name
         self._reverse_map: Dict[str, str] = {}  # multipass_name -> requestor_name
         self._lock = asyncio.Lock()
-        self._storage_path = storage_path
+        self._storage_path = db_path
         
         # Load existing mappings if storage path provided
-        if storage_path and storage_path.exists():
+        if db_path and db_path.exists():
             try:
-                with open(storage_path) as f:
+                with open(db_path) as f:
                     data = json.load(f)
                     self._name_map = data.get('name_map', {})
                     self._reverse_map = data.get('reverse_map', {})

provider/vm/port_manager.py

@@ -47,10 +47,13 @@ class PortManager:
         self._existing_ports = existing_ports or set()
 
         # Initialize port verifier with default servers
-        self.port_check_servers = port_check_servers or [
-            "http://localhost:9000",  # Local development server
-            "http://195.201.39.101:9000",  # Production servers
-        ]
+        if settings.DEV_MODE:
+            self.port_check_servers = ["http://localhost:9000"]
+        else:
+            self.port_check_servers = port_check_servers or [
+                "http://localhost:9000",  # Local development server
+                "http://195.201.39.101:9000",  # Production servers
+            ]
         self.discovery_port = discovery_port or settings.PORT
         self.skip_verification = skip_verification
         self.port_verifier = PortVerifier(
@@ -105,7 +108,21 @@ class PortManager:
 
             # Clear existing verified ports before verification
             self.verified_ports.clear()
-            results = await self.port_verifier.verify_ports(ssh_ports)
+            results = {}
+            if not self.skip_verification:
+                try:
+                    results = await self.port_verifier.verify_ports(ssh_ports)
+                except RuntimeError as e:
+                    logger.error(f"Port verification failed: {e}")
+                    display.print_summary(
+                        PortVerificationResult(
+                            port=self.discovery_port,
+                            accessible=False,
+                            error=str(e)
+                        ),
+                        {}
+                    )
+                    return False
 
         # Add provider port as verified since we already checked it
         results[self.discovery_port] = PortVerificationResult(
@@ -237,7 +254,8 @@ class PortManager:
             used_ports = self._get_used_ports()
 
             # Find first available verified port
-            ports_to_check = sorted(self.verified_ports) if not self.skip_verification else range(self.start_port, self.end_port)
+            ports_to_check = sorted(list(self.verified_ports)) if not self.skip_verification else range(
+                self.start_port, self.end_port)
             for port in ports_to_check:
                 if port not in used_ports:
                     # Quick check if port is actually available

provider/vm/provider.py

@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+from typing import Dict, List
+
+from .models import VMConfig, VMInfo, VMResources
+
+
+class VMProvider(ABC):
+    """Abstract base class for VM providers."""
+
+    @abstractmethod
+    async def initialize(self) -> None:
+        """Initialize the VM provider."""
+        pass
+
+    @abstractmethod
+    async def create_vm(self, config: VMConfig) -> VMInfo:
+        """Create a new VM."""
+        pass
+
+    @abstractmethod
+    async def delete_vm(self, vm_id: str) -> None:
+        """Delete a VM."""
+        pass
+
+    @abstractmethod
+    async def start_vm(self, vm_id: str) -> VMInfo:
+        """Start a VM."""
+        pass
+
+    @abstractmethod
+    async def stop_vm(self, vm_id: str) -> VMInfo:
+        """Stop a VM."""
+        pass
+
+    @abstractmethod
+    async def get_vm_status(self, vm_id: str) -> VMInfo:
+        """Get the status of a VM."""
+        pass
+
+    @abstractmethod
+    def get_all_vms_resources(self) -> Dict[str, VMResources]:
+        """Get resources for all running VMs."""
+        pass
+
+    @abstractmethod
+    async def cleanup(self) -> None:
+        """Cleanup resources used by the provider."""
+        pass

provider/vm/proxy_manager.py

@@ -308,7 +308,7 @@ class PythonProxyManager:
         """
         try:
             # Use provided port or allocate one
-            if port is None or port == 0:
+            if port is None:
                 allocated_port = self.port_manager.allocate_port(vm_id)
                 if allocated_port is None:
                     logger.error(f"Failed to allocate port for VM {vm_id}")

provider/vm/service.py

@@ -0,0 +1,91 @@
+from datetime import datetime
+from typing import Dict, List
+
+from ..discovery.resource_tracker import ResourceTracker
+from ..utils.logging import setup_logger
+from .models import VMConfig, VMInfo, VMResources, VMNotFoundError
+from .provider import VMProvider
+from .name_mapper import VMNameMapper
+from .cloud_init import generate_cloud_init, cleanup_cloud_init
+
+logger = setup_logger(__name__)
+
+
+class VMService:
+    """Service for managing the lifecycle of VMs."""
+
+    def __init__(
+        self,
+        provider: VMProvider,
+        resource_tracker: ResourceTracker,
+        name_mapper: VMNameMapper,
+    ):
+        self.provider = provider
+        self.resource_tracker = resource_tracker
+        self.name_mapper = name_mapper
+
+    async def create_vm(self, config: VMConfig) -> VMInfo:
+        """Create a new VM."""
+        if not await self.resource_tracker.allocate(config.resources, config.name):
+            raise ValueError("Insufficient resources available on provider")
+
+        multipass_name = f"golem-{config.name}-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
+        await self.name_mapper.add_mapping(config.name, multipass_name)
+
+        cloud_init_path, config_id = generate_cloud_init(
+            hostname=config.name,
+            ssh_key=config.ssh_key
+        )
+        config.cloud_init_path = cloud_init_path
+
+        try:
+            vm_info = await self.provider.create_vm(config)
+            return vm_info
+        except Exception as e:
+            logger.error(f"Failed to create VM, deallocating resources", exc_info=True)
+            await self.resource_tracker.deallocate(config.resources, config.name)
+            raise
+        finally:
+            cleanup_cloud_init(cloud_init_path, config_id)
+
+    async def delete_vm(self, vm_id: str) -> None:
+        """Delete a VM."""
+        multipass_name = await self.name_mapper.get_multipass_name(vm_id)
+        if not multipass_name:
+            logger.warning(f"No multipass name found for VM {vm_id}")
+            return
+
+        try:
+            vm_info = await self.provider.get_vm_status(multipass_name)
+            await self.provider.delete_vm(multipass_name)
+            await self.resource_tracker.deallocate(vm_info.resources, vm_id)
+        except VMNotFoundError:
+            logger.warning(f"VM {multipass_name} not found on provider, cleaning up resources")
+            # If the VM is not found, we still need to deallocate the resources we have tracked for it
+            # Since we can't get the resources from the provider, we'll have to assume the resources are what we have tracked
+            # This is not ideal, but it's the best we can do in this situation
+            # A better solution would be to store the resources in the name mapper
+            pass
+        finally:
+            await self.name_mapper.remove_mapping(vm_id)
+
+    async def list_vms(self) -> List[VMInfo]:
+        """List all VMs."""
+        return await self.provider.list_vms()
+
+    async def get_vm_status(self, vm_id: str) -> VMInfo:
+        """Get the status of a VM."""
+        multipass_name = await self.name_mapper.get_multipass_name(vm_id)
+        if not multipass_name:
+            from .models import VMNotFoundError
+            raise VMNotFoundError(f"VM {vm_id} not found")
+        return await self.provider.get_vm_status(multipass_name)
+
+    async def get_all_vms_resources(self) -> Dict[str, VMResources]:
+        """Get resources for all running VMs."""
+        return await self.provider.get_all_vms_resources()
+    async def initialize(self):
+        await self.provider.initialize()
+
+    async def shutdown(self):
+        await self.provider.cleanup()

golem_vm_provider-0.1.26.dist-info/RECORD

@@ -1,30 +0,0 @@
-provider/__init__.py,sha256=HO1fkPpZqPO3z8O8-eVIyx8xXSMIVuTR_b1YF0RtXOg,45
-provider/api/__init__.py,sha256=ssX1ugDqEPt8Fn04IymgmG-Ev8PiXLsCSaiZVvHQnec,344
-provider/api/models.py,sha256=9799o6ZnBfG2LZCoqm2-6ONNNgqKB7bn5oIs_Ocq24s,3519
-provider/api/routes.py,sha256=P27RQvNqFWn6PacRwr1PaVz-yv5KAWsp9KeORejkXSI,6452
-provider/config.py,sha256=pFW0Qwej9eTZfx1BXTExhsXwul-drqn0Lu8wPhx399Y,16908
-provider/discovery/__init__.py,sha256=VR3NRoQtZRH5Vs8FG7jnGLR7p7wn7XeZdLaBb3t8e1g,123
-provider/discovery/advertiser.py,sha256=yv7RbRf1K43qOLAEa2Olj9hhN8etl2qsBuoHok0xoVs,6784
-provider/discovery/golem_base_advertiser.py,sha256=UpSJyO6wyujBJ2xDrb6lhKC7zYv1vZg0db4t0k--5dk,5830
-provider/discovery/golem_base_utils.py,sha256=xk7vznhMgzrn0AuGyk6-9N9ukp9oPdBbbk1RI-sVjp0,607
-provider/discovery/resource_tracker.py,sha256=8dYhJxoe_jLRwisHoA0jr575YhUKmLIqSXfW88KshcQ,6000
-provider/main.py,sha256=7Fj9h13uVL3dPeukAc2U3q_HonMsRhTSnj86lj6Hqtc,9898
-provider/network/port_verifier.py,sha256=3l6WNwBHydggJRFYkAsuBp1eCxaU619kjWuM-zSVj2o,13267
-provider/security/ethereum.py,sha256=EwPZj4JR8OEpto6LhKjuuT3Z9pBX6P7-UQaqJtqFkYQ,1242
-provider/security/faucet.py,sha256=O2DgP3bIrRUm9tdLCdgnda9em0rPyeW42sWhO1EQJaA,5363
-provider/utils/ascii_art.py,sha256=ykBFsztk57GIiz1NJ-EII5UvN74iECqQL4h9VmiW6Z8,3161
-provider/utils/logging.py,sha256=C_elr0sJROHKQgErYpHJQvfujgh0k4Zf2gg8ZKfrmVk,2590
-provider/utils/port_display.py,sha256=eLtkpYk5va1KWDQhEbxTYGymaXiPVpsYbvmdXM29IPo,12067
-provider/utils/retry.py,sha256=ekP2ucaSJNN-lBcrIvyHa4QYPKNITMl1a5V1X6BBvsw,1560
-provider/utils/setup.py,sha256=Z5dLuBQkb5vdoQsu1HJZwXmu9NWsiBYJ7Vq9-C-_tY8,2932
-provider/vm/__init__.py,sha256=JGs50tUmzOR1rQ_w4fMY_3XWylmiA1G7KKWZkVw51mY,501
-provider/vm/cloud_init.py,sha256=E5dDH7dqStRcJNDfbarBBe83-c9N63W8B5ycIrHI8eU,4627
-provider/vm/models.py,sha256=RnfRQrziGBQZV_z6whQ8m328Abn2pItKZEZhT4S8AkU,6266
-provider/vm/multipass.py,sha256=DOL_9fRXNu9LyWdW_HGCxG9C-iuuPnAF0u95sObb6Fk,16940
-provider/vm/name_mapper.py,sha256=MrshNeJ4Dw-WBsyiIVcn9N5xyOxaBKX4Yqhyh_m5IFg,4103
-provider/vm/port_manager.py,sha256=H7uLe-jJgBPtvnWrIHhRP-04sq11wMunOf0CFa_eDWE,11820
-provider/vm/proxy_manager.py,sha256=aSzMVggH8BgE6L92yCSRtultEQiwHtPP8fnQzSF102Y,14941
-golem_vm_provider-0.1.26.dist-info/METADATA,sha256=p4DYiuNE7QKUyIyev2sZVXiTMP3gshKOokTWGMepJ4M,10645
-golem_vm_provider-0.1.26.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-golem_vm_provider-0.1.26.dist-info/entry_points.txt,sha256=5Jiie1dIXygmxmDW66bKKxQpmBLJ7leSKRrb8bkQALw,52
-golem_vm_provider-0.1.26.dist-info/RECORD,,