golem-vm-provider 0.1.19__tar.gz → 0.1.21__tar.gz

{golem_vm_provider-0.1.19 → golem_vm_provider-0.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: golem-vm-provider
-Version: 0.1.19
+Version: 0.1.21
 Summary: VM on Golem Provider Node - Run your own provider node to offer VMs on the Golem Network
 Keywords: golem,vm,provider,cloud,decentralized
 Author: Phillip Jensen

{golem_vm_provider-0.1.19 → golem_vm_provider-0.1.21}/provider/main.py

@@ -18,28 +18,44 @@ app = FastAPI(title="VM on Golem Provider")
 async def setup_provider() -> None:
     """Setup and initialize the provider components."""
     try:
-        # Port manager is already initialized and verified in startup_event
-        port_manager = app.state.port_manager
-        
-        # Create resource tracker
+        # Create resource tracker first
         logger.process("🔄 Initializing resource tracker...")
         resource_tracker = ResourceTracker()
         app.state.resource_tracker = resource_tracker
-        
-        # Create provider with resource tracker and port manager
+
+        # Create provider with resource tracker and temporary port manager
         logger.process("🔄 Initializing VM provider...")
-        provider = MultipassProvider(resource_tracker, port_manager=port_manager)
+        provider = MultipassProvider(resource_tracker, port_manager=None)  # Will be set later
+        
         try:
+            # Initialize provider (without port operations)
             await asyncio.wait_for(provider.initialize(), timeout=30)
             
-            # Store provider and proxy manager references
+            # Store provider reference
             app.state.provider = provider
             app.state.proxy_manager = provider.proxy_manager
             
-            # Restore proxy configurations
+            # Restore proxy configurations first
             logger.process("🔄 Restoring proxy configurations...")
             await app.state.proxy_manager._load_state()
             
+            # Now initialize port manager with knowledge of restored proxies
+            logger.process("🔄 Initializing port manager...")
+            port_manager = PortManager(
+                start_port=settings.PORT_RANGE_START,
+                end_port=settings.PORT_RANGE_END,
+                discovery_port=settings.PORT,
+                existing_ports=app.state.proxy_manager.get_active_ports()
+            )
+            
+            if not await port_manager.initialize():
+                raise RuntimeError("Port verification failed")
+            
+            # Update provider and proxy manager with verified port manager
+            app.state.port_manager = port_manager
+            provider.port_manager = port_manager
+            app.state.proxy_manager.port_manager = port_manager
+            
         except asyncio.TimeoutError:
             logger.error("Provider initialization timed out")
             raise

{golem_vm_provider-0.1.19 → golem_vm_provider-0.1.21}/provider/network/port_verifier.py

@@ -198,15 +198,21 @@ class PortVerifier:
                 ))
                 logger.warning(error_msg)
         
-        # If no successful verifications, mark all ports as inaccessible
-        if not any(result.accessible for result in results.values()):
+        # If no servers were successful, fail verification
+        if not any(attempt.success for attempt in attempts):
             error_msg = (
-                "Failed to verify ports with any server. Please ensure:\n"
+                "Failed to connect to any port check servers. Please ensure:\n"
                 "1. At least one port check server is running and accessible\n"
                 "2. Your network connection is stable\n"
                 "3. The server URLs are correct"
             )
             logger.error(error_msg)
+            raise RuntimeError(error_msg)
+
+        # If no successful verifications but servers were reachable, mark ports as inaccessible
+        if not any(result.accessible for result in results.values()):
+            error_msg = "No ports were verified as accessible"
+            logger.error(error_msg)
             results = {
                 port: PortVerificationResult(
                     port=port,

{golem_vm_provider-0.1.19 → golem_vm_provider-0.1.21}/provider/vm/multipass.py

@@ -37,8 +37,11 @@ class MultipassProvider(VMProvider):
         self.vm_data_dir.mkdir(parents=True, exist_ok=True)
 
         # Initialize managers
-        self.proxy_manager = PythonProxyManager(port_manager=port_manager)
         self.name_mapper = VMNameMapper(self.vm_data_dir / "vm_names.json")
+        self.proxy_manager = PythonProxyManager(
+            port_manager=port_manager,
+            name_mapper=self.name_mapper
+        )
 
     def _verify_installation(self) -> None:
         """Verify multipass is installed and get version."""

{golem_vm_provider-0.1.19 → golem_vm_provider-0.1.21}/provider/vm/port_manager.py

@@ -23,7 +23,8 @@ class PortManager:
         end_port: int = 50900,
         state_file: Optional[str] = None,
         port_check_servers: Optional[List[str]] = None,
-        discovery_port: Optional[int] = None
+        discovery_port: Optional[int] = None,
+        existing_ports: Optional[Set[int]] = None
     ):
         """Initialize the port manager.
 
@@ -32,6 +33,8 @@ class PortManager:
             end_port: End of port range (exclusive)
             state_file: Path to persist port assignments
             port_check_servers: List of URLs for port checking services
+            discovery_port: Port used for discovery service
+            existing_ports: Set of ports that should be considered in use
         """
         self.start_port = start_port
         self.end_port = end_port
@@ -40,12 +43,12 @@ class PortManager:
         self.lock = Lock()
         self._used_ports: dict[str, int] = {}  # vm_id -> port
         self.verified_ports: Set[int] = set()
+        self._existing_ports = existing_ports or set()
 
         # Initialize port verifier with default servers
         self.port_check_servers = port_check_servers or [
             "http://localhost:9000",  # Local development server
             "http://195.201.39.101:9000",  # Production servers
-
         ]
         self.discovery_port = discovery_port or settings.PORT
         self.port_verifier = PortVerifier(
@@ -53,7 +56,14 @@ class PortManager:
             discovery_port=self.discovery_port
         )
 
+        # Load state after setting existing ports
         self._load_state()
+        
+        # Mark existing ports as used and remove from verified ports
+        for port in self._existing_ports:
+            if port in self.verified_ports:
+                self.verified_ports.remove(port)
+                logger.debug(f"Marked port {port} as in use from existing ports")
 
     async def initialize(self) -> bool:
         """Initialize port manager with verification.
@@ -70,8 +80,9 @@ class PortManager:
         )
         display.print_header()
 
-        # Only verify SSH ports since provider port was already verified
-        ssh_ports = list(range(self.start_port, self.end_port))
+        # Only verify ports that aren't already marked as in use
+        available_ports = set(range(self.start_port, self.end_port)) - self._existing_ports
+        ssh_ports = list(available_ports)
         logger.info(f"Starting port verification...")
         logger.info(f"SSH ports range: {self.start_port}-{self.end_port}")
         logger.info(

{golem_vm_provider-0.1.19 → golem_vm_provider-0.1.21}/provider/vm/proxy_manager.py

@@ -3,7 +3,7 @@ import json
 import asyncio
 import logging
 from pathlib import Path
-from typing import Optional, Dict
+from typing import Optional, Dict, Set
 from asyncio import Task, Transport, Protocol
 
 from .port_manager import PortManager
@@ -146,59 +146,145 @@ class PythonProxyManager:
     
     def __init__(
         self,
-        port_manager: PortManager,
+        port_manager: Optional[PortManager],
+        name_mapper: "VMNameMapper",
         state_file: Optional[str] = None
     ):
         """Initialize the proxy manager.
         
         Args:
-            port_manager: Port allocation manager
+            port_manager: Port allocation manager (optional during startup)
+            name_mapper: VM name mapping manager
             state_file: Path to persist proxy state
         """
         self.port_manager = port_manager
+        self.name_mapper = name_mapper
         self.state_file = state_file or os.path.expanduser("~/.golem/provider/proxy_state.json")
-        self._proxies: Dict[str, ProxyServer] = {}  # vm_id -> ProxyServer
-        # Note: _load_state is now async and will be called explicitly during provider setup
+        self._proxies: Dict[str, ProxyServer] = {}  # multipass_name -> ProxyServer
+        self._state_version = 1  # For future state schema migrations
+        self._active_ports: Dict[str, int] = {}  # multipass_name -> port
     
+    def get_active_ports(self) -> Set[int]:
+        """Get set of ports that should be considered in use.
+        
+        Returns:
+            Set of ports that are allocated to VMs
+        """
+        return set(self._active_ports.values())
+
     async def _load_state(self) -> None:
         """Load and restore proxy state from file."""
         try:
             state_path = Path(self.state_file)
-            if state_path.exists():
-                with open(state_path, 'r') as f:
-                    state = json.load(f)
-                    # Restore proxy servers from saved state
-                    restore_tasks = []
-                    for vm_id, proxy_info in state.items():
-                        # Create task to restore proxy
-                        task = self.add_vm(
-                            vm_id=vm_id,
-                            vm_ip=proxy_info['target'],
-                            port=proxy_info['port']
-                        )
-                        restore_tasks.append(task)
-                    
-                    # Wait for all proxies to be restored
-                    if restore_tasks:
-                        results = await asyncio.gather(*restore_tasks, return_exceptions=True)
-                        successful = sum(1 for r in results if r is True)
-                        logger.info(f"Restored {successful}/{len(state)} proxy configurations")
+            if not state_path.exists():
+                return
+
+            with open(state_path, 'r') as f:
+                state = json.load(f)
+
+            # Check state version for future migrations
+            if state.get('version', 1) != self._state_version:
+                logger.warning(f"State version mismatch: {state.get('version')} != {self._state_version}")
+
+            # First load all port allocations
+            for requestor_name, proxy_info in state.get('proxies', {}).items():
+                multipass_name = await self.name_mapper.get_multipass_name(requestor_name)
+                if multipass_name:
+                    self._active_ports[multipass_name] = proxy_info['port']
+
+            # Then attempt to restore proxies with retries
+            restore_tasks = []
+            for requestor_name, proxy_info in state.get('proxies', {}).items():
+                multipass_name = await self.name_mapper.get_multipass_name(requestor_name)
+                if multipass_name:
+                    task = self._restore_proxy_with_retry(
+                        multipass_name=multipass_name,
+                        vm_ip=proxy_info['target'],
+                        port=proxy_info['port']
+                    )
+                    restore_tasks.append(task)
+                else:
+                    logger.warning(f"No multipass name found for requestor VM {requestor_name}")
+
+            # Wait for all restore attempts
+            if restore_tasks:
+                results = await asyncio.gather(*restore_tasks, return_exceptions=True)
+                successful = sum(1 for r in results if r is True)
+                logger.info(f"Restored {successful}/{len(state.get('proxies', {}))} proxy configurations")
+
         except Exception as e:
             logger.error(f"Failed to load proxy state: {e}")
+
+    async def _restore_proxy_with_retry(
+        self,
+        multipass_name: str,
+        vm_ip: str,
+        port: int,
+        max_retries: int = 3,
+        initial_delay: float = 1.0
+    ) -> bool:
+        """Attempt to restore a proxy with exponential backoff retry.
+        
+        Args:
+            multipass_name: Multipass VM name
+            vm_ip: VM IP address
+            port: Port to use
+            max_retries: Maximum number of retry attempts
+            initial_delay: Initial delay between retries (doubles each attempt)
+            
+        Returns:
+            bool: True if restoration was successful
+        """
+        delay = initial_delay
+        for attempt in range(max_retries):
+            try:
+                if attempt > 0:
+                    logger.info(f"Retry attempt {attempt + 1} for {multipass_name} on port {port}")
+                    await asyncio.sleep(delay)
+                    delay *= 2  # Exponential backoff
+
+                # Attempt to create proxy
+                proxy = ProxyServer(port, vm_ip)
+                await proxy.start()
+                
+                self._proxies[multipass_name] = proxy
+                logger.info(f"Successfully restored proxy for {multipass_name} on port {port}")
+                return True
+
+            except Exception as e:
+                logger.warning(f"Attempt {attempt + 1} failed for {multipass_name}: {e}")
+                if attempt == max_retries - 1:
+                    logger.error(f"Failed to restore proxy for {multipass_name} after {max_retries} attempts")
+                    # Remove from active ports if all retries failed
+                    self._active_ports.pop(multipass_name, None)
+                    return False
     
-    def _save_state(self) -> None:
-        """Save current proxy state to file."""
+    async def _save_state(self) -> None:
+        """Save current proxy state to file using requestor names."""
         try:
             state = {
-                vm_id: {
-                    'port': proxy.listen_port,
-                    'target': proxy.target_host
-                }
-                for vm_id, proxy in self._proxies.items()
+                'version': self._state_version,
+                'proxies': {}
             }
+            
+            for multipass_name, proxy in self._proxies.items():
+                requestor_name = await self.name_mapper.get_requestor_name(multipass_name)
+                if requestor_name:
+                    state['proxies'][requestor_name] = {
+                        'port': proxy.listen_port,
+                        'target': proxy.target_host
+                    }
+            
+            # Save to temporary file first
+            temp_file = f"{self.state_file}.tmp"
             os.makedirs(os.path.dirname(self.state_file), exist_ok=True)
-            with open(self.state_file, 'w') as f:
-                json.dump(state, f)
+            
+            with open(temp_file, 'w') as f:
+                json.dump(state, f, indent=2)
+            
+            # Atomic rename
+            os.replace(temp_file, self.state_file)
+            
         except Exception as e:
             logger.error(f"Failed to save proxy state: {e}")
     
@@ -206,7 +292,7 @@ class PythonProxyManager:
         """Add proxy configuration for a new VM.
         
         Args:
-            vm_id: Unique identifier for the VM
+            vm_id: Unique identifier for the VM (multipass name)
             vm_ip: IP address of the VM
             port: Optional specific port to use, if not provided one will be allocated
             
@@ -226,7 +312,7 @@ class PythonProxyManager:
             await proxy.start()
             
             self._proxies[vm_id] = proxy
-            self._save_state()
+            await self._save_state()
             
             logger.info(f"Started proxy for VM {vm_id} on port {port}")
             return True
@@ -242,14 +328,14 @@ class PythonProxyManager:
         """Remove proxy configuration for a VM.
         
         Args:
-            vm_id: Unique identifier for the VM
+            vm_id: Unique identifier for the VM (multipass name)
         """
         try:
             if vm_id in self._proxies:
                 proxy = self._proxies.pop(vm_id)
                 await proxy.stop()
                 self.port_manager.deallocate_port(vm_id)
-                self._save_state()
+                await self._save_state()
                 logger.info(f"Removed proxy for VM {vm_id}")
         except Exception as e:
             logger.error(f"Failed to remove proxy for VM {vm_id}: {e}")
@@ -270,7 +356,7 @@ class PythonProxyManager:
                 cleanup_errors.append(f"Failed to remove proxy for VM {vm_id}: {e}")
         
         try:
-            self._save_state()
+            await self._save_state()
         except Exception as e:
             cleanup_errors.append(f"Failed to save state: {e}")
             

{golem_vm_provider-0.1.19 → golem_vm_provider-0.1.21}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "golem-vm-provider"
-version = "0.1.19"
+version = "0.1.21"
 description = "VM on Golem Provider Node - Run your own provider node to offer VMs on the Golem Network"
 authors = ["Phillip Jensen <phillip+vm-on-golem@golemgrid.com>"]
 readme = "README.md"