golem-vm-provider 0.1.24__py3-none-any.whl → 0.1.27__py3-none-any.whl

provider/container.py

@@ -0,0 +1,84 @@
+import os
+from dependency_injector import containers, providers
+from pathlib import Path
+
+from .config import settings
+from .discovery.resource_tracker import ResourceTracker
+from .discovery.golem_base_advertiser import GolemBaseAdvertiser
+from .discovery.advertiser import DiscoveryServerAdvertiser
+from .discovery.service import AdvertisementService
+from .service import ProviderService
+from .vm.multipass_adapter import MultipassAdapter
+from .vm.service import VMService
+from .vm.name_mapper import VMNameMapper
+from .vm.port_manager import PortManager
+from .vm.proxy_manager import PythonProxyManager
+
+
+class Container(containers.DeclarativeContainer):
+    """Dependency injection container."""
+
+    config = providers.Configuration()
+
+    resource_tracker = providers.Singleton(ResourceTracker)
+
+    advertiser = providers.Selector(
+        config.ADVERTISER_TYPE,
+        golem_base=providers.Singleton(
+            GolemBaseAdvertiser,
+            resource_tracker=resource_tracker,
+        ),
+        discovery_server=providers.Singleton(
+            DiscoveryServerAdvertiser,
+            resource_tracker=resource_tracker,
+        ),
+    )
+
+    advertisement_service = providers.Singleton(
+        AdvertisementService,
+        advertiser=advertiser,
+    )
+
+    vm_name_mapper = providers.Singleton(
+        VMNameMapper,
+        db_path=Path(settings.VM_DATA_DIR) / "vm_names.json",
+    )
+
+    port_manager = providers.Singleton(
+        PortManager,
+        start_port=config.PORT_RANGE_START,
+        end_port=config.PORT_RANGE_END,
+        state_file=providers.Callable(
+            os.path.join,
+            config.PROXY_STATE_DIR,
+            "ports.json"
+        ),
+        discovery_port=config.PORT,
+        skip_verification=config.SKIP_PORT_VERIFICATION,
+    )
+
+    proxy_manager = providers.Singleton(
+        PythonProxyManager,
+        port_manager=port_manager,
+        name_mapper=vm_name_mapper,
+    )
+
+    vm_provider = providers.Singleton(
+        MultipassAdapter,
+        proxy_manager=proxy_manager,
+        name_mapper=vm_name_mapper,
+    )
+
+    vm_service = providers.Singleton(
+        VMService,
+        provider=vm_provider,
+        resource_tracker=resource_tracker,
+        name_mapper=vm_name_mapper,
+    )
+
+    provider_service = providers.Singleton(
+        ProviderService,
+        vm_service=vm_service,
+        advertisement_service=advertisement_service,
+        port_manager=port_manager,
+    )

provider/discovery/__init__.py

@@ -1,6 +1,12 @@
-from .advertiser import ResourceMonitor, ResourceAdvertiser
+from .advertiser import Advertiser, DiscoveryServerAdvertiser
+from .golem_base_advertiser import GolemBaseAdvertiser
+from .resource_monitor import ResourceMonitor
+from .service import AdvertisementService
 
 __all__ = [
+    "Advertiser",
+    "DiscoveryServerAdvertiser",
+    "GolemBaseAdvertiser",
     "ResourceMonitor",
-    "ResourceAdvertiser"
+    "AdvertisementService",
 ]

provider/discovery/advertiser.py

@@ -1,90 +1,71 @@
 import aiohttp
 import asyncio
 import logging
-import psutil
-from datetime import datetime
-from typing import Dict, Optional
+from abc import ABC, abstractmethod
+from typing import Optional
 
 from ..config import settings
 from ..utils.retry import async_retry
+from .resource_tracker import ResourceTracker
 
 logger = logging.getLogger(__name__)
 
-class ResourceMonitor:
-    """Monitor system resources."""
-    
-    @staticmethod
-    def get_cpu_count() -> int:
-        """Get number of CPU cores."""
-        return psutil.cpu_count()
-
-    @staticmethod
-    def get_memory_gb() -> int:
-        """Get available memory in GB."""
-        return psutil.virtual_memory().available // (1024 ** 3)
-
-    @staticmethod
-    def get_storage_gb() -> int:
-        """Get available storage in GB."""
-        return psutil.disk_usage("/").free // (1024 ** 3)
-
-    @staticmethod
-    def get_cpu_percent() -> float:
-        """Get CPU usage percentage."""
-        return psutil.cpu_percent(interval=1)
-
-    @staticmethod
-    def get_memory_percent() -> float:
-        """Get memory usage percentage."""
-        return psutil.virtual_memory().percent
-
-    @staticmethod
-    def get_storage_percent() -> float:
-        """Get storage usage percentage."""
-        return psutil.disk_usage("/").percent
-
-class ResourceAdvertiser:
-    """Advertise available resources to discovery service."""
+class Advertiser(ABC):
+    """Abstract base class for advertisers."""
+
+    @abstractmethod
+    async def initialize(self):
+        """Initialize the advertiser."""
+        pass
+
+    @abstractmethod
+    async def start_loop(self):
+        """Start the advertising loop."""
+        pass
+
+    @abstractmethod
+    async def stop(self):
+        """Stop the advertising loop."""
+        pass
+
+    @abstractmethod
+    async def post_advertisement(self):
+        """Post a single advertisement."""
+        pass
+
+class DiscoveryServerAdvertiser(Advertiser):
+    """Advertise available resources to a discovery service."""
     
     def __init__(
         self,
         resource_tracker: 'ResourceTracker',
         discovery_url: Optional[str] = None,
         provider_id: Optional[str] = None,
-        update_interval: Optional[int] = None
     ):
         self.resource_tracker = resource_tracker
         self.discovery_url = discovery_url or settings.DISCOVERY_URL
         self.provider_id = provider_id or settings.PROVIDER_ID
-        self.update_interval = update_interval or settings.ADVERTISEMENT_INTERVAL
         self.session: Optional[aiohttp.ClientSession] = None
         self._stop_event = asyncio.Event()
 
-    async def start(self):
-        """Start advertising resources."""
+    async def initialize(self):
+        """Initialize the advertiser."""
         self.session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10))
-        # Register for resource updates
-        self.resource_tracker.on_update(self._post_advertisement)
-        
-        # Test discovery service connection with retries
+        self.resource_tracker.on_update(
+            lambda: asyncio.create_task(self.post_advertisement())
+        )
         try:
             await self._check_discovery_health()
         except Exception as e:
             logger.warning(f"Could not connect to discovery service after retries, continuing without advertising: {e}")
             return
-            
+
+    async def start_loop(self):
+        """Start advertising resources in a loop."""
         try:
             while not self._stop_event.is_set():
-                try:
-                    await self._post_advertisement()
-                except aiohttp.ClientError as e:
-                    logger.error(f"Network error posting advertisement: {e}")
-                    await asyncio.sleep(min(60, self.update_interval))
-                except Exception as e:
-                    logger.error(f"Failed to post advertisement: {e}")
-                    await asyncio.sleep(min(60, self.update_interval))
-                else:
-                    await asyncio.sleep(self.update_interval)
+                await self.post_advertisement()
+                await asyncio.sleep(settings.ADVERTISEMENT_INTERVAL)
         finally:
             await self.stop()
 
@@ -106,19 +87,17 @@ class ResourceAdvertiser:
                 raise Exception(f"Discovery service health check failed: {response.status}")
 
     @async_retry(retries=3, delay=1.0, backoff=2.0, exceptions=(aiohttp.ClientError, asyncio.TimeoutError))
-    async def _post_advertisement(self):
+    async def post_advertisement(self):
         """Post resource advertisement to discovery service."""
         if not self.session:
             raise RuntimeError("Session not initialized")
 
         resources = self.resource_tracker.get_available_resources()
         
-        # Don't advertise if resources are too low
         if not self.resource_tracker._meets_minimum_requirements(resources):
             logger.warning("Resources too low, skipping advertisement")
             return
 
-        # Get public IP with retries
         try:
             ip_address = await self._get_public_ip()
         except Exception as e:
@@ -130,7 +109,7 @@ class ResourceAdvertiser:
                 f"{self.discovery_url}/api/v1/advertisements",
                 headers={
                     "X-Provider-ID": self.provider_id,
-                    "X-Provider-Signature": "signature",  # TODO: Implement signing
+                    "X-Provider-Signature": "signature",
                     "Content-Type": "application/json"
                 },
                 json={
@@ -138,7 +117,7 @@ class ResourceAdvertiser:
                     "country": settings.PROVIDER_COUNTRY,
                     "resources": resources
                 },
-                timeout=aiohttp.ClientTimeout(total=5)  # 5 second timeout for advertisement
+                timeout=aiohttp.ClientTimeout(total=5)
             ) as response:
                 if not response.ok:
                     error_text = await response.text()
@@ -159,7 +138,6 @@ class ResourceAdvertiser:
         if not self.session:
             raise RuntimeError("Session not initialized")
 
-        # Try multiple IP services in case one fails
         services = [
             "https://api.ipify.org",
             "https://ifconfig.me/ip",

provider/discovery/golem_base_advertiser.py

@@ -0,0 +1,135 @@
+import asyncio
+from typing import Optional
+
+from golem_base_sdk import GolemBaseClient, GolemBaseCreate, GolemBaseUpdate, GolemBaseDelete, Annotation
+from .advertiser import Advertiser
+from .golem_base_utils import get_provider_entity_keys
+from ..config import settings
+from ..utils.logging import setup_logger
+
+logger = setup_logger(__name__)
+
+
+class GolemBaseAdvertiser(Advertiser):
+    """Advertise available resources to the Golem Base network."""
+
+    def __init__(self, resource_tracker: "ResourceTracker"):
+        self.resource_tracker = resource_tracker
+        self.client: Optional[GolemBaseClient] = None
+        self._stop_event = asyncio.Event()
+
+    async def initialize(self):
+        """Initialize the advertiser."""
+        private_key_hex = settings.ETHEREUM_PRIVATE_KEY.replace("0x", "")
+        private_key_bytes = bytes.fromhex(private_key_hex)
+        self.client = await GolemBaseClient.create(
+            rpc_url=settings.GOLEM_BASE_RPC_URL,
+            ws_url=settings.GOLEM_BASE_WS_URL,
+            private_key=private_key_bytes,
+        )
+        self.resource_tracker.on_update(
+            lambda: asyncio.create_task(self.post_advertisement())
+        )
+
+    async def start_loop(self):
+        """Start advertising resources in a loop."""
+        try:
+            while not self._stop_event.is_set():
+                await self.post_advertisement()
+                await asyncio.sleep(settings.ADVERTISEMENT_INTERVAL)
+        finally:
+            await self.stop()
+
+    async def stop(self):
+        """Stop advertising resources."""
+        self._stop_event.set()
+        if self.client:
+            await self.client.disconnect()
+
+    async def post_advertisement(self):
+        """Post or update resource advertisement on the Golem Base network."""
+        if not self.client:
+            raise RuntimeError("Golem Base client not initialized")
+
+        resources = self.resource_tracker.get_available_resources()
+        if not self.resource_tracker._meets_minimum_requirements(resources):
+            logger.warning("Resources too low, skipping advertisement")
+            return
+
+        ip_address = settings.PUBLIC_IP
+        if not ip_address:
+            logger.error("Could not get public IP, skipping advertisement")
+            return
+
+        try:
+            existing_keys = await get_provider_entity_keys(self.client, settings.PROVIDER_ID)
+
+            string_annotations = [
+                Annotation(key="golem_type", value="provider"),
+                Annotation(key="golem_provider_id", value=settings.PROVIDER_ID),
+                Annotation(key="golem_ip_address", value=ip_address),
+                Annotation(key="golem_country", value=settings.PROVIDER_COUNTRY),
+                Annotation(key="golem_provider_name", value=settings.PROVIDER_NAME),
+            ]
+            numeric_annotations = [
+                Annotation(key="golem_cpu", value=resources["cpu"]),
+                Annotation(key="golem_memory", value=resources["memory"]),
+                Annotation(key="golem_storage", value=resources["storage"]),
+            ]
+
+            if len(existing_keys) > 1:
+                logger.warning(f"Found {len(existing_keys)} advertisements. Cleaning up and creating a new one.")
+                deletes = [GolemBaseDelete(entity_key=key) for key in existing_keys]
+                await self.client.delete_entities(deletes)
+                await self._create_advertisement(string_annotations, numeric_annotations)
+
+            elif len(existing_keys) == 1:
+                entity_key = existing_keys[0]
+                metadata = await self.client.get_entity_metadata(entity_key)
+                
+                current_annotations = {ann.key: ann.value for ann in metadata.numeric_annotations}
+                current_annotations.update({ann.key: ann.value for ann in metadata.string_annotations})
+
+                # Full comparison of all annotations
+                expected_annotations = {ann.key: ann.value for ann in string_annotations}
+                expected_annotations.update({ann.key: ann.value for ann in numeric_annotations})
+
+                # Debugging logs to compare annotations
+                logger.info(f"IP address from settings: {ip_address}")
+                logger.info(f"Current on-chain annotations: {current_annotations}")
+                logger.info(f"Expected annotations based on current config: {expected_annotations}")
+
+                if sorted(current_annotations.items()) == sorted(expected_annotations.items()):
+                    logger.info("Advertisement is up-to-date. Waiting for expiration.")
+                else:
+                    logger.info("Advertisement is outdated. Updating.")
+                    update = GolemBaseUpdate(
+                        entity_key=entity_key,
+                        data=b"",
+                        btl=settings.ADVERTISEMENT_INTERVAL * 2,
+                        string_annotations=string_annotations,
+                        numeric_annotations=numeric_annotations,
+                    )
+                    await self.client.update_entities([update])
+                    logger.info(f"Updated advertisement. Entity key: {entity_key}")
+
+            else: # No existing keys
+                await self._create_advertisement(string_annotations, numeric_annotations)
+
+        except Exception as e:
+            logger.error(f"Failed to post or update advertisement on Golem Base: {e}")
+
+    async def _create_advertisement(self, string_annotations, numeric_annotations):
+        """Helper to create a new advertisement."""
+        entity = GolemBaseCreate(
+            data=b"",
+            btl=settings.ADVERTISEMENT_INTERVAL * 2,
+            string_annotations=string_annotations,
+            numeric_annotations=numeric_annotations,
+        )
+        receipts = await self.client.create_entities([entity])
+        if receipts:
+            receipt = receipts[0]
+            logger.info(f"Posted new advertisement. Entity key: {receipt.entity_key}")
+        else:
+            logger.error("Failed to post advertisement: no receipt received")

provider/discovery/golem_base_utils.py

@@ -0,0 +1,10 @@
+from typing import List
+from golem_base_sdk import GolemBaseClient, EntityKey, QueryEntitiesResult, GenericBytes
+from ..config import settings
+
+async def get_provider_entity_keys(client: GolemBaseClient, provider_id: str) -> List[EntityKey]:
+    """Get all entity keys for a given provider ID."""
+    query = f'golem_provider_id="{provider_id}"'
+    results: list[QueryEntitiesResult] = await client.query_entities(query)
+    # The entity_key from query_entities is a hex string, convert it to an EntityKey object
+    return [EntityKey(GenericBytes.from_hex_string(result.entity_key)) for result in results]

provider/discovery/resource_monitor.py

@@ -0,0 +1,34 @@
+import psutil
+
+class ResourceMonitor:
+    """Monitor system resources."""
+    
+    @staticmethod
+    def get_cpu_count() -> int:
+        """Get number of CPU cores."""
+        return psutil.cpu_count()
+
+    @staticmethod
+    def get_memory_gb() -> int:
+        """Get available memory in GB."""
+        return psutil.virtual_memory().available // (1024 ** 3)
+
+    @staticmethod
+    def get_storage_gb() -> int:
+        """Get available storage in GB."""
+        return psutil.disk_usage("/").free // (1024 ** 3)
+
+    @staticmethod
+    def get_cpu_percent() -> float:
+        """Get CPU usage percentage."""
+        return psutil.cpu_percent(interval=1)
+
+    @staticmethod
+    def get_memory_percent() -> float:
+        """Get memory usage percentage."""
+        return psutil.virtual_memory().percent
+
+    @staticmethod
+    def get_storage_percent() -> float:
+        """Get storage usage percentage."""
+        return psutil.disk_usage("/").percent

provider/discovery/resource_tracker.py

@@ -11,7 +11,7 @@ class ResourceTracker:
 
     def __init__(self):
         """Initialize resource tracker."""
-        from .advertiser import ResourceMonitor
+        from .resource_monitor import ResourceMonitor
         self.total_resources = {
             "cpu": ResourceMonitor.get_cpu_count(),
             "memory": ResourceMonitor.get_memory_gb(),

provider/discovery/service.py

@@ -0,0 +1,24 @@
+import asyncio
+from typing import Optional
+
+from .advertiser import Advertiser
+from ..config import settings
+
+class AdvertisementService:
+    """Service for managing the advertisement lifecycle."""
+
+    def __init__(self, advertiser: Advertiser):
+        self.advertiser = advertiser
+        self._task: Optional[asyncio.Task] = None
+
+    async def start(self):
+        """Initialize and start the advertiser."""
+        await self.advertiser.initialize()
+        self._task = asyncio.create_task(self.advertiser.start_loop())
+
+    async def stop(self):
+        """Stop the advertiser."""
+        if self._task:
+            self._task.cancel()
+            await self._task
+        await self.advertiser.stop()