request-vm-on-golem 0.1.24__py3-none-any.whl → 0.1.26__py3-none-any.whl

requestor/config.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from typing import Optional, Dict
-from pydantic import BaseSettings, Field
+from pydantic_settings import BaseSettings
+from pydantic import Field, validator
 
 class RequestorConfig(BaseSettings):
     """Configuration settings for the requestor node."""
@@ -19,11 +20,44 @@ class RequestorConfig(BaseSettings):
         default=False,
         description="Force localhost for provider URLs in development mode"
     )
+
+    @property
+    def DEV_MODE(self) -> bool:
+        return self.environment == "development"
     
     # Discovery Service
+    discovery_driver: str = Field(
+        default="golem-base",
+        description="Discovery driver: 'central' or 'golem-base'"
+    )
     discovery_url: str = Field(
         default="http://195.201.39.101:9001",
-        description="URL of the discovery service"
+        description="URL of the discovery service (for 'central' driver)"
+    )
+
+    @validator("discovery_url", always=True)
+    def set_discovery_url(cls, v: str, values: dict) -> str:
+        """Prefix discovery URL with DEVMODE if in development."""
+        if values.get("environment") == "development":
+            return f"DEVMODE-{v}"
+        return v
+
+    # Golem Base Settings
+    golem_base_rpc_url: str = Field(
+        default="https://ethwarsaw.holesky.golemdb.io/rpc",
+        description="Golem Base RPC URL"
+    )
+    golem_base_ws_url: str = Field(
+        default="wss://ethwarsaw.holesky.golemdb.io/rpc/ws",
+        description="Golem Base WebSocket URL"
+    )
+    advertisement_interval: int = Field(
+        default=240,
+        description="Advertisement interval in seconds (should match provider)"
+    )
+    ethereum_private_key: str = Field(
+        default="0x0000000000000000000000000000000000000000000000000000000000000001",
+        description="Private key for Golem Base"
     )
     
     # Base Directory

requestor/db/sqlite.py

@@ -11,47 +11,17 @@ class Database:
     async def init(self):
         """Initialize database and handle migrations."""
         async with aiosqlite.connect(self.db_path) as db:
-            # Check if table exists
-            async with db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='vms'") as cursor:
-                table_exists = await cursor.fetchone() is not None
-
-            if not table_exists:
-                # Create new table without ssh_key_name
-                await db.execute("""
-                    CREATE TABLE vms (
-                        name TEXT PRIMARY KEY,
-                        provider_ip TEXT NOT NULL,
-                        vm_id TEXT NOT NULL,
-                        config TEXT NOT NULL,
-                        status TEXT NOT NULL DEFAULT 'running',
-                        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-                    )
-                """)
-            else:
-                # Check if ssh_key_name column exists
-                async with db.execute("PRAGMA table_info(vms)") as cursor:
-                    columns = await cursor.fetchall()
-                    has_ssh_key_name = any(col[1] == 'ssh_key_name' for col in columns)
-
-                if has_ssh_key_name:
-                    # Migrate existing data
-                    await db.execute("""
-                        CREATE TABLE vms_new (
-                            name TEXT PRIMARY KEY,
-                            provider_ip TEXT NOT NULL,
-                            vm_id TEXT NOT NULL,
-                            config TEXT NOT NULL,
-                            status TEXT NOT NULL DEFAULT 'running',
-                            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-                        )
-                    """)
-                    await db.execute("""
-                        INSERT INTO vms_new (name, provider_ip, vm_id, config, status, created_at)
-                        SELECT name, provider_ip, vm_id, config, status, created_at FROM vms
-                    """)
-                    await db.execute("DROP TABLE vms")
-                    await db.execute("ALTER TABLE vms_new RENAME TO vms")
-
+            # Create VMs table if it doesn't exist
+            await db.execute("""
+                CREATE TABLE IF NOT EXISTS vms (
+                    name TEXT PRIMARY KEY,
+                    provider_ip TEXT NOT NULL,
+                    vm_id TEXT NOT NULL,
+                    config TEXT NOT NULL,
+                    status TEXT NOT NULL DEFAULT 'running',
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """)
             await db.commit()
 
     async def save_vm(
@@ -73,6 +43,28 @@ class Database:
             )
             await db.commit()
 
+    async def execute(self, query: str, params: tuple = None) -> None:
+        """Execute a raw SQL query."""
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute(query, params or ())
+            await db.commit()
+
+    async def fetchone(self, query: str, params: tuple = None) -> Optional[Dict]:
+        """Fetch a single row as a dictionary."""
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            async with db.execute(query, params or ()) as cursor:
+                row = await cursor.fetchone()
+                return dict(row) if row else None
+
+    async def fetchall(self, query: str, params: tuple = None) -> List[Dict]:
+        """Fetch all rows as dictionaries."""
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            async with db.execute(query, params or ()) as cursor:
+                rows = await cursor.fetchall()
+                return [dict(row) for row in rows]
+
     async def get_vm(self, name: str) -> Optional[Dict]:
         """Get VM details."""
         async with aiosqlite.connect(self.db_path) as db:

requestor/run.py

@@ -32,9 +32,11 @@ def check_requirements():
 def main():
     """Run the requestor CLI."""
     try:
-        # Load environment variables from .env file
-        env_path = Path(__file__).parent / '.env'
+        # Load environment variables from .env.dev file if it exists, otherwise use .env
+        dev_env_path = Path(__file__).parent.parent / '.env.dev'
+        env_path = dev_env_path if dev_env_path.exists() else Path(__file__).parent.parent / '.env'
         load_dotenv(dotenv_path=env_path)
+        logger.info(f"Loading environment variables from: {env_path}")
 
         # Check requirements
         if not check_requirements():

requestor/services/__init__.py

@@ -0,0 +1,6 @@
+"""Service layer for VM on Golem requestor."""
+
+from .vm_service import VMService
+from .provider_service import ProviderService
+
+__all__ = ['VMService', 'ProviderService']

requestor/services/database_service.py

@@ -0,0 +1,91 @@
+"""Database service for VM management."""
+from typing import Dict, List, Optional
+from pathlib import Path
+
+from ..db.sqlite import Database
+from ..errors import DatabaseError
+
+class DatabaseService:
+    """Service for database operations."""
+    
+    def __init__(self, db_path: Path):
+        self.db = Database(db_path)
+
+    async def init(self):
+        """Initialize database."""
+        try:
+            await self.db.init()
+        except Exception as e:
+            raise DatabaseError(f"Failed to initialize database: {str(e)}")
+
+    async def save_vm(
+        self,
+        name: str,
+        provider_ip: str,
+        vm_id: str,
+        config: Dict,
+        status: str = 'running'
+    ) -> None:
+        """Save VM details."""
+        try:
+            await self.db.save_vm(
+                name=name,
+                provider_ip=provider_ip,
+                vm_id=vm_id,
+                config=config,
+                status=status
+            )
+        except Exception as e:
+            raise DatabaseError(f"Failed to save VM: {str(e)}")
+
+    async def get_vm(self, name: str) -> Optional[Dict]:
+        """Get VM details by name."""
+        try:
+            vm = await self.db.get_vm(name)
+            if not vm:
+                return None
+            return vm
+        except Exception as e:
+            raise DatabaseError(f"Failed to get VM details: {str(e)}")
+
+    async def delete_vm(self, name: str) -> None:
+        """Delete VM from database."""
+        try:
+            await self.db.delete_vm(name)
+        except Exception as e:
+            raise DatabaseError(f"Failed to delete VM: {str(e)}")
+
+    async def update_vm_status(self, name: str, status: str) -> None:
+        """Update VM status."""
+        try:
+            await self.db.update_vm_status(name, status)
+        except Exception as e:
+            raise DatabaseError(f"Failed to update VM status: {str(e)}")
+
+    async def list_vms(self) -> List[Dict]:
+        """List all VMs."""
+        try:
+            return await self.db.list_vms()
+        except Exception as e:
+            raise DatabaseError(f"Failed to list VMs: {str(e)}")
+
+    async def execute(self, query: str, params: tuple = None) -> None:
+        """Execute a raw SQL query."""
+        try:
+            await self.db.execute(query, params)
+        except Exception as e:
+            raise DatabaseError(f"Failed to execute query: {str(e)}")
+
+    async def fetchone(self, query: str, params: tuple = None) -> Optional[Dict]:
+        """Fetch a single row as a dictionary."""
+        try:
+            return await self.db.fetchone(query, params)
+        except Exception as e:
+            raise DatabaseError(f"Failed to fetch row: {str(e)}")
+
+    async def fetchall(self, query: str, params: tuple = None) -> List[Dict]:
+        """Fetch all rows as dictionaries."""
+        try:
+            return await self.db.fetchall(query, params)
+        except Exception as e:
+            raise DatabaseError(f"Failed to fetch rows: {str(e)}")

requestor/services/provider_service.py

@@ -0,0 +1,265 @@
+"""Provider discovery and management service."""
+from typing import Dict, List, Optional
+import aiohttp
+import time
+from datetime import datetime, timezone
+from ..errors import DiscoveryError, ProviderError
+from ..config import config
+from golem_base_sdk import GolemBaseClient
+from golem_base_sdk.types import EntityKey, GenericBytes
+
+
+class ProviderService:
+    """Service for provider operations."""
+
+    def __init__(self):
+        self.session = None
+        self.golem_base_client = None
+
+    async def __aenter__(self):
+        self.session = aiohttp.ClientSession()
+        # The GolemBaseClient is now initialized on-demand in find_providers
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+        if self.golem_base_client:
+            await self.golem_base_client.disconnect()
+
+    async def find_providers(
+        self,
+        cpu: Optional[int] = None,
+        memory: Optional[int] = None,
+        storage: Optional[int] = None,
+        country: Optional[str] = None,
+        driver: Optional[str] = None
+    ) -> List[Dict]:
+        """Find providers matching requirements."""
+        discovery_driver = driver or config.discovery_driver
+        if discovery_driver == "golem-base":
+            if not self.golem_base_client:
+                private_key_hex = config.ethereum_private_key.replace("0x", "")
+                private_key_bytes = bytes.fromhex(private_key_hex)
+                self.golem_base_client = await GolemBaseClient.create(
+                    rpc_url=config.golem_base_rpc_url,
+                    ws_url=config.golem_base_ws_url,
+                    private_key=private_key_bytes,
+                )
+            return await self._find_providers_golem_base(cpu, memory, storage, country)
+        else:
+            return await self._find_providers_central(cpu, memory, storage, country)
+
+    async def _find_providers_golem_base(
+        self,
+        cpu: Optional[int] = None,
+        memory: Optional[int] = None,
+        storage: Optional[int] = None,
+        country: Optional[str] = None
+    ) -> List[Dict]:
+        """Find providers using Golem Base."""
+        try:
+            query = 'golem_type="provider"'
+            if cpu:
+                query += f' && golem_cpu>={cpu}'
+            if memory:
+                query += f' && golem_memory>={memory}'
+            if storage:
+                query += f' && golem_storage>={storage}'
+            if country:
+                query += f' && golem_country="{country}"'
+
+            results = await self.golem_base_client.query_entities(query)
+
+            providers = []
+            for result in results:
+                entity_key = EntityKey(
+                    GenericBytes.from_hex_string(result.entity_key)
+                )
+                metadata = await self.golem_base_client.get_entity_metadata(entity_key)
+                annotations = {
+                    ann.key: ann.value for ann in metadata.string_annotations}
+                annotations.update(
+                    {ann.key: ann.value for ann in metadata.numeric_annotations})
+                provider = {
+                    'provider_id': annotations.get('golem_provider_id'),
+                    'provider_name': annotations.get('golem_provider_name'),
+                    'ip_address': '127.0.0.1' if 'DEVMODE' in annotations.get('golem_provider_name', '') else annotations.get('golem_ip_address'),
+                    'country': annotations.get('golem_country'),
+                    'resources': {
+                        'cpu': int(annotations.get('golem_cpu', 0)),
+                        'memory': int(annotations.get('golem_memory', 0)),
+                        'storage': int(annotations.get('golem_storage', 0)),
+                    },
+                    'created_at_block': metadata.expires_at_block - (config.advertisement_interval * 2)
+                }
+                if provider['provider_id']:
+                    providers.append(provider)
+
+            return providers
+        except Exception as e:
+            raise DiscoveryError(
+                f"Error finding providers on Golem Base: {str(e)}")
+
+    async def _find_providers_central(
+        self,
+        cpu: Optional[int] = None,
+        memory: Optional[int] = None,
+        storage: Optional[int] = None,
+        country: Optional[str] = None
+    ) -> List[Dict]:
+        """Find providers using the central discovery service."""
+        try:
+            # Build query parameters
+            params = {
+                k: v for k, v in {
+                    'cpu': cpu,
+                    'memory': memory,
+                    'storage': storage,
+                    'country': country
+                }.items() if v is not None
+            }
+
+            # Query discovery service
+            async with self.session.get(
+                f"{config.discovery_url}/api/v1/advertisements",
+                params=params
+            ) as response:
+                if not response.ok:
+                    raise DiscoveryError(
+                        f"Failed to query discovery service: {await response.text()}"
+                    )
+                providers = await response.json()
+
+            # Process provider IPs based on environment
+            for provider in providers:
+                provider['ip_address'] = (
+                    'localhost' if config.environment == "development"
+                    else provider.get('ip_address')
+                )
+
+            return providers
+
+        except aiohttp.ClientError as e:
+            raise DiscoveryError(
+                f"Failed to connect to discovery service: {str(e)}")
+        except Exception as e:
+            raise DiscoveryError(f"Error finding providers: {str(e)}")
+
+    async def verify_provider(self, provider_id: str) -> Dict:
+        """Verify provider exists and is available."""
+        try:
+            providers = await self.find_providers()
+            provider = next(
+                (p for p in providers if p['provider_id'] == provider_id),
+                None
+            )
+
+            if not provider:
+                raise ProviderError(f"Provider {provider_id} not found")
+
+            return provider
+
+        except Exception as e:
+            if isinstance(e, ProviderError):
+                raise
+            raise ProviderError(f"Failed to verify provider: {str(e)}")
+
+    async def get_provider_resources(self, provider_id: str) -> Dict:
+        """Get current resource availability for a provider."""
+        try:
+            provider = await self.verify_provider(provider_id)
+            return {
+                'cpu': provider['resources']['cpu'],
+                'memory': provider['resources']['memory'],
+                'storage': provider['resources']['storage']
+            }
+        except Exception as e:
+            raise ProviderError(f"Failed to get provider resources: {str(e)}")
+
+    async def check_resource_availability(
+        self,
+        provider_id: str,
+        cpu: int,
+        memory: int,
+        storage: int
+    ) -> bool:
+        """Check if provider has sufficient resources."""
+        try:
+            resources = await self.get_provider_resources(provider_id)
+
+            return (
+                resources['cpu'] >= cpu and
+                resources['memory'] >= memory and
+                resources['storage'] >= storage
+            )
+
+        except Exception as e:
+            raise ProviderError(
+                f"Failed to check resource availability: {str(e)}"
+            )
+
+    async def _format_block_timestamp(self, block_number: int) -> str:
+        """Format a block number into a human-readable 'time ago' string."""
+        if not self.golem_base_client:
+            return "N/A"
+        try:
+            latest_block = await self.golem_base_client.http_client().eth.get_block('latest')
+            block_diff = latest_block.number - block_number
+            seconds_ago = block_diff * 2  # Approximate block time
+            
+            if seconds_ago < 60:
+                return f"{int(seconds_ago)}s ago"
+            elif seconds_ago < 3600:
+                return f"{int(seconds_ago / 60)}m ago"
+            elif seconds_ago < 86400:
+                return f"{int(seconds_ago / 3600)}h ago"
+            else:
+                return f"{int(seconds_ago / 86400)}d ago"
+        except Exception:
+            return "N/A"
+
+    async def format_provider_row(self, provider: Dict, colorize: bool = False) -> List:
+        """Format provider information for display."""
+        from click import style
+
+        updated_at_str = await self._format_block_timestamp(provider.get('created_at_block', 0))
+
+        row = [
+            provider['provider_id'],
+            provider['provider_name'],
+            provider['ip_address'] or 'N/A',
+            provider['country'],
+            provider['resources']['cpu'],
+            provider['resources']['memory'],
+            provider['resources']['storage'],
+            updated_at_str
+        ]
+
+        if colorize:
+            # Format Provider ID
+            row[0] = style(row[0], fg="yellow")
+
+            # Format resources with icons and colors
+            row[4] = style(f"💻 {row[4]}", fg="cyan", bold=True)
+            row[5] = style(f"🧠 {row[5]}", fg="cyan", bold=True)
+            row[6] = style(f"💾 {row[6]}", fg="cyan", bold=True)
+
+            # Format location info
+            row[3] = style(f"🌍 {row[3]}", fg="green", bold=True)
+
+        return row
+
+    @property
+    def provider_headers(self) -> List[str]:
+        """Get headers for provider display."""
+        return [
+            "Provider ID",
+            "Name",
+            "IP Address",
+            "Country",
+            "CPU",
+            "Memory (GB)",
+            "Storage (GB)",
+            "Updated"
+        ]

requestor/services/ssh_service.py

@@ -0,0 +1,121 @@
+"""SSH connection service."""
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from ..ssh.manager import SSHKeyManager
+from ..errors import SSHError
+
+class SSHService:
+    """Service for handling SSH connections."""
+    
+    def __init__(self, ssh_key_dir: Path):
+        self.ssh_manager = SSHKeyManager(ssh_key_dir)
+
+    async def get_key_pair(self):
+        """Get or create SSH key pair."""
+        try:
+            return await self.ssh_manager.get_key_pair()
+        except Exception as e:
+            raise SSHError(f"Failed to get SSH key pair: {str(e)}")
+
+    def connect_to_vm(
+        self,
+        host: str,
+        port: int,
+        private_key_path: Path,
+        username: str = "ubuntu"
+    ) -> None:
+        """Connect to VM via SSH."""
+        try:
+            cmd = [
+                "ssh",
+                "-i", str(private_key_path),
+                "-p", str(port),
+                "-o", "StrictHostKeyChecking=no",
+                "-o", "UserKnownHostsFile=/dev/null",
+                f"{username}@{host}"
+            ]
+            subprocess.run(cmd)
+        except Exception as e:
+            raise SSHError(f"Failed to establish SSH connection: {str(e)}")
+
+    def format_ssh_command(
+        self,
+        host: str,
+        port: int,
+        private_key_path: Path,
+        username: str = "ubuntu",
+        colorize: bool = False
+    ) -> str:
+        """Format SSH command for display."""
+        from click import style
+
+        command = (
+            f"ssh -i {private_key_path} "
+            f"-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "
+            f"-p {port} {username}@{host}"
+        )
+
+        if colorize:
+            return style(command, fg="yellow")
+        return command
+
+    def get_vm_stats(
+        self,
+        host: str,
+        port: int,
+        private_key_path: Path,
+        username: str = "ubuntu"
+    ) -> Dict:
+        """Get VM stats via SSH."""
+        try:
+            cmd = [
+                "ssh",
+                "-i", str(private_key_path),
+                "-p", str(port),
+                "-o", "StrictHostKeyChecking=no",
+                "-o", "UserKnownHostsFile=/dev/null",
+                f"{username}@{host}",
+                "top -b -n 1; df -h /"
+            ]
+            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+            return self._parse_stats(result.stdout)
+        except subprocess.CalledProcessError as e:
+            raise SSHError(f"Failed to get VM stats: {e.stderr}")
+        except Exception as e:
+            raise SSHError(f"An unexpected error occurred: {str(e)}")
+
+    def _parse_stats(self, stats_output: str) -> Dict:
+        """Parse the output of the stats command."""
+        lines = stats_output.strip().split('\n')
+        stats = {'cpu': {}, 'memory': {}, 'disk': {}}
+
+        for line in lines:
+            if line.startswith('%Cpu(s):'):
+                parts = line.split(',')
+                for part in parts:
+                    if 'id' in part:
+                        idle_str = part.strip().split()[0]
+                        try:
+                            idle = float(idle_str)
+                            stats['cpu']['usage'] = f"{100.0 - idle:.1f}%"
+                        except ValueError:
+                            stats['cpu']['usage'] = "N/A"
+                        break
+            elif 'MiB Mem' in line:
+                parts = line.split(',')
+                total_mem_str = parts[0].split(':')[1].strip().split()[0]
+                used_mem_str = parts[2].strip().split()[0]
+                stats['memory'] = {
+                    'total': f"{float(total_mem_str):.1f} MiB",
+                    'used': f"{float(used_mem_str):.1f} MiB",
+                }
+            elif line.startswith('/dev/'):
+                parts = line.split()
+                stats['disk'] = {
+                    'total': parts[1],
+                    'used': parts[2],
+                }
+
+        return stats