unitlab 2.3.0__tar.gz → 2.3.3__tar.gz

{unitlab-2.3.0/src/unitlab.egg-info → unitlab-2.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: unitlab
-Version: 2.3.0
+Version: 2.3.3
 Home-page: https://github.com/teamunitlab/unitlab-sdk
 Author: Unitlab Inc.
 Author-email: team@unitlab.ai
@@ -21,11 +21,5 @@ Requires-Dist: requests
 Requires-Dist: tqdm
 Requires-Dist: typer
 Requires-Dist: validators
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: license-file
-Dynamic: requires-dist
+Requires-Dist: psutil
+Requires-Dist: pyyaml

{unitlab-2.3.0 → unitlab-2.3.3}/README.md

@@ -26,6 +26,35 @@ Once you have successfully installed the Unitlab package, you can conveniently h
 ## Quickstart 
 Follow [the quickstart guide for the Python SDK](https://docs.unitlab.ai/cli-python-sdk/unitlab-python-sdk).
 
+## CLI Commands
+
+### Agent Commands
+
+The agent module provides commands for running device agents with Jupyter, SSH tunnels, and metrics reporting.
+
+#### Run Agent
+
+Run a full device agent that sets up Jupyter notebooks, SSH tunnels, and system metrics reporting:
+
+```bash
+unitlab agent run --api-key YOUR_API_KEY [OPTIONS]
+```
+
+**Options:**
+- `--api-key` (required): Your Unitlab API key
+
+**Example:**
+```bash
+# Run with auto-generated device ID
+unitlab agent run  your-api-key-here
+
+
+
+The agent will:
+- Initialize Jupyter notebook server
+- Set up SSH tunnels for remote access
+- Collect and report system metrics
+- Handle graceful shutdown on interruption
 
 ## Documentation 
 [The documentation](https://docs.unitlab.ai/) provides comprehensive instructions on how to utilize the Unilab SDK effectively.

{unitlab-2.3.0 → unitlab-2.3.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="unitlab",
-    version="2.3.0",
+    version="2.3.3",
     license="MIT",
     author="Unitlab Inc.",
     author_email="team@unitlab.ai",
@@ -29,6 +29,8 @@ setup(
         "tqdm",
         "typer",
         "validators",
+        'psutil',
+        'pyyaml',
     ],
     entry_points={
         "console_scripts": ["unitlab=unitlab.main:app"],

unitlab-2.3.3/src/unitlab/client.py

@@ -0,0 +1,633 @@
+import asyncio
+import glob
+import logging
+import os
+import urllib.parse
+import aiofiles
+import aiohttp
+import requests
+import tqdm
+import socket
+import subprocess
+import signal
+import re
+import time
+import threading
+import psutil
+from datetime import datetime, timezone
+from .tunnel_config import CloudflareTunnel
+from .utils import get_api_url, handle_exceptions
+
+
+try:
+    import GPUtil
+    HAS_GPU = True
+except ImportError:
+    HAS_GPU = False
+
+
+logger = logging.getLogger(__name__)
+
+class UnitlabClient:
+    """A client with a connection to the Unitlab.ai platform.
+
+    Note:
+        Please refer to the `Python SDK quickstart <https://docs.unitlab.ai/cli-python-sdk/unitlab-python-sdk>`__ for a full example of working with the Python SDK.
+
+    First install the SDK.
+
+    .. code-block:: bash
+
+        pip install --upgrade unitlab
+
+    Import the ``unitlab`` package in your python file and set up a client with an API key. An API key can be created on <https://unitlab.ai/>`__.
+
+    .. code-block:: python
+
+        from unitlab import UnitlabClient
+        api_key = 'YOUR_API_KEY'
+        client = UnitlabClient(api_key)
+
+    Or store your Unitlab API key in your environment (``UNITLAB_API_KEY = 'YOUR_API_KEY'``):
+
+    .. code-block:: python
+
+        from unitlab import UnitlabClient
+        client = UnitlabClient()
+
+    Args:
+        api_key: Your Unitlab.ai API key. If no API key given, reads ``UNITLAB_API_KEY`` from the environment. Defaults to :obj:`None`.
+    Raises:
+        :exc:`~unitlab.exceptions.AuthenticationError`: If an invalid API key is used or (when not passing the API key directly) if ``UNITLAB_API_KEY`` is not found in your environment.
+    """
+
+    def __init__(self, api_key, api_url=None):
+        self.api_key = api_key
+        self.api_url = api_url or get_api_url()
+        self.api_session = requests.Session()
+        adapter = requests.adapters.HTTPAdapter(max_retries=3)
+        self.api_session.mount("http://", adapter)
+        self.api_session.mount("https://", adapter)
+        
+        # Device agent attributes (initialized when needed)
+        self.device_id = None
+        self.base_domain = None
+        self.server_url = None
+        self.hostname = socket.gethostname()
+        self.tunnel_manager = None
+        self.jupyter_url = None
+        self.ssh_url = None
+        self.jupyter_proc = None
+        self.tunnel_proc = None
+        self.jupyter_port = None
+        self.running = True
+        self.metrics_thread = None
+
+    def close(self) -> None:
+        """Close :class:`UnitlabClient` connections.
+
+        You can manually close the Unitlab client's connections:
+
+        .. code-block:: python
+
+            client = UnitlabClient()
+            client.projects()
+            client.close()
+
+        Or use the client as a context manager:
+
+        .. code-block:: python
+
+            with UnitlabClient() as client:
+                client.projects()
+        """
+        self.api_session.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(
+        self,
+        exc_type,
+        exc_value,
+        traceback,
+    ) -> None:
+        self.close()
+
+    def _get_headers(self):
+        return {"Authorization": f"Api-Key {self.api_key}"}
+
+    @handle_exceptions
+    def _get(self, endpoint):
+        return self.api_session.get(
+            urllib.parse.urljoin(self.api_url, endpoint), headers=self._get_headers()
+        )
+
+    @handle_exceptions
+    def _post(self, endpoint, data=None):
+        return self.api_session.post(
+            urllib.parse.urljoin(self.api_url, endpoint),
+            json=data or {},
+            headers=self._get_headers(),
+        )
+
+    def projects(self, pretty=0):
+        return self._get(f"/api/sdk/projects/?pretty={pretty}")
+
+    def project(self, project_id, pretty=0):
+        return self._get(f"/api/sdk/projects/{project_id}/?pretty={pretty}")
+
+    def project_members(self, project_id, pretty=0):
+        return self._get(f"/api/sdk/projects/{project_id}/members/?pretty={pretty}")
+
+    def project_upload_data(self, project_id, directory, batch_size=100):
+        if not os.path.isdir(directory):
+            raise ValueError(f"Directory {directory} does not exist")
+
+        files = [
+            file
+            for files_list in (
+                glob.glob(os.path.join(directory, "") + extension)
+                for extension in ["*jpg", "*png", "*jpeg", "*webp"]
+            )
+            for file in files_list
+        ]
+        filtered_files = []
+        for file in files:
+            file_size = os.path.getsize(file) / 1024 / 1024
+            if file_size > 6:
+                logger.warning(
+                    f"File {file} is too large ({file_size:.4f} megabytes) skipping, max size is 6 MB"
+                )
+                continue
+            filtered_files.append(file)
+
+        num_files = len(filtered_files)
+        num_batches = (num_files + batch_size - 1) // batch_size
+
+        async def post_file(session: aiohttp.ClientSession, file: str, project_id: str):
+            async with aiofiles.open(file, "rb") as f:
+                form_data = aiohttp.FormData()
+                form_data.add_field("project", project_id)
+                form_data.add_field(
+                    "file", await f.read(), filename=os.path.basename(file)
+                )
+                try:
+                    await asyncio.sleep(0.1)
+                    async with session.post(
+                        urllib.parse.urljoin(self.api_url, "/api/sdk/upload-data/"),
+                        data=form_data,
+                    ) as response:
+                        response.raise_for_status()
+                        return 1
+                except Exception as e:
+                    logger.error(f"Error uploading file {file} - {e}")
+                    return 0
+
+        async def main():
+            logger.info(f"Uploading {num_files} files to project {project_id}")
+            with tqdm.tqdm(total=num_files, ncols=80) as pbar:
+                async with aiohttp.ClientSession(
+                    headers=self._get_headers()
+                ) as session:
+                    for i in range(num_batches):
+                        tasks = []
+                        for file in filtered_files[
+                            i * batch_size : min((i + 1) * batch_size, num_files)
+                        ]:
+                            tasks.append(
+                                post_file(
+                                    session=session, file=file, project_id=project_id
+                                )
+                            )
+                        for f in asyncio.as_completed(tasks):
+                            pbar.update(await f)
+
+        asyncio.run(main())
+
+    def datasets(self, pretty=0):
+        return self._get(f"/api/sdk/datasets/?pretty={pretty}")
+
+    def dataset_download(self, dataset_id, export_type):
+        response = self._post(
+            f"/api/sdk/datasets/{dataset_id}/",
+            data={"download_type": "annotation", "export_type": export_type},
+        )
+
+        with self.api_session.get(url=response["file"], stream=True) as r:
+            r.raise_for_status()
+            filename = f"dataset-{dataset_id}.json"
+            with open(filename, "wb") as f:
+                for chunk in r.iter_content(chunk_size=1024 * 1024):
+                    f.write(chunk)
+            logger.info(f"File: {os.path.abspath(filename)}")
+            return os.path.abspath(filename)
+
+    def dataset_download_files(self, dataset_id):
+        response = self._post(
+            f"/api/sdk/datasets/{dataset_id}/", data={"download_type": "files"}
+        )
+        folder = f"dataset-files-{dataset_id}"
+        os.makedirs(folder, exist_ok=True)
+        dataset_files = [
+            dataset_file
+            for dataset_file in response
+            if not os.path.isfile(os.path.join(folder, dataset_file["file_name"]))
+        ]
+
+        async def download_file(session: aiohttp.ClientSession, dataset_file: dict):
+            async with session.get(url=dataset_file["source"]) as r:
+                try:
+                    r.raise_for_status()
+                except Exception as e:
+                    logger.error(
+                        f"Error downloading file {dataset_file['file_name']} - {e}"
+                    )
+                    return 0
+                async with aiofiles.open(
+                    os.path.join(folder, dataset_file["file_name"]), "wb"
+                ) as f:
+                    async for chunk in r.content.iter_any():
+                        await f.write(chunk)
+                    return 1
+
+        async def main():
+            with tqdm.tqdm(total=len(dataset_files), ncols=80) as pbar:
+                async with aiohttp.ClientSession() as session:
+                    tasks = [
+                        download_file(session=session, dataset_file=dataset_file)
+                        for dataset_file in dataset_files
+                    ]
+                    for f in asyncio.as_completed(tasks):
+                        pbar.update(await f)
+
+        asyncio.run(main())
+    
+    def initialize_device_agent(self, server_url: str, device_id: str, base_domain: str):
+        """Initialize device agent configuration"""
+        self.server_url = server_url.rstrip('/')
+        self.device_id = device_id
+        self.base_domain = base_domain
+        
+        # Initialize tunnel manager if available
+        if CloudflareTunnel:
+            self.tunnel_manager = CloudflareTunnel(base_domain, device_id)
+            self.jupyter_url = self.tunnel_manager.jupyter_url
+            self.ssh_url = self.tunnel_manager.ssh_url
+        else:
+            self.tunnel_manager = None
+            self.jupyter_url = f"https://jupyter-{device_id}.{base_domain}"
+            self.ssh_url = f"https://ssh-{device_id}.{base_domain}"
+        
+        # Setup signal handlers
+        signal.signal(signal.SIGINT, self._handle_shutdown)
+        signal.signal(signal.SIGTERM, self._handle_shutdown)
+    
+    def _handle_shutdown(self, signum, frame):
+        """Handle shutdown signals"""
+        _ = frame  # Unused but required by signal handler signature
+        logger.info(f"Received signal {signum}, shutting down...")
+        self.running = False
+    
+    def _get_device_headers(self):
+        """Get headers for device agent API requests"""
+        headers = {
+            'Content-Type': 'application/json',
+            'User-Agent': f'UnitlabDeviceAgent/{self.device_id}'
+        }
+        
+        # Add API key if provided
+        if self.api_key:
+            headers['Authorization'] = f'Api-Key {self.api_key}'
+        
+        return headers
+    
+    def _post_device(self, endpoint, data=None):
+        """Make authenticated POST request for device agent"""
+        full_url = urllib.parse.urljoin(self.server_url, endpoint)
+        logger.debug(f"Posting to {full_url} with data: {data}")
+        
+        try:
+            response = self.api_session.post(
+                full_url,
+                json=data or {},
+                headers=self._get_device_headers(),
+            )
+            logger.debug(f"Response status: {response.status_code}, Response: {response.text}")
+            response.raise_for_status()
+            return response
+        except Exception as e:
+            logger.error(f"POST request failed to {full_url}: {e}")
+            raise
+    
+    def start_jupyter(self) -> bool:
+        """Start Jupyter notebook server"""
+        try:
+            logger.info("Starting Jupyter notebook...")
+            
+            cmd = [
+                "jupyter", "notebook",
+                "--no-browser",
+                "--ServerApp.token=''",
+                "--ServerApp.password=''",
+                "--ServerApp.allow_origin='*'",
+                "--ServerApp.ip='0.0.0.0'"
+            ]
+            
+            self.jupyter_proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True
+            )
+            
+            # Wait for Jupyter to start and get the port
+            start_time = time.time()
+            while time.time() - start_time < 30:
+                line = self.jupyter_proc.stdout.readline()
+                if not line:
+                    break
+                
+                # Look for the port in the output
+                match = re.search(r'http://.*:(\d+)/', line)
+                if match:
+                    self.jupyter_port = match.group(1)
+                    logger.info(f"✅ Jupyter started on port {self.jupyter_port}")
+                    return True
+            
+            raise Exception("Timeout waiting for Jupyter to start")
+            
+        except Exception as e:
+            logger.error(f"Failed to start Jupyter: {e}")
+            if self.jupyter_proc:
+                self.jupyter_proc.terminate()
+                self.jupyter_proc = None
+            return False
+    
+    def setup_tunnels(self) -> bool:
+        """Setup Cloudflare tunnels"""
+        try:
+            if not self.jupyter_port:
+                logger.error("Jupyter port not available")
+                return False
+            
+            if not self.tunnel_manager:
+                logger.warning("CloudflareTunnel not available, skipping tunnel setup")
+                return True
+            
+            logger.info("Setting up Cloudflare tunnels...")
+            self.tunnel_proc = self.tunnel_manager.setup(self.jupyter_port)
+            
+            if self.tunnel_proc:
+                logger.info("✅ Tunnels established")
+                self.report_services()
+                return True
+            
+            return False
+            
+        except Exception as e:
+            logger.error(f"Tunnel setup failed: {e}")
+            return False
+    
+    def check_ssh(self) -> bool:
+        """Check if SSH service is available"""
+        try:
+            # Check if SSH is running
+            result = subprocess.run(
+                ["systemctl", "is-active", "ssh"],
+                capture_output=True,
+                text=True,
+                timeout=5
+            )
+            
+            if result.stdout.strip() == "active":
+                logger.info("✅ SSH service is active")
+                return True
+            else:
+                logger.warning("SSH service is not active")
+                # Try to start SSH
+                subprocess.run(["sudo", "systemctl", "start", "ssh"], timeout=10)
+                time.sleep(2)
+                return False
+                
+        except Exception as e:
+            logger.error(f"SSH check failed: {e}")
+            return False
+    
+    def report_services(self):
+        """Report services to the server"""
+        try:
+            # Report Jupyter service
+            jupyter_data = {
+                'service_type': 'jupyter',
+                'service_name': f'jupyter-{self.device_id}',
+                'local_port': int(self.jupyter_port) if self.jupyter_port else 8888,
+                'tunnel_url': self.jupyter_url,
+                'status': 'online'
+            }
+            
+            logger.info(f"Reporting Jupyter service with URL: {self.jupyter_url}")
+            jupyter_response = self._post_device(
+                f"/api/tunnel/agent/jupyter/{self.device_id}/", 
+                jupyter_data
+            )
+            logger.info(f"Reported Jupyter service: {jupyter_response.status_code if hasattr(jupyter_response, 'status_code') else jupyter_response}")
+            
+            # Report SSH service (always report, even if SSH is not running locally)
+            # Remove https:// prefix for SSH hostname
+            ssh_hostname = self.ssh_url.replace('https://', '')
+            
+            # Get current system username
+            import getpass
+            current_user = getpass.getuser()
+            
+            # Create SSH connection command
+            ssh_connection_cmd = f"ssh -o ProxyCommand='cloudflared access ssh --hostname {ssh_hostname}' {current_user}@{ssh_hostname}"
+            
+            # Check if SSH is available
+            ssh_available = self.check_ssh()
+            
+            ssh_data = {
+                'service_type': 'ssh',
+                'service_name': f'ssh-{self.device_id}',
+                'local_port': 22,
+                'tunnel_url': ssh_connection_cmd,  # Send the SSH command instead of URL
+                'status': 'online' if ssh_available else 'offline'
+            }
+            
+            logger.info(f"Reporting SSH service with command: {ssh_connection_cmd}")
+            ssh_response = self._post_device(
+                f"/api/tunnel/agent/ssh/{self.device_id}/",
+                ssh_data
+            )
+            logger.info(f"Reported SSH service: {ssh_response.status_code if hasattr(ssh_response, 'status_code') else ssh_response}")
+            
+        except Exception as e:
+            logger.error(f"Failed to report services: {e}", exc_info=True)
+    
+    def collect_metrics(self) -> dict:
+        """Collect system metrics"""
+        metrics = {}
+        
+        # CPU metrics
+        metrics['cpu'] = {
+            'percent': psutil.cpu_percent(interval=1),
+            'count': psutil.cpu_count(),
+            'timestamp': datetime.now(timezone.utc).isoformat()
+        }
+        
+        # Memory metrics
+        mem = psutil.virtual_memory()
+        metrics['ram'] = {
+            'total': mem.total,
+            'used': mem.used,
+            'available': mem.available,
+            'percent': mem.percent,
+            'timestamp': datetime.now(timezone.utc).isoformat()
+        }
+        
+        # GPU metrics (if available)
+        if HAS_GPU:
+            try:
+                gpus = GPUtil.getGPUs()
+                if gpus:
+                    gpu = gpus[0]
+                    metrics['gpu'] = {
+                        'name': gpu.name,
+                        'load': gpu.load * 100,
+                        'memory_used': gpu.memoryUsed,
+                        'memory_total': gpu.memoryTotal,
+                        'temperature': gpu.temperature,
+                        'timestamp': datetime.now(timezone.utc).isoformat()
+                    }
+            except Exception as e:
+                logger.debug(f"GPU metrics unavailable: {e}")
+        
+        return metrics
+    
+    def send_metrics(self):
+        """Send metrics to server"""
+        try:
+            metrics = self.collect_metrics()
+            
+            # Send CPU metrics
+            if 'cpu' in metrics:
+                self._post_device(f"/api/tunnel/agent/cpu/{self.device_id}/", metrics['cpu'])
+            
+            # Send RAM metrics  
+            if 'ram' in metrics:
+                self._post_device(f"/api/tunnel/agent/ram/{self.device_id}/", metrics['ram'])
+            
+            # Send GPU metrics if available
+            if 'gpu' in metrics and metrics['gpu']:
+                self._post_device(f"/api/tunnel/agent/gpu/{self.device_id}/", metrics['gpu'])
+            
+            logger.debug(f"Metrics sent - CPU: {metrics['cpu']['percent']:.1f}%, RAM: {metrics['ram']['percent']:.1f}%")
+            
+        except Exception as e:
+            logger.error(f"Failed to send metrics: {e}")
+    
+    def metrics_loop(self):
+        """Background thread for sending metrics"""
+        logger.info("Starting metrics thread")
+        
+        while self.running:
+            try:
+                self.send_metrics()
+                
+                # Check if processes are still running
+                if self.jupyter_proc and self.jupyter_proc.poll() is not None:
+                    logger.warning("Jupyter process died")
+                    self.jupyter_proc = None
+                
+                if self.tunnel_proc and self.tunnel_proc.poll() is not None:
+                    logger.warning("Tunnel process died")
+                    self.tunnel_proc = None
+                
+            except Exception as e:
+                logger.error(f"Metrics loop error: {e}")
+            
+            # Wait for next interval (default 5 seconds)
+            for _ in range(3):
+                if not self.running:
+                    break
+                time.sleep(1)
+        
+        logger.info("Metrics thread stopped")
+    
+    def run_device_agent(self):
+        """Main run method for device agent"""
+        logger.info("=" * 50)
+        logger.info("Starting Device Agent")
+        logger.info(f"Device ID: {self.device_id}")
+        logger.info(f"Server: {self.server_url}")
+        logger.info(f"Domain: {self.base_domain}")
+        logger.info("=" * 50)
+        
+        # Check SSH
+        self.check_ssh()
+        
+        # Start Jupyter
+        if not self.start_jupyter():
+            logger.error("Failed to start Jupyter")
+            return
+        
+        # Wait a moment for Jupyter to fully initialize
+        time.sleep(1)
+        
+        # Setup tunnels
+        if not self.setup_tunnels():
+            logger.error("Failed to setup tunnels")
+            self.cleanup_device_agent()
+            return
+        
+        # Print access information
+        logger.info("=" * 50)
+        logger.info("🎉 All services started successfully!")
+        logger.info(f"📔 Jupyter: {self.jupyter_url}")
+        logger.info(f"🔐 SSH: {self.ssh_url}")
+        # Remove https:// prefix for SSH command display
+        ssh_hostname = self.ssh_url.replace('https://', '')
+        import getpass
+        current_user = getpass.getuser()
+        logger.info(f"🔐 SSH Command: ssh -o ProxyCommand='cloudflared access ssh --hostname {ssh_hostname}' {current_user}@{ssh_hostname}")
+        logger.info("=" * 50)
+        
+        # Start metrics thread
+        self.metrics_thread = threading.Thread(target=self.metrics_loop, daemon=True)
+        self.metrics_thread.start()
+        
+        # Main loop
+        try:
+            while self.running:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            logger.info("Interrupted by user")
+        
+        self.cleanup_device_agent()
+    
+    def cleanup_device_agent(self):
+        """Clean up device agent resources"""
+        logger.info("Cleaning up...")
+        
+        self.running = False
+        
+        # Stop Jupyter
+        if self.jupyter_proc:
+            logger.info("Stopping Jupyter...")
+            self.jupyter_proc.terminate()
+            try:
+                self.jupyter_proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self.jupyter_proc.kill()
+        
+        # Stop tunnel
+        if self.tunnel_proc:
+            logger.info("Stopping tunnel...")
+            self.tunnel_proc.terminate()
+            try:
+                self.tunnel_proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self.tunnel_proc.kill()
+        
+        logger.info("Cleanup complete")

{unitlab-2.3.0 → unitlab-2.3.3}/src/unitlab/main.py

@@ -1,6 +1,8 @@
 from enum import Enum
 from pathlib import Path
 from uuid import UUID
+import logging
+import os
 
 import typer
 import validators
@@ -9,12 +11,16 @@ from typing_extensions import Annotated
 from . import utils
 from .client import UnitlabClient
 
+
 app = typer.Typer()
 project_app = typer.Typer()
 dataset_app = typer.Typer()
+agent_app = typer.Typer()
+
 
 app.add_typer(project_app, name="project", help="Project commands")
 app.add_typer(dataset_app, name="dataset", help="Dataset commands")
+app.add_typer(agent_app, name="agent", help="Agent commands")
 
 
 API_KEY = Annotated[
@@ -42,7 +48,7 @@ class AnnotationType(str, Enum):
 @app.command(help="Configure the credentials")
 def configure(
     api_key: Annotated[str, typer.Option(help="The api-key obtained from unitlab.ai")],
-    api_url: Annotated[str, typer.Option()] = "https://api.unitlab.ai",
+    api_url: Annotated[str, typer.Option()] = "https://localhost/",
 ):
     if not validators.url(api_url, simple_host=True):
         raise typer.BadParameter("Invalid api url")
@@ -105,5 +111,65 @@ def dataset_download(
     get_client(api_key).dataset_download_files(pk)
 
 
+def send_metrics_to_server(server_url: str, device_id: str, metrics: dict):
+    """Standalone function to send metrics to server using client"""
+    client = UnitlabClient(api_key="dummy")  # API key not needed for metrics
+    return client.send_metrics_to_server(server_url, device_id, metrics)
+
+
+def send_metrics_into_server():
+    """Standalone function to collect system metrics using client"""
+    client = UnitlabClient(api_key="dummy")  # API key not needed for metrics
+    return client.collect_system_metrics()
+
+
+@agent_app.command(name="run", help="Run the device agent with Jupyter, SSH tunnels and metrics")
+def run_agent(
+    api_key: str,
+    device_id: Annotated[str, typer.Option(help="Device ID")] = None,
+    base_domain: Annotated[str, typer.Option(help="Base domain for tunnels")] = "1scan.uz",
+  
+):
+    """Run the full device agent with Jupyter, SSH tunnels and metrics reporting"""
+    
+    # Setup logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        handlers=[logging.StreamHandler()]
+    )
+
+    # Get server URL from environment or use default
+    server_url = 'https://api-dev.unitlab.ai/'
+    
+    # Generate unique device ID if not provided
+    if not device_id:
+        import uuid
+        import platform
+        # Try environment variable first
+        device_id = os.getenv('DEVICE_ID')
+        if not device_id:
+            # Generate a unique ID based on hostname and random UUID
+            hostname = platform.node().replace('.', '-').replace(' ', '-')[:20]
+            random_suffix = str(uuid.uuid4())[:8]
+            device_id = f"{hostname}-{random_suffix}"
+          
+    
+    # Create client and initialize device agent
+    client = UnitlabClient(api_key=api_key)
+    client.initialize_device_agent(
+        server_url=server_url,
+        device_id=device_id,
+        base_domain=base_domain
+    )
+    
+    try:
+        client.run_device_agent()
+    except Exception as e:
+        logging.error(f"Fatal error: {e}")
+        client.cleanup_device_agent()
+        raise typer.Exit(1)
+
+
 if __name__ == "__main__":
     app()

unitlab-2.3.3/src/unitlab/tunnel_config.py

@@ -0,0 +1,238 @@
+"""
+Cloudflare Tunnel Configuration for persistent subdomains
+"""
+
+import json
+import subprocess
+import socket
+import time
+import yaml
+from pathlib import Path
+
+
+class CloudflareTunnel:
+    def __init__(self, base_domain, device_id):
+        # Hardcode the base domain here
+        self.base_domain = "1scan.uz"  # HARDCODED - ignore the passed base_domain
+        self.device_id = device_id
+        self.hostname = socket.gethostname()
+        self.tunnel_name = f"device-{device_id}"
+        self.config_dir = Path.home() / ".cloudflared"
+        self.config_dir.mkdir(exist_ok=True)
+        
+        # Subdomain names
+        self.jupyter_subdomain = f"jupyter-{device_id}"
+        self.ssh_subdomain = f"ssh-{device_id}"
+        
+        # Full URLs - using hardcoded base_domain
+        self.jupyter_url = f"https://{self.jupyter_subdomain}.{self.base_domain}"
+        self.ssh_url = f"https://{self.ssh_subdomain}.{self.base_domain}"
+        
+        self.tunnel_uuid = None
+        self.credentials_file = None
+
+    def login(self):
+        """Login to Cloudflare (one-time setup)"""
+        try:
+            print("🔐 Checking Cloudflare authentication...")
+            result = subprocess.run(
+                ["cloudflared", "tunnel", "login"],
+                capture_output=True,
+                text=True
+            )
+            if result.returncode == 0:
+                print("✅ Cloudflare authentication successful")
+                return True
+            else:
+                print("❌ Cloudflare authentication failed")
+                return False
+        except Exception as e:
+            print(f"❌ Error during Cloudflare login: {e}")
+            return False
+
+    def create_tunnel(self):
+        """Create a named tunnel"""
+        try:
+            print(f"🚇 Creating tunnel: {self.tunnel_name}")
+            
+            # Check if tunnel already exists
+            list_result = subprocess.run(
+                ["cloudflared", "tunnel", "list", "--output", "json"],
+                capture_output=True,
+                text=True
+            )
+            
+            if list_result.returncode == 0:
+                tunnels = json.loads(list_result.stdout)
+                for tunnel in tunnels:
+                    if tunnel.get("name") == self.tunnel_name:
+                        self.tunnel_uuid = tunnel.get("id")
+                        print(f"✅ Tunnel already exists with ID: {self.tunnel_uuid}")
+                        self.credentials_file = self.config_dir / f"{self.tunnel_uuid}.json"
+                        return True
+            
+            # Create new tunnel
+            result = subprocess.run(
+                ["cloudflared", "tunnel", "create", self.tunnel_name],
+                capture_output=True,
+                text=True
+            )
+            
+            if result.returncode == 0:
+                for line in result.stdout.split('\n'):
+                    if "Created tunnel" in line and "with id" in line:
+                        self.tunnel_uuid = line.split("with id")[1].strip()
+                        break
+                
+                if not self.tunnel_uuid:
+                    list_result = subprocess.run(
+                        ["cloudflared", "tunnel", "list", "--output", "json"],
+                        capture_output=True,
+                        text=True
+                    )
+                    if list_result.returncode == 0:
+                        tunnels = json.loads(list_result.stdout)
+                        for tunnel in tunnels:
+                            if tunnel.get("name") == self.tunnel_name:
+                                self.tunnel_uuid = tunnel.get("id")
+                                break
+                
+                if self.tunnel_uuid:
+                    self.credentials_file = self.config_dir / f"{self.tunnel_uuid}.json"
+                    print(f"✅ Tunnel created with ID: {self.tunnel_uuid}")
+                    return True
+                    
+            print(f"❌ Failed to create tunnel: {result.stderr}")
+            return False
+            
+        except Exception as e:
+            print(f"❌ Error creating tunnel: {e}")
+            return False
+
+    def configure_dns(self):
+        """Configure DNS routes for the tunnel"""
+        try:
+            print("🌐 Configuring DNS routes...")
+            
+            # Route for Jupyter
+            jupyter_result = subprocess.run(
+                ["cloudflared", "tunnel", "route", "dns", 
+                 self.tunnel_name, f"{self.jupyter_subdomain}.{self.base_domain}"],
+                capture_output=True,
+                text=True
+            )
+            
+            if jupyter_result.returncode == 0:
+                print(f"✅ Jupyter route configured: {self.jupyter_url}")
+            else:
+                print(f"⚠️  Jupyter route may already exist or failed: {jupyter_result.stderr}")
+            
+            # Route for SSH
+            ssh_result = subprocess.run(
+                ["cloudflared", "tunnel", "route", "dns",
+                 self.tunnel_name, f"{self.ssh_subdomain}.{self.base_domain}"],
+                capture_output=True,
+                text=True
+            )
+            
+            if ssh_result.returncode == 0:
+                print(f"✅ SSH route configured: {self.ssh_url}")
+            else:
+                print(f"⚠️  SSH route may already exist or failed: {ssh_result.stderr}")
+                
+            return True
+            
+        except Exception as e:
+            print(f"❌ Error configuring DNS: {e}")
+            return False
+
+    def create_config_file(self, jupyter_port):
+        """Create tunnel configuration file"""
+        config = {
+            "tunnel": self.tunnel_uuid,
+            "credentials-file": str(self.credentials_file),
+            "ingress": [
+                {
+                    "hostname": f"{self.jupyter_subdomain}.{self.base_domain}",
+                    "service": f"http://localhost:{jupyter_port}",
+                    "originRequest": {
+                        "noTLSVerify": True
+                    }
+                },
+                {
+                    "hostname": f"{self.ssh_subdomain}.{self.base_domain}",
+                    "service": "ssh://localhost:22",
+                    "originRequest": {
+                        "noTLSVerify": True
+                    }
+                },
+                {
+                    "service": "http_status:404"
+                }
+            ]
+        }
+        
+        config_file = self.config_dir / f"config-{self.device_id}.yml"
+        with open(config_file, 'w') as f:
+            yaml.dump(config, f, default_flow_style=False)
+        
+        print(f"📝 Configuration saved to: {config_file}")
+        return config_file
+
+    def start_tunnel(self, config_file):
+        """Start the tunnel with the configuration"""
+        try:
+            print("🚀 Starting Cloudflare tunnel...")
+            
+            cmd = ["cloudflared", "tunnel", "--config", str(config_file), "run"]
+            
+            process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                bufsize=1
+            )
+            
+            # Wait for tunnel to establish
+            time.sleep(5)
+            
+            if process.poll() is None:
+                print("✅ Tunnel is running")
+                return process
+            else:
+                print("❌ Tunnel failed to start")
+                return None
+                
+        except Exception as e:
+            print(f"❌ Error starting tunnel: {e}")
+            return None
+
+    def setup(self, jupyter_port):
+        """Complete setup process"""
+        # Check if we need to login
+        if not (self.config_dir / "cert.pem").exists():
+            if not self.login():
+                return None
+        
+        # Create tunnel
+        if not self.create_tunnel():
+            return None
+        
+        # Configure DNS
+        if not self.configure_dns():
+            return None
+        
+        # Create config file
+        config_file = self.create_config_file(jupyter_port)
+        
+        # Start tunnel
+        tunnel_process = self.start_tunnel(config_file)
+        
+        if tunnel_process:
+            print("\n✅ Tunnel setup complete!")
+            print(f"📌 Jupyter URL: {self.jupyter_url}")
+            print(f"📌 SSH URL: {self.ssh_url}")
+            return tunnel_process
+        
+        return None

{unitlab-2.3.0 → unitlab-2.3.3}/src/unitlab/utils.py

@@ -1,6 +1,8 @@
 import logging
 from configparser import ConfigParser
 from pathlib import Path
+import logging
+import requests
 
 import requests
 
@@ -63,3 +65,4 @@ def get_api_url() -> str:
         return config.get("default", "api_url")
     except Exception:
         return "https://api.unitlab.ai"
+

{unitlab-2.3.0 → unitlab-2.3.3/src/unitlab.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: unitlab
-Version: 2.3.0
+Version: 2.3.3
 Home-page: https://github.com/teamunitlab/unitlab-sdk
 Author: Unitlab Inc.
 Author-email: team@unitlab.ai
@@ -21,11 +21,5 @@ Requires-Dist: requests
 Requires-Dist: tqdm
 Requires-Dist: typer
 Requires-Dist: validators
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: license-file
-Dynamic: requires-dist
+Requires-Dist: psutil
+Requires-Dist: pyyaml

{unitlab-2.3.0 → unitlab-2.3.3}/src/unitlab.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ src/unitlab/__main__.py
 src/unitlab/client.py
 src/unitlab/exceptions.py
 src/unitlab/main.py
+src/unitlab/tunnel_config.py
 src/unitlab/utils.py
 src/unitlab.egg-info/PKG-INFO
 src/unitlab.egg-info/SOURCES.txt

{unitlab-2.3.0 → unitlab-2.3.3}/src/unitlab.egg-info/requires.txt

@@ -4,3 +4,5 @@ requests
 tqdm
 typer
 validators
+psutil
+pyyaml

unitlab-2.3.0/src/unitlab/client.py

@@ -1,236 +0,0 @@
-import asyncio
-import glob
-import logging
-import os
-import urllib.parse
-
-import aiofiles
-import aiohttp
-import requests
-import tqdm
-
-from .utils import get_api_url, handle_exceptions
-
-logger = logging.getLogger(__name__)
-
-
-class UnitlabClient:
-    """A client with a connection to the Unitlab.ai platform.
-
-    Note:
-        Please refer to the `Python SDK quickstart <https://docs.unitlab.ai/cli-python-sdk/unitlab-python-sdk>`__ for a full example of working with the Python SDK.
-
-    First install the SDK.
-
-    .. code-block:: bash
-
-        pip install --upgrade unitlab
-
-    Import the ``unitlab`` package in your python file and set up a client with an API key. An API key can be created on <https://unitlab.ai/>`__.
-
-    .. code-block:: python
-
-        from unitlab import UnitlabClient
-        api_key = 'YOUR_API_KEY'
-        client = UnitlabClient(api_key)
-
-    Or store your Unitlab API key in your environment (``UNITLAB_API_KEY = 'YOUR_API_KEY'``):
-
-    .. code-block:: python
-
-        from unitlab import UnitlabClient
-        client = UnitlabClient()
-
-    Args:
-        api_key: Your Unitlab.ai API key. If no API key given, reads ``UNITLAB_API_KEY`` from the environment. Defaults to :obj:`None`.
-    Raises:
-        :exc:`~unitlab.exceptions.AuthenticationError`: If an invalid API key is used or (when not passing the API key directly) if ``UNITLAB_API_KEY`` is not found in your environment.
-    """
-
-    def __init__(self, api_key, api_url=None):
-        self.api_key = api_key
-        self.api_url = api_url or get_api_url()
-        self.api_session = requests.Session()
-        adapter = requests.adapters.HTTPAdapter(max_retries=3)
-        self.api_session.mount("http://", adapter)
-        self.api_session.mount("https://", adapter)
-
-    def close(self) -> None:
-        """Close :class:`UnitlabClient` connections.
-
-        You can manually close the Unitlab client's connections:
-
-        .. code-block:: python
-
-            client = UnitlabClient()
-            client.projects()
-            client.close()
-
-        Or use the client as a context manager:
-
-        .. code-block:: python
-
-            with UnitlabClient() as client:
-                client.projects()
-        """
-        self.api_session.close()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(
-        self,
-        exc_type,
-        exc_value,
-        traceback,
-    ) -> None:
-        self.close()
-
-    def _get_headers(self):
-        return {"Authorization": f"Api-Key {self.api_key}"}
-
-    @handle_exceptions
-    def _get(self, endpoint):
-        return self.api_session.get(
-            urllib.parse.urljoin(self.api_url, endpoint), headers=self._get_headers()
-        )
-
-    @handle_exceptions
-    def _post(self, endpoint, data=None):
-        return self.api_session.post(
-            urllib.parse.urljoin(self.api_url, endpoint),
-            json=data or {},
-            headers=self._get_headers(),
-        )
-
-    def projects(self, pretty=0):
-        return self._get(f"/api/sdk/projects/?pretty={pretty}")
-
-    def project(self, project_id, pretty=0):
-        return self._get(f"/api/sdk/projects/{project_id}/?pretty={pretty}")
-
-    def project_members(self, project_id, pretty=0):
-        return self._get(f"/api/sdk/projects/{project_id}/members/?pretty={pretty}")
-
-    def project_upload_data(self, project_id, directory, batch_size=100):
-        if not os.path.isdir(directory):
-            raise ValueError(f"Directory {directory} does not exist")
-
-        files = [
-            file
-            for files_list in (
-                glob.glob(os.path.join(directory, "") + extension)
-                for extension in ["*jpg", "*png", "*jpeg", "*webp"]
-            )
-            for file in files_list
-        ]
-        filtered_files = []
-        for file in files:
-            file_size = os.path.getsize(file) / 1024 / 1024
-            if file_size > 6:
-                logger.warning(
-                    f"File {file} is too large ({file_size:.4f} megabytes) skipping, max size is 6 MB"
-                )
-                continue
-            filtered_files.append(file)
-
-        num_files = len(filtered_files)
-        num_batches = (num_files + batch_size - 1) // batch_size
-
-        async def post_file(session: aiohttp.ClientSession, file: str, project_id: str):
-            async with aiofiles.open(file, "rb") as f:
-                form_data = aiohttp.FormData()
-                form_data.add_field("project", project_id)
-                form_data.add_field(
-                    "file", await f.read(), filename=os.path.basename(file)
-                )
-                try:
-                    await asyncio.sleep(0.1)
-                    async with session.post(
-                        urllib.parse.urljoin(self.api_url, "/api/sdk/upload-data/"),
-                        data=form_data,
-                    ) as response:
-                        response.raise_for_status()
-                        return 1
-                except Exception as e:
-                    logger.error(f"Error uploading file {file} - {e}")
-                    return 0
-
-        async def main():
-            logger.info(f"Uploading {num_files} files to project {project_id}")
-            with tqdm.tqdm(total=num_files, ncols=80) as pbar:
-                async with aiohttp.ClientSession(
-                    headers=self._get_headers()
-                ) as session:
-                    for i in range(num_batches):
-                        tasks = []
-                        for file in filtered_files[
-                            i * batch_size : min((i + 1) * batch_size, num_files)
-                        ]:
-                            tasks.append(
-                                post_file(
-                                    session=session, file=file, project_id=project_id
-                                )
-                            )
-                        for f in asyncio.as_completed(tasks):
-                            pbar.update(await f)
-
-        asyncio.run(main())
-
-    def datasets(self, pretty=0):
-        return self._get(f"/api/sdk/datasets/?pretty={pretty}")
-
-    def dataset_download(self, dataset_id, export_type):
-        response = self._post(
-            f"/api/sdk/datasets/{dataset_id}/",
-            data={"download_type": "annotation", "export_type": export_type},
-        )
-
-        with self.api_session.get(url=response["file"], stream=True) as r:
-            r.raise_for_status()
-            filename = f"dataset-{dataset_id}.json"
-            with open(filename, "wb") as f:
-                for chunk in r.iter_content(chunk_size=1024 * 1024):
-                    f.write(chunk)
-            logger.info(f"File: {os.path.abspath(filename)}")
-            return os.path.abspath(filename)
-
-    def dataset_download_files(self, dataset_id):
-        response = self._post(
-            f"/api/sdk/datasets/{dataset_id}/", data={"download_type": "files"}
-        )
-        folder = f"dataset-files-{dataset_id}"
-        os.makedirs(folder, exist_ok=True)
-        dataset_files = [
-            dataset_file
-            for dataset_file in response
-            if not os.path.isfile(os.path.join(folder, dataset_file["file_name"]))
-        ]
-
-        async def download_file(session: aiohttp.ClientSession, dataset_file: dict):
-            async with session.get(url=dataset_file["source"]) as r:
-                try:
-                    r.raise_for_status()
-                except Exception as e:
-                    logger.error(
-                        f"Error downloading file {dataset_file['file_name']} - {e}"
-                    )
-                    return 0
-                async with aiofiles.open(
-                    os.path.join(folder, dataset_file["file_name"]), "wb"
-                ) as f:
-                    async for chunk in r.content.iter_any():
-                        await f.write(chunk)
-                    return 1
-
-        async def main():
-            with tqdm.tqdm(total=len(dataset_files), ncols=80) as pbar:
-                async with aiohttp.ClientSession() as session:
-                    tasks = [
-                        download_file(session=session, dataset_file=dataset_file)
-                        for dataset_file in dataset_files
-                    ]
-                    for f in asyncio.as_completed(tasks):
-                        pbar.update(await f)
-
-        asyncio.run(main())