nedo-vision-worker 1.0.0__py3-none-any.whl

nedo_vision_worker/worker/SystemUsageManager.py

@@ -0,0 +1,111 @@
+import logging
+import threading
+import time
+from ..util.SystemMonitor import SystemMonitor
+from ..services.SystemUsageClient import SystemUsageClient
+from ..services.GrpcClientBase import GrpcClientBase
+from ..util.Networking import Networking 
+
+logger = logging.getLogger(__name__)
+
+def safe_join_thread(thread, timeout=5):
+    """Safely join a thread, avoiding RuntimeError when joining current thread."""
+    if thread and thread != threading.current_thread():
+        thread.join(timeout=timeout)
+    elif thread == threading.current_thread():
+        logging.info("🛑 [APP] Thread stopping from within itself, skipping join.")
+
+class SystemUsageManager:
+    def __init__(self, server_host: str, device_id: str, token: str):
+        """
+        Handles system usage monitoring, latency tracking, and reporting.
+
+        Args:
+            server_host (str): The gRPC server host.
+            device_id (str): Unique ID of the device (passed externally).
+            token (str): Authentication token for the worker.
+        """
+        if not device_id:
+            raise ValueError("⚠️ [APP] 'device_id' cannot be empty.")
+        if not token:
+            raise ValueError("⚠️ [APP] 'token' cannot be empty.")
+
+        self.system_monitor = SystemMonitor()
+        self.system_usage_client = SystemUsageClient(server_host)
+        self.server_host = server_host
+        self.device_id = device_id
+        self.token = token
+        self.latency = None
+        self.latency_lock = threading.Lock()
+        self.stop_event = threading.Event()
+        self.latency_thread = None  
+
+        self._start_latency_monitoring()
+
+    def _start_latency_monitoring(self):
+        """Starts a background thread to monitor network latency."""
+        if self.latency_thread and self.latency_thread.is_alive():
+            logger.warning("⚠️ [APP] Latency monitoring thread is already running.")
+            return
+
+        self.latency_thread = threading.Thread(target=self._monitor_latency, daemon=True)
+        self.latency_thread.start()
+        logger.info("📡 [APP] Latency monitoring started.")
+
+    def _monitor_latency(self):
+        """Periodically checks the network latency using gRPC and updates the latency variable."""
+        server_port = 50051  # Default gRPC port
+
+        while not self.stop_event.is_set():
+            try:
+                latency_value = Networking.check_grpc_latency(self.server_host, server_port)
+                with self.latency_lock:
+                    self.latency = latency_value
+                # logger.info(f"🔄 [APP] Updated network latency: {latency_value} ms")
+
+            except Exception as e:
+                logger.error("🚨 [APP] Error checking gRPC latency.", exc_info=True)
+                with self.latency_lock:
+                    self.latency = None  
+            time.sleep(10) 
+
+    def process_system_usage(self):
+        """Collect and send system usage data to the server, including network latency."""
+        try:
+            usage = self.system_monitor.get_system_usage()
+            cpu_usage = usage["cpu"]["usage_percent"]
+            ram_usage = usage["ram"]
+            gpu_usage = usage.get("gpu", [])
+
+            with self.latency_lock:
+                latency = self.latency if self.latency is not None else -1
+
+            response = self.system_usage_client.send_system_usage(
+                device_id=self.device_id,
+                cpu_usage=cpu_usage,
+                ram_usage=ram_usage,
+                gpu_usage=gpu_usage,
+                latency=latency,
+                token=self.token,
+            )
+
+            if not response or not response.get("success"):
+                error_message = GrpcClientBase.get_error_message(response)
+                logger.error(f"❌ [APP] Failed to send system usage: {error_message}")
+            #else:
+            #    logger.info("✅ [APP] System usage sent successfully.")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Error sending system usage.", exc_info=True)
+
+    def close(self):
+        """Closes the system usage client and stops the latency thread."""
+        self.stop_event.set()
+
+        if self.latency_thread and self.latency_thread.is_alive():
+            safe_join_thread(self.latency_thread)
+            logger.info("🔌 [APP] Latency monitoring thread stopped.")
+
+        if self.system_usage_client:
+            self.system_usage_client.close_client()
+            logger.info("✅ [APP] SystemUsageClient closed.")

nedo_vision_worker/worker/VideoStreamWorker.py

@@ -0,0 +1,139 @@
+import os
+import threading
+import logging
+import json
+from ..database.DatabaseManager import get_storage_path
+from ..services.FileToRTMPServer import FileToRTMPStreamer
+from .RabbitMQListener import RabbitMQListener
+from ..services.RTSPtoRTMPStreamer import RTSPtoRTMPStreamer
+
+logger = logging.getLogger(__name__)
+
+def safe_join_thread(thread, timeout=5):
+    """Safely join a thread, avoiding RuntimeError when joining current thread."""
+    if thread and thread != threading.current_thread():
+        thread.join(timeout=timeout)
+    elif thread == threading.current_thread():
+        logging.info("🛑 [APP] Thread stopping from within itself, skipping join.")
+
+class VideoStreamWorker:
+    def __init__(self, config: dict, stream_duration=300):
+        """
+        Initialize Video Stream Worker.
+
+        Args:
+            config (dict): Configuration object containing settings.
+            stream_duration (int): Default stream duration in seconds.
+        """
+        if not isinstance(config, dict):
+            raise ValueError("⚠️ [APP] config must be a dictionary.")
+
+        self.config = config
+        self.worker_id = self.config.get("worker_id")
+        self.source_file_path = get_storage_path("files") / "source_files"
+
+        if not self.worker_id:
+            raise ValueError("⚠️ [APP] Configuration is missing 'worker_id'.")
+
+        self.stream_duration = stream_duration
+        self.rtmp_server = self.config.get("rtmp_server")
+        
+        if not self.rtmp_server:
+            raise ValueError("⚠️ [APP] RTMP server URL is required but not provided in configuration.")
+
+        self.thread = None
+        self.stop_event = threading.Event()
+        self.lock = threading.Lock()
+
+        # Initialize RabbitMQ listener
+        self.listener = RabbitMQListener(
+            self.config, self.worker_id, self.stop_event, self._process_video_preview_message
+        )
+
+    def start(self):
+        """Start the Video Stream Worker."""
+        with self.lock:
+            if self.thread and self.thread.is_alive():
+                logger.warning("⚠️ [APP] Stream Worker is already running.")
+                return
+
+            self.stop_event.clear()
+            self.thread = threading.Thread(target=self._run, daemon=True)  # ✅ Run as daemon
+            self.thread.start()
+            logger.info(f"🚀 [APP] Stream Worker started (Device: {self.worker_id}).")
+
+    def stop(self):
+        """Stop the Video Stream Worker."""
+        with self.lock:
+            if not self.thread or not self.thread.is_alive():
+                logger.warning("⚠️ [APP] Stream Worker is not running.")
+                return
+
+            self.stop_event.set()
+            self.listener.stop_listening()
+
+            safe_join_thread(self.thread)  # Ensures the thread stops gracefully
+            self.thread = None
+            logger.info(f"🛑 [APP] Stream Worker stopped (Device: {self.worker_id}).")
+
+    def _run(self):
+        """Main loop to manage RabbitMQ listener."""
+        try:
+            while not self.stop_event.is_set():
+                logger.info("📡 [APP] Waiting for video stream messages...")
+                self.listener.start_listening(exchange_name="nedo.worker.stream.preview", queue_name=f"nedo.worker.preview.{self.worker_id}")
+                safe_join_thread(self.listener.listener_thread)
+        except Exception as e:
+            logger.error("🚨 [APP] Unexpected error in Stream Worker loop.", exc_info=True)
+
+    def _process_video_preview_message(self, message):
+        """Process messages related to video preview streaming."""
+        try:
+            data = json.loads(message)
+            worker_id = data.get("workerId")
+            url = data.get("url")
+            uuid = data.get("uuid")
+            stream_duration = int(data.get("duration", self.stream_duration)) 
+
+            logger.info(f"📡 [APP] Received video preview message ({data})")
+
+            if not url or (not url.startswith("rtsp://") and not url.startswith("worker-source/")):
+                logger.error(f"⚠️ [APP] Invalid URL: {url}")
+                return
+
+            if stream_duration <= 0:
+                logger.warning(f"⚠️ [APP] Invalid stream duration {stream_duration}. Using default {self.stream_duration}s.")
+                stream_duration = self.stream_duration
+
+            logger.info(f"📡 [APP] Forwarding to RTMP (Worker: {worker_id}, UUID: {uuid})")
+
+            # Start a streaming thread
+            threading.Thread(
+                target=self._start_stream,
+                args=(url, self.rtmp_server, uuid, stream_duration, worker_id),
+                daemon=True,
+            ).start()
+
+        except json.JSONDecodeError:
+            logger.error("⚠️ [APP] Invalid JSON message format.")
+        except Exception as e:
+            logger.error("🚨 [APP] Error processing video preview message.", exc_info=True)
+
+    def _start_stream(self, url, rtmp_server, stream_key, stream_duration, worker_id):
+        """Runs RTSP-to-RTMP streaming in a separate thread."""
+        try:
+            logger.info(f"🎥 [APP] Starting RTMP stream (Worker: {worker_id})")
+            
+            if url.startswith("worker-source/"):
+                streamer = FileToRTMPStreamer(self.source_file_path / os.path.basename(url), rtmp_server, stream_key, stream_duration)
+            else:
+                streamer = RTSPtoRTMPStreamer(url, rtmp_server, stream_key, stream_duration)
+
+            streamer.start_stream()
+
+            # Schedule stream stop
+            threading.Timer(stream_duration, streamer.stop_stream).start()
+            logger.info(f"⏳ [APP] Stopping RTMP stream in {stream_duration}s.")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Error in stream worker.", exc_info=True)

nedo_vision_worker/worker/WorkerManager.py

@@ -0,0 +1,155 @@
+import logging
+from ..services.WorkerStatusClient import WorkerStatusClient
+from ..services.GrpcClientManager import GrpcClientManager
+from ..services.GrpcClientBase import GrpcClientBase
+from .PipelineActionWorker import PipelineActionWorker
+from .DataSyncWorker import DataSyncWorker
+from .DataSenderWorker import DataSenderWorker
+from .PipelineImageWorker import PipelineImageWorker
+from .VideoStreamWorker import VideoStreamWorker
+from .CoreActionWorker import CoreActionWorker
+from .DatasetFrameWorker import DatasetFrameWorker
+
+logger = logging.getLogger(__name__)
+
+class WorkerManager:
+    def __init__(self, config):
+        """Initialize all worker threads with the given config."""
+        self.config = config
+        self.worker_id = self.config.get("worker_id")
+        self.server_host = self.config.get("server_host")
+        self.token = self.config.get("token")
+
+        if not self.worker_id:
+            raise ValueError("⚠️ [APP] Configuration is missing 'worker_id'.")
+        if not self.server_host:
+            raise ValueError("⚠️ [APP] Configuration is missing 'server_host'.")
+        if not self.token:
+            raise ValueError("⚠️ [APP] Configuration is missing 'token'.")
+
+        # Configure the centralized gRPC client manager
+        self.client_manager = GrpcClientManager.get_instance()
+        self.client_manager.configure(self.server_host)
+        
+        # Get shared client instance
+        self.status_client = self.client_manager.get_client(WorkerStatusClient)
+
+        self.data_sync_worker = DataSyncWorker(config, sync_interval=10)
+        self.data_sender_worker = DataSenderWorker(config, send_interval=10)
+        self.video_stream_worker = VideoStreamWorker(config)
+        self.pipeline_image_worker = PipelineImageWorker(config)
+        self.pipeline_action_worker = PipelineActionWorker(config)
+        self.core_action_worker = CoreActionWorker(config, self._start_workers, self._stop_workers)
+        self.dataset_frame_worker = DatasetFrameWorker(config)
+
+    def _start_workers(self):
+        """Start processing workers while keeping monitoring workers running."""
+        try:
+            self.video_stream_worker.start()
+            logger.info("🚀 [APP] Video Stream Worker started.")
+
+            self.pipeline_image_worker.start()
+            logger.info("🚀 [APP] Pipeline Image Worker started.")
+
+            self.data_sender_worker.start_updating()
+            logger.info("🚀 [APP] Data Sender Worker started updating.")
+            
+            self.dataset_frame_worker.start()
+            logger.info("🚀 [APP] Dataset Frame Worker started.")
+            
+            self._update_status("run")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Failed to start processing workers.", exc_info=True)
+
+    def _stop_workers(self):
+        """Stop processing workers while keeping monitoring workers running."""
+        try:
+            self.video_stream_worker.stop()
+            logger.info("🛑 [APP] Video Stream Worker stopped.")
+
+            self.pipeline_image_worker.stop()
+            logger.info("🛑 [APP] Pipeline Image Worker stopped.")
+
+            self.data_sender_worker.stop_updating()
+            logger.info("🛑 [APP] Data Sender Worker stopped updating.")
+
+            self.dataset_frame_worker.stop()
+            logger.info("🛑 [APP] Dataset Frame Worker stopped.")
+
+            self._update_status("stop")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Failed to stop processing workers.", exc_info=True)
+
+    def start_all(self):
+        """Start all workers including monitoring workers."""
+        try:
+            # Start monitoring workers first
+            self.core_action_worker.start()
+            logging.info("🚀 [APP] Core Action Worker started and listening for commands.")
+
+            self.data_sync_worker.start()
+            logger.info("🚀 [APP] Data Sync Worker started.")
+
+            self.data_sender_worker.start()
+            logger.info("🚀 [APP] Data Sender Worker started.")
+
+            self.pipeline_action_worker.start()
+            logger.info("🚀 [APP] Pipeline Action Worker started.")
+
+            self._start_workers()
+
+            logger.info("✅ [APP] All workers started successfully.")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Failed to start all workers.", exc_info=True)
+
+    def stop_all(self):
+        """Stop all workers including monitoring workers."""
+        try:
+            self.core_action_worker.stop()
+            logger.info("🛑 [APP] Core Action Worker stopped.")
+
+            self.data_sync_worker.stop()
+            logger.info("🛑 [APP] Data Sync Worker stopped.")
+
+            self.data_sender_worker.stop()
+            logger.info("🛑 [APP] Data Sender Worker stopped.")
+
+            self.pipeline_action_worker.stop()
+            logger.info("🛑 [APP] Pipeline Action Worker stopped.")
+
+            self._stop_workers()
+
+            logger.info("✅ [APP] All workers stopped successfully.")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Failed to stop all workers.", exc_info=True)
+        finally:
+            # Cleanup: close gRPC clients when workers are stopped
+            try:
+                logger.info("🔌 [APP] Closing gRPC client connections...")
+                self.client_manager.close_all_clients()
+            except Exception as e:
+                logger.warning(f"⚠️ [APP] Error closing gRPC clients: {e}")
+    
+    def _update_status(self, status_code):
+        """
+        Update the worker status via gRPC.
+        
+        Args:
+            status_code (str): Status code to report to the server
+        """
+        try:
+            logger.info(f"📡 [APP] Updating worker status to {status_code}")
+            result = self.status_client.update_worker_status(self.worker_id, status_code, self.token)
+            
+            if result["success"]:
+                logger.info(f"✅ [APP] Status update successful: {result['message']}")
+            else:
+                error_message = GrpcClientBase.get_error_message(result)
+                logger.warning(f"⚠️ [APP] Status update failed: {error_message}")
+                
+        except Exception as e:
+            logger.error(f"🚨 [APP] Error updating worker status: {str(e)}")

nedo_vision_worker/worker/__init__.py

@@ -0,0 +1 @@
+ 

nedo_vision_worker/worker_service.py

@@ -0,0 +1,264 @@
+import time
+import multiprocessing
+import signal
+import sys
+import logging
+
+# Set multiprocessing start method to 'spawn' for CUDA compatibility
+try:
+    multiprocessing.set_start_method('spawn', force=True)
+except RuntimeError:
+    # Method already set, ignore
+    pass
+
+from .initializer.AppInitializer import AppInitializer
+from .worker.WorkerManager import WorkerManager
+from .config.ConfigurationManager import ConfigurationManager
+from .util.HardwareID import HardwareID
+from .services.GrpcClientBase import set_auth_failure_callback
+from .database.DatabaseManager import set_storage_path
+from . import models  # Ensure all models are registered before DB init
+
+
+class WorkerService:
+    """
+    Main worker service class that manages the worker agent lifecycle.
+    Uses hardware ID-based authentication and configuration management.
+    """
+    
+    def __init__(
+        self,
+        server_host: str = "be.vision.sindika.co.id",
+        token: str = None,
+        system_usage_interval: int = 30,
+        rtmp_server: str = None,
+        storage_path: str = "data",
+    ):
+        """
+        Initialize the worker service.
+        
+        Args:
+            server_host: Manager server host (default: 'be.vision.sindika.co.id')
+            token: Authentication token for the worker (obtained from frontend)
+            system_usage_interval: Interval for system usage reporting (default: 30)
+            rtmp_server: RTMP server URL for video streaming (required)
+            storage_path: Storage path for databases and files (default: 'data')
+        """
+        # Set the global storage path before any database operations
+        set_storage_path(storage_path)
+        
+        self.logger = self._setup_logging()
+        self.worker_manager = None
+        self.running = False
+        self.server_host = server_host
+        self.token = token
+        self.system_usage_interval = system_usage_interval
+        self.rtmp_server = rtmp_server
+        self.storage_path = storage_path
+        self.config = None
+        self.auth_failure_detected = False
+        
+        # Validate required parameters
+        if not rtmp_server:
+            raise ValueError("RTMP server URL is required. Please provide --rtmp-server parameter.")
+        
+        # Register authentication failure callback
+        set_auth_failure_callback(self._on_authentication_failure)
+        
+        # Setup signal handlers
+        signal.signal(signal.SIGINT, self._signal_handler)
+        signal.signal(signal.SIGTERM, self._signal_handler)
+
+    def _on_authentication_failure(self):
+        """Called when an authentication failure is detected."""
+        if not self.auth_failure_detected:
+            self.auth_failure_detected = True
+            self.logger.error("🔑 [APP] Authentication failure detected. Shutting down service...")
+            self.stop()
+
+    def _setup_logging(self):
+        """Configure logging settings (allows inline emojis)."""
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s [%(levelname)s] %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+        # Only show warnings and errors
+        logging.getLogger("sqlalchemy").setLevel(logging.WARNING)
+        logging.getLogger("pika").setLevel(logging.WARNING)
+        logging.getLogger("grpc").setLevel(logging.FATAL)
+        logging.getLogger("ffmpeg").setLevel(logging.FATAL)
+        logging.getLogger("subprocess").setLevel(logging.FATAL)
+        
+        return logging.getLogger(__name__)
+
+    def _initialize_configuration(self):
+        """Initialize the application configuration."""
+        self.logger.info("🚀 [APP] Initializing application...")
+
+        # Initialize database
+        ConfigurationManager.init_database()
+
+        # Load all configurations at once
+        config = ConfigurationManager.get_all_configs()
+        
+        # Use the server_host parameter directly
+        server_host = self.server_host
+        self.logger.info(f"🌐 [APP] Using server host: {server_host}")
+
+        # Check if configuration exists
+        if not config:
+            self.logger.info("⚙️ [APP] Configuration not found. Performing first-time setup...")
+
+            # Get hardware ID
+            hardware_id = HardwareID.get_unique_id()
+
+            self.logger.info(f"🖥️ [APP] Detected Hardware ID: {hardware_id}")
+            self.logger.info(f"🌐 [APP] Using Server Host: {server_host}")
+
+            # Check if token is provided
+            if not self.token:
+                raise ValueError("Token is required for worker initialization. Please provide a token obtained from the frontend.")
+
+            # Initialize with token
+            AppInitializer.initialize_configuration(hardware_id, server_host, self.token)
+
+            # Get configuration
+            config = ConfigurationManager.get_all_configs()
+        else:
+            # Check if server_host or token has changed and update if needed
+            config_updated = False
+            
+            if config['server_host'] != server_host:
+                ConfigurationManager.set_config("server_host", server_host)
+                config_updated = True
+                self.logger.info(f"✅ [APP] Updated server host to: {server_host}")
+            
+            # Check if token has changed and update if needed
+            if self.token and config.get('token') != self.token:
+                ConfigurationManager.set_config("token", self.token)
+                config_updated = True
+                self.logger.info("✅ [APP] Updated authentication token")
+            
+            if config_updated:
+                config = ConfigurationManager.get_all_configs()
+                self.logger.info("✅ [APP] Configuration updated successfully")
+            else:
+                self.logger.info("✅ [APP] Configuration found. No changes needed.")
+        
+        # Add runtime parameters to config
+        config['rtmp_server'] = self.rtmp_server
+        
+        return config
+
+    def initialize(self) -> bool:
+        """
+        Initialize the worker service components.
+        
+        Returns:
+            bool: True if initialization successful, False otherwise
+        """
+        try:
+            self.logger.info("Worker service initialization started")
+            
+            # Initialize configuration
+            self.config = self._initialize_configuration()
+            
+            if not self.config:
+                raise RuntimeError("Failed to initialize configuration")
+
+            # Initialize WorkerManager
+            self.worker_manager = WorkerManager(self.config)
+
+            self.logger.info("Worker service initialization completed successfully")
+            return True
+        except Exception as e:
+            self.logger.error(f"Failed to initialize worker service: {e}")
+            import traceback
+            self.logger.error(traceback.format_exc())
+            return False
+    
+    def start(self):
+        """Start the worker service"""
+        if not self.running:
+            self.running = True
+            self.logger.info("Worker service started")
+            try:
+                # Start all workers via WorkerManager
+                self.worker_manager.start_all()
+                # Block main thread to keep process alive
+                while self.running and not self.auth_failure_detected:
+                    time.sleep(1)
+                
+                # If authentication failure was detected, exit with error code
+                if self.auth_failure_detected:
+                    self.logger.error("🔑 [APP] Service terminated due to authentication failure")
+                    sys.exit(1)
+            except Exception as e:
+                self.logger.error(f"Error in worker service: {e}")
+                import traceback
+                self.logger.error(traceback.format_exc())
+                self.stop()
+        else:
+            self.logger.info("Service already running.")
+    
+    def stop(self):
+        """Stop the worker service"""
+        if self.running:
+            self.running = False
+            self.logger.info("Worker service stopping...")
+            try:
+                # Stop all workers via WorkerManager
+                if hasattr(self, 'worker_manager'):
+                    self.worker_manager.stop_all()
+                self.logger.info("Worker service stopped")
+            except Exception as e:
+                self.logger.error(f"Error stopping worker service: {e}")
+                import traceback
+                self.logger.error(traceback.format_exc())
+        else:
+            self.logger.info("Service already stopped.")
+    
+    def _signal_handler(self, signum, frame):
+        """Handle system signals for graceful shutdown"""
+        self.logger.info(f"Received signal {signum}, shutting down...")
+        self.stop()
+        sys.exit(0)
+    
+    def run(self):
+        """Run the worker service"""
+        if self.initialize():
+            self.start()
+        else:
+            self.logger.error("Failed to initialize worker service")
+            sys.exit(1)
+
+
+def main():
+    """Main entry point for the worker service"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Nedo Vision Worker Service")
+    parser.add_argument(
+        "--server-host", 
+        default="be.vision.sindika.co.id",
+        help="Manager server host (default: be.vision.sindika.co.id)"
+    )
+    parser.add_argument(
+        "--system-usage-interval",
+        type=int,
+        default=30,
+        help="System usage reporting interval in seconds (default: 30)"
+    )
+    args = parser.parse_args()
+    
+    # Create and run worker service
+    service = WorkerService(
+        server_host=args.server_host,
+        system_usage_interval=args.system_usage_interval
+    )
+    service.run()
+
+
+if __name__ == "__main__":
+    main()