nedo-vision-worker 1.0.0__py3-none-any.whl

nedo_vision_worker/util/VideoProbeUtil.py

@@ -0,0 +1,120 @@
+import cv2
+import subprocess
+import logging
+import json
+import fractions
+import shutil
+from urllib.parse import urlparse
+
+class VideoProbeUtil:
+    """Utility to extract metadata from video URLs using OpenCV and ffmpeg."""
+    
+    @staticmethod
+    def get_video_metadata(video_url: str) -> dict:
+        """Extracts resolution and frame rate from a video URL using OpenCV or ffmpeg."""
+        try:
+            # metadata = VideoProbeUtil._get_metadata_opencv(video_url)
+            metadata = VideoProbeUtil._get_metadata_ffmpeg(video_url)
+            return metadata
+        
+        except Exception as e:
+            logging.error(f"🚨 [APP] Error probing video {video_url}: {e}", exc_info=True)
+            return None
+
+    @staticmethod
+    def _get_metadata_opencv(video_url: str) -> dict:
+        cap = cv2.VideoCapture(video_url)
+        if not cap.isOpened():
+            logging.warning(f"⚠️ [APP] OpenCV failed to open video: {video_url}")
+            return None
+
+        # Read first frame to ensure the video is valid
+        ret, frame = cap.read()
+        if not ret or frame is None:
+            logging.warning(f"⚠️ [APP] OpenCV failed to read a frame from {video_url}")
+            cap.release()
+            return None
+
+        height, width = frame.shape[:2]
+        frame_rate = round(cap.get(cv2.CAP_PROP_FPS), 2)
+        cap.release()
+        
+        return {
+            "resolution": f"{width}x{height}" if width and height else None,
+            "frame_rate": frame_rate if frame_rate > 0 else None,
+            "timestamp": None
+        }
+    
+    @staticmethod
+    def _detect_stream_type(video_url: str) -> str:
+        """Detect whether the URL is RTSP, local file, or other type."""
+        # Convert PosixPath to string if needed
+        if hasattr(video_url, '__str__'):
+            video_url = str(video_url)
+        
+        parsed_url = urlparse(video_url)
+        if parsed_url.scheme == "rtsp":
+            return "rtsp"
+        elif parsed_url.scheme in ["http", "https"]:
+            return "http"
+        else:
+            return "file"
+    
+    @staticmethod
+    def _get_metadata_ffmpeg(video_url: str) -> dict:
+        # Check if ffprobe is available
+        if not shutil.which("ffprobe"):
+            logging.error("⚠️ [APP] ffprobe is not installed or not found in PATH.")
+            return None
+
+        # Detect stream type
+        stream_type = VideoProbeUtil._detect_stream_type(video_url)
+        
+        # Build ffprobe command based on stream type
+        cmd = ["ffprobe", "-v", "error", "-select_streams", "v:0", 
+               "-show_entries", "stream=width,height,avg_frame_rate", "-of", "json"]
+        
+        # Add RTSP transport option only for RTSP streams
+        if stream_type == "rtsp":
+            cmd.insert(1, "-rtsp_transport")
+            cmd.insert(2, "tcp")
+        
+        # Add the video URL
+        cmd.append(video_url)
+
+        try:
+            # Run ffprobe command
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
+
+            # Check for errors
+            if result.returncode != 0 or not result.stdout.strip():
+                logging.warning(f"⚠️ [APP] ffprobe failed for {video_url}: {result.stderr.strip()}")
+                return None
+
+            # Parse JSON output
+            metadata = json.loads(result.stdout)
+            streams = metadata.get("streams", [{}])[0]
+
+            # Extract metadata
+            width = streams.get("width")
+            height = streams.get("height")
+            avg_fps = streams.get("avg_frame_rate", "0/1")
+
+            # Convert FPS safely
+            try:
+                frame_rate = round(float(fractions.Fraction(avg_fps)), 2)
+            except (ValueError, ZeroDivisionError):
+                frame_rate = None
+
+            return {
+                "resolution": f"{width}x{height}" if width and height else None,
+                "frame_rate": frame_rate,
+                "timestamp": None  # Placeholder if you need a timestamp later
+            }
+
+        except subprocess.TimeoutExpired:
+            logging.warning(f"⚠️ [APP] ffprobe timeout for {video_url}")
+        except json.JSONDecodeError:
+            logging.error(f"❌ [APP] Failed to parse ffprobe output for {video_url}")
+
+        return None

nedo_vision_worker/util/__init__.py

@@ -0,0 +1 @@
+ 

nedo_vision_worker/worker/CoreActionWorker.py

@@ -0,0 +1,125 @@
+import threading
+import logging
+import json
+from .RabbitMQListener import RabbitMQListener
+
+logger = logging.getLogger(__name__)
+
+def safe_join_thread(thread, timeout=5):
+    """Safely join a thread, avoiding RuntimeError when joining current thread."""
+    if thread and thread != threading.current_thread():
+        thread.join(timeout=timeout)
+    elif thread == threading.current_thread():
+        logging.info("🛑 [APP] Thread stopping from within itself, skipping join.")
+
+class CoreActionWorker:
+    def __init__(self, config: dict, start_cb, stop_cb):
+        """
+        Initialize Core Action Worker.
+
+        Args:
+            config (dict): Configuration object containing settings.
+            start_cb: Callback function to start the worker.
+            stop_cb: Callback function to stop the worker.
+        """
+        if not isinstance(config, dict):
+            raise ValueError("⚠️ [APP] config must be a dictionary.")
+
+        self.config = config
+        self.worker_id = self.config.get("worker_id")
+        self.start_cb = start_cb
+        self.stop_cb = stop_cb
+
+        if not self.worker_id:
+            raise ValueError("⚠️ [APP] Configuration is missing 'worker_id'.")
+
+        self.thread = None
+        self.stop_event = threading.Event()
+        self.lock = threading.Lock()
+
+        # Initialize RabbitMQ listener
+        self.listener = RabbitMQListener(
+            self.config, self.worker_id, self.stop_event, self._process_core_action_message
+        )
+
+    def start(self):
+        """Start the Core Action Worker."""
+        with self.lock:
+            if self.thread and self.thread.is_alive():
+                logger.warning("⚠️ [APP] Core Action Worker is already running.")
+                return
+
+            self.stop_event.clear()
+            self.thread = threading.Thread(target=self._run, daemon=True)
+            self.thread.start()
+            logger.info(f"🚀 [APP] Core Action Worker started (Device: {self.worker_id}).")
+
+    def stop(self):
+        """Stop the Core Action Worker."""
+        with self.lock:
+            if not self.thread or not self.thread.is_alive():
+                logger.warning("⚠️ [APP] Core Action Worker is not running.")
+                return
+
+            self.stop_event.set()
+            self.listener.stop_listening()
+
+            safe_join_thread(self.thread)
+            self.thread = None
+            logger.info(f"🛑 [APP] Core Action Worker stopped (Device: {self.worker_id}).")
+
+    def _run(self):
+        """Main loop to manage RabbitMQ listener."""
+        try:
+            while not self.stop_event.is_set():
+                logger.info("📡 [APP] Waiting for core action messages...")
+                self.listener.start_listening(
+                    exchange_name="nedo.worker.core.action", 
+                    queue_name=f"nedo.worker.core.{self.worker_id}"
+                )
+                safe_join_thread(self.listener.listener_thread)
+        except Exception as e:
+            logger.error("🚨 [APP] Unexpected error in Core Action Worker loop.", exc_info=True)
+
+    def _process_core_action_message(self, message):
+        """
+        Process received core action messages.
+        
+        Args:
+            message (str): JSON message containing action and timestamp
+        """
+        try:
+            data = json.loads(message)
+            action = data.get('action')
+            timestamp = data.get('timestamp')
+
+            logger.info(f"📥 [APP] Received core action: {action} at {timestamp}")
+
+            if action == "start":
+                logger.info(f"🚀 [APP] Starting processing workers")
+                self.start_cb()
+                logger.info(f"✅ [APP] Started processing workers")
+                
+            elif action == "stop":
+                logger.info(f"🛑 [APP] Stopping processing workers")
+                self.stop_cb()
+                logger.info(f"✅ [APP] Stopped processing workers")
+                
+            elif action == "restart":
+                logger.info(f"🔄 [APP] Restarting processing workers")
+                self.start_cb()
+                self.stop_cb()
+                logger.info(f"✅ [APP] Restarted processing workers")
+                
+            elif action == "debug":
+                # TODO: do something on debugging, not now
+                logger.info(f"🔍 [APP] Debugging")
+                
+            else:
+                logger.warning(f"⚠️ [APP] Unknown core action received: {action}")
+
+        except json.JSONDecodeError:
+            logger.error("🚨 [APP] Failed to parse core action message JSON")
+        except Exception as e:
+            logger.error(f"🚨 [APP] Error processing core action: {str(e)}")
+

nedo_vision_worker/worker/DataSenderWorker.py

@@ -0,0 +1,168 @@
+import threading
+import time
+import logging
+from .RestrictedAreaManager import RestrictedAreaManager
+from .SystemUsageManager import SystemUsageManager
+from .PPEDetectionManager import PPEDetectionManager
+from .DatasetFrameSender import DatasetFrameSender
+from ..util.ImageUploader import ImageUploader
+from ..services.ImageUploadClient import ImageUploadClient
+from ..services.WorkerSourceUpdater import WorkerSourceUpdater
+
+def safe_join_thread(thread, timeout=5):
+    """Safely join a thread, avoiding RuntimeError when joining current thread."""
+    if thread and thread != threading.current_thread():
+        thread.join(timeout=timeout)
+    elif thread == threading.current_thread():
+        logging.info("🛑 [APP] Thread stopping from within itself, skipping join.")
+
+logger = logging.getLogger(__name__)
+
+class DataSenderWorker:
+    def __init__(self, config: dict, send_interval=5, update_interval=10):
+        """
+        Initializes the Data Sender Worker.
+
+        Args:
+            config (dict): Configuration dictionary.
+            send_interval (int): Interval (in seconds) for sending system usage & images.
+            update_interval (int): Interval (in seconds) for updating worker sources.
+        """
+        if not isinstance(config, dict):
+            raise ValueError("⚠️ [APP] config must be a dictionary.")
+
+        self.config = config
+        self.worker_id = self.config.get("worker_id")
+        self.server_host = self.config.get("server_host")
+        self.token = self.config.get("token")
+
+        if not self.worker_id:
+            raise ValueError("⚠️ [APP] Configuration is missing 'worker_id'.")
+        if not self.server_host:
+            raise ValueError("⚠️ [APP] Configuration is missing 'server_host'.")
+        if not self.token:
+            raise ValueError("⚠️ [APP] Configuration is missing 'token'.")
+        
+        self.should_update = True
+
+        self.send_interval = send_interval
+        self.update_interval = update_interval
+
+        self.main_thread = None
+        self.worker_update_thread = None
+        self.stop_event = threading.Event()
+        self.lock = threading.Lock()
+
+        # Initialize services
+        self.system_usage_manager = SystemUsageManager(self.server_host, self.worker_id, self.token)
+        self.image_upload_client = ImageUploadClient(self.server_host)
+        self.image_uploader = ImageUploader(self.image_upload_client, self.worker_id)
+        self.ppe_detection_manager = PPEDetectionManager(self.server_host, self.worker_id, "worker_source_id", self.token) 
+        self.restricted_area_manager = RestrictedAreaManager(self.server_host, self.worker_id, "worker_source_id", self.token)
+        self.dataset_frame_sender = DatasetFrameSender(self.server_host, self.token)
+
+        self.source_updater = WorkerSourceUpdater(self.server_host, self.token)
+
+    def start(self):
+        """Start the Data Sender Worker threads."""
+        with self.lock:
+            if self.main_thread and self.main_thread.is_alive():
+                logger.warning("⚠️ [APP] Data Sender Worker is already running.")
+                return
+
+            self.stop_event.clear()
+
+            # ✅ Start the main worker thread (System usage + Image upload)
+            self.main_thread = threading.Thread(target=self._run_main_worker, daemon=True)
+            self.main_thread.start()
+
+            # ✅ Start the worker source update thread
+            self.worker_update_thread = threading.Thread(target=self._run_worker_source_updater, daemon=True)
+            self.worker_update_thread.start()
+
+            logger.info(f"🚀 [APP] Data Sender Worker started (Device: {self.worker_id}).")
+
+    def stop(self):
+        """Stop the Data Sender Worker and Worker Source Updater threads."""
+        with self.lock:
+            if not self.main_thread or not self.main_thread.is_alive():
+                logger.warning("⚠️ [APP] Data Sender Worker is not running.")
+                return
+
+            self.stop_event.set()
+
+            # ✅ Stop the main worker thread
+            if self.main_thread:
+                safe_join_thread(self.main_thread, timeout=5)
+
+            # ✅ Stop the worker source update thread
+            if self.worker_update_thread:
+                safe_join_thread(self.worker_update_thread, timeout=5)
+
+            self.main_thread = None
+            self.worker_update_thread = None
+
+            logger.info(f"🛑 [APP] Data Sender Worker stopped (Device: {self.worker_id}).")
+
+    def start_updating(self):
+        """Start updating worker sources."""
+        self.should_update = True
+
+    def stop_updating(self):
+        """Stop updating worker sources."""
+        self.should_update = False
+        self.source_updater.stop_worker_sources()
+
+    def _run_main_worker(self):
+        """Main loop for sending system usage and uploading images."""
+        try:
+            while not self.stop_event.is_set():
+                self.system_usage_manager.process_system_usage()
+                self.ppe_detection_manager.send_ppe_detection_batch()  
+                self.restricted_area_manager.send_violation_batch()  
+                self._process_image_upload()
+                self._process_dataset_frames()
+                time.sleep(self.send_interval)
+        except Exception as e:
+            logger.error("🚨 [APP] Unexpected error in main worker loop.", exc_info=True)
+
+    def _run_worker_source_updater(self):
+        """Dedicated loop for updating worker sources at a different interval."""
+        try:
+            while not self.stop_event.is_set():
+                if self.should_update:
+                    self._update_worker_sources()
+                
+                time.sleep(self.update_interval)
+        except Exception as e:
+            logger.error("🚨 [APP] Unexpected error in Worker Source Updater loop.", exc_info=True)
+
+    def _process_image_upload(self):
+        """Check and upload images to the server."""
+        try:
+            self.image_uploader.check_and_upload_images()
+        except Exception as e:
+            logger.error("🚨 [APP] Error uploading images.", exc_info=True)
+
+    def _process_dataset_frames(self):
+        """Send pending dataset frames to the server."""
+        try:
+            stats = self.dataset_frame_sender.send_pending_frames(max_batch_size=5)
+            
+            if stats:
+                total_sent = sum(stats.values())
+                logger.info(f"📤 [APP] Sent {total_sent} dataset frames: {stats}")
+            else:
+                pending_count = self.dataset_frame_sender.get_pending_frame_count()
+                if pending_count > 0:
+                    logger.debug(f"📋 [APP] {pending_count} dataset frames pending")
+                    
+        except Exception as e:
+            logger.error("🚨 [APP] Error processing dataset frames.", exc_info=True)
+
+    def _update_worker_sources(self):
+        """Synchronize and update worker sources."""
+        try:
+            self.source_updater.update_worker_sources()
+        except Exception as e:
+            logger.error("🚨 [APP] Error updating worker sources.", exc_info=True)

nedo_vision_worker/worker/DataSyncWorker.py

@@ -0,0 +1,143 @@
+import threading
+import time
+import logging
+from ..services.AIModelClient import AIModelClient
+from ..services.WorkerSourceClient import WorkerSourceClient
+from ..services.WorkerSourcePipelineClient import WorkerSourcePipelineClient
+from ..services.GrpcClientBase import GrpcClientBase
+
+def safe_join_thread(thread, timeout=5):
+    """Safely join a thread, avoiding RuntimeError when joining current thread."""
+    if thread and thread != threading.current_thread():
+        thread.join(timeout=timeout)
+    elif thread == threading.current_thread():
+        logging.info("🛑 [APP] Thread stopping from within itself, skipping join.")
+
+logger = logging.getLogger(__name__)
+
+class DataSyncWorker:
+    def __init__(self, config: dict, sync_interval=10):
+        """
+        Initializes the Data Sync Worker.
+
+        Args:
+            config (dict): Configuration dictionary.
+            sync_interval (int): Interval (in seconds) for synchronization.
+        """
+        if not isinstance(config, dict):
+            raise ValueError("⚠️ [APP] config must be a dictionary.")
+
+        self.config = config
+        self.worker_id = self.config.get("worker_id")
+        self.server_host = self.config.get("server_host")
+        self.token = self.config.get("token")
+
+        if not self.worker_id:
+            raise ValueError("⚠️ [APP] Configuration is missing 'worker_id'.")
+        if not self.server_host:
+            raise ValueError("⚠️ [APP] Configuration is missing 'server_host'.")
+        if not self.token:
+            raise ValueError("⚠️ [APP] Configuration is missing 'token'.")
+
+        self.ai_model_client = AIModelClient(self.token, self.server_host)
+        self.worker_source_client = WorkerSourceClient(self.server_host)
+        self.worker_source_pipeline_client = WorkerSourcePipelineClient(self.server_host)
+
+        self.sync_interval = sync_interval
+        self.thread = None
+        self.stop_event = threading.Event()
+        self.lock = threading.Lock()
+
+    def start(self):
+        """Start the data synchronization worker thread."""
+        with self.lock:
+            if self.thread and self.thread.is_alive():
+                logger.warning("⚠️ [APP] Sync Worker is already running.")
+                return
+
+            self.stop_event.clear()
+            self.thread = threading.Thread(target=self._run, daemon=True)
+            self.thread.start()
+            logger.info(f"🚀 [APP] Sync Worker started (Device: {self.worker_id}).")
+
+    def stop(self):
+        """Stop the data sync worker."""
+        logging.info("🛑 [DATA SYNC] Stopping DataSyncWorker.")
+        self.running = False
+        safe_join_thread(self.thread)
+        logging.info("🛑 [DATA SYNC] DataSyncWorker stopped.")
+
+    def _run(self):
+        """Main loop for syncing worker sources and pipelines."""
+        try:
+            while not self.stop_event.is_set():
+                self._sync_ai_models()
+                self._sync_worker_sources()
+                self._sync_worker_source_pipelines()
+                self._sync_worker_source_pipelines_debug()
+                self._sync_worker_source_pipelines_detection()
+                time.sleep(self.sync_interval)
+        except Exception as e:
+            logger.error("🚨 [APP] Unexpected error in Sync Worker main loop.", exc_info=True)
+
+    def _sync_ai_models(self):
+        """Synchronize worker sources from the server."""
+        try:
+            response = self.ai_model_client.sync_ai_models(self.worker_id)
+            
+            if not response or not response.get("success"):
+                error_message = GrpcClientBase.get_error_message(response)
+                logger.error(f"❌ [APP] Failed to sync AI Models: {error_message}")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Error syncing AI Models.", exc_info=True)
+
+    def _sync_worker_sources(self):
+        """Synchronize worker sources from the server."""
+        try:
+            response = self.worker_source_client.sync_worker_sources(self.worker_id, self.token)
+            
+            if not response or not response.get("success"):
+                error_message = GrpcClientBase.get_error_message(response)
+                logger.error(f"❌ [APP] Failed to sync worker sources: {error_message}")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Error syncing worker sources.", exc_info=True)
+
+
+    def _sync_worker_source_pipelines(self):
+        """Synchronize worker source pipelines from the server."""
+        try:
+            response = self.worker_source_pipeline_client.get_worker_source_pipeline_list(self.worker_id, self.token)
+
+            if not response or not response.get("success"):
+                error_message = GrpcClientBase.get_error_message(response)
+                logger.error(f"❌ [APP] Failed to sync worker source pipelines: {error_message}")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Error syncing worker source pipelines.", exc_info=True)
+
+    def _sync_worker_source_pipelines_debug(self):
+        """Synchronize worker source pipelines debug with the server."""
+        try:
+            response = self.worker_source_pipeline_client.sync_pipeline_debug(self.token)
+
+            if not response or not response.get("success"):
+                error_message = GrpcClientBase.get_error_message(response)
+                logger.error(f"❌ [APP] Failed to sync restricted area violations: {error_message}")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Error syncing worker source pipelines debug.", exc_info=True)
+
+
+    def _sync_worker_source_pipelines_detection(self):
+        """Synchronize worker source pipelines detection with the server."""
+        try:
+            response = self.worker_source_pipeline_client.sync_pipeline_detection(self.token)
+
+            if not response or not response.get("success"):
+                error_message = GrpcClientBase.get_error_message(response)
+                logger.error(f"❌ [APP] Failed to sync dataset sources: {error_message}")
+
+        except Exception as e:
+            logger.error("🚨 [APP] Error syncing worker source pipelines detection.", exc_info=True)