nedo-vision-worker-core 0.3.2__tar.gz → 0.3.4__tar.gz

{nedo_vision_worker_core-0.3.2 → nedo_vision_worker_core-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker-core
-Version: 0.3.2
+Version: 0.3.4
 Summary: Nedo Vision Worker Core Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>

{nedo_vision_worker_core-0.3.2 → nedo_vision_worker_core-0.3.4}/nedo_vision_worker_core/__init__.py

@@ -7,7 +7,7 @@ A library for running AI vision processing and detection in the Nedo Vision plat
 from .core_service import CoreService
 from .callbacks import DetectionType, CallbackTrigger, DetectionData, IntervalMetadata
 
-__version__ = "0.3.2"
+__version__ = "0.3.4"
 __all__ = [
     "CoreService", 
     "DetectionType", 

{nedo_vision_worker_core-0.3.2 → nedo_vision_worker_core-0.3.4}/nedo_vision_worker_core/detection/RFDETRDetector.py

@@ -9,6 +9,7 @@ except ImportError:
 
 from ..database.DatabaseManager import DatabaseManager
 from ..models.ai_model import AIModelEntity
+from ..util.PlatformDetector import PlatformDetector
 from .BaseDetector import BaseDetector
 
 logging.getLogger("ultralytics").setLevel(logging.WARNING)
@@ -25,6 +26,8 @@ class RFDETRDetector(BaseDetector):
             raise TypeError("model must be an instance of AIModelEntity")
         self.model = None
         self.metadata = None
+        self.device = PlatformDetector.get_device()
+        logging.info(f"ℹ️ RFDETRDetector will use '{self.device}' device.")
 
         if model:
             self.load_model(model)

nedo_vision_worker_core-0.3.4/nedo_vision_worker_core/pipeline/ModelManager.py

@@ -0,0 +1,139 @@
+import logging
+from typing import Dict, Optional, Set
+
+from ..repositories.AIModelRepository import AIModelRepository
+from ..detection.BaseDetector import BaseDetector
+from ..detection.YOLODetector import YOLODetector
+from ..detection.RFDETRDetector import RFDETRDetector
+from ..models.ai_model import AIModelEntity
+
+
+class ModelManager:
+    """Manages loading and caching of AI models to avoid redundant loads."""
+
+    def __init__(self):
+        self._detector_cache: Dict[str, BaseDetector] = {}
+        self._model_repo = AIModelRepository()
+        logging.info("🤖 ModelManager initialized.")
+
+    def get_detector(self, model_id: str) -> Optional[BaseDetector]:
+        """
+        Retrieves a detector by its model ID.
+
+        This method implements a lazy-loading and cache-validation strategy:
+        1. It fetches the latest model metadata from the database.
+        2. If a detector is already cached, it validates its metadata against the DB version.
+        3. If the cached version is stale (e.g., version or classes changed), it's evicted.
+        4. If no detector is cached or the cache was stale, it loads the detector on-demand.
+        """
+        if not model_id:
+            return None
+
+        # 1. Fetch the current model state from the database
+        db_model: AIModelEntity = self._model_repo.get_model(model_id)
+        if not db_model:
+            # If the model doesn't exist in DB, ensure it's not in cache either
+            if model_id in self._detector_cache:
+                logging.info(f"🧹 Removing detector for deleted model {model_id} from cache.")
+                del self._detector_cache[model_id]
+            return None
+
+        # 2. Check if a detector is cached
+        if model_id in self._detector_cache:
+            cached_detector = self._detector_cache[model_id]
+            
+            # 3. Check if the cached version is stale
+            if self._has_metadata_changed(cached_detector.metadata, db_model):
+                logging.info(f"Reloading detector for model {model_id} due to metadata changes.")
+                del self._detector_cache[model_id]
+                # Fall through to load the new version
+            else:
+                # Cache is fresh, return it
+                logging.debug(f"🧠 Detector for model {model_id} found in cache and is fresh.")
+                return cached_detector
+
+        # 4. If not cached or was stale, load it now
+        return self._load_and_cache_detector(model_id, db_model)
+
+    def _load_and_cache_detector(self, model_id: str, db_model: AIModelEntity) -> Optional[BaseDetector]:
+        """Creates a detector from a DB model entity and caches it."""
+        logging.info(f"🔄 Loading model {model_id} (version: {db_model.version}) from database to create detector...")
+
+        # Check model readiness before attempting to load
+        if not db_model.is_ready_for_use():
+            if db_model.is_downloading():
+                logging.warning(f"⏳ Model {model_id} is still downloading. Skipping detector load.")
+            elif db_model.has_download_failed():
+                logging.error(f"❌ Model {model_id} download failed: {db_model.download_error}")
+            else:
+                logging.warning(f"⚠️ Model {model_id} is not ready for use (status: {db_model.download_status})")
+            return None
+
+        detector_type = db_model.type.lower()
+        detector: Optional[BaseDetector] = None
+        try:
+            if detector_type == "yolo":
+                detector = YOLODetector(db_model)
+            elif detector_type == "rf_detr":
+                detector = RFDETRDetector(db_model)
+            else:
+                raise ValueError(f"Unsupported model type: {detector_type}")
+
+            if detector and detector.model is not None:
+                self._detector_cache[model_id] = detector
+                logging.info(f"✅ Detector for model {model_id} loaded and cached successfully.")
+                return detector
+            else:
+                logging.error(f"❌ Failed to load detector for model: {db_model.name}")
+                return None
+
+        except Exception as e:
+            logging.error(f"❌ Error creating detector for model {db_model.name}: {e}")
+            return None
+
+    def _has_metadata_changed(self, cached_model: AIModelEntity, db_model: AIModelEntity) -> bool:
+        """Check if critical model metadata has changed."""
+        if cached_model.version != db_model.version:
+            logging.info(
+                f"🔄 Model {db_model.id} version changed "
+                f"({cached_model.version} -> {db_model.version})."
+            )
+            return True
+
+        # Compare classes
+        cached_classes = set(cached_model.get_classes() or [])
+        db_classes = set(db_model.get_classes() or [])
+        if cached_classes != db_classes:
+            logging.info(f"🔄 Model {db_model.id} classes changed.")
+            return True
+        
+        # Compare PPE class groups
+        cached_ppe_groups = cached_model.get_ppe_class_groups() or {}
+        db_ppe_groups = db_model.get_ppe_class_groups() or {}
+        if cached_ppe_groups != db_ppe_groups:
+            logging.info(f"🔄 Model {db_model.id} PPE groups changed.")
+            return True
+        
+        # Compare main class
+        if cached_model.get_main_class() != db_model.get_main_class():
+            logging.info(f"🔄 Model {db_model.id} main class changed.")
+            return True
+        
+        return False
+
+    def sync_cache(self, active_model_ids: Set[str]):
+        """
+        Removes detectors from the cache if their corresponding models are no longer in the database
+        or are not being used by any active pipeline.
+        """
+        cached_ids = set(self._detector_cache.keys())
+        stale_ids = cached_ids - active_model_ids
+        
+        for model_id in stale_ids:
+            del self._detector_cache[model_id]
+            logging.info(f"🧹 Removed unused detector for model {model_id} from cache.")
+
+    def clear_cache(self):
+        """Clears the detector cache."""
+        logging.info("🧹 Clearing all detectors from cache.")
+        self._detector_cache.clear()

{nedo_vision_worker_core-0.3.2 → nedo_vision_worker_core-0.3.4}/nedo_vision_worker_core/pipeline/PipelineManager.py

@@ -19,12 +19,12 @@ class PipelineManager:
         self._stop_lock = threading.Lock()  # Lock for thread-safe pipeline stopping
         self.on_pipeline_stopped = on_pipeline_stopped
 
-    def start_pipeline(self, pipeline, model):
+    def start_pipeline(self, pipeline, detector):
         """
         Start a pipeline processing.
         Args:
             pipeline: The pipeline object (contains id, worker_source_id, name, etc.)
-            model: The AI model to use for processing.
+            detector: The detector instance to use for processing.
         """
         pipeline_id = pipeline.id
         worker_source_id = pipeline.worker_source_id
@@ -39,7 +39,7 @@ class PipelineManager:
 
         logging.info(f"🚀 Starting Pipeline processing for pipeline: {pipeline_id} | Source: {worker_source_id} ({pipeline.name})")
 
-        processor = PipelineProcessor(pipeline_id, worker_source_id, model, False)
+        processor = PipelineProcessor(pipeline, detector, False)
         processor.frame_drawer.location_name = pipeline.location_name
         self.processors[pipeline_id] = processor  # Store processor instance
 

{nedo_vision_worker_core-0.3.2 → nedo_vision_worker_core-0.3.4}/nedo_vision_worker_core/pipeline/PipelineProcessor.py

@@ -12,18 +12,19 @@ from ..streams.VideoStreamManager import VideoStreamManager
 from ..ai.VideoDebugger import VideoDebugger
 from ..ai.FrameDrawer import FrameDrawer
 from ..tracker.TrackerManager import TrackerManager
-from ..detection.DetectionManager import DetectionManager
+from ..detection.BaseDetector import BaseDetector
 from ..streams.RTMPStreamer import RTMPStreamer
 
 
 class PipelineProcessor:
     """Handles pipeline processing including preprocessing, AI model inference, tracking, and video stream processing."""
 
-    def __init__(self, pipeline_id, worker_source_id, model, enable_visualization=True):
+    def __init__(self, pipeline, detector: BaseDetector, enable_visualization=True):
+        self._pipeline = pipeline
         self.running = True
         self.video_debugger = VideoDebugger(enable_visualization)
         self.tracker_manager = TrackerManager()
-        self.detection_manager = DetectionManager(model)
+        self.detector = detector
         self.config_manager = PipelineConfigManager()
         self.preprocessor = PipelinePrepocessor()
         self.detection_processor = None
@@ -36,8 +37,8 @@ class PipelineProcessor:
         self.detection_thread = None
         self.frame_counter = 0
         self.frame_drawer = FrameDrawer()
-        self.pipeline_id = pipeline_id
-        self.worker_source_id = worker_source_id
+        self.pipeline_id = pipeline.id
+        self.worker_source_id = pipeline.worker_source_id
 
         self.rtmp_streamer = None
 
@@ -61,11 +62,16 @@ class PipelineProcessor:
         self.last_hevc_recovery = 0
         self.hevc_recovery_cooldown = 30.0  # 30 seconds between HEVC recovery attempts
 
-    def load_model(self, model):
-        logging.info(f"🔄 Loading new model for pipeline {self.pipeline_id}: {getattr(model, 'name', None) or 'Unknown'}")
-        self.detection_manager.load_model(model)
+    def update_config(self, pipeline):
+        """Updates the pipeline configuration."""
+        self._pipeline = pipeline
+        self._update_config_internal()
+
+    def load_detector(self, detector: BaseDetector):
+        logging.info(f"🔄 Loading new detector for pipeline {self.pipeline_id}")
+        self.detector = detector
         self._update_detection_processor()
-        logging.info(f"✅ Model updated for pipeline {self.pipeline_id}")
+        logging.info(f"✅ Detector updated for pipeline {self.pipeline_id}")
 
     def _get_detection_processor_code(self):
         for code in self.detection_processor_codes:
@@ -107,7 +113,7 @@ class PipelineProcessor:
             self.frame_drawer.update_config()
             self.tracker_manager.update_config([], [], [])
 
-    def _update_config(self):
+    def _update_config_internal(self):
         self.config_manager.update(self.pipeline_id)
         self.preprocessor.update(self.config_manager)
         self.detection_interval = self._get_detection_interval()
@@ -117,7 +123,7 @@ class PipelineProcessor:
         self.consecutive_frame_failures = 0
         self.last_successful_frame_time = time.time()
 
-        ai_model = self.detection_manager.model_metadata
+        ai_model = self.detector.metadata if self.detector else None
         if self.detection_processor:
             config = self.config_manager.get_feature_config(self.detection_processor.code)
             self.detection_processor.update(self.config_manager, ai_model)
@@ -133,7 +139,7 @@ class PipelineProcessor:
         worker_source_id = self.worker_source_id
         logging.info(f"🎯 Running pipeline processing for pipeline {pipeline_id} | Source: {worker_source_id}")
 
-        self._update_config()
+        self._update_config_internal()
         self.consecutive_frame_failures = 0
         self.last_successful_frame_time = time.time()
 
@@ -243,7 +249,7 @@ class PipelineProcessor:
         processed_frame = self.preprocessor.apply(frame)
         
         class_thresholds = {}
-        ai_model = self.detection_manager.model_metadata
+        ai_model = self.detector.metadata if self.detector else None
         
         if self.detection_processor:
             if self.detection_processor.code == PPEDetectionProcessor.code:
@@ -253,7 +259,10 @@ class PipelineProcessor:
                 if main_threshold and ai_model and ai_model.get_main_class():
                     class_thresholds[ai_model.get_main_class()] = main_threshold
         
-        detections = self.detection_manager.detect_objects(processed_frame, self.threshold, class_thresholds)
+        detections = []
+        if self.detector:
+            detections = self.detector.detect_objects(processed_frame, self.threshold, class_thresholds)
+        
         detections = self.preprocessor.revert_detections_bboxes(detections, dimension)
         
         if self.detection_processor:
@@ -277,9 +286,8 @@ class PipelineProcessor:
 
                 # Update config periodically
                 if (current_time - last_config_update_time) >= config_update_interval:
-                    self._update_config()
+                    self._update_config_internal()
                     last_config_update_time = current_time
-                    logging.info(f"🔄 Updated pipeline config for {pipeline_id}")
 
                 # Keep only the latest frame if we fell behind
                 try:
@@ -324,25 +332,24 @@ class PipelineProcessor:
             except Exception as e:
                 logging.error(f"❌ Error in detection thread for pipeline {pipeline_id}: {e}", exc_info=True)
     
-    def _wait_for_frame(self, video_manager, max_retries=10, sleep_time=3):
+    def _wait_for_frame(self, video_manager, max_wait_time=30.0):
         logging.info(f"⏳ Waiting for initial frame from {self.worker_source_id}...")
-        for retry_count in range(max_retries):
+        
+        is_ready = video_manager.wait_for_stream_ready(self.worker_source_id, timeout=max_wait_time)
+
+        if is_ready:
             frame = video_manager.get_frame(self.worker_source_id)
             if frame is not None:
                 logging.info(f"✅ Initial frame received from {self.worker_source_id}")
                 return frame
-            
-            if not video_manager.has_stream(self.worker_source_id):
-                logging.error(f"❌ Stream {self.worker_source_id} not found in video manager")
-                return None
-            
-            logging.warning(f"⚠️ Waiting for video stream {self.worker_source_id} (Attempt {retry_count + 1}/{max_retries})...")
-            if retry_count >= 3:
+            else:
+                logging.error(f"❌ Stream {self.worker_source_id} reported ready, but the first frame could not be retrieved.")
                 self._log_stream_diagnostics(video_manager, self.worker_source_id)
-            time.sleep(sleep_time)
-
-        logging.error(f"❌ Failed to get initial frame from {self.worker_source_id} after {max_retries} attempts")
-        return None
+                return None
+        else:
+            logging.error(f"❌ Timed out after {max_wait_time}s waiting for first frame from {self.worker_source_id}.")
+            self._log_stream_diagnostics(video_manager, self.worker_source_id)
+            return None
 
     def _handle_frame_failure(self, video_manager, worker_source_id):
         """Handle frame retrieval failures with progressive backoff and recovery attempts."""

nedo_vision_worker_core-0.3.4/nedo_vision_worker_core/pipeline/PipelineSyncThread.py

@@ -0,0 +1,186 @@
+import json
+import logging
+import time
+import threading
+from typing import Dict, Set, Optional
+from ..repositories.WorkerSourcePipelineDebugRepository import WorkerSourcePipelineDebugRepository
+from ..repositories.WorkerSourcePipelineRepository import WorkerSourcePipelineRepository
+from .PipelineManager import PipelineManager
+from .ModelManager import ModelManager
+from ..streams.VideoStreamManager import VideoStreamManager
+
+
+class PipelineSyncThread(threading.Thread):
+    """Thread responsible for synchronizing worker source pipelines from the database in real-time."""
+
+    def __init__(self, video_manager: VideoStreamManager, polling_interval=5, max_workers=4):
+        super().__init__(daemon=True)  # Runs as a daemon
+        self.video_manager = video_manager
+        self.polling_interval = polling_interval
+        self.pipeline_repo = WorkerSourcePipelineRepository()
+        self.debug_repo = WorkerSourcePipelineDebugRepository()
+        self.model_manager = ModelManager()
+        self.running = True
+        self.pipeline_manager = PipelineManager(video_manager, self.on_pipeline_stopped, max_workers)
+
+    def _parse_json(self, value: str) -> Optional[dict]:
+        """Attempts to parse the value as JSON if applicable."""
+        if not value:
+            return None
+        
+        value = value.strip()  # Remove leading/trailing spaces
+        if (value.startswith("{") and value.endswith("}")) or (value.startswith("[") and value.endswith("]")):
+            try:
+                return json.loads(value)  # Parse JSON object or list
+            except json.JSONDecodeError:
+                pass  # Keep as string if parsing fails
+        return None
+        
+    def on_pipeline_stopped(self, pipeline_id: str) -> None:
+        """Set the pipeline as stopped in the database."""
+        pipeline = self.pipeline_repo.get_worker_source_pipeline(pipeline_id)
+        pipeline.pipeline_status_code = "run" if pipeline.pipeline_status_code == "restart" else "stop"
+        self.pipeline_repo.session.commit()
+
+    def run(self) -> None:
+        """Continuously updates pipelines based on database changes."""
+        while self.running:
+            try:
+                db_pipelines = {p.id: p for p in self.pipeline_repo.get_all_pipelines()}
+                
+                # Get pipeline IDs for comparison
+                local_pipeline_ids = set(self.pipeline_manager.get_active_pipelines())
+                db_pipeline_ids = set(db_pipelines.keys())
+
+                restarted_pipeline = False
+
+                # Process pipeline changes
+                self._add_new_pipelines(db_pipeline_ids - local_pipeline_ids, db_pipelines, restarted_pipeline)
+                self._remove_deleted_pipelines(local_pipeline_ids - db_pipeline_ids)
+                self._update_existing_pipelines(db_pipeline_ids & local_pipeline_ids, db_pipelines)
+
+                if restarted_pipeline:
+                    self.pipeline_repo.session.commit()
+
+                # Sync the cache to remove unused detectors
+                active_model_ids = {p.ai_model_id for p in db_pipelines.values() if p.pipeline_status_code == 'run'}
+                self.model_manager.sync_cache(active_model_ids)
+
+            except Exception as e:
+                logging.error(f"⚠️ Error syncing pipelines from database: {e}", exc_info=True)
+
+            time.sleep(self.polling_interval) 
+
+    def _add_new_pipelines(self, pipeline_ids: Set[str], db_pipelines: Dict[str, object], 
+                           restarted_pipeline: bool) -> None:
+        """Add new pipelines that exist in DB but not locally."""
+        for pid in pipeline_ids:
+            pipeline = db_pipelines[pid]
+
+            if pipeline.pipeline_status_code == 'restart':
+                pipeline.pipeline_status_code = 'run'
+                restarted_pipeline = True
+
+            if pipeline.pipeline_status_code == 'run':
+                detector = self.model_manager.get_detector(pipeline.ai_model_id)
+                
+                if not detector and pipeline.ai_model_id:
+                    logging.warning(f"⚠️ Could not load detector for pipeline {pid} ({pipeline.name}). Skipping.")
+                    continue
+                
+                logging.info(f"🟢 Adding new pipeline: {pid} ({pipeline.name})")
+                self.pipeline_manager.start_pipeline(pipeline, detector)
+
+    def _remove_deleted_pipelines(self, pipeline_ids: Set[str]) -> None:
+        """Remove pipelines that exist locally but not in DB."""
+        for pid in pipeline_ids:
+            logging.info(f"🔴 Removing deleted pipeline: {pid}")
+            self.pipeline_manager.stop_pipeline(pid)
+
+    def _update_existing_pipelines(self, pipeline_ids: Set[str], db_pipelines: Dict[str, object]) -> None:
+        """Update existing pipelines that need changes."""
+        debug_pipeline_ids = self.debug_repo.get_pipeline_ids_to_debug()
+
+        for pid in pipeline_ids:
+            db_pipeline = db_pipelines[pid]
+            processor = self.pipeline_manager.processors.get(pid)
+            if not processor:
+                continue
+
+            local_detector = processor.detector
+
+            self.update_pipeline(pid, db_pipeline, local_detector)
+            if pid in debug_pipeline_ids:
+                processor.enable_debug()
+
+    def update_pipeline(self, pid: str, db_pipeline: object, local_detector: object) -> None:
+        """Updates a single pipeline if necessary."""
+        processor = self.pipeline_manager.processors.get(pid)
+        if not processor:
+            return
+
+        # Stop/start based on status change
+        if db_pipeline.pipeline_status_code != processor._pipeline.pipeline_status_code:
+            if db_pipeline.pipeline_status_code == 'run':
+                logging.info(f"▶️ Resuming pipeline: {pid}")
+                self.pipeline_manager.start_pipeline(db_pipeline, self.model_manager.get_detector(db_pipeline.ai_model_id))
+            elif db_pipeline.pipeline_status_code in ['stop', 'restart']:
+                logging.info(f"⏹️ Stopping pipeline: {pid}")
+                self.pipeline_manager.stop_pipeline(pid)
+                if db_pipeline.pipeline_status_code == 'restart':
+                    # This will be picked up by the 'add_new_pipelines' logic in the next cycle
+                    return 
+            else:
+                 processor.update_config(db_pipeline) # Update config for non-running pipelines
+            return
+        elif db_pipeline.pipeline_status_code != 'run':
+            processor.update_config(db_pipeline)
+            return
+
+        # Check for significant changes that require a restart
+        db_detector = self.model_manager.get_detector(db_pipeline.ai_model_id)
+
+        requires_restart = any([
+            db_pipeline.ai_model_id != processor._pipeline.ai_model_id,
+            db_pipeline.worker_source_id != processor._pipeline.worker_source_id,
+            local_detector != db_detector
+        ])
+
+        if requires_restart and db_pipeline.pipeline_status_code == 'run':
+            logging.info(f"🔄 Restarting pipeline due to significant changes: {pid}")
+            self.pipeline_manager.stop_pipeline(pid)
+            self.pipeline_manager.start_pipeline(db_pipeline, db_detector)
+        else:
+            # Update config for minor changes that don't require restart
+            processor.update_config(db_pipeline)
+
+
+    def _has_pipeline_changed(self, local_pipeline, db_pipeline):
+        """Checks if the pipeline configuration has changed."""
+        if not local_pipeline or db_pipeline.pipeline_status_code == "restart":
+            return True
+
+        local_configs = local_pipeline.worker_source_pipeline_configs
+        db_configs = db_pipeline.worker_source_pipeline_configs
+
+        # Convert config objects to comparable structures
+        local_config_values = [
+            (config.pipeline_config_id, config.is_enabled, config.value, 
+             config.pipeline_config_name, config.pipeline_config_code)
+            for config in local_configs
+        ]
+
+        db_config_values = [
+            (config.pipeline_config_id, config.is_enabled, config.value, 
+             config.pipeline_config_name, config.pipeline_config_code)
+            for config in db_configs
+        ]
+
+        return sorted(local_config_values) != sorted(db_config_values)
+
+    def stop(self):
+        """Stops the synchronization thread and shuts down pipelines properly."""
+        logging.info("🛑 Stopping PipelineSyncThread...")
+        self.running = False
+        self.video_manager.stop_all()
+        self.pipeline_manager.shutdown()

{nedo_vision_worker_core-0.3.2 → nedo_vision_worker_core-0.3.4}/nedo_vision_worker_core/repositories/AIModelRepository.py

@@ -28,4 +28,24 @@ class AIModelRepository:
             return models
         except SQLAlchemyError as e:
             logging.error(f"Error retrieving models: {e}")
-            return []
+            return []
+
+    def get_model(self, model_id: str) -> AIModelEntity | None:
+        """
+        Retrieves a single AI model by its ID.
+
+        Args:
+            model_id: The ID of the model to retrieve.
+
+        Returns:
+            An AIModelEntity object or None if not found.
+        """
+        try:
+            self.session.expire_all()
+            model = self.session.query(AIModelEntity).filter_by(id=model_id).first()
+            if model:
+                self.session.expunge(model)
+            return model
+        except SQLAlchemyError as e:
+            logging.error(f"Error retrieving model {model_id}: {e}")
+            return None