nedo-vision-worker-core 0.2.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

nedo_vision_worker_core/detection/detection_processing/PPEDetectionProcessor.py

@@ -6,6 +6,7 @@ from .DetectionProcessor import DetectionProcessor
 from ...pipeline.PipelineConfigManager import PipelineConfigManager
 from ...repositories.PPEDetectionRepository import PPEDetectionRepository
 from ...util.PersonAttributeMatcher import PersonAttributeMatcher
+from ...callbacks import DetectionType, DetectionAttribute, BoundingBox, DetectionData
 
 class PPEDetectionProcessor(DetectionProcessor):
     code = "ppe"
@@ -15,22 +16,118 @@ class PPEDetectionProcessor(DetectionProcessor):
         "vest": "icons/vest-green.png",
         "no_vest": "icons/vest-red.png"
     }
-    labels = ["helmet", "no_helmet", "vest", "no_vest", "gloves", "no_gloves", "goggles", "no_goggles", "boots", "no_boots"]
-    violation_labels = ["no_helmet", "no_vest", "no_gloves", "no_goggles", "no_boots"]
-    compliance_labels = ["helmet", "vest", "gloves", "goggles", "boots"]
-    exclusive_labels = [("helmet", "no_helmet"), ("vest", "no_vest"), ("gloves", "no_gloves"), ("goggles", "no_goggles"), ("boots", "no_boots")]
-
+    
     def __init__(self):
         self.ppe_storage = PPEDetectionRepository()
         self.types = []
+        self.ppe_groups = {}
+        self.group_thresholds = {}
+        self.main_class_threshold = 0.7
+        self.main_class = "person"
+        
+        self.labels = ["helmet", "no_helmet", "vest", "no_vest", "gloves", "no_gloves", "goggles", "no_goggles", "boots", "no_boots"]
+        self.violation_labels = ["no_helmet", "no_vest", "no_gloves", "no_goggles", "no_boots"]
+        self.compliance_labels = ["helmet", "vest", "gloves", "goggles", "boots"]
+        self.exclusive_labels = [("helmet", "no_helmet"), ("vest", "no_vest"), ("gloves", "no_gloves"), ("goggles", "no_goggles"), ("boots", "no_boots")]
+
+    def update(self, config_manager: PipelineConfigManager, ai_model=None):
+        config = config_manager.get_feature_config(self.code, {})
+        
+        # Update from AI model
+        if ai_model:
+            self._update_from_ai_model(ai_model)
+        
+        # Update PPE type configuration
+        ppe_type_configs = config.get("ppeType", [])
+        self.types = []
+        self.group_thresholds = {}
+        
+        for ppe_config in ppe_type_configs:
+            if isinstance(ppe_config, dict):
+                group = ppe_config.get("group")
+                threshold = ppe_config.get("confidenceThreshold", 0.7)
+                if group:
+                    self.types.append(group)
+                    self.group_thresholds[group] = threshold
+            elif isinstance(ppe_config, str):
+                # Backward compatibility
+                self.types.append(ppe_config)
+                self.group_thresholds[ppe_config] = 0.7
+        
+        # Update main class threshold
+        self.main_class_threshold = config.get("mainClassConfidenceThreshold", 0.7)
+
+    def _update_from_ai_model(self, ai_model):
+        """Update processor settings from AI model configuration"""
+        if ai_model and hasattr(ai_model, 'ppe_groups') and ai_model.ppe_groups:
+            self.ppe_groups = {group.group_name: group for group in ai_model.ppe_groups}
+            self._build_labels_from_groups()
+            
+        if ai_model and hasattr(ai_model, 'main_class') and ai_model.main_class:
+            self.main_class = ai_model.main_class
 
-    def update(self, config_manager: PipelineConfigManager):
-        config = config_manager.get_feature_config(self.code, [])
-        self.types = config.get("ppeType", [])
+    def _build_labels_from_groups(self):
+        """Build standard PPE labels from AI model PPE groups"""
+        if not self.ppe_groups:
+            return
+            
+        labels = []
+        violation_labels = []
+        compliance_labels = []
+        exclusive_labels = []
+        
+        for group_name in self.ppe_groups.items():
+            compliance_class = group_name  
+            violation_class = f"no_{group_name}" 
+            
+            # Build label lists using standard naming
+            labels.extend([compliance_class, violation_class])
+            compliance_labels.append(compliance_class)
+            violation_labels.append(violation_class)
+            exclusive_labels.append((compliance_class, violation_class))
+        
+        # Update instance variables with dynamic labels
+        self.labels = labels
+        self.violation_labels = violation_labels
+        self.compliance_labels = compliance_labels
+        self.exclusive_labels = exclusive_labels
+
+    def get_multi_instance_classes(self):
+        """Get PPE classes that can have multiple instances per person"""
+        multi_instance_base = ["boots", "gloves", "goggles"]
+        multi_instance = []
+        
+        for label in self.labels:
+            base_label = label.replace("no_", "") if label.startswith("no_") else label
+            if base_label in multi_instance_base:
+                multi_instance.append(label)
+        return multi_instance
 
     def process(self, detections: List[Dict[str, Any]], dimension: Tuple[int, int]) -> List[Dict[str, Any]]:
-        persons = [d for d in detections if d["label"] == "person"]
-        ppe_attributes = [d for d in detections if any(x in d["label"] for x in self.types)]
+        persons = [d for d in detections if d["label"] == self.main_class]
+        
+        ppe_attributes = []
+        for detection in detections:
+            label = detection["label"]
+            
+            for group_name in self.types:
+                if group_name in self.ppe_groups:
+                    group_config = self.ppe_groups[group_name]
+                    
+                    original_compliance = group_config.get("compliance")
+                    original_violation = group_config.get("violation")
+                    
+                    if label in [original_compliance, original_violation]:
+                        if label == original_compliance:
+                            detection["label"] = group_name
+                        elif label == original_violation:
+                            detection["label"] = f"no_{group_name}"
+                        
+                        ppe_attributes.append(detection)
+                        break
+                elif label == group_name or label == f"no_{group_name}":
+                    ppe_attributes.append(detection)
+                    break
 
         matched_results = PersonAttributeMatcher.match_persons_with_attributes(
             persons, ppe_attributes, coverage_threshold=0.5
@@ -38,7 +135,73 @@ class PPEDetectionProcessor(DetectionProcessor):
 
         return matched_results
 
+    def get_class_thresholds(self):
+        """Get confidence thresholds for each class using original AI model class names"""
+        thresholds = {}
+        
+        for group_name, threshold in self.group_thresholds.items():
+            if group_name in self.ppe_groups:
+                group_config = self.ppe_groups[group_name]
+                
+                original_compliance = group_config.get("compliance")
+                original_violation = group_config.get("violation")
+                
+                if original_compliance:
+                    thresholds[original_compliance] = threshold
+                if original_violation:
+                    thresholds[original_violation] = threshold
+                    
+                thresholds[group_name] = threshold
+                thresholds[f"no_{group_name}"] = threshold
+            else:
+                thresholds[group_name] = threshold
+                if not group_name.startswith("no_"):
+                    thresholds[f"no_{group_name}"] = threshold
+        
+        return thresholds
+
     def save_to_db(self, pipeline_id: str, worker_source_id: str, frame_counter: int, tracked_objects: List[Dict[str, Any]], frame: np.ndarray, frame_drawer: FrameDrawer):
         self.ppe_storage.save_ppe_detection(
             pipeline_id, worker_source_id, frame_counter, tracked_objects, frame, frame_drawer
         )
+
+    @staticmethod
+    def create_detection_data(pipeline_id: str, worker_source_id: str, person_id: str, 
+                                    detection_id: str, tracked_obj: Dict[str, Any], 
+                                    image_path: str = "", image_tile_path: str = "",
+                                    frame_id: int = 0) -> DetectionData:
+        """Create DetectionData from PPE detection data."""
+        bbox = BoundingBox.from_list(tracked_obj["bbox"])
+        
+        attributes = []
+        for attr in tracked_obj.get("attributes", []):
+            attr_bbox = None
+            if "bbox" in attr:
+                attr_bbox = BoundingBox.from_list(attr["bbox"])
+            
+            # Determine if this is a violation based on label
+            is_violation = attr["label"].startswith("no_") or attr["label"] in [
+                "no_helmet", "no_vest", "no_gloves", "no_goggles", "no_boots"
+            ]
+            
+            attributes.append(DetectionAttribute(
+                label=attr["label"],
+                confidence=attr.get("confidence", 1.0),
+                count=attr.get("count", 0),
+                bbox=attr_bbox,
+                is_violation=is_violation
+            ))
+        
+        return DetectionData(
+            detection_type=DetectionType.PPE_DETECTION,
+            detection_id=detection_id,
+            person_id=person_id,
+            pipeline_id=pipeline_id,
+            worker_source_id=worker_source_id,
+            confidence_score=tracked_obj.get("confidence", 1.0),
+            bbox=bbox,
+            attributes=attributes,
+            image_path=image_path,
+            image_tile_path=image_tile_path,
+            frame_id=frame_id
+        )

nedo_vision_worker_core/models/ai_model.py

@@ -1,6 +1,6 @@
 import uuid
+import json
 from sqlalchemy import Column, String, DateTime
-from datetime import datetime
 from ..database.DatabaseManager import Base
 
 class AIModelEntity(Base):
@@ -15,6 +15,9 @@ class AIModelEntity(Base):
     download_status = Column(String, nullable=True, default="completed")  # pending, downloading, completed, failed
     last_download_attempt = Column(DateTime, nullable=True)
     download_error = Column(String, nullable=True)
+    classes = Column(String, nullable=True) 
+    ppe_class_groups = Column(String, nullable=True) 
+    main_class = Column(String, nullable=True)
 
     def __repr__(self):
         return (
@@ -38,4 +41,22 @@ class AIModelEntity(Base):
 
     def has_download_failed(self) -> bool:
         """Check if the model download has failed."""
-        return self.download_status == "failed"
+        return self.download_status == "failed"
+
+    def set_classes(self, classes_list):
+        self.classes = json.dumps(classes_list)
+
+    def get_classes(self):
+        return json.loads(self.classes) if self.classes else []
+
+    def set_ppe_class_groups(self, groups_list):
+        self.ppe_class_groups = json.dumps(groups_list)
+
+    def get_ppe_class_groups(self):
+        return json.loads(self.ppe_class_groups) if self.ppe_class_groups else []
+
+    def set_main_class(self, main_class):
+        self.main_class = main_class
+
+    def get_main_class(self):
+        return self.main_class or None

nedo_vision_worker_core/pipeline/PipelineProcessor.py

@@ -54,6 +54,32 @@ class PipelineProcessor:
         self.debug_flag = False
         self.debug_repo = WorkerSourcePipelineDebugRepository()
         self.detection_repo = WorkerSourcePipelineDetectionRepository()
+        
+        # Frame recovery mechanism
+        self.consecutive_frame_failures = 0
+        self.max_consecutive_failures = 150  # 1.5 seconds at 0.01s intervals
+        self.last_successful_frame_time = time.time()
+        self.stream_recovery_timeout = 30.0  # 30 seconds timeout for stream recovery
+        
+        # HEVC error tracking
+        self.hevc_error_count = 0
+        self.last_hevc_recovery = 0
+        self.hevc_recovery_cooldown = 30.0  # 30 seconds between HEVC recovery attempts
+
+    def load_model(self, model):
+        """
+        Load a new AI model into the detection manager.
+        This allows runtime model updates without restarting the pipeline.
+        
+        :param model: The new AI model to load
+        """
+        logging.info(f"🔄 Loading new model for pipeline {self.pipeline_id}: {model.name if model else 'None'}")
+        self.detection_manager.load_model(model)
+        
+        # Re-initialize detection processor to use the new model configuration
+        self._update_detection_processor()
+        
+        logging.info(f"✅ Model updated for pipeline {self.pipeline_id}")
 
     def _get_detection_processor_code(self):
         for code in self.detection_processor_codes:
@@ -83,18 +109,31 @@ class PipelineProcessor:
                 violation_labels=self.detection_processor.violation_labels,
                 compliance_labels=self.detection_processor.compliance_labels,
             )
-            self.tracker_manager.attribute_labels = self.detection_processor.labels
-            self.tracker_manager.exclusive_attribute_groups = self.detection_processor.exclusive_labels
+            multi_instance_classes = []
+            if hasattr(self.detection_processor, 'get_multi_instance_classes'):
+                multi_instance_classes = self.detection_processor.get_multi_instance_classes()
+            
+            self.tracker_manager.update_config(
+                attribute_labels=self.detection_processor.labels,
+                exclusive_attribute_groups=self.detection_processor.exclusive_labels,
+                multi_instance_classes=multi_instance_classes
+            )
         
     def _update_config(self):
         self.config_manager.update(self.pipeline_id)
         self.preprocessor.update(self.config_manager)
         self.detection_interval = self._get_detection_interval()
         self._update_detection_processor()
+        
+        # Reset frame failure counters on config update
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
+
+        ai_model = self.detection_manager.model_metadata
 
         if self.detection_processor:
             config = self.config_manager.get_feature_config(self.detection_processor.code)
-            self.detection_processor.update(self.config_manager)
+            self.detection_processor.update(self.config_manager, ai_model)
             self.threshold = config.get("minimumDetectionConfidence", 0.7)
 
             if self.detection_processor.code == HumanDetectionProcessor.code:
@@ -102,8 +141,11 @@ class PipelineProcessor:
         else:
             self.threshold = 0.7
             self.frame_drawer.update_config()
-            self.tracker_manager.attribute_labels = []
-            self.tracker_manager.exclusive_attribute_groups = []
+            self.tracker_manager.update_config(
+                attribute_labels=[], 
+                exclusive_attribute_groups=[], 
+                multi_instance_classes=[]
+            )
 
     def process_pipeline(self, video_manager: VideoStreamManager):
         """
@@ -115,6 +157,10 @@ class PipelineProcessor:
         logging.info(f"🎯 Running pipeline processing for pipeline {pipeline_id} | Source: {worker_source_id}")
 
         self._update_config()
+        
+        # Reset failure counters at start
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
 
         initial_frame = self._wait_for_frame(video_manager)
         if initial_frame is None:
@@ -136,14 +182,14 @@ class PipelineProcessor:
                 frame = video_manager.get_frame(worker_source_id)
 
                 if frame is None:
-                    logging.warning(f"⚠️ No frame available for {worker_source_id}. Retrying...")
-                    # Check if stream was removed
-                    if not video_manager.has_stream(worker_source_id):
-                        logging.info(f"🛑 Stream {worker_source_id} was removed, stopping pipeline")
+                    if not self._handle_frame_failure(video_manager, worker_source_id):
                         break
-                    time.sleep(0.01)
                     continue
 
+                # Reset failure counters on successful frame
+                self.consecutive_frame_failures = 0
+                self.last_successful_frame_time = time.time()
+
                 self.frame_counter += 1
 
                 self.frame_drawer.draw_polygons(frame)
@@ -195,10 +241,26 @@ class PipelineProcessor:
         dimension = frame.shape[:2]
 
         processed_frame = self.preprocessor.apply(frame)
-        detections = self.detection_manager.detect_objects(processed_frame, self.threshold)    
+        
+        class_thresholds = {}
+        ai_model = self.detection_manager.model_metadata
+        
+        if self.detection_processor:
+            if self.detection_processor.code == PPEDetectionProcessor.code:
+                class_thresholds.update(self.detection_processor.get_class_thresholds())
+            elif self.detection_processor.code == HumanDetectionProcessor.code:
+                main_threshold = self.detection_processor.get_main_class_threshold(ai_model)
+                if main_threshold and ai_model and ai_model.get_main_class():
+                    class_thresholds[ai_model.get_main_class()] = main_threshold
+        
+        detections = self.detection_manager.detect_objects(processed_frame, self.threshold, class_thresholds)    
         detections = self.preprocessor.revert_detections_bboxes(detections, dimension)
-        matched_results = self.detection_processor.process(detections, dimension)
-        return self.tracker_manager.track_objects(matched_results)
+        
+        if self.detection_processor:
+            matched_results = self.detection_processor.process(detections, dimension)
+            return self.tracker_manager.track_objects(matched_results)
+        else:
+            return self.tracker_manager.track_objects(detections)
 
 
     def _detection_worker(self):
@@ -262,15 +324,200 @@ class PipelineProcessor:
     
     def _wait_for_frame(self, video_manager, max_retries=10, sleep_time=3):
         """Waits until a frame is available from the video source."""
+        logging.info(f"⏳ Waiting for initial frame from {self.worker_source_id}...")
+        
         for retry_count in range(max_retries):
             frame = video_manager.get_frame(self.worker_source_id)
             if frame is not None:
+                logging.info(f"✅ Initial frame received from {self.worker_source_id}")
                 return frame
+            
+            # Check if stream exists
+            if not video_manager.has_stream(self.worker_source_id):
+                logging.error(f"❌ Stream {self.worker_source_id} not found in video manager")
+                return None
+            
             logging.warning(f"⚠️ Waiting for video stream {self.worker_source_id} (Attempt {retry_count + 1}/{max_retries})...")
+            
+            # Log stream diagnostics on later attempts
+            if retry_count >= 3:
+                self._log_stream_diagnostics(video_manager, self.worker_source_id)
+            
             time.sleep(sleep_time)
 
+        logging.error(f"❌ Failed to get initial frame from {self.worker_source_id} after {max_retries} attempts")
         return None
 
+    def _handle_frame_failure(self, video_manager, worker_source_id):
+        """
+        Handle frame retrieval failures with progressive backoff and recovery attempts.
+        Returns False if pipeline should stop, True to continue.
+        """
+        self.consecutive_frame_failures += 1
+        
+        # Check if stream was removed
+        if not video_manager.has_stream(worker_source_id):
+            logging.info(f"🛑 Stream {worker_source_id} was removed, stopping pipeline")
+            return False
+        
+        # Check for stream recovery timeout
+        time_since_last_frame = time.time() - self.last_successful_frame_time
+        if time_since_last_frame > self.stream_recovery_timeout:
+            logging.error(f"❌ Stream {worker_source_id} recovery timeout ({self.stream_recovery_timeout}s). Stopping pipeline.")
+            return False
+        
+        # Progressive logging and backoff
+        if self.consecutive_frame_failures <= 10:
+            # First 10 failures: minimal logging, fast retry
+            if self.consecutive_frame_failures % 5 == 1:  # Log every 5th failure
+                logging.debug(f"⚠️ No frame available for {worker_source_id} (attempt {self.consecutive_frame_failures})")
+            time.sleep(0.01)
+        elif self.consecutive_frame_failures <= 50:
+            # 11-50 failures: moderate logging, slightly longer wait
+            if self.consecutive_frame_failures % 10 == 1:  # Log every 10th failure
+                logging.warning(f"⚠️ No frame available for {worker_source_id} (attempt {self.consecutive_frame_failures}). Stream may be reconnecting...")
+            time.sleep(0.05)
+        elif self.consecutive_frame_failures <= self.max_consecutive_failures:
+            # 51-150 failures: more frequent logging, longer wait
+            if self.consecutive_frame_failures % 20 == 1:  # Log every 20th failure
+                logging.warning(f"⚠️ Persistent frame issues for {worker_source_id} (attempt {self.consecutive_frame_failures}). Checking stream health...")
+                self._log_stream_diagnostics(video_manager, worker_source_id)
+                
+                # Attempt HEVC recovery on severe persistent failures (every 60 failures to avoid too frequent reconnections)
+                if self.consecutive_frame_failures % 60 == 1:
+                    # Check if we should attempt HEVC recovery based on error patterns and cooldown
+                    if self._should_attempt_hevc_recovery(video_manager, worker_source_id):
+                        logging.info(f"🔧 Attempting HEVC-specific recovery for persistent frame failures...")
+                        recovery_success = self._handle_hevc_recovery(video_manager, worker_source_id)
+                        if recovery_success:
+                            logging.info(f"✅ HEVC recovery successful, continuing pipeline...")
+                            return True  # Continue processing after successful recovery
+                    
+            time.sleep(0.1)
+        else:
+            # Over max failures: critical logging and stop
+            logging.error(f"❌ Too many consecutive frame failures for {worker_source_id} ({self.consecutive_frame_failures}). Stopping pipeline.")
+            self._log_stream_diagnostics(video_manager, worker_source_id)
+            return False
+        
+        return True
+    
+    def _log_stream_diagnostics(self, video_manager, worker_source_id):
+        """Log diagnostic information about the stream state."""
+        try:
+            stream_url = video_manager.get_stream_url(worker_source_id)
+            is_file = video_manager.is_video_file(worker_source_id)
+            
+            # Get stream object for more detailed diagnostics
+            if hasattr(video_manager, 'streams') and worker_source_id in video_manager.streams:
+                stream = video_manager.streams[worker_source_id]
+                state = stream.get_state() if hasattr(stream, 'get_state') else "unknown"
+                is_connected = stream.is_connected() if hasattr(stream, 'is_connected') else "unknown"
+                
+                logging.info(f"📊 Stream diagnostics for {worker_source_id}:")
+                logging.info(f"   URL: {stream_url}")
+                logging.info(f"   Type: {'Video file' if is_file else 'Live stream'}")
+                logging.info(f"   State: {state}")
+                logging.info(f"   Connected: {is_connected}")
+                logging.info(f"   Time since last frame: {time.time() - self.last_successful_frame_time:.1f}s")
+                
+                # Check for HEVC/codec specific issues
+                if hasattr(stream, 'get_codec_info'):
+                    codec_info = stream.get_codec_info()
+                    if codec_info:
+                        logging.info(f"   Codec: {codec_info}")
+                        if 'hevc' in str(codec_info).lower() or 'h265' in str(codec_info).lower():
+                            logging.warning(f"   ⚠️ HEVC stream detected - may experience QP delta or POC reference errors")
+                
+                # Log recent error patterns if available
+                if hasattr(stream, 'get_recent_errors'):
+                    recent_errors = stream.get_recent_errors()
+                    if recent_errors:
+                        hevc_errors = [err for err in recent_errors if 'cu_qp_delta' in str(err.get('error', '')) or 'Could not find ref with POC' in str(err.get('error', ''))]
+                        if hevc_errors:
+                            logging.warning(f"   🔥 Recent HEVC errors detected: {len(hevc_errors)} codec-related errors")
+                            self.hevc_error_count += len(hevc_errors)
+                            
+                            # Log sample of recent HEVC errors for debugging
+                            for i, err in enumerate(hevc_errors[-3:]):  # Show last 3 errors
+                                logging.warning(f"   🔥 HEVC Error {i+1}: {err.get('error', '')[:100]}...")
+            else:
+                logging.info(f"📊 Stream {worker_source_id} not found in regular streams, checking direct device streams...")
+                
+        except Exception as e:
+            logging.error(f"Error getting stream diagnostics: {e}")
+
+    def _should_attempt_hevc_recovery(self, video_manager, worker_source_id) -> bool:
+        """
+        Determine if HEVC recovery should be attempted based on error patterns and cooldown.
+        """
+        current_time = time.time()
+        
+        # Check cooldown period
+        if current_time - self.last_hevc_recovery < self.hevc_recovery_cooldown:
+            logging.debug(f"HEVC recovery on cooldown ({current_time - self.last_hevc_recovery:.1f}s elapsed)")
+            return False
+        
+        # Check if stream has HEVC-related errors
+        if hasattr(video_manager, 'streams') and worker_source_id in video_manager.streams:
+            stream = video_manager.streams[worker_source_id]
+            if hasattr(stream, 'get_recent_errors'):
+                recent_errors = stream.get_recent_errors(max_age_seconds=60)  # Last minute
+                hevc_errors = [err for err in recent_errors if 
+                             'cu_qp_delta' in str(err.get('error', '')) or 
+                             'Could not find ref with POC' in str(err.get('error', ''))]
+                
+                if len(hevc_errors) >= 3:  # Threshold for HEVC errors
+                    logging.info(f"HEVC recovery warranted: {len(hevc_errors)} HEVC errors in last minute")
+                    return True
+        
+        # Check if we have accumulated enough general HEVC errors
+        if self.hevc_error_count >= 5:
+            logging.info(f"HEVC recovery warranted: {self.hevc_error_count} total HEVC errors detected")
+            return True
+        
+        return False
+
+    def _handle_hevc_recovery(self, video_manager, worker_source_id):
+        """
+        Handle HEVC-specific recovery strategies for codec errors.
+        This method attempts to recover from common HEVC issues like QP delta and POC reference errors.
+        """
+        try:
+            self.last_hevc_recovery = time.time()  # Update recovery timestamp
+            logging.info(f"🔧 Attempting HEVC stream recovery for {worker_source_id}")
+            
+            # Get the stream URL for recreation
+            stream_url = video_manager.get_stream_url(worker_source_id)
+            if not stream_url:
+                logging.error(f"   Cannot get stream URL for {worker_source_id}")
+                return False
+            
+            # Strategy 1: Remove and re-add the stream to reset decoder state
+            logging.info(f"   Recreating stream {worker_source_id} to reset decoder state...")
+            video_manager.remove_stream(worker_source_id)
+            time.sleep(1.0)  # Give time for cleanup
+            
+            # Re-add the stream
+            video_manager.add_stream(worker_source_id, stream_url)
+            time.sleep(2.0)  # Give time for stream to initialize
+            
+            # Strategy 2: Check if stream was successfully recreated
+            if not video_manager.has_stream(worker_source_id):
+                logging.error(f"   Failed to recreate stream {worker_source_id}")
+                return False
+            
+            # Strategy 3: Reset failure counters and error counts after recovery attempt
+            self.reset_frame_failure_counters()
+            self.hevc_error_count = 0  # Reset HEVC error counter
+            
+            logging.info(f"✅ HEVC recovery attempt completed for {worker_source_id}")
+            return True
+            
+        except Exception as e:
+            logging.error(f"❌ HEVC recovery failed for {worker_source_id}: {e}")
+            return False
+
     def stop(self):
         """Stops the Pipeline processor and cleans up resources."""
         if not self.running:  # Prevent multiple stops
@@ -335,4 +582,42 @@ class PipelineProcessor:
 
     def enable_debug(self):
         """Enable debug mode for this pipeline."""
-        self.debug_flag = True
+        self.debug_flag = True
+        # Reset failure counters when debug is enabled as it may help with recovery
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
+    
+    def reset_frame_failure_counters(self):
+        """Reset frame failure counters. Can be called externally to help with recovery."""
+        logging.info(f"🔄 Resetting frame failure counters for pipeline {self.pipeline_id}")
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
+        self.hevc_error_count = 0  # Also reset HEVC error count
+    
+    def get_hevc_diagnostics(self, video_manager) -> dict:
+        """Get HEVC-specific diagnostics for the pipeline."""
+        diagnostics = {
+            'hevc_error_count': self.hevc_error_count,
+            'last_hevc_recovery': self.last_hevc_recovery,
+            'time_since_last_recovery': time.time() - self.last_hevc_recovery,
+            'recovery_cooldown_remaining': max(0, self.hevc_recovery_cooldown - (time.time() - self.last_hevc_recovery)),
+            'consecutive_failures': self.consecutive_frame_failures,
+            'time_since_last_frame': time.time() - self.last_successful_frame_time,
+        }
+        
+        # Add stream-specific HEVC information
+        if hasattr(video_manager, 'streams') and self.worker_source_id in video_manager.streams:
+            stream = video_manager.streams[self.worker_source_id]
+            
+            if hasattr(stream, 'get_codec_info'):
+                diagnostics['codec'] = stream.get_codec_info()
+                
+            if hasattr(stream, 'get_recent_errors'):
+                recent_errors = stream.get_recent_errors(max_age_seconds=300)  # Last 5 minutes
+                hevc_errors = [err for err in recent_errors if 
+                             'cu_qp_delta' in str(err.get('error', '')) or 
+                             'Could not find ref with POC' in str(err.get('error', ''))]
+                diagnostics['recent_hevc_errors'] = len(hevc_errors)
+                diagnostics['total_recent_errors'] = len(recent_errors)
+        
+        return diagnostics

nedo_vision_worker_core/pipeline/PipelineSyncThread.py

@@ -171,6 +171,38 @@ class PipelineSyncThread(threading.Thread):
             else:
                 logging.warning(f"⚠️ Pipeline {pid}: {readiness['reason']}")
 
+        # Case 7: Model metadata has changed (same ID and version, but different properties)
+        elif local_model and db_model and local_model.id == db_model.id and local_model.version == db_model.version:
+            # Check if model metadata (classes, PPE groups, main_class) has changed
+            if self._has_model_metadata_changed(local_model, db_model):
+                readiness = ModelReadinessChecker.check_model_readiness(db_model)
+                if readiness["ready"]:
+                    local_proc.load_model(db_model)
+                    logging.info(f"🔄 Model metadata updated for pipeline {pid}: {db_pipeline.name} "
+                                 f"(same version {db_model.version}, updated properties)")
+                else:
+                    logging.warning(f"⚠️ Pipeline {pid}: {readiness['reason']}")
+
+    def _has_model_metadata_changed(self, local_model, db_model):
+        """Check if model metadata has changed without version change."""
+        # Compare classes
+        local_classes = set(local_model.get_classes() or [])
+        db_classes = set(db_model.get_classes() or [])
+        if local_classes != db_classes:
+            return True
+        
+        # Compare PPE class groups
+        local_ppe_groups = local_model.get_ppe_class_groups() or {}
+        db_ppe_groups = db_model.get_ppe_class_groups() or {}
+        if local_ppe_groups != db_ppe_groups:
+            return True
+        
+        # Compare main class
+        if local_model.get_main_class() != db_model.get_main_class():
+            return True
+        
+        return False
+
     def _has_pipeline_changed(self, local_pipeline, db_pipeline):
         """Checks if the pipeline configuration has changed."""
         if db_pipeline.pipeline_status_code == "restart":