nedo-vision-worker-core 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

nedo_vision_worker_core/__init__.py

@@ -7,7 +7,7 @@ A library for running AI vision processing and detection in the Nedo Vision plat
 from .core_service import CoreService
 from .callbacks import DetectionType, CallbackTrigger, DetectionData, IntervalMetadata
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"
 __all__ = [
     "CoreService", 
     "DetectionType", 

nedo_vision_worker_core/cli.py

@@ -91,61 +91,15 @@ Detection Callbacks:
     
     run_parser.add_argument(
         "--rtmp-server",
-        default="rtmp://localhost:1935/live",
-        help="RTMP server URL for video streaming (default: rtmp://localhost:1935/live)"
+        default="rtmp://live.vision.sindika.co.id:1935/live",
+        help="RTMP server URL for video streaming (default: rtmp://live.vision.sindika.co.id:1935/live)"
     )
     
     run_parser.add_argument(
-        "--enable-video-sharing-daemon",
-        action="store_true",
-        default=True,
-        help="Enable automatic video sharing daemon management (default: True)"
-    )
-    
-    run_parser.add_argument(
-        "--disable-video-sharing-daemon",
+        "--disable_video_sharing_daemon",
         action="store_true",
         default=False,
-        help="Disable automatic video sharing daemon management"
-    )
-    
-    # Add legacy arguments for backward compatibility (when no subcommand is used)
-    parser.add_argument(
-        "--drawing-assets",
-        help="(Legacy) Path to drawing assets directory (optional, uses bundled assets by default)"
-    )
-    
-    parser.add_argument(
-        "--log-level",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
-        default="INFO",
-        help="(Legacy) Logging level (default: INFO)"
-    )
-    
-    parser.add_argument(
-        "--storage-path",
-        default="data",
-        help="(Legacy) Storage path for databases and files (default: data)"
-    )
-    
-    parser.add_argument(
-        "--rtmp-server",
-        default="rtmp://localhost:1935/live",
-        help="(Legacy) RTMP server URL for video streaming (default: rtmp://localhost:1935/live)"
-    )
-    
-    parser.add_argument(
-        "--enable-video-sharing-daemon",
-        action="store_true",
-        default=True,
-        help="(Legacy) Enable automatic video sharing daemon management (default: True)"
-    )
-    
-    parser.add_argument(
-        "--disable-video-sharing-daemon",
-        action="store_true",
-        default=False,
-        help="(Legacy) Disable automatic video sharing daemon management"
+        help="Disable automatic video sharing daemon management (default: False)"
     )
     
     parser.add_argument(
@@ -161,11 +115,7 @@ Detection Callbacks:
         run_core_service(args)
     elif args.command == 'doctor':
         run_doctor()
-    elif hasattr(args, 'drawing_assets') and args.drawing_assets is not None:  # Legacy mode - if any arguments are provided without subcommand
-        print("⚠️  Warning: Using legacy command format. Consider using 'nedo-core run --drawing-assets ...' instead.")
-        run_core_service(args)
     else:
-        # If no subcommand provided, show help
         parser.print_help()
         sys.exit(1)
 
@@ -197,8 +147,6 @@ def run_core_service(args):
         enable_daemon = True  # Default
         if hasattr(args, 'disable_video_sharing_daemon') and args.disable_video_sharing_daemon:
             enable_daemon = False
-        elif hasattr(args, 'enable_video_sharing_daemon'):
-            enable_daemon = args.enable_video_sharing_daemon
         
         # Create and start the core service
         service = CoreService(

nedo_vision_worker_core/doctor.py

@@ -19,7 +19,7 @@ def check_python_version():
     print("🐍 Checking Python version...")
     
     version = sys.version_info
-    min_version = (3, 8)
+    min_version = (3, 10)
     
     if version >= min_version:
         print(f"   ✅ Python {version.major}.{version.minor}.{version.micro} (meets requirement >= {min_version[0]}.{min_version[1]})")

nedo_vision_worker_core/pipeline/PipelineProcessor.py

@@ -20,13 +20,6 @@ class PipelineProcessor:
     """Handles pipeline processing including preprocessing, AI model inference, tracking, and video stream processing."""
 
     def __init__(self, pipeline_id, worker_source_id, model, enable_visualization=True):
-        """
-        Initializes the PipelineProcessor with configurable detection labels.
-
-        :param model: The model to use for inference.
-        :param enable_visualization: Flag to enable visualization.
-        :param detection_labels: List of object labels to detect.
-        """
         self.running = True
         self.video_debugger = VideoDebugger(enable_visualization)
         self.tracker_manager = TrackerManager()
@@ -36,7 +29,9 @@ class PipelineProcessor:
         self.detection_processor = None
         self.threshold = 0.7
 
+        # Keep the latest frame for detection; size=1 and we overwrite when full
         self.frame_queue = queue.Queue(maxsize=1)
+
         self.tracked_objects_render = []
         self.detection_thread = None
         self.frame_counter = 0
@@ -54,27 +49,28 @@ class PipelineProcessor:
         self.debug_flag = False
         self.debug_repo = WorkerSourcePipelineDebugRepository()
         self.detection_repo = WorkerSourcePipelineDetectionRepository()
+        
+        # Frame recovery mechanism
+        self.consecutive_frame_failures = 0
+        self.max_consecutive_failures = 150  # 1.5 seconds at 0.01s intervals
+        self.last_successful_frame_time = time.time()
+        self.stream_recovery_timeout = 30.0  # 30 seconds timeout for stream recovery
+        
+        # HEVC error tracking
+        self.hevc_error_count = 0
+        self.last_hevc_recovery = 0
+        self.hevc_recovery_cooldown = 30.0  # 30 seconds between HEVC recovery attempts
 
     def load_model(self, model):
-        """
-        Load a new AI model into the detection manager.
-        This allows runtime model updates without restarting the pipeline.
-        
-        :param model: The new AI model to load
-        """
-        logging.info(f"🔄 Loading new model for pipeline {self.pipeline_id}: {model.name if model else 'None'}")
+        logging.info(f"🔄 Loading new model for pipeline {self.pipeline_id}: {getattr(model, 'name', None) or 'Unknown'}")
         self.detection_manager.load_model(model)
-        
-        # Re-initialize detection processor to use the new model configuration
         self._update_detection_processor()
-        
         logging.info(f"✅ Model updated for pipeline {self.pipeline_id}")
 
     def _get_detection_processor_code(self):
         for code in self.detection_processor_codes:
             if self.config_manager.is_feature_enabled(code):
                 return code
-            
         return None
     
     def _get_detection_processor(self, code):
@@ -87,7 +83,6 @@ class PipelineProcessor:
     
     def _update_detection_processor(self):
         code = self._get_detection_processor_code()
-
         if self.detection_processor and self.detection_processor.code == code:
             return
         
@@ -107,15 +102,22 @@ class PipelineProcessor:
                 exclusive_attribute_groups=self.detection_processor.exclusive_labels,
                 multi_instance_classes=multi_instance_classes
             )
-        
+        else:
+            # Reset drawer/tracker when no processor enabled
+            self.frame_drawer.update_config()
+            self.tracker_manager.update_config([], [], [])
+
     def _update_config(self):
         self.config_manager.update(self.pipeline_id)
         self.preprocessor.update(self.config_manager)
         self.detection_interval = self._get_detection_interval()
         self._update_detection_processor()
+        
+        # Reset failure counters on config update
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
 
         ai_model = self.detection_manager.model_metadata
-
         if self.detection_processor:
             config = self.config_manager.get_feature_config(self.detection_processor.code)
             self.detection_processor.update(self.config_manager, ai_model)
@@ -125,37 +127,30 @@ class PipelineProcessor:
                 self.frame_drawer.polygons = [((0, 0, 255), p) for p in self.detection_processor.restricted_areas]
         else:
             self.threshold = 0.7
-            self.frame_drawer.update_config()
-            self.tracker_manager.update_config(
-                attribute_labels=[], 
-                exclusive_attribute_groups=[], 
-                multi_instance_classes=[]
-            )
 
     def process_pipeline(self, video_manager: VideoStreamManager):
-        """
-        Runs the full pipeline processing including preprocessing, detection and tracking.
-        """
         pipeline_id = self.pipeline_id
         worker_source_id = self.worker_source_id
-
         logging.info(f"🎯 Running pipeline processing for pipeline {pipeline_id} | Source: {worker_source_id}")
 
         self._update_config()
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
 
         initial_frame = self._wait_for_frame(video_manager)
         if initial_frame is None:
             logging.error(f"❌ Pipeline {pipeline_id} | Source {worker_source_id}: No initial frame available. Exiting...")
             return
 
+        # Start RTMP (internal pacer thread will begin on first push_frame())
         self.rtmp_streamer = RTMPStreamer(pipeline_id)
 
-        # Start detection in a separate thread
+        # Start detection thread
         self.detection_thread = threading.Thread(
             target=self._detection_worker,
-            name=f"detection-{pipeline_id}"
+            name=f"detection-{pipeline_id}",
+            daemon=True
         )
-        self.detection_thread.daemon = True
         self.detection_thread.start()
 
         try:
@@ -163,64 +158,88 @@ class PipelineProcessor:
                 frame = video_manager.get_frame(worker_source_id)
 
                 if frame is None:
-                    logging.warning(f"⚠️ No frame available for {worker_source_id}. Retrying...")
-                    # Check if stream was removed
-                    if not video_manager.has_stream(worker_source_id):
-                        logging.info(f"🛑 Stream {worker_source_id} was removed, stopping pipeline")
+                    if not self._handle_frame_failure(video_manager, worker_source_id):
                         break
-                    time.sleep(0.01)
+                    # no frame this tick—just continue (the streamer will repeat last good frame)
                     continue
-
+                    
+                # cv2.imshow("AA", frame)
+                # cv2.waitKey(1)
+                # continue
+                
+                # successful frame
+                self.consecutive_frame_failures = 0
+                self.last_successful_frame_time = time.time()
                 self.frame_counter += 1
 
-                self.frame_drawer.draw_polygons(frame)
-                drawn_frame = self.frame_drawer.draw_frame(
-                    frame.copy(),
-                    self.tracked_objects_render,
-                    with_trails=True,
-                    trail_length=int(2 / self.detection_interval)
-                )
+                # draw annotations
+                try:
+                    self.frame_drawer.draw_polygons(frame)
+                    drawn_frame = self.frame_drawer.draw_frame(
+                        frame.copy(),
+                        self.tracked_objects_render,
+                        with_trails=True,
+                        trail_length=int(max(1, 2 / self.detection_interval))
+                    )
+                except Exception as e:
+                    logging.error(f"❌ Draw failed, using raw frame: {e}")
+                    drawn_frame = frame
 
+                # debug snapshot if requested
                 if self.debug_flag:
                     tracked_objects_render = self._process_frame(frame)
-
-                    self.debug_repo.update_debug_entries_by_pipeline_id(
-                        self.pipeline_id, 
-                        self.frame_drawer.draw_frame(
-                            frame.copy(),
+                    try:
+                        self.debug_repo.update_debug_entries_by_pipeline_id(
+                            self.pipeline_id,
+                            self.frame_drawer.draw_frame(frame.copy(), tracked_objects_render),
                             tracked_objects_render
-                        ),
-                        tracked_objects_render
-                    )
+                        )
+                    except Exception as e:
+                        logging.warning(f"Debug save failed: {e}")
                     self.debug_flag = False
 
-                # Check RTMP streamer status before sending frame
-                if self.rtmp_streamer:
+                # ---- RTMP push (latest-only; pacing handled inside RTMPStreamer) ----
+                if self.rtmp_streamer is None or not self.rtmp_streamer.is_active():
+                    # (re)create; it will learn WxH on first push
+                    self.rtmp_streamer = RTMPStreamer(self.pipeline_id)
+
+                try:
+                    self.rtmp_streamer.push_frame(drawn_frame)
+                except Exception as e:
+                    logging.error(f"❌ RTMP push error: {e}")
                     try:
-                        self.rtmp_streamer.send_frame(drawn_frame)
-                    except Exception as e:
-                        logging.error(f"❌ RTMP streaming error: {e}")
-                        # Stop RTMP streamer on error
                         self.rtmp_streamer.stop_stream()
-                        self.rtmp_streamer = None
+                    except Exception:
+                        pass
+                    self.rtmp_streamer = None
 
-                # Only put frame in queue if detection thread is still running
+                # feed detection worker with latest-only behavior
                 if self.detection_thread and self.detection_thread.is_alive():
-                    if not self.frame_queue.full():
-                        self.frame_queue.put(frame, block=False)
-
+                    try:
+                        self.frame_queue.put_nowait(frame)
+                    except queue.Full:
+                        try:
+                            _ = self.frame_queue.get_nowait()
+                        except queue.Empty:
+                            pass
+                        try:
+                            self.frame_queue.put_nowait(frame)
+                        except queue.Full:
+                            pass
+
+                # visualize
                 try:
                     self.video_debugger.show_frame(pipeline_id, worker_source_id, drawn_frame)
                 except Exception as e:
                     logging.error(f"⚠️ Failed to render frame for pipeline {pipeline_id}: {e}")
+                
+                time.sleep(0.1)
 
-                time.sleep(0.01)
         except Exception as e:
             logging.error(f"❌ Error in pipeline {pipeline_id}: {e}", exc_info=True)
 
     def _process_frame(self, frame):
         dimension = frame.shape[:2]
-
         processed_frame = self.preprocessor.apply(frame)
         
         class_thresholds = {}
@@ -234,7 +253,7 @@ class PipelineProcessor:
                 if main_threshold and ai_model and ai_model.get_main_class():
                     class_thresholds[ai_model.get_main_class()] = main_threshold
         
-        detections = self.detection_manager.detect_objects(processed_frame, self.threshold, class_thresholds)    
+        detections = self.detection_manager.detect_objects(processed_frame, self.threshold, class_thresholds)
         detections = self.preprocessor.revert_detections_bboxes(detections, dimension)
         
         if self.detection_processor:
@@ -243,17 +262,13 @@ class PipelineProcessor:
         else:
             return self.tracker_manager.track_objects(detections)
 
-
     def _detection_worker(self):
-        """
-        Runs detection in a separate thread and updates configuration periodically.
-        Applies preprocessing based on pipeline configuration.
-        """
+        """Runs detection in a separate thread and updates configuration periodically."""
         pipeline_id = self.pipeline_id
         worker_source_id = self.worker_source_id
         last_detection_time = time.time()
-        last_config_update_time = time.time()  
-        config_update_interval = 5  # Update configuration every 5 seconds
+        last_config_update_time = time.time()
+        config_update_interval = 5  # seconds
 
         while self.running:
             try:
@@ -263,16 +278,22 @@ class PipelineProcessor:
                 # Update config periodically
                 if (current_time - last_config_update_time) >= config_update_interval:
                     self._update_config()
-                    last_config_update_time = current_time 
+                    last_config_update_time = current_time
                     logging.info(f"🔄 Updated pipeline config for {pipeline_id}")
 
-                # Process detection only if enough time has passed since last detection
-                # detection_interval is the time in seconds between consecutive detections
+                # Keep only the latest frame if we fell behind
+                try:
+                    while True:
+                        newer = self.frame_queue.get_nowait()
+                        frame = newer
+                except queue.Empty:
+                    pass
+
+                # Respect detection interval
                 if (current_time - last_detection_time) < self.detection_interval:
                     continue
+                last_detection_time = current_time
 
-                last_detection_time = current_time 
-                
                 if self.detection_processor is None or frame is None or frame.size == 0:
                     self.tracked_objects_render = []
                     continue
@@ -304,33 +325,165 @@ class PipelineProcessor:
                 logging.error(f"❌ Error in detection thread for pipeline {pipeline_id}: {e}", exc_info=True)
     
     def _wait_for_frame(self, video_manager, max_retries=10, sleep_time=3):
-        """Waits until a frame is available from the video source."""
+        logging.info(f"⏳ Waiting for initial frame from {self.worker_source_id}...")
         for retry_count in range(max_retries):
             frame = video_manager.get_frame(self.worker_source_id)
             if frame is not None:
+                logging.info(f"✅ Initial frame received from {self.worker_source_id}")
                 return frame
+            
+            if not video_manager.has_stream(self.worker_source_id):
+                logging.error(f"❌ Stream {self.worker_source_id} not found in video manager")
+                return None
+            
             logging.warning(f"⚠️ Waiting for video stream {self.worker_source_id} (Attempt {retry_count + 1}/{max_retries})...")
+            if retry_count >= 3:
+                self._log_stream_diagnostics(video_manager, self.worker_source_id)
             time.sleep(sleep_time)
 
+        logging.error(f"❌ Failed to get initial frame from {self.worker_source_id} after {max_retries} attempts")
         return None
 
+    def _handle_frame_failure(self, video_manager, worker_source_id):
+        """Handle frame retrieval failures with progressive backoff and recovery attempts."""
+        self.consecutive_frame_failures += 1
+        
+        if not video_manager.has_stream(worker_source_id):
+            logging.info(f"🛑 Stream {worker_source_id} was removed, stopping pipeline")
+            return False
+        
+        time_since_last_frame = time.time() - self.last_successful_frame_time
+        if time_since_last_frame > self.stream_recovery_timeout:
+            logging.error(f"❌ Stream {worker_source_id} recovery timeout ({self.stream_recovery_timeout}s). Stopping pipeline.")
+            return False
+        
+        if self.consecutive_frame_failures <= 10:
+            if self.consecutive_frame_failures % 5 == 1:
+                logging.debug(f"⚠️ No frame for {worker_source_id} (attempt {self.consecutive_frame_failures})")
+            time.sleep(0.01)
+        elif self.consecutive_frame_failures <= 50:
+            if self.consecutive_frame_failures % 10 == 1:
+                logging.warning(f"⚠️ No frame for {worker_source_id} (attempt {self.consecutive_frame_failures}). Stream may be reconnecting...")
+            time.sleep(0.05)
+        elif self.consecutive_frame_failures <= self.max_consecutive_failures:
+            if self.consecutive_frame_failures % 20 == 1:
+                logging.warning(f"⚠️ Persistent frame issues for {worker_source_id} (attempt {self.consecutive_frame_failures}). Checking stream health...")
+                self._log_stream_diagnostics(video_manager, worker_source_id)
+                if self.consecutive_frame_failures % 60 == 1:
+                    if self._should_attempt_hevc_recovery(video_manager, worker_source_id):
+                        logging.info("🔧 Attempting HEVC-specific recovery for persistent frame failures...")
+                        if self._handle_hevc_recovery(video_manager, worker_source_id):
+                            logging.info("✅ HEVC recovery successful, continuing pipeline...")
+                            return True
+            time.sleep(0.1)
+        else:
+            logging.error(f"❌ Too many consecutive frame failures for {worker_source_id} ({self.consecutive_frame_failures}). Stopping pipeline.")
+            self._log_stream_diagnostics(video_manager, worker_source_id)
+            return False
+        
+        return True
+    
+    def _log_stream_diagnostics(self, video_manager, worker_source_id):
+        try:
+            stream_url = video_manager.get_stream_url(worker_source_id)
+            is_file = video_manager.is_video_file(worker_source_id)
+            if hasattr(video_manager, 'streams') and worker_source_id in video_manager.streams:
+                stream = video_manager.streams[worker_source_id]
+                state = stream.get_state() if hasattr(stream, 'get_state') else "unknown"
+                is_connected = stream.is_connected() if hasattr(stream, 'is_connected') else "unknown"
+                
+                logging.info(f"📊 Stream diagnostics for {worker_source_id}:")
+                logging.info(f"   URL: {stream_url}")
+                logging.info(f"   Type: {'Video file' if is_file else 'Live stream'}")
+                logging.info(f"   State: {state}")
+                logging.info(f"   Connected: {is_connected}")
+                logging.info(f"   Time since last frame: {time.time() - self.last_successful_frame_time:.1f}s")
+                
+                if hasattr(stream, 'get_codec_info'):
+                    codec_info = stream.get_codec_info()
+                    if codec_info:
+                        logging.info(f"   Codec: {codec_info}")
+                        if 'hevc' in str(codec_info).lower() or 'h265' in str(codec_info).lower():
+                            logging.warning("   ⚠️ HEVC stream detected - potential QP/POC errors")
+                
+                if hasattr(stream, 'get_recent_errors'):
+                    recent_errors = stream.get_recent_errors()
+                    if recent_errors:
+                        hevc_errors = [err for err in recent_errors if 'cu_qp_delta' in str(err.get('error', '')) or 'Could not find ref with POC' in str(err.get('error', ''))]
+                        if hevc_errors:
+                            logging.warning(f"   🔥 Recent HEVC errors: {len(hevc_errors)}")
+                            self.hevc_error_count += len(hevc_errors)
+                            for i, err in enumerate(hevc_errors[-3:]):
+                                logging.warning(f"   🔥 HEVC Error {i+1}: {err.get('error', '')[:100]}...")
+            else:
+                logging.info(f"📊 Stream {worker_source_id} not found in registry; checking device directly...")
+        except Exception as e:
+            logging.error(f"Error getting stream diagnostics: {e}")
+
+    def _should_attempt_hevc_recovery(self, video_manager, worker_source_id) -> bool:
+        current_time = time.time()
+        if current_time - self.last_hevc_recovery < self.hevc_recovery_cooldown:
+            logging.debug(f"HEVC recovery on cooldown ({current_time - self.last_hevc_recovery:.1f}s elapsed)")
+            return False
+        
+        if hasattr(video_manager, 'streams') and worker_source_id in video_manager.streams:
+            stream = video_manager.streams[worker_source_id]
+            if hasattr(stream, 'get_recent_errors'):
+                recent_errors = stream.get_recent_errors(max_age_seconds=60)
+                hevc_errors = [err for err in recent_errors if 
+                               'cu_qp_delta' in str(err.get('error', '')) or 
+                               'Could not find ref with POC' in str(err.get('error', ''))]
+                if len(hevc_errors) >= 3:
+                    logging.info(f"HEVC recovery warranted: {len(hevc_errors)} HEVC errors in last minute")
+                    return True
+        
+        if self.hevc_error_count >= 5:
+            logging.info(f"HEVC recovery warranted: {self.hevc_error_count} total HEVC errors detected")
+            return True
+        
+        return False
+
+    def _handle_hevc_recovery(self, video_manager, worker_source_id):
+        try:
+            self.last_hevc_recovery = time.time()
+            logging.info(f"🔧 Attempting HEVC stream recovery for {worker_source_id}")
+            stream_url = video_manager.get_stream_url(worker_source_id)
+            if not stream_url:
+                logging.error(f"   Cannot get stream URL for {worker_source_id}")
+                return False
+            
+            video_manager.remove_stream(worker_source_id)
+            time.sleep(1.0)
+            video_manager.add_stream(worker_source_id, stream_url)
+            time.sleep(2.0)
+            
+            if not video_manager.has_stream(worker_source_id):
+                logging.error(f"   Failed to recreate stream {worker_source_id}")
+                return False
+            
+            self.reset_frame_failure_counters()
+            self.hevc_error_count = 0
+            logging.info(f"✅ HEVC recovery attempt completed for {worker_source_id}")
+            return True
+        except Exception as e:
+            logging.error(f"❌ HEVC recovery failed for {worker_source_id}: {e}")
+            return False
+
     def stop(self):
         """Stops the Pipeline processor and cleans up resources."""
-        if not self.running:  # Prevent multiple stops
+        if not self.running:
             return
-            
         logging.info("🛑 Stopping PipelineProcessor...")
         self.running = False
 
-        # Stop RTMP streamer first
         if hasattr(self, 'rtmp_streamer') and self.rtmp_streamer:
             try:
                 self.rtmp_streamer.stop_stream()
-                self.rtmp_streamer = None
             except Exception as e:
                 logging.error(f"Error stopping RTMP streamer: {e}")
+            finally:
+                self.rtmp_streamer = None
 
-        # Clear frame queue before joining thread
         try:
             while True:
                 try:
@@ -340,10 +493,9 @@ class PipelineProcessor:
         except Exception as e:
             logging.error(f"Error clearing frame queue: {e}")
 
-        # Wait for detection thread with increased timeout
         if self.detection_thread and self.detection_thread.is_alive():
             try:
-                self.detection_thread.join(timeout=5.0)  # Increased timeout
+                self.detection_thread.join(timeout=5.0)
                 if self.detection_thread.is_alive():
                     logging.warning("Detection thread did not terminate cleanly")
             except Exception as e:
@@ -351,10 +503,8 @@ class PipelineProcessor:
             finally:
                 self.detection_thread = None
 
-        # Clear tracked objects
-        self.tracked_objects_render.clear()  # Use clear() instead of reassignment
+        self.tracked_objects_render.clear()
 
-        # Close video debugger windows last
         try:
             if hasattr(self, 'video_debugger'):
                 self.video_debugger.close_all()
@@ -364,18 +514,41 @@ class PipelineProcessor:
         logging.info("✅ PipelineProcessor stopped successfully")
         
     def _get_detection_interval(self):
-        """
-        Get detection interval from configuration.
-        Converts frames per second to seconds per frame.
-        """
         config = self.config_manager.get_feature_config("processing_speed")
         fps = config.get("decimal", 1.0)
-        
         if fps <= 0:
-            return 1 / 10  # Default to 10 frame per second if fps is zero or negative 
-        
-        return 1.0 / fps  # Convert fps to seconds per frame
+            return 1.0 / 10.0  # default 10 fps
+        return 1.0 / fps
 
     def enable_debug(self):
-        """Enable debug mode for this pipeline."""
-        self.debug_flag = True
+        self.debug_flag = True
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
+    
+    def reset_frame_failure_counters(self):
+        logging.info(f"🔄 Resetting frame failure counters for pipeline {self.pipeline_id}")
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
+        self.hevc_error_count = 0
+    
+    def get_hevc_diagnostics(self, video_manager) -> dict:
+        diagnostics = {
+            'hevc_error_count': self.hevc_error_count,
+            'last_hevc_recovery': self.last_hevc_recovery,
+            'time_since_last_recovery': time.time() - self.last_hevc_recovery,
+            'recovery_cooldown_remaining': max(0, self.hevc_recovery_cooldown - (time.time() - self.last_hevc_recovery)),
+            'consecutive_failures': self.consecutive_frame_failures,
+            'time_since_last_frame': time.time() - self.last_successful_frame_time,
+        }
+        if hasattr(video_manager, 'streams') and self.worker_source_id in video_manager.streams:
+            stream = video_manager.streams[self.worker_source_id]
+            if hasattr(stream, 'get_codec_info'):
+                diagnostics['codec'] = stream.get_codec_info()
+            if hasattr(stream, 'get_recent_errors'):
+                recent_errors = stream.get_recent_errors(max_age_seconds=300)
+                hevc_errors = [err for err in recent_errors if 
+                               'cu_qp_delta' in str(err.get('error', '')) or 
+                               'Could not find ref with POC' in str(err.get('error', ''))]
+                diagnostics['recent_hevc_errors'] = len(hevc_errors)
+                diagnostics['total_recent_errors'] = len(recent_errors)
+        return diagnostics