nedo-vision-worker-core 0.3.0__tar.gz → 0.3.1__tar.gz

{nedo_vision_worker_core-0.3.0 → nedo_vision_worker_core-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker-core
-Version: 0.3.0
+Version: 0.3.1
 Summary: Nedo Vision Worker Core Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>

{nedo_vision_worker_core-0.3.0 → nedo_vision_worker_core-0.3.1}/nedo_vision_worker_core/__init__.py

@@ -7,7 +7,7 @@ A library for running AI vision processing and detection in the Nedo Vision plat
 from .core_service import CoreService
 from .callbacks import DetectionType, CallbackTrigger, DetectionData, IntervalMetadata
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 __all__ = [
     "CoreService", 
     "DetectionType", 

{nedo_vision_worker_core-0.3.0 → nedo_vision_worker_core-0.3.1}/nedo_vision_worker_core/cli.py

@@ -91,61 +91,15 @@ Detection Callbacks:
     
     run_parser.add_argument(
         "--rtmp-server",
-        default="rtmp://localhost:1935/live",
-        help="RTMP server URL for video streaming (default: rtmp://localhost:1935/live)"
+        default="rtmp://live.vision.sindika.co.id:1935/live",
+        help="RTMP server URL for video streaming (default: rtmp://live.vision.sindika.co.id:1935/live)"
     )
     
     run_parser.add_argument(
-        "--enable-video-sharing-daemon",
-        action="store_true",
-        default=True,
-        help="Enable automatic video sharing daemon management (default: True)"
-    )
-    
-    run_parser.add_argument(
-        "--disable-video-sharing-daemon",
+        "--disable_video_sharing_daemon",
         action="store_true",
         default=False,
-        help="Disable automatic video sharing daemon management"
-    )
-    
-    # Add legacy arguments for backward compatibility (when no subcommand is used)
-    parser.add_argument(
-        "--drawing-assets",
-        help="(Legacy) Path to drawing assets directory (optional, uses bundled assets by default)"
-    )
-    
-    parser.add_argument(
-        "--log-level",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
-        default="INFO",
-        help="(Legacy) Logging level (default: INFO)"
-    )
-    
-    parser.add_argument(
-        "--storage-path",
-        default="data",
-        help="(Legacy) Storage path for databases and files (default: data)"
-    )
-    
-    parser.add_argument(
-        "--rtmp-server",
-        default="rtmp://localhost:1935/live",
-        help="(Legacy) RTMP server URL for video streaming (default: rtmp://localhost:1935/live)"
-    )
-    
-    parser.add_argument(
-        "--enable-video-sharing-daemon",
-        action="store_true",
-        default=True,
-        help="(Legacy) Enable automatic video sharing daemon management (default: True)"
-    )
-    
-    parser.add_argument(
-        "--disable-video-sharing-daemon",
-        action="store_true",
-        default=False,
-        help="(Legacy) Disable automatic video sharing daemon management"
+        help="Disable automatic video sharing daemon management (default: False)"
     )
     
     parser.add_argument(
@@ -161,11 +115,7 @@ Detection Callbacks:
         run_core_service(args)
     elif args.command == 'doctor':
         run_doctor()
-    elif hasattr(args, 'drawing_assets') and args.drawing_assets is not None:  # Legacy mode - if any arguments are provided without subcommand
-        print("⚠️  Warning: Using legacy command format. Consider using 'nedo-core run --drawing-assets ...' instead.")
-        run_core_service(args)
     else:
-        # If no subcommand provided, show help
         parser.print_help()
         sys.exit(1)
 
@@ -197,8 +147,6 @@ def run_core_service(args):
         enable_daemon = True  # Default
         if hasattr(args, 'disable_video_sharing_daemon') and args.disable_video_sharing_daemon:
             enable_daemon = False
-        elif hasattr(args, 'enable_video_sharing_daemon'):
-            enable_daemon = args.enable_video_sharing_daemon
         
         # Create and start the core service
         service = CoreService(

{nedo_vision_worker_core-0.3.0 → nedo_vision_worker_core-0.3.1}/nedo_vision_worker_core/pipeline/PipelineProcessor.py

@@ -54,6 +54,17 @@ class PipelineProcessor:
         self.debug_flag = False
         self.debug_repo = WorkerSourcePipelineDebugRepository()
         self.detection_repo = WorkerSourcePipelineDetectionRepository()
+        
+        # Frame recovery mechanism
+        self.consecutive_frame_failures = 0
+        self.max_consecutive_failures = 150  # 1.5 seconds at 0.01s intervals
+        self.last_successful_frame_time = time.time()
+        self.stream_recovery_timeout = 30.0  # 30 seconds timeout for stream recovery
+        
+        # HEVC error tracking
+        self.hevc_error_count = 0
+        self.last_hevc_recovery = 0
+        self.hevc_recovery_cooldown = 30.0  # 30 seconds between HEVC recovery attempts
 
     def load_model(self, model):
         """
@@ -113,6 +124,10 @@ class PipelineProcessor:
         self.preprocessor.update(self.config_manager)
         self.detection_interval = self._get_detection_interval()
         self._update_detection_processor()
+        
+        # Reset frame failure counters on config update
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
 
         ai_model = self.detection_manager.model_metadata
 
@@ -142,6 +157,10 @@ class PipelineProcessor:
         logging.info(f"🎯 Running pipeline processing for pipeline {pipeline_id} | Source: {worker_source_id}")
 
         self._update_config()
+        
+        # Reset failure counters at start
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
 
         initial_frame = self._wait_for_frame(video_manager)
         if initial_frame is None:
@@ -163,14 +182,14 @@ class PipelineProcessor:
                 frame = video_manager.get_frame(worker_source_id)
 
                 if frame is None:
-                    logging.warning(f"⚠️ No frame available for {worker_source_id}. Retrying...")
-                    # Check if stream was removed
-                    if not video_manager.has_stream(worker_source_id):
-                        logging.info(f"🛑 Stream {worker_source_id} was removed, stopping pipeline")
+                    if not self._handle_frame_failure(video_manager, worker_source_id):
                         break
-                    time.sleep(0.01)
                     continue
 
+                # Reset failure counters on successful frame
+                self.consecutive_frame_failures = 0
+                self.last_successful_frame_time = time.time()
+
                 self.frame_counter += 1
 
                 self.frame_drawer.draw_polygons(frame)
@@ -305,15 +324,200 @@ class PipelineProcessor:
     
     def _wait_for_frame(self, video_manager, max_retries=10, sleep_time=3):
         """Waits until a frame is available from the video source."""
+        logging.info(f"⏳ Waiting for initial frame from {self.worker_source_id}...")
+        
         for retry_count in range(max_retries):
             frame = video_manager.get_frame(self.worker_source_id)
             if frame is not None:
+                logging.info(f"✅ Initial frame received from {self.worker_source_id}")
                 return frame
+            
+            # Check if stream exists
+            if not video_manager.has_stream(self.worker_source_id):
+                logging.error(f"❌ Stream {self.worker_source_id} not found in video manager")
+                return None
+            
             logging.warning(f"⚠️ Waiting for video stream {self.worker_source_id} (Attempt {retry_count + 1}/{max_retries})...")
+            
+            # Log stream diagnostics on later attempts
+            if retry_count >= 3:
+                self._log_stream_diagnostics(video_manager, self.worker_source_id)
+            
             time.sleep(sleep_time)
 
+        logging.error(f"❌ Failed to get initial frame from {self.worker_source_id} after {max_retries} attempts")
         return None
 
+    def _handle_frame_failure(self, video_manager, worker_source_id):
+        """
+        Handle frame retrieval failures with progressive backoff and recovery attempts.
+        Returns False if pipeline should stop, True to continue.
+        """
+        self.consecutive_frame_failures += 1
+        
+        # Check if stream was removed
+        if not video_manager.has_stream(worker_source_id):
+            logging.info(f"🛑 Stream {worker_source_id} was removed, stopping pipeline")
+            return False
+        
+        # Check for stream recovery timeout
+        time_since_last_frame = time.time() - self.last_successful_frame_time
+        if time_since_last_frame > self.stream_recovery_timeout:
+            logging.error(f"❌ Stream {worker_source_id} recovery timeout ({self.stream_recovery_timeout}s). Stopping pipeline.")
+            return False
+        
+        # Progressive logging and backoff
+        if self.consecutive_frame_failures <= 10:
+            # First 10 failures: minimal logging, fast retry
+            if self.consecutive_frame_failures % 5 == 1:  # Log every 5th failure
+                logging.debug(f"⚠️ No frame available for {worker_source_id} (attempt {self.consecutive_frame_failures})")
+            time.sleep(0.01)
+        elif self.consecutive_frame_failures <= 50:
+            # 11-50 failures: moderate logging, slightly longer wait
+            if self.consecutive_frame_failures % 10 == 1:  # Log every 10th failure
+                logging.warning(f"⚠️ No frame available for {worker_source_id} (attempt {self.consecutive_frame_failures}). Stream may be reconnecting...")
+            time.sleep(0.05)
+        elif self.consecutive_frame_failures <= self.max_consecutive_failures:
+            # 51-150 failures: more frequent logging, longer wait
+            if self.consecutive_frame_failures % 20 == 1:  # Log every 20th failure
+                logging.warning(f"⚠️ Persistent frame issues for {worker_source_id} (attempt {self.consecutive_frame_failures}). Checking stream health...")
+                self._log_stream_diagnostics(video_manager, worker_source_id)
+                
+                # Attempt HEVC recovery on severe persistent failures (every 60 failures to avoid too frequent reconnections)
+                if self.consecutive_frame_failures % 60 == 1:
+                    # Check if we should attempt HEVC recovery based on error patterns and cooldown
+                    if self._should_attempt_hevc_recovery(video_manager, worker_source_id):
+                        logging.info(f"🔧 Attempting HEVC-specific recovery for persistent frame failures...")
+                        recovery_success = self._handle_hevc_recovery(video_manager, worker_source_id)
+                        if recovery_success:
+                            logging.info(f"✅ HEVC recovery successful, continuing pipeline...")
+                            return True  # Continue processing after successful recovery
+                    
+            time.sleep(0.1)
+        else:
+            # Over max failures: critical logging and stop
+            logging.error(f"❌ Too many consecutive frame failures for {worker_source_id} ({self.consecutive_frame_failures}). Stopping pipeline.")
+            self._log_stream_diagnostics(video_manager, worker_source_id)
+            return False
+        
+        return True
+    
+    def _log_stream_diagnostics(self, video_manager, worker_source_id):
+        """Log diagnostic information about the stream state."""
+        try:
+            stream_url = video_manager.get_stream_url(worker_source_id)
+            is_file = video_manager.is_video_file(worker_source_id)
+            
+            # Get stream object for more detailed diagnostics
+            if hasattr(video_manager, 'streams') and worker_source_id in video_manager.streams:
+                stream = video_manager.streams[worker_source_id]
+                state = stream.get_state() if hasattr(stream, 'get_state') else "unknown"
+                is_connected = stream.is_connected() if hasattr(stream, 'is_connected') else "unknown"
+                
+                logging.info(f"📊 Stream diagnostics for {worker_source_id}:")
+                logging.info(f"   URL: {stream_url}")
+                logging.info(f"   Type: {'Video file' if is_file else 'Live stream'}")
+                logging.info(f"   State: {state}")
+                logging.info(f"   Connected: {is_connected}")
+                logging.info(f"   Time since last frame: {time.time() - self.last_successful_frame_time:.1f}s")
+                
+                # Check for HEVC/codec specific issues
+                if hasattr(stream, 'get_codec_info'):
+                    codec_info = stream.get_codec_info()
+                    if codec_info:
+                        logging.info(f"   Codec: {codec_info}")
+                        if 'hevc' in str(codec_info).lower() or 'h265' in str(codec_info).lower():
+                            logging.warning(f"   ⚠️ HEVC stream detected - may experience QP delta or POC reference errors")
+                
+                # Log recent error patterns if available
+                if hasattr(stream, 'get_recent_errors'):
+                    recent_errors = stream.get_recent_errors()
+                    if recent_errors:
+                        hevc_errors = [err for err in recent_errors if 'cu_qp_delta' in str(err.get('error', '')) or 'Could not find ref with POC' in str(err.get('error', ''))]
+                        if hevc_errors:
+                            logging.warning(f"   🔥 Recent HEVC errors detected: {len(hevc_errors)} codec-related errors")
+                            self.hevc_error_count += len(hevc_errors)
+                            
+                            # Log sample of recent HEVC errors for debugging
+                            for i, err in enumerate(hevc_errors[-3:]):  # Show last 3 errors
+                                logging.warning(f"   🔥 HEVC Error {i+1}: {err.get('error', '')[:100]}...")
+            else:
+                logging.info(f"📊 Stream {worker_source_id} not found in regular streams, checking direct device streams...")
+                
+        except Exception as e:
+            logging.error(f"Error getting stream diagnostics: {e}")
+
+    def _should_attempt_hevc_recovery(self, video_manager, worker_source_id) -> bool:
+        """
+        Determine if HEVC recovery should be attempted based on error patterns and cooldown.
+        """
+        current_time = time.time()
+        
+        # Check cooldown period
+        if current_time - self.last_hevc_recovery < self.hevc_recovery_cooldown:
+            logging.debug(f"HEVC recovery on cooldown ({current_time - self.last_hevc_recovery:.1f}s elapsed)")
+            return False
+        
+        # Check if stream has HEVC-related errors
+        if hasattr(video_manager, 'streams') and worker_source_id in video_manager.streams:
+            stream = video_manager.streams[worker_source_id]
+            if hasattr(stream, 'get_recent_errors'):
+                recent_errors = stream.get_recent_errors(max_age_seconds=60)  # Last minute
+                hevc_errors = [err for err in recent_errors if 
+                             'cu_qp_delta' in str(err.get('error', '')) or 
+                             'Could not find ref with POC' in str(err.get('error', ''))]
+                
+                if len(hevc_errors) >= 3:  # Threshold for HEVC errors
+                    logging.info(f"HEVC recovery warranted: {len(hevc_errors)} HEVC errors in last minute")
+                    return True
+        
+        # Check if we have accumulated enough general HEVC errors
+        if self.hevc_error_count >= 5:
+            logging.info(f"HEVC recovery warranted: {self.hevc_error_count} total HEVC errors detected")
+            return True
+        
+        return False
+
+    def _handle_hevc_recovery(self, video_manager, worker_source_id):
+        """
+        Handle HEVC-specific recovery strategies for codec errors.
+        This method attempts to recover from common HEVC issues like QP delta and POC reference errors.
+        """
+        try:
+            self.last_hevc_recovery = time.time()  # Update recovery timestamp
+            logging.info(f"🔧 Attempting HEVC stream recovery for {worker_source_id}")
+            
+            # Get the stream URL for recreation
+            stream_url = video_manager.get_stream_url(worker_source_id)
+            if not stream_url:
+                logging.error(f"   Cannot get stream URL for {worker_source_id}")
+                return False
+            
+            # Strategy 1: Remove and re-add the stream to reset decoder state
+            logging.info(f"   Recreating stream {worker_source_id} to reset decoder state...")
+            video_manager.remove_stream(worker_source_id)
+            time.sleep(1.0)  # Give time for cleanup
+            
+            # Re-add the stream
+            video_manager.add_stream(worker_source_id, stream_url)
+            time.sleep(2.0)  # Give time for stream to initialize
+            
+            # Strategy 2: Check if stream was successfully recreated
+            if not video_manager.has_stream(worker_source_id):
+                logging.error(f"   Failed to recreate stream {worker_source_id}")
+                return False
+            
+            # Strategy 3: Reset failure counters and error counts after recovery attempt
+            self.reset_frame_failure_counters()
+            self.hevc_error_count = 0  # Reset HEVC error counter
+            
+            logging.info(f"✅ HEVC recovery attempt completed for {worker_source_id}")
+            return True
+            
+        except Exception as e:
+            logging.error(f"❌ HEVC recovery failed for {worker_source_id}: {e}")
+            return False
+
     def stop(self):
         """Stops the Pipeline processor and cleans up resources."""
         if not self.running:  # Prevent multiple stops
@@ -378,4 +582,42 @@ class PipelineProcessor:
 
     def enable_debug(self):
         """Enable debug mode for this pipeline."""
-        self.debug_flag = True
+        self.debug_flag = True
+        # Reset failure counters when debug is enabled as it may help with recovery
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
+    
+    def reset_frame_failure_counters(self):
+        """Reset frame failure counters. Can be called externally to help with recovery."""
+        logging.info(f"🔄 Resetting frame failure counters for pipeline {self.pipeline_id}")
+        self.consecutive_frame_failures = 0
+        self.last_successful_frame_time = time.time()
+        self.hevc_error_count = 0  # Also reset HEVC error count
+    
+    def get_hevc_diagnostics(self, video_manager) -> dict:
+        """Get HEVC-specific diagnostics for the pipeline."""
+        diagnostics = {
+            'hevc_error_count': self.hevc_error_count,
+            'last_hevc_recovery': self.last_hevc_recovery,
+            'time_since_last_recovery': time.time() - self.last_hevc_recovery,
+            'recovery_cooldown_remaining': max(0, self.hevc_recovery_cooldown - (time.time() - self.last_hevc_recovery)),
+            'consecutive_failures': self.consecutive_frame_failures,
+            'time_since_last_frame': time.time() - self.last_successful_frame_time,
+        }
+        
+        # Add stream-specific HEVC information
+        if hasattr(video_manager, 'streams') and self.worker_source_id in video_manager.streams:
+            stream = video_manager.streams[self.worker_source_id]
+            
+            if hasattr(stream, 'get_codec_info'):
+                diagnostics['codec'] = stream.get_codec_info()
+                
+            if hasattr(stream, 'get_recent_errors'):
+                recent_errors = stream.get_recent_errors(max_age_seconds=300)  # Last 5 minutes
+                hevc_errors = [err for err in recent_errors if 
+                             'cu_qp_delta' in str(err.get('error', '')) or 
+                             'Could not find ref with POC' in str(err.get('error', ''))]
+                diagnostics['recent_hevc_errors'] = len(hevc_errors)
+                diagnostics['total_recent_errors'] = len(recent_errors)
+        
+        return diagnostics

{nedo_vision_worker_core-0.3.0 → nedo_vision_worker_core-0.3.1}/nedo_vision_worker_core/streams/VideoStream.py

@@ -50,6 +50,11 @@ class VideoStream(threading.Thread):
         
         self.is_file = self._is_file_source()
         
+        # Error tracking for diagnostics
+        self._recent_errors = []
+        self._max_error_history = 50
+        self._codec_info = None
+        
     def _is_file_source(self) -> bool:
         """Check if source is a file path."""
         if isinstance(self.source, int):
@@ -90,6 +95,32 @@ class VideoStream(threading.Thread):
         """Configure capture properties and determine FPS."""
         detected_fps = self.capture.get(cv2.CAP_PROP_FPS)
         
+        # Configure error-resilient settings for video streams
+        if not self.is_file:
+            try:
+                # Set buffer size to reduce latency and handle corrupted frames better
+                self.capture.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+                
+                # Set codec-specific properties for better error handling
+                # These help with HEVC streams that have QP delta and POC reference errors
+                if hasattr(cv2, 'CAP_PROP_FOURCC'):
+                    # Try to get current codec
+                    fourcc = self.capture.get(cv2.CAP_PROP_FOURCC)
+                    codec_str = "".join([chr((int(fourcc) >> 8 * i) & 0xFF) for i in range(4)])
+                    
+                    # Log codec information for debugging
+                    logging.info(f"Stream codec detected: {codec_str} (fourcc: {int(fourcc)})")
+                    self._codec_info = codec_str
+                    
+                    # For HEVC streams, we can't change the codec but we can optimize buffering
+                    if 'hevc' in codec_str.lower() or 'h265' in codec_str.lower():
+                        logging.info("HEVC stream detected - applying error-resilient settings")
+                        # Reduce buffer to minimize impact of corrupted frames
+                        self.capture.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+                        
+            except Exception as e:
+                logging.warning(f"Could not configure capture properties: {e}")
+        
         if self.target_fps:
             self.fps = self.target_fps
             if hasattr(cv2, 'CAP_PROP_FPS'):
@@ -205,13 +236,17 @@ class VideoStream(threading.Thread):
                     break
                     
             except cv2.error as e:
-                logging.error(f"OpenCV error: {e}")
+                error_msg = f"OpenCV error: {e}"
+                logging.error(error_msg)
+                self._add_error_to_history(error_msg)
                 self._cleanup_capture()
                 if not self._sleep_interruptible(1.0):
                     break
                     
             except Exception as e:
-                logging.error(f"Unexpected error: {e}", exc_info=True)
+                error_msg = f"Unexpected error: {e}"
+                logging.error(error_msg, exc_info=True)
+                self._add_error_to_history(error_msg)
                 if not self._sleep_interruptible(self.reconnect_interval):
                     break
         
@@ -283,4 +318,28 @@ class VideoStream(threading.Thread):
     
     def __exit__(self, exc_type, exc_val, exc_tb):
         """Context manager exit."""
-        self.stop()
+        self.stop()
+    
+    def _add_error_to_history(self, error_msg: str):
+        """Add error to recent error history for diagnostics."""
+        with self._lock:
+            self._recent_errors.append({
+                'timestamp': time.time(),
+                'error': error_msg
+            })
+            # Keep only recent errors
+            if len(self._recent_errors) > self._max_error_history:
+                self._recent_errors = self._recent_errors[-self._max_error_history:]
+    
+    def get_recent_errors(self, max_age_seconds: float = 300) -> list:
+        """Get recent errors within the specified time window."""
+        current_time = time.time()
+        with self._lock:
+            return [
+                err for err in self._recent_errors 
+                if current_time - err['timestamp'] <= max_age_seconds
+            ]
+    
+    def get_codec_info(self) -> Optional[str]:
+        """Get codec information for the stream."""
+        return self._codec_info

{nedo_vision_worker_core-0.3.0 → nedo_vision_worker_core-0.3.1}/nedo_vision_worker_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker-core
-Version: 0.3.0
+Version: 0.3.1
 Summary: Nedo Vision Worker Core Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>