matrice-streaming 0.1.68__tar.gz → 0.1.69__tar.gz

{matrice_streaming-0.1.68 → matrice_streaming-0.1.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.68
+Version: 0.1.69
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.68 → matrice_streaming-0.1.69}/matrice_streaming.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.68
+Version: 0.1.69
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.68 → matrice_streaming-0.1.69}/src/matrice_streaming/streaming_gateway/camera_streamer/nvdec.py

@@ -96,8 +96,23 @@ except (ImportError, AttributeError, RuntimeError, TypeError):
     CudaIpcRingBuffer = None
     GlobalFrameCounter = None
 
+try:
+    from matrice_common.stream.gpu_camera_map import GpuCameraMap
+    GPU_CAMERA_MAP_AVAILABLE = True
+except ImportError:
+    GPU_CAMERA_MAP_AVAILABLE = False
+    GpuCameraMap = None
+
 logger = logging.getLogger(__name__)
 
+# =============================================================================
+# Fixed dimensions for NV12 output - ALL cameras resize to this size
+# This matches the TensorRT model input requirements (640x640)
+# =============================================================================
+TARGET_WIDTH = 640
+TARGET_HEIGHT = 640
+NV12_HEIGHT = TARGET_HEIGHT + TARGET_HEIGHT // 2  # 960 = H * 1.5 for NV12 format
+
 def setup_logging(quiet: bool = True):
     """Configure logging level based on quiet mode."""
     level = logging.WARNING if quiet else logging.INFO
@@ -407,6 +422,8 @@ class StreamState:
     width: int = 640
     height: int = 640
     empty_packets: int = 0
+    decode_errors: int = 0  # Consecutive decode errors
+    MAX_DECODE_ERRORS: int = 5  # Restart demuxer after this many consecutive errors
 
 
 # =============================================================================
@@ -664,18 +681,44 @@ class NVDECDecoderPool:
                             break
 
                     frames_before = frames_this_stream
-                    for surface in decoder.Decode(packet):
-                        tensor = surface_to_nv12(surface, target_h, target_w)
-
-                        if tensor is not None:
-                            decoded_frames.append((stream.camera_id, tensor))
-                            frames_this_stream += 1
-                            stream.frames_decoded += 1
-                            total_frames += 1
-                            stream.empty_packets = 0
 
-                        if frames_this_stream >= frames_per_stream:
-                            break
+                    # Wrap decode loop in try/except to catch decode errors
+                    try:
+                        for surface in decoder.Decode(packet):
+                            tensor = surface_to_nv12(surface, target_h, target_w)
+
+                            if tensor is not None:
+                                decoded_frames.append((stream.camera_id, tensor))
+                                frames_this_stream += 1
+                                stream.frames_decoded += 1
+                                total_frames += 1
+                                stream.empty_packets = 0
+                                stream.decode_errors = 0  # Reset on success
+
+                            if frames_this_stream >= frames_per_stream:
+                                break
+
+                    except Exception as decode_err:
+                        # Handle decode errors - these can happen with corrupted packets
+                        stream.decode_errors += 1
+                        if stream.decode_errors == 1:
+                            # Only log first error per stream to avoid spam
+                            logger.warning(f"{stream.camera_id}: Decode error: {decode_err}")
+
+                        if stream.decode_errors >= stream.MAX_DECODE_ERRORS:
+                            # Too many consecutive errors - restart demuxer
+                            logger.warning(
+                                f"{stream.camera_id}: {stream.decode_errors} consecutive decode errors, "
+                                f"restarting demuxer"
+                            )
+                            try:
+                                stream.demuxer = nvc.CreateDemuxer(stream.video_path)
+                                stream.decode_errors = 0
+                                stream.empty_packets = 0
+                                logger.info(f"{stream.camera_id}: Demuxer restarted successfully")
+                            except Exception as restart_err:
+                                logger.error(f"{stream.camera_id}: Failed to restart demuxer: {restart_err}")
+                                break  # Exit this stream's loop
 
                     if frames_this_stream == frames_before:
                         stream.empty_packets += 1
@@ -683,7 +726,17 @@ class NVDECDecoderPool:
                             stream.demuxer = nvc.CreateDemuxer(stream.video_path)
                             stream.empty_packets = 0
 
-                except Exception:
+                except Exception as demux_err:
+                    # Handle demux errors
+                    logger.warning(f"{stream.camera_id}: Demux error: {demux_err}")
+                    stream.decode_errors += 1
+                    if stream.decode_errors >= stream.MAX_DECODE_ERRORS:
+                        try:
+                            stream.demuxer = nvc.CreateDemuxer(stream.video_path)
+                            stream.decode_errors = 0
+                            logger.info(f"{stream.camera_id}: Demuxer restarted after demux errors")
+                        except Exception:
+                            pass
                     break
 
                 if frames_this_stream >= frames_per_stream:
@@ -778,16 +831,28 @@ def nvdec_pool_worker(
 
         try:
             with cuda_stream:
+                # Always use fixed TARGET dimensions to ensure consistent NV12 output
                 num_frames, decoded_frames = pool.decode_round(
                     decoder_idx,
                     frames_per_stream=burst_size,
-                    target_h=target_h,
-                    target_w=target_w
+                    target_h=TARGET_HEIGHT,  # Always 640
+                    target_w=TARGET_WIDTH    # Always 640
                 )
 
                 for cam_id, tensor in decoded_frames:
                     if cam_id in ring_buffers:
                         try:
+                            # Validate frame shape matches expected NV12 dimensions
+                            expected_shape = (NV12_HEIGHT, TARGET_WIDTH, 1)  # (960, 640, 1)
+                            if tensor.shape != expected_shape:
+                                local_errors += 1
+                                if local_errors <= 5:
+                                    logger.error(
+                                        f"Worker {worker_id} shape mismatch for {cam_id}: "
+                                        f"got {tensor.shape}, expected {expected_shape}"
+                                    )
+                                continue  # Skip this frame
+
                             ring_buffers[cam_id].write_frame_fast(tensor, sync=False)
                             local_frames += 1
                             frames_since_counter_update += 1
@@ -932,30 +997,64 @@ def nvdec_pool_process(
         })
         return
 
-    # Create NV12 ring buffers: (H + H/2, W, 1) = 0.6 MB/frame
+    # Create NV12 ring buffers with FIXED dimensions (640x640 -> 960x640 for NV12)
+    # ALL cameras resize to TARGET_WIDTH x TARGET_HEIGHT regardless of source resolution
     ring_buffers: Dict[str, CudaIpcRingBuffer] = {}
-    frame_size_mb = target_h * target_w * 1.5 / 1e6
+    frame_size_mb = TARGET_WIDTH * NV12_HEIGHT * 1 / 1e6  # 640 * 960 * 1 channel
 
     try:
+        # Initialize GPU camera map (producer side) - process 0 creates, others connect
+        gpu_map = None
+        if GPU_CAMERA_MAP_AVAILABLE:
+            gpu_map = GpuCameraMap(is_producer=True)
+            if process_id == 0:
+                if gpu_map.initialize():
+                    logger.info(f"Process {process_id}: GpuCameraMap initialized")
+                else:
+                    logger.warning(f"Process {process_id}: Failed to initialize GpuCameraMap")
+                    gpu_map = None
+            else:
+                # Wait for process 0 to initialize
+                for retry in range(50):
+                    if gpu_map.connect():
+                        logger.info(f"Process {process_id}: Connected to GpuCameraMap")
+                        break
+                    time.sleep(0.1)
+                else:
+                    logger.warning(f"Process {process_id}: Failed to connect to GpuCameraMap")
+                    gpu_map = None
+
+        # Collect GPU mappings for bulk write
+        gpu_mappings = {}
+
         for i, config in enumerate(camera_configs):
+            # Use fixed dimensions - ALL cameras output 640x640 (960x640 for NV12)
             rb = CudaIpcRingBuffer.create_producer(
                 config.camera_id,
                 gpu_id=config.gpu_id,
                 num_slots=num_slots,
-                width=config.width,
-                height=config.height + config.height // 2,  # H * 1.5 for NV12
+                width=TARGET_WIDTH,      # Always 640
+                height=NV12_HEIGHT,      # Always 960 (640 * 1.5 for NV12)
                 channels=1,
             )
             ring_buffers[config.camera_id] = rb
 
+            # Track GPU assignment for this camera
+            gpu_mappings[config.camera_id] = config.gpu_id
+
             pool.assign_stream(
                 stream_id=i,
                 camera_id=config.camera_id,
                 video_path=config.video_path,
-                width=config.width,
-                height=config.height
+                width=TARGET_WIDTH,   # Use fixed width
+                height=TARGET_HEIGHT  # Use fixed height
             )
 
+        # Write all GPU mappings at once
+        if gpu_map and gpu_mappings:
+            gpu_map.set_bulk_mapping(gpu_mappings)
+            logger.info(f"Process {process_id}: Wrote {len(gpu_mappings)} camera-GPU mappings")
+
         logger.info(f"Process {process_id}: {pool.actual_pool_size} decoders, "
                    f"{len(camera_configs)} streams, NV12 ({frame_size_mb:.1f} MB/frame)")