matrice-streaming 0.1.67__tar.gz → 0.1.69__tar.gz

{matrice_streaming-0.1.67 → matrice_streaming-0.1.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.67
+Version: 0.1.69
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.67 → matrice_streaming-0.1.69}/matrice_streaming.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.67
+Version: 0.1.69
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.67 → matrice_streaming-0.1.69}/src/matrice_streaming/streaming_gateway/camera_streamer/nvdec.py

@@ -96,8 +96,23 @@ except (ImportError, AttributeError, RuntimeError, TypeError):
     CudaIpcRingBuffer = None
     GlobalFrameCounter = None
 
+try:
+    from matrice_common.stream.gpu_camera_map import GpuCameraMap
+    GPU_CAMERA_MAP_AVAILABLE = True
+except ImportError:
+    GPU_CAMERA_MAP_AVAILABLE = False
+    GpuCameraMap = None
+
 logger = logging.getLogger(__name__)
 
+# =============================================================================
+# Fixed dimensions for NV12 output - ALL cameras resize to this size
+# This matches the TensorRT model input requirements (640x640)
+# =============================================================================
+TARGET_WIDTH = 640
+TARGET_HEIGHT = 640
+NV12_HEIGHT = TARGET_HEIGHT + TARGET_HEIGHT // 2  # 960 = H * 1.5 for NV12 format
+
 def setup_logging(quiet: bool = True):
     """Configure logging level based on quiet mode."""
     level = logging.WARNING if quiet else logging.INFO
@@ -407,6 +422,8 @@ class StreamState:
     width: int = 640
     height: int = 640
     empty_packets: int = 0
+    decode_errors: int = 0  # Consecutive decode errors
+    MAX_DECODE_ERRORS: int = 5  # Restart demuxer after this many consecutive errors
 
 
 # =============================================================================
@@ -420,8 +437,8 @@ def _get_nv12_resize_kernel():
     """Get or compile the NV12 resize kernel.
 
     This kernel resizes NV12 directly (no color conversion).
-    Output: concatenated Y (H×W) + UV ((H/2)×W) = H×W×1.5 bytes
-    This is 50% smaller than RGB (H×W×3 bytes).
+    Output: concatenated Y (H*W) + UV ((H/2)*W) = H*W*1.5 bytes
+    This is 50% smaller than RGB (H*W*3 bytes).
 
     Consumer will do: NV12→RGB→CHW→FP16 in one fused kernel.
     """
@@ -429,9 +446,9 @@ def _get_nv12_resize_kernel():
     if _nv12_resize_kernel is None and CUPY_AVAILABLE:
         _nv12_resize_kernel = cp.RawKernel(r'''
 extern "C" __global__ void nv12_resize(
-    const unsigned char* src_y,    // Source Y plane
-    const unsigned char* src_uv,   // Source UV plane (interleaved)
-    unsigned char* dst,            // Output: Y (H×W) followed by UV ((H/2)×W)
+    const unsigned char* src_y,    
+    const unsigned char* src_uv,   
+    unsigned char* dst,            
     int src_h, int src_w,
     int dst_h, int dst_w,
     int y_stride, int uv_stride
@@ -477,8 +494,8 @@ def nv12_resize(y_plane: cp.ndarray, uv_plane: cp.ndarray,
                 dst_h: int = 640, dst_w: int = 640) -> cp.ndarray:
     """Resize NV12 without color conversion.
 
-    Output: concatenated Y (H×W) + UV ((H/2)×W) as single buffer.
-    Total size: H×W + (H/2)×W = H×W×1.5 bytes (50% of RGB).
+    Output: concatenated Y (H*W) + UV ((H/2)*W) as single buffer.
+    Total size: H*W + (H/2)*W = H*W*1.5 bytes (50% of RGB).
     """
     kernel = _get_nv12_resize_kernel()
     if kernel is None:
@@ -504,7 +521,7 @@ def surface_to_nv12(frame, target_h: int = 640, target_w: int = 640) -> Optional
     """Convert NVDEC surface to resized NV12 (50% smaller than RGB).
 
     Output: (H + H/2, W) uint8 - concatenated Y + UV planes.
-    Total size: H×W×1.5 bytes (vs H×W×3 for RGB).
+    Total size: H*W*1.5 bytes (vs H*W*3 for RGB).
     """
     if not CUPY_AVAILABLE or frame is None:
         return None
@@ -552,11 +569,14 @@ def surface_to_nv12(frame, target_h: int = 640, target_w: int = 640) -> Optional
         return nv12_frame[:, :, cp.newaxis] if nv12_frame is not None else None
 
     except Exception as e:
-        # Safely encode error message (some CUDA errors contain non-ASCII chars like '×')
+        # Safely handle any characters in error message (CUDA errors may contain Unicode like ×)
         try:
             err_msg = str(e).encode('ascii', errors='replace').decode('ascii')
         except Exception:
-            err_msg = "unknown error"
+            try:
+                err_msg = repr(str(e))[:200]
+            except Exception:
+                err_msg = "failed to format error message"
         logger.warning(f"surface_to_nv12 failed: {err_msg}")
         return None
 
@@ -569,7 +589,7 @@ class NVDECDecoderPool:
     """Pool of NVDEC decoders that time-multiplex streams.
 
     Each decoder is exclusively owned by one worker thread.
-    Outputs NV12: 1.5×H×W bytes (50% smaller than RGB).
+    Outputs NV12: 1.5*H*W bytes (50% smaller than RGB).
     """
 
     def __init__(self, pool_size: int, gpu_id: int = 0):
@@ -661,18 +681,44 @@ class NVDECDecoderPool:
                             break
 
                     frames_before = frames_this_stream
-                    for surface in decoder.Decode(packet):
-                        tensor = surface_to_nv12(surface, target_h, target_w)
-
-                        if tensor is not None:
-                            decoded_frames.append((stream.camera_id, tensor))
-                            frames_this_stream += 1
-                            stream.frames_decoded += 1
-                            total_frames += 1
-                            stream.empty_packets = 0
 
-                        if frames_this_stream >= frames_per_stream:
-                            break
+                    # Wrap decode loop in try/except to catch decode errors
+                    try:
+                        for surface in decoder.Decode(packet):
+                            tensor = surface_to_nv12(surface, target_h, target_w)
+
+                            if tensor is not None:
+                                decoded_frames.append((stream.camera_id, tensor))
+                                frames_this_stream += 1
+                                stream.frames_decoded += 1
+                                total_frames += 1
+                                stream.empty_packets = 0
+                                stream.decode_errors = 0  # Reset on success
+
+                            if frames_this_stream >= frames_per_stream:
+                                break
+
+                    except Exception as decode_err:
+                        # Handle decode errors - these can happen with corrupted packets
+                        stream.decode_errors += 1
+                        if stream.decode_errors == 1:
+                            # Only log first error per stream to avoid spam
+                            logger.warning(f"{stream.camera_id}: Decode error: {decode_err}")
+
+                        if stream.decode_errors >= stream.MAX_DECODE_ERRORS:
+                            # Too many consecutive errors - restart demuxer
+                            logger.warning(
+                                f"{stream.camera_id}: {stream.decode_errors} consecutive decode errors, "
+                                f"restarting demuxer"
+                            )
+                            try:
+                                stream.demuxer = nvc.CreateDemuxer(stream.video_path)
+                                stream.decode_errors = 0
+                                stream.empty_packets = 0
+                                logger.info(f"{stream.camera_id}: Demuxer restarted successfully")
+                            except Exception as restart_err:
+                                logger.error(f"{stream.camera_id}: Failed to restart demuxer: {restart_err}")
+                                break  # Exit this stream's loop
 
                     if frames_this_stream == frames_before:
                         stream.empty_packets += 1
@@ -680,7 +726,17 @@ class NVDECDecoderPool:
                             stream.demuxer = nvc.CreateDemuxer(stream.video_path)
                             stream.empty_packets = 0
 
-                except Exception:
+                except Exception as demux_err:
+                    # Handle demux errors
+                    logger.warning(f"{stream.camera_id}: Demux error: {demux_err}")
+                    stream.decode_errors += 1
+                    if stream.decode_errors >= stream.MAX_DECODE_ERRORS:
+                        try:
+                            stream.demuxer = nvc.CreateDemuxer(stream.video_path)
+                            stream.decode_errors = 0
+                            logger.info(f"{stream.camera_id}: Demuxer restarted after demux errors")
+                        except Exception:
+                            pass
                     break
 
                 if frames_this_stream >= frames_per_stream:
@@ -775,16 +831,28 @@ def nvdec_pool_worker(
 
         try:
             with cuda_stream:
+                # Always use fixed TARGET dimensions to ensure consistent NV12 output
                 num_frames, decoded_frames = pool.decode_round(
                     decoder_idx,
                     frames_per_stream=burst_size,
-                    target_h=target_h,
-                    target_w=target_w
+                    target_h=TARGET_HEIGHT,  # Always 640
+                    target_w=TARGET_WIDTH    # Always 640
                 )
 
                 for cam_id, tensor in decoded_frames:
                     if cam_id in ring_buffers:
                         try:
+                            # Validate frame shape matches expected NV12 dimensions
+                            expected_shape = (NV12_HEIGHT, TARGET_WIDTH, 1)  # (960, 640, 1)
+                            if tensor.shape != expected_shape:
+                                local_errors += 1
+                                if local_errors <= 5:
+                                    logger.error(
+                                        f"Worker {worker_id} shape mismatch for {cam_id}: "
+                                        f"got {tensor.shape}, expected {expected_shape}"
+                                    )
+                                continue  # Skip this frame
+
                             ring_buffers[cam_id].write_frame_fast(tensor, sync=False)
                             local_frames += 1
                             frames_since_counter_update += 1
@@ -808,8 +876,14 @@ def nvdec_pool_worker(
                             if local_errors <= 3:
                                 logger.error(f"Worker {worker_id} write error: {e}")
 
-                if decoded_frames and len(ring_buffers) > 0:
-                    next(iter(ring_buffers.values())).sync_writes()
+                if decoded_frames:
+                    # Sync all buffers that received frames in this round
+                    # Critical for cross-container IPC - ensures GPU writes are visible
+                    synced_cams = set()
+                    for cam_id, _ in decoded_frames:
+                        if cam_id in ring_buffers and cam_id not in synced_cams:
+                            ring_buffers[cam_id].sync_writes()
+                            synced_cams.add(cam_id)
 
             if num_frames == 0:
                 time.sleep(0.0001)
@@ -923,30 +997,64 @@ def nvdec_pool_process(
         })
         return
 
-    # Create NV12 ring buffers: (H + H/2, W, 1) = 0.6 MB/frame
+    # Create NV12 ring buffers with FIXED dimensions (640x640 -> 960x640 for NV12)
+    # ALL cameras resize to TARGET_WIDTH x TARGET_HEIGHT regardless of source resolution
     ring_buffers: Dict[str, CudaIpcRingBuffer] = {}
-    frame_size_mb = target_h * target_w * 1.5 / 1e6
+    frame_size_mb = TARGET_WIDTH * NV12_HEIGHT * 1 / 1e6  # 640 * 960 * 1 channel
 
     try:
+        # Initialize GPU camera map (producer side) - process 0 creates, others connect
+        gpu_map = None
+        if GPU_CAMERA_MAP_AVAILABLE:
+            gpu_map = GpuCameraMap(is_producer=True)
+            if process_id == 0:
+                if gpu_map.initialize():
+                    logger.info(f"Process {process_id}: GpuCameraMap initialized")
+                else:
+                    logger.warning(f"Process {process_id}: Failed to initialize GpuCameraMap")
+                    gpu_map = None
+            else:
+                # Wait for process 0 to initialize
+                for retry in range(50):
+                    if gpu_map.connect():
+                        logger.info(f"Process {process_id}: Connected to GpuCameraMap")
+                        break
+                    time.sleep(0.1)
+                else:
+                    logger.warning(f"Process {process_id}: Failed to connect to GpuCameraMap")
+                    gpu_map = None
+
+        # Collect GPU mappings for bulk write
+        gpu_mappings = {}
+
         for i, config in enumerate(camera_configs):
+            # Use fixed dimensions - ALL cameras output 640x640 (960x640 for NV12)
             rb = CudaIpcRingBuffer.create_producer(
                 config.camera_id,
                 gpu_id=config.gpu_id,
                 num_slots=num_slots,
-                width=config.width,
-                height=config.height + config.height // 2,  # H * 1.5 for NV12
+                width=TARGET_WIDTH,      # Always 640
+                height=NV12_HEIGHT,      # Always 960 (640 * 1.5 for NV12)
                 channels=1,
             )
             ring_buffers[config.camera_id] = rb
 
+            # Track GPU assignment for this camera
+            gpu_mappings[config.camera_id] = config.gpu_id
+
             pool.assign_stream(
                 stream_id=i,
                 camera_id=config.camera_id,
                 video_path=config.video_path,
-                width=config.width,
-                height=config.height
+                width=TARGET_WIDTH,   # Use fixed width
+                height=TARGET_HEIGHT  # Use fixed height
             )
 
+        # Write all GPU mappings at once
+        if gpu_map and gpu_mappings:
+            gpu_map.set_bulk_mapping(gpu_mappings)
+            logger.info(f"Process {process_id}: Wrote {len(gpu_mappings)} camera-GPU mappings")
+
         logger.info(f"Process {process_id}: {pool.actual_pool_size} decoders, "
                    f"{len(camera_configs)} streams, NV12 ({frame_size_mb:.1f} MB/frame)")