matrice-streaming 0.1.64__tar.gz → 0.1.65__tar.gz

{matrice_streaming-0.1.64 → matrice_streaming-0.1.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.64
+Version: 0.1.65
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.64 → matrice_streaming-0.1.65}/matrice_streaming.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.64
+Version: 0.1.65
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.64 → matrice_streaming-0.1.65}/src/matrice_streaming/streaming_gateway/camera_streamer/nvdec.py

@@ -714,12 +714,17 @@ def nvdec_pool_worker(
     target_w: int = 640,
     target_fps: int = 0,
     shared_frame_count: Optional[mp.Value] = None,
+    gpu_frame_count: Optional[mp.Value] = None,
 ):
     """NVDEC worker thread.
 
     Decodes frames and writes NV12 tensors to ring buffers.
     Uses dedicated CUDA stream per worker for kernel overlap.
     Supports FPS limiting when target_fps > 0.
+
+    Args:
+        shared_frame_count: Global counter (all GPUs)
+        gpu_frame_count: Per-GPU counter (this GPU only)
     """
     if CUPY_AVAILABLE:
         cp.cuda.Device(pool.gpu_id).use()
@@ -779,11 +784,16 @@ def nvdec_pool_worker(
                             local_frames += 1
                             frames_since_counter_update += 1
 
-                            # Update shared counter for real-time progress
+                            # Update global counter (all GPUs)
                             if shared_frame_count is not None:
                                 with shared_frame_count.get_lock():
                                     shared_frame_count.value += 1
 
+                            # Update per-GPU counter (this GPU only)
+                            if gpu_frame_count is not None:
+                                with gpu_frame_count.get_lock():
+                                    gpu_frame_count.value += 1
+
                             # Update next frame time for FPS limiting
                             if fps_limit_enabled:
                                 next_frame_time += frame_interval
@@ -839,10 +849,19 @@ def nvdec_pool_process(
     num_slots: int = 32,
     target_fps: int = 0,
     shared_frame_count: Optional[mp.Value] = None,
+    gpu_frame_counts: Optional[Dict[int, mp.Value]] = None,
+    total_num_streams: int = 0,
+    total_num_gpus: int = 1,
 ):
     """NVDEC process for one GPU.
 
     Creates NV12 ring buffers: (H*1.5, W) = 0.6 MB/frame.
+
+    Args:
+        gpu_frame_counts: Dict mapping gpu_id -> per-GPU frame counter (for per-GPU stats)
+        shared_frame_count: Global frame counter (for overall stats)
+        total_num_streams: Total streams across ALL GPUs (for global per-stream calc)
+        total_num_gpus: Total number of GPUs (for context in logging)
     """
     if not camera_configs:
         return
@@ -851,6 +870,9 @@ def nvdec_pool_process(
     target_h = camera_configs[0].height
     target_w = camera_configs[0].width
 
+    # Get per-GPU counter (or fall back to shared if not provided)
+    gpu_frame_count = gpu_frame_counts.get(gpu_id) if gpu_frame_counts else None
+
     if CUPY_AVAILABLE:
         cp.cuda.Device(gpu_id).use()
 
@@ -944,6 +966,7 @@ def nvdec_pool_process(
                     target_w,
                     target_fps,
                     shared_frame_count,
+                    gpu_frame_count,  # Per-GPU counter
                 )
             )
             t.start()
@@ -952,10 +975,13 @@ def nvdec_pool_process(
         # Progress monitoring loop with current/avg FPS tracking
         start_time = time.perf_counter()
         last_report_time = start_time
-        last_frame_count = 0
+        last_gpu_frame_count = 0
+        last_global_frame_count = 0
         report_interval = 5.0
         processing_start_time = None
-        frames_at_processing_start = 0
+        gpu_frames_at_start = 0
+        global_frames_at_start = 0
+        num_gpu_streams = len(camera_configs)
 
         while not stop_event.is_set():
             current_time = time.perf_counter()
@@ -965,34 +991,58 @@ def nvdec_pool_process(
             # Periodic progress report with current and average FPS
             if current_time - last_report_time >= report_interval:
                 elapsed = current_time - start_time
+                remaining = max(0, duration_sec - elapsed)
+
+                # Get per-GPU frame count (this GPU only)
+                gpu_frames = gpu_frame_count.value if gpu_frame_count else 0
+                gpu_interval_frames = gpu_frames - last_gpu_frame_count
+                gpu_interval_fps = gpu_interval_frames / report_interval
+                gpu_per_stream_fps = gpu_interval_fps / num_gpu_streams if num_gpu_streams > 0 else 0
+
+                # Get global frame count (all GPUs)
+                global_frames = shared_frame_count.value if shared_frame_count else 0
+                global_interval_frames = global_frames - last_global_frame_count
+                global_interval_fps = global_interval_frames / report_interval
+                global_per_stream_fps = global_interval_fps / total_num_streams if total_num_streams > 0 else 0
+
+                # Track when processing actually starts (exclude warmup)
+                if processing_start_time is None and gpu_frames > 0:
+                    processing_start_time = last_report_time
+                    gpu_frames_at_start = last_gpu_frame_count
+                    global_frames_at_start = last_global_frame_count
+
+                # Calculate average FPS excluding warmup
+                if processing_start_time is not None:
+                    processing_elapsed = current_time - processing_start_time
 
-                if shared_frame_count is not None:
-                    current_frames = shared_frame_count.value
-                    interval_frames = current_frames - last_frame_count
-                    interval_fps = interval_frames / report_interval
-                    per_stream_fps = interval_fps / len(camera_configs) if camera_configs else 0
-
-                    # Track when processing actually starts (exclude warmup)
-                    if processing_start_time is None and current_frames > 0:
-                        processing_start_time = last_report_time
-                        frames_at_processing_start = last_frame_count
-
-                    # Calculate average FPS excluding warmup
-                    if processing_start_time is not None:
-                        processing_elapsed = current_time - processing_start_time
-                        processing_frames = current_frames - frames_at_processing_start
-                        avg_fps = processing_frames / processing_elapsed if processing_elapsed > 0 else 0
-                        avg_per_stream = avg_fps / len(camera_configs) if camera_configs else 0
-                        remaining = max(0, duration_sec - elapsed)
-
+                    # Per-GPU averages
+                    gpu_processing_frames = gpu_frames - gpu_frames_at_start
+                    gpu_avg_fps = gpu_processing_frames / processing_elapsed if processing_elapsed > 0 else 0
+                    gpu_avg_per_stream = gpu_avg_fps / num_gpu_streams if num_gpu_streams > 0 else 0
+
+                    # Global averages
+                    global_processing_frames = global_frames - global_frames_at_start
+                    global_avg_fps = global_processing_frames / processing_elapsed if processing_elapsed > 0 else 0
+                    global_avg_per_stream = global_avg_fps / total_num_streams if total_num_streams > 0 else 0
+
+                    # Log per-GPU stats
+                    logger.info(
+                        f"GPU{gpu_id} [{elapsed:5.1f}s] {gpu_frames:,} frames ({num_gpu_streams} cams) | "
+                        f"cur: {gpu_interval_fps:,.0f} FPS ({gpu_per_stream_fps:.1f}/cam) | "
+                        f"avg: {gpu_avg_fps:,.0f} FPS ({gpu_avg_per_stream:.1f}/cam)"
+                    )
+
+                    # Log global stats (only from GPU0 to avoid spam)
+                    if gpu_id == 0:
                         logger.info(
-                            f"GPU{gpu_id} [{elapsed:5.1f}s] {current_frames:,} frames | "
-                            f"cur: {interval_fps:,.0f} FPS ({per_stream_fps:.1f}/stream) | "
-                            f"avg: {avg_fps:,.0f} FPS ({avg_per_stream:.1f}/stream) | "
+                            f"GLOBAL [{elapsed:5.1f}s] {global_frames:,} frames ({total_num_streams} cams, {total_num_gpus} GPUs) | "
+                            f"cur: {global_interval_fps:,.0f} FPS ({global_per_stream_fps:.1f}/cam) | "
+                            f"avg: {global_avg_fps:,.0f} FPS ({global_avg_per_stream:.1f}/cam) | "
                             f"{remaining:.0f}s left"
                         )
 
-                    last_frame_count = current_frames
+                last_gpu_frame_count = gpu_frames
+                last_global_frame_count = global_frames
                 last_report_time = current_time
 
             time.sleep(0.1)

{matrice_streaming-0.1.64 → matrice_streaming-0.1.65}/src/matrice_streaming/streaming_gateway/camera_streamer/nvdec_worker_manager.py

@@ -123,6 +123,7 @@ class NVDECWorkerManager:
         self._stop_event: Optional[mp.Event] = None
         self._result_queue: Optional[mp.Queue] = None
         self._shared_frame_count: Optional[mp.Value] = None
+        self._gpu_frame_counts: Dict[int, mp.Value] = {}  # Per-GPU counters
         self._start_time: Optional[float] = None
         self._is_running = False
 
@@ -200,9 +201,20 @@ class NVDECWorkerManager:
         ctx = mp.get_context("spawn")
         self._stop_event = ctx.Event()
         self._result_queue = ctx.Queue()
-        self._shared_frame_count = ctx.Value('i', 0)
+        self._shared_frame_count = ctx.Value('L', 0)  # Global counter (all GPUs)
         self._start_time = time.perf_counter()
 
+        # Create per-GPU frame counters
+        self._gpu_frame_counts = {}
+        for gpu_id in range(self.num_gpus):
+            if self._gpu_camera_assignments[gpu_id]:  # Only if GPU has cameras
+                self._gpu_frame_counts[gpu_id] = ctx.Value('L', 0)
+
+        total_num_streams = len(self._stream_configs)
+        total_num_gpus = len([g for g in range(self.num_gpus) if self._gpu_camera_assignments[g]])
+
+        logger.info(f"Starting NVDEC: {total_num_streams} cameras across {total_num_gpus} GPUs")
+
         # Start one process per GPU that has cameras
         for gpu_id in range(self.num_gpus):
             gpu_cameras = self._gpu_camera_assignments[gpu_id]
@@ -221,7 +233,10 @@ class NVDECWorkerManager:
                     self.nvdec_burst_size,      # burst_size
                     self.num_slots,             # num_slots
                     self.target_fps,            # target_fps
-                    self._shared_frame_count,   # shared_frame_count
+                    self._shared_frame_count,   # shared_frame_count (global)
+                    self._gpu_frame_counts,     # gpu_frame_counts (per-GPU dict)
+                    total_num_streams,          # total_num_streams
+                    total_num_gpus,             # total_num_gpus
                 ),
                 name=f"NVDECWorker-GPU{gpu_id}",
                 daemon=False,
@@ -311,6 +326,23 @@ class NVDECWorkerManager:
                     if self._stream_configs else 0
                 )
 
+        # Add per-GPU frame counts and FPS
+        if self._gpu_frame_counts and self._start_time:
+            elapsed = time.perf_counter() - self._start_time
+            gpu_stats = {}
+            for gpu_id, counter in self._gpu_frame_counts.items():
+                gpu_frames = counter.value
+                num_cams = len(self._gpu_camera_assignments.get(gpu_id, []))
+                gpu_fps = gpu_frames / elapsed if elapsed > 0 else 0
+                gpu_per_cam = gpu_fps / num_cams if num_cams > 0 else 0
+                gpu_stats[f'GPU{gpu_id}'] = {
+                    'frames': gpu_frames,
+                    'cameras': num_cams,
+                    'fps': gpu_fps,
+                    'fps_per_cam': gpu_per_cam,
+                }
+            stats['per_gpu_stats'] = gpu_stats
+
         # Collect any available results from queue (non-blocking)
         gpu_results = []
         if self._result_queue: