matrice-streaming 0.1.63__tar.gz → 0.1.65__tar.gz

{matrice_streaming-0.1.63 → matrice_streaming-0.1.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.63
+Version: 0.1.65
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.63 → matrice_streaming-0.1.65}/matrice_streaming.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.63
+Version: 0.1.65
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.63 → matrice_streaming-0.1.65}/src/matrice_streaming/streaming_gateway/camera_streamer/nvdec.py

@@ -547,7 +547,12 @@ def surface_to_nv12(frame, target_h: int = 640, target_w: int = 640) -> Optional
         return nv12_frame[:, :, cp.newaxis] if nv12_frame is not None else None
 
     except Exception as e:
-        logger.warning(f"surface_to_nv12 failed: {e}")
+        # Safely encode error message (some CUDA errors contain non-ASCII chars like '×')
+        try:
+            err_msg = str(e).encode('ascii', errors='replace').decode('ascii')
+        except Exception:
+            err_msg = "unknown error"
+        logger.warning(f"surface_to_nv12 failed: {err_msg}")
         return None
 
 
@@ -709,12 +714,17 @@ def nvdec_pool_worker(
     target_w: int = 640,
     target_fps: int = 0,
     shared_frame_count: Optional[mp.Value] = None,
+    gpu_frame_count: Optional[mp.Value] = None,
 ):
     """NVDEC worker thread.
 
     Decodes frames and writes NV12 tensors to ring buffers.
     Uses dedicated CUDA stream per worker for kernel overlap.
     Supports FPS limiting when target_fps > 0.
+
+    Args:
+        shared_frame_count: Global counter (all GPUs)
+        gpu_frame_count: Per-GPU counter (this GPU only)
     """
     if CUPY_AVAILABLE:
         cp.cuda.Device(pool.gpu_id).use()
@@ -727,7 +737,6 @@ def nvdec_pool_worker(
     frames_since_counter_update = 0
     counter_batch_size = 100
     start_time = time.perf_counter()
-    last_log_time = start_time
     camera_ids = pool.get_camera_ids_for_decoder(decoder_idx)
     num_streams = len(camera_ids)
 
@@ -745,19 +754,12 @@ def nvdec_pool_worker(
         next_frame_time = 0
         fps_mode = ", unlimited FPS"
 
-    logger.info(f"Worker {worker_id}: decoder={decoder_idx}, cams={num_streams}{fps_mode}")
+    logger.debug(f"Worker {worker_id}: decoder={decoder_idx}, cams={num_streams}{fps_mode}")
 
     while not stop_event.is_set():
         if time.perf_counter() - start_time >= duration_sec:
             break
 
-        now = time.perf_counter()
-        if now - last_log_time >= 5.0:
-            elapsed = now - start_time
-            fps = local_frames / elapsed if elapsed > 0 else 0
-            logger.info(f"Worker {worker_id}: {local_frames} frames, {fps:.0f} FPS")
-            last_log_time = now
-
         # FPS limiting: wait until next scheduled frame time
         if fps_limit_enabled:
             current_time = time.perf_counter()
@@ -782,11 +784,16 @@ def nvdec_pool_worker(
                             local_frames += 1
                             frames_since_counter_update += 1
 
-                            # Update shared counter for real-time progress
+                            # Update global counter (all GPUs)
                             if shared_frame_count is not None:
                                 with shared_frame_count.get_lock():
                                     shared_frame_count.value += 1
 
+                            # Update per-GPU counter (this GPU only)
+                            if gpu_frame_count is not None:
+                                with gpu_frame_count.get_lock():
+                                    gpu_frame_count.value += 1
+
                             # Update next frame time for FPS limiting
                             if fps_limit_enabled:
                                 next_frame_time += frame_interval
@@ -842,10 +849,19 @@ def nvdec_pool_process(
     num_slots: int = 32,
     target_fps: int = 0,
     shared_frame_count: Optional[mp.Value] = None,
+    gpu_frame_counts: Optional[Dict[int, mp.Value]] = None,
+    total_num_streams: int = 0,
+    total_num_gpus: int = 1,
 ):
     """NVDEC process for one GPU.
 
     Creates NV12 ring buffers: (H*1.5, W) = 0.6 MB/frame.
+
+    Args:
+        gpu_frame_counts: Dict mapping gpu_id -> per-GPU frame counter (for per-GPU stats)
+        shared_frame_count: Global frame counter (for overall stats)
+        total_num_streams: Total streams across ALL GPUs (for global per-stream calc)
+        total_num_gpus: Total number of GPUs (for context in logging)
     """
     if not camera_configs:
         return
@@ -854,6 +870,9 @@ def nvdec_pool_process(
     target_h = camera_configs[0].height
     target_w = camera_configs[0].width
 
+    # Get per-GPU counter (or fall back to shared if not provided)
+    gpu_frame_count = gpu_frame_counts.get(gpu_id) if gpu_frame_counts else None
+
     if CUPY_AVAILABLE:
         cp.cuda.Device(gpu_id).use()
 
@@ -947,15 +966,85 @@ def nvdec_pool_process(
                     target_w,
                     target_fps,
                     shared_frame_count,
+                    gpu_frame_count,  # Per-GPU counter
                 )
             )
             t.start()
             threads.append(t)
 
+        # Progress monitoring loop with current/avg FPS tracking
         start_time = time.perf_counter()
+        last_report_time = start_time
+        last_gpu_frame_count = 0
+        last_global_frame_count = 0
+        report_interval = 5.0
+        processing_start_time = None
+        gpu_frames_at_start = 0
+        global_frames_at_start = 0
+        num_gpu_streams = len(camera_configs)
+
         while not stop_event.is_set():
-            if time.perf_counter() - start_time >= duration_sec:
+            current_time = time.perf_counter()
+            if current_time - start_time >= duration_sec:
                 break
+
+            # Periodic progress report with current and average FPS
+            if current_time - last_report_time >= report_interval:
+                elapsed = current_time - start_time
+                remaining = max(0, duration_sec - elapsed)
+
+                # Get per-GPU frame count (this GPU only)
+                gpu_frames = gpu_frame_count.value if gpu_frame_count else 0
+                gpu_interval_frames = gpu_frames - last_gpu_frame_count
+                gpu_interval_fps = gpu_interval_frames / report_interval
+                gpu_per_stream_fps = gpu_interval_fps / num_gpu_streams if num_gpu_streams > 0 else 0
+
+                # Get global frame count (all GPUs)
+                global_frames = shared_frame_count.value if shared_frame_count else 0
+                global_interval_frames = global_frames - last_global_frame_count
+                global_interval_fps = global_interval_frames / report_interval
+                global_per_stream_fps = global_interval_fps / total_num_streams if total_num_streams > 0 else 0
+
+                # Track when processing actually starts (exclude warmup)
+                if processing_start_time is None and gpu_frames > 0:
+                    processing_start_time = last_report_time
+                    gpu_frames_at_start = last_gpu_frame_count
+                    global_frames_at_start = last_global_frame_count
+
+                # Calculate average FPS excluding warmup
+                if processing_start_time is not None:
+                    processing_elapsed = current_time - processing_start_time
+
+                    # Per-GPU averages
+                    gpu_processing_frames = gpu_frames - gpu_frames_at_start
+                    gpu_avg_fps = gpu_processing_frames / processing_elapsed if processing_elapsed > 0 else 0
+                    gpu_avg_per_stream = gpu_avg_fps / num_gpu_streams if num_gpu_streams > 0 else 0
+
+                    # Global averages
+                    global_processing_frames = global_frames - global_frames_at_start
+                    global_avg_fps = global_processing_frames / processing_elapsed if processing_elapsed > 0 else 0
+                    global_avg_per_stream = global_avg_fps / total_num_streams if total_num_streams > 0 else 0
+
+                    # Log per-GPU stats
+                    logger.info(
+                        f"GPU{gpu_id} [{elapsed:5.1f}s] {gpu_frames:,} frames ({num_gpu_streams} cams) | "
+                        f"cur: {gpu_interval_fps:,.0f} FPS ({gpu_per_stream_fps:.1f}/cam) | "
+                        f"avg: {gpu_avg_fps:,.0f} FPS ({gpu_avg_per_stream:.1f}/cam)"
+                    )
+
+                    # Log global stats (only from GPU0 to avoid spam)
+                    if gpu_id == 0:
+                        logger.info(
+                            f"GLOBAL [{elapsed:5.1f}s] {global_frames:,} frames ({total_num_streams} cams, {total_num_gpus} GPUs) | "
+                            f"cur: {global_interval_fps:,.0f} FPS ({global_per_stream_fps:.1f}/cam) | "
+                            f"avg: {global_avg_fps:,.0f} FPS ({global_avg_per_stream:.1f}/cam) | "
+                            f"{remaining:.0f}s left"
+                        )
+
+                last_gpu_frame_count = gpu_frames
+                last_global_frame_count = global_frames
+                last_report_time = current_time
+
             time.sleep(0.1)
 
         thread_stop_event.set()

{matrice_streaming-0.1.63 → matrice_streaming-0.1.65}/src/matrice_streaming/streaming_gateway/camera_streamer/nvdec_worker_manager.py

@@ -123,6 +123,7 @@ class NVDECWorkerManager:
         self._stop_event: Optional[mp.Event] = None
         self._result_queue: Optional[mp.Queue] = None
         self._shared_frame_count: Optional[mp.Value] = None
+        self._gpu_frame_counts: Dict[int, mp.Value] = {}  # Per-GPU counters
         self._start_time: Optional[float] = None
         self._is_running = False
 
@@ -200,9 +201,20 @@ class NVDECWorkerManager:
         ctx = mp.get_context("spawn")
         self._stop_event = ctx.Event()
         self._result_queue = ctx.Queue()
-        self._shared_frame_count = ctx.Value('i', 0)
+        self._shared_frame_count = ctx.Value('L', 0)  # Global counter (all GPUs)
         self._start_time = time.perf_counter()
 
+        # Create per-GPU frame counters
+        self._gpu_frame_counts = {}
+        for gpu_id in range(self.num_gpus):
+            if self._gpu_camera_assignments[gpu_id]:  # Only if GPU has cameras
+                self._gpu_frame_counts[gpu_id] = ctx.Value('L', 0)
+
+        total_num_streams = len(self._stream_configs)
+        total_num_gpus = len([g for g in range(self.num_gpus) if self._gpu_camera_assignments[g]])
+
+        logger.info(f"Starting NVDEC: {total_num_streams} cameras across {total_num_gpus} GPUs")
+
         # Start one process per GPU that has cameras
         for gpu_id in range(self.num_gpus):
             gpu_cameras = self._gpu_camera_assignments[gpu_id]
@@ -221,7 +233,10 @@ class NVDECWorkerManager:
                     self.nvdec_burst_size,      # burst_size
                     self.num_slots,             # num_slots
                     self.target_fps,            # target_fps
-                    self._shared_frame_count,   # shared_frame_count
+                    self._shared_frame_count,   # shared_frame_count (global)
+                    self._gpu_frame_counts,     # gpu_frame_counts (per-GPU dict)
+                    total_num_streams,          # total_num_streams
+                    total_num_gpus,             # total_num_gpus
                 ),
                 name=f"NVDECWorker-GPU{gpu_id}",
                 daemon=False,
@@ -311,6 +326,23 @@ class NVDECWorkerManager:
                     if self._stream_configs else 0
                 )
 
+        # Add per-GPU frame counts and FPS
+        if self._gpu_frame_counts and self._start_time:
+            elapsed = time.perf_counter() - self._start_time
+            gpu_stats = {}
+            for gpu_id, counter in self._gpu_frame_counts.items():
+                gpu_frames = counter.value
+                num_cams = len(self._gpu_camera_assignments.get(gpu_id, []))
+                gpu_fps = gpu_frames / elapsed if elapsed > 0 else 0
+                gpu_per_cam = gpu_fps / num_cams if num_cams > 0 else 0
+                gpu_stats[f'GPU{gpu_id}'] = {
+                    'frames': gpu_frames,
+                    'cameras': num_cams,
+                    'fps': gpu_fps,
+                    'fps_per_cam': gpu_per_cam,
+                }
+            stats['per_gpu_stats'] = gpu_stats
+
         # Collect any available results from queue (non-blocking)
         gpu_results = []
         if self._result_queue: