matrice-streaming 0.1.62__tar.gz → 0.1.64__tar.gz

{matrice_streaming-0.1.62 → matrice_streaming-0.1.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.62
+Version: 0.1.64
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.62 → matrice_streaming-0.1.64}/matrice_streaming.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_streaming
-Version: 0.1.62
+Version: 0.1.64
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_streaming-0.1.62 → matrice_streaming-0.1.64}/src/matrice_streaming/streaming_gateway/camera_streamer/nvdec.py

@@ -56,11 +56,21 @@ import os
 import time
 import threading
 import queue as thread_queue
+import hashlib
+import tempfile
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Any
+from urllib.parse import urlparse, urlunparse
 
 import numpy as np
 
+try:
+    import requests
+    REQUESTS_AVAILABLE = True
+except ImportError:
+    REQUESTS_AVAILABLE = False
+
 try:
     import cupy as cp
     CUPY_AVAILABLE = True
@@ -93,6 +103,267 @@ def setup_logging(quiet: bool = True):
     logging.getLogger('cuda_shm_ring_buffer').setLevel(logging.WARNING if quiet else logging.INFO)
 
 
+# =============================================================================
+# Video Downloader for HTTPS URLs (PyNvVideoCodec's FFmpeg lacks HTTPS support)
+# =============================================================================
+
+class VideoDownloader:
+    """Downloads and caches video files from HTTPS URLs.
+
+    PyNvVideoCodec uses a bundled FFmpeg that doesn't have HTTPS support.
+    This class downloads HTTPS videos to local files before passing them
+    to the NVDEC demuxer.
+
+    Features:
+    - URL deduplication: same video URL (ignoring query params) is only downloaded once
+    - Disk caching: reuses existing files across runs
+    - Progress tracking for large files
+    - Dynamic timeout based on file size
+    """
+
+    # Configuration
+    DOWNLOAD_TIMEOUT = 300  # Base timeout in seconds
+    DOWNLOAD_TIMEOUT_PER_100MB = 300  # Additional seconds per 100MB
+    MAX_DOWNLOAD_TIMEOUT = 6000  # 100 minutes max
+    DOWNLOAD_CHUNK_SIZE = 8192
+
+    # Singleton instance for process-wide caching
+    _instance: Optional['VideoDownloader'] = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        """Singleton pattern for process-wide cache sharing."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        """Initialize the video downloader."""
+        if self._initialized:
+            return
+
+        self._initialized = True
+        self.downloaded_files: Dict[str, str] = {}
+        self._normalized_url_to_path: Dict[str, str] = {}
+        self._download_lock = threading.Lock()
+        self.temp_dir = Path(tempfile.gettempdir()) / "nvdec_video_cache"
+        self.temp_dir.mkdir(exist_ok=True)
+        logger.info(f"VideoDownloader initialized, cache dir: {self.temp_dir}")
+
+    def prepare_source(self, video_path: str, camera_id: str) -> str:
+        """Prepare video source, downloading HTTPS URLs if needed.
+
+        Args:
+            video_path: Video file path, RTSP URL, or HTTPS URL
+            camera_id: Camera identifier for logging
+
+        Returns:
+            Local file path (downloaded if HTTPS) or original path
+        """
+        if not self._is_https_url(video_path):
+            return video_path
+
+        if not REQUESTS_AVAILABLE:
+            logger.warning(f"requests module not available, cannot download HTTPS URL for {camera_id}")
+            return video_path
+
+        local_path = self._download_video(video_path, camera_id)
+        if local_path:
+            return local_path
+
+        logger.warning(f"Failed to download {video_path} for {camera_id}, will try URL directly (may fail)")
+        return video_path
+
+    def _is_https_url(self, source: str) -> bool:
+        """Check if source is an HTTPS URL."""
+        return source.startswith('https://')
+
+    def _normalize_url(self, url: str) -> str:
+        """Normalize URL by stripping query parameters for deduplication."""
+        parsed = urlparse(url)
+        return urlunparse((
+            parsed.scheme,
+            parsed.netloc,
+            parsed.path,
+            '', '', ''  # params, query, fragment
+        ))
+
+    def _get_url_hash(self, normalized_url: str) -> str:
+        """Generate a short hash for consistent file naming."""
+        return hashlib.md5(normalized_url.encode()).hexdigest()[:12]
+
+    def _download_video(self, url: str, camera_id: str) -> Optional[str]:
+        """Download video file from HTTPS URL with caching.
+
+        Thread-safe: uses lock to prevent duplicate downloads.
+
+        Args:
+            url: HTTPS video URL
+            camera_id: Camera identifier for logging
+
+        Returns:
+            Local file path or None if download failed
+        """
+        normalized_url = self._normalize_url(url)
+        file_ext = Path(url.split('?')[0]).suffix or '.mp4'
+        url_hash = self._get_url_hash(normalized_url)
+        expected_path = self.temp_dir / f"video_{url_hash}{file_ext}"
+        expected_path_str = str(expected_path)
+
+        # Quick check: file already on disk
+        if expected_path.exists():
+            existing_size = expected_path.stat().st_size
+            logger.info(
+                f"[{camera_id}] Reusing cached video: {expected_path.name} "
+                f"({existing_size / (1024*1024):.1f}MB)"
+            )
+            with self._download_lock:
+                self.downloaded_files[url] = expected_path_str
+                self._normalized_url_to_path[normalized_url] = expected_path_str
+            return expected_path_str
+
+        # Check memory cache
+        with self._download_lock:
+            if url in self.downloaded_files:
+                local_path = self.downloaded_files[url]
+                if os.path.exists(local_path):
+                    logger.debug(f"[{camera_id}] Using cached path (exact URL match)")
+                    return local_path
+
+            if normalized_url in self._normalized_url_to_path:
+                local_path = self._normalized_url_to_path[normalized_url]
+                if os.path.exists(local_path):
+                    logger.info(f"[{camera_id}] Reusing download (same base URL)")
+                    self.downloaded_files[url] = local_path
+                    return local_path
+
+        # Need to download - acquire lock to prevent duplicate downloads
+        with self._download_lock:
+            # Double-check after acquiring lock
+            if expected_path.exists():
+                self.downloaded_files[url] = expected_path_str
+                self._normalized_url_to_path[normalized_url] = expected_path_str
+                return expected_path_str
+
+            return self._do_download(url, expected_path, camera_id)
+
+    def _do_download(self, url: str, dest_path: Path, camera_id: str) -> Optional[str]:
+        """Perform the actual download. Must be called with _download_lock held."""
+        content_length = 0
+        file_size_mb = 0.0
+        bytes_downloaded = 0
+        timeout = self.DOWNLOAD_TIMEOUT
+
+        try:
+            # HEAD request to get file size
+            try:
+                head_response = requests.head(url, timeout=10, allow_redirects=True)
+                content_length = int(head_response.headers.get('Content-Length', 0))
+                file_size_mb = content_length / (1024 * 1024)
+            except Exception as e:
+                logger.debug(f"[{camera_id}] HEAD request failed: {e}")
+
+            # Calculate dynamic timeout
+            if content_length > 0:
+                timeout = min(
+                    self.DOWNLOAD_TIMEOUT + int(file_size_mb // 100) * self.DOWNLOAD_TIMEOUT_PER_100MB,
+                    self.MAX_DOWNLOAD_TIMEOUT
+                )
+                logger.info(f"[{camera_id}] Downloading {file_size_mb:.1f}MB (timeout: {timeout}s)")
+            else:
+                logger.info(f"[{camera_id}] Downloading video (size unknown, timeout: {timeout}s)")
+
+            # Download with progress tracking
+            response = requests.get(url, stream=True, timeout=timeout)
+            response.raise_for_status()
+
+            if content_length == 0:
+                content_length = int(response.headers.get('Content-Length', 0))
+                file_size_mb = content_length / (1024 * 1024) if content_length > 0 else 0
+
+            last_progress_log = 0
+
+            with open(dest_path, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=self.DOWNLOAD_CHUNK_SIZE):
+                    f.write(chunk)
+                    bytes_downloaded += len(chunk)
+
+                    # Log progress every 50MB for large files
+                    if content_length > 50_000_000:
+                        mb_downloaded = bytes_downloaded // (1024 * 1024)
+                        if mb_downloaded - last_progress_log >= 50:
+                            progress = (bytes_downloaded / content_length * 100) if content_length else 0
+                            logger.info(
+                                f"[{camera_id}] Download progress: "
+                                f"{mb_downloaded}MB / {file_size_mb:.0f}MB ({progress:.1f}%)"
+                            )
+                            last_progress_log = mb_downloaded
+
+            # Update caches
+            normalized_url = self._normalize_url(url)
+            dest_path_str = str(dest_path)
+            self.downloaded_files[url] = dest_path_str
+            self._normalized_url_to_path[normalized_url] = dest_path_str
+
+            logger.info(
+                f"[{camera_id}] Downloaded: {dest_path.name} "
+                f"({bytes_downloaded / (1024*1024):.1f}MB)"
+            )
+            return dest_path_str
+
+        except requests.Timeout:
+            logger.error(
+                f"[{camera_id}] Download timeout: {file_size_mb:.1f}MB, "
+                f"got {bytes_downloaded/(1024*1024):.1f}MB in {timeout}s"
+            )
+        except requests.HTTPError as e:
+            logger.error(f"[{camera_id}] HTTP error: {e.response.status_code} - {e.response.reason}")
+        except IOError as e:
+            logger.error(f"[{camera_id}] Disk I/O error: {e}")
+        except Exception as e:
+            logger.error(f"[{camera_id}] Download failed: {type(e).__name__}: {e}")
+
+        # Cleanup partial download
+        try:
+            if dest_path.exists():
+                dest_path.unlink()
+        except Exception:
+            pass
+
+        return None
+
+    def cleanup(self):
+        """Clean up downloaded temporary files."""
+        unique_files = set(self.downloaded_files.values())
+        unique_files.update(self._normalized_url_to_path.values())
+
+        for filepath in unique_files:
+            try:
+                if os.path.exists(filepath):
+                    os.remove(filepath)
+                    logger.debug(f"Removed temp file: {filepath}")
+            except Exception as e:
+                logger.warning(f"Failed to remove temp file {filepath}: {e}")
+
+        self.downloaded_files.clear()
+        self._normalized_url_to_path.clear()
+
+
+# Global video downloader instance
+_video_downloader: Optional[VideoDownloader] = None
+
+
+def get_video_downloader() -> VideoDownloader:
+    """Get or create the global VideoDownloader instance."""
+    global _video_downloader
+    if _video_downloader is None:
+        _video_downloader = VideoDownloader()
+    return _video_downloader
+
+
 @dataclass
 class StreamConfig:
     """Configuration for a single video stream."""
@@ -276,7 +547,12 @@ def surface_to_nv12(frame, target_h: int = 640, target_w: int = 640) -> Optional
         return nv12_frame[:, :, cp.newaxis] if nv12_frame is not None else None
 
     except Exception as e:
-        logger.warning(f"surface_to_nv12 failed: {e}")
+        # Safely encode error message (some CUDA errors contain non-ASCII chars like '×')
+        try:
+            err_msg = str(e).encode('ascii', errors='replace').decode('ascii')
+        except Exception:
+            err_msg = "unknown error"
+        logger.warning(f"surface_to_nv12 failed: {err_msg}")
         return None
 
 
@@ -320,13 +596,22 @@ class NVDECDecoderPool:
 
     def assign_stream(self, stream_id: int, camera_id: str, video_path: str,
                       width: int = 640, height: int = 640) -> bool:
-        """Assign a stream to a decoder (round-robin)."""
+        """Assign a stream to a decoder (round-robin).
+
+        Automatically downloads HTTPS URLs to local files since PyNvVideoCodec's
+        bundled FFmpeg doesn't support HTTPS protocol.
+        """
         if self.actual_pool_size == 0:
             return False
 
         decoder_idx = stream_id % self.actual_pool_size
+
+        # Download HTTPS URLs to local files (PyNvVideoCodec lacks HTTPS support)
+        downloader = get_video_downloader()
+        local_path = downloader.prepare_source(video_path, camera_id)
+
         try:
-            demuxer = nvc.CreateDemuxer(video_path)
+            demuxer = nvc.CreateDemuxer(local_path)
         except Exception as e:
             logger.error(f"Failed to create demuxer for {camera_id}: {e}")
             return False
@@ -334,7 +619,7 @@ class NVDECDecoderPool:
         stream_state = StreamState(
             stream_id=stream_id,
             camera_id=camera_id,
-            video_path=video_path,
+            video_path=local_path,  # Store local path for video looping
             demuxer=demuxer,
             width=width,
             height=height
@@ -447,7 +732,6 @@ def nvdec_pool_worker(
     frames_since_counter_update = 0
     counter_batch_size = 100
     start_time = time.perf_counter()
-    last_log_time = start_time
     camera_ids = pool.get_camera_ids_for_decoder(decoder_idx)
     num_streams = len(camera_ids)
 
@@ -465,19 +749,12 @@ def nvdec_pool_worker(
         next_frame_time = 0
         fps_mode = ", unlimited FPS"
 
-    logger.info(f"Worker {worker_id}: decoder={decoder_idx}, cams={num_streams}{fps_mode}")
+    logger.debug(f"Worker {worker_id}: decoder={decoder_idx}, cams={num_streams}{fps_mode}")
 
     while not stop_event.is_set():
         if time.perf_counter() - start_time >= duration_sec:
             break
 
-        now = time.perf_counter()
-        if now - last_log_time >= 5.0:
-            elapsed = now - start_time
-            fps = local_frames / elapsed if elapsed > 0 else 0
-            logger.info(f"Worker {worker_id}: {local_frames} frames, {fps:.0f} FPS")
-            last_log_time = now
-
         # FPS limiting: wait until next scheduled frame time
         if fps_limit_enabled:
             current_time = time.perf_counter()
@@ -672,10 +949,52 @@ def nvdec_pool_process(
             t.start()
             threads.append(t)
 
+        # Progress monitoring loop with current/avg FPS tracking
         start_time = time.perf_counter()
+        last_report_time = start_time
+        last_frame_count = 0
+        report_interval = 5.0
+        processing_start_time = None
+        frames_at_processing_start = 0
+
         while not stop_event.is_set():
-            if time.perf_counter() - start_time >= duration_sec:
+            current_time = time.perf_counter()
+            if current_time - start_time >= duration_sec:
                 break
+
+            # Periodic progress report with current and average FPS
+            if current_time - last_report_time >= report_interval:
+                elapsed = current_time - start_time
+
+                if shared_frame_count is not None:
+                    current_frames = shared_frame_count.value
+                    interval_frames = current_frames - last_frame_count
+                    interval_fps = interval_frames / report_interval
+                    per_stream_fps = interval_fps / len(camera_configs) if camera_configs else 0
+
+                    # Track when processing actually starts (exclude warmup)
+                    if processing_start_time is None and current_frames > 0:
+                        processing_start_time = last_report_time
+                        frames_at_processing_start = last_frame_count
+
+                    # Calculate average FPS excluding warmup
+                    if processing_start_time is not None:
+                        processing_elapsed = current_time - processing_start_time
+                        processing_frames = current_frames - frames_at_processing_start
+                        avg_fps = processing_frames / processing_elapsed if processing_elapsed > 0 else 0
+                        avg_per_stream = avg_fps / len(camera_configs) if camera_configs else 0
+                        remaining = max(0, duration_sec - elapsed)
+
+                        logger.info(
+                            f"GPU{gpu_id} [{elapsed:5.1f}s] {current_frames:,} frames | "
+                            f"cur: {interval_fps:,.0f} FPS ({per_stream_fps:.1f}/stream) | "
+                            f"avg: {avg_fps:,.0f} FPS ({avg_per_stream:.1f}/stream) | "
+                            f"{remaining:.0f}s left"
+                        )
+
+                    last_frame_count = current_frames
+                last_report_time = current_time
+
             time.sleep(0.1)
 
         thread_stop_event.set()