matrice-streaming 0.1.14__py3-none-any.whl → 0.1.65__py3-none-any.whl

matrice_streaming/streaming_gateway/camera_streamer/stream_statistics.py

@@ -8,6 +8,7 @@ class StreamStatistics:
     """Manages streaming statistics and timing data."""
 
     STATS_LOG_INTERVAL = 50
+    MAX_HISTORY_SIZE = 1000  # Maximum entries per stream to prevent memory growth
 
     def __init__(self):
         """Initialize statistics tracker."""
@@ -25,11 +26,13 @@ class StreamStatistics:
         self.last_frame_sizes: Dict[str, int] = {}
 
         # History storage for accurate statistics (accumulated between reporting intervals)
+        # These are bounded to MAX_HISTORY_SIZE entries per stream
         self.read_times_history: Dict[str, List[float]] = {}
         self.write_times_history: Dict[str, List[float]] = {}
         self.process_times_history: Dict[str, List[float]] = {}
         self.frame_sizes_history: Dict[str, List[int]] = {}
         self.frame_timestamps_history: Dict[str, List[float]] = {}
+        self.encoding_times_history: Dict[str, List[float]] = {}  # NEW: encoding time tracking
 
         # Per-stream input order tracking
         self.input_order: Dict[str, int] = {}
@@ -58,7 +61,8 @@ class StreamStatistics:
         read_time: float,
         write_time: float,
         process_time: float,
-        frame_size: Optional[int] = None
+        frame_size: Optional[int] = None,
+        encoding_time: float = 0.0
     ):
         """Update timing statistics for a stream.
 
@@ -68,6 +72,7 @@ class StreamStatistics:
             write_time: Time spent writing/sending frame
             process_time: Total processing time
             frame_size: Size of encoded frame in bytes (ACG frame size)
+            encoding_time: Time spent encoding frame (NEW)
         """
         key = self._normalize_key(stream_key)
         timestamp = time.time()
@@ -79,21 +84,34 @@ class StreamStatistics:
         if frame_size is not None:
             self.last_frame_sizes[key] = frame_size
 
-        # Append to history for accurate statistics
+        # Append to history for accurate statistics (bounded to prevent memory growth)
         if key not in self.read_times_history:
             self.read_times_history[key] = []
             self.write_times_history[key] = []
             self.process_times_history[key] = []
             self.frame_sizes_history[key] = []
             self.frame_timestamps_history[key] = []
+            self.encoding_times_history[key] = []
 
         self.read_times_history[key].append(read_time)
         self.write_times_history[key].append(write_time)
         self.process_times_history[key].append(process_time)
         self.frame_timestamps_history[key].append(timestamp)
+        self.encoding_times_history[key].append(encoding_time)
 
         if frame_size is not None:
             self.frame_sizes_history[key].append(frame_size)
+
+        # Enforce size limits to prevent unbounded growth
+        # Keep only the last MAX_HISTORY_SIZE entries
+        if len(self.read_times_history[key]) > self.MAX_HISTORY_SIZE:
+            self.read_times_history[key] = self.read_times_history[key][-self.MAX_HISTORY_SIZE:]
+            self.write_times_history[key] = self.write_times_history[key][-self.MAX_HISTORY_SIZE:]
+            self.process_times_history[key] = self.process_times_history[key][-self.MAX_HISTORY_SIZE:]
+            self.frame_timestamps_history[key] = self.frame_timestamps_history[key][-self.MAX_HISTORY_SIZE:]
+            self.encoding_times_history[key] = self.encoding_times_history[key][-self.MAX_HISTORY_SIZE:]
+            if len(self.frame_sizes_history[key]) > self.MAX_HISTORY_SIZE:
+                self.frame_sizes_history[key] = self.frame_sizes_history[key][-self.MAX_HISTORY_SIZE:]
     
     def get_timing(self, stream_key: str) -> Tuple[float, float, float]:
         """Get timing data for a stream.
@@ -142,7 +160,7 @@ class StreamStatistics:
         write_time: float
     ):
         """Log periodic statistics.
-        
+
         Args:
             stream_key: Stream identifier
             read_time: Time spent reading frame
@@ -157,7 +175,82 @@ class StreamStatistics:
                 f"Timing: read={read_time*1000:.1f}ms, encode={encoding_time*1000:.1f}ms, "
                 f"write={write_time*1000:.1f}ms"
             )
-    
+
+    def log_detailed_stats(self, stream_key: str) -> None:
+        """Log comprehensive metrics for a stream.
+
+        Args:
+            stream_key: Stream identifier
+        """
+        stats = self.get_timing_statistics(stream_key)
+        if not stats:
+            return
+
+        # Calculate additional metrics
+        total_frames = self.frames_sent + self.frames_skipped + self.frames_diff_sent
+        skip_rate = (self.frames_skipped / total_frames * 100) if total_frames > 0 else 0
+
+        # FPS metrics
+        fps_stats = stats.get("fps", {})
+        fps_current = fps_stats.get("avg", 0)
+        fps_min = fps_stats.get("min", 0)
+        fps_max = fps_stats.get("max", 0)
+
+        # Latency breakdown (ms)
+        read_ms = stats.get("read_time_ms", {}).get("avg", 0)
+        encoding_ms = stats.get("encoding_time_ms", {}).get("avg", 0)
+        write_ms = stats.get("write_time_ms", {}).get("avg", 0)
+        process_ms = stats.get("process_time_ms", {}).get("avg", 0)
+
+        # Frame size stats (KB)
+        frame_size_stats = stats.get("frame_size_bytes", {})
+        frame_size_avg_kb = frame_size_stats.get("avg", 0) / 1024
+        frame_size_min_kb = frame_size_stats.get("min", 0) / 1024
+        frame_size_max_kb = frame_size_stats.get("max", 0) / 1024
+
+        # Throughput (KB/s)
+        throughput_kbps = (frame_size_avg_kb * fps_current) if fps_current > 0 else 0
+
+        self.logger.info(
+            f"Stream Metrics [{stream_key}]: "
+            f"FPS={fps_current:.1f} (min={fps_min:.1f}, max={fps_max:.1f}) | "
+            f"Latency: read={read_ms:.1f}ms, encode={encoding_ms:.1f}ms, write={write_ms:.1f}ms, total={process_ms:.1f}ms | "
+            f"Frames: sent={self.frames_sent}, skipped={self.frames_skipped} ({skip_rate:.1f}%) | "
+            f"Frame size: {frame_size_avg_kb:.1f}KB (min={frame_size_min_kb:.1f}, max={frame_size_max_kb:.1f}) | "
+            f"Throughput: {throughput_kbps:.1f} KB/s"
+        )
+
+    def log_aggregated_stats(self) -> None:
+        """Log aggregated metrics across all streams."""
+        total_frames_sent = self.frames_sent
+        total_frames_skipped = self.frames_skipped
+        total_frames_diff = self.frames_diff_sent
+        total_frames = total_frames_sent + total_frames_skipped + total_frames_diff
+
+        if total_frames == 0:
+            return
+
+        skip_rate = (total_frames_skipped / total_frames * 100)
+        diff_rate = (total_frames_diff / total_frames * 100)
+
+        # Aggregate FPS across all streams
+        all_fps = []
+        for stream_key in self.last_read_times.keys():
+            stats = self.get_timing_statistics(stream_key)
+            if stats and "fps" in stats:
+                fps_avg = stats["fps"].get("avg", 0)
+                if fps_avg > 0:
+                    all_fps.append(fps_avg)
+
+        avg_fps = sum(all_fps) / len(all_fps) if all_fps else 0
+
+        self.logger.info(
+            f"Gateway Aggregate Metrics: "
+            f"Total frames: {total_frames} (sent={total_frames_sent}, skipped={total_frames_skipped}, diff={total_frames_diff}) | "
+            f"Skip rate: {skip_rate:.1f}%, Diff rate: {diff_rate:.1f}% | "
+            f"Avg FPS across {len(all_fps)} streams: {avg_fps:.1f}"
+        )
+
     def get_transmission_stats(self, video_codec: str, active_streams: int) -> Dict[str, Any]:
         """Get comprehensive transmission statistics.
         
@@ -231,6 +324,7 @@ class StreamStatistics:
         result = {
             "read_time_ms": {"min": 0, "max": 0, "avg": 0, "count": 0},
             "write_time_ms": {"min": 0, "max": 0, "avg": 0, "count": 0},
+            "encoding_time_ms": {"min": 0, "max": 0, "avg": 0, "count": 0},  # NEW
             "process_time_ms": {"min": 0, "max": 0, "avg": 0, "count": 0},
             "frame_size_bytes": {"min": 0, "max": 0, "avg": 0, "count": 0},
             "fps": {"min": 0, "max": 0, "avg": 0},
@@ -266,6 +360,16 @@ class StreamStatistics:
                 "count": len(process_times),
             }
 
+        # Calculate encoding time statistics
+        if key in self.encoding_times_history and self.encoding_times_history[key]:
+            encoding_times = self.encoding_times_history[key]
+            result["encoding_time_ms"] = {
+                "min": min(encoding_times) * 1000,
+                "max": max(encoding_times) * 1000,
+                "avg": (sum(encoding_times) / len(encoding_times)) * 1000,
+                "count": len(encoding_times),
+            }
+
         # Calculate frame size statistics
         if key in self.frame_sizes_history and self.frame_sizes_history[key]:
             frame_sizes = self.frame_sizes_history[key]
@@ -318,6 +422,7 @@ class StreamStatistics:
             self.process_times_history.clear()
             self.frame_sizes_history.clear()
             self.frame_timestamps_history.clear()
+            self.encoding_times_history.clear()
             self.logger.debug("Cleared timing history for all streams")
         else:
             # Clear specific stream
@@ -332,6 +437,8 @@ class StreamStatistics:
                 self.frame_sizes_history[key].clear()
             if key in self.frame_timestamps_history:
                 self.frame_timestamps_history[key].clear()
+            if key in self.encoding_times_history:
+                self.encoding_times_history[key].clear()
             self.logger.debug(f"Cleared timing history for stream: {stream_key}")
     
     def reset(self):

matrice_streaming/streaming_gateway/camera_streamer/video_capture_manager.py

@@ -5,7 +5,9 @@ import cv2
 import requests
 import os
 import tempfile
+import hashlib
 from pathlib import Path
+from urllib.parse import urlparse, urlunparse
 from typing import Union, Optional, Tuple, Dict, Any
 
 
@@ -14,18 +16,28 @@ class VideoSourceConfig:
     MAX_CAPTURE_RETRIES = 3
     CAPTURE_RETRY_DELAY = 2.0
     MAX_CONSECUTIVE_FAILURES = 10
-    DOWNLOAD_TIMEOUT = 30
+    DOWNLOAD_TIMEOUT = 30  # Base timeout in seconds
+    DOWNLOAD_TIMEOUT_PER_100MB = 30  # Additional seconds per 100MB
+    MAX_DOWNLOAD_TIMEOUT = 600  # 10 minutes max
     DOWNLOAD_CHUNK_SIZE = 8192
-    DEFAULT_BUFFER_SIZE = 1
+    DEFAULT_BUFFER_SIZE = 5  # Increased from 1 to 5 for better throughput
     DEFAULT_FPS = 30
 
 
 class VideoCaptureManager:
-    """Manages video capture from various sources with retry logic and caching."""
+    """Manages video capture from various sources with retry logic and caching.
+    
+    Features URL deduplication: if multiple cameras use the same video URL
+    (ignoring query parameters like AWS signed URL tokens), the video is only
+    downloaded once and the local path is shared between cameras.
+    """
     
     def __init__(self):
         """Initialize video capture manager."""
+        # Maps full URL -> local file path (for backwards compatibility)
         self.downloaded_files: Dict[str, str] = {}
+        # Maps normalized URL (without query params) -> local file path (for deduplication)
+        self._normalized_url_to_path: Dict[str, str] = {}
         self.temp_dir = Path(tempfile.gettempdir()) / "matrice_streaming_cache"
         self.temp_dir.mkdir(exist_ok=True)
         self.logger = logging.getLogger(__name__)
@@ -43,7 +55,7 @@ class VideoCaptureManager:
         if isinstance(source, str) and self._is_downloadable_url(source):
             local_path = self._download_video_file(source, stream_key)
             if local_path:
-                self.logger.info(f"Using downloaded file: {local_path}")
+                self.logger.debug(f"Using downloaded file: {local_path}")
                 return local_path
             else:
                 self.logger.warning(f"Failed to download {source}, will try to use URL directly")
@@ -82,11 +94,28 @@ class VideoCaptureManager:
                 return cap, source_type
                 
             except Exception as e:
-                self.logger.error(f"Attempt {attempt + 1}/{VideoSourceConfig.MAX_CAPTURE_RETRIES} failed: {e}")
+                # Gather detailed source info for error logging
+                source_info = ""
+                if isinstance(source, str):
+                    if os.path.exists(source):
+                        file_size = os.path.getsize(source)
+                        source_info = f" | File exists: {file_size/(1024*1024):.1f}MB"
+                    elif source.startswith("rtsp://") or source.startswith("http://") or source.startswith("https://"):
+                        source_info = f" | Network source"
+                    else:
+                        source_info = " | File does not exist"
+
+                self.logger.error(
+                    f"Attempt {attempt + 1}/{VideoSourceConfig.MAX_CAPTURE_RETRIES} failed to open "
+                    f"{source_type} source: {type(e).__name__}: {e}{source_info}"
+                )
                 if attempt < VideoSourceConfig.MAX_CAPTURE_RETRIES - 1:
                     time.sleep(VideoSourceConfig.CAPTURE_RETRY_DELAY)
                 else:
-                    raise RuntimeError(f"Failed to open source after {VideoSourceConfig.MAX_CAPTURE_RETRIES} attempts")
+                    raise RuntimeError(
+                        f"Failed to open source after {VideoSourceConfig.MAX_CAPTURE_RETRIES} attempts: "
+                        f"{type(e).__name__}: {e}{source_info}"
+                    )
     
     def get_video_properties(self, cap: cv2.VideoCapture) -> Dict[str, Any]:
         """Extract video properties from capture.
@@ -123,14 +152,20 @@ class VideoCaptureManager:
     
     def cleanup(self):
         """Clean up downloaded temporary files."""
-        for filepath in self.downloaded_files.values():
+        # Collect unique file paths (since multiple URLs may point to the same file)
+        unique_files = set(self.downloaded_files.values())
+        unique_files.update(self._normalized_url_to_path.values())
+        
+        for filepath in unique_files:
             try:
                 if os.path.exists(filepath):
                     os.remove(filepath)
                     self.logger.debug(f"Removed temp file: {filepath}")
             except Exception as e:
                 self.logger.warning(f"Failed to remove temp file {filepath}: {e}")
+        
         self.downloaded_files.clear()
+        self._normalized_url_to_path.clear()
     
     # Private methods
     
@@ -138,36 +173,197 @@ class VideoCaptureManager:
         """Check if source is a downloadable URL (not RTSP)."""
         return (source.startswith('http://') or source.startswith('https://')) and not source.startswith('rtsp')
     
+    def _normalize_url(self, url: str) -> str:
+        """Normalize URL by stripping query parameters.
+        
+        This allows URLs that point to the same file but have different
+        query parameters (e.g., AWS signed URLs with different tokens)
+        to be recognized as the same resource.
+        
+        Args:
+            url: Full URL with potential query parameters
+            
+        Returns:
+            Normalized URL without query parameters
+        """
+        parsed = urlparse(url)
+        # Rebuild URL without query string and fragment
+        normalized = urlunparse((
+            parsed.scheme,
+            parsed.netloc,
+            parsed.path,
+            '',  # params
+            '',  # query
+            ''   # fragment
+        ))
+        return normalized
+    
+    def _get_url_hash(self, normalized_url: str) -> str:
+        """Generate a short hash for the normalized URL.
+        
+        This is used for creating unique but consistent file names.
+        
+        Args:
+            normalized_url: URL without query parameters
+            
+        Returns:
+            Short hash string
+        """
+        return hashlib.md5(normalized_url.encode()).hexdigest()[:12]
+    
     def _download_video_file(self, url: str, stream_key: str) -> Optional[str]:
-        """Download video file from URL and cache it locally."""
+        """Download video file from URL and cache it locally.
+
+        Uses URL deduplication: if the same video (by normalized URL without
+        query parameters) has already been downloaded, returns the existing
+        local path instead of downloading again.
+
+        Features dynamic timeout calculation based on file size and progress
+        tracking for large files.
+
+        Args:
+            url: Video file URL (may include query parameters like AWS signatures)
+            stream_key: Stream identifier
+
+        Returns:
+            Local file path or None if download failed
+        """
+        # Initialize tracking variables for error reporting
+        content_length = 0
+        file_size_mb = 0.0
+        bytes_downloaded = 0
+        timeout = VideoSourceConfig.DOWNLOAD_TIMEOUT
+        expected_path = None
+
         try:
-            # Check cache
+            # Normalize URL to check for duplicate downloads
+            # (same file but different query params, e.g., different AWS signatures)
+            normalized_url = self._normalize_url(url)
+
+            # Generate a consistent filename using URL hash
+            file_ext = Path(url.split('?')[0]).suffix or '.mp4'
+            url_hash = self._get_url_hash(normalized_url)
+            expected_path = self.temp_dir / f"video_{url_hash}{file_ext}"
+            expected_path_str = str(expected_path)
+
+            # Quick check: if file already exists on disk, use it
+            if os.path.exists(expected_path):
+                existing_size = os.path.getsize(expected_path)
+                self.logger.debug(
+                    f"Reusing existing video file for {stream_key}: {expected_path} "
+                    f"({existing_size / (1024*1024):.1f}MB, already downloaded)"
+                )
+                # Update caches
+                self.downloaded_files[url] = expected_path_str
+                self._normalized_url_to_path[normalized_url] = expected_path_str
+                return expected_path_str
+
+            # Check memory cache for exact URL
             if url in self.downloaded_files:
                 local_path = self.downloaded_files[url]
                 if os.path.exists(local_path):
-                    self.logger.info(f"Using cached video file: {local_path}")
+                    self.logger.debug(f"Using cached video file (exact URL match): {local_path}")
                     return local_path
-            
-            # Download
-            self.logger.info(f"Downloading video file from: {url}")
-            response = requests.get(url, stream=True, timeout=VideoSourceConfig.DOWNLOAD_TIMEOUT)
+
+            # Check memory cache for normalized URL
+            if normalized_url in self._normalized_url_to_path:
+                local_path = self._normalized_url_to_path[normalized_url]
+                if os.path.exists(local_path):
+                    self.logger.debug(
+                        f"Reusing previously downloaded file for {stream_key}: {local_path} "
+                        f"(same base URL, different query params)"
+                    )
+                    self.downloaded_files[url] = local_path
+                    return local_path
+
+            # HEAD request to get file size for dynamic timeout calculation
+            try:
+                head_response = requests.head(url, timeout=10, allow_redirects=True)
+                content_length = int(head_response.headers.get('Content-Length', 0))
+                file_size_mb = content_length / (1024 * 1024)
+            except Exception as head_err:
+                self.logger.debug(f"HEAD request failed for {stream_key}: {head_err}")
+                content_length = 0
+                file_size_mb = 0
+
+            # Calculate dynamic timeout based on file size
+            if content_length > 0:
+                # Base timeout + additional time per 100MB
+                timeout = min(
+                    VideoSourceConfig.DOWNLOAD_TIMEOUT +
+                    int(file_size_mb // 100) * VideoSourceConfig.DOWNLOAD_TIMEOUT_PER_100MB,
+                    VideoSourceConfig.MAX_DOWNLOAD_TIMEOUT
+                )
+                self.logger.info(
+                    f"Downloading video file for {stream_key}: {file_size_mb:.1f}MB "
+                    f"(timeout: {timeout}s)"
+                )
+            else:
+                timeout = VideoSourceConfig.DOWNLOAD_TIMEOUT
+                self.logger.info(f"Downloading video file for {stream_key} (size unknown, timeout: {timeout}s)")
+
+            # Download the file with progress tracking
+            response = requests.get(url, stream=True, timeout=timeout)
             response.raise_for_status()
-            
-            # Save to temp directory
-            file_ext = Path(url.split('?')[0]).suffix or '.mp4'
-            local_path = self.temp_dir / f"{stream_key}_{int(time.time())}{file_ext}"
-            
-            with open(local_path, 'wb') as f:
+
+            # Update content_length from response if HEAD failed
+            if content_length == 0:
+                content_length = int(response.headers.get('Content-Length', 0))
+                file_size_mb = content_length / (1024 * 1024) if content_length > 0 else 0
+
+            last_progress_log = 0
+
+            with open(expected_path, 'wb') as f:
                 for chunk in response.iter_content(chunk_size=VideoSourceConfig.DOWNLOAD_CHUNK_SIZE):
                     f.write(chunk)
-            
-            # Cache
-            self.downloaded_files[url] = str(local_path)
-            self.logger.info(f"Downloaded video file to: {local_path}")
-            return str(local_path)
-            
+                    bytes_downloaded += len(chunk)
+
+                    # Log progress every 50MB for large files (>50MB)
+                    if content_length > 50_000_000:
+                        mb_downloaded = bytes_downloaded // (1024 * 1024)
+                        if mb_downloaded - last_progress_log >= 50:
+                            progress = (bytes_downloaded / content_length * 100) if content_length else 0
+                            self.logger.info(
+                                f"Download progress for {stream_key}: "
+                                f"{mb_downloaded}MB / {file_size_mb:.0f}MB ({progress:.1f}%)"
+                            )
+                            last_progress_log = mb_downloaded
+
+            # Cache for both exact URL and normalized URL
+            self.downloaded_files[url] = expected_path_str
+            self._normalized_url_to_path[normalized_url] = expected_path_str
+
+            self.logger.info(
+                f"Downloaded video file for {stream_key}: {expected_path} "
+                f"({bytes_downloaded / (1024*1024):.1f}MB)"
+            )
+            return expected_path_str
+
+        except requests.Timeout as e:
+            self.logger.error(
+                f"Download timeout for {stream_key}: {e} | "
+                f"File size: {file_size_mb:.1f}MB, Downloaded: {bytes_downloaded/(1024*1024):.1f}MB, "
+                f"Timeout: {timeout}s"
+            )
+            return None
+        except requests.HTTPError as e:
+            self.logger.error(
+                f"HTTP error downloading {stream_key}: {e.response.status_code} - {e.response.reason} | "
+                f"URL: {url[:100]}..."
+            )
+            return None
+        except IOError as e:
+            self.logger.error(
+                f"Disk I/O error downloading {stream_key}: {e} | "
+                f"Downloaded: {bytes_downloaded/(1024*1024):.1f}MB, Path: {expected_path}"
+            )
+            return None
         except Exception as e:
-            self.logger.error(f"Failed to download video file: {e}")
+            size_info = f"{file_size_mb:.1f}MB" if content_length > 0 else "unknown"
+            self.logger.error(
+                f"Failed to download video file for {stream_key}: {type(e).__name__}: {e} | "
+                f"File size: {size_info}, Downloaded: {bytes_downloaded/(1024*1024):.1f}MB"
+            )
             return None
     
     def _detect_source_type(self, source: Union[str, int]) -> str: