matrice-inference 0.1.22__py3-none-any.whl → 0.1.23__py3-none-any.whl

matrice_inference/server/server.py

@@ -131,6 +131,7 @@ class MatriceDeployServer:
             self.streaming_pipeline = None
             self.app_deployment = None
             self.stream_manager = None
+            self.camera_config_monitor = None
 
             # Initialize utilities
             self.utils = None
@@ -401,6 +402,9 @@ class MatriceDeployServer:
             
             logging.info("Streaming pipeline initialized successfully")
             
+            # Start camera config monitor if app deployment is available
+            self._start_camera_config_monitor()
+            
         except Exception as e:
             logging.error(f"Failed to initialize streaming pipeline: {str(e)}")
             raise
@@ -415,6 +419,37 @@ class MatriceDeployServer:
         finally:
             loop.close()
 
+    def _start_camera_config_monitor(self):
+        """Start the camera config monitor if app deployment is available."""
+        try:
+            if not self.app_deployment:
+                logging.info("No app deployment configured, skipping camera config monitor")
+                return
+            
+            if not self.streaming_pipeline:
+                logging.warning("Streaming pipeline not initialized, skipping camera config monitor")
+                return
+            
+            # Get check interval from job params (default: 60 seconds)
+            check_interval = self.job_params.get("camera_config_check_interval", 60)
+            
+            # Import and create the monitor
+            from matrice_inference.server.stream.camera_config_monitor import CameraConfigMonitor
+            
+            self.camera_config_monitor = CameraConfigMonitor(
+                app_deployment=self.app_deployment,
+                streaming_pipeline=self.streaming_pipeline,
+                check_interval=check_interval
+            )
+            
+            # Start monitoring
+            self.camera_config_monitor.start()
+            logging.info(f"Camera config monitor started (check interval: {check_interval}s)")
+            
+        except Exception as e:
+            logging.error(f"Failed to start camera config monitor: {str(e)}")
+            # Don't raise - monitor is optional
+
     def _stop_pipeline_in_new_loop(self):
         """Stop the pipeline in a new event loop (for use when already in an event loop)."""
         import asyncio
@@ -462,6 +497,14 @@ class MatriceDeployServer:
             # Signal shutdown to all components
             self._shutdown_event.set()
             
+            # Stop camera config monitor
+            if self.camera_config_monitor:
+                try:
+                    self.camera_config_monitor.stop()
+                    logging.info("Camera config monitor stopped")
+                except Exception as exc:
+                    logging.error("Error stopping camera config monitor: %s", str(exc))
+            
             # Stop streaming pipeline
             if self.streaming_pipeline:
                 try:

matrice_inference/server/stream/camera_config_monitor.py

@@ -0,0 +1,221 @@
+"""Background monitor for camera configuration updates."""
+
+import hashlib
+import json
+import logging
+import threading
+import time
+from typing import Dict, Optional
+
+from matrice_inference.server.stream.utils import CameraConfig
+
+
+class CameraConfigMonitor:
+    """Monitors and syncs camera configurations from app deployment API."""
+
+    DEFAULT_CHECK_INTERVAL = 120  # seconds
+
+    def __init__(
+        self,
+        app_deployment,
+        streaming_pipeline,
+        check_interval: int = DEFAULT_CHECK_INTERVAL
+    ):
+        """Initialize the camera config monitor.
+        
+        Args:
+            app_deployment: AppDeployment instance to fetch configs
+            streaming_pipeline: StreamingPipeline instance to update
+            check_interval: Seconds between config checks
+        """
+        self.app_deployment = app_deployment
+        self.streaming_pipeline = streaming_pipeline
+        self.check_interval = max(10, int(check_interval))  # Minimum 10 seconds
+        self.running = False
+        self.thread: Optional[threading.Thread] = None
+        self.logger = logging.getLogger(__name__)
+        
+        # Track camera configs by hash to detect changes (thread-safe access)
+        self.camera_hashes: Dict[str, str] = {}
+        self._hashes_lock = threading.Lock()
+
+    def start(self) -> None:
+        """Start the background monitoring thread."""
+        if self.running:
+            self.logger.warning("Camera config monitor already running")
+            return
+        
+        self.running = True
+        self.thread = threading.Thread(
+            target=self._monitor_loop,
+            name="CameraConfigMonitor",
+            daemon=False
+        )
+        self.thread.start()
+        self.logger.info(f"Started camera config monitor (check interval: {self.check_interval}s)")
+
+    def stop(self) -> None:
+        """Stop the background monitoring thread."""
+        if not self.running:
+            return
+        
+        self.running = False
+        if self.thread:
+            self.thread.join(timeout=5.0)
+            self.logger.info("Stopped camera config monitor")
+
+    def _monitor_loop(self) -> None:
+        """Main monitoring loop - periodically sync camera configs."""
+        while self.running:
+            try:
+                self._sync_camera_configs()
+            except Exception as e:
+                self.logger.error(f"Error syncing camera configs: {e}")
+            
+            # Sleep in small intervals to allow quick shutdown
+            for _ in range(self.check_interval):
+                if not self.running:
+                    break
+                time.sleep(1)
+
+    def _sync_camera_configs(self) -> None:
+        """Fetch latest configs from API and sync with pipeline."""
+        try:
+            # Fetch current configs from app deployment API
+            latest_configs = self.app_deployment.get_camera_configs()
+            
+            if not latest_configs:
+                self.logger.debug("No camera configs returned from API")
+                return
+            
+            # Process each camera config
+            for camera_id, camera_config in latest_configs.items():
+                self._process_camera_config(camera_id, camera_config)
+            
+            # Optional: Remove cameras that are no longer in API
+            # Uncomment if you want to auto-remove deleted cameras
+            # self._remove_deleted_cameras(latest_configs)
+            
+        except Exception as e:
+            self.logger.error(f"Failed to sync camera configs: {e}")
+
+    def _process_camera_config(self, camera_id: str, camera_config: CameraConfig) -> None:
+        """Process a single camera config - add new or update changed."""
+        try:
+            # Calculate config hash to detect changes
+            config_hash = self._hash_camera_config(camera_config)
+            
+            # Thread-safe read of previous hash
+            with self._hashes_lock:
+                previous_hash = self.camera_hashes.get(camera_id)
+            
+            # Check if this is a new camera or config changed
+            if previous_hash is None:
+                # New camera - add it
+                self._add_new_camera(camera_id, camera_config, config_hash)
+            elif previous_hash != config_hash:
+                # Config changed - update it
+                self._update_changed_camera(camera_id, camera_config, config_hash)
+            else:
+                # No change - skip
+                self.logger.debug(f"Camera {camera_id} config unchanged")
+                
+        except Exception as e:
+            self.logger.error(f"Error processing camera {camera_id}: {e}")
+
+    def _add_new_camera(self, camera_id: str, camera_config: CameraConfig, config_hash: str) -> None:
+        """Add a new camera to the pipeline."""
+        try:
+            # Use asyncio to add camera config
+            import asyncio
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            try:
+                success = loop.run_until_complete(
+                    self.streaming_pipeline.add_camera_config(camera_config)
+                )
+                if success:
+                    # Thread-safe write
+                    with self._hashes_lock:
+                        self.camera_hashes[camera_id] = config_hash
+                    self.logger.info(f"Added new camera: {camera_id}")
+                else:
+                    self.logger.warning(f"Failed to add camera: {camera_id}")
+            finally:
+                loop.close()
+                
+        except Exception as e:
+            self.logger.error(f"Error adding camera {camera_id}: {e}")
+
+    def _update_changed_camera(self, camera_id: str, camera_config: CameraConfig, config_hash: str) -> None:
+        """Update an existing camera with changed config."""
+        try:
+            # Use asyncio to update camera config
+            import asyncio
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            try:
+                success = loop.run_until_complete(
+                    self.streaming_pipeline.update_camera_config(camera_config)
+                )
+                if success:
+                    # Thread-safe write
+                    with self._hashes_lock:
+                        self.camera_hashes[camera_id] = config_hash
+                    self.logger.info(f"Updated camera config: {camera_id}")
+                else:
+                    self.logger.warning(f"Failed to update camera: {camera_id}")
+            finally:
+                loop.close()
+                
+        except Exception as e:
+            self.logger.error(f"Error updating camera {camera_id}: {e}")
+
+    def _remove_deleted_cameras(self, latest_configs: Dict[str, CameraConfig]) -> None:
+        """Remove cameras that are no longer in the API response."""
+        # Thread-safe read
+        with self._hashes_lock:
+            current_camera_ids = set(self.camera_hashes.keys())
+        
+        latest_camera_ids = set(latest_configs.keys())
+        deleted_camera_ids = current_camera_ids - latest_camera_ids
+        
+        for camera_id in deleted_camera_ids:
+            try:
+                import asyncio
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                try:
+                    success = loop.run_until_complete(
+                        self.streaming_pipeline.remove_camera_config(camera_id)
+                    )
+                    if success:
+                        # Thread-safe delete
+                        with self._hashes_lock:
+                            del self.camera_hashes[camera_id]
+                        self.logger.info(f"Removed deleted camera: {camera_id}")
+                finally:
+                    loop.close()
+            except Exception as e:
+                self.logger.error(f"Error removing camera {camera_id}: {e}")
+
+    def _hash_camera_config(self, camera_config: CameraConfig) -> str:
+        """Generate a hash of the camera config to detect changes."""
+        try:
+            # Create a dict with relevant config fields
+            config_dict = {
+                "camera_id": camera_config.camera_id,
+                "input_topic": camera_config.input_topic,
+                "output_topic": camera_config.output_topic,
+                "stream_config": camera_config.stream_config,
+                "enabled": camera_config.enabled
+            }
+            
+            # Convert to JSON string (sorted for consistency) and hash
+            config_str = json.dumps(config_dict, sort_keys=True)
+            return hashlib.md5(config_str.encode()).hexdigest()
+            
+        except Exception as e:
+            self.logger.error(f"Error hashing camera config: {e}")
+            return ""
+

matrice_inference/server/stream/consumer_worker.py

@@ -13,7 +13,32 @@ from typing import Dict, Any, Optional
 from matrice_inference.server.stream.utils import CameraConfig, StreamMessage
 
 class ConsumerWorker:
-    """Handles message consumption from streams with optimized processing."""
+    """Handles message consumption from streams with optimized processing.
+    
+    Frame ID Management:
+    -------------------
+    This worker ensures frame_id uniqueness and consistency throughout the pipeline:
+    
+    1. Frame ID Source Priority:
+       - Upstream frame_id (from message data) - preferred
+       - Message key (if UUID-like)
+       - Generated unique ID (camera_id + worker_id + uuid4)
+    
+    2. Frame Caching:
+       - Frames are cached to Redis using: stream:frames:{frame_id}
+       - frame_id MUST be unique across all apps and cameras
+       - The same frame_id is used throughout the entire pipeline
+    
+    3. Multi-App Safety:
+       - Each app deployment has unique camera_ids
+       - Generated IDs include camera_id + worker_id + uuid4 for uniqueness
+       - Redis prefix ensures isolation between different frame types
+    
+    4. Frame ID Flow:
+       Consumer → Inference → Post-Processing → Producer
+       The frame_id is preserved in task_data["frame_id"] at each stage
+       and included in the final output message for client retrieval.
+    """
 
     DEFAULT_PRIORITY = 1
     DEFAULT_DB = 0
@@ -209,7 +234,25 @@ class ConsumerWorker:
             return b""
     
     def _process_message(self, message_data: Dict[str, Any]) -> None:
-        """Process incoming message and add to inference queue."""
+        """Process incoming message and add to inference queue.
+        
+        This method:
+        1. Extracts/generates a unique frame_id
+        2. Handles codec-specific processing (H.264, H.265, JPEG, etc.)
+        3. Caches the frame content to Redis with the frame_id
+        4. Enqueues the task for inference with frame_id preserved
+        
+        Frame ID Consistency:
+        - The frame_id is determined once at the start of processing
+        - The same frame_id is used for cache writes and task data
+        - frame_id is propagated through the entire pipeline
+        - Output messages include the frame_id for client retrieval
+        
+        Multi-App Safety:
+        - frame_id uniqueness ensures no collisions between apps
+        - Redis prefix (stream:frames:) provides namespace isolation
+        - Cache writes are non-blocking to prevent pipeline delays
+        """
         try:
             message_key = self._extract_message_key(message_data)
             data = self._parse_message_data(message_data)
@@ -394,16 +437,36 @@ class ConsumerWorker:
         return {}
 
     def _determine_frame_id(self, data: Dict[str, Any], message_data: Dict[str, Any]) -> str:
-        """Determine frame ID from message data."""
+        """Determine frame ID from message data with guaranteed uniqueness.
+        
+        Priority:
+        1. Existing frame_id from upstream (UUID expected)
+        2. Message key (if it looks like a UUID)
+        3. Generate unique ID with camera context
+        """
+        # First priority: explicit frame_id from upstream
         frame_id = data.get("frame_id")
-        if frame_id:
-            return frame_id
+        if frame_id and isinstance(frame_id, str) and frame_id.strip():
+            self.logger.debug(f"Using upstream frame_id: {frame_id}")
+            return str(frame_id).strip()
 
+        # Second priority: message key (if it's a UUID-like string)
         fallback_key = message_data.get("key") or data.get("input_name")
         if fallback_key:
-            return str(fallback_key)
-
-        return f"{self.camera_id}_{int(time.time() * 1000)}"
+            key_str = str(fallback_key)
+            # Check if it looks like a UUID (contains dashes and right length)
+            if "-" in key_str and len(key_str) >= 32:
+                self.logger.debug(f"Using message key as frame_id: {key_str}")
+                return key_str
+        
+        # Last resort: generate unique ID with camera, worker, and high-precision timestamp
+        import uuid
+        generated_id = f"{self.camera_id}_{self.worker_id}_{uuid.uuid4()}"
+        self.logger.warning(
+            f"No upstream frame_id found, generated unique ID: {generated_id} "
+            f"(message_key: {fallback_key})"
+        )
+        return generated_id
 
     def _enrich_input_stream(self, input_stream: Dict[str, Any], frame_id: str) -> None:
         """Add frame_id to input_stream if not present."""
@@ -444,15 +507,44 @@ class ConsumerWorker:
             frame_id: Unique frame identifier (uuid expected)
             content_b64: Base64-encoded JPEG string
         """
+        if not self.frame_cache:
+            self.logger.debug("Frame cache not configured, skipping cache write")
+            return
+            
+        # Validate frame_id
+        if not frame_id or not isinstance(frame_id, str):
+            self.logger.warning(
+                f"Invalid frame_id for caching: {frame_id!r} (type: {type(frame_id).__name__})"
+            )
+            return
+        
+        frame_id = frame_id.strip()
+        if not frame_id:
+            self.logger.warning("Empty frame_id after stripping, skipping cache")
+            return
+            
+        # Validate content
+        if not content_b64 or not isinstance(content_b64, str):
+            self.logger.warning(
+                f"Invalid content for frame_id={frame_id}: "
+                f"type={type(content_b64).__name__}, "
+                f"len={len(content_b64) if content_b64 else 0}"
+            )
+            return
+        
         try:
-            if not self.frame_cache:
-                return
-            if not frame_id or not isinstance(frame_id, str):
-                return
-            if not content_b64 or not isinstance(content_b64, str):
-                return
+            content_len = len(content_b64)
+            self.logger.debug(
+                f"Caching frame: frame_id={frame_id}, camera={self.camera_id}, "
+                f"worker={self.worker_id}, content_size={content_len} bytes"
+            )
             self.frame_cache.put(frame_id, content_b64)
+            self.logger.debug(f"Successfully queued frame {frame_id} for caching")
         except Exception as e:
             # Do not block pipeline on cache errors
-            self.logger.debug(f"Frame cache put failed for frame_id={frame_id}: {e}")
+            self.logger.error(
+                f"Frame cache put failed: frame_id={frame_id}, camera={self.camera_id}, "
+                f"worker={self.worker_id}, error={e}",
+                exc_info=True
+            )
 

matrice_inference/server/stream/frame_cache.py

@@ -1,7 +1,8 @@
 import logging
 import queue
 import threading
-from typing import Optional
+import time
+from typing import Optional, Dict, Any
 
 try:
     import redis  # type: ignore
@@ -47,6 +48,17 @@ class RedisFrameCache:
         self.queue: queue.Queue = queue.Queue(maxsize=max_queue)
         self.threads: list = []
         self._client: Optional[redis.Redis] = None
+        
+        # Metrics for monitoring and debugging
+        self._metrics = {
+            "frames_queued": 0,
+            "frames_cached": 0,
+            "frames_failed": 0,
+            "frames_dropped": 0,
+            "last_cache_time": None,
+            "last_frame_id": None,
+        }
+        self._metrics_lock = threading.Lock()
 
         if not self._is_redis_available():
             return
@@ -92,11 +104,21 @@ class RedisFrameCache:
 
     def start(self) -> None:
         """Start the frame cache with worker threads."""
-        if not self._client or self.running:
+        if not self._client:
+            self.logger.warning("Cannot start frame cache: Redis client not initialized")
+            return
+        
+        if self.running:
+            self.logger.warning("Frame cache already running")
             return
 
         self.running = True
         self._start_worker_threads()
+        
+        self.logger.info(
+            f"Started RedisFrameCache: prefix={self.prefix}, ttl={self.ttl_seconds}s, "
+            f"workers={self._worker_threads}, queue_size={self.queue.maxsize}"
+        )
 
     def _start_worker_threads(self) -> None:
         """Start worker threads for processing cache operations."""
@@ -138,18 +160,36 @@ class RedisFrameCache:
         """Enqueue a cache write for the given frame.
 
         Args:
-            frame_id: unique identifier for the frame
+            frame_id: unique identifier for the frame (must be unique across all apps)
             base64_content: base64-encoded image string
         """
         if not self._is_cache_ready():
+            self.logger.debug(
+                f"Cache not ready for frame_id={frame_id}, skipping "
+                f"(running={self.running}, client={self._client is not None})"
+            )
             return
 
         if not self._validate_input(frame_id, base64_content):
             return
 
         try:
+            # Build Redis key with prefix to avoid collisions
             key = f"{self.prefix}{frame_id}"
-            self.queue.put_nowait((key, base64_content))
+            content_len = len(base64_content)
+            
+            self.queue.put_nowait((key, base64_content, frame_id))
+            
+            # Update metrics
+            with self._metrics_lock:
+                self._metrics["frames_queued"] += 1
+                self._metrics["last_frame_id"] = frame_id
+            
+            self.logger.debug(
+                f"Queued frame for caching: frame_id={frame_id}, "
+                f"redis_key={key}, content_size={content_len}, "
+                f"queue_size={self.queue.qsize()}"
+            )
         except queue.Full:
             self._handle_queue_full(frame_id)
 
@@ -159,17 +199,28 @@ class RedisFrameCache:
 
     def _validate_input(self, frame_id: str, base64_content: str) -> bool:
         """Validate input parameters."""
-        if not frame_id:
-            self.logger.warning("Empty frame_id provided")
+        if not frame_id or not isinstance(frame_id, str) or not frame_id.strip():
+            self.logger.warning(
+                f"Invalid frame_id: {frame_id!r} (type: {type(frame_id).__name__})"
+            )
             return False
-        if not base64_content:
-            self.logger.warning("Empty base64_content provided")
+        if not base64_content or not isinstance(base64_content, str):
+            self.logger.warning(
+                f"Invalid base64_content for frame_id={frame_id}: "
+                f"type={type(base64_content).__name__}, "
+                f"len={len(base64_content) if base64_content else 0}"
+            )
             return False
         return True
 
     def _handle_queue_full(self, frame_id: str) -> None:
         """Handle queue full condition."""
-        self.logger.debug(f"Frame cache queue full; dropping frame_id={frame_id}")
+        with self._metrics_lock:
+            self._metrics["frames_dropped"] += 1
+        self.logger.warning(
+            f"Frame cache queue full (size={self.queue.maxsize}); "
+            f"dropping frame_id={frame_id}. Consider increasing max_queue or worker_threads."
+        )
 
     def _worker(self) -> None:
         """Worker thread for processing cache operations."""
@@ -195,22 +246,72 @@ class RedisFrameCache:
 
     def _process_cache_item(self, item: tuple) -> None:
         """Process a single cache item."""
+        frame_id = "unknown"
         try:
-            key, base64_content = item
-            self._store_frame_data(key, base64_content)
+            key, base64_content, frame_id = item
+            self._store_frame_data(key, base64_content, frame_id)
+        except ValueError as e:
+            # Handle old tuple format without frame_id for backwards compatibility
+            try:
+                key, base64_content = item
+                frame_id = key.replace(self.prefix, "") if key.startswith(self.prefix) else key
+                self._store_frame_data(key, base64_content, frame_id)
+            except Exception as inner_e:
+                self.logger.error(f"Failed to unpack cache item: {inner_e}")
+                with self._metrics_lock:
+                    self._metrics["frames_failed"] += 1
         except Exception as e:
-            self.logger.debug(f"Failed to process cache item: {e}")
+            self.logger.error(f"Failed to process cache item for frame_id={frame_id}: {e}")
+            with self._metrics_lock:
+                self._metrics["frames_failed"] += 1
         finally:
             self._mark_task_done()
 
-    def _store_frame_data(self, key: str, base64_content: str) -> None:
-        """Store frame data in Redis with TTL."""
+    def _store_frame_data(self, key: str, base64_content: str, frame_id: str) -> None:
+        """Store frame data in Redis with TTL.
+        
+        Uses Redis HSET + EXPIRE for atomic TTL management.
+        Multiple apps can safely write to different frame_ids without conflicts.
+        """
+        start_time = time.time()
         try:
+            content_len = len(base64_content)
+            self.logger.debug(
+                f"Writing to Redis: frame_id={frame_id}, key={key}, "
+                f"content_size={content_len}, ttl={self.ttl_seconds}s"
+            )
+            
             # Store base64 string in Redis hash field 'frame', then set TTL
             self._client.hset(key, "frame", base64_content)
             self._client.expire(key, self.ttl_seconds)
+            
+            elapsed = time.time() - start_time
+            
+            # Update metrics
+            with self._metrics_lock:
+                self._metrics["frames_cached"] += 1
+                self._metrics["last_cache_time"] = time.time()
+                self._metrics["last_frame_id"] = frame_id
+            
+            self.logger.info(
+                f"Successfully cached frame: frame_id={frame_id}, key={key}, "
+                f"content_size={content_len}, ttl={self.ttl_seconds}s, "
+                f"elapsed={elapsed:.3f}s"
+            )
+        except redis.RedisError as e:
+            with self._metrics_lock:
+                self._metrics["frames_failed"] += 1
+            self.logger.error(
+                f"Redis error caching frame: frame_id={frame_id}, key={key}, "
+                f"error={e.__class__.__name__}: {e}"
+            )
         except Exception as e:
-            self.logger.debug(f"Failed to cache frame {key}: {e}")
+            with self._metrics_lock:
+                self._metrics["frames_failed"] += 1
+            self.logger.error(
+                f"Unexpected error caching frame: frame_id={frame_id}, key={key}, "
+                f"error={e}", exc_info=True
+            )
 
     def _mark_task_done(self) -> None:
         """Mark queue task as done."""
@@ -218,5 +319,32 @@ class RedisFrameCache:
             self.queue.task_done()
         except Exception:
             pass
+    
+    def get_metrics(self) -> Dict[str, Any]:
+        """Get cache performance metrics for monitoring and debugging.
+        
+        Returns:
+            Dictionary containing cache metrics including:
+            - frames_queued: Total frames queued for caching
+            - frames_cached: Total frames successfully cached
+            - frames_failed: Total frames that failed to cache
+            - frames_dropped: Total frames dropped due to queue full
+            - queue_size: Current queue size
+            - last_cache_time: Timestamp of last successful cache
+            - last_frame_id: Last frame_id cached
+        """
+        with self._metrics_lock:
+            metrics = dict(self._metrics)
+        
+        metrics.update({
+            "running": self.running,
+            "queue_size": self.queue.qsize(),
+            "queue_maxsize": self.queue.maxsize,
+            "worker_threads": self._worker_threads,
+            "prefix": self.prefix,
+            "ttl_seconds": self.ttl_seconds,
+        })
+        
+        return metrics
 
 

matrice_inference/server/stream/inference_worker.py

@@ -107,8 +107,8 @@ class InferenceWorker:
     def _create_postprocessing_task(
         self, task_data: Dict[str, Any], result: Dict[str, Any], processing_time: float
     ) -> Dict[str, Any]:
-        """Create post-processing task from inference result."""
-        return {
+        """Create post-processing task from inference result, preserving frame_id."""
+        postproc_task = {
             "original_message": task_data["message"],
             "model_result": result["model_result"],
             "metadata": result["metadata"],
@@ -117,6 +117,15 @@ class InferenceWorker:
             "stream_key": task_data["stream_key"],
             "camera_config": task_data["camera_config"]
         }
+        
+        # Preserve frame_id from task_data (critical for cache retrieval)
+        if "frame_id" in task_data:
+            postproc_task["frame_id"] = task_data["frame_id"]
+            self.logger.debug(f"Preserved frame_id in postproc task: {task_data['frame_id']}")
+        else:
+            self.logger.warning("No frame_id in task_data to preserve")
+        
+        return postproc_task
 
     def _run_inference(self, task_data: Dict[str, Any]) -> Dict[str, Any]:
         """Run inference in thread pool with proper error handling and cleanup."""

matrice_inference/server/stream/post_processing_worker.py

@@ -126,6 +126,13 @@ class PostProcessingWorker:
             "processing_time_sec": task_data["processing_time"],
             "metadata": task_data.get("metadata", {})
         }
+        
+        # Verify frame_id is present in output
+        if not frame_id:
+            self.logger.warning(
+                f"Output task missing frame_id for camera={task_data['original_message'].camera_id}, "
+                f"message_key={task_data['original_message'].message_key}"
+            )
 
         return {
             "camera_id": task_data["original_message"].camera_id,

matrice_inference/server/stream/stream_pipeline.py

@@ -353,8 +353,8 @@ class StreamingPipeline:
             del self.consumer_workers[camera_id]
     
     def get_metrics(self) -> Dict[str, Any]:
-        """Get pipeline metrics."""
-        return {
+        """Get pipeline metrics including frame cache statistics."""
+        metrics = {
             "running": self.running,
             "camera_count": len(self.camera_configs),
             "enabled_cameras": sum(1 for config in self.camera_configs.values() if config.enabled),
@@ -383,6 +383,18 @@ class StreamingPipeline:
                 for camera_id, config in self.camera_configs.items()
             }
         }
+        
+        # Add frame cache metrics if available
+        if self.frame_cache:
+            try:
+                metrics["frame_cache"] = self.frame_cache.get_metrics()
+            except Exception as e:
+                self.logger.warning(f"Failed to get frame cache metrics: {e}")
+                metrics["frame_cache"] = {"error": str(e)}
+        else:
+            metrics["frame_cache"] = {"enabled": False}
+        
+        return metrics
 
     def _initialize_frame_cache(self) -> None:
         """Initialize RedisFrameCache with TTL 10 minutes, deriving connection from Redis cameras if available."""

{matrice_inference-0.1.22.dist-info → matrice_inference-0.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_inference
-Version: 0.1.22
+Version: 0.1.23
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_inference-0.1.22.dist-info → matrice_inference-0.1.23.dist-info}/RECORD

@@ -3,7 +3,7 @@ matrice_inference/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 matrice_inference/server/__init__.py,sha256=dzLoM3Ma5DjZj2QP6hTXt9dlPQR6QHzNsKkqFahTnZ8,799
 matrice_inference/server/inference_interface.py,sha256=FOoR_5EPqJItgiDyWPQeLFYayAK15dYQAvcbArs2PVc,7842
 matrice_inference/server/proxy_interface.py,sha256=lZ1Yq77JmahHLSqksmQs-Bl0WQ3QmTiWVgRZeQw6oBo,14911
-matrice_inference/server/server.py,sha256=es6dJua2hyz7bwDNmObMiHERXqQCnAh8i7rzK5FkWeg,41806
+matrice_inference/server/server.py,sha256=qAF3TF0wGCBkXcyCAcqQQrbNyWyfRG-e85SYSH_WKzQ,43754
 matrice_inference/server/model/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 matrice_inference/server/model/model_manager.py,sha256=RbJi9EKCTiW49mCqd7gfEOY1a1sXQXclN7kpmiZVA2E,11725
 matrice_inference/server/model/model_manager_wrapper.py,sha256=b0stRwLuoXnwC2AP2HRULa3Azgcm__PIQv7mwRQDcSo,23119
@@ -11,12 +11,13 @@ matrice_inference/server/model/triton_model_manager.py,sha256=rxDj7gFCBDHPbk-dtj
 matrice_inference/server/model/triton_server.py,sha256=KtauLqgRchfRFtRYnzpjJCbRaBHdDNWLaHwSBv5DvVY,55560
 matrice_inference/server/stream/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 matrice_inference/server/stream/app_deployment.py,sha256=b6-VDmI9xYry-6_law68Q5r-d8Fo7EpY9oW6r-260Wo,10631
-matrice_inference/server/stream/consumer_worker.py,sha256=5YzK05K8X8_HG01HHWRsIqSSCdCoFweo7SgVm73QCqs,20842
-matrice_inference/server/stream/frame_cache.py,sha256=YWkpMytiKRLebh6jmLMzfaQOQawpHBMsTltQyMxkdqo,7183
-matrice_inference/server/stream/inference_worker.py,sha256=jIQ6FTNXJSp95gpofpc-gbYoIthfJMCVl7Tcv1i9ta0,10161
-matrice_inference/server/stream/post_processing_worker.py,sha256=99ieOnXCbj3m_Sx7C8WpXf3ca1wEKkU3rKyeqKhmMCk,12407
+matrice_inference/server/stream/camera_config_monitor.py,sha256=GFdSgOJ6hg5IszKqkQ3VJ2wPrPciAvIUQj7_L7xh9TU,8813
+matrice_inference/server/stream/consumer_worker.py,sha256=YPHFdSIBHnBuQK0zgwUuxYalwyFGb7N0gv-Dvi0jG_0,24895
+matrice_inference/server/stream/frame_cache.py,sha256=R3AdXNZsJTE3OSEhIRJdy7Q-AXWZRhX9d5hbTzud92o,12563
+matrice_inference/server/stream/inference_worker.py,sha256=udFbXhAH_fJeqxzZh2MsG85UE62qC0qQCnftiqKtuSk,10591
+matrice_inference/server/stream/post_processing_worker.py,sha256=Tt-ymq2dHpY_v-dzG4zKQAAhHUGhE7OhWuQ1BR_7K3k,12713
 matrice_inference/server/stream/producer_worker.py,sha256=wjS5c3zNhdA5TCFVINnAkJmaVCxHpx_y6GoszEdeDaY,8220
-matrice_inference/server/stream/stream_pipeline.py,sha256=KopK-_irjtgOvC_VY-cvExsWeCSaTEoJVk05d5y8EZk,16968
+matrice_inference/server/stream/stream_pipeline.py,sha256=5eiUyKEgFuSYrLa0o3HuYg4XOvTeWmTGmJjStEloq9o,17453
 matrice_inference/server/stream/utils.py,sha256=z2qKhnyk_6MpQfPvOWHkRltnB2oVH6MW_9jwecxqAlI,507
 matrice_inference/tmp/abstract_model_manager.py,sha256=_UUWAa0pjld5SfOWNGG-9WuMaOiT5PlrRWPoAzeYg-M,1733
 matrice_inference/tmp/batch_manager.py,sha256=r6TU_UFZoDGYAfEhBT3WGXf8NmpT-h3UvRyFJmL9Uew,9639
@@ -30,8 +31,8 @@ matrice_inference/tmp/aggregator/latency.py,sha256=ObRb_2gZZTqoT2ezHd2G9I4tcz3wp
 matrice_inference/tmp/aggregator/pipeline.py,sha256=qleXlCvPbqidOd12Hy0MvVzr1LqIMIazK1xc_YpjpBc,44825
 matrice_inference/tmp/aggregator/publisher.py,sha256=MrFXqM8w_cXVMLYEGZoAX4huLjKrZsjC_yXKuxIGh-M,19305
 matrice_inference/tmp/aggregator/synchronizer.py,sha256=HiGvWIsABxWQZ1-jT-9uBexmDCgN7GJnWkvKGfuVoUA,26899
-matrice_inference-0.1.22.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
-matrice_inference-0.1.22.dist-info/METADATA,sha256=Brjkeqi-SID-zNj0QGQgZiAc02Gj-G2W5qluTUCFST4,1031
-matrice_inference-0.1.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice_inference-0.1.22.dist-info/top_level.txt,sha256=h7i-nE-ZEKLmslZGWAdfa78m5oY8Bes04TTL8TtBvjM,18
-matrice_inference-0.1.22.dist-info/RECORD,,
+matrice_inference-0.1.23.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
+matrice_inference-0.1.23.dist-info/METADATA,sha256=BDk8QRdcRZEG-nO_emElH8W4yzloNIkJUoHaP05TOLw,1031
+matrice_inference-0.1.23.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice_inference-0.1.23.dist-info/top_level.txt,sha256=h7i-nE-ZEKLmslZGWAdfa78m5oY8Bes04TTL8TtBvjM,18
+matrice_inference-0.1.23.dist-info/RECORD,,