matrice 1.0.99150__py3-none-any.whl → 1.0.99152__py3-none-any.whl

matrice/deploy/__init__.py

@@ -8,13 +8,14 @@ dependencies_check(
         "fastapi",
         "uvicorn",
         "pillow",
-        "confluent_kafka",
+        "confluent_kafka[snappy]",
         "aiokafka",
         "filterpy",
         "scipy",
         "scikit-learn",
         "matplotlib",
         "scikit-image",
+        "python-snappy"
     ]
 )
 

matrice/deploy/aggregator/aggregator.py

@@ -35,6 +35,7 @@ class ResultsAggregator:
         self._aggregation_thread: Optional[threading.Thread] = None
         self._is_running = False
         self._lock = threading.RLock()
+        self._sent_keys = set()
 
         # Statistics
         self.stats = {
@@ -149,6 +150,20 @@ class ResultsAggregator:
             stream_key = sync_result.get("stream_key")
             input_order = sync_result.get("input_order")
             stream_group_key = sync_result.get("stream_group_key")
+            
+            key = (stream_group_key, stream_key, input_order)
+            if key in self._sent_keys:
+                logging.debug(f"Skipping duplicate result: {key}")
+                return None
+            self._sent_keys.add(key)
+            
+            # Basic memory management - prevent unbounded growth
+            if len(self._sent_keys) > 10000:
+                # Remove oldest entries (this is a simple approach)
+                keys_to_remove = list(self._sent_keys)[:2000]
+                for old_key in keys_to_remove:
+                    self._sent_keys.discard(old_key)
+                logging.debug(f"Cleaned up {len(keys_to_remove)} old keys from _sent_keys")
 
             if not stream_key or input_order is None:
                 logging.warning("Missing stream_key or input_order in synchronized result")
@@ -265,5 +280,8 @@ class ResultsAggregator:
                 self.aggregated_results_queue.get_nowait()
         except Exception:
             pass
+        
+        # Clear tracking data
+        self._sent_keys.clear()
 
         logging.info("Results aggregator cleanup completed") 

matrice/deploy/aggregator/ingestor.py

@@ -34,7 +34,6 @@ class ResultsIngestor:
             )
 
         self.consumer_timeout = consumer_timeout
-        self.max_queue_size = 1000  # Maximum queue size for health monitoring
 
         # Result queues for each deployment (now using PriorityQueue)
         self.results_queues: Dict[str, PriorityQueue] = {}
@@ -76,36 +75,6 @@ class ResultsIngestor:
             self.results_queues[deployment_id] = PriorityQueue()
             self.stats["queue_sizes"][deployment_id] = 0
     
-    def _get_priority_counter(self, deployment_id: str, stream_key: str, input_order: int) -> Tuple[int, int, int]:
-        """
-        Get priority tuple for queue ordering with reset detection.
-        
-        Returns:
-            Tuple[session_id, temporal_counter, input_order]: Priority tuple for queue ordering
-        """
-        key = (deployment_id, stream_key)
-        
-        # Initialize counters if needed
-        if key not in self._counters:
-            self._counters[key] = itertools.count()
-            self._last_input_order[key] = -1
-            self._session_counters[key] = 0
-        
-        # Get current temporal counter (always increments)
-        temporal_counter = next(self._counters[key])
-        
-        # Detect input_order reset (significant decrease suggests reset)
-        last_input_order = self._last_input_order[key]
-        if last_input_order != -1 and input_order < last_input_order - 5:  # Allow some tolerance for out-of-order
-            # Input order reset detected - increment session counter
-            self._session_counters[key] += 1
-            logging.info(f"Input order reset detected for {deployment_id}:{stream_key} - {last_input_order} -> {input_order}, session: {self._session_counters[key]}")
-        
-        # Update last seen input_order
-        self._last_input_order[key] = input_order
-        
-        # Return (session_id, temporal_counter, input_order) for proper ordering
-        return (self._session_counters[key], temporal_counter, input_order)
 
     def start_streaming(self) -> bool:
         """
@@ -189,6 +158,21 @@ class ResultsIngestor:
 
         logging.info("Results streaming stopped")
 
+    def _get_priority_counter(self, deployment_id: str, stream_key: str, stream_group_key: str) -> int:
+        """
+        Get priority tuple for queue ordering with reset detection.
+        
+        Returns:
+            Tuple[session_id, temporal_counter, input_order]: Priority tuple for queue ordering
+        """
+        key = (deployment_id, stream_key, stream_group_key)
+        
+        # Initialize counters if needed
+        if key not in self._counters:
+            self._counters[key] = itertools.count()
+        
+        return next(self._counters[key])
+
     def _stream_results_to_queue(
         self, deployment_id: str, results_queue: PriorityQueue
     ):
@@ -213,39 +197,31 @@ class ResultsIngestor:
                     result_value = result.get("value", {})
                     input_streams = result_value.get("input_streams", [])
                     input_stream = input_streams[0]["input_stream"] if input_streams else {}
-                    input_order = input_stream.get("input_order")
+                    # input_order = input_stream.get("input_order")
                     camera_info = input_stream.get("camera_info")
                     stream_key = camera_info.get("camera_name")
-                    stream_group_key = camera_info.get("camera_group")
+                    stream_group_key = camera_info.get("camera_group") or "default_group"
 
-                    if not stream_key or input_order is None:
+                    if not stream_key:
                         logging.warning(
-                            f"Missing stream_key or input_order for deployment {deployment_id}, skipping result. "
-                            f"Stream key: {stream_key}, Input order: {input_order}, Stream group: {stream_group_key}"
+                            f"Missing stream_key for deployment {deployment_id}, skipping result. "
+                            f"Stream key: {stream_key}, Stream group: {stream_group_key}"
                         )
                         continue
                     
-
-                    # Get priority tuple with reset detection
-                    priority_tuple = self._get_priority_counter(deployment_id, stream_key, input_order)
-                    session_id, temporal_counter, _ = priority_tuple
-
+                    order = self._get_priority_counter(deployment_id, stream_key, stream_group_key)
                     # Create enhanced result object with the structured response
                     enhanced_result = {
                         "deployment_id": deployment_id,
                         "stream_key": stream_key,
                         "stream_group_key": stream_group_key,
-                        "input_order": input_order,
-                        "session_id": session_id,  # Add session ID for proper synchronization
-                        "temporal_counter": temporal_counter,  # Add temporal counter for debugging
+                        "input_order": order,
                         "timestamp": time.time(),
                         "result": result_value, # TODO: check if should send this or just agg_summary
                     }
 
-                    # Add to priority queue (non-blocking) with proper ordering
-                    # Priority: (session_id, temporal_counter, input_order) ensures correct ordering even with resets
                     try:
-                        results_queue.put((priority_tuple, enhanced_result), block=False)
+                        results_queue.put((order, enhanced_result), block=False)
 
                         with self._lock:
                             self.stats["results_consumed"] += 1
@@ -367,10 +343,10 @@ class ResultsIngestor:
                 total_queue_size += queue_size
 
                 # Mark as degraded if queue is getting full
-                if queue_size > self.max_queue_size * 0.8:
+                if queue_size > 1000:
                     health["status"] = "degraded"
-                    health["reason"] = f"Queue for {deployment_id} nearly full ({queue_size}/{self.max_queue_size})"
-                    logging.warning(f"Ingestor degraded: {deployment_id} queue has {queue_size} items (threshold: {self.max_queue_size * 0.8})")
+                    health["reason"] = f"Queue for {deployment_id} nearly full ({queue_size})"
+                    logging.warning(f"Ingestor degraded: {deployment_id} queue has {queue_size} items")
 
         # Check for recent errors (within last 60 seconds)
         if (
@@ -419,4 +395,10 @@ class ResultsIngestor:
                 pass
 
         self.results_queues.clear()
+        
+        # Clear tracking data
+        self._counters.clear()
+        self._last_input_order.clear()
+        self._session_counters.clear()
+        
         logging.info("Results streamer cleanup completed")

matrice/deploy/aggregator/synchronizer.py

@@ -15,7 +15,7 @@ class ResultsSynchronizer:
     def __init__(
         self,
         results_queues: Dict[str, PriorityQueue],
-        sync_timeout: float = 10.0,
+        sync_timeout: float = 60.0,
     ):
         """
         Initialize the results synchronizer.
@@ -79,18 +79,10 @@ class ResultsSynchronizer:
                         result = priority_result[1]  # Get actual result (2nd element, after priority tuple)
 
                         stream_key = result.get("stream_key")
-                        input_order = result.get("input_order")
-                        session_id = result.get("session_id")
                         stream_group_key = result.get("stream_group_key")
+                        input_order = result.get("input_order")
 
-                        if stream_key is None or input_order is None or session_id is None:
-                            logging.warning(
-                                f"Result missing stream_key, input_order, or session_id from {deployment_id}, skipping. "
-                                f"Stream key: {stream_key}, Input order: {input_order}, Session ID: {session_id}, Stream group: {stream_group_key}"
-                            )
-                            continue
-
-                        key = (stream_key, session_id, input_order)
+                        key = (stream_group_key, stream_key, input_order)
                         current_time = time.time()
 
                         with self._lock:
@@ -105,7 +97,7 @@ class ResultsSynchronizer:
                             results_collected += 1
 
                         logging.debug(
-                            f"Collected result from {deployment_id} for stream {stream_key}, session {session_id}, order {input_order}"
+                            f"Collected result from {deployment_id} for stream {stream_key}, stream group {stream_group_key}"
                         )
 
                     except Empty:
@@ -127,7 +119,7 @@ class ResultsSynchronizer:
         is_timeout: bool,
     ) -> Dict:
         """Create a synchronized result dictionary with enhanced metadata."""
-        stream_key, session_id, input_order = key
+        stream_group_key, stream_key, input_order = key
         current_time = time.time()
         sync_start_time = self._result_timestamps.get(key, current_time)
         sync_duration = current_time - sync_start_time
@@ -155,7 +147,6 @@ class ResultsSynchronizer:
         synchronized_result = {
             "stream_key": stream_key,
             "input_order": input_order,
-            "session_id": session_id,  # Include session ID for reset tracking
             "stream_group_key": stream_group_key,
             "deployment_results": deployment_results.copy(),
             "synchronization_metadata": {
@@ -177,7 +168,6 @@ class ResultsSynchronizer:
                 ),
                 "sync_completeness_ratio": len(deployment_results) / len(self.deployment_ids),
                 "synchronizer_version": "2.0",  # Updated version for session support
-                "session_id": session_id,  # Include session ID in metadata
             },
         }
 

matrice/deploy/server/proxy/proxy_utils.py

@@ -73,7 +73,7 @@ class AuthKeyValidator:
             else:
                 self.auth_keys_info = []
             if not self.auth_keys_info:
-                logging.warning("No auth keys found for deployment")
+                logging.debug("No auth keys found for deployment")
                 return
             current_time = time.time()
             self.auth_keys.clear()

matrice/deploy/utils/kafka_utils.py

@@ -64,17 +64,19 @@ class KafkaUtils:
         """
         producer_config = {
             "bootstrap.servers": self.bootstrap_servers,
-            "acks": "all",
+            "acks": "1",
             "retries": 1,
             "retry.backoff.ms": 500,
             "request.timeout.ms": 30000,
             "max.in.flight.requests.per.connection": 1,
-            "linger.ms": 5,
-            "batch.size": 65536, # 1MB
+            "linger.ms": 50,
+            "batch.size": 8388608, # 8MB
             "queue.buffering.max.ms": 100,
             "message.max.bytes": 25000000, # 25MB
-            'queue.buffering.max.messages': 2000000
-            # "compression.type": "snappy" # TODO: add the reqs for it
+            'queue.buffering.max.messages': 10000,
+            "delivery.timeout.ms": 600000,
+            "request.timeout.ms": 600000,
+            "compression.type": "snappy"
         }
         
 

matrice/deploy/utils/post_processing/usecases/fire_detection.py

@@ -272,7 +272,7 @@ class FireSmokeUseCase(BaseProcessor):
 
             for category, threshold in config.alert_config.count_thresholds.items():
                 if category == "all" and total > threshold:  
-
+                    print("-----------ALERTS--INNN-----------")
                     alerts.append({
                         "alert_type": getattr(config.alert_config, 'alert_type', ['Default']) if hasattr(config.alert_config, 'alert_type') else ['Default'],
                         "alert_id": "alert_"+category+'_'+frame_key,
@@ -284,6 +284,7 @@ class FireSmokeUseCase(BaseProcessor):
                                     }                    
                     })
                 elif category in summary.get("per_category_count", {}):
+                    print("-----------ALERTS--INNN 2-----------")
                     count = summary.get("per_category_count", {})[category]
                     if count > threshold:  # Fixed logic: alert when EXCEEDING threshold
                         alerts.append({
@@ -333,6 +334,7 @@ class FireSmokeUseCase(BaseProcessor):
 
             for category, threshold in config.alert_config.count_thresholds.items():
                 if category in summary.get("per_category_count", {}):
+                    print("-----------INCIDENTSS--INNN-----------")
                     #count = summary.get("per_category_count", {})[category]
                     start_timestamp = self._get_start_timestamp_str(stream_info)
                     if start_timestamp and self.current_incident_end_timestamp=='N/A':
@@ -581,11 +583,17 @@ class FireSmokeUseCase(BaseProcessor):
                 det for det in data
                 if det.get("category", "").lower() in valid_categories
             ]
+            counts = {}
+            for det in detections:
+                cat = det.get('category', 'unknown')
+                counts[cat] = counts.get(cat, 0) + 1
+            
 
             summary = {
                 "total_objects": len(detections),
                 "by_category": {},
                 "detections": detections,
+                "per_category_count": counts,
             }
 
             # Count by each category defined in config

{matrice-1.0.99150.dist-info → matrice-1.0.99152.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice
-Version: 1.0.99150
+Version: 1.0.99152
 Summary: SDK for connecting to matrice.ai services
 Home-page: https://github.com/matrice-ai/python-sdk
 Author: Matrice.ai

{matrice-1.0.99150.dist-info → matrice-1.0.99152.dist-info}/RECORD

@@ -89,13 +89,13 @@ matrice/data_processing/data_formats/video_mot_tracking.py,sha256=WlTikrOyVU6O_n
 matrice/data_processing/data_formats/video_mscoco_detection.py,sha256=Z4e0GXcNrj-awwM-rIOlNFJqrb0cZD3KeFrnkYJjT1A,17066
 matrice/data_processing/data_formats/video_youtube_bb_tracking.py,sha256=6RbuBOW1kfw0E626jGEZ0i5P3upgx8D5EQ0hsNVc9vs,10353
 matrice/data_processing/data_formats/yolo_detection.py,sha256=qUeZA7_8Of_QWGZlHh-mhRnBFtb5A_u89Oihx-Meg3c,10018
-matrice/deploy/__init__.py,sha256=3GQ-YF7DcrCqI_UBb4kCDopxS5nNU0iPqGTzQPcDvAg,1206
+matrice/deploy/__init__.py,sha256=7rpz_oCPpW5q_NYuLz7CvRh2VRQnPgYbWIdo_PHvOmU,1238
 matrice/deploy/aggregator/__init__.py,sha256=HFz-ufUMeSpSyRINcrx6NdmrcuVZtPfyIXxYu5UNLTc,508
-matrice/deploy/aggregator/aggregator.py,sha256=nrrmrHJYKYMQcT5hcKBfq8VbrbOJCrG4vc2pGgsyK8k,10653
-matrice/deploy/aggregator/ingestor.py,sha256=0bagAWb8tPONSLnpwvLgXrYRZUuLjbnn5zdU4PjVQVc,16969
+matrice/deploy/aggregator/aggregator.py,sha256=T5S4qLBkCKs_lFaT83AS1dp422mzhty1NmJkGU00KhU,11447
+matrice/deploy/aggregator/ingestor.py,sha256=4SwCDvo1ZBwNUJECbHYGcvxGz6U7CSugV5SFmLauBpY,15423
 matrice/deploy/aggregator/pipeline.py,sha256=xOKDqb4Qre5Ek3Fr-03C-jxpZNXdQ2_Dk5EaOn4ssYE,33948
 matrice/deploy/aggregator/publisher.py,sha256=SRPnyindD_R_QNK5MJ_WZAeAaGtj79CZ1VNoltsQtXM,15587
-matrice/deploy/aggregator/synchronizer.py,sha256=Pxh37J6aGbAzyMy7FHPShPcf7FoEtGY3vzXFK3edL8Y,20352
+matrice/deploy/aggregator/synchronizer.py,sha256=tPFwTWPHnF9c4memBoVB0b6n7v-vvUbdNeWPP4CH6bE,19662
 matrice/deploy/client/__init__.py,sha256=d7yxlWNHYKOoAM9T_AlrSQw1_9cfLEop4zBE0QqrTVw,1330
 matrice/deploy/client/client.py,sha256=OnDxQf9m0mGR8a7BlYgsQI8hop6G42cbqHDsYRiW8cI,28533
 matrice/deploy/client/client_stream_utils.py,sha256=FlG6GkVDMVG9EgKV0-hDi-fP3wJ_AEM0XaTe_TV6pJc,37735
@@ -119,9 +119,9 @@ matrice/deploy/server/inference/model_manager.py,sha256=gvCteBmXAAEtjmfWbJqE2CbY
 matrice/deploy/server/inference/triton_utils.py,sha256=fwnrydcdkm9ZdfyRxW2ddTZhN9sVLejAY72ZCGF4XBs,23342
 matrice/deploy/server/proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 matrice/deploy/server/proxy/proxy_interface.py,sha256=f_kNp5Y5WDBXKokoZXBMIS7SGwoYtvSMLhNzKqm9oso,8495
-matrice/deploy/server/proxy/proxy_utils.py,sha256=vrSySzG7mhbXy4ld_Xz07-WUrBTcL72Hq1CLJ5P2E5Q,10425
+matrice/deploy/server/proxy/proxy_utils.py,sha256=-ne310WJilwhEK-4vn3-shsSi5V8PmJPtc6OspRYSXE,10423
 matrice/deploy/utils/__init__.py,sha256=-qzjYt4wl6dxSgmWQzcE5f3-Fp25zmXwtDMZAvreS4E,675
-matrice/deploy/utils/kafka_utils.py,sha256=qs8GPqAeN-l4W6yquEpyIcA-lmdyYaQ864ECRFOpReY,56412
+matrice/deploy/utils/kafka_utils.py,sha256=e5ex7oZLchlIVSvuHmm_FquvQiKk5iZdMcfttROwEVE,56467
 matrice/deploy/utils/boundary_drawing_internal/__init__.py,sha256=4mUOm5_T-vf-XA-kS8EKuuDwUIEV2cTYcYsfApe2CbU,1588
 matrice/deploy/utils/boundary_drawing_internal/boundary_drawing_internal.py,sha256=5SPGXS9EIhJJtvC5qTBBmOTQqSKU2byxHIFgo6Bmt-U,43944
 matrice/deploy/utils/boundary_drawing_internal/boundary_drawing_tool.py,sha256=eY0VQGZ8BfTmR4_ThIAXaumBjh8_c7w69w-d3kta8p0,15421
@@ -175,7 +175,7 @@ matrice/deploy/utils/post_processing/usecases/emergency_vehicle_detection.py,sha
 matrice/deploy/utils/post_processing/usecases/face_emotion.py,sha256=eRfqBdryB0uNoOlz_y-JMuZL1BhPWrI-odqgx_9LT7s,39132
 matrice/deploy/utils/post_processing/usecases/fashion_detection.py,sha256=f9gpzMDhIW-gyn46k9jgf8nY7YeoqAnTxGOzksabFbE,40457
 matrice/deploy/utils/post_processing/usecases/field_mapping.py,sha256=JDwYX8pd2W-waDvBh98Y_o_uchJu7wEYbFxOliA4Iq4,39822
-matrice/deploy/utils/post_processing/usecases/fire_detection.py,sha256=ztFoCNxEKyyLtQKp5c8qIH6nZUCvlMYQhppmvZvm9kk,39046
+matrice/deploy/utils/post_processing/usecases/fire_detection.py,sha256=w0vCthr1adHpMgJNSh_b9Ej2s_17q6GGN90iCsKnaLI,39467
 matrice/deploy/utils/post_processing/usecases/flare_analysis.py,sha256=-egmS3Hs_iGOLeCMfapbkfQ04EWtZx97QRuUcDa-jMU,45340
 matrice/deploy/utils/post_processing/usecases/flower_segmentation.py,sha256=4I7qMx9Ztxg_hy9KTVX-3qBhAN-QwDt_Yigf9fFjLus,52017
 matrice/deploy/utils/post_processing/usecases/gender_detection.py,sha256=DEnCTRew6B7DtPcBQVCTtpd_IQMvMusBcu6nadUg2oM,40107
@@ -227,8 +227,8 @@ matrice/deployment/camera_manager.py,sha256=ReBZqm1CNXRImKcbcZ4uWAT3TUWkof1D28oB
 matrice/deployment/deployment.py,sha256=PLIUD-PxTaC2Zxb3Y12wUddsryV-OJetjCjLoSUh7S4,48103
 matrice/deployment/inference_pipeline.py,sha256=bXLgd29ViA7o0c7YWLFJl1otBUQfTPb61jS6VawQB0Y,37918
 matrice/deployment/streaming_gateway_manager.py,sha256=w5swGsuFVfZIdOm2ZuBHRHlRdYYJMLopLsf2gb91lQ8,20946
-matrice-1.0.99150.dist-info/licenses/LICENSE.txt,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
-matrice-1.0.99150.dist-info/METADATA,sha256=kKKCIinYcZumDM6ZDoUljBf0juk-dppjw3C5gc4NMRk,14624
-matrice-1.0.99150.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice-1.0.99150.dist-info/top_level.txt,sha256=P97js8ur6o5ClRqMH3Cjoab_NqbJ6sOQ3rJmVzKBvMc,8
-matrice-1.0.99150.dist-info/RECORD,,
+matrice-1.0.99152.dist-info/licenses/LICENSE.txt,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
+matrice-1.0.99152.dist-info/METADATA,sha256=4fSj25PDfdJgCXR6iIiqvshkaPJ21CRocpbIqqOkyEI,14624
+matrice-1.0.99152.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice-1.0.99152.dist-info/top_level.txt,sha256=P97js8ur6o5ClRqMH3Cjoab_NqbJ6sOQ3rJmVzKBvMc,8
+matrice-1.0.99152.dist-info/RECORD,,