matrice-analytics 0.1.55__py3-none-any.whl → 0.1.58__py3-none-any.whl

matrice_analytics/post_processing/face_reg/embedding_manager.py

@@ -56,7 +56,26 @@ class EmbeddingConfig:
 
 
 class EmbeddingManager:
-    """Manages face embeddings, search operations, and caching."""
+    """
+    Manages face embeddings, search operations, and caching.
+    
+    CRITICAL INITIALIZATION FLOW:
+    1. __init__() creates the manager but does NOT load embeddings or start background refresh
+    2. External caller MUST call await _load_staff_embeddings() to load embeddings synchronously
+    3. After successful load, caller SHOULD call start_background_refresh() for periodic updates
+    4. The _embeddings_loaded flag tracks whether embeddings are ready for use
+    5. All search operations check _embeddings_loaded before proceeding
+    
+    This design prevents race conditions where:
+    - Background thread tries to load while main thread is loading
+    - Search operations are called before embeddings are loaded
+    - Multiple threads compete for the embeddings_lock during initialization
+    
+    Thread Safety:
+    - _embeddings_lock protects embeddings_matrix and embedding_metadata
+    - _cache_lock protects track_id_cache
+    - _embeddings_loaded is set only after successful load under lock
+    """
     
     def __init__(self, config: EmbeddingConfig, face_client: FacialRecognitionClient = None):
         self.config = config
@@ -89,11 +108,47 @@ class EmbeddingManager:
         self._is_running = False
         self._stop_event = threading.Event()
         
-        # Start background refresh if enabled
-        if self.config.enable_background_refresh and self.face_client:
-            self.start_background_refresh()
-            self.logger.info(f"Background embedding refresh enabled - interval: {self.config.background_refresh_interval}s")
+        # Initialization status flag
+        self._embeddings_loaded = False
+        
+        # DON'T start background refresh yet - wait for initial load in initialize()
+        # This prevents race conditions where background thread interferes with main init
+        self.logger.info(f"EmbeddingManager created - background refresh will start after initial load (interval: {self.config.background_refresh_interval}s)")
+        
+    def is_ready(self) -> bool:
+        """
+        Check if embeddings are loaded and ready for use.
         
+        Returns:
+            True if embeddings are loaded and matrix is valid, False otherwise
+        """
+        return (
+            self._embeddings_loaded 
+            and self.embeddings_matrix is not None 
+            and len(self.embedding_metadata) > 0
+        )
+    
+    def get_status(self) -> Dict[str, Any]:
+        """
+        Get detailed status of embedding manager for debugging and health checks.
+        
+        Returns:
+            Dictionary with status information
+        """
+        with self._embeddings_lock:
+            matrix_shape = self.embeddings_matrix.shape if self.embeddings_matrix is not None else None
+            
+        return {
+            "embeddings_loaded": self._embeddings_loaded,
+            "embeddings_count": len(self.staff_embeddings),
+            "matrix_shape": matrix_shape,
+            "metadata_count": len(self.embedding_metadata),
+            "cache_size": len(self.track_id_cache),
+            "last_update": self.staff_embeddings_last_update,
+            "is_running": self._is_running,
+            "is_ready": self.is_ready(),
+        }
+    
     def set_face_client(self, face_client: FacialRecognitionClient):
         """Set the face recognition client."""
         self.face_client = face_client
@@ -175,11 +230,14 @@ class EmbeddingManager:
         """Load all staff embeddings from API and cache them."""
         if not self.face_client:
             self.logger.error("Face client not available for loading staff embeddings")
+            print("ERROR: Face client not available for loading staff embeddings")
             return False
             
         try:
             self.logger.info("Loading staff embeddings from API...")
+            print("=============== LOADING STAFF EMBEDDINGS FROM API ===============")
             response = await self.face_client.get_all_staff_embeddings()
+            print(f"API RESPONSE TYPE: {type(response)}, IS_LIST: {isinstance(response, list)}, LEN: {len(response) if isinstance(response, list) else 'N/A'}")
             
             # Robust response handling: accept dict with data or raw list
             embeddings_data: List[Dict[str, Any]] = []
@@ -269,27 +327,33 @@ class EmbeddingManager:
                     
                     self.embedding_metadata = self.staff_embeddings.copy()
                     self.staff_embeddings_last_update = time.time()
+                    self._embeddings_loaded = True  # Mark as successfully loaded
+                    
                     self.logger.info(f"Successfully loaded and cached {len(self.staff_embeddings)} staff embeddings (dim={self.embeddings_matrix.shape[1]})")
+                    print(f"=============== SUCCESS: LOADED {len(self.staff_embeddings)} EMBEDDINGS, MATRIX SHAPE: {self.embeddings_matrix.shape} ===============")
                     try:
                         # Quick sanity metrics
                         row0_sum = float(np.sum(self.embeddings_matrix[0])) if self.embeddings_matrix.shape[0] > 0 else 0.0
+                        row0_norm = float(np.linalg.norm(self.embeddings_matrix[0])) if self.embeddings_matrix.shape[0] > 0 else 0.0
+                        print(f"SANITY CHECK: row0_sum={row0_sum:.4f}, row0_norm={row0_norm:.4f} (should be ~1.0 after normalization)")
                         self.logger.debug(f"Embeddings matrix shape: {self.embeddings_matrix.shape}, dtype={self.embeddings_matrix.dtype}, row0_sum={row0_sum:.4f}")
-                    except Exception:
-                        pass
+                    except Exception as e:
+                        print(f"ERROR in sanity check: {e}")
                     return True
                 else:
                     # Build diagnostics and raise to stop pipeline early with actionable info
                     dims_summary: Dict[int, int] = {}
                     for d in dims_observed:
                         dims_summary[d] = dims_summary.get(d, 0) + 1
-                    self.logger.error(
-                        "No valid staff embeddings loaded. Observed dimension distribution: "
-                        f"{dims_summary}. Expected_dim={expected_dim}. Mismatch examples (staffId, dim): "
-                        f"{mismatch_examples[:5]}"
-                    )
-                    raise RuntimeError(
-                        f"Failed to load staff embeddings due to dimension mismatch. Observed dims: {dims_summary}"
+                    error_msg = (
+                        f"No valid staff embeddings loaded. Observed dimension distribution: {dims_summary}. "
+                        f"Expected_dim={expected_dim}. Mismatch examples (staffId, dim): {mismatch_examples[:5]}"
                     )
+                    self.logger.error(error_msg)
+                    print(f"=============== ERROR: NO VALID EMBEDDINGS ===============")
+                    print(f"ERROR: {error_msg}")
+                    print(f"=============== STOPPING PIPELINE ===============")
+                    raise RuntimeError(f"Failed to load staff embeddings due to dimension mismatch. Observed dims: {dims_summary}")
                 
         except Exception as e:
             self.logger.error(f"Error loading staff embeddings: {e}", exc_info=True)
@@ -329,8 +393,16 @@ class EmbeddingManager:
     
     def _find_best_local_match(self, query_embedding: List[float]) -> Optional[Tuple[StaffEmbedding, float]]:
         """Find best matching staff member using optimized matrix operations (thread-safe)."""
+        # Check if embeddings are loaded at all
+        if not self._embeddings_loaded:
+            print(f"ERROR: _find_best_local_match called but embeddings not loaded yet (_embeddings_loaded={self._embeddings_loaded})")
+            self.logger.error("Embeddings not loaded - _find_best_local_match cannot proceed")
+            return None
+            
         with self._embeddings_lock:
             if self.embeddings_matrix is None or len(self.embedding_metadata) == 0:
+                print(f"ERROR: _find_best_local_match - embeddings_matrix is None={self.embeddings_matrix is None}, metadata_len={len(self.embedding_metadata)}, _embeddings_loaded={self._embeddings_loaded}")
+                self.logger.error(f"Embeddings matrix is None despite _embeddings_loaded={self._embeddings_loaded}")
                 return None
                 
             # Create local copies to avoid issues with concurrent modifications
@@ -338,6 +410,7 @@ class EmbeddingManager:
             embedding_metadata = self.embedding_metadata.copy()
             
         if embeddings_matrix is None:
+            print("ERROR: _find_best_local_match - embeddings_matrix copy is None")
             return None
             
         try:
@@ -345,6 +418,7 @@ class EmbeddingManager:
             # Dimension check
             if embeddings_matrix.shape[1] != query_array.shape[1]:
                 self.logger.warning(f"Query embedding dim mismatch: query={query_array.shape[1]} staff={embeddings_matrix.shape[1]}")
+                print(f"ERROR: DIMENSION MISMATCH - query={query_array.shape[1]} staff={embeddings_matrix.shape[1]}")
                 return None
             
             # Normalize query embedding
@@ -382,6 +456,7 @@ class EmbeddingManager:
         try:
             query_array = np.array(query_embedding, dtype=np.float32).reshape(1, -1)
             if embeddings_matrix.shape[1] != query_array.shape[1]:
+                print(f"ERROR: get_best_similarity DIMENSION MISMATCH - query={query_array.shape[1]} staff={embeddings_matrix.shape[1]}")
                 return 0.0
             qn = np.linalg.norm(query_array)
             if qn == 0:

matrice_analytics/post_processing/face_reg/face_recognition.py

@@ -143,6 +143,13 @@ class TemporalIdentityManager:
         
         # PRIMARY PATH: Local similarity search using EmbeddingManager (FAST - ~1-5ms)
         if self.embedding_manager:
+            # Defensive check: ensure embeddings are loaded before attempting search
+            if not self.embedding_manager.is_ready():
+                status = self.embedding_manager.get_status()
+                self.logger.error(f"EmbeddingManager not ready for search - status: {status}")
+                print(f"ERROR: _compute_best_identity - embeddings not ready. Status: {status}")
+                return None, "Unknown", 0.0, None, {}, "unknown"
+                
             try:
                 local_match = self.embedding_manager._find_best_local_match(emb)
                 
@@ -540,11 +547,30 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
         Async initialization method to set up face client and all components.
         Must be called after __init__ before process() can be called.
         
+        CRITICAL INITIALIZATION SEQUENCE:
+        1. Initialize face client and update deployment
+        2. Create EmbeddingManager (does NOT load embeddings yet)
+        3. Synchronously load embeddings with _load_staff_embeddings() - MUST succeed
+        4. Verify embeddings are actually loaded (fail-fast if not)
+        5. Start background refresh thread (only after successful load)
+        6. Initialize TemporalIdentityManager with loaded EmbeddingManager
+        7. Final verification of all components
+        
+        This sequence ensures:
+        - No race conditions between main load and background thread
+        - Fail-fast behavior if embeddings can't be loaded
+        - All components have verified embeddings before use
+        
         Args:
             config: Optional config to use. If not provided, uses config from __init__.
+            
+        Raises:
+            RuntimeError: If embeddings fail to load or verification fails
         """
+        print("=============== INITIALIZE() CALLED ===============")
         if self._initialized:
             self.logger.debug("Use case already initialized, skipping")
+            print("=============== ALREADY INITIALIZED, SKIPPING ===============")
             return
             
         # Use provided config or fall back to default config from __init__
@@ -558,10 +584,12 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
             raise TypeError(f"Invalid config type for initialization: {type(init_config)}, expected FaceRecognitionEmbeddingConfig")
             
         self.logger.info("Initializing face recognition use case with provided config")
+        print("=============== STEP 1: INITIALIZING FACE CLIENT ===============")
         
         # Initialize face client (includes deployment update)
         try:
             self.face_client = await self._get_facial_recognition_client(init_config)
+            print(f"=============== FACE CLIENT INITIALIZED: {self.face_client is not None} ===============")
             
             # Initialize People activity logging if enabled
             if init_config.enable_people_activity_logging:
@@ -570,7 +598,9 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
                 self.logger.info("People activity logging enabled and started")
             
             # Initialize EmbeddingManager
+            print("=============== STEP 2: INITIALIZING EMBEDDING MANAGER ===============")
             if not init_config.embedding_config:
+                print("=============== CREATING EMBEDDING CONFIG ===============")
                 init_config.embedding_config = EmbeddingConfig(
                     similarity_threshold=init_config.similarity_threshold,
                     confidence_threshold=init_config.confidence_threshold,
@@ -581,16 +611,42 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
                     staff_embeddings_cache_ttl=43200,
                 )
             self.embedding_manager = EmbeddingManager(init_config.embedding_config, self.face_client)
+            print(f"=============== EMBEDDING MANAGER CREATED: {self.embedding_manager is not None} ===============")
             self.logger.info("Embedding manager initialized")
             
             # Load staff embeddings immediately for fast startup (avoid race conditions)
+            # This MUST succeed before we can proceed - fail fast if it doesn't
+            print("=============== STEP 3: CALLING _load_staff_embeddings() ===============")
             embeddings_loaded = await self.embedding_manager._load_staff_embeddings()
-            if embeddings_loaded:
-                self.logger.info(f"Loaded {len(self.embedding_manager.staff_embeddings)} staff embeddings at initialization")
-            else:
-                self.logger.warning("Failed to load staff embeddings at initialization - will retry in background")
+            print(f"=============== EMBEDDINGS LOADED: {embeddings_loaded} ===============")
+            
+            if not embeddings_loaded:
+                error_msg = "CRITICAL: Failed to load staff embeddings at initialization - cannot proceed without embeddings"
+                print(f"=============== {error_msg} ===============")
+                self.logger.error(error_msg)
+                raise RuntimeError(error_msg)
+            
+            # Verify embeddings are actually loaded using is_ready() method
+            if not self.embedding_manager.is_ready():
+                status = self.embedding_manager.get_status()
+                error_msg = f"CRITICAL: Embeddings not ready after load - status: {status}"
+                print(f"=============== {error_msg} ===============")
+                self.logger.error(error_msg)
+                raise RuntimeError(error_msg)
+            
+            print(f"=============== STAFF EMBEDDINGS COUNT: {len(self.embedding_manager.staff_embeddings)} ===============")
+            print(f"=============== EMBEDDINGS MATRIX SHAPE: {self.embedding_manager.embeddings_matrix.shape} ===============")
+            print(f"=============== EMBEDDINGS LOADED FLAG: {self.embedding_manager._embeddings_loaded} ===============")
+            self.logger.info(f"Successfully loaded {len(self.embedding_manager.staff_embeddings)} staff embeddings at initialization")
+            
+            # NOW start background refresh after successful initial load (prevents race conditions)
+            if init_config.embedding_config.enable_background_refresh:
+                print("=============== STEP 4: STARTING BACKGROUND REFRESH ===============")
+                self.embedding_manager.start_background_refresh()
+                self.logger.info("Background embedding refresh started after successful initial load")
             
             # Initialize TemporalIdentityManager with EmbeddingManager for fast local search
+            print("=============== STEP 5: INITIALIZING TEMPORAL IDENTITY MANAGER ===============")
             self.temporal_identity_manager = TemporalIdentityManager(
                 face_client=self.face_client,
                 embedding_manager=self.embedding_manager,
@@ -602,8 +658,26 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
             )
             self.logger.info("Temporal identity manager initialized with embedding manager for local similarity search")
             
+            # Final verification before marking as initialized
+            print("=============== STEP 6: FINAL VERIFICATION ===============")
+            if not self.embedding_manager.is_ready():
+                status = self.embedding_manager.get_status()
+                error_msg = f"CRITICAL: Final verification failed - embeddings not ready. Status: {status}"
+                print(f"=============== {error_msg} ===============")
+                self.logger.error(error_msg)
+                raise RuntimeError(error_msg)
+            
+            # Log detailed status for debugging
+            status = self.embedding_manager.get_status()
+            print(f"=============== FINAL CHECKS PASSED ===============")
+            print(f"  - Face client: {self.face_client is not None}")
+            print(f"  - Embedding manager: {self.embedding_manager is not None}")
+            print(f"  - Embedding manager status: {status}")
+            print(f"  - Temporal identity manager: {self.temporal_identity_manager is not None}")
+            
             self._initialized = True
-            self.logger.info("Face recognition use case fully initialized")
+            self.logger.info("Face recognition use case fully initialized and verified")
+            print("=============== INITIALIZATION COMPLETE ===============")
             
         except Exception as e:
             self.logger.error(f"Error during use case initialization: {e}", exc_info=True)
@@ -747,7 +821,7 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
             self.logger.debug("Applied category filtering")
 
         print("------------------TILL TRACKER MS----------------------------")
-        print("LATENCY:",(time.time() - processing_start)*1000,"| Throughput fps:",(1.0 / (time.time() - processing_start)) if (time.time() - processing_start) > 0 else None)
+        print(self._initialized,"LATENCY:",(time.time() - processing_start)*1000,"| Throughput fps:",(1.0 / (time.time() - processing_start)) if (time.time() - processing_start) > 0 else None)
         print("------------------TILL TRACKER MS----------------------------")
         # Advanced tracking (BYTETracker-like) - only if enabled
         if config.enable_face_tracking:
@@ -794,6 +868,19 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
 
         # Process face recognition for each detection (if enabled)
         if config.enable_face_recognition:
+            # Additional safety check: verify embeddings are still loaded and ready
+            if not self.embedding_manager or not self.embedding_manager.is_ready():
+                status = self.embedding_manager.get_status() if self.embedding_manager else {}
+                error_msg = f"CRITICAL: Cannot process face recognition - embeddings not ready. Status: {status}"
+                self.logger.error(error_msg)
+                print(f"ERROR: {error_msg}")
+                return self.create_error_result(
+                    error_msg,
+                    usecase=self.name,
+                    category=self.category,
+                    context=context,
+                )
+            
             face_recognition_result = await self._process_face_recognition(
                 processed_data, config, stream_info, input_bytes
             )
@@ -1603,11 +1690,11 @@ class FaceRecognitionEmbeddingUseCase(BaseProcessor):
         human_text_lines.append(f"\tRecognized: {face_summary.get('session_totals',{}).get('total_recognized', 0)}")  
         human_text_lines.append(f"\tUnknown: {face_summary.get('session_totals',{}).get('total_unknown', 0)}")
         # Additional counts similar to compare_similarity HUD
-        try:
-            human_text_lines.append(f"\tCurrent Faces (detections): {total_detections}")
-            human_text_lines.append(f"\tTotal Unique Tracks: {cumulative_total}")
-        except Exception:
-            pass
+        # try:
+        #     human_text_lines.append(f"\tCurrent Faces (detections): {total_detections}")
+        #     human_text_lines.append(f"\tTotal Unique Tracks: {cumulative_total}")
+        # except Exception:
+        #     pass
 
         human_text = "\n".join(human_text_lines)
 

{matrice_analytics-0.1.55.dist-info → matrice_analytics-0.1.58.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_analytics
-Version: 0.1.55
+Version: 0.1.58
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_analytics-0.1.55.dist-info → matrice_analytics-0.1.58.dist-info}/RECORD

@@ -28,8 +28,8 @@ matrice_analytics/post_processing/core/config.py,sha256=uyxWndO-DE9PeGD_h5K3TeB0
 matrice_analytics/post_processing/core/config_utils.py,sha256=QuAS-_JKSoNOtfUWgr7Alf_wsqODzN2rHlQu-cHRK0s,34311
 matrice_analytics/post_processing/face_reg/__init__.py,sha256=yntaiGlW9vdjBpPZQXNuovALihJPzRlFyUE88l3MhBA,1364
 matrice_analytics/post_processing/face_reg/compare_similarity.py,sha256=NlFc8b2a74k0PqSFAbuM_fUbA1BT3pr3VUgvSqRpJzQ,23396
-matrice_analytics/post_processing/face_reg/embedding_manager.py,sha256=KhSV-JXykqvhRC1iwvbAyJZrFl3Ntv1eiLE5semKOHE,40533
-matrice_analytics/post_processing/face_reg/face_recognition.py,sha256=ILkQumMY2ij_QrhJxzLifOv_q5rXh7N4mW94hr6EcgQ,103585
+matrice_analytics/post_processing/face_reg/embedding_manager.py,sha256=zd4KIqZijRssPzA7QJsg9-DCJ983aIyUPoo8WuRW2rs,44956
+matrice_analytics/post_processing/face_reg/face_recognition.py,sha256=qkFl7f3EkyK9V6nWVCCC61ODV8R2sRffp1OB-ZVPjSU,109247
 matrice_analytics/post_processing/face_reg/face_recognition_client.py,sha256=eF2NYju1uWKXhILndI1rh4_VhWrKSGidui2jjbPQXgM,27596
 matrice_analytics/post_processing/face_reg/people_activity_logging.py,sha256=vZbIvkK1h3h58ROeF0_ygF3lqr19O2h5222bN8XyIis,13675
 matrice_analytics/post_processing/ocr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -189,8 +189,8 @@ matrice_analytics/post_processing/utils/format_utils.py,sha256=UTF7A5h9j0_S12xH9
 matrice_analytics/post_processing/utils/geometry_utils.py,sha256=BWfdM6RsdJTTLR1GqkWfdwpjMEjTCJyuBxA4zVGKdfk,9623
 matrice_analytics/post_processing/utils/smoothing_utils.py,sha256=78U-yucAcjUiZ0NIAc9NOUSIT0PWP1cqyIPA_Fdrjp0,14699
 matrice_analytics/post_processing/utils/tracking_utils.py,sha256=rWxuotnJ3VLMHIBOud2KLcu4yZfDp7hVPWUtNAq_2xw,8288
-matrice_analytics-0.1.55.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
-matrice_analytics-0.1.55.dist-info/METADATA,sha256=XlOk1g_bwwjiRWoXw5loC_kJd9weHzxZC58JgkhOzhA,14378
-matrice_analytics-0.1.55.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice_analytics-0.1.55.dist-info/top_level.txt,sha256=STAPEU-e-rWTerXaspdi76T_eVRSrEfFpURSP7_Dt8E,18
-matrice_analytics-0.1.55.dist-info/RECORD,,
+matrice_analytics-0.1.58.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
+matrice_analytics-0.1.58.dist-info/METADATA,sha256=K3BFU6zoK1DBwLnrxRRi0bM8biLmRcJ2rHbuzPVinsI,14378
+matrice_analytics-0.1.58.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice_analytics-0.1.58.dist-info/top_level.txt,sha256=STAPEU-e-rWTerXaspdi76T_eVRSrEfFpURSP7_Dt8E,18
+matrice_analytics-0.1.58.dist-info/RECORD,,