nedo-vision-worker 1.2.9__py3-none-any.whl → 1.3.0__py3-none-any.whl

nedo_vision_worker/__init__.py

@@ -6,5 +6,5 @@ A library for running worker agents in the Nedo Vision platform.
 
 from .worker_service import WorkerService
 
-__version__ = "1.2.9"
+__version__ = "1.3.0"
 __all__ = ["WorkerService"] 

nedo_vision_worker/repositories/PPEDetectionRepository.py

@@ -19,19 +19,22 @@ class PPEDetectionRepository:
         os.makedirs(self.storage_dir, exist_ok=True) 
 
     
-    def get_latest_5_detections(self) -> list:
+    def get_latest_detections(self, limit: int = 50) -> list:
         """
-        Retrieves the latest 5 PPE detections ordered by the 'created_at' timestamp.
+        Retrieves the latest PPE detections ordered by the 'created_at' timestamp.
+
+        Args:
+            limit (int): Maximum number of detections to retrieve. Default is 50.
 
         Returns:
-            list: A list of PPEDetectionEntity objects.
+            list: A list of PPE detection dictionaries.
         """
         try:
             latest_detections = (
                 self.session.query(PPEDetectionEntity)
                 .options(joinedload(PPEDetectionEntity.ppe_labels)) 
                 .order_by(asc(PPEDetectionEntity.created_at))
-                .limit(5)
+                .limit(limit)
                 .all()
             )
             
@@ -67,9 +70,22 @@ class PPEDetectionRepository:
 
         except SQLAlchemyError as e:
             self.session.rollback()
-            logging.error(f"❌ Database error while retrieving latest 5 detections: {e}")
+            logging.error(f"❌ Database error while retrieving detections: {e}")
             return []
 
+    def get_total_pending_count(self) -> int:
+        """
+        Returns the total count of pending PPE detections in the database.
+
+        Returns:
+            int: Total count of pending detections.
+        """
+        try:
+            return self.session.query(PPEDetectionEntity).count()
+        except SQLAlchemyError as e:
+            logging.error(f"❌ Error counting pending detections: {e}")
+            return 0
+
     def delete_records_from_db(self, detection_data: list):
         """
         Deletes PPE detection records from the database based on detection data.
@@ -103,7 +119,6 @@ class PPEDetectionRepository:
                 if os.path.exists(image_path):
                     try:
                         os.remove(image_path)
-                        logging.info(f"Deleted image file: {image_path}")
                     except OSError as e:
                         logging.warning(f"Failed to delete image file {image_path}: {e}")
                 else:
@@ -117,7 +132,6 @@ class PPEDetectionRepository:
                     if os.path.exists(image_tile_path):
                         try:
                             os.remove(image_tile_path)
-                            logging.info(f"Deleted image tile file: {image_tile_path}")
                         except OSError as e:
                             logging.warning(f"Failed to delete image tile file {image_tile_path}: {e}")
 

nedo_vision_worker/repositories/RestrictedAreaRepository.py

@@ -16,9 +16,12 @@ class RestrictedAreaRepository:
         self.session: Session = self.db_manager.get_session("default")
         os.makedirs(self.storage_dir, exist_ok=True)
 
-    def get_latest_5_violations(self) -> list:
+    def get_latest_violations(self, limit: int = 50) -> list:
         """
-        Retrieves the latest 5 restricted area violations ordered by the 'created_at' timestamp.
+        Retrieves the latest restricted area violations ordered by the 'created_at' timestamp.
+
+        Args:
+            limit (int): Maximum number of violations to retrieve. Default is 50.
 
         Returns:
             list: A list of dictionaries representing restricted area violations.
@@ -27,7 +30,7 @@ class RestrictedAreaRepository:
             latest_violations = (
                 self.session.query(RestrictedAreaViolationEntity)
                 .order_by(asc(RestrictedAreaViolationEntity.created_at))
-                .limit(5)
+                .limit(limit)
                 .all()
             )
 
@@ -51,9 +54,22 @@ class RestrictedAreaRepository:
 
         except SQLAlchemyError as e:
             self.session.rollback()
-            logging.error(f"❌ Database error while retrieving latest 5 violations: {e}")
+            logging.error(f"❌ Database error while retrieving violations: {e}")
             return []
 
+    def get_total_pending_count(self) -> int:
+        """
+        Returns the total count of pending restricted area violations in the database.
+
+        Returns:
+            int: Total count of pending violations.
+        """
+        try:
+            return self.session.query(RestrictedAreaViolationEntity).count()
+        except SQLAlchemyError as e:
+            logging.error(f"❌ Error counting pending violations: {e}")
+            return 0
+
     def delete_records_from_db(self, violation_data: list):
         """
         Deletes restricted area violation records from the database based on the provided data.
@@ -81,7 +97,6 @@ class RestrictedAreaRepository:
                 if os.path.exists(image_path):
                     try:
                         os.remove(image_path)
-                        logging.info(f"Deleted image file: {image_path}")
                     except OSError as e:
                         logging.warning(f"Failed to delete image file {image_path}: {e}")
                 else:
@@ -95,7 +110,6 @@ class RestrictedAreaRepository:
                     if os.path.exists(image_tile_path):
                         try:
                             os.remove(image_tile_path)
-                            logging.info(f"Deleted image tile file: {image_tile_path}")
                         except OSError as e:
                             logging.warning(f"Failed to delete image tile file {image_tile_path}: {e}")
 

nedo_vision_worker/worker/DataSenderWorker.py

@@ -118,14 +118,63 @@ class DataSenderWorker:
         try:
             while not self.stop_event.is_set():
                 self.system_usage_manager.process_system_usage()
-                self.ppe_detection_manager.send_ppe_detection_batch()  
-                self.restricted_area_manager.send_violation_batch()  
+                
+                ppe_pending = self._send_with_adaptive_interval(
+                    self.ppe_detection_manager.send_ppe_detection_batch,
+                    self.ppe_detection_manager.ppe_detection_repo.get_total_pending_count
+                )
+                
+                violation_pending = self._send_with_adaptive_interval(
+                    self.restricted_area_manager.send_violation_batch,
+                    self.restricted_area_manager.repo.get_total_pending_count
+                )
+                
                 self._process_image_upload()
                 self._process_dataset_frames()
-                time.sleep(self.send_interval)
+                
+                sleep_time = self._calculate_sleep_interval(ppe_pending, violation_pending)
+                if sleep_time < self.send_interval and (ppe_pending + violation_pending) > 0:
+                    logger.debug(f"⚡ [APP] Catch-up mode: {ppe_pending + violation_pending} pending, sleeping {sleep_time}s")
+                time.sleep(sleep_time)
         except Exception as e:
             logger.error("🚨 [APP] Unexpected error in main worker loop.", exc_info=True)
 
+    def _send_with_adaptive_interval(self, send_func, count_func):
+        """Send data with adaptive interval based on backlog."""
+        try:
+            pending = count_func()
+            if pending > 0:
+                send_func()
+            return pending
+        except Exception as e:
+            logger.error(f"🚨 [APP] Error in adaptive sending: {e}", exc_info=True)
+            return 0
+
+    def _calculate_sleep_interval(self, ppe_pending: int, violation_pending: int) -> float:
+        """
+        Calculate optimal sleep interval based on pending data.
+        Conservative approach to prevent server overload.
+        
+        Args:
+            ppe_pending: Number of pending PPE detections
+            violation_pending: Number of pending violations
+            
+        Returns:
+            float: Sleep interval in seconds
+        """
+        total_pending = ppe_pending + violation_pending
+        
+        if total_pending == 0:
+            return self.send_interval
+        elif total_pending < 50:
+            return self.send_interval
+        elif total_pending < 200:
+            return 3.0
+        elif total_pending < 500:
+            return 2.0
+        else:
+            return 1.5
+
     def _run_worker_source_updater(self):
         """Dedicated loop for updating worker sources at a different interval."""
         try:

nedo_vision_worker/worker/PPEDetectionManager.py

@@ -50,14 +50,42 @@ class PPEDetectionManager:
 
         logger.info("📡 [APP] PPE detection monitoring started.")
 
+    def _calculate_batch_size(self, pending_count: int) -> int:
+        """
+        Calculates optimal batch size based on pending items.
+        Conservative limits to prevent server overload.
+        
+        Args:
+            pending_count (int): Number of pending detections
+            
+        Returns:
+            int: Optimal batch size
+        """
+        if pending_count < 20:
+            return 10
+        elif pending_count < 100:
+            return 30
+        elif pending_count < 500:
+            return 50
+        else:
+            return 100
 
     def send_ppe_detection_batch(self):
-        """Sends a batch of collected PPE detection data to the server."""
+        """Sends a batch of collected PPE detection data to the server with dynamic batch sizing."""
         try:
-            self.ppe_detection_data = self.ppe_detection_repo.get_latest_5_detections()
+            pending_count = self.ppe_detection_repo.get_total_pending_count()
+            
+            if pending_count == 0:
+                return
+            
+            batch_size = self._calculate_batch_size(pending_count)
+            self.ppe_detection_data = self.ppe_detection_repo.get_latest_detections(batch_size)
+            
             if not self.ppe_detection_data:
                 return
 
+            logger.info(f"📤 [APP] Sending {len(self.ppe_detection_data)} PPE detections ({pending_count} pending)")
+
             response = self.ppe_detection_client.send_upsert_batch(
                 worker_id=self.worker_id,
                 worker_source_id=self.worker_source_id,
@@ -66,7 +94,7 @@ class PPEDetectionManager:
             )
 
             if response.get("success"):
-                logger.info("✅ [APP] Successfully sent PPE detection batch.")
+                logger.info(f"✅ [APP] Successfully sent {len(self.ppe_detection_data)} PPE detections")
                 self.ppe_detection_data.clear() 
             else:
                 logger.error(f"❌ [APP] Failed to send PPE detection batch: {response.get('message')}")

nedo_vision_worker/worker/RestrictedAreaManager.py

@@ -50,13 +50,42 @@ class RestrictedAreaManager:
 
         logger.info("📡 [APP] Restricted area violation monitoring started.")
 
+    def _calculate_batch_size(self, pending_count: int) -> int:
+        """
+        Calculates optimal batch size based on pending items.
+        Conservative limits to prevent server overload.
+        
+        Args:
+            pending_count (int): Number of pending violations
+            
+        Returns:
+            int: Optimal batch size
+        """
+        if pending_count < 20:
+            return 10
+        elif pending_count < 100:
+            return 30
+        elif pending_count < 500:
+            return 50
+        else:
+            return 100
+
     def send_violation_batch(self):
-        """Sends a batch of collected violation data to the server."""
+        """Sends a batch of collected violation data to the server with dynamic batch sizing."""
         try:
-            self.violations_data = self.repo.get_latest_5_violations()
+            pending_count = self.repo.get_total_pending_count()
+            
+            if pending_count == 0:
+                return
+            
+            batch_size = self._calculate_batch_size(pending_count)
+            self.violations_data = self.repo.get_latest_violations(batch_size)
+            
             if not self.violations_data:
                 return
 
+            logger.info(f"📤 [APP] Sending {len(self.violations_data)} violations ({pending_count} pending)")
+
             response = self.client.send_upsert_batch(
                 worker_id=self.worker_id,
                 worker_source_id=self.worker_source_id,
@@ -65,7 +94,7 @@ class RestrictedAreaManager:
             )
 
             if response.get("success"):
-                logger.info("✅ [APP] Successfully sent restricted area violation batch.")
+                logger.info(f"✅ [APP] Successfully sent {len(self.violations_data)} violations")
                 self.violations_data.clear()
             else:
                 logger.error(f"❌ [APP] Failed to send restricted area violation batch: {response.get('message')}")

{nedo_vision_worker-1.2.9.dist-info → nedo_vision_worker-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker
-Version: 1.2.9
+Version: 1.3.0
 Summary: Nedo Vision Worker Service Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>

{nedo_vision_worker-1.2.9.dist-info → nedo_vision_worker-1.3.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-nedo_vision_worker/__init__.py,sha256=oLzUhGDH-bezYa_Smc3pWfjWYzwQGUZ0LiTwVuu5AYA,203
+nedo_vision_worker/__init__.py,sha256=lM4SwuHDgc4TdI91GAS_FS--pQKK2RFKliCp8sJY1-o,203
 nedo_vision_worker/cli.py,sha256=ddWspJmSgVkcUYvRdkvTtMNuMTDvNCqLLuMVU9KE3Ik,7457
 nedo_vision_worker/doctor.py,sha256=wNkpe8gLVd76Y_ViyK2h1ZFdqeSl37MnzZN5frWKu30,48410
 nedo_vision_worker/worker_service.py,sha256=9zz8hKwDwqwpfS0KPQfftGJtRci0uj_wiwcr_TGf-E0,11039
@@ -40,8 +40,8 @@ nedo_vision_worker/protos/WorkerSourceService_pb2_grpc.py,sha256=23Zj3WSogfhmp_Y
 nedo_vision_worker/protos/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 nedo_vision_worker/repositories/AIModelRepository.py,sha256=35cQ2EnJvMdArewumPrVIaiwueGy6cXU_g5J1Qcj6_Y,1415
 nedo_vision_worker/repositories/DatasetSourceRepository.py,sha256=MOUbXY1-7WUPhY15aMpdXl6rl0afSvu3Cq8jNtp9nzM,8595
-nedo_vision_worker/repositories/PPEDetectionRepository.py,sha256=s4iCaTNYoJJgGr9ojWgoj6wtUQYFbloxSpnuntX26yw,5792
-nedo_vision_worker/repositories/RestrictedAreaRepository.py,sha256=XudFBscLCXCheUTAPC_CbwBkJMn2uIx2HCnU0NH4riE,4766
+nedo_vision_worker/repositories/PPEDetectionRepository.py,sha256=WYPrmMs3wSsaALR4ixsgMNOUbHbmYDGPbjMQdjSnIE4,6161
+nedo_vision_worker/repositories/RestrictedAreaRepository.py,sha256=nk8DV5dkBfAm2AT8wY9la6yv6Q7pFi00YXLbvofqNkc,5158
 nedo_vision_worker/repositories/WorkerSourcePipelineDebugRepository.py,sha256=kOlVEnPOoDRZdZIm8uWXlc89GMvBPI-36QyKecX7ucE,3350
 nedo_vision_worker/repositories/WorkerSourcePipelineDetectionRepository.py,sha256=cbgg_7p0eNUIgCHoPDZBaRZ1b2Y68p_dfSxpvuGMtRE,1773
 nedo_vision_worker/repositories/WorkerSourcePipelineRepository.py,sha256=xfmEvgnyt-DdfSApGyFfy0H0dXjFFkjeo4LMr0fVFXU,10053
@@ -80,22 +80,22 @@ nedo_vision_worker/util/SystemMonitor.py,sha256=2kkqj9mOlywAS2fHdN1TaIXSXvCApcIH
 nedo_vision_worker/util/VideoProbeUtil.py,sha256=cF-vJ7hIDlXfEJby2a0s9tqwkPGVz_6B3Vv4D5pMmIw,12876
 nedo_vision_worker/util/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 nedo_vision_worker/worker/CoreActionWorker.py,sha256=lb7zPY3yui6I3F4rX4Ii7JwpWZahLEO72rh3iWOgFmg,5441
-nedo_vision_worker/worker/DataSenderWorker.py,sha256=9FudRRItiMOcQx5UfVyu4p0Enb9BbgwZZ5EgX6Ho2U4,7160
+nedo_vision_worker/worker/DataSenderWorker.py,sha256=BNA5mL2guBVp0O5MKULEWd78dY32YjiyWZX7ECFjVFc,9089
 nedo_vision_worker/worker/DataSyncWorker.py,sha256=LmDPt2J1frmXwuR46L6b0MjlFOHfgG-4_0MGQa78zF4,6288
 nedo_vision_worker/worker/DatasetFrameSender.py,sha256=1SFYj8LJFNi-anBTapsbq8U_NGMM7mnoMKg9NeFAHys,8087
 nedo_vision_worker/worker/DatasetFrameWorker.py,sha256=Hh_wZuMjwovxsEKFqXSuTRin9eYRBZCbcFKm3CKLMbE,19335
-nedo_vision_worker/worker/PPEDetectionManager.py,sha256=fAolWlrsY5SQAWygvjNBNU56IlC0HLlhPgpz7shL-gk,3588
+nedo_vision_worker/worker/PPEDetectionManager.py,sha256=sO0ygzUKNNkUsrv-QJk9b-r9QnTw80VsniD5--OQJi0,4569
 nedo_vision_worker/worker/PipelineActionWorker.py,sha256=xgvryjKtEsMj4BKqWzDIaK_lFny-DfMCj5Y2DxHnWww,5651
 nedo_vision_worker/worker/PipelineImageWorker.py,sha256=J8VBUG0cwcH3qOJp2zTl30B-XhmPFyvJLjxitKJYq0E,5642
 nedo_vision_worker/worker/PipelinePreviewWorker.py,sha256=owFiBbktcOZkdImQeykZSeBIR2-mpt6HNkmYIkLRKzE,6397
 nedo_vision_worker/worker/RabbitMQListener.py,sha256=9gR49MDplgpyb-D5HOH0K77-DJQFvhS2E7biL92SjSU,6950
-nedo_vision_worker/worker/RestrictedAreaManager.py,sha256=3yoXgQ459tV1bOa5choEzR9gE6LklrtHR_e0472U3L0,3521
+nedo_vision_worker/worker/RestrictedAreaManager.py,sha256=Y739KzbPzm2UF1a9oT3YEk7trfKqQjGUQ6MTwEDMkj4,4463
 nedo_vision_worker/worker/SystemUsageManager.py,sha256=mkh4sT-HkIEY1CJHMEG6LP9ATu39YXvLRLyf995OkoQ,5315
 nedo_vision_worker/worker/VideoStreamWorker.py,sha256=5n6v1PNO7IB-jj_McALLkUP-cBjJoIEw4UiSAs3vTb0,7606
 nedo_vision_worker/worker/WorkerManager.py,sha256=2bxXi19fp3p1qjYBStYRdVVgko8dnevXx1_M_sqH5og,5521
 nedo_vision_worker/worker/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
-nedo_vision_worker-1.2.9.dist-info/METADATA,sha256=ySyI3I8lzUmhonotTgOrhH2Tt-AxL_l9oYifd_qaHcU,14728
-nedo_vision_worker-1.2.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nedo_vision_worker-1.2.9.dist-info/entry_points.txt,sha256=LrglS-8nCi8C_PL_pa6uxdgCe879hBETHDVXAckvs-8,60
-nedo_vision_worker-1.2.9.dist-info/top_level.txt,sha256=vgilhlkyD34YsEKkaBabmhIpcKSvF3XpzD2By68L-XI,19
-nedo_vision_worker-1.2.9.dist-info/RECORD,,
+nedo_vision_worker-1.3.0.dist-info/METADATA,sha256=MiQg-EA4V0jEqMkrupbyhwLvIDD1UfTpQ_cub90gh8E,14728
+nedo_vision_worker-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nedo_vision_worker-1.3.0.dist-info/entry_points.txt,sha256=LrglS-8nCi8C_PL_pa6uxdgCe879hBETHDVXAckvs-8,60
+nedo_vision_worker-1.3.0.dist-info/top_level.txt,sha256=vgilhlkyD34YsEKkaBabmhIpcKSvF3XpzD2By68L-XI,19
+nedo_vision_worker-1.3.0.dist-info/RECORD,,